[med-svn] [ncbi-vdb] 02/06: New upstream version 2.8.0+dfsg
Andreas Tille
tille at debian.org
Fri Nov 4 08:31:05 UTC 2016
This is an automated email from the git hooks/post-receive script.
tille pushed a commit to branch master
in repository ncbi-vdb.
commit 78461da6923957496e5a2755ba31c7bdd4795403
Author: Andreas Tille <tille at debian.org>
Date: Fri Nov 4 08:26:05 2016 +0100
New upstream version 2.8.0+dfsg
---
.gitignore | 1 +
CHANGES | 60 -
CHANGES.md | 20 +
README | 55 -
README.md | 25 +-
build/Makefile.gcc | 4 +-
build/Makefile.install | 28 +-
build/Makefile.vers | 2 +-
build/ld.linux.exe_cmd.sh | 1 +
interfaces/kdb/rowset-impl.h | 141 ++
interfaces/kdb/rowset.h | 186 +++
interfaces/kfc/rc.h | 25 +-
interfaces/kfc/xc.h | 6 +
interfaces/kfg/kfg-priv.h | 6 +
interfaces/kfg/properties.h | 6 +
interfaces/klib/rc.h | 28 +-
interfaces/klib/status.h | 2 +-
interfaces/klib/text.h | 6 +
interfaces/kns/endpoint.h | 1 +
interfaces/kns/http.h | 35 +-
interfaces/kns/kns-mgr-priv.h | 46 +
interfaces/kns/manager.h | 2 +-
interfaces/kns/socket.h | 25 +-
interfaces/kns/{manager.h => tls.h} | 69 +-
interfaces/kproc/timeout.h | 10 +-
interfaces/ktst/unit_test_suite.hpp | 5 +-
interfaces/vdb/manager.h | 12 +-
interfaces/vdb/vdb.vschema | 2 +-
interfaces/vfs/manager-priv.h | 9 +
interfaces/vfs/resolver.h | 60 +-
libs/align/bam.c | 9 +-
libs/ascp/unix/connect.c | 2 +-
libs/blast/blast-mgr.c | 2 +-
libs/kapp/loader-file.c | 10 +-
libs/kdb/Makefile | 2 +
libs/kdb/libkdb.vers.h | 2 +-
libs/kdb/rowset-it.c_old | 385 +++++
libs/kdb/rowset-priv.h_old | 116 ++
libs/kdb/rowset-simple.c | 434 ++++++
libs/kdb/rowset.c | 494 +++++++
libs/kdb/rowset.c_old | 1513 ++++++++++++++++++++
libs/kdb/table.c | 3 +
libs/kdb/wtable.c | 3 +
libs/kfc/refcount.c | 1 +
libs/kfc/unix/sysrsrc.c | 2 +-
libs/kfc/vtbl.c | 9 +-
libs/kfg/Makefile | 24 +-
libs/kfg/certs.kfg | 31 +
libs/kfg/config-aws.c | 34 +-
libs/kfg/config.c | 255 ++--
libs/kfg/default.kfg | 10 +-
libs/kfg/properties.c | 47 +
libs/kfg/repository.c | 10 +
libs/kfs/tar.c | 48 +-
libs/klib/rc-idx-tbl.c | 2 +
libs/klib/rc-tbl.c | 2 +
libs/klib/release-vers.h | 2 +-
libs/klib/report-klib.c | 8 +-
libs/klib/status-rc-strings.c | 1 +
libs/klib/status.c | 7 +-
libs/klib/text.c | 35 +
libs/klib/writer.c | 3 +-
libs/kns/Makefile | 3 +-
libs/kns/http-client.c | 526 ++++++-
libs/kns/http-file.c | 5 +-
libs/kns/http-priv.h | 40 +-
libs/kns/http.c | 47 +-
libs/kns/kns_manager-ext.c | 2 +-
libs/kns/manager.c | 631 ++++++--
libs/kns/mgr-priv.h | 15 +-
libs/kns/stream-priv.h | 12 +
libs/kns/sysmgr.h | 2 +-
interfaces/kproc/timeout.h => libs/kns/tls-priv.h | 52 +-
libs/kns/tls.c | 1087 ++++++++++++++
libs/kns/unix/syssock.c | 907 +++++++-----
libs/kproc/procmgr.c | 101 +-
libs/kproc/unix/systimeout.c | 42 +
libs/ncbi-vdb/Makefile | 3 +
libs/ncbi-vdb/libncbi-vdb.vers | 2 +-
libs/ngs-jni/jni_String.c | 4 +-
libs/ngs/BAM_Alignment.c | 66 +-
libs/ngs/CSRA1_Alignment.c | 21 +
libs/ngs/CSRA1_Read.c | 123 +-
libs/ngs/CSRA1_Reference.c | 322 +++--
libs/ngs/CSRA1_ReferenceWindow.c | 251 ++--
libs/ngs/EBI_ReferenceSequence.c | 41 +-
libs/ngs/Makefile | 1 +
libs/ngs/NGS_Alignment.c | 29 +-
libs/ngs/NGS_Alignment.h | 5 +-
libs/ngs/NGS_Cursor.c | 38 +-
libs/ngs/NGS_Cursor.h | 44 +-
libs/ngs/NGS_FragmentBlob.c | 84 +-
libs/ngs/NGS_ReadCollection.c | 6 +-
libs/ngs/NGS_Statistics.c | 1 +
libs/ngs/SRA_Read.c | 271 ++--
libs/ngs/SRA_Read.h | 60 +-
libs/ngs/SRA_ReferenceSequence.c | 46 +-
libs/ngs/SRA_Statistics.c | 51 +-
libs/ngs/VByteBlob.c | 117 ++
libs/{kns/sysmgr.h => ngs/VByteBlob.h} | 16 +-
libs/sra/sramgr-cmn.c | 2 +-
libs/svdb/svdb.c | 2 +-
libs/vdb/database-cmn.c | 4 +-
libs/vdb/database-depend.c | 2 +-
libs/vdb/dbmgr-cmn.c | 26 +
libs/vdb/libvdb.vers.h | 2 +-
libs/vdb/prod-cmn.c | 181 ++-
{test/vxf => libs/vdb/test}/Makefile | 41 +-
libs/vdb/test/test-prod-cmn.c | 10 +
libs/vdb/wdatabase.c | 2 +-
libs/vfs/manager.c | 219 ++-
libs/vfs/path-priv.h | 1 +
libs/vfs/path.c | 15 +-
libs/vfs/resolver-priv.h | 3 +
libs/vfs/resolver.c | 345 ++++-
libs/vxf/entrez.c | 2 +-
libs/vxf/fzip.c | 2 +-
libs/vxf/irzip.c | 4 +-
libs/vxf/izip.c | 2 +-
libs/xfs/path.c | 14 +-
libs/xfs/unix/operations.c | 9 +-
libs/xfs/xgap.c | 4 +-
libs/xfs/xhttp.c | 16 +-
test/{vxf => align}/Makefile | 40 +-
test/align/indextest.cpp | 113 ++
test/cc/asm-test.cpp | 15 +
test/kdb/Makefile | 23 +-
test/kdb/remote_open_test.cpp | 4 +-
test/kdb/rowsettest.cpp | 819 +++++++++++
test/kfg/flat-sra-kfg.cpp | 44 +-
test/kfg/kfgtest.cpp | 4 +-
test/kfs/cacheteetest.cpp | 13 +-
test/kns/Makefile | 50 +
test/kns/TestProxy.hpp | 227 +++
test/kns/http-client.c | 17 +
test/kns/httptest.cpp | 5 +-
test/kns/knstest.cpp | 47 +-
test/kns/test-proxy-with-env.cpp | 112 ++
test/kns/test-proxy.cpp | 263 ++++
test/kns/test-proxy.h | 7 +
test/kns/test-proxy/2-path-no-env/config | 1 +
test/kns/test-proxy/2-path-no-env/expected | 2 +
test/kns/test-proxy/2-path-no-port-no-env/config | 1 +
test/kns/test-proxy/2-path-no-port-no-env/expected | 2 +
.../MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/config | 1 +
.../all.MULTIPLE_PROXIES/environment | 4 +
.../MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/expected | 10 +
.../disabled.MULTIPLE_PROXIES/config | 2 +
.../disabled.MULTIPLE_PROXIES/environment | 4 +
.../disabled.MULTIPLE_PROXIES/expected | 0
.../enabled.MULTIPLE_PROXIES/config | 2 +
.../enabled.MULTIPLE_PROXIES/environment | 4 +
.../enabled.MULTIPLE_PROXIES/expected | 10 +
.../env-kfg.MULTIPLE_PROXIES/config | 2 +
.../env-kfg.MULTIPLE_PROXIES/environment | 4 +
.../env-kfg.MULTIPLE_PROXIES/expected | 10 +
.../env-only.MULTIPLE_PROXIES/config | 2 +
.../env-only.MULTIPLE_PROXIES/environment | 4 +
.../env-only.MULTIPLE_PROXIES/expected | 8 +
.../ignore.MULTIPLE_PROXIES/config | 2 +
.../ignore.MULTIPLE_PROXIES/environment | 4 +
.../ignore.MULTIPLE_PROXIES/expected | 0
.../kfg-env.MULTIPLE_PROXIES/config | 2 +
.../kfg-env.MULTIPLE_PROXIES/environment | 4 +
.../kfg-env.MULTIPLE_PROXIES/expected | 10 +
.../proxy-not-only.MULTIPLE_PROXIES/config | 2 +
.../proxy-not-only.MULTIPLE_PROXIES/environment | 4 +
.../proxy-not-only.MULTIPLE_PROXIES/expected | 10 +
.../proxy-only.MILTIPLE_PROXIES/config | 2 +
.../proxy-only.MILTIPLE_PROXIES/environment | 4 +
.../proxy-only.MILTIPLE_PROXIES/expected | 10 +
test/kns/test-proxy/all/config | 1 +
test/kns/test-proxy/all/environment | 4 +
test/kns/test-proxy/all/expected | 5 +
test/kns/test-proxy/comma-error/environment | 1 +
test/kns/test-proxy/comma-error/expected | 0
test/kns/test-proxy/disabled/config | 2 +
test/kns/test-proxy/disabled/environment | 4 +
test/kns/test-proxy/disabled/expected | 0
test/kns/test-proxy/enabled/config | 2 +
test/kns/test-proxy/enabled/environment | 4 +
test/kns/test-proxy/enabled/expected | 5 +
test/kns/test-proxy/env-kfg/config | 2 +
test/kns/test-proxy/env-kfg/environment | 4 +
test/kns/test-proxy/env-kfg/expected | 5 +
test/kns/test-proxy/env-only/config | 2 +
test/kns/test-proxy/env-only/environment | 4 +
test/kns/test-proxy/env-only/expected | 4 +
test/kns/test-proxy/error/config | 1 +
test/kns/test-proxy/error/environment | 1 +
test/kns/test-proxy/error/expected | 0
test/kns/test-proxy/http_proxy/environment | 1 +
test/kns/test-proxy/http_proxy/expected | 1 +
test/kns/test-proxy/ignore/config | 2 +
test/kns/test-proxy/ignore/environment | 4 +
test/kns/test-proxy/ignore/expected | 0
test/kns/test-proxy/kfg-env/config | 2 +
test/kns/test-proxy/kfg-env/environment | 4 +
test/kns/test-proxy/kfg-env/expected | 5 +
test/kns/test-proxy/kfg-only/config | 2 +
test/kns/test-proxy/kfg-only/environment | 4 +
test/kns/test-proxy/kfg-only/expected | 1 +
test/kns/test-proxy/no-env/config | 1 +
test/kns/test-proxy/no-env/expected | 1 +
test/kns/test-proxy/no-port-no-env/config | 1 +
test/kns/test-proxy/no-port-no-env/expected | 1 +
test/kns/test-proxy/proxy-not-only/config | 2 +
test/kns/test-proxy/proxy-not-only/environment | 4 +
test/kns/test-proxy/proxy-not-only/expected | 5 +
test/kns/test-proxy/proxy-only/config | 2 +
test/kns/test-proxy/proxy-only/environment | 4 +
test/kns/test-proxy/proxy-only/expected | 5 +
test/kproc/kproctest.cpp | 111 ++
test/krypto/test-aes-ciphers.c | 2 +-
test/krypto/test-modes.c | 2 +-
test/ngs-c++/CSRA1test.cpp | 281 ++--
test/ngs-c++/Makefile | 28 +-
test/ngs-c++/SRAtest.cpp | 120 +-
test/ngs-java/ngs_test_CSRA1.java | 103 ++
test/ngs/ngstest.cpp | 125 +-
test/ngs/ngstest_csra1.cpp | 121 +-
test/ngs/ngstest_csra1_refwin.cpp | 429 +++---
test/ngs/ngstest_reference.cpp | 31 +-
test/ngs/ngstest_sra.cpp | 36 +-
test/only-remote-repository.kfg | 2 +-
test/vdb/.gitignore | 1 +
test/vdb/Makefile | 25 +-
test/vdb/test-VDB-3060.cpp | 33 +-
test/vdb/test-VDB-3061.cpp | 249 ++++
test/vdb/test-wvdb.cpp | 205 ++-
test/vfs/Makefile | 11 +
.../redirect-rejected-names-cgi-http-to-https.cpp | 189 +++
.../incomplete.kfg | 1 +
test/vxf/Makefile | 25 +-
test/vxf/VDB-2915.cpp | 508 +++++++
test/vxf/local_config/local.kfg | 2 +
236 files changed, 12378 insertions(+), 2661 deletions(-)
diff --git a/.gitignore b/.gitignore
index 4a39773..40b02ab 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,4 @@ user.status
\#*
.\#*
core
+*.pyc
diff --git a/CHANGES b/CHANGES
deleted file mode 100644
index 6803b60..0000000
--- a/CHANGES
+++ /dev/null
@@ -1,60 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-The NCBI SRA Software Development Kit
-Contact: sra-tools at ncbi.nlm.nih.gov
-
-
-NCBI External Developer Release: NCBI VDB 2.7.0
-July 12, 2016
-
- blast, kfg, ncbi-vdb, sra-tools, vfs: restored possibility to disable local caching
- doc, ncbi-vdb: created a Wiki page illustrating how to use the API to set up logging
- kdb, kfs, kns: Added blob validation for data being fetched remotely
- kfg: When loading configuration files on Windows USERPROFILE environment variable is used before HOME
- kfg: modified auxiliary remote access configuration to use load-balanced servers
- kns: Fixed a bug when KHttpRequestPOST generated an incorrect Content-Length after retry
- ngs, search, sra-search: sra-search was modified to support multiple threads.
- ngs-engine, ngs-tools, sra-tools, vfs: The "auxiliary" nodes in configuration are now ignored
- ngs-engine: Added support for blob-by-blob access to SEQUENCE table
- ngs-engine: removed a potential memory leak in NGS_CursorMake()
- search: now supports multi-threaded search
- vdb: fixed a bug in VCursorFindNextRowIdDirect where it returned a wrong rowId
- vdb: fixed a bug in the code used to iterate over blobs where rowmap expansion cache would reset iteration to initial row instead of respecting sequence
- vfs: environment variable VDB_PWFILE is no longer used
-
-
-NCBI External Developer Release: NCBI VDB 2.6.3
-May 25, 2016
-
- vdb: Fixed a bound on memory cache that would never flush under certain access modes
-
-
-NCBI External Developer Release: NCBI VDB 2.6.2
-April 20, 2016
-
- kdb: Fixed VTableDropColumn, so that it can drop static columns
- kfs, kns: Fixed thread safety issues for both cache and http files
- kproc: Fixed KQueue to wake waiters when sealed, fixed KSemaphore to wake waiters when canceled
diff --git a/CHANGES.md b/CHANGES.md
index bca80f7..8feb8b8 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,5 +1,25 @@
# NCBI External Developer Release:
+## NCBI VDB 2.8.0
+**October 7, 2016**
+
+### HTTPS-ENABLED RELEASE
+
+ **kfg, kns**: Use environment variables to determine the proxy for a http protocol
+ **kfg, vdb-config**: vdb-config-GUI has now option to pick up proxy-setting from environment
+ **kns**: All tools and libraries now support https
+ **kns**: replaced all direct uses of sleep() within code to enforce standardization upon mS.
+ **kproc, ncbi-vdb**: Fixed KCondition to generate timeout error on Windows when timeout exhausted
+ **ngs-engine**: improved diagnostic messages
+ **test**: updated tests to not fail outside of NCBI
+ **test-sra**: test-sra prints network information
+ **test-sra**: test-sra prints version of ncbi-vdb or ngs-sdk dynamic library
+ **vdb**: improved parameter checking on VDatabaseOpenTableRead()
+ **vdb**: new function: "VDBManagerDeleteCacheOlderThan()"
+ **vdb**: problem with buffer-overrun when compressing random data fixed
+ **vdb**: remote/aux nodes have been removed from configuration
+
+
## NCBI VDB 2.7.0
**July 12, 2016**
diff --git a/README b/README
deleted file mode 100644
index 0c4b070..0000000
--- a/README
+++ /dev/null
@@ -1,55 +0,0 @@
-# ===========================================================================
-#
-# PUBLIC DOMAIN NOTICE
-# National Center for Biotechnology Information
-#
-# This software/database is a "United States Government Work" under the
-# terms of the United States Copyright Act. It was written as part of
-# the author's official duties as a United States Government employee and
-# thus cannot be copyrighted. This software/database is freely available
-# to the public for use. The National Library of Medicine and the U.S.
-# Government have not placed any restriction on its use or reproduction.
-#
-# Although all reasonable efforts have been taken to ensure the accuracy
-# and reliability of the software and data, the NLM and the U.S.
-# Government do not and cannot warrant the performance or results that
-# may be obtained by using this software or data. The NLM and the U.S.
-# Government disclaim all warranties, express or implied, including
-# warranties of performance, merchantability or fitness for any particular
-# purpose.
-#
-# Please cite the author in any work or product based on this material.
-#
-# ===========================================================================
-
-
-The NCBI SRA ( Sequence Read Archive )
-
-
-Contact: sra-tools at ncbi.nlm.nih.gov
-http://www.ncbi.nlm.nih.gov/Traces/sra/?view=toolkit_doc
-
-The SRA Toolkit and SDK from NCBI is a collection of tools and libraries for
-using data in the INSDC Sequence Read Archives.
-
-Much of the data submitted these days, for example in BAM, Illumina export.txt,
-and Complete Genomics formats, contain alignment information. With aligned data,
-NCBI uses Compression by Reference, which only stores the differences in base
-pairs between sequence data and the segment it aligns to. The process to
-restore original data, for example as FastQ, requires fast access to the
-reference sequences that the original data was aligned to. NCBI recommends that
-SRA users dedicate local disk space to store references downloaded from the NCBI
-SRA site. As of Fall 2014, the complete collection of these reference sequences
-is 66 GB. While it isn't usually necessary to download the entirety of the
-reference sequences, this should give you an idea of the scale of the storage
-requirement. By default, the Toolkit will download missing reference sequences
-on demand and cache them in the user's home directory. The location of this
-cache is configurable, as is whether the download is automatic or manual.
-
-For additional information on using, configuring, and building the toolkit,
-please visit our web site at:
-
- http://www.ncbi.nlm.nih.gov/Traces/sra/?view=toolkit_doc&f=std
-
-
-SRA Toolkit Development Team
diff --git a/README.md b/README.md
index f772a80..227a88e 100644
--- a/README.md
+++ b/README.md
@@ -10,19 +10,18 @@ Please check the CHANGES.md file for change history.
The SRA Toolkit and SDK from NCBI is a collection of tools and libraries for
using data in the INSDC Sequence Read Archives.
-Much of the data submitted these days, for example in BAM, Illumina export.txt,
-and Complete Genomics formats, contain alignment information. With aligned data,
-NCBI uses Compression by Reference, which only stores the differences in base
-pairs between sequence data and the segment it aligns to. The process to
-restore original data, for example as FastQ, requires fast access to the
-reference sequences that the original data was aligned to. NCBI recommends that
-SRA users dedicate local disk space to store references downloaded from the NCBI
-SRA site. As of Fall 2014, the complete collection of these reference sequences
-is 66 GB. While it isn't usually necessary to download the entirety of the
-reference sequences, this should give you an idea of the scale of the storage
-requirement. By default, the Toolkit will download missing reference sequences
-on demand and cache them in the user's home directory. The location of this
-cache is configurable, as is whether the download is automatic or manual.
+### IMPORTANT ANNOUNCEMENT
+As was announced in [OMB memorandum M-15-13](https://www.whitehouse.gov/sites/default/files/omb/memoranda/2015/m-15-13.pdf) on June 8, 2015, NCBI and all Federal systems will be transitioning to using HTTPS-only protocols before the end of 2016. This change will affect any software that uses NCBI APIs such as the E-utilities or NCBI software toolkits such as `sra-tools`, `ncbi-vdb` or `ngs`.
+
+The NLM and NCBI may implement the switch to HTTPS-only as early as September 30, 2016.
+
+In particular, software products that depend on `sra-tools`, `ncbi-vdb` or `ngs` may not function as expected after September 30 unless they are properly updated from this site or by the software provider.
+
+If you use software that accesses NCBI SRA data in any way, your software will likely be affected by this change. Please check with your software provider for recent udpates or patches, and be sure to acquire these before September 30.
+
+If you develop software that relies on `sra-tools`, `ncbi-vdb` or `ngs` in any way, you will likely need to update your code so that it accesses NCBI using HTTPS.
+
+We will be releasing new tools that are HTTPS compatible and `M-15-13` compliant in early September, 2016. Please be certain to update all of your binaries and configuration files.
For additional information on using, configuring, and building the toolkit,
please visit our [wiki](https://github.com/ncbi/sra-tools/wiki)
diff --git a/build/Makefile.gcc b/build/Makefile.gcc
index 3f7a72c..91b2a19 100644
--- a/build/Makefile.gcc
+++ b/build/Makefile.gcc
@@ -75,10 +75,10 @@ ifeq (dbg, $(BUILD))
PED = -std=gnu99 -pedantic # -fdiagnostics-show-option
else
-OPT = -O3 -Wno-variadic-macros -fno-strict-aliasing -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1
+OPT = -O3 -Wno-variadic-macros -fno-strict-aliasing -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=1
ifeq (x86_64, $(ARCH))
OPT += -Wall
-endif
+endif
endif
diff --git a/build/Makefile.install b/build/Makefile.install
index 5b007e7..743ff72 100644
--- a/build/Makefile.install
+++ b/build/Makefile.install
@@ -60,7 +60,7 @@ copyincludes: checkversion
INCLUDE_SYMLINK = $(ROOT)/usr/include/ncbi-vdb
PROFILE_FILE = $(ROOT)/etc/profile.d/ncbi-vdb
-KONFIG_DIR = $(ROOT)/etc/ncbi/
+KONFIG_DIR = $(ROOT)/etc/ncbi
ifeq (linux, $(OS))
ifeq (0, $(shell id -u))
@@ -78,7 +78,14 @@ ifeq (true, $(LINUX_ROOT))
@ # install configuration file(s)
@ echo "Installing configuration files to $(KONFIG_DIR)"
@ mkdir -p $(KONFIG_DIR)
- @ cp $(TOP)/libs/kfg/default.kfg $(KONFIG_DIR)/ncbi-vdb.kfg
+ if [ -f $(KONFIG_DIR)/certs.kfg ] ; \
+ then mv -v $(KONFIG_DIR)/certs.kfg $(KONFIG_DIR)/certs.kfg.orig ; fi
+ if [ -f $(KONFIG_DIR)/default.kfg ] ; \
+ then mv -v $(KONFIG_DIR)/default.kfg $(KONFIG_DIR)/default.kfg.orig ; fi
+ if [ -f $(KONFIG_DIR)/ncbi-vdb.kfg ] ; \
+ then mv -v $(KONFIG_DIR)/ncbi-vdb.kfg $(KONFIG_DIR)/ncbi-vdb.kfg.orig ; fi
+ @ cp $(TOP)/libs/kfg/certs.kfg $(KONFIG_DIR)
+ @ cp $(TOP)/libs/kfg/default.kfg $(KONFIG_DIR)
@ #
@ echo "Updating $(PROFILE_FILE).[c]sh"
@ printf \
@@ -102,7 +109,17 @@ else
@ #
@ echo "Installing configuration files to $(LIB_TARGET)/ncbi/"
@ mkdir -p $(LIB_TARGET)/ncbi
- @ cp $(TOP)/libs/kfg/default.kfg $(LIB_TARGET)/ncbi/ncbi-vdb.kfg
+ if [ -f $(LIB_TARGET)/ncbi/certs.kfg ] ; then \
+ mv -v $(LIB_TARGET)/ncbi/certs.kfg \
+ $(LIB_TARGET)/ncbi/certs.kfg.orig ; fi
+ if [ -f $(LIB_TARGET)/ncbi/default.kfg ] ; then \
+ mv -v $(LIB_TARGET)/ncbi/default.kfg \
+ $(LIB_TARGET)/ncbi/default.kfg.orig ; fi
+ if [ -f $(LIB_TARGET)/ncbi/ncbi-vdb.kfg ] ; then \
+ mv -v $(LIB_TARGET)/ncbi/ncbi-vdb.kfg \
+ $(LIB_TARGET)/ncbi/ncbi-vdb.kfg.orig ; fi
+ @ cp $(TOP)/libs/kfg/certs.kfg $(LIB_TARGET)/ncbi
+ @ cp $(TOP)/libs/kfg/default.kfg $(LIB_TARGET)/ncbi
@ #
@ echo "Please add $(LIB_TARGET) to your LD_LIBRARY_PATH, e.g.:"
@ echo " export LD_LIBRARY_PATH=$(LIB_TARGET):\$$LD_LIBRARY_PATH"
@@ -117,7 +134,10 @@ endif
#
TO_UNINSTALL = $(LIB_TARGET)/libncbi-vdb.* $(LIB_TARGET)/libncbi-wvdb.* $(LIB_TARGET)/libncbi-ngs-c++.*
-TO_UNINSTALL_AS_ROOT = $(INCLUDE_SYMLINK) $(KONFIG_DIR)/ncbi-vdb.kfg $(PROFILE_FILE).sh $(PROFILE_FILE).csh
+ifneq (true, $(LINUX_ROOT))
+ TO_UNINSTALL += $(LIB_TARGET)/ncbi/ncbi-vdb.kfg $(LIB_TARGET)/ncbi/default.kfg
+endif
+TO_UNINSTALL_AS_ROOT = $(INCLUDE_SYMLINK) $(KONFIG_DIR)/ncbi-vdb.kfg $(KONFIG_DIR)/default.kfg $(PROFILE_FILE).sh $(PROFILE_FILE).csh
uninstall:
@ echo "Uninstalling $(TO_UNINSTALL) ..."
diff --git a/build/Makefile.vers b/build/Makefile.vers
index 6cfa60e..bc4c86c 100644
--- a/build/Makefile.vers
+++ b/build/Makefile.vers
@@ -23,4 +23,4 @@
# ===========================================================================
# NCBI-VDB and library version
-VERSION = 2.7.0
+VERSION = 2.8.0
diff --git a/build/ld.linux.exe_cmd.sh b/build/ld.linux.exe_cmd.sh
new file mode 100644
index 0000000..70a5b23
--- /dev/null
+++ b/build/ld.linux.exe_cmd.sh
@@ -0,0 +1 @@
+EXE_CMD="$LD -static-libstdc++ -static-libgcc"
diff --git a/interfaces/kdb/rowset-impl.h b/interfaces/kdb/rowset-impl.h
new file mode 100644
index 0000000..da77b29
--- /dev/null
+++ b/interfaces/kdb/rowset-impl.h
@@ -0,0 +1,141 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#ifndef _h_kdb_rowset_impl_
+#define _h_kdb_rowset_impl_
+
+#ifndef _h_kdb_extern_
+#include <kdb/extern.h>
+#endif
+
+#ifndef _h_kfc_refcount_impl_
+#include <kfc/refcount-impl.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef KROWSET
+#define KROWSET KRowSet
+#endif
+
+#ifndef KROWSET_IT
+#define KROWSET_IT KRowSetIterator
+#endif
+
+#ifndef KROWSET_DATA
+#define KROWSET_DATA void
+#endif
+
+/* we do not want to expand the RC codes now */
+#define rcRowSet rcVector
+\
+
+/*--------------------------------------------------------------------------
+ * forwards
+ */
+typedef struct KRowSet_v1_vt KRowSet_v1_vt;
+typedef struct KRowSetIterator_v1_vt KRowSetIterator_v1_vt;
+
+extern KITFTOK_DECL ( KRowSet );
+extern KITFTOK_DECL ( KRowSetIterator );
+
+/*--------------------------------------------------------------------------
+ * KRowSet
+ */
+struct KRowSet
+{
+ KRefcount_v1 dad;
+ const struct KTable * table;
+
+ /* "morphable" state */
+ KROWSET_DATA * data;
+};
+
+
+/*--------------------------------------------------------------------------
+ * KRowSet_v1_vt
+ */
+struct KRowSet_v1_vt
+{
+ KVTable dad;
+
+ /* start minor version == 0 */
+ void ( CC * destroy_data ) ( KROWSET_DATA * self, ctx_t ctx );
+ /* probably need a serialization function */
+ void ( CC * add_row_id_range ) ( struct KROWSET * self, ctx_t ctx, int64_t start_row_id, uint64_t count );
+ uint64_t ( CC * get_num_rows ) ( const struct KROWSET * self, ctx_t ctx );
+ bool ( CC * has_row_id ) ( const struct KROWSET * self, ctx_t ctx, int64_t row_id );
+ struct KROWSET_IT * ( CC * get_iterator ) ( const struct KROWSET * self, ctx_t ctx );
+ /* end minor version == 0 */
+};
+
+/* Init
+ * initialize a newly allocated RowSet object
+ */
+KDB_EXTERN void CC KRowSetInit ( struct KRowSet *self, ctx_t ctx, const KVTable *kvt,
+ const char *classname );
+
+/* MakeRowSet
+ * create specific instance of a rowset
+ */
+KDB_EXTERN KRowSet * CC KTableMakeRowSetSimple ( struct KTable const * self, ctx_t ctx );
+
+/*--------------------------------------------------------------------------
+ * KRowSetIterator
+ */
+struct KRowSetIterator
+{
+ KRefcount_v1 dad;
+};
+
+/*--------------------------------------------------------------------------
+ * KRowSetIterator_v1_vt
+ */
+struct KRowSetIterator_v1_vt
+{
+ KVTable dad;
+
+ /* start minor version == 0 */
+ bool ( CC * next ) ( struct KROWSET_IT * self, ctx_t ctx );
+ bool ( CC * is_valid ) ( const struct KROWSET_IT * self );
+ int64_t ( CC * get_row_id ) ( const struct KROWSET_IT * self, ctx_t ctx );
+ /* end minor version == 0 */
+};
+
+/* Init
+ * initialize a newly allocated RowSet object
+ */
+KDB_EXTERN void CC KRowSetIteratorInit ( struct KRowSetIterator *self, ctx_t ctx, const KVTable *kvt,
+ const char *classname );
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _h_kdb_rowset_impl_ */
diff --git a/interfaces/kdb/rowset.h b/interfaces/kdb/rowset.h
new file mode 100644
index 0000000..ab35cd6
--- /dev/null
+++ b/interfaces/kdb/rowset.h
@@ -0,0 +1,186 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#ifndef _h_kdb_rowset_
+#define _h_kdb_rowset_
+
+#ifndef _h_kfc_refcount_
+#include <kfc/refcount.h>
+#endif
+
+#ifndef _h_kdb_extern_
+#include <kdb/extern.h>
+#endif
+
+#ifndef _h_klib_defs_
+#include <klib/defs.h>
+#endif
+
+#include <stdarg.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*--------------------------------------------------------------------------
+ * forwards
+ */
+struct KTable;
+
+/*--------------------------------------------------------------------------
+ * KRowSet
+ * a subset of matching rows
+ */
+typedef struct KRowSet KRowSet;
+
+/* MakeRowSet
+ * may add others...
+ */
+KDB_EXTERN KRowSet * CC KTableMakeRowSet ( struct KTable const * self, ctx_t ctx );
+
+
+/* Duplicate
+ * Release
+ * ignores NULL references
+ */
+static __inline__
+KRowSet * KRowSetDuplicate ( const KRowSet * self, ctx_t ctx, caps_t rm )
+{
+ return ( KRowSet * ) KRefcountDuplicate_v1 ( TO_REFCOUNT_V1 ( self ), ctx, rm );
+}
+
+static __inline__
+void KRowSetRelease ( const KRowSet * self, ctx_t ctx )
+{
+ KRefcountRelease_v1 ( TO_REFCOUNT_V1 ( self ), ctx );
+}
+
+/* AddRowId
+ * add a single row to set
+ *
+ * "row_id" [ IN ] - row-id to be added
+ */
+KDB_EXTERN void CC KRowSetAddRowId ( KRowSet * self, ctx_t ctx, int64_t row_id );
+
+
+/* AddRowIdRange
+ * adds row-ids within specified range
+ *
+ * "row_id" [ IN ] and "count" [ IN ] - range of row-ids to be added
+ */
+KDB_EXTERN void CC KRowSetAddRowIdRange ( KRowSet * self, ctx_t ctx, int64_t row_id, uint64_t count );
+
+/* needed (? maybe ?)
+ * add an array of row-ids in random order
+ */
+
+
+/* GetNumRowIds
+ * return the number of elements in set
+ */
+KDB_EXTERN uint64_t CC KRowSetGetNumRowIds ( const KRowSet * self, ctx_t ctx );
+
+/* HasRowId
+ * checks if element is present in set
+ */
+KDB_EXTERN bool CC KRowSetHasRowId ( const KRowSet * self, ctx_t ctx, int64_t row_id );
+
+/* Visit
+ * execute a function on each row-id in set
+ */
+KDB_EXTERN void CC KRowSetVisit ( const KRowSet * self, ctx_t ctx,
+ void ( CC * f ) ( ctx_t ctx, int64_t row_id, void * data ), void * data );
+
+/* Intersect
+ * performs an intersection between two sets and returns the result
+ */
+KDB_EXTERN KRowSet * CC KRowSetIntersect ( ctx_t ctx, const KRowSet * a, const KRowSet * b );
+
+/* Union
+ * performs a union between two sets and returns the result
+ */
+KDB_EXTERN KRowSet * CC KRowSetUnion ( ctx_t ctx, const KRowSet * a, const KRowSet * b );
+
+/*--------------------------------------------------------------------------
+ * KRowSetIterator
+ * an iterator over rowset
+ */
+typedef struct KRowSetIterator KRowSetIterator;
+
+
+/* MakeIterator
+ * create an iterator on set
+ * initially set to first row-id in set
+ * iterator always goes over rows in sorted order
+ * after creating an iterator, any modifications of rowset will result in undefined behavior
+ * when accessing the iterator (except IteratorRelease function)
+ */
+KDB_EXTERN KRowSetIterator * CC KRowSetMakeIterator ( const KRowSet * self, ctx_t ctx );
+
+
+/* Duplicate
+ * Release
+ * ignores NULL references
+ */
+static __inline__
+KRowSet * KRowSetIteratorDuplicate ( const KRowSetIterator * self, ctx_t ctx, caps_t rm )
+{
+ return ( KRowSet * ) KRefcountDuplicate_v1 ( TO_REFCOUNT_V1 ( self ), ctx, rm );
+}
+
+static __inline__
+void KRowSetIteratorRelease ( const KRowSetIterator * self, ctx_t ctx )
+{
+ KRefcountRelease_v1 ( TO_REFCOUNT_V1 ( self ), ctx );
+}
+
+/* Next
+ * advance iterator to next row-id
+
+ * advance to first row-id on initial invocation
+ * advance to next row-id subsequently
+ * returns false if no more row-ids are available.
+ */
+KDB_EXTERN bool CC KRowSetIteratorNext ( KRowSetIterator * self, ctx_t ctx );
+
+/*IsValid
+ * check if iterator points to a valid row
+ *
+ * returns false when iterator points somewhere outside of a row set
+ */
+KDB_EXTERN bool CC KRowSetIteratorIsValid ( const KRowSetIterator * self );
+
+/* GetRowId
+ * report current row id
+ */
+KDB_EXTERN int64_t CC KRowSetIteratorGetRowId ( const KRowSetIterator * self, ctx_t ctx );
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _h_kdb_rowset_ */
diff --git a/interfaces/kfc/rc.h b/interfaces/kfc/rc.h
index 4527445..a6a82d1 100644
--- a/interfaces/kfc/rc.h
+++ b/interfaces/kfc/rc.h
@@ -43,6 +43,9 @@ extern "C" {
#undef RC_ENTRY
#define RC_ENTRY( id, txt ) id,
+#undef RC_VFIRST
+#define RC_VFIRST( id1, id2 ) id1 = ( int ) ( ( id2 ) - 1 ),
+
#undef RC_VLAST
#define RC_VLAST( id ) id,
@@ -80,7 +83,8 @@ RC_ENUM ( RCModule )
RC_ENTRY ( rcXML, "xml support" )
RC_ENTRY ( rcSRA, "short read archive" )
RC_VLAST ( rcLastModule_v1_0 )
- RC_ENTRY ( rcKFG = rcLastModule_v1_0, "configuration" )
+ RC_VFIRST ( rcFirstModule_v1_1, rcLastModule_v1_0 )
+ RC_ENTRY ( rcKFG, "configuration" )
RC_ENTRY ( rcAlign, "alignment" )
RC_ENTRY ( rcKrypto, "cryptographic" )
RC_ENTRY ( rcRDBMS, "RDBMS" )
@@ -149,14 +153,15 @@ RC_ENUM ( RCTarget )
RC_ENTRY ( rcDylib, "dynamic library" )
RC_ENTRY ( rcExpression, "expression" )
RC_VLAST ( rcLastTarget_v1_0 )
- RC_ENTRY ( rcProduction = rcLastTarget_v1_0, "schema production" )
+ RC_VFIRST ( rcFirstTarget_v1_1, rcLastTarget_v1_0 )
+ RC_ENTRY ( rcProduction, "schema production" )
RC_ENTRY ( rcEncryptionKey, "encryption key" )
RC_ENTRY ( rcRng, "random number generator" )
RC_ENTRY ( rcCmd, "command" )
RC_ENTRY ( rcData, "data" )
RC_ENTRY ( rcQuery, "query" )
RC_ENTRY ( rcUri, "uri" )
- RC_LAST ( rcLastTarget_v1_1 )
+ RC_LAST ( rcLastTarget_v1_1 )
};
/* RCContext
@@ -223,7 +228,8 @@ RC_ENUM ( RCContext )
RC_ENTRY ( rcEvaluating, "evaluating" )
RC_ENTRY ( rcInflating, "inflating" )
RC_VLAST ( rcLastContext_v1_0 )
- RC_ENTRY ( rcFlushing = rcLastContext_v1_0, "flushing" )
+ RC_VFIRST ( rcFirstContext_v1_1, rcLastContext_v1_0 )
+ RC_ENTRY ( rcFlushing, "flushing" )
RC_ENTRY ( rcAppending, "appending" )
RC_ENTRY ( rcEncrypting, "encrypting" )
RC_ENTRY ( rcDecrypting, "decrypting" )
@@ -232,7 +238,7 @@ RC_ENUM ( RCContext )
RC_ENTRY ( rcRetrieving, "retrieving" )
RC_ENTRY ( rcSending, "sending" )
RC_ENTRY ( rcProcessing, "processing" )
- RC_ENTRY ( rcIdentifying, "type identifying" )
+ RC_ENTRY ( rcIdentifying, "identifying" )
RC_LAST ( rcLastContext_v1_1 )
};
@@ -242,7 +248,8 @@ RC_ENUM ( RCContext )
RC_ENUM ( RCObject )
{
RC_ENTRY ( rcNoObj, NULL )
- RC_ENTRY ( rcLink = ( int ) rcLastTarget_v1_1, "symbolic link" )
+ RC_VFIRST ( rcFirstObject, rcLastTarget_v1_1 )
+ RC_ENTRY ( rcLink, "symbolic link" )
RC_ENTRY ( rcSelf, "self" )
RC_ENTRY ( rcParam, "param" )
RC_ENTRY ( rcOffset, "offset" )
@@ -263,7 +270,8 @@ RC_ENUM ( RCObject )
RC_ENTRY ( rcRow, "row" )
RC_ENTRY ( rcLibrary, "loadable library" )
RC_VLAST ( rcLastObject_v1_0 )
- RC_ENTRY ( rcItem = rcLastObject_v1_0, "item" )
+ RC_VFIRST ( rcFirstObject_v1_1, rcLastObject_v1_0 )
+ RC_ENTRY ( rcItem, "item" )
RC_ENTRY ( rcMode, "mode" )
RC_ENTRY ( rcEncryption, "encryption" )
RC_ENTRY ( rcCrc, "crc" )
@@ -325,7 +333,8 @@ RC_ENUM ( RCState )
RC_ENTRY ( rcIgnored, "ignored" )
RC_ENTRY ( rcOutofrange, "out of range" )
RC_VLAST ( rcLastState_v1_0 )
- RC_ENTRY ( rcOpen = rcLastState_v1_0, "open" )
+ RC_VFIRST ( rcFirstState_v1_1, rcLastState_v1_0 )
+ RC_ENTRY ( rcOpen, "open" )
RC_ENTRY ( rcOutoforder, "out of order" )
RC_ENTRY ( rcNotOpen, "not open" )
RC_ENTRY ( rcUndefined, "undefined" )
diff --git a/interfaces/kfc/xc.h b/interfaces/kfc/xc.h
index 33bfd51..a57e845 100644
--- a/interfaces/kfc/xc.h
+++ b/interfaces/kfc/xc.h
@@ -109,6 +109,8 @@ XSTATE ( xsInconsistent, "inconsistent", rcInconsistent );
XSTATE ( xsUnsupported, "unsupported", rcUnsupported );
XSTATE_EXT ( xsInitFailed, xsFailed, "init failed", rcFailed );
XSTATE ( xsNoPrimary, "missing primary", rcInvalid );
+XSTATE ( xsBadVersion, "bad version", rcBadVersion );
+XSTATE ( xsViolated, "violated", rcViolated );
/*--------------------------------------------------------------------------
* XC
@@ -178,6 +180,10 @@ XC ( xcFileAccessFailed, xoFile, xsAccessFailed );
XC ( xcZlibInitFailed, xoZlib, xsInitFailed );
XC ( xcZlibDataInvalid, xoZlib, xsInvalid );
XC ( xcSecondaryAlignmentMissingPrimary, xoAlignment, xsNoPrimary );
+XC ( xcInterfaceBadVersion, xoInterface, xsBadVersion );
+XC ( xcRefcountViolated, xoRefcount, xsViolated );
+XC ( xcSelfInvalid, xoSelf, xsInvalid );
+XC ( xcParamIncorrect, xoParam, xsIncorrect );
#ifdef __cplusplus
}
diff --git a/interfaces/kfg/kfg-priv.h b/interfaces/kfg/kfg-priv.h
index 8a8dcba..21cbb51 100644
--- a/interfaces/kfg/kfg-priv.h
+++ b/interfaces/kfg/kfg-priv.h
@@ -78,6 +78,12 @@ KFG_EXTERN rc_t CC KConfigGetLoadPath ( const KConfig *self,
KFG_EXTERN rc_t CC KConfigMakeLocal ( KConfig **cfg,
struct KDirectory const * cfgdir );
+
+/* make sure ResolverCgiNode is https */
+KFG_EXTERN rc_t KConfigFixMainResolverCgiNode ( KConfig * self );
+KFG_EXTERN rc_t KConfigFixProtectedResolverCgiNode ( KConfig * self );
+
+
/* PrintDebug
* print configuration including some internal nodes information
*/
diff --git a/interfaces/kfg/properties.h b/interfaces/kfg/properties.h
index 88cf630..3f3e792 100644
--- a/interfaces/kfg/properties.h
+++ b/interfaces/kfg/properties.h
@@ -56,6 +56,12 @@ KFG_EXTERN rc_t CC KConfig_Get_Http_Proxy_Enabled
KFG_EXTERN rc_t CC KConfig_Set_Http_Proxy_Enabled
( KConfig *self, bool enabled );
+/* get/set priority of environmnet vs. configuration for HTTP proxy */
+KFG_EXTERN rc_t CC KConfig_Has_Http_Proxy_Env_Higher_Priority
+ ( const KConfig *self, bool *enabled );
+KFG_EXTERN rc_t CC KConfig_Set_Http_Proxy_Env_Higher_Priority
+ ( KConfig *self, bool enabled );
+
/* get the %HOME% path from config
*/
diff --git a/interfaces/klib/rc.h b/interfaces/klib/rc.h
index 08236ec..5972170 100644
--- a/interfaces/klib/rc.h
+++ b/interfaces/klib/rc.h
@@ -94,14 +94,20 @@ KLIB_EXTERN bool CC GetUnreadRCInfo ( rc_t *rc, const char **filename, const cha
#endif
#ifdef assert
-#define ASSERT_MOD_TARG_CTX() \
- assert ( ( int ) rcLastModule_v1_1 < ( 1 << 5 ) ), \
- assert ( ( int ) rcLastTarget_v1_1 < ( 1 << 6 ) ), \
- assert ( ( int ) rcLastContext_v1_1 < ( 1 << 7 ) )
+#define ASSERT_MOD_TARG_CTX() \
+ assert ( ( int ) rcKFG == ( int ) rcSRA + 1 ), \
+ assert ( ( int ) rcProduction == ( int ) rcExpression + 1 ), \
+ assert ( ( int ) rcFlushing == ( int ) rcInflating + 1 ), \
+ assert ( ( int ) rcLastModule_v1_1 <= ( 1 << 5 ) ), \
+ assert ( ( int ) rcLastTarget_v1_1 <= ( 1 << 6 ) ), \
+ assert ( ( int ) rcLastContext_v1_1 <= ( 1 << 7 ) )
#define ASSERT_OBJ_STATE() \
- assert ( ( int ) rcLastObject_v1_1 < ( 1 << 8 ) ), \
- assert ( ( int ) rcLastState_v1_1 < ( 1 << 6 ) )
+ assert ( ( int ) rcLink == ( int ) rcUri + 1 ), \
+ assert ( ( int ) rcItem == ( int ) rcLibrary + 1 ), \
+ assert ( ( int ) rcOpen == ( int ) rcOutofrange + 1 ), \
+ assert ( ( int ) rcLastObject_v1_1 <= ( 1 << 8 ) ), \
+ assert ( ( int ) rcLastState_v1_1 <= ( 1 << 6 ) )
#else
#define ASSERT_MOD_TARG_CTX() ( void ) 0
@@ -131,6 +137,16 @@ KLIB_EXTERN bool CC GetUnreadRCInfo ( rc_t *rc, const char **filename, const cha
( ( rc_t ) ( obj ) << 6 ) | /* 8 bits */ \
( ( rc_t ) ( state ) ) ) ) /* 6 bits */
+/* RC_FROM_CTX
+ * form an rc but take input from existing CTX()
+ */
+#define RC_FROM_CTX( ctx, obj, state ) \
+ ( rc_t ) ( ASSERT_OBJ_STATE (), \
+ SET_RC_FILE_FUNC_LINE ( \
+ ( ctx ) | /* 18 bits */ \
+ ( ( rc_t ) ( obj ) << 6 ) | /* 8 bits */ \
+ ( ( rc_t ) ( state ) ) ) ) /* 6 bits */
+
/* ResetRCContext
* rewrite rc to reflect different context
* typically used to pass out return codes
diff --git a/interfaces/klib/status.h b/interfaces/klib/status.h
index 736b8bd..f56489a 100644
--- a/interfaces/klib/status.h
+++ b/interfaces/klib/status.h
@@ -68,7 +68,7 @@ KLIB_EXTERN void* CC KStsLibDataGet (void);
/*
* Init()
- * Initialize the debug messages module to a known state
+ * Initialize the status messages module to a known state
*/
KLIB_EXTERN rc_t CC KStsInit (void);
diff --git a/interfaces/klib/text.h b/interfaces/klib/text.h
index d2ae45f..cf554f7 100644
--- a/interfaces/klib/text.h
+++ b/interfaces/klib/text.h
@@ -111,6 +111,12 @@ KLIB_EXTERN rc_t CC StringConcat ( const String **cat,
KLIB_EXTERN String* CC StringSubstr ( const String *str,
String *sub, uint32_t idx, uint32_t len );
+/* StringTrim
+ * trims ascii white-space from both ends
+ * returns trimmed string in "trimmed"
+ */
+KLIB_EXTERN String * CC StringTrim ( const String * str, String * trimmed );
+
/* StringHead
* access the first character
*
diff --git a/interfaces/kns/endpoint.h b/interfaces/kns/endpoint.h
index ff84042..0e4c29c 100644
--- a/interfaces/kns/endpoint.h
+++ b/interfaces/kns/endpoint.h
@@ -50,6 +50,7 @@ struct KNSManager;
typedef uint32_t KEndPointType;
enum
{
+ epInvalid,
epIPV4,
epIPV6,
epIPC
diff --git a/interfaces/kns/http.h b/interfaces/kns/http.h
index ad83600..75b7b18 100644
--- a/interfaces/kns/http.h
+++ b/interfaces/kns/http.h
@@ -71,6 +71,8 @@ KNS_EXTERN rc_t CC KNSManagerSetHTTPTimeouts ( struct KNSManager * self,
* return status is 0 if the path is valid, non-zero otherwise
*
* returned reference to String must be freed via StringWhack.
+ *
+ * DEPRECATED : THIS FUNCTION IS INCORRECT !!!
*/
KNS_EXTERN rc_t CC KNSManagerGetHTTPProxyPath ( struct KNSManager const * self,
struct String const ** proxy );
@@ -149,12 +151,41 @@ KNS_EXTERN rc_t CC KNSManagerMakeTimedClientHttp ( struct KNSManager const *self
struct String const *host, uint32_t port );
+/* MakeClientHttps
+ * create an HTTPS protocol
+ *
+ * "https" [ OUT ] - return parameter for HTTPS object
+ *
+ * "opt_conn" [ IN, NULL OKAY ] - previously opened stream for communications.
+ *
+ * "vers" [ IN ] - https version
+ * the only legal types are 1.0 ( 0x01000000 ) and 1.1 ( 0x01010000 )
+ *
+ * "readMillis" [ IN ] and "writeMillis" - when negative, infinite timeout
+ * when 0, return immediately, positive gives maximum wait time in mS
+ * for reads and writes respectively.
+ *
+ * "host" [ IN ] - parameter to give the host dns name for the connection
+ *
+ * "port" [ IN, DEFAULT ZERO ] - if zero, defaults to standard for scheme
+ * if non-zero, is taken as explicit port specification
+ */
+KNS_EXTERN rc_t CC KNSManagerMakeClientHttps ( struct KNSManager const *self,
+ KClientHttp **https, struct KStream *conn, ver_t vers,
+ struct String const *host, uint32_t port );
+
+KNS_EXTERN rc_t CC KNSManagerMakeTimedClientHttps ( struct KNSManager const *self,
+ KClientHttp **https, struct KStream *opt_conn, ver_t vers,
+ int32_t readMillis, int32_t writeMillis,
+ struct String const *host, uint32_t port );
+
+
/* AddRef
* Release
* ignores NULL references
*/
-KNS_EXTERN rc_t CC KClientHttpAddRef ( const KHttp *self );
-KNS_EXTERN rc_t CC KClientHttpRelease ( const KHttp *self );
+KNS_EXTERN rc_t CC KClientHttpAddRef ( const KClientHttp *self );
+KNS_EXTERN rc_t CC KClientHttpRelease ( const KClientHttp *self );
/* compatibility for existing code */
#define KNSManagerMakeHttp KNSManagerMakeClientHttp
diff --git a/interfaces/kns/kns-mgr-priv.h b/interfaces/kns/kns-mgr-priv.h
index 674ce15..c6404ff 100644
--- a/interfaces/kns/kns-mgr-priv.h
+++ b/interfaces/kns/kns-mgr-priv.h
@@ -34,6 +34,10 @@
#include <klib/defs.h>
#endif
+#ifndef _h_klib_text_
+#include <klib/text.h>
+#endif
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -96,6 +100,48 @@ rc_t KHttpRetrierInit ( KHttpRetrier * self, const char * url, const struct KNSM
bool KHttpRetrierWait ( KHttpRetrier * self, uint32_t status );
rc_t KHttpRetrierDestroy ( KHttpRetrier * self );
+
+typedef struct HttpProxy HttpProxy;
+const HttpProxy * KNSManagerGetHttpProxy ( const struct KNSManager * self );
+const HttpProxy * HttpProxyGetNextHttpProxy ( const HttpProxy * self );
+
+/* N.B.: DO NOT WHACK THE RETURNED http_proxy String !!! */
+void HttpProxyGet ( const HttpProxy * self,
+ const String ** http_proxy, uint16_t * http_proxy_port );
+
+
+/*--------------------------------------------------------------------------
+ * URLBlock
+ * RFC 3986
+ */
+typedef enum
+{
+ st_NONE,
+ st_HTTP,
+ st_HTTPS,
+ st_S3
+} SchemeType;
+
+typedef struct URLBlock URLBlock;
+struct URLBlock
+{
+ String scheme;
+ String host;
+ String path; /* Path includes any parameter portion */
+ String query;
+ String fragment;
+
+ uint32_t port;
+
+ SchemeType scheme_type;
+ bool tls;
+
+ bool port_dflt;
+};
+extern void URLBlockInit ( URLBlock *self );
+extern rc_t ParseUrl ( URLBlock * b, const char * url, size_t url_size );
+
+
#ifdef __cplusplus
}
#endif
diff --git a/interfaces/kns/manager.h b/interfaces/kns/manager.h
index 6718714..6fc6bf4 100644
--- a/interfaces/kns/manager.h
+++ b/interfaces/kns/manager.h
@@ -86,7 +86,7 @@ KNS_EXTERN bool CC KNSManagerIsVerbose ( const KNSManager *self );
* for connects, reads and writes respectively.
*/
KNS_EXTERN rc_t CC KNSManagerSetConnectionTimeouts ( KNSManager *self,
- int32_t connectSecs, int32_t readMillis, int32_t writeMillis );
+ int32_t connectMillis, int32_t readMillis, int32_t writeMillis );
/* Set/Get UserAgent
diff --git a/interfaces/kns/socket.h b/interfaces/kns/socket.h
index 9505f92..1baee5e 100644
--- a/interfaces/kns/socket.h
+++ b/interfaces/kns/socket.h
@@ -42,6 +42,7 @@ extern "C" {
/*--------------------------------------------------------------------------
* forwards
*/
+struct timeout_t;
struct KStream;
struct KEndPoint;
struct KNSManager;
@@ -53,15 +54,19 @@ typedef struct KSocket KSocket;
/* MakeConnection
+ * create a connection-oriented stream
* MakeTimedConnection
+ * create a connection-oriented stream having specific read/write timeouts
* MakeRetryConnection
+ * create a connection-oriented stream before stated timeout, retrying as necessary
* MakeRetryTimedConnection
- * create a connection-oriented stream
+ * create a connection-oriented stream having specific read/write timeouts,
+ * before stated timeout, retrying as necessary
*
* "conn" [ OUT ] - a stream for communication with the server
*
- * "retryTimeout" [ IN ] - if connection is refused, retry with 1ms intervals: when negative, retry infinitely,
- * when 0, do not retry, positive gives maximum wait time in seconds
+ * "retryTimeout" [ IN ] - the connect request should be repeated upon failure
+ * until this timeout expires.
*
* "readMillis" [ IN ] and "writeMillis" - when negative, infinite timeout
* when 0, return immediately, positive gives maximum wait time in mS
@@ -74,18 +79,20 @@ typedef struct KSocket KSocket;
* both endpoints have to be of type epIP; creates a TCP connection
*/
KNS_EXTERN rc_t CC KNSManagerMakeConnection ( struct KNSManager const * self,
- struct KSocket **conn, struct KEndPoint const *from, struct KEndPoint const *to );
+ struct KSocket ** conn, struct KEndPoint const * from, struct KEndPoint const * to );
KNS_EXTERN rc_t CC KNSManagerMakeTimedConnection ( struct KNSManager const * self,
- struct KSocket **conn, int32_t readMillis, int32_t writeMillis,
- struct KEndPoint const *from, struct KEndPoint const *to );
+ struct KSocket ** conn, int32_t readMillis, int32_t writeMillis,
+ struct KEndPoint const * from, struct KEndPoint const * to );
KNS_EXTERN rc_t CC KNSManagerMakeRetryConnection ( struct KNSManager const * self,
- struct KSocket **conn, int32_t retryTimeout, struct KEndPoint const *from, struct KEndPoint const *to );
+ struct KSocket ** conn, struct timeout_t * retryTimeout,
+ struct KEndPoint const * from, struct KEndPoint const * to );
KNS_EXTERN rc_t CC KNSManagerMakeRetryTimedConnection ( struct KNSManager const * self,
- struct KSocket **conn, int32_t retryTimeout, int32_t readMillis, int32_t writeMillis,
- struct KEndPoint const *from, struct KEndPoint const *to );
+ struct KSocket ** conn, struct timeout_t * retryTimeout,
+ int32_t readMillis, int32_t writeMillis,
+ struct KEndPoint const * from, struct KEndPoint const * to );
/* AddRef
diff --git a/interfaces/kns/manager.h b/interfaces/kns/tls.h
similarity index 54%
copy from interfaces/kns/manager.h
copy to interfaces/kns/tls.h
index 6718714..0f51dc6 100644
--- a/interfaces/kns/manager.h
+++ b/interfaces/kns/tls.h
@@ -23,8 +23,8 @@
* ===========================================================================
*
*/
-#ifndef _h_kns_manager_
-#define _h_kns_manager_
+#ifndef _h_kns_tls_
+#define _h_kns_tls_
#ifndef _h_kns_extern_
#include <kns/extern.h>
@@ -42,62 +42,59 @@ extern "C" {
/*--------------------------------------------------------------------------
* forwards
*/
+struct String;
+struct KSocket;
+struct KStream;
+struct KNSManager;
/*--------------------------------------------------------------------------
- * KNSManager
- * manages the network system
+ * KTLSStream
+ * a stream encrypted with TLS
*/
-typedef struct KNSManager KNSManager;
+typedef struct KTLSStream KTLSStream;
-/* Make
- * create a manager instance
+/* MakeTLSStream
+ * create a TLS wrapper upon an existing KSocket
+ *
+ * data read from or written to TLS stream is in plaintext
+ * data from to or wrapped stream is in ciphertext
+ *
+ * "plaintext" [ OUT ] - return parameter for TLS stream
+ *
+ * "ciphertext" [ IN ] - existing stream of ciphertext
+ *
+ * "host" [ IN ] - host dns name
*/
-KNS_EXTERN rc_t CC KNSManagerMake ( KNSManager **mgr );
+KNS_EXTERN rc_t CC KNSManagerMakeTLSStream ( struct KNSManager const * self,
+ KTLSStream ** plaintext, struct KSocket const * ciphertext,
+ struct String const * host );
/* AddRef
* Release
* ignores NULL references
*/
-KNS_EXTERN rc_t CC KNSManagerAddRef ( const KNSManager *self );
-KNS_EXTERN rc_t CC KNSManagerRelease ( const KNSManager *self );
-
+KNS_EXTERN rc_t CC KTLSStreamAddRef ( const KTLSStream * self );
+KNS_EXTERN rc_t CC KTLSStreamRelease ( const KTLSStream * self );
-/* SetVerbose
- * set/clear verbosity flag of manager ( dflt is false )...
- * the network-code has to request it
- */
-KNS_EXTERN void CC KNSManagerSetVerbose ( KNSManager *self, bool verbosity );
-
-/* IsVerbose
- * request the verbosity flag of manager ( dflt is false )...
- */
-KNS_EXTERN bool CC KNSManagerIsVerbose ( const KNSManager *self );
-
-
-/* SetConnectionTimeouts
- * sets default connect/read/write timeouts to supply to sockets
- *
- * "connectMillis", "readMillis", "writeMillis" [ IN ] - when negative, infinite timeout
- * when 0, return immediately, positive gives maximum wait time in sec/mS
- * for connects, reads and writes respectively.
+/* VerifyCACert
*/
-KNS_EXTERN rc_t CC KNSManagerSetConnectionTimeouts ( KNSManager *self,
- int32_t connectSecs, int32_t readMillis, int32_t writeMillis );
+KNS_EXTERN rc_t CC KTLSStreamVerifyCACert ( const KTLSStream * self );
-/* Set/Get UserAgent
- * for http connections
+/* GetStream
+ * reference-counted cast operation
+ * creates a reference to a KStream from a KTLSStream
*/
-KNS_EXTERN rc_t CC KNSManagerSetUserAgent ( KNSManager *self, const char * fmt, ... );
-KNS_EXTERN rc_t CC KNSManagerGetUserAgent ( const char ** user_agent );
+KNS_EXTERN rc_t CC KTLSStreamGetStream ( const KTLSStream * self,
+ struct KStream ** strm );
#ifdef __cplusplus
}
#endif
-#endif /* _h_kns_manager_ */
+#endif /* _h_kns_tls_ */
diff --git a/interfaces/kproc/timeout.h b/interfaces/kproc/timeout.h
index 738c9c3..7ba6084 100644
--- a/interfaces/kproc/timeout.h
+++ b/interfaces/kproc/timeout.h
@@ -52,13 +52,19 @@ typedef struct timeout_t timeout_t;
/* Init
* initialize a timeout in milliseconds
*/
-KPROC_EXTERN rc_t CC TimeoutInit ( timeout_t *tm, uint32_t msec );
+KPROC_EXTERN rc_t CC TimeoutInit ( timeout_t * tm, uint32_t msec );
/* Prepare
* ensures that a timeout is prepared with an absolute value
*/
-KPROC_EXTERN rc_t CC TimeoutPrepare ( timeout_t *tm );
+KPROC_EXTERN rc_t CC TimeoutPrepare ( timeout_t * tm );
+
+
+/* Remaining
+ * ask how many milliseconds remain before timeout expires
+ */
+KPROC_EXTERN uint32_t TimeoutRemaining ( timeout_t * tm );
#ifdef __cplusplus
diff --git a/interfaces/ktst/unit_test_suite.hpp b/interfaces/ktst/unit_test_suite.hpp
index d534fc7..9967c0c 100644
--- a/interfaces/ktst/unit_test_suite.hpp
+++ b/interfaces/ktst/unit_test_suite.hpp
@@ -72,7 +72,7 @@
#define THROW_ON_RC(call) \
do { \
if ( ( rc_t ) ( call ) != ( rc_t ) 0 ) \
- throw std::logic_error ( string ( __func__ ) + #call + " failed" ); \
+ throw std::logic_error ( std::string ( __func__ ) + #call + " failed" ); \
} while (0)
////////////////////////////////////////////////////////////////////////////////
@@ -446,7 +446,8 @@ public:
const std::string& GetName(void) const { return _name; }
ncbi::NK::counter_t GetErrorCounter(void) { return _ec; }
protected:
- void SetErrorCounter(ncbi::NK::counter_t ec) { _ec = ec; }
+ void SetErrorCounter(ncbi::NK::counter_t ec)
+ { _ec = ec; }
private:
const std::string _name;
ncbi::NK::counter_t _ec;
diff --git a/interfaces/vdb/manager.h b/interfaces/vdb/manager.h
index a21f769..bf94a3a 100644
--- a/interfaces/vdb/manager.h
+++ b/interfaces/vdb/manager.h
@@ -214,10 +214,18 @@ VDB_EXTERN rc_t CC VDBManagerGetCacheRoot ( const VDBManager * self,
* forgotten. If the user wants to make the change permanent, he has to obtain
* a reference to the config-manager 'KConfig' and call KConfigCommit().
*/
-
VDB_EXTERN rc_t CC VDBManagerSetCacheRoot ( const VDBManager * self,
struct VPath const * path );
-
+
+/* DeleteCacheOlderThan
+ * delete old cached files
+ *
+ * days [ IN ] - files older than this will be deleted
+ *
+ */
+VDB_EXTERN rc_t CC VDBManagerDeleteCacheOlderThan ( const VDBManager * self,
+ uint32_t days );
+
#ifdef __cplusplus
}
diff --git a/interfaces/vdb/vdb.vschema b/interfaces/vdb/vdb.vschema
index 15d8f67..be3225d 100644
--- a/interfaces/vdb/vdb.vschema
+++ b/interfaces/vdb/vdb.vschema
@@ -354,7 +354,7 @@ function < type T >
T integral #1.0 ( T in )
= vdb:integral;
function < type T >
-T integral_0 #1.0.1 ( T in )
+T integral_0 #1.1 ( T in )
= vdb:integral_0;
/* delta
diff --git a/interfaces/vfs/manager-priv.h b/interfaces/vfs/manager-priv.h
index c146dd0..f76003f 100644
--- a/interfaces/vfs/manager-priv.h
+++ b/interfaces/vfs/manager-priv.h
@@ -144,6 +144,15 @@ VFS_EXTERN rc_t CC VFSManagerGetCacheRoot ( const struct VFSManager * self,
VFS_EXTERN rc_t CC VFSManagerSetCacheRoot ( const struct VFSManager * self,
struct VPath const * path );
+/* DeleteCacheOlderThan
+ * delete old cached files
+ *
+ * days [ IN ] - files older than this will be deleted
+ *
+ */
+VFS_EXTERN rc_t CC VFSManagerDeleteCacheOlderThan ( const struct VFSManager * self,
+ uint32_t days );
+
#ifdef __cplusplus
}
diff --git a/interfaces/vfs/resolver.h b/interfaces/vfs/resolver.h
index 46024d5..4611165 100644
--- a/interfaces/vfs/resolver.h
+++ b/interfaces/vfs/resolver.h
@@ -45,6 +45,7 @@ extern "C" {
*/
struct KFile;
struct VPath;
+struct String;
struct KConfig;
struct KRepository;
struct VFSManager;
@@ -72,17 +73,64 @@ VFS_EXTERN rc_t CC VResolverAddRef ( const VResolver * self );
VFS_EXTERN rc_t CC VResolverRelease ( const VResolver * self );
-/* Accepted protocol list ordered by preference */
+/* VRemoteProtocols
+ * accepted protocol list
+ * there is a simple set of protocols
+ * where multiple protocols are involved,
+ * they are ordered by preference from LSB toward MSB
+ */
typedef uint32_t VRemoteProtocols;
enum
{
- eProtocolHttp
- , eProtocolFasp
- , eProtocolFaspHttp
- , eProtocolHttpFasp
- , eProtocolLastDefined
+ /* version 1.1 protocols */
+ eProtocolNone = 0
+ , eProtocolHttp = 1
+ , eProtocolFasp = 2
+
+ /* version 1.2 protocols */
+ , eProtocolHttps = 3
+
+ /* values 3..7 are available for future */
+
+ , eProtocolLast
+ , eProtocolMax = eProtocolLast - 1
+ , eProtocolMask = 7
+
+ , eProtocolMaxPref = 3
+
+ /* macros for building multi-protocol constants
+ ordered by preference from least to most significant bits */
+#define VRemoteProtocolsMake2( p1, p2 ) \
+ ( ( ( VRemoteProtocols ) ( p1 ) & eProtocolMask ) | \
+ ( ( ( VRemoteProtocols ) ( p2 ) & eProtocolMask ) << ( 3 * 1 ) ) )
+
+#define VRemoteProtocolsMake3( p1, p2, p3 ) \
+ ( VRemoteProtocolsMake2 ( p1, p2 ) | \
+ ( ( ( VRemoteProtocols ) ( p3 ) & eProtocolMask ) << ( 3 * 2 ) ) )
+
+ , eProtocolFaspHttp = VRemoteProtocolsMake2 ( eProtocolFasp, eProtocolHttp )
+ , eProtocolHttpFasp = VRemoteProtocolsMake2 ( eProtocolHttp, eProtocolFasp )
+ , eProtocolHttpsHttp = VRemoteProtocolsMake2 ( eProtocolHttps, eProtocolHttp )
+ , eProtocolHttpHttps = VRemoteProtocolsMake2 ( eProtocolHttp, eProtocolHttps )
+ , eProtocolFaspHttps = VRemoteProtocolsMake2 ( eProtocolFasp, eProtocolHttps )
+ , eProtocolHttpsFasp = VRemoteProtocolsMake2 ( eProtocolHttps, eProtocolFasp )
+ , eProtocolFaspHttpHttps = VRemoteProtocolsMake3 ( eProtocolFasp, eProtocolHttp, eProtocolHttps )
+ , eProtocolHttpFaspHttps = VRemoteProtocolsMake3 ( eProtocolHttp, eProtocolFasp, eProtocolHttps )
+ , eProtocolFaspHttpsHttp = VRemoteProtocolsMake3 ( eProtocolFasp, eProtocolHttps, eProtocolHttp )
+ , eProtocolHttpHttpsFasp = VRemoteProtocolsMake3 ( eProtocolHttp, eProtocolHttps, eProtocolFasp )
+ , eProtocolHttpsFaspHttp = VRemoteProtocolsMake3 ( eProtocolHttps, eProtocolFasp, eProtocolHttp )
+ , eProtocolHttpsHttpFasp = VRemoteProtocolsMake3 ( eProtocolHttps, eProtocolHttp, eProtocolFasp )
};
+/* Parse
+ * parses a comma-separated list of case-insensitive protocols:
+ * 'http', 'https', 'fasp'
+ *
+ * trims white-space, ignores unrecognized and empty terms
+ *
+ * returns an ordered list of valid protocols
+ */
+VFS_EXTERN VRemoteProtocols CC VRemoteProtocolsParse ( struct String const * protos );
/* Query
* resolve object location to either an existing local path,
diff --git a/libs/align/bam.c b/libs/align/bam.c
index ebae8b2..51c85dd 100644
--- a/libs/align/bam.c
+++ b/libs/align/bam.c
@@ -3990,6 +3990,8 @@ rc_t LoadIndex1(const uint8_t data[], size_t dlen, unsigned refNo,
ctx->intervals = (ctx->self->refSeq[refNo].length + 16383) >> 14;
if (type == bai_16kIntervals && elements > ctx->intervals)
return RC(rcAlign, rcIndex, rcReading, rcData, rcExcessive);
+ if (type == bai_StartStopPairs && bin2ival(binNo) > ctx->intervals)
+ return RC(rcAlign, rcIndex, rcReading, rcData, rcExcessive);
}
return 0;
}
@@ -4067,7 +4069,7 @@ rc_t LoadIndex2a(const uint8_t data[], size_t dlen, unsigned refNo,
memset(minOffset, 0, sizeof(minOffset));
for (i = 0; i != MAX_BIN; ++i) {
const unsigned ival = bin2ival(i);
- unsigned n_ival = bin_ival_count(i);
+ unsigned const n_ival = bin_ival_count(i);
cp = ctx->bins[i];
if (cp == 0)
@@ -4075,9 +4077,6 @@ rc_t LoadIndex2a(const uint8_t data[], size_t dlen, unsigned refNo,
if (n_ival <= 1)
break;
- if (ival + n_ival > max_ival)
- n_ival = max_ival - ival;
-
chunk_count = LE2HI32(ctx->base + cp + 4); cp += 8;
for (k = 0; k < chunk_count; ++k) {
const uint64_t start = get_pos(ctx->base + cp);
@@ -4085,7 +4084,7 @@ rc_t LoadIndex2a(const uint8_t data[], size_t dlen, unsigned refNo,
unsigned l;
cp += 16;
- for (l = 0; l != n_ival; ++l) {
+ for (l = 0; ival + l < max_ival; ++l) {
if (start < ctx->refSeq[refNo][ival + l] &&
ctx->refSeq[refNo][ival + l] <= end &&
(start < minOffset[ival + l] ||
diff --git a/libs/ascp/unix/connect.c b/libs/ascp/unix/connect.c
index 26415c5..9ca4773 100644
--- a/libs/ascp/unix/connect.c
+++ b/libs/ascp/unix/connect.c
@@ -427,7 +427,7 @@ rc_t run_ascp(const char *path, const char *key,
got = true;
}
if (!got) {
- sleep(1);
+ KSleepMs(1000);
++i;
if ((heartbeat > 0 && i >= heartbeat) || (i > 99)) {
uint64_t size = 0;
diff --git a/libs/blast/blast-mgr.c b/libs/blast/blast-mgr.c
index 584fec4..1db5838 100644
--- a/libs/blast/blast-mgr.c
+++ b/libs/blast/blast-mgr.c
@@ -50,7 +50,7 @@
#include <stdio.h> /* fprintf */
#include <string.h> /* memset */
-#define TOOLKIT "sratoolkit2_7_0"
+#define TOOLKIT "sratoolkit2_8_0"
/******************************************************************************/
diff --git a/libs/kapp/loader-file.c b/libs/kapp/loader-file.c
index 8d94bb9..7edde50 100644
--- a/libs/kapp/loader-file.c
+++ b/libs/kapp/loader-file.c
@@ -49,11 +49,11 @@
#if _DEBUGGING
# ifndef KLoaderFile_BUFFERSIZE
-# define KLoaderFile_BUFFERSIZE (256 * 1024)
+# define KLoaderFile_BUFFERSIZE (64 * 1024 * 1024)
# endif
#else
# undef KLoaderFile_BUFFERSIZE
-# define KLoaderFile_BUFFERSIZE (256 * 1024)
+# define KLoaderFile_BUFFERSIZE (64 * 1024 * 1024)
#endif
struct KLoaderFile
@@ -426,13 +426,13 @@ LIB_EXPORT rc_t CC KLoaderFile_Readline(const KLoaderFile* cself, const void** b
uint8_t* nl;
bool refilled = false;
bool eof;
-
+
rc = KLoaderFile_IsEof(cself, &eof);
if( rc == 0 && eof ) {
*buffer = NULL;
return 0;
}
-
+
while( rc == 0 ) {
bool CR_last = false;
int i, cnt = self->avail - self->eol;
@@ -471,7 +471,7 @@ LIB_EXPORT rc_t CC KLoaderFile_Readline(const KLoaderFile* cself, const void** b
if( self->buffer_size == self->avail ) {
/* buffer could be copied and next call will provide tail of line */
rc = RC( rcApp, rcFile, rcReading, rcString, rcTooLong);
- }
+ }
} else {
*length = nl - (uint8_t*)*buffer;
self->eol = (uint32_t) ( nl - self->buffer_pos + 1 );
diff --git a/libs/kdb/Makefile b/libs/kdb/Makefile
index 9dbfb99..84a66f3 100644
--- a/libs/kdb/Makefile
+++ b/libs/kdb/Makefile
@@ -78,6 +78,8 @@ $(ILIBDIR)/libkdb: $(addprefix $(ILIBDIR)/libkdb.,$(ILIBEXT))
KDB_CMN = \
btree \
+ rowset \
+ rowset-simple \
dbmgr-cmn \
# database-cmn
diff --git a/libs/kdb/libkdb.vers.h b/libs/kdb/libkdb.vers.h
index 43643a6..cdea506 100644
--- a/libs/kdb/libkdb.vers.h
+++ b/libs/kdb/libkdb.vers.h
@@ -24,4 +24,4 @@
*
*/
-#define LIBKDB_VERS 0x02070014
+#define LIBKDB_VERS 0x02070015
diff --git a/libs/kdb/rowset-it.c_old b/libs/kdb/rowset-it.c_old
new file mode 100644
index 0000000..f7c3bff
--- /dev/null
+++ b/libs/kdb/rowset-it.c_old
@@ -0,0 +1,385 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include <kdb/rowset.h>
+#include "rowset-priv.h"
+#include <klib/refcount.h>
+#include <klib/rc.h>
+#include <klib/out.h>
+
+#include <string.h>
+#include <assert.h>
+
+/*--------------------------------------------------------------------------
+ * KRowSetIterator
+ */
+
+struct KRowSetIterator
+{
+ KRefcount refcount;
+ const KRowSet * rowset;
+ const KRowSetTreeLeaf * leaf; // will be NULL in empty iterator
+ int64_t current_row_id;
+ ptrdiff_t eol; // end of "list"; will be non-zero when there are no more rows and we moved after valid row-id sequence dimensions
+
+ // only for leaf type RangeArray
+ struct {
+ int8_t current_range_index;
+ } range_array;
+};
+
+static
+void KRowSetIteratorStart ( KRowSetIterator * self )
+{
+ assert ( self != NULL );
+ self->current_row_id = -1;
+ self->leaf = KRowSetTreeGetFirstLeaf ( self->rowset );
+ self->eol = self->leaf == NULL ? -1 : 0;
+}
+
+static
+void KRowSetIteratorEnd ( KRowSetIterator * self )
+{
+ assert ( self != NULL );
+ self->current_row_id = -1;
+ self->leaf = KRowSetTreeGetLastLeaf ( self->rowset );
+ self->eol = self->leaf == NULL ? 1 : 0;
+}
+
+static
+bool CC KRowSetIteratorMove ( KRowSetIterator * iter, bool backward )
+{
+ int step = !backward ? 1 : -1;
+
+ assert ( iter->rowset != NULL );
+
+ if ( iter->eol != 0 )
+ {
+ iter->eol += step;
+ if ( iter->eol == 0 )
+ {
+ if ( !backward )
+ KRowSetIteratorStart ( iter );
+ else
+ KRowSetIteratorEnd ( iter );
+ }
+ }
+
+ if ( iter->eol != 0 )
+ return false;
+
+ assert ( iter->leaf != NULL );
+
+ while ( iter->eol == 0 )
+ {
+ int i;
+ int j;
+ int64_t last_row_id;
+ int last_leaf_bt = -1;
+ int64_t leaf_id;
+ int64_t row_id;
+ const KRowSetTreeLeaf * leaf = iter->leaf;
+
+ last_row_id = iter->current_row_id;
+ if ( last_row_id != -1 )
+ last_leaf_bt = last_row_id & 0xFFFF;
+
+ leaf_id = leaf->header.leaf_id;
+ row_id = leaf_id << 16;
+
+ switch ( leaf->header.type )
+ {
+ case LeafType_Bitmap:
+ {
+ int from_i;
+ int from_j;
+ int to_i;
+ int to_j;
+
+
+ if ( last_row_id != -1 && (last_row_id >> 16) == leaf_id )
+ {
+ assert ( last_leaf_bt != -1 );
+ from_i = last_leaf_bt >> 3;
+ from_j = (last_leaf_bt & 7) + step;
+ }
+ else
+ {
+ from_i = !backward ? 0 : LEAF_DATA_SZ_BT - 1;
+ from_j = !backward ? 0 : 8 - 1;
+ }
+
+ if ( !backward )
+ {
+ to_i = LEAF_DATA_SZ_BT;
+ to_j = 8;
+ }
+ else
+ {
+ to_i = -1;
+ to_j = -1;
+ }
+
+ for ( i = from_i; i != to_i; i += step )
+ {
+ if ( leaf->data.bitmap[i] )
+ {
+ int64_t new_row_id = row_id | i << 3;
+ for ( j = from_j; j != to_j; j += step )
+ {
+ if ( leaf->data.bitmap[i] & (1 << j) )
+ {
+ iter->current_row_id = (int64_t)( new_row_id | j );
+ return true;
+ }
+ }
+ }
+ from_j = !backward ? 0 : 8 - 1;
+ }
+
+ break;
+ }
+ case LeafType_ArrayRanges:
+ {
+ int from_i; // range index
+ int from_j;
+ int to_i;
+ int to_j;
+ int len = leaf->data.array_ranges.len;
+
+ if ( last_row_id != -1 && (last_row_id >> 16) == leaf_id )
+ {
+ assert ( last_leaf_bt != -1 );
+ assert ( iter->range_array.current_range_index != -1 );
+
+ from_i = iter->range_array.current_range_index;
+ from_j = last_leaf_bt + step;
+ }
+ else
+ {
+ from_i = !backward ? 0 : len - 1;
+ from_j = !backward ? leaf->data.array_ranges.ranges[from_i].start : leaf->data.array_ranges.ranges[from_i].end;
+ }
+
+ if ( !backward )
+ {
+ to_i = len;
+ }
+ else
+ {
+ to_i = -1;
+ }
+
+ for ( i = from_i; i != to_i; i += step )
+ {
+ const struct KRowSetTreeLeafRowRange * range = &leaf->data.array_ranges.ranges[i];
+ assert ( range->start <= range->end );
+ to_j = (!backward ? range->end : range->start) + step;
+ // single loop iteration just to check conditions
+ for ( j = from_j; j != to_j; j += step )
+ {
+ assert ( j == (j & 0xFFFF) );
+ iter->current_row_id = row_id | j;
+ iter->range_array.current_range_index = i;
+ return true;
+ }
+ from_j = !backward ? (range + step)->start : (range + step)->end;
+ }
+ break;
+ }
+ default:
+ assert ( false );
+ }
+
+ if ( !backward )
+ iter->leaf = KRowSetTreeLeafGetNext ( leaf );
+ else
+ iter->leaf = KRowSetTreeLeafGetPrev ( leaf );
+
+ if ( iter->leaf == NULL )
+ {
+ iter->eol = step;
+ }
+ }
+
+ return false;
+}
+
+KDB_EXTERN rc_t CC KRowSetMakeIterator ( const KRowSet * self, KRowSetIterator ** iter )
+{
+ rc_t rc;
+ KRowSetIterator * it;
+ uint64_t num_rows;
+
+ if ( self == NULL )
+ return RC ( rcDB, rcIterator, rcConstructing, rcSelf, rcNull );
+
+ if ( iter == NULL )
+ return RC ( rcDB, rcIterator, rcConstructing, rcParam, rcNull );
+
+ rc = KRowSetGetNumRowIds ( self, &num_rows );
+ if ( rc != 0 )
+ return rc;
+
+ if ( num_rows == 0 )
+ return RC ( rcDB, rcIterator, rcConstructing, rcItem, rcNotFound );
+
+ it = calloc ( 1, sizeof *it );
+ if ( it == NULL )
+ rc = RC ( rcDB, rcIterator, rcConstructing, rcMemory, rcExhausted );
+ else
+ {
+ KRefcountInit ( &it->refcount, 1, "KRowSetIterator", "new", "" );
+
+ rc = KRowSetAddRef ( self );
+ if ( rc == 0 )
+ {
+ it->rowset = self;
+ KRowSetIteratorFirst ( it );
+ assert ( KRowSetIteratorIsValid ( it ) );
+
+ *iter = it;
+ return 0;
+ }
+
+ free ( it );
+ }
+
+ return rc;
+}
+
+/**
+ * Whack
+ */
+static
+rc_t KRowSetIteratorWhack ( KRowSetIterator * iter )
+{
+ assert ( iter != NULL );
+
+ KRowSetRelease ( iter->rowset );
+ iter->rowset = NULL;
+ free ( iter );
+
+ return 0;
+}
+
+KDB_EXTERN rc_t CC KRowSetIteratorAddRef ( const KRowSetIterator * iter )
+{
+ if ( iter != NULL ) switch ( KRefcountAdd ( & iter -> refcount, "KRowSetIterator" ) )
+ {
+ case krefOkay:
+ break;
+ default:
+ return RC ( rcDB, rcRowSet, rcAttaching, rcConstraint, rcViolated );
+ }
+ return 0;
+}
+KDB_EXTERN rc_t CC KRowSetIteratorRelease ( const KRowSetIterator * iter )
+{
+ if ( iter != NULL ) switch ( KRefcountDrop ( & iter -> refcount, "KRowSetIterator" ) )
+ {
+ case krefOkay:
+ break;
+ case krefWhack:
+ return KRowSetIteratorWhack ( ( KRowSetIterator* ) iter );
+ default:
+ return RC ( rcDB, rcRowSet, rcReleasing, rcConstraint, rcViolated );
+ }
+ return 0;
+}
+
+KDB_EXTERN rc_t CC KRowSetIteratorFirst ( KRowSetIterator * self )
+{
+ if ( self == NULL )
+ return RC ( rcDB, rcIterator, rcPositioning, rcSelf, rcNull );
+
+ KRowSetIteratorStart ( self );
+ KRowSetIteratorNext ( self );
+
+ return 0;
+}
+
+KDB_EXTERN rc_t CC KRowSetIteratorLast ( KRowSetIterator * self )
+{
+ if ( self == NULL )
+ return RC ( rcDB, rcIterator, rcPositioning, rcSelf, rcNull );
+
+ KRowSetIteratorEnd ( self );
+ KRowSetIteratorPrev ( self );
+
+ return 0;
+}
+
+KDB_EXTERN rc_t CC KRowSetIteratorNext ( KRowSetIterator * self )
+{
+ bool moved;
+ if ( self == NULL )
+ return RC ( rcDB, rcIterator, rcPositioning, rcSelf, rcNull );
+
+ moved = KRowSetIteratorMove ( self, false );
+
+ if ( !moved )
+ return RC ( rcDB, rcIterator, rcPositioning, rcItem, rcDone );
+
+ return 0;
+}
+
+KDB_EXTERN rc_t CC KRowSetIteratorPrev ( KRowSetIterator * self )
+{
+ bool moved;
+
+ if ( self == NULL )
+ return RC ( rcDB, rcIterator, rcPositioning, rcSelf, rcNull );
+
+ moved = KRowSetIteratorMove ( self, true );
+
+ if ( !moved )
+ return RC ( rcDB, rcIterator, rcPositioning, rcItem, rcDone );
+
+ return 0;
+}
+
+KDB_EXTERN bool CC KRowSetIteratorIsValid ( const KRowSetIterator * self )
+{
+ return self != NULL && !self->eol;
+}
+
+KDB_EXTERN rc_t CC KRowSetIteratorRowId ( const KRowSetIterator * self, int64_t * row_id )
+{
+ if ( self == NULL )
+ return RC ( rcDB, rcIterator, rcAccessing, rcSelf, rcNull );
+
+ if ( row_id == NULL )
+ return RC ( rcDB, rcIterator, rcAccessing, rcParam, rcNull );
+
+ if ( !KRowSetIteratorIsValid ( self ) )
+ return RC ( rcDB, rcIterator, rcAccessing, rcSelf, rcInvalid );
+
+ assert ( self->current_row_id != -1 );
+
+ *row_id = self->current_row_id;
+ return 0;
+}
diff --git a/libs/kdb/rowset-priv.h_old b/libs/kdb/rowset-priv.h_old
new file mode 100644
index 0000000..c57f5b7
--- /dev/null
+++ b/libs/kdb/rowset-priv.h_old
@@ -0,0 +1,116 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#ifndef _h_kdb_rowset_priv_
+#define _h_kdb_rowset_priv_
+
+#include <klib/container.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if _DEBUGGING
+#define CHECK_NODE_MARKS 1
+#endif
+
+#if CHECK_NODE_MARKS
+#define NODE_MARK 55
+#define LEAF_MARK 99
+#endif
+
+#define LEAF_DEPTH 6 // at which depth leaves are located
+#define LEAF_DATA_SZ_BT 8192
+
+// the maximum value that leaf can represent
+// (leaf data only stores lowest bytes of row_id)
+#define ROW_LEAF_MAX 0xFFFFLL
+
+/*--------------------------------------------------------------------------
+ * KRowSet
+ */
+typedef enum KRowSetTreeLeafType
+{
+ LeafType_Bitmap,
+ LeafType_ArrayRanges
+} KRowSetTreeLeafType;
+
+// Holds bites of row_id followed by current byte (radix trie compression)
+typedef struct KRowSetTreeNodeTransition
+{
+ int8_t size;
+ uint8_t data[5];
+} KRowSetTreeNodeTransition;
+
+typedef struct KRowSetTreeNode
+{
+ void* children[256];
+ KRowSetTreeNodeTransition transitions[256];
+
+#if CHECK_NODE_MARKS
+ uint8_t node_mark;
+#endif
+} KRowSetTreeNode;
+
+typedef struct KRowSetTreeLeafHeader
+{
+ DLNode dad;
+ int64_t leaf_id; // this is basically a row_id >> 16
+ uint64_t leaf_rows; // number of rows set in this leaf
+ uint8_t type; // types are defined in KRowSetTreeLeafType
+
+#if CHECK_NODE_MARKS
+ uint8_t leaf_mark;
+#endif
+} KRowSetTreeLeafHeader;
+
+
+typedef struct KRowSetTreeLeaf
+{
+ KRowSetTreeLeafHeader header;
+ union u
+ {
+ uint8_t bitmap[LEAF_DATA_SZ_BT];
+ struct KRowSetTreeLeafArrayRanges {
+ struct KRowSetTreeLeafRowRange {
+ uint16_t start;
+ uint16_t end;
+ } ranges[8];
+ uint8_t len;
+ } array_ranges;
+ } data;
+} KRowSetTreeLeaf;
+
+const KRowSetTreeLeaf * KRowSetTreeGetFirstLeaf ( const KRowSet * self );
+const KRowSetTreeLeaf * KRowSetTreeGetLastLeaf ( const KRowSet * self );
+const KRowSetTreeLeaf * KRowSetTreeLeafGetNext ( const KRowSetTreeLeaf * leaf );
+const KRowSetTreeLeaf * KRowSetTreeLeafGetPrev ( const KRowSetTreeLeaf * leaf );
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _h_kdb_rowset_priv_ */
diff --git a/libs/kdb/rowset-simple.c b/libs/kdb/rowset-simple.c
new file mode 100644
index 0000000..b05aaa8
--- /dev/null
+++ b/libs/kdb/rowset-simple.c
@@ -0,0 +1,434 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#define KROWSET_IT KRowSetSimpleIterator
+
+#include <kdb/extern.h>
+
+#include <kdb/rowset.h>
+#include <kdb/rowset-impl.h>
+#include <kdb/table.h>
+#include <kfc/ctx.h>
+#include <kfc/except.h>
+#include <kfc/xc.h>
+#include <klib/out.h>
+#include <klib/sort.h>
+
+#include <string.h>
+#include <assert.h>
+
+#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y))
+#define MAX(X, Y) (((X) > (Y)) ? (X) : (Y))
+
+/*--------------------------------------------------------------------------
+ * forwards
+ */
+typedef struct KRowSetSimpleData KRowSetSimpleData;
+typedef struct KRowSetSimpleIterator KRowSetSimpleIterator;
+
+static
+KRowSetSimpleIterator * CC KRowSetSimpleGetIterator ( const struct KRowSet * self, ctx_t ctx );
+
+/*--------------------------------------------------------------------------
+ * KRowSet
+ */
+
+struct KRowSetSimpleData
+{
+ uint64_t size;
+ uint64_t num_rows;
+ bool sorted;
+ int64_t rows_array[1];
+};
+
+static
+KRowSetSimpleData * KRowSetSimpleAllocateData ( ctx_t ctx, uint64_t size )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcAllocating );
+ KRowSetSimpleData * data;
+
+ if ( size <= 0 )
+ INTERNAL_ERROR ( xcParamInvalid, "data size should be greater than zero" );
+ else
+ {
+ data = malloc ( sizeof *data + (size - 1) * sizeof(int64_t) );
+ if ( data == NULL )
+ SYSTEM_ERROR ( xcNoMemory, "out of memory" );
+ else
+ {
+ data -> size = size;
+ data -> num_rows = 0;
+ data -> sorted = true;
+ return data;
+ }
+ }
+
+ return NULL;
+}
+
+static
+KRowSetSimpleData * GetRowSetSimpleData ( KRowSet *self, ctx_t ctx )
+{
+ if ( self == NULL || self -> data == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcAccessing );
+ INTERNAL_ERROR ( xcSelfNull, "failed to get rowset data" );
+ return NULL;
+ }
+
+ return self -> data;
+}
+
+static
+void CC KRowSetSimpleDestroyData ( void *data, ctx_t ctx )
+{
+ free ( data );
+}
+
+static
+bool CC KRowSetSimpleHasRowId ( const KRowSet * self, ctx_t ctx, int64_t row_id )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcSearching );
+ const KRowSetSimpleData * data;
+
+ TRY ( data = GetRowSetSimpleData ( (KRowSet *)self, ctx ) )
+ {
+ /* TBD - only want this algorithm when count > some threshold... */
+ if ( data -> sorted )
+ {
+ uint64_t start, end, idx;
+
+ start = 0;
+ end = data -> num_rows;
+ while (start < end)
+ {
+ idx = (start + end) / 2;
+ if (row_id < data -> rows_array[idx])
+ end = idx;
+ else if (row_id > data -> rows_array[idx])
+ start = idx + 1;
+ else
+ return true;
+ }
+ }
+ else
+ {
+ uint64_t i;
+ for ( i = 0; i < data -> num_rows; ++i )
+ {
+ if ( data -> rows_array[i] == row_id )
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+static
+void KRowSetSimpleDataSort ( KRowSetSimpleData * data )
+{
+ if ( data == NULL || data -> sorted || data -> num_rows <= 1 )
+ return;
+
+ ksort_int64_t ( data -> rows_array, data -> num_rows );
+ data -> sorted = true;
+}
+
+static
+void AppendRowId ( KRowSet *self, ctx_t ctx, int64_t row_id )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcInserting );
+ KRowSetSimpleData * data;
+ TRY ( data = GetRowSetSimpleData ( self, ctx ) )
+ {
+ if ( data -> size - data -> num_rows == 0 )
+ {
+ KRowSetSimpleData * old_data = data;
+ uint64_t new_size;
+ if ( data -> size == SIZE_MAX )
+ {
+ INTERNAL_ERROR ( xcIntegerOutOfBounds, "cannot allocate bigger simple container for row id insertion" );
+ return;
+ }
+ if ( SIZE_MAX - data -> size > data -> size )
+ new_size = data -> size * 2;
+ else
+ new_size = SIZE_MAX;
+
+ TRY ( data = KRowSetSimpleAllocateData ( ctx, new_size ) )
+ {
+ memcpy ( &data -> rows_array, &old_data -> rows_array, old_data -> num_rows * sizeof(int64_t) );
+ data -> num_rows = old_data -> num_rows;
+ data -> sorted = old_data -> sorted;
+ self -> data = data;
+ ON_FAIL ( KRowSetSimpleDestroyData ( old_data, ctx ) )
+ return;
+ }
+ else
+ return;
+ }
+ if ( data -> sorted )
+ data -> sorted = data -> num_rows == 0 || data -> rows_array[ data -> num_rows - 1 ] <= row_id;
+ data -> rows_array[ data -> num_rows++ ] = row_id;
+ }
+}
+
+static
+void CC KRowSetSimpleAddRowIdRange ( KRowSet *self, ctx_t ctx, int64_t start_row_id, uint64_t count )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcInserting );
+ uint64_t i;
+
+ if ( (uint64_t)INT64_MAX - start_row_id < count )
+ {
+ USER_ERROR ( xcIntegerOutOfBounds, "bad row id range" );
+ return;
+ }
+
+
+ for ( i = 0; i < count; ++i )
+ {
+ int64_t row_id = i + start_row_id;
+ bool row_exists;
+ TRY ( row_exists = KRowSetSimpleHasRowId ( self, ctx, row_id ) )
+ {
+ if ( row_exists )
+ continue;
+
+ ON_FAIL ( AppendRowId ( self, ctx, row_id ) )
+ break;
+ }
+ }
+}
+
+static
+uint64_t CC KRowSetSimpleGetNumRowIds ( const KRowSet * self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcAccessing );
+ const KRowSetSimpleData * data;
+ TRY ( data = GetRowSetSimpleData ( (KRowSet *) self, ctx ) )
+ {
+ return data -> num_rows;
+ }
+
+ return 0;
+}
+
+static
+void CC KRowSetSimpleDestroy ( KRefcount_v1 *self_refcount, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcDestroying );
+
+ KRowSet * self = (KRowSet *)self_refcount;
+
+ const KRowSet_v1_vt * vt = KVTABLE_CAST ( TO_REFCOUNT_V1 ( self ) -> vt, ctx, KRowSet );
+ if ( vt == NULL )
+ INTERNAL_ERROR ( xcInterfaceIncorrect, "this object does not support the KRowSet interface" );
+ else
+ {
+ KTableRelease ( self -> table );
+ self -> table = NULL;
+
+ assert ( vt != NULL );
+ assert ( vt -> destroy_data != NULL );
+
+ ( * vt -> destroy_data ) ( self -> data, ctx );
+ self -> data = NULL;
+ }
+}
+
+static KRefcount_v1_vt vtKRowSetSimpleRefCount =
+{
+ KVTABLE_INITIALIZER ( KRowSetSimpleRefCount, KRefcount_v1, 0, NULL ),
+ KRowSetSimpleDestroy,
+ NULL
+};
+
+static KRowSet_v1_vt vtKRowSetSimple =
+{
+ KVTABLE_INITIALIZER ( KRowSetSimple, KRowSet, 0, &vtKRowSetSimpleRefCount.dad ),
+
+ KRowSetSimpleDestroyData,
+ KRowSetSimpleAddRowIdRange,
+ KRowSetSimpleGetNumRowIds,
+ KRowSetSimpleHasRowId,
+ KRowSetSimpleGetIterator
+};
+
+KDB_EXTERN KRowSet * CC KTableMakeRowSetSimple ( struct KTable const * self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcConstructing );
+ if ( self == NULL )
+ INTERNAL_ERROR ( xcSelfNull, "failed to construct rowset: table is NULL" );
+ else
+ {
+ KRowSet *r;
+
+ r = calloc ( 1, sizeof *r );
+ if ( r == NULL )
+ SYSTEM_ERROR ( xcNoMemory, "out of memory" );
+ else
+ {
+ TRY ( KRowSetInit ( r, ctx, &vtKRowSetSimple.dad, "KRowSetSimple" ) )
+ {
+ TRY ( r -> data = KRowSetSimpleAllocateData ( ctx, 16 ) )
+ {
+ rc_t rc;
+ r -> table = self;
+ rc = KTableAddRef ( r -> table );
+ if ( rc != 0 )
+ INTERNAL_ERROR ( xcUnexpected, "unknown result from KTableAddRef: rc = %R", rc );
+ else
+ return r;
+ }
+ }
+ free ( r );
+ }
+ }
+
+ return NULL;
+}
+
+/*--------------------------------------------------------------------------
+ * KRowSetSimpleIterator
+ */
+
+struct KRowSetSimpleIterator
+{
+ KRowSetIterator dad;
+ const KRowSet * rowset;
+ const KRowSetSimpleData * rowset_data;
+ uint64_t array_index;
+};
+
+static
+void CC KRowSetSimpleIteratorDestroy ( KRefcount_v1 *self_refcount, ctx_t ctx )
+{
+ KRowSetSimpleIterator * self = (KRowSetSimpleIterator *)self_refcount;
+ if ( self == NULL || self -> rowset == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcDestroying );
+ INTERNAL_ERROR ( xcSelfNull, "failed to destroy rowset iterator" );
+ }
+ else
+ {
+ KRowSetRelease ( self -> rowset, ctx );
+ }
+}
+
+static
+bool CC KRowSetSimpleIteratorNext ( struct KRowSetSimpleIterator * self, ctx_t ctx )
+{
+ if ( self == NULL || self -> rowset_data == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcPositioning );
+ INTERNAL_ERROR ( xcSelfNull, "failed to move rowset iterator" );
+ }
+ else if ( self -> array_index == self -> rowset_data -> num_rows )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcPositioning );
+ INTERNAL_ERROR ( xcIteratorExhausted, "failed to move rowset iterator - no more elements" );
+ }
+ else
+ return ++self -> array_index < self -> rowset_data -> num_rows;
+
+ return false;
+}
+
+static
+bool CC KRowSetSimpleIteratorIsValid ( const struct KRowSetSimpleIterator * self )
+{
+ if (self == NULL || self -> rowset_data == NULL || self -> rowset == NULL || self -> rowset_data != self -> rowset -> data )
+ return false;
+
+ return self -> array_index < self -> rowset_data -> num_rows;
+}
+
+static
+int64_t CC KRowSetSimpleIteratorGetRowId ( const struct KRowSetSimpleIterator * self, ctx_t ctx )
+{
+ if ( !KRowSetSimpleIteratorIsValid ( self ) )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcAccessing );
+ if (self == NULL || self -> rowset_data == NULL || self -> rowset == NULL )
+ INTERNAL_ERROR ( self == NULL ? xcSelfNull : xcSelfInvalid, "cannot get row id from iterator" );
+ else if ( self -> rowset_data != self -> rowset -> data )
+ USER_ERROR ( xcSelfInvalid, "usage of iterator after modifying rowset" );
+ else
+ USER_ERROR ( xcSelfInvalid, "usage of iterator beyond rowset range" );
+ return -1;
+ }
+ return self -> rowset_data -> rows_array[self -> array_index];
+}
+
+static KRefcount_v1_vt vtKRowSetIteratorSimpleRefCount =
+{
+ KVTABLE_INITIALIZER ( KRowSetIteratorSimpleRefCount, KRefcount_v1, 0, NULL ),
+ KRowSetSimpleIteratorDestroy,
+ NULL
+};
+
+static KRowSetIterator_v1_vt vtKRowSetIteratorSimple =
+{
+ KVTABLE_INITIALIZER ( KRowSetIteratorSimple, KRowSetIterator, 0, &vtKRowSetIteratorSimpleRefCount.dad ),
+
+ KRowSetSimpleIteratorNext,
+ KRowSetSimpleIteratorIsValid,
+ KRowSetSimpleIteratorGetRowId
+};
+
+static
+KRowSetSimpleIterator * CC KRowSetSimpleGetIterator ( const struct KRowSet * self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcConstructing );
+
+ if ( self == NULL || self -> data == NULL )
+ INTERNAL_ERROR ( xcSelfNull, "failed to get rowset data" );
+ else
+ {
+ KRowSetSimpleIterator *r;
+ r = calloc ( 1, sizeof *r );
+ if ( r == NULL )
+ SYSTEM_ERROR ( xcNoMemory, "out of memory" );
+ else
+ {
+ TRY ( KRowSetIteratorInit ( &r -> dad, ctx, &vtKRowSetIteratorSimple.dad, "KRowSetIteratorSimple" ) )
+ {
+ KRowSetSimpleDataSort ( self -> data ); /* make sure rowset is sorted */
+ r -> rowset_data = self -> data;
+ r -> array_index = 0;
+ TRY ( r -> rowset = KRowSetDuplicate ( self, ctx, 0 ) )
+ {
+ return r;
+ }
+ }
+ free ( r );
+ }
+ }
+
+ return NULL;
+}
+
diff --git a/libs/kdb/rowset.c b/libs/kdb/rowset.c
new file mode 100644
index 0000000..c391917
--- /dev/null
+++ b/libs/kdb/rowset.c
@@ -0,0 +1,494 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include <kdb/extern.h>
+
+#include <kdb/rowset.h>
+#include <kdb/rowset-impl.h>
+#include <kfc/ctx.h>
+#include <kfc/except.h>
+#include <kfc/xc.h>
+
+KITFTOK_DEF ( KRowSet );
+KITFTOK_DEF ( KRowSetIterator );
+
+/* MakeRowSet
+ * may add others...
+ */
+KDB_EXTERN KRowSet * CC KTableMakeRowSet ( struct KTable const * self, ctx_t ctx )
+{
+ return KTableMakeRowSetSimple ( self, ctx );
+}
+
+/* Init
+ * initialize a newly allocated rowset object
+ */
+KDB_EXTERN void CC KRowSetInit ( KRowSet *self, ctx_t ctx, const KVTable *kvt,
+ const char *classname )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcInitializing );
+
+ if ( self == NULL )
+ INTERNAL_ERROR ( xcSelfNull, "KRowSetInit failed" );
+ else if ( kvt == NULL )
+ INTERNAL_ERROR ( xcParamNull, "KRowSetInit failed: vt is NULL" );
+ else
+ {
+ TRY ( KRefcountInit_v1 ( & self -> dad, ctx, kvt, classname ) )
+ {
+ const KRowSet_v1_vt *vt = KVTABLE_CAST ( kvt, ctx, KRowSet );
+ if ( vt == NULL )
+ INTERNAL_ERROR ( xcInterfaceIncorrect, "vtable does not appear to implement KRowSet" );
+ else switch ( vt -> dad . min )
+ {
+ case 0:
+#if _DEBUGGING
+ if ( vt -> destroy_data == NULL ||
+ vt -> add_row_id_range == NULL ||
+ vt -> get_num_rows == NULL ||
+ vt -> has_row_id == NULL ||
+ vt -> get_iterator == NULL )
+ {
+ INTERNAL_ERROR ( xcInterfaceInvalid, "null method pointer(s)" );
+ return;
+ }
+#endif
+ break;
+ default:
+ INTERNAL_ERROR ( xcInterfaceInvalid, "rowset has an invalid version" );
+ }
+
+ }
+ }
+}
+
+/* AddRowId
+ * add a single row to set
+ *
+ * "row_id" [ IN ] - row-id to be added
+ */
+KDB_EXTERN void CC KRowSetAddRowId ( KRowSet * self, ctx_t ctx, int64_t row_id )
+{
+ KRowSetAddRowIdRange ( self, ctx, row_id, 1 );
+}
+
+/* AddRowIdRange
+ * adds row-ids within specified range
+ *
+ * "row_id" [ IN ] and "count" [ IN ] - range of row-ids to be added
+ */
+KDB_EXTERN void CC KRowSetAddRowIdRange ( KRowSet * self, ctx_t ctx, int64_t row_id, uint64_t count )
+{
+ if ( self == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcInserting );
+ INTERNAL_ERROR ( xcSelfNull, "failed to insert rows into rowset" );
+ }
+ else
+ {
+ const KRowSet_v1_vt * vt = KVTABLE_CAST ( TO_REFCOUNT_V1 ( self ) -> vt, ctx, KRowSet );
+ if ( vt == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcInserting );
+ INTERNAL_ERROR ( xcInterfaceIncorrect, "this object does not support the KRowSet interface" );
+ }
+ else
+ vt -> add_row_id_range ( self, ctx, row_id, count );
+ }
+}
+
+/* GetNumRowIds
+ * return the number of elements in set
+ */
+KDB_EXTERN uint64_t CC KRowSetGetNumRowIds ( const KRowSet * self, ctx_t ctx )
+{
+ if ( self == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcAccessing );
+ INTERNAL_ERROR ( xcSelfNull, "failed to get number of rows in rowset" );
+ }
+ else
+ {
+ const KRowSet_v1_vt * vt = KVTABLE_CAST ( TO_REFCOUNT_V1 ( self ) -> vt, ctx, KRowSet );
+ if ( vt == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcAccessing );
+ INTERNAL_ERROR ( xcInterfaceIncorrect, "this object does not support the KRowSet interface" );
+ }
+ else
+ return vt -> get_num_rows ( self, ctx );
+ }
+
+ return 0;
+}
+
+/* HasRowId
+ * checks if element is present in set
+ */
+KDB_EXTERN bool CC KRowSetHasRowId ( const KRowSet * self, ctx_t ctx, int64_t row_id )
+{
+ if ( self == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcAccessing );
+ INTERNAL_ERROR ( xcSelfNull, "failed to find row in rowset" );
+ }
+ else
+ {
+ const KRowSet_v1_vt * vt = KVTABLE_CAST ( TO_REFCOUNT_V1 ( self ) -> vt, ctx, KRowSet );
+ if ( vt == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcAccessing );
+ INTERNAL_ERROR ( xcInterfaceIncorrect, "this object does not support the KRowSet interface" );
+ }
+ else
+ return vt -> has_row_id ( self, ctx, row_id );
+ }
+
+ return false;
+}
+
+/* Visit
+ * execute a function on each row-id in set
+ */
+KDB_EXTERN void CC KRowSetVisit ( const KRowSet * self, ctx_t ctx,
+ void ( CC * f ) ( ctx_t ctx, int64_t row_id, void * data ), void * data )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcAccessing );
+ KRowSetIterator * it;
+
+ if ( self == NULL )
+ INTERNAL_ERROR ( xcSelfNull, "failed to iterate over rowset" );
+ else
+ {
+ TRY ( it = KRowSetMakeIterator ( self, ctx ) )
+ {
+ while ( !FAILED() && KRowSetIteratorIsValid ( it ) )
+ {
+ int64_t row_id;
+ ON_FAIL ( row_id = KRowSetIteratorGetRowId ( it, ctx ) )
+ break;
+
+ ON_FAIL ( f ( ctx, row_id, data ) )
+ break;
+
+ KRowSetIteratorNext ( it, ctx );
+ }
+
+ KRowSetIteratorRelease ( it, ctx );
+ }
+ }
+}
+
+/* Intersect
+ * performs an intersection between two sets and returns the result
+ */
+KDB_EXTERN KRowSet * CC KRowSetIntersect ( ctx_t ctx, const KRowSet * a, const KRowSet * b )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcProcessing );
+ KRowSet * result;
+
+ if ( a -> table != b -> table )
+ USER_ERROR ( xcParamIncorrect, "cannot intersect rowsets from different tables" );
+ else
+ {
+ TRY ( result = KTableMakeRowSet ( a -> table, ctx ) )
+ {
+ KRowSetIterator * a_it;
+ KRowSetIterator * b_it;
+ TRY ( a_it = KRowSetMakeIterator ( a, ctx ) )
+ {
+ TRY ( b_it = KRowSetMakeIterator ( b, ctx ) )
+ {
+ while ( !FAILED() && KRowSetIteratorIsValid ( a_it ) && KRowSetIteratorIsValid ( b_it ) )
+ {
+ int64_t a_row_id;
+ int64_t b_row_id;
+ ON_FAIL ( a_row_id = KRowSetIteratorGetRowId ( a_it, ctx ) )
+ break;
+
+ ON_FAIL ( b_row_id = KRowSetIteratorGetRowId ( b_it, ctx ) )
+ break;
+
+ if ( a_row_id < b_row_id )
+ KRowSetIteratorNext ( a_it, ctx );
+ else if ( a_row_id > b_row_id )
+ KRowSetIteratorNext ( b_it, ctx );
+ else
+ {
+ TRY ( KRowSetAddRowId ( result, ctx, a_row_id ) )
+ {
+ TRY ( KRowSetIteratorNext ( a_it, ctx ) )
+ {
+ KRowSetIteratorNext ( b_it, ctx );
+ }
+ }
+ }
+ }
+ KRowSetIteratorRelease ( b_it, ctx );
+ }
+ KRowSetIteratorRelease ( a_it, ctx );
+ }
+ if ( !FAILED() )
+ return result;
+
+ KRowSetRelease ( result, ctx );
+ }
+ }
+
+ return NULL;
+}
+
+/* Union
+ * performs a union between two sets and returns the result
+ */
+KDB_EXTERN KRowSet * CC KRowSetUnion ( ctx_t ctx, const KRowSet * a, const KRowSet * b )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcRowSet, rcProcessing );
+ KRowSet * result;
+
+ if ( a -> table != b -> table )
+ USER_ERROR ( xcParamIncorrect, "cannot intersect rowsets from different tables" );
+ else
+ {
+ TRY ( result = KTableMakeRowSet ( a -> table, ctx ) )
+ {
+ KRowSetIterator * a_it;
+ KRowSetIterator * b_it;
+ TRY ( a_it = KRowSetMakeIterator ( a, ctx ) )
+ {
+ TRY ( b_it = KRowSetMakeIterator ( b, ctx ) )
+ {
+ while ( !FAILED() && KRowSetIteratorIsValid ( a_it ) && KRowSetIteratorIsValid ( b_it ) )
+ {
+ int64_t a_row_id;
+ int64_t b_row_id;
+ ON_FAIL ( a_row_id = KRowSetIteratorGetRowId ( a_it, ctx ) )
+ break;
+
+ ON_FAIL ( b_row_id = KRowSetIteratorGetRowId ( b_it, ctx ) )
+ break;
+
+ if ( a_row_id < b_row_id )
+ {
+ TRY ( KRowSetAddRowId ( result, ctx, a_row_id ) )
+ {
+ KRowSetIteratorNext ( a_it, ctx );
+ }
+ }
+ else if ( a_row_id > b_row_id )
+ {
+ TRY ( KRowSetAddRowId ( result, ctx, b_row_id ) )
+ {
+ KRowSetIteratorNext ( b_it, ctx );
+ }
+ }
+ else
+ {
+ TRY ( KRowSetAddRowId ( result, ctx, a_row_id ) )
+ {
+ TRY ( KRowSetIteratorNext ( a_it, ctx ) )
+ {
+ KRowSetIteratorNext ( b_it, ctx );
+ }
+ }
+ }
+ }
+ while ( !FAILED() && KRowSetIteratorIsValid ( a_it ) )
+ {
+ int64_t a_row_id;
+ TRY ( a_row_id = KRowSetIteratorGetRowId ( a_it, ctx ) )
+ {
+ TRY ( KRowSetAddRowId ( result, ctx, a_row_id ) )
+ {
+ KRowSetIteratorNext ( a_it, ctx );
+ }
+ }
+ }
+ while ( !FAILED() && KRowSetIteratorIsValid ( b_it ) )
+ {
+ int64_t b_row_id;
+ TRY ( b_row_id = KRowSetIteratorGetRowId ( b_it, ctx ) )
+ {
+ TRY ( KRowSetAddRowId ( result, ctx, b_row_id ) )
+ {
+ KRowSetIteratorNext ( b_it, ctx );
+ }
+ }
+ }
+ KRowSetIteratorRelease ( b_it, ctx );
+ }
+ KRowSetIteratorRelease ( a_it, ctx );
+ }
+ if ( !FAILED() )
+ return result;
+
+ KRowSetRelease ( result, ctx );
+ }
+ }
+
+ return NULL;
+}
+
+/* MakeIterator
+ * create an iterator on set
+ * initially set to first row-id in set
+ */
+KDB_EXTERN KRowSetIterator * CC KRowSetMakeIterator ( const KRowSet * self, ctx_t ctx )
+{
+ if ( self == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcCreating );
+ INTERNAL_ERROR ( xcSelfNull, "failed to create rowset iterator" );
+ }
+ else
+ {
+ const KRowSet_v1_vt * vt = KVTABLE_CAST ( TO_REFCOUNT_V1 ( self ) -> vt, ctx, KRowSet );
+ if ( vt == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcCreating );
+ INTERNAL_ERROR ( xcInterfaceIncorrect, "this object does not support the KRowSet interface" );
+ }
+ else
+ return vt -> get_iterator ( self, ctx );
+ }
+
+ return NULL;
+}
+
+/* Init
+ * initialize a newly allocated rowset iterator object
+ */
+KDB_EXTERN void CC KRowSetIteratorInit ( KRowSetIterator *self, ctx_t ctx, const KVTable *kvt,
+ const char *classname )
+{
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcInitializing );
+
+ if ( self == NULL )
+ INTERNAL_ERROR ( xcSelfNull, "KRowSetIteratorInit failed" );
+ else if ( kvt == NULL )
+ INTERNAL_ERROR ( xcParamNull, "KRowSetIteratorInit failed: vt is NULL" );
+ else
+ {
+ TRY ( KRefcountInit_v1 ( & self -> dad, ctx, kvt, classname ) )
+ {
+ const KRowSetIterator_v1_vt *vt = KVTABLE_CAST ( kvt, ctx, KRowSetIterator );
+ if ( vt == NULL )
+ INTERNAL_ERROR ( xcInterfaceIncorrect, "vtable does not appear to implement KRowSetIterator" );
+ else switch ( vt -> dad . min )
+ {
+ case 0:
+#if _DEBUGGING
+ if ( vt -> next == NULL ||
+ vt -> is_valid == NULL ||
+ vt -> get_row_id == NULL )
+ {
+ INTERNAL_ERROR ( xcInterfaceNull, "KRowSetIteratorInit failed" );
+ return;
+ }
+#endif
+ break;
+ default:
+ INTERNAL_ERROR ( xcInterfaceBadVersion, "rowset iterator has an invalid version" );
+ }
+
+ }
+ }
+}
+
+/* Next
+ * advance iterator to next row-id
+
+ * advance to first row-id on initial invocation
+ * advance to next row-id subsequently
+ * returns rcDone if no more row-ids are available.
+ */
+KDB_EXTERN bool CC KRowSetIteratorNext ( KRowSetIterator * self, ctx_t ctx )
+{
+
+ if ( self == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcPositioning );
+ INTERNAL_ERROR ( xcSelfNull, "failed to move rowset iterator" );
+ }
+ else
+ {
+ const KRowSetIterator_v1_vt * vt = KVTABLE_CAST ( TO_REFCOUNT_V1 ( self ) -> vt, ctx, KRowSetIterator );
+ if ( vt == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcPositioning );
+ INTERNAL_ERROR ( xcInterfaceIncorrect, "this object does not support the KRowSetIterator interface" );
+ }
+ else
+ return vt -> next ( self, ctx );
+ }
+
+ return false;
+}
+
+/*IsValid
+ * check if iterator points to a valid row
+ *
+ * returns false when iterator points somewhere outside of a row set
+ */
+KDB_EXTERN bool CC KRowSetIteratorIsValid ( const KRowSetIterator * self )
+{
+ if ( self != NULL )
+ {
+ HYBRID_FUNC_ENTRY ( rcDB, rcIterator, rcAccessing );
+ const KRowSetIterator_v1_vt * vt = KVTABLE_CAST ( TO_REFCOUNT_V1 ( self ) -> vt, ctx, KRowSetIterator );
+ if ( vt == NULL )
+ INTERNAL_ERROR ( xcInterfaceIncorrect, "this object does not support the KRowSetIterator interface" );
+ else
+ return vt -> is_valid ( self );
+ }
+
+ return false;
+}
+
+/* RowId
+ * report current row id
+ */
+KDB_EXTERN int64_t CC KRowSetIteratorGetRowId ( const KRowSetIterator * self, ctx_t ctx )
+{
+ if ( self == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcAccessing );
+ INTERNAL_ERROR ( xcSelfNull, "failed to access rowset iterator" );
+ }
+ else
+ {
+ const KRowSetIterator_v1_vt * vt = KVTABLE_CAST ( TO_REFCOUNT_V1 ( self ) -> vt, ctx, KRowSetIterator );
+ if ( vt == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcDB, rcIterator, rcAccessing );
+ INTERNAL_ERROR ( xcInterfaceIncorrect, "this object does not support the KRowSetIterator interface" );
+ }
+ else
+ return vt -> get_row_id ( self, ctx );
+ }
+
+ return -1;
+}
diff --git a/libs/kdb/rowset.c_old b/libs/kdb/rowset.c_old
new file mode 100644
index 0000000..2d2a2ba
--- /dev/null
+++ b/libs/kdb/rowset.c_old
@@ -0,0 +1,1513 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+#include <kdb/rowset.h>
+#include "rowset-priv.h"
+#include <klib/refcount.h>
+#include <klib/rc.h>
+#include <klib/out.h>
+
+#include <string.h>
+#include <assert.h>
+
+#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y))
+#define MAX(X, Y) (((X) > (Y)) ? (X) : (Y))
+
+/*--------------------------------------------------------------------------
+ * KRowSet
+ */
+
+struct KRowSet
+{
+ KRefcount refcount;
+ KRowSetTreeNode * root_node;
+ DLList leaf_nodes;
+
+ uint64_t number_rows;
+ size_t number_leaves;
+};
+
+KDB_EXTERN rc_t CC KTableMakeRowSet ( struct KTable const * self, KRowSet ** rowset )
+{
+ rc_t rc;
+ KRowSet * self_rowset;
+
+ if ( rowset == NULL )
+ rc = RC ( rcDB, rcRowSet, rcConstructing, rcSelf, rcNull );
+ else
+ {
+ self_rowset = calloc ( 1, sizeof *self_rowset );
+ if ( self_rowset == NULL )
+ rc = RC ( rcDB, rcRowSet, rcConstructing, rcMemory, rcExhausted );
+ else
+ {
+ KRefcountInit ( &self_rowset->refcount, 1, "KRowSet", "new", "" );
+ DLListInit ( &self_rowset->leaf_nodes );
+
+ *rowset = self_rowset;
+ return 0;
+ }
+ }
+
+ return rc;
+}
+
+/**
+ * Allocates and initializes single leaf
+ */
+static
+rc_t KRowSetAllocateLeaf ( KRowSetTreeLeafType type, int64_t leaf_id, KRowSetTreeLeaf ** leaf )
+{
+ KRowSetTreeLeaf * allocated_leaf;
+ size_t size = sizeof allocated_leaf->header;
+
+ assert ( leaf != NULL );
+
+ switch ( type )
+ {
+ case LeafType_Bitmap:
+ size += sizeof allocated_leaf->data.bitmap;
+ break;
+ case LeafType_ArrayRanges:
+ size += sizeof allocated_leaf->data.array_ranges;
+ break;
+ default:
+ assert ( false );
+ }
+
+ allocated_leaf = calloc ( 1, size );
+ if ( allocated_leaf == NULL )
+ return RC ( rcDB, rcRowSet, rcInserting, rcMemory, rcExhausted );
+
+#if CHECK_NODE_MARKS
+ allocated_leaf->header.leaf_mark = LEAF_MARK;
+#endif
+ allocated_leaf->header.type = type;
+ allocated_leaf->header.leaf_id = leaf_id;
+
+ *leaf = allocated_leaf;
+ return 0;
+}
+
+/* Whack
+ */
+
+/*
+ * When data is NULL, it means that leaf is freeing during freeing the whole list
+ * and DLListUnlink does not need to be called. Otherwise, we are removing a single leaf
+ */
+static
+void KRowSetLeafWhack ( DLNode * node, void * data )
+{
+ KRowSetTreeLeaf * leaf = (KRowSetTreeLeaf *) node;
+#if CHECK_NODE_MARKS
+ assert ( leaf->header.leaf_mark == LEAF_MARK );
+#endif
+
+ if ( data != NULL )
+ {
+ DLList * leaf_nodes = data;
+ DLListUnlink ( leaf_nodes, node );
+ }
+
+ free ( leaf );
+}
+
+/**
+ * Recursively deallocates all trie nodes and leaves when "free_leaves" is true.
+ * Otherwise only deallocates nodes.
+ */
+static
+void KRowSetNodeWhack ( KRowSet * self, KRowSetTreeNode * node, int depth, bool free_leaves )
+{
+ int i;
+
+ assert ( self != NULL );
+ assert ( depth < LEAF_DEPTH );
+
+#if CHECK_NODE_MARKS
+ assert ( node->node_mark == NODE_MARK );
+#endif
+ assert ( depth < LEAF_DEPTH );
+
+ for ( i = 0; i < 256; ++i )
+ {
+ assert ( (i & 0xFF) == i );
+
+ if ( node->children[i] )
+ {
+ int new_depth = depth + node->transitions[i].size + 1;
+ assert ( new_depth <= LEAF_DEPTH );
+
+ if ( new_depth < LEAF_DEPTH )
+ KRowSetNodeWhack ( self, node->children[i], new_depth, free_leaves );
+ else if ( free_leaves )
+ KRowSetLeafWhack ( &((KRowSetTreeLeaf*) node->children[i])->header.dad, &self->leaf_nodes );
+ }
+ }
+
+ free ( node );
+}
+
+/**
+ * Whack
+ * Frees all leaves first and then walks through the trie
+ * and frees all remaining nodes
+ */
+static
+rc_t KRowSetWhack ( KRowSet * self )
+{
+ assert ( self != NULL );
+
+ if ( self->root_node )
+ {
+ KRowSetNodeWhack ( self, self->root_node, 0, false );
+ DLListWhack ( &self->leaf_nodes, KRowSetLeafWhack, NULL );
+ }
+ free ( self );
+ return 0;
+}
+
+/* AddRef
+ * Release
+ * ignores NULL references
+ */
+KDB_EXTERN rc_t CC KRowSetAddRef ( const KRowSet *self )
+{
+ if ( self != NULL ) switch ( KRefcountAdd ( & self -> refcount, "KRowSet" ) )
+ {
+ case krefOkay:
+ break;
+ default:
+ return RC ( rcDB, rcRowSet, rcAttaching, rcConstraint, rcViolated );
+ }
+ return 0;
+}
+
+KDB_EXTERN rc_t CC KRowSetRelease ( const KRowSet *self )
+{
+ if ( self != NULL ) switch ( KRefcountDrop ( & self -> refcount, "KRowSet" ) )
+ {
+ case krefOkay:
+ break;
+ case krefWhack:
+ return KRowSetWhack ( ( KRowSet* ) self );
+ default:
+ return RC ( rcDB, rcRowSet, rcReleasing, rcConstraint, rcViolated );
+ }
+ return 0;
+}
+
+/**
+ * Get byte of "leaf_id" which represents node's child at "depth"
+ */
+static
+inline uint8_t KRowSetGetLeafIdByte ( int64_t leaf_id, int depth )
+{
+ return leaf_id >> (LEAF_DEPTH - depth - 1) * 8 & 0xFF;
+}
+
+/**
+ * Searches a nearest neighbor of just inserted "leaf",
+ * by walking through nodes_stack from the most deep node to the root one.
+ *
+ * Is used to inserter newly created "leaf" to a linked list.
+ */
+static
+void KRowSetFindNearestLeaf ( KRowSet * self, KRowSetTreeNode * nodes_stack[], int nodes_depth[], int nodes_stack_size, KRowSetTreeLeaf * leaf, KRowSetTreeLeaf ** result, bool * result_left )
+{
+ assert ( self != NULL );
+ assert ( nodes_stack != NULL );
+ assert ( leaf != NULL );
+ assert ( result != NULL );
+ assert ( result_left != NULL );
+
+ *result = NULL;
+
+ if ( self->number_leaves == 0 )
+ return;
+
+ if ( self->number_leaves == 1 )
+ {
+ *result = (KRowSetTreeLeaf *) DLListHead ( &self->leaf_nodes );
+ *result_left = (*result)->header.leaf_id < leaf->header.leaf_id;
+ return;
+ }
+
+ {
+ int i;
+ int64_t leaf_id = leaf->header.leaf_id;
+ KRowSetTreeLeaf * nearest_leaf = NULL;
+ bool nearest_leaf_left;
+ for ( i = nodes_stack_size - 1; i >= 0; --i )
+ {
+ int depth;
+ int j;
+ int j_max;
+ uint8_t bt;
+ KRowSetTreeNode * current_node = nodes_stack[i];
+ void * nearest_subtree = NULL;
+ bool nearest_subtre_left;
+ int nearest_subtree_depth;
+
+ assert ( current_node != NULL );
+
+ depth = nodes_depth[i];
+ bt = KRowSetGetLeafIdByte ( leaf_id, depth );
+
+ assert ( current_node->children[bt] != NULL );
+
+ if ( 255 - bt > bt )
+ j_max = 255 - bt;
+ else
+ j_max = bt;
+
+ for (j = 1; j <= j_max; ++j)
+ {
+ if ( bt + j <= 255 && current_node->children[bt + j] != NULL )
+ {
+ nearest_subtre_left = false;
+ nearest_subtree = current_node->children[bt + j];
+ nearest_subtree_depth = depth + 1 + current_node->transitions[bt + j].size;
+ break;
+ }
+
+ if ( bt - j >= 0 && current_node->children[bt - j] != NULL )
+ {
+ nearest_subtre_left = true;
+ nearest_subtree = current_node->children[bt - j];
+ nearest_subtree_depth = depth + 1 + current_node->transitions[bt - j].size;
+ break;
+ }
+ }
+
+ if ( nearest_subtree != NULL )
+ {
+ for ( j = nearest_subtree_depth; j < LEAF_DEPTH; ++j )
+ {
+ int search_i;
+ int search_start = nearest_subtre_left ? 255 : 0;
+ int search_stop = nearest_subtre_left ? 0 : 255;
+ int search_step = nearest_subtre_left ? -1 : 1;
+ void * nearest_subtree_next = NULL;
+ KRowSetTreeNodeTransition * nearest_subtree_next_tr;
+
+#if CHECK_NODE_MARKS
+ assert ( ((KRowSetTreeNode *) nearest_subtree)->node_mark == NODE_MARK );
+#endif
+
+ for ( search_i = search_start; search_step > 0 ? search_i <= search_stop : search_i >= search_stop; search_i += search_step )
+ {
+ if ( ((KRowSetTreeNode *) nearest_subtree)->children[search_i] != NULL )
+ {
+ nearest_subtree_next = ((KRowSetTreeNode *) nearest_subtree)->children[search_i];
+ nearest_subtree_next_tr = &((KRowSetTreeNode *) nearest_subtree)->transitions[search_i];
+ break;
+ }
+ }
+
+ assert ( nearest_subtree_next != NULL );
+ nearest_subtree = nearest_subtree_next;
+ j += nearest_subtree_next_tr->size;
+ }
+
+ assert ( j == LEAF_DEPTH );
+
+ nearest_leaf = nearest_subtree;
+ nearest_leaf_left = nearest_subtre_left;
+
+ break;
+ }
+ }
+
+ // empty tree is handled by previous "if"s
+ assert ( nearest_leaf != NULL );
+
+#if CHECK_NODE_MARKS
+ assert ( ((KRowSetTreeLeaf *) nearest_leaf)->header.leaf_mark == LEAF_MARK );
+#endif
+ *result = nearest_leaf;
+ *result_left = nearest_leaf_left;
+ }
+
+ return;
+}
+
+/**
+ * Checks if node's child pointed by "node_depth"'s "leaf_id" byte matches "leaf_id".
+ * It is possible that transition uses more than one byte of "leaf_id".
+ * In such case, we have to make sure that those bytes match.
+ */
+static
+bool KRowSetNodeIsTransitionMatches ( int64_t leaf_id, int node_depth, const KRowSetTreeNodeTransition * tr, int * size_matched )
+{
+ int matched;
+
+ assert ( node_depth + tr->size < LEAF_DEPTH );
+
+ for ( matched = 0; matched < tr->size; ++matched )
+ {
+ if ( KRowSetGetLeafIdByte ( leaf_id, matched + node_depth + 1 ) != tr->data[matched] )
+ break;
+ }
+
+ if ( size_matched != NULL )
+ *size_matched = matched;
+
+ return matched == tr->size;
+}
+
+/**
+ * Assigns a new child to a "node".
+ * It will note create any extra node's even if child_depth - node_depth > 1
+ * (because of radix trie compression)
+ */
+static
+void KRowSetNodeSetChild ( KRowSetTreeNode * node, void * child, int64_t leaf_id, int node_depth, int child_depth )
+{
+ uint8_t bt = KRowSetGetLeafIdByte ( leaf_id, node_depth );
+ int i;
+
+ node->children[bt] = child;
+ node->transitions[bt].size = child_depth - 1 - node_depth;
+
+ for ( i = node_depth; i < child_depth - 1; ++i )
+ {
+ node->transitions[bt].data[i - node_depth] = KRowSetGetLeafIdByte ( leaf_id, i + 1 );
+ }
+}
+
+/**
+ * Splits the node, by inserting "allocated_interm_node" between "node" and it's child
+ * Corrects transitions after insertion.
+ *
+ * NB - To ease error handling and deallocation of resources in case of errors,
+ * "allocated_interm_node" has to be allocated before calling this function.
+ */
+static
+void KRowSetSplitNode ( KRowSetTreeNode * node, int node_depth, KRowSetTreeNode * allocated_interm_node, int interm_node_depth, int64_t leaf_id )
+{
+ uint8_t bt = KRowSetGetLeafIdByte ( leaf_id, node_depth );
+ uint8_t interm_bt;
+ void * child = node->children[bt];
+ int first_tr_size = interm_node_depth - node_depth - 1;
+
+#if CHECK_NODE_MARKS
+ allocated_interm_node->node_mark = NODE_MARK;
+#endif
+
+ assert ( allocated_interm_node != NULL );
+ assert ( node->children[bt] != NULL );
+
+ interm_bt = node->transitions[bt].data[first_tr_size];
+ allocated_interm_node->transitions[interm_bt].size = node->transitions[bt].size - first_tr_size - 1;
+ for ( int i = 0; i < allocated_interm_node->transitions[interm_bt].size; ++i )
+ {
+ allocated_interm_node->transitions[interm_bt].data[i] = node->transitions[bt].data[first_tr_size + i + 1];
+ }
+ allocated_interm_node->children[interm_bt] = child;
+
+ KRowSetNodeSetChild ( node, allocated_interm_node, leaf_id, node_depth, interm_node_depth );
+}
+
+/**
+ * Collapses node, by linking (parent) "node" to "collapse_node"'s the only child
+ * and deallocates "collapse_node"
+ */
+static
+void KRowSetCollapseNode ( KRowSetTreeNode * node, int node_depth, KRowSetTreeNode * collapse_node, int collapse_node_depth, uint8_t collapse_node_bt, int64_t leaf_id )
+{
+ int i;
+ uint8_t node_bt = KRowSetGetLeafIdByte ( leaf_id, node_depth );
+
+#if CHECK_NODE_MARKS
+ node->node_mark = NODE_MARK;
+ collapse_node->node_mark = NODE_MARK;
+#endif
+ assert ( node->children[node_bt] == collapse_node );
+ assert ( collapse_node->children[collapse_node_bt] != NULL );
+
+ node->children[node_bt] = collapse_node->children[collapse_node_bt];
+ node->transitions[node_bt].data[node->transitions[node_bt].size++] = collapse_node_bt;
+ for ( i = 0; i < collapse_node->transitions[collapse_node_bt].size; ++i )
+ {
+ node->transitions[node_bt].data[node->transitions[node_bt].size++] = collapse_node->transitions[collapse_node_bt].data[i];
+ }
+
+ free ( collapse_node );
+}
+
+/**
+ * Sets "allocated_leaf" members and inserts it as a child to a previously found "node" that fits leaf_id.
+ * Also inserts "allocated_leaf" to leaves linked list.
+ *
+ * NB - To ease error handling and deallocation of resources in case of errors,
+ * "allocated_leaf" has to be allocated before calling this function.
+ */
+static
+void KRowSetInsertLeaf ( KRowSet * self, int64_t leaf_id, KRowSetTreeNode * node, int node_depth, KRowSetTreeNode * nodes_stack[], int nodes_depth[], int nodes_stack_size, KRowSetTreeLeaf * allocated_leaf )
+{
+ KRowSetTreeLeaf * nearest_leaf;
+ bool nearest_leaf_left;
+ uint8_t bt;
+ bt = KRowSetGetLeafIdByte ( leaf_id, node_depth );
+
+ assert ( allocated_leaf != NULL );
+ assert ( node->children[bt] == NULL );
+
+ KRowSetNodeSetChild ( node, allocated_leaf, leaf_id, node_depth, LEAF_DEPTH );
+ node->children[bt] = allocated_leaf;
+
+ KRowSetFindNearestLeaf ( self, nodes_stack, nodes_depth, nodes_stack_size, allocated_leaf, &nearest_leaf, &nearest_leaf_left );
+ if ( nearest_leaf == NULL )
+ DLListPushTail ( &self->leaf_nodes, &allocated_leaf->header.dad );
+ else if (nearest_leaf_left)
+ DLListInsertNodeAfter ( &self->leaf_nodes, &nearest_leaf->header.dad, &allocated_leaf->header.dad );
+ else
+ DLListInsertNodeBefore ( &self->leaf_nodes, &nearest_leaf->header.dad, &allocated_leaf->header.dad );
+
+ ++self->number_leaves;
+ assert ( KRowSetNodeIsTransitionMatches ( leaf_id, node_depth, &node->transitions[bt], NULL ) );
+}
+
+/**
+ * Searches for a leaf by its id.
+ * Returns rcNotFound when leaf is not in the tree and "insert_when_needed" is false,
+ * otherwise inserts a new leaf and returns it in "leaf_found".
+ *
+ * NB - This function preallocates all required resources
+ * and only after that changes the RowSet data structure.
+ *
+ * Using this approach we can simply return from the function,
+ * since all the resources are being allocated once.
+ */
+static
+rc_t KRowSetGetLeaf ( KRowSet * self, int64_t leaf_id, bool insert_when_needed, KRowSetTreeLeaf ** leaf_found, KRowSetTreeNode * nodes_stack[], int nodes_depth[], int * nodes_stack_size )
+{
+ rc_t rc;
+ KRowSetTreeNode * node;
+ KRowSetTreeLeaf * new_leaf;
+ uint8_t bt;
+ int depth = 0;
+ int nodes_stack_size_int;
+ KRowSetTreeNode * nodes_stack_int[LEAF_DEPTH];
+ int nodes_depth_int[LEAF_DEPTH];
+
+ if ( self == NULL )
+ return RC ( rcDB, rcRowSet, rcSelecting, rcSelf, rcNull );
+
+ if ( leaf_found == NULL )
+ return RC ( rcDB, rcRowSet, rcSelecting, rcParam, rcNull );
+
+ if ( nodes_stack == NULL && nodes_depth == NULL && nodes_stack_size == NULL )
+ {
+ nodes_stack = nodes_stack_int;
+ nodes_depth = nodes_depth_int;
+ nodes_stack_size = &nodes_stack_size_int;
+ }
+ else if ( nodes_stack == NULL || nodes_depth == NULL || nodes_stack_size == NULL )
+ return RC ( rcDB, rcRowSet, rcSelecting, rcParam, rcNull );
+
+ *nodes_stack_size = 0;
+
+ // empty tree
+ if ( self->root_node == NULL )
+ {
+ KRowSetTreeNode * root;
+ if ( !insert_when_needed )
+ return RC ( rcDB, rcRowSet, rcSelecting, rcItem, rcNotFound );
+
+ root = calloc ( 1, sizeof ( KRowSetTreeNode ) );
+ if (root == NULL)
+ return RC ( rcDB, rcRowSet, rcInserting, rcMemory, rcExhausted );
+
+ // pre-allocate leaf here
+ rc = KRowSetAllocateLeaf ( LeafType_ArrayRanges, leaf_id, &new_leaf );
+ if ( rc != 0 )
+ {
+ free ( root );
+ return rc;
+ }
+
+#if CHECK_NODE_MARKS
+ root->node_mark = NODE_MARK;
+#endif
+
+ self->root_node = root;
+
+ node = self->root_node;
+ nodes_stack[*nodes_stack_size] = node;
+ nodes_depth[(*nodes_stack_size)++] = depth;
+
+ KRowSetInsertLeaf ( self, leaf_id, node, depth, nodes_stack, nodes_depth, *nodes_stack_size, new_leaf );
+ *leaf_found = new_leaf;
+ return 0;
+ }
+
+ node = self->root_node;
+ nodes_stack[*nodes_stack_size] = node;
+ nodes_depth[(*nodes_stack_size)++] = depth;
+#if CHECK_NODE_MARKS
+ assert ( node->node_mark == NODE_MARK );
+#endif
+
+ for ( ; depth < LEAF_DEPTH; )
+ {
+ int tr_size_matched;
+ bt = KRowSetGetLeafIdByte ( leaf_id, depth );
+ // no child at a given transition, let's insert leaf here
+ if ( node->children[bt] == NULL )
+ {
+ if ( !insert_when_needed )
+ return RC ( rcDB, rcRowSet, rcSelecting, rcItem, rcNotFound );
+
+ // pre-allocate leaf here
+ rc = KRowSetAllocateLeaf ( LeafType_ArrayRanges, leaf_id, &new_leaf );
+ if ( rc != 0 )
+ return rc;
+
+ KRowSetInsertLeaf ( self, leaf_id, node, depth, nodes_stack, nodes_depth, *nodes_stack_size, new_leaf );
+ *leaf_found = new_leaf;
+ return 0;
+ }
+
+ // transition does not match, split the node
+ if ( !KRowSetNodeIsTransitionMatches ( leaf_id, depth, &node->transitions[bt], &tr_size_matched ) )
+ {
+ KRowSetTreeNode * interm_node;
+ int interm_node_depth = depth + 1 + tr_size_matched;
+
+ if ( !insert_when_needed )
+ return RC ( rcDB, rcRowSet, rcSelecting, rcItem, rcNotFound );
+
+ // pre-allocate node and leaf here
+ interm_node = calloc ( 1, sizeof ( KRowSetTreeNode ) );
+ if ( interm_node == NULL )
+ return RC ( rcDB, rcRowSet, rcInserting, rcMemory, rcExhausted );
+
+ rc = KRowSetAllocateLeaf ( LeafType_ArrayRanges, leaf_id, &new_leaf );
+ if ( rc != 0 )
+ {
+ free ( interm_node );
+ return rc;
+ }
+
+ KRowSetSplitNode ( node, depth, interm_node, interm_node_depth, leaf_id );
+
+ node = interm_node;
+ depth = interm_node_depth;
+#if CHECK_NODE_MARKS
+ assert ( node->node_mark == NODE_MARK );
+#endif
+ assert ( depth < LEAF_DEPTH );
+ nodes_stack[*nodes_stack_size] = node;
+ nodes_depth[(*nodes_stack_size)++] = depth;
+
+ KRowSetInsertLeaf ( self, leaf_id, node, depth, nodes_stack, nodes_depth, *nodes_stack_size, new_leaf );
+ *leaf_found = new_leaf;
+ return 0;
+
+ }
+
+ // check if transition leads to a leaf or follow the transition
+ if ( depth + 1 + tr_size_matched == LEAF_DEPTH )
+ {
+ *leaf_found = node->children[bt];
+ return 0;
+ }
+
+ node = node->children[bt];
+ depth += 1 + tr_size_matched;
+#if CHECK_NODE_MARKS
+ assert ( node->node_mark == NODE_MARK );
+#endif
+ assert ( depth < LEAF_DEPTH );
+ nodes_stack[*nodes_stack_size] = node;
+ nodes_depth[(*nodes_stack_size)++] = depth;
+ }
+
+ // "for" loop iterates up to a maximum possible depth,
+ // so it must come to a leaf
+ assert ( false );
+ return 0;
+}
+
+/**
+ * Transforms leaf from LeafType_ArrayRanges to LeafType_Bitmap
+ */
+static
+rc_t KRowSetTreeLeafTransform ( KRowSet * self, KRowSetTreeLeaf ** p_leaf, KRowSetTreeNode * parent_node, int parent_depth )
+{
+ rc_t rc;
+ int i;
+ int j;
+ int len;
+ uint8_t parent_bt;
+ uint16_t leaf_bt;
+ KRowSetTreeLeaf * leaf;
+ KRowSetTreeLeaf * new_leaf;
+
+ assert ( p_leaf != NULL );
+ leaf = *p_leaf;
+ assert ( leaf != NULL );
+
+ parent_bt = KRowSetGetLeafIdByte ( leaf->header.leaf_id, parent_depth );
+ assert ( KRowSetNodeIsTransitionMatches ( leaf->header.leaf_id, parent_depth, &parent_node->transitions[parent_bt], NULL ) );
+
+ rc = KRowSetAllocateLeaf( LeafType_Bitmap, leaf->header.leaf_id, &new_leaf );
+ if ( rc != 0 )
+ return rc;
+
+ // copy rows to a new leaf
+ len = leaf->data.array_ranges.len;
+ for ( i = 0; i < len; ++i )
+ {
+ struct KRowSetTreeLeafRowRange * range = &leaf->data.array_ranges.ranges[i];
+ for ( j = range->start; j <= range->end; ++j )
+ {
+ leaf_bt = j;
+ new_leaf->data.bitmap[leaf_bt >> 3] |= 1 << (leaf_bt & 7);
+ }
+ }
+ new_leaf->header.leaf_rows = leaf->header.leaf_rows;
+
+ DLListInsertNodeBefore ( &self->leaf_nodes, &leaf->header.dad, &new_leaf->header.dad );
+ KRowSetLeafWhack ( &leaf->header.dad, &self->leaf_nodes );
+ leaf = new_leaf;
+ parent_node->children[parent_bt] = leaf;
+ *p_leaf = leaf;
+
+ return 0;
+}
+
+static
+int KRowSetBitmapByteRows ( uint8_t bt )
+{
+ static int NIBBLE_LOOKUP[] = {
+ 0, 1, 1, 2, 1, 2, 2, 3,
+ 1, 2, 2, 3, 2, 3, 3, 4
+ };
+ return NIBBLE_LOOKUP[bt & 0x0F] + NIBBLE_LOOKUP[bt >> 4];
+}
+
+
+/**
+ * NB- it may reallocate leaf in case when leaf is transformed,
+ * so leaf pointer may change as a result of this function
+ */
+static
+rc_t KRowSetTreeLeafAddRows ( KRowSet * self, KRowSetTreeLeaf ** p_leaf, uint16_t leaf_row_start, uint16_t leaf_row_end, KRowSetTreeNode * parent_node, int parent_depth, int * rows_inserted )
+{
+ KRowSetTreeLeaf * leaf = *p_leaf;
+ int i;
+ uint16_t leaf_row;
+
+ assert ( rows_inserted != NULL );
+ assert ( leaf_row_end >= leaf_row_start );
+
+ *rows_inserted = 0;
+
+ switch ( leaf->header.type )
+ {
+ case LeafType_Bitmap:
+ {
+ uint8_t* mem_ptr;
+ size_t mem_size;
+
+ // try to align to the nearest 8 rows from left
+ if ( (leaf_row_start | 7) < leaf_row_end )
+ {
+ for ( i = leaf_row_start; i <= (leaf_row_start | 7); ++i )
+ {
+ leaf_row = i;
+ if ( ( leaf->data.bitmap[leaf_row >> 3] & (1 << (leaf_row & 7)) ) != 0 )
+ continue;
+
+ leaf->data.bitmap[leaf_row >> 3] |= 1 << (leaf_row & 7);
+ ++leaf->header.leaf_rows;
+ ++(*rows_inserted);
+ }
+
+ leaf_row_start = (leaf_row_start | 7) + 1;
+ }
+
+ // if we are left aligned and still have more than 8 rows,
+ // then use memset to set multiple rows at once
+ if ( (leaf_row_start & 7) == 0 && leaf_row_end + 1 - leaf_row_start >= 8 )
+ {
+ int range_rows_prev = 0;
+
+ mem_ptr = &leaf->data.bitmap[leaf_row_start >> 3];
+ mem_size = (leaf_row_end + 1 - leaf_row_start) / 8;
+
+ assert ( mem_size > 0 );
+
+ for ( i = 0; i < mem_size; ++i )
+ {
+ range_rows_prev += KRowSetBitmapByteRows ( mem_ptr[i] );
+ }
+
+ memset ( mem_ptr, 0xFF, mem_size );
+ leaf_row_start += mem_size * 8;
+ *rows_inserted += mem_size * 8 - range_rows_prev;
+ leaf->header.leaf_rows += mem_size * 8 - range_rows_prev;
+ }
+
+ // insert all remaining rows
+ {
+ for ( i = leaf_row_start; i <= leaf_row_end; ++i )
+ {
+ leaf_row = i;
+ if ( ( leaf->data.bitmap[leaf_row >> 3] & (1 << (leaf_row & 7)) ) != 0 )
+ continue;
+
+ leaf->data.bitmap[leaf_row >> 3] |= 1 << (leaf_row & 7);
+ ++leaf->header.leaf_rows;
+ ++(*rows_inserted);
+ }
+ return 0;
+ }
+
+ break;
+ }
+ case LeafType_ArrayRanges:
+ {
+ int j;
+ int insert_i;
+ int len = leaf->data.array_ranges.len;
+ int max_len = 8;
+ for ( i = 0; i < len; ++i )
+ {
+ struct KRowSetTreeLeafRowRange * range = &leaf->data.array_ranges.ranges[i];
+
+ // we can just increase existing range to the left
+ if ( range->start > 0 && leaf_row_end >= range->start - 1 && leaf_row_start < range->start )
+ {
+ if ( i == 0 || (range - 1)->end < leaf_row_start )
+ {
+ *rows_inserted += range->start - leaf_row_start;
+ leaf->header.leaf_rows += range->start - leaf_row_start;
+ range->start = leaf_row_start;
+ }
+ }
+
+ // or to the right
+ // but lets first check next range, maybe we can just merge them both
+ if ( leaf_row_start <= range->end + 1 && leaf_row_end > range->end )
+ {
+ if ( i + 1 < len && leaf_row_end >= (range+1)->start - 1 )
+ {
+ // just merge two ranges
+ *rows_inserted += (range+1)->start - range->end - 1;
+
+ range->end = (range+1)->end;
+
+ --len;
+ --leaf->data.array_ranges.len;
+ for ( j = i + 1; j < len; --j )
+ {
+ leaf->data.array_ranges.ranges[j] = leaf->data.array_ranges.ranges[j + 1];
+ }
+ --i;
+ continue;
+ }
+
+ leaf->header.leaf_rows += leaf_row_end - range->end;
+ *rows_inserted += leaf_row_end - range->end;
+ range->end = leaf_row_end;
+ }
+
+ // not found - insert new range
+ if ( range->start > leaf_row_end && ( i == 0 || (range - 1)->end < leaf_row_start ) )
+ {
+ break;
+ }
+ }
+
+ // check if we are done
+ if ( i > 0 && leaf->data.array_ranges.ranges[i-1].end >= leaf_row_end )
+ break;
+
+ // came here because we need to insert a new range
+ // (or transform a leaf to a bitmap)
+ insert_i = i;
+
+ if ( len == max_len )
+ {
+ rc_t rc;
+ rc = KRowSetTreeLeafTransform ( self, p_leaf, parent_node, parent_depth );
+ if ( rc != 0 )
+ return rc;
+
+ return KRowSetTreeLeafAddRows ( self, p_leaf, leaf_row_start, leaf_row_end, parent_node, parent_depth, rows_inserted );
+ }
+
+ ++len;
+ ++leaf->data.array_ranges.len;
+ for ( i = len - 1; i > insert_i; --i )
+ {
+ leaf->data.array_ranges.ranges[i] = leaf->data.array_ranges.ranges[i - 1];
+ }
+
+ leaf->data.array_ranges.ranges[insert_i].start = leaf_row_start;
+ leaf->data.array_ranges.ranges[insert_i].end = leaf_row_end;
+ leaf->header.leaf_rows += leaf_row_end - leaf_row_start + 1;
+ *rows_inserted += leaf_row_end - leaf_row_start + 1;
+
+ break;
+ }
+ default:
+ assert ( false );
+ }
+
+ return 0;
+}
+
+KDB_EXTERN rc_t CC KRowSetAddRowIdRange ( KRowSet * self, int64_t row_id_start,
+ uint64_t row_id_count, uint64_t * optional_inserted )
+{
+ rc_t rc = 0;
+ KRowSetTreeLeaf * leaf;
+ int64_t current_range_start = row_id_start;
+ uint64_t total_inserted = 0;
+ uint16_t leaf_row_start;
+ uint16_t leaf_row_end;
+ int leaf_row_count;
+ int leaf_rows_inserted;
+
+ int nodes_stack_size;
+ KRowSetTreeNode * nodes_stack[LEAF_DEPTH];
+ int nodes_depth[LEAF_DEPTH];
+
+ if ( row_id_start < 0 || row_id_count <= 0 )
+ return RC ( rcDB, rcRowSet, rcInserting, rcParam, rcInvalid );
+
+ if ( row_id_start + row_id_count < row_id_start )
+ return RC ( rcDB, rcRowSet, rcInserting, rcParam, rcOutofrange );
+
+ if ( optional_inserted != NULL )
+ *optional_inserted = 0;
+
+ while ( rc == 0 && current_range_start < row_id_start + row_id_count )
+ {
+ leaf_row_start = current_range_start & ROW_LEAF_MAX;
+ leaf_row_count = ROW_LEAF_MAX - leaf_row_start + 1;
+ if ( current_range_start + leaf_row_count > row_id_start + row_id_count )
+ leaf_row_count = row_id_start + row_id_count - current_range_start;
+
+ assert ( leaf_row_start + leaf_row_count - 1 <= ROW_LEAF_MAX );
+ leaf_row_end = leaf_row_start + leaf_row_count - 1;
+
+ rc = KRowSetGetLeaf ( self, current_range_start >> 16, true, &leaf, nodes_stack, nodes_depth, &nodes_stack_size );
+ if ( rc == 0 )
+ {
+ rc = KRowSetTreeLeafAddRows ( self, &leaf, leaf_row_start, leaf_row_end, nodes_stack[nodes_stack_size-1], nodes_depth[nodes_stack_size-1], &leaf_rows_inserted );
+ }
+
+ if ( rc == 0 )
+ assert ( leaf_row_count >= leaf_rows_inserted );
+ else
+ assert ( leaf_row_count > leaf_rows_inserted );
+
+ current_range_start += leaf_row_count;
+ total_inserted += leaf_rows_inserted;
+ }
+
+ self->number_rows += total_inserted;
+ if ( optional_inserted != NULL )
+ *optional_inserted = total_inserted;
+
+ return rc;
+}
+
+KDB_EXTERN rc_t CC KRowSetAddRowId ( KRowSet * self, int64_t row_id, bool * optional_inserted )
+{
+ rc_t rc;
+ uint64_t num_inserted;
+
+ if ( optional_inserted != NULL )
+ *optional_inserted = false;
+
+ rc = KRowSetAddRowIdRange ( self, row_id, 1, &num_inserted );
+ if ( rc != 0 )
+ return rc;
+
+ if ( optional_inserted != NULL )
+ *optional_inserted = num_inserted > 0;
+
+ return 0;
+}
+
+KDB_EXTERN rc_t CC KRowSetGetNumRowIds ( const KRowSet * self, uint64_t * num_rows )
+{
+ if ( self == NULL )
+ return RC ( rcDB, rcRowSet, rcAccessing, rcSelf, rcNull );
+
+ if ( num_rows == NULL )
+ return RC ( rcDB, rcRowSet, rcAccessing, rcParam, rcNull );
+
+ *num_rows = self->number_rows;
+ return 0;
+}
+
+KDB_EXTERN rc_t CC KRowSetVisit ( const KRowSet * self, bool reverse,
+ void ( CC * cb ) ( int64_t row_id, void * data ), void * data )
+{
+ rc_t rc;
+ KRowSetIterator * it;
+
+ if ( self == NULL )
+ return RC ( rcDB, rcRowSet, rcAccessing, rcSelf, rcNull );
+
+ if ( self->number_rows == 0 )
+ return 0;
+
+ rc = KRowSetMakeIterator ( self, &it );
+ if ( rc != 0 )
+ return rc;
+
+ if ( !reverse )
+ rc = KRowSetIteratorFirst ( it );
+ else
+ rc = KRowSetIteratorLast ( it );
+
+ while ( rc == 0 )
+ {
+ int64_t row_id;
+ rc = KRowSetIteratorRowId ( it, &row_id );
+ if ( rc != 0 )
+ break;
+
+ cb ( row_id, data );
+
+ if ( !reverse )
+ rc = KRowSetIteratorNext ( it );
+ else
+ rc = KRowSetIteratorPrev ( it );
+
+ if ( GetRCState ( rc ) == rcDone )
+ {
+ rc = 0;
+ break;
+ }
+ }
+
+ KRowSetIteratorRelease ( it );
+
+ return rc;
+}
+
+/**
+ * Tries to intersect array-range leaf (self) with bitmap leaf (other).
+ * Returns true if succeeds to produce array-range result and stores it in target_ranges/target_rows
+ */
+static
+bool KRowSetIntersectRowRangeAndBitmapLeaves ( KRowSet * self, const KRowSetTreeLeaf * self_leaf, const KRowSetTreeLeaf * other_leaf, struct KRowSetTreeLeafArrayRanges * target_ranges, int * target_rows )
+{
+ const struct KRowSetTreeLeafRowRange * self_range;
+ const struct KRowSetTreeLeafRowRange * self_end_range;
+
+ struct KRowSetTreeLeafRowRange * target_range;
+ const struct KRowSetTreeLeafRowRange * target_first_range;
+ const struct KRowSetTreeLeafRowRange * target_end_range;
+
+ assert ( self_leaf != NULL );
+ assert ( other_leaf != NULL );
+ assert ( target_ranges != NULL );
+ assert ( target_rows != NULL );
+
+#if CHECK_NODE_MARKS
+ assert ( self_leaf->header.leaf_mark == LEAF_MARK );
+ assert ( other_leaf->header.leaf_mark == LEAF_MARK );
+#endif
+
+ assert ( self_leaf->header.type == LeafType_ArrayRanges );
+ assert ( other_leaf->header.type == LeafType_Bitmap );
+
+ self_range = &self_leaf->data.array_ranges.ranges[0];
+ self_end_range = &self_leaf->data.array_ranges.ranges[self_leaf->data.array_ranges.len];
+
+ target_range = &target_ranges->ranges[0];
+ target_first_range = &target_ranges->ranges[0];
+ target_end_range = &target_ranges->ranges[8];
+
+ target_ranges->len = 0;
+ *target_rows = 0;
+
+ int i;
+ for ( i = 0; i < LEAF_DATA_SZ_BT; ++i )
+ {
+ uint16_t bt_first_row = i << 3;
+ uint16_t bt_last_row = bt_first_row | 7;
+ int j;
+
+ if ( other_leaf->data.bitmap[i] == 0 )
+ continue;
+
+ while ( self_range != self_end_range && bt_first_row > self_range->end )
+ ++self_range;
+
+ if ( self_range == self_end_range || bt_first_row > self_range->end )
+ {
+ return true;
+ }
+
+ if ( bt_last_row < self_range->start )
+ {
+ i = (self_range->start >> 3) - 1;
+ continue;
+ }
+
+ for ( j = 0; j < 8; ++j )
+ {
+ uint16_t bt_row = bt_first_row + j;
+ if ( (other_leaf->data.bitmap[i] & (1 << j)) != 0 && bt_row <= self_range->end && bt_row >= self_range->start )
+ {
+ if ( target_range != target_first_range && (target_range-1)->end == bt_row - 1 )
+ {
+ ++(target_range-1)->end;
+ ++(*target_rows);
+ continue;
+ }
+
+ if ( target_range == target_end_range )
+ {
+ return false;
+ }
+
+ target_range->start = target_range->end = bt_row;
+ ++(*target_rows);
+ ++target_range;
+ ++target_ranges->len;
+ }
+ }
+ }
+
+
+ return true;
+}
+
+/**
+ * Tries to intersect array-range leaf (self) with array-range leaf (other).
+ * Returns true if succeeds to produce array-range result and stores it in target_ranges/target_rows
+ */
+static
+bool KRowSetIntersectRowRangeLeaves ( KRowSet * self, const KRowSetTreeLeaf * self_leaf, const KRowSetTreeLeaf * other_leaf, struct KRowSetTreeLeafArrayRanges * target_ranges, int * target_rows )
+{
+ const struct KRowSetTreeLeafRowRange * self_range;
+ const struct KRowSetTreeLeafRowRange * self_end_range;
+ const struct KRowSetTreeLeafRowRange * other_range;
+ const struct KRowSetTreeLeafRowRange * other_end_range;
+
+ struct KRowSetTreeLeafRowRange * target_range;
+ const struct KRowSetTreeLeafRowRange * target_end_range;
+
+ assert ( self_leaf != NULL );
+ assert ( other_leaf != NULL );
+ assert ( target_ranges != NULL );
+ assert ( target_rows != NULL );
+
+#if CHECK_NODE_MARKS
+ assert ( self_leaf->header.leaf_mark == LEAF_MARK );
+ assert ( other_leaf->header.leaf_mark == LEAF_MARK );
+#endif
+
+ assert ( self_leaf->header.type == LeafType_ArrayRanges );
+ assert ( other_leaf->header.type == LeafType_ArrayRanges );
+
+ self_range = &self_leaf->data.array_ranges.ranges[0];
+ self_end_range = &self_leaf->data.array_ranges.ranges[self_leaf->data.array_ranges.len];
+ other_range = &other_leaf->data.array_ranges.ranges[0];
+ other_end_range = &other_leaf->data.array_ranges.ranges[other_leaf->data.array_ranges.len];
+
+ target_range = &target_ranges->ranges[0];
+ target_end_range = &target_ranges->ranges[8];
+
+ target_ranges->len = 0;
+ *target_rows = 0;
+
+ while ( self_range != self_end_range && other_range != other_end_range )
+ {
+ if ( self_range->end < other_range->start )
+ {
+ ++self_range;
+ continue;
+ }
+
+ if ( other_range->end < self_range->start )
+ {
+ ++other_range;
+ continue;
+ }
+
+ if ( target_range == target_end_range )
+ return false;
+
+ target_range->start = MAX ( self_range->start, other_range->start );
+ target_range->end = MIN ( self_range->end, other_range->end );
+ *target_rows += target_range->end - target_range->start + 1;
+ ++target_range;
+ ++target_ranges->len;
+
+ if ( self_range->end < other_range->end )
+ ++self_range;
+ else if ( other_range->end < self_range->end )
+ ++other_range;
+ else
+ {
+ ++self_range;
+ ++other_range;
+ }
+ }
+
+ return true;
+}
+
+/**
+ * Performs mutable intersection of self_leaf with other_leaf.
+ * Result of intersection is saved to self_leaf.
+ *
+ * NB - it checks types of self and other leaves and performs intersection depending on types.
+ */
+static
+rc_t KRowSetOpAndIntersectLeaf ( KRowSet * self, KRowSetTreeLeaf ** self_leaf_p, const KRowSetTreeLeaf * other_leaf )
+{
+ KRowSetTreeLeaf * self_leaf;
+ int64_t leaf_id;
+
+ assert ( self_leaf_p != NULL );
+ self_leaf = *self_leaf_p;
+ assert ( self_leaf != NULL );
+ assert ( other_leaf != NULL );
+
+#if CHECK_NODE_MARKS
+ assert ( self_leaf->header.leaf_mark == LEAF_MARK );
+ assert ( other_leaf->header.leaf_mark == LEAF_MARK );
+#endif
+
+ leaf_id = self_leaf->header.leaf_id;
+ assert ( leaf_id == other_leaf->header.leaf_id );
+
+ switch ( self_leaf->header.type )
+ {
+ case LeafType_Bitmap:
+ if ( other_leaf->header.type == LeafType_Bitmap )
+ {
+ int i;
+ for ( i = 0; i < LEAF_DATA_SZ_BT; ++i )
+ {
+ int rows_before = KRowSetBitmapByteRows ( self_leaf->data.bitmap[i] );
+ int rows_after;
+ self_leaf->data.bitmap[i] &= other_leaf->data.bitmap[i];
+ rows_after = KRowSetBitmapByteRows ( self_leaf->data.bitmap[i] );
+ self_leaf->header.leaf_rows -= rows_before - rows_after;
+ self->number_rows -= rows_before - rows_after;
+ }
+ return 0;
+
+ }
+
+ if ( other_leaf->header.type == LeafType_ArrayRanges )
+ {
+ int i;
+ const struct KRowSetTreeLeafRowRange * other_range = &other_leaf->data.array_ranges.ranges[0];
+ const struct KRowSetTreeLeafRowRange * other_end_range = &other_leaf->data.array_ranges.ranges[other_leaf->data.array_ranges.len];
+ for ( i = 0; i < LEAF_DATA_SZ_BT; ++i )
+ {
+ uint16_t bt_first_row = i << 3;
+ uint16_t bt_last_row = bt_first_row | 7;
+ int j;
+
+ if ( self_leaf->data.bitmap[i] == 0 )
+ continue;
+
+ while ( other_range != other_end_range && bt_first_row > other_range->end )
+ ++other_range;
+
+ if ( other_range == other_end_range || bt_first_row > other_range->end )
+ {
+ for ( ; i < LEAF_DATA_SZ_BT; ++i )
+ {
+ int bt_rows = KRowSetBitmapByteRows ( self_leaf->data.bitmap[i] );
+ self_leaf->data.bitmap[i] = 0;
+ self_leaf->header.leaf_rows -= bt_rows;
+ self->number_rows -= bt_rows;
+ }
+ break;
+ }
+
+ if ( bt_last_row < other_range->start )
+ {
+ int bt_rows = KRowSetBitmapByteRows ( self_leaf->data.bitmap[i] );
+ self_leaf->data.bitmap[i] = 0;
+ self_leaf->header.leaf_rows -= bt_rows;
+ self->number_rows -= bt_rows;
+ continue;
+ }
+
+ for ( j = 0; j < 8; ++j )
+ {
+ uint16_t bt_row = bt_first_row + j;
+ if ( (self_leaf->data.bitmap[i] & (1 << j))== 0 )
+ continue;
+
+ if ( bt_row > other_range->end || bt_row < other_range->start )
+ {
+ self_leaf->data.bitmap[i] &= ~(1 << j);
+ --self_leaf->header.leaf_rows;
+ --self->number_rows;
+ }
+ }
+ }
+ return 0;
+ }
+
+ assert ( false );
+ break;
+ case LeafType_ArrayRanges:
+ {
+ struct KRowSetTreeLeafArrayRanges result_ranges;
+ int result_rows;
+ bool can_produce_ranges;
+ if ( other_leaf->header.type == LeafType_Bitmap )
+ {
+ can_produce_ranges = KRowSetIntersectRowRangeAndBitmapLeaves ( self, self_leaf, other_leaf, &result_ranges, &result_rows );
+ }
+ else if ( other_leaf->header.type == LeafType_ArrayRanges )
+ {
+ can_produce_ranges = KRowSetIntersectRowRangeLeaves ( self, self_leaf, other_leaf, &result_ranges, &result_rows );
+ }
+ else
+ assert ( false );
+
+ if ( can_produce_ranges )
+ {
+ int i;
+ self_leaf->data.array_ranges.len = result_ranges.len;
+ for ( i = 0; i < result_ranges.len; ++i )
+ {
+ assert ( result_ranges.ranges[i].start <= result_ranges.ranges[i].end );
+ self_leaf->data.array_ranges.ranges[i] = result_ranges.ranges[i];
+ }
+ self->number_rows += result_rows - self_leaf->header.leaf_rows;
+ self_leaf->header.leaf_rows = result_rows;
+ return 0;
+ }
+
+ // transform array range leaf into bitmap and perform operation once again
+ {
+ rc_t rc;
+
+ int nodes_stack_size;
+ KRowSetTreeNode * nodes_stack[LEAF_DEPTH];
+ int nodes_depth[LEAF_DEPTH];
+ KRowSetTreeLeaf * search_leaf;
+
+ rc = KRowSetGetLeaf ( self, self_leaf->header.leaf_id, false, &search_leaf, nodes_stack, nodes_depth, &nodes_stack_size );
+ if ( GetRCObject ( rc ) == rcItem && GetRCState ( rc ) == rcNotFound )
+ return RC ( rcDB, rcRowSet, rcUpdating, rcItem, rcInconsistent );
+
+ if ( rc != 0 )
+ return rc;
+
+ if ( search_leaf != self_leaf )
+ return RC ( rcDB, rcRowSet, rcUpdating, rcItem, rcInconsistent );
+
+ rc = KRowSetTreeLeafTransform ( self, self_leaf_p, nodes_stack[nodes_stack_size-1], nodes_depth[nodes_stack_size-1] );
+ if ( rc != 0 )
+ return rc;
+
+ self_leaf = *self_leaf_p;
+
+ return KRowSetOpAndIntersectLeaf ( self, self_leaf_p, other_leaf );
+ }
+ break;
+ }
+ default:
+ assert ( false );
+ }
+
+ assert ( false );
+ return 0;
+}
+
+/**
+ * Removes a leaf (and its rows) from the tree and deallocates/collapses intermediate nodes when necessary
+ */
+static
+rc_t KRowSetRemoveLeaf ( KRowSet * self, KRowSetTreeLeaf * leaf )
+{
+ rc_t rc;
+ int i;
+ int64_t leaf_id;
+
+ int nodes_stack_size;
+ KRowSetTreeNode * nodes_stack[LEAF_DEPTH];
+ int nodes_depth[LEAF_DEPTH];
+ KRowSetTreeLeaf * search_leaf;
+
+ assert ( leaf != NULL );
+#if CHECK_NODE_MARKS
+ assert ( leaf->header.leaf_mark == LEAF_MARK );
+#endif
+ leaf_id = leaf->header.leaf_id;
+
+ rc = KRowSetGetLeaf ( self, leaf->header.leaf_id, false, &search_leaf, nodes_stack, nodes_depth, &nodes_stack_size );
+ if ( GetRCObject ( rc ) == rcItem && GetRCState ( rc ) == rcNotFound )
+ return RC ( rcDB, rcRowSet, rcUpdating, rcItem, rcInconsistent );
+
+ if ( rc != 0 )
+ return rc;
+
+ if ( search_leaf != leaf )
+ return RC ( rcDB, rcRowSet, rcUpdating, rcItem, rcInconsistent );
+
+ for ( i = nodes_stack_size - 1; i >= 0; --i )
+ {
+ int number_children;
+ int j;
+ int found_j; // this will be used in case when there is only a single child exists
+ uint8_t bt;
+
+ KRowSetTreeNode * node = nodes_stack[i];
+ assert ( node != NULL );
+#if CHECK_NODE_MARKS
+ assert ( node->node_mark == NODE_MARK );
+#endif
+
+ bt = KRowSetGetLeafIdByte ( leaf_id, nodes_depth[i] );
+
+ assert ( node->children[bt] != NULL );
+
+ node->children[bt] = NULL;
+ node->transitions[bt].size = 0;
+
+ number_children = 0;
+ found_j = -1;
+ for ( j = 0; j < 256; ++j )
+ {
+ if ( node->children[j] != NULL )
+ {
+ ++number_children;
+ found_j = j;
+ }
+ }
+
+ if ( number_children == 0 )
+ {
+ assert ( i != 0 || node == self->root_node );
+ KRowSetNodeWhack ( self, node, nodes_depth[i], false );
+ if ( i == 0 )
+ self->root_node = NULL;
+ continue;
+ }
+
+ // it has only a single child, need to collapse
+ if ( number_children == 1 && i > 0 )
+ KRowSetCollapseNode ( nodes_stack[i-1], nodes_depth[i-1], node, nodes_depth[i], found_j, leaf_id );
+
+ break;
+ }
+
+ self->number_rows -= leaf->header.leaf_rows;
+ --self->number_leaves;
+ KRowSetLeafWhack ( (DLNode *) leaf, &self->leaf_nodes );
+
+ return 0;
+}
+
+KDB_EXTERN rc_t CC KRowSetOpAnd ( KRowSet * self, const KRowSet * other )
+{
+ rc_t rc;
+ KRowSetTreeLeaf * self_leaf;
+ const KRowSetTreeLeaf * other_leaf;
+
+ if ( self == NULL )
+ return RC ( rcDB, rcRowSet, rcUpdating, rcSelf, rcNull );
+
+ if ( other == NULL )
+ return RC ( rcDB, rcRowSet, rcUpdating, rcParam, rcNull );
+
+ self_leaf = (KRowSetTreeLeaf *) KRowSetTreeGetFirstLeaf ( self );
+ other_leaf = KRowSetTreeGetFirstLeaf ( other );
+
+ while ( self_leaf != NULL )
+ {
+ KRowSetTreeLeaf * next_leaf;
+ if ( other_leaf == NULL || other_leaf->header.leaf_id > self_leaf->header.leaf_id )
+ {
+ next_leaf = (KRowSetTreeLeaf *) KRowSetTreeLeafGetNext ( self_leaf );
+ KRowSetRemoveLeaf ( self, self_leaf );
+ self_leaf = next_leaf;
+ continue;
+ }
+
+ if ( other_leaf->header.leaf_id < self_leaf->header.leaf_id )
+ {
+ other_leaf = KRowSetTreeLeafGetNext ( other_leaf );
+ continue;
+ }
+
+ // leafs are aligned
+ assert ( other_leaf->header.leaf_id == self_leaf->header.leaf_id );
+ rc = KRowSetOpAndIntersectLeaf ( self, &self_leaf, other_leaf );
+ if ( rc != 0 )
+ return rc;
+
+ other_leaf = KRowSetTreeLeafGetNext ( other_leaf );
+
+ next_leaf = (KRowSetTreeLeaf *) KRowSetTreeLeafGetNext ( self_leaf );
+ if ( self_leaf->header.leaf_rows == 0 )
+ KRowSetRemoveLeaf ( self, self_leaf );
+ self_leaf = next_leaf;
+ }
+
+ return 0;
+}
+
+const KRowSetTreeLeaf * KRowSetTreeGetFirstLeaf ( const KRowSet * self )
+{
+ assert ( self != NULL );
+ return (const KRowSetTreeLeaf * )DLListHead ( &self->leaf_nodes );
+}
+
+const KRowSetTreeLeaf * KRowSetTreeGetLastLeaf ( const KRowSet * self )
+{
+ assert ( self != NULL );
+ return (const KRowSetTreeLeaf * )DLListTail ( &self->leaf_nodes );
+}
+
+const KRowSetTreeLeaf * KRowSetTreeLeafGetNext ( const KRowSetTreeLeaf * leaf )
+{
+ assert ( leaf != NULL );
+ return (const KRowSetTreeLeaf * ) DLNodeNext ( &leaf->header.dad );
+}
+
+const KRowSetTreeLeaf * KRowSetTreeLeafGetPrev ( const KRowSetTreeLeaf * leaf )
+{
+ assert ( leaf != NULL );
+ return (const KRowSetTreeLeaf * ) DLNodePrev ( &leaf->header.dad );
+}
diff --git a/libs/kdb/table.c b/libs/kdb/table.c
index fe86a74..c77f809 100644
--- a/libs/kdb/table.c
+++ b/libs/kdb/table.c
@@ -306,6 +306,9 @@ LIB_EXPORT rc_t CC KDatabaseVOpenTableRead ( const KDatabase *self,
if ( self == NULL )
return RC ( rcDB, rcDatabase, rcOpening, rcSelf, rcNull );
+ if ( name == NULL )
+ return RC ( rcDB, rcDatabase, rcOpening, rcParam, rcNull );
+
rc = KDBVMakeSubPath ( self -> dir,
path, sizeof path, "tbl", 3, name, args );
if ( rc == 0 )
diff --git a/libs/kdb/wtable.c b/libs/kdb/wtable.c
index 0632043..5bb6f87 100644
--- a/libs/kdb/wtable.c
+++ b/libs/kdb/wtable.c
@@ -760,6 +760,9 @@ LIB_EXPORT rc_t CC KDatabaseVOpenTableRead ( const KDatabase *self,
if ( self == NULL )
return RC ( rcDB, rcDatabase, rcOpening, rcSelf, rcNull );
+ if ( name == NULL )
+ return RC ( rcDB, rcDatabase, rcOpening, rcParam, rcNull );
+
rc = KDBVMakeSubPath ( self -> dir,
path, sizeof path, "tbl", 3, name, args );
if ( rc == 0 )
diff --git a/libs/kfc/refcount.c b/libs/kfc/refcount.c
index 107ac35..8a25c12 100644
--- a/libs/kfc/refcount.c
+++ b/libs/kfc/refcount.c
@@ -160,6 +160,7 @@ LIB_EXPORT void CC KRefcountInit_v1 ( KRefcount_v1 *self, ctx_t ctx,
}
KRefcountInit ( & self -> refcount, 1, kvt -> name, "init", instance_name ? instance_name : "" );
+ self -> vt = kvt;
#if _DEBUGGING && _ARCH_BITS > 32
memset ( & self -> align, 0, sizeof self -> align );
#endif
diff --git a/libs/kfc/unix/sysrsrc.c b/libs/kfc/unix/sysrsrc.c
index 967d396..212a27b 100644
--- a/libs/kfc/unix/sysrsrc.c
+++ b/libs/kfc/unix/sysrsrc.c
@@ -133,7 +133,7 @@ void KRsrcGlobalWhack ( ctx_t ctx )
s_rsrc . proc = NULL;
/* run destructor tasks while we still have other mgrs */
- /*KProcMgrWhack ();*/
+ KProcMgrWhack ();
/* tear it down */
KRsrcWhack ( & s_rsrc, ctx );
diff --git a/libs/kfc/vtbl.c b/libs/kfc/vtbl.c
index 4daeacc..6a0d6b3 100644
--- a/libs/kfc/vtbl.c
+++ b/libs/kfc/vtbl.c
@@ -152,10 +152,13 @@ uint32_t KVTableDepth ( KVTable * self /*, ctx_t ctx */ )
assert ( self -> itf != NULL );
idx = self -> itf -> idx;
- if ( idx == 0 && self -> parent != NULL )
+ if ( idx == 0 )
{
KItfTok * itf = ( KItfTok * ) self -> itf;
- idx = KVTableDepth ( ( KVTable* ) self -> parent /*, ctx */ ) + 1;
+ if ( self -> parent != NULL )
+ idx = KVTableDepth ( ( KVTable* ) self -> parent /*, ctx */ ) + 1;
+ else
+ idx = 1;
itf -> idx = idx;
}
@@ -178,7 +181,7 @@ void KVTableResolve ( const KVTable * cself, ctx_t ctx )
uint32_t idx = KVTableDepth ( self /*, ctx */ );
KHierCache * cache;
- size_t bytes = sizeof * cache - sizeof cache -> parent + idx + sizeof cache -> parent [ 0 ];
+ size_t bytes = sizeof * cache - sizeof cache -> parent + idx * sizeof cache -> parent [ 0 ];
cache = malloc ( bytes );
if ( cache == NULL )
diff --git a/libs/kfg/Makefile b/libs/kfg/Makefile
index 264ffdb..d30c7ef 100644
--- a/libs/kfg/Makefile
+++ b/libs/kfg/Makefile
@@ -63,10 +63,10 @@ $(TARGDIR)/std: \
#INTERM_SRC = \
$(SRCDIR)/config-lex.c \
$(SRCDIR)/config-grammar.c \
- $(SRCDIR)/config-tokens.h
+ $(SRCDIR)/config-tokens.h
clean: stdclean
- @ rm -f $(addsuffix /default.kfg,$(NCBIDIR)) $(TEST_BINDIR)/ncbi/default.kfg
+ @ rm -f $(addsuffix /default.kfg,$(NCBIDIR))
.PHONY: clean
@@ -75,18 +75,20 @@ clean: stdclean
#-------------------------------------------------------------------------------
# kfg
#
-$(ILIBDIR)/libkfg: $(TARGDIR)/default.kfg $(addprefix $(ILIBDIR)/libkfg.,$(ILIBEXT))
+$(ILIBDIR)/libkfg: \
+ $(TARGDIR)/default.kfg \
+ $(TARGDIR)/certs.kfg \
+ $(addprefix $(ILIBDIR)/libkfg.,$(ILIBEXT))
-$(TARGDIR)/default.kfg: $(addsuffix /default.kfg,$(NCBIDIR)) $(TEST_BINDIR)/ncbi/default.kfg
+$(TARGDIR)/default.kfg: $(addsuffix /default.kfg,$(NCBIDIR))
+
+$(TARGDIR)/certs.kfg: $(addsuffix /certs.kfg,$(NCBIDIR))
$(addsuffix /default.kfg,$(NCBIDIR)): $(SRCDIR)/default.kfg
cp $^ $@
-ifneq ($(NCBIDIR),$(TEST_BINDIR)/ncbi)
-$(TEST_BINDIR)/ncbi/default.kfg: $(SRCDIR)/default.kfg
- mkdir -p $(TEST_BINDIR)/ncbi
+$(addsuffix /certs.kfg,$(NCBIDIR)): $(SRCDIR)/certs.kfg
cp $^ $@
-endif
KFG_SRC = \
config-lex \
@@ -98,14 +100,14 @@ KFG_SRC = \
properties \
ngc \
report-kfg \
- keystore
+ keystore
# flex/bison should only be invoked manually in an environment ensures the correct versions:
# bison 2.5, flex 2.5.35
bison:
- bison -o $(SRCDIR)/config-grammar.c --defines=$(SRCDIR)/config-tokens.h -v --no-lines $(SRCDIR)/config-grammar.y
+ bison -o $(SRCDIR)/config-grammar.c --defines=$(SRCDIR)/config-tokens.h -v --no-lines $(SRCDIR)/config-grammar.y
-flex:
+flex:
flex -t $(SRCDIR)/config-lex.l | grep -v '^#line' > $(SRCDIR)/config-lex.c
#$(SRCDIR)/config-grammar.c $(SRCDIR)/config-tokens.h: $(SRCDIR)/config-grammar.y
diff --git a/libs/kfg/certs.kfg b/libs/kfg/certs.kfg
new file mode 100644
index 0000000..96d4122
--- /dev/null
+++ b/libs/kfg/certs.kfg
@@ -0,0 +1,31 @@
+# ===========================================================================
+#
+# PUBLIC DOMAIN NOTICE
+# National Center for Biotechnology Information
+#
+# This software/database is a "United States Government Work" under the
+# terms of the United States Copyright Act. It was written as part of
+# the author's official duties as a United States Government employee and
+# thus cannot be copyrighted. This software/database is freely available
+# to the public for use. The National Library of Medicine and the U.S.
+# Government have not placed any restriction on its use or reproduction.
+#
+# Although all reasonable efforts have been taken to ensure the accuracy
+# and reliability of the software and data, the NLM and the U.S.
+# Government do not and cannot warrant the performance or results that
+# may be obtained by using this software or data. The NLM and the U.S.
+# Government disclaim all warranties, express or implied, including
+# warranties of performance, merchantability or fitness for any particular
+# purpose.
+#
+# Please cite the author in any work or product based on this material.
+#
+# ===========================================================================
+
+
+##################
+# trusted CA certs
+
+/tls/ca.crt/ncbi1 = "-----BEGIN CERTIFICATE-----\r\nMIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh\r\nMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3\r\nd3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD\r\nQTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT\r\nMRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j\r\nb20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG\r\n9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvh [...]
+
+/tls/ca.crt/ncbi2 = "-----BEGIN CERTIFICATE-----\r\nMIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs\r\nMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3\r\nd3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j\r\nZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL\r\nMAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3\r\nLmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug\r\nRVYgUm9vdCBDQTCCASIwDQYJKoZIhvcN [...]
diff --git a/libs/kfg/config-aws.c b/libs/kfg/config-aws.c
index 4271d15..fb702f7 100644
--- a/libs/kfg/config-aws.c
+++ b/libs/kfg/config-aws.c
@@ -51,34 +51,6 @@ rc_t aws_KConfigNodeUpdateChild ( KConfigNode *self, String *name, String *value
return rc;
}
-
-static
-void StringTrim ( String *trim, const String *source )
-{
- uint32_t str_len = source -> len;
- const char *start = source -> addr;
- const char *end = start + source -> size;
-
- while ( start < end )
- {
- if ( ! isspace ( * start ) )
- break;
-
- ++ start;
- -- str_len;
- }
-
- while ( end > start )
- {
- if ( ! isspace ( end [ - 1 ] ) )
- break;
-
- -- end;
- -- str_len;
- }
-
- StringInit ( trim, start, end - start, str_len );
-}
static
rc_t aws_extract_key_value_pair ( const String *source, String *key, String *val )
@@ -93,13 +65,13 @@ rc_t aws_extract_key_value_pair ( const String *source, String *key, String *val
/* key */
StringInit ( &k, start, eql - start, string_len ( start, eql - start ) );
- StringTrim ( key, &k );
+ StringTrim ( &k, key );
start = eql + 1;
/* value */
StringInit ( &v, start, end - start, string_len ( start, end - start ) );
- StringTrim ( val, &v );
+ StringTrim ( &v, val );
return 0;
}
@@ -123,7 +95,7 @@ void aws_parse_file ( const KFile *self, KConfigNode *aws_node,
StringInit ( &string, start, sep - start, string_len ( start, sep - start ) );
- StringTrim ( &trim, &string );
+ StringTrim ( &string, &trim );
/* check for comment line and skip */
if ( StringLength ( & trim ) != 0 && trim . addr [ 0 ] == '#' )
diff --git a/libs/kfg/config.c b/libs/kfg/config.c
index ed9d239..cd04301 100644
--- a/libs/kfg/config.c
+++ b/libs/kfg/config.c
@@ -32,17 +32,18 @@ struct KfgConfigNamelist;
#include <klib/namelist.h>
#include <klib/impl.h>
-#include <klib/token.h>
#include <klib/container.h>
#include <klib/data-buffer.h> /* KDataBuffer */
-#include <klib/refcount.h>
-#include <klib/text.h>
-#include <klib/printf.h>
-#include <klib/rc.h>
#include <klib/debug.h>
-#include <klib/log.h>
+#include <klib/log.h>
#include <klib/out.h> /* OUTMSG */
+#include <klib/printf.h>
+#include <klib/rc.h>
+#include <klib/refcount.h>
+#include <klib/text.h>
+#include <klib/token.h>
#include <klib/klib-priv.h>
+
#include <kfs/directory.h>
#include <kfs/gzip.h> /* KFileMakeGzipForRead */
#include <kfs/subfile.h> /* KFileMakeSubRead */
@@ -89,16 +90,9 @@ static const char default_kfg[] = {
"/repository/user/main/public/apps/wgs/volumes/wgsFlat = \"wgs\"\n"
"/repository/user/main/public/root = \"$(HOME)/ncbi/public\"\n"
"/repository/remote/main/CGI/resolver-cgi = "
- "\"http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi\"\n"
-"/repository/remote/aux/NCBI/apps/nakmer/volumes/fuseNAKMER = \"sadb\"\n"
-"/repository/remote/aux/NCBI/apps/nannot/volumes/fuseNANNOT = \"sadb\"\n"
-"/repository/remote/aux/NCBI/apps/refseq/volumes/refseq = \"refseq\"\n"
-"/repository/remote/aux/NCBI/apps/sra/volumes/fuse1000 = "
- "\"sra-instant/reads/ByRun/sra\"\n"
-"/repository/remote/aux/NCBI/apps/wgs/volumes/fuseWGS = \"wgs\"\n"
-"/repository/remote/aux/NCBI/root = \"http://ftp-trace.ncbi.nlm.nih.gov/sra\"\n"
+ "\"https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi\"\n"
"/repository/remote/protected/CGI/resolver-cgi = "
- "\"http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi\"\n"
+ "\"https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi\"\n"
"/tools/ascp/max_rate = \"1000m\"\n"
};
/*----------------------------------------------------------------------------*/
@@ -413,32 +407,6 @@ rc_t find_home_directory ( KDyld *dyld,
}
rc = KDyldHomeDirectory ( dyld, dir, ( fptr_t ) KConfigMake );
-
- if ( rc != 0
- || (KDirectoryPathType ( * dir, "ncbi" ) & ~kptAlias) != kptDir )
- {
- KDylib * lib;
- if ( rc == 0 )
- {
- /* Nominally succeeded, but got a useless directory
- * (for a statically linked executable?); try again. */
- KDirectoryRelease ( * dir );
- }
- rc = KDyldLoadLib ( dyld, & lib, LPFX "kfg-beacon" SHLX );
- if ( rc == 0 )
- {
- KSymAddr * sym;
- if ( ( rc = KDylibSymbol ( lib, & sym, "KConfigBeacon" ) ) == 0 )
- {
- fptr_t fp;
- KSymAddrAsFunc ( sym, & fp );
- rc = KDyldHomeDirectory ( dyld, dir, fp );
- KSymAddrRelease ( sym );
- }
- KDylibRelease ( lib );
- }
- }
-
if ( rc == 0 && KDirectoryAddRef ( * dir ) == 0 )
{
cached_dir = * dir;
@@ -614,7 +582,7 @@ KToken *KConfigNodeFind ( const KConfigNode *self, const KConfigNode **n, KToken
/* Create
*/
static
-KToken *KConfigNodeCreate ( KConfigNode *self, KConfigNode **n, KTokenSource *src, KToken *t,
+KToken *KConfigNodeCreate ( KConfigNode *self, KConfigNode **n, KTokenSource *src, KToken *t,
KConfigIncluded *current_file )
{
bool created = false;
@@ -645,7 +613,7 @@ KToken *KConfigNodeCreate ( KConfigNode *self, KConfigNode **n, KTokenSource *sr
rc_t rc = KConfigNodeMake ( & child, & t -> str );
if ( rc != 0 )
return t;
- BSTreeInsert ( & self -> children, & child -> n, KConfigNodeSort );
+ BSTreeInsert ( & self -> children, & child -> n, KConfigNodeSort );
child -> dad = self;
self = child;
created = true;
@@ -715,7 +683,7 @@ rc_t KConfigNodeVOpenNodeReadInt ( const KConfigNode *self, const KConfig *mgr,
KTokenText tt;
KTokenSource src;
char full [ 4096 ];
-
+
rc = init_token_source ( & tt, & src, full, sizeof full, "", path, args );
if ( rc == 0 )
{
@@ -887,7 +855,7 @@ rc_t KConfigNodeVOpenNodeUpdateInt ( KConfigNode *self, KConfig *mgr,
KTokenText tt;
KTokenSource src;
char full [ 4096 ];
-
+
rc = init_token_source ( & tt, & src, full, sizeof full, "", path, args );
if ( rc == 0 )
{
@@ -1101,9 +1069,9 @@ LIB_EXPORT rc_t CC KConfigNodeWrite ( KConfigNode *self, const char *buffer, siz
assert ( self -> val_buffer != NULL );
string_copy ( self -> val_buffer, self -> value . size + 1, buffer, size );
self -> value . len = string_len ( self -> val_buffer, size );
-
+
KConfigNodeSetDirty ( self );
-
+
rc = 0;
}
@@ -1160,7 +1128,7 @@ LIB_EXPORT rc_t CC KConfigNodeAppend ( KConfigNode *self, const char *buffer, si
string_copy ( & self -> val_buffer [ self -> value . size ], self -> value . size + size + 1, buffer, size );
self -> value . size += size;
self -> value . len = string_len ( self -> val_buffer, self -> value . size );
-
+
KConfigNodeSetDirty ( self );
rc = 0;
@@ -1212,7 +1180,7 @@ LIB_EXPORT rc_t CC KConfigNodeDropAll ( KConfigNode *self )
{
if ( self == NULL )
return RC ( rcKFG, rcNode, rcClearing, rcSelf, rcNull );
- BSTreeWhack ( & self->children, KConfigNodeWhack, self->mgr);
+ BSTreeWhack ( & self->children, KConfigNodeWhack, self->mgr);
return 0;
}
@@ -1270,8 +1238,8 @@ update_node ( KConfig* self, const char* key, const char* value,
rc_t rc = KConfigOpenNodeUpdate ( self, &node, "%s", key);
if (rc == 0)
{
-/* pLogMsg(klogInfo, "updating config key $(KEY) with '$(VALUE)'",
- "KEY=%s,VALUE=%s",
+/* pLogMsg(klogInfo, "updating config key $(KEY) with '$(VALUE)'",
+ "KEY=%s,VALUE=%s",
key, value);*/
rc = KConfigNodeWrite (node, value, string_size(value));
node -> internal = internal;
@@ -1334,7 +1302,7 @@ rc_t write_nvp(void* pself, const char* name, size_t nameLen, VNamelist* values)
{ /* create the node */
String* nameStr;
-
+
/* some old config files may have "dbGaP" in their repository keys misspelled as "dbGap" - fix if seen */
const char* oldGaPprefix = "/repository/user/protected/dbGap-";
size_t size = sizeof("/repository/user/protected/dbGap-") - 1;
@@ -1345,7 +1313,7 @@ rc_t write_nvp(void* pself, const char* name, size_t nameLen, VNamelist* values)
StringCopy((const String**)&nameStr, &tmp);
if (needsFix)
((char*)(nameStr->addr)) [ size - 2 ] = 'P';
-
+
rc = update_node(self, nameStr->addr, buf, false);
if (needsFix)
{
@@ -1360,7 +1328,7 @@ rc_t write_nvp(void* pself, const char* name, size_t nameLen, VNamelist* values)
}
StringWhack(nameStr);
}
-
+
free(buf);
return rc;
}
@@ -1372,7 +1340,7 @@ bool look_up_var(void * self, struct KFGToken* pb)
rc_t rc = KConfigOpenNodeRead((KConfig*)self, &node, "%.*s", pb->tokenLength-3, pb->tokenText+2);
if (rc == 0)
{
- pb->tokenText = node->value.addr;
+ pb->tokenText = node->value.addr;
pb->tokenLength = node->value.len;
pb->tokenId = kfgVAR_REF;
}
@@ -1383,13 +1351,13 @@ bool look_up_var(void * self, struct KFGToken* pb)
static
void CC report_error(KFGScanBlock* sb, const char* msg)
{
- pLogMsg(klogErr, "$(file):$(line):$(column): error: token='$(token)', msg='$(msg)'",
- "file=%s,line=%d,column=%d,token=%.*s,msg=%s",
- sb->file,
- sb->lastToken->line_no,
- sb->lastToken->column_no,
- sb->lastToken->tokenLength,
- sb->lastToken->tokenText,
+ pLogMsg(klogErr, "$(file):$(line):$(column): error: token='$(token)', msg='$(msg)'",
+ "file=%s,line=%d,column=%d,token=%.*s,msg=%s",
+ sb->file,
+ sb->lastToken->line_no,
+ sb->lastToken->column_no,
+ sb->lastToken->tokenLength,
+ sb->lastToken->tokenText,
msg);
}
@@ -1406,16 +1374,16 @@ typedef struct
/* total bytes in buffer */
size_t buffered;
-
+
rc_t rc;
-
+
/* buffer */
char buffer [ 32 * 1024 ];
-
+
} PrintBuff;
/* Init
- * initialize your structure
+ * initialize your structure
*/
static
void PrintBuffInit ( PrintBuff *pb, KFile *f )
@@ -1448,7 +1416,7 @@ static rc_t PrintBuffFlush ( PrintBuff *self )
rc = KFileWriteAll ( self -> f, self -> flushed,
self -> buffer, self -> buffered, & num_writ );
}
-
+
if ( rc == 0 )
{
if ( num_writ != self -> buffered )
@@ -1482,7 +1450,7 @@ rc_t PrintBuffPrint ( PrintBuff *self, const char *fmt, ... )
va_start ( args1, fmt );
va_copy ( args2, args1 );
- rc = string_vprintf ( & self -> buffer [ self -> buffered ],
+ rc = string_vprintf ( & self -> buffer [ self -> buffered ],
sizeof self -> buffer - self -> buffered, & num_writ, fmt, args1 );
if ( rc == 0 )
self -> buffered += num_writ;
@@ -1589,9 +1557,9 @@ static rc_t _printNodeData(const char *name, const char *data, uint32_t dlen,
size_t l3 = sizeof d3 - 1;
if ((string_cmp(name,
- string_measure(name, NULL), d1, l1, (uint32_t)l1) == 0) ||
+ string_measure(name, NULL), d1, l1, (uint32_t)l1) == 0) ||
(string_cmp(name,
- string_measure(name, NULL), d2, l2, (uint32_t)l2) == 0) ||
+ string_measure(name, NULL), d2, l2, (uint32_t)l2) == 0) ||
(string_cmp(name,
string_measure(name, NULL), d3, l3, (uint32_t)l3) == 0))
{
@@ -1761,7 +1729,7 @@ static rc_t KConfigNodePrint(const KConfigNode *self, int indent,
}
static rc_t CC KConfigPrintImpl(const KConfig* self,
- int indent, const char *root, bool debug, bool native,
+ int indent, const char *root, bool debug, bool native,
PrintBuff *pb, uint32_t skipCount, va_list args)
{
rc_t rc = 0;
@@ -1834,10 +1802,10 @@ rc_t parse_file ( KConfig * self, const char* path, const char * src )
{ /* out of memory is the only reason we can get false here */
return RC ( rcKFG, rcMgr, rcParsing, rcMemory, rcExhausted );
}
-
+
KFG_parse(&pb, &sb); /* may have reported parse errors into log, but we should have been able to extract enough data to proceed regardless */
KFGScan_yylex_destroy(&sb);
-
+
return 0;
}
@@ -2164,7 +2132,7 @@ bool CC WriteDirtyNode ( BSTNode *n, void *data )
PrintBuff *pb = data;
if ( self -> dirty
- || ( self -> came_from != NULL && self -> came_from -> is_magic_file
+ || ( self -> came_from != NULL && self -> came_from -> is_magic_file
&& ! self -> internal ) )
{
if ( KConfigNodePrintPath ( self, pb ) )
@@ -2172,10 +2140,10 @@ bool CC WriteDirtyNode ( BSTNode *n, void *data )
if ( PrintBuffPrint ( pb, " = " ) != 0 )
return true;
-
+
if ( PrintBuffPrintQuoted ( pb, &self->value ) != 0 )
return true;
-
+
if ( PrintBuffPrint ( pb, "\n" ) != 0 )
return true;
}
@@ -2209,7 +2177,7 @@ rc_t path_to_magic_file ( const KConfig *self, char *path, size_t buffer_size, s
rc = RC ( rcKFG, rcNode, rcReading, rcBuffer, rcInsufficient );
path[num_read] = '\0';
-
+
KConfigNodeRelease ( node );
}
return rc;
@@ -2247,8 +2215,8 @@ LIB_EXPORT rc_t CC KConfigCommit ( KConfig *self )
rc = string_printf ( tmp_file_path, sizeof tmp_file_path, NULL, "%s.tmp", magic_file_path );
if ( rc == 0 )
{
- KDirectory *dir;
-
+ KDirectory *dir;
+
/* get current directory */
rc = KDirectoryNativeDir ( & dir );
if ( rc == 0 )
@@ -2271,12 +2239,12 @@ LIB_EXPORT rc_t CC KConfigCommit ( KConfig *self )
if ( BSTreeDoUntil ( & self -> tree, false, WriteDirtyNode, & pb ) )
/* failure */
rc = pb . rc;
- else
+ else
rc = PrintBuffFlush ( & pb );
}
-
+
KFileRelease ( tmp );
-
+
if ( rc == 0 )
{
/* Rename file */
@@ -2285,15 +2253,15 @@ LIB_EXPORT rc_t CC KConfigCommit ( KConfig *self )
{
/* clear dirty flags */
BSTreeForEach ( & self -> tree, false, UnsetDirtyNode, & pb);
-
+
self -> dirty = false;
}
}
-
+
if ( rc != 0 )
KDirectoryRemove ( dir, true, "%s", tmp_file_path );
}
-
+
/* release dir */
KDirectoryRelease ( dir );
}
@@ -2361,7 +2329,7 @@ bool load_from_file_path ( KConfig *self, const KDirectory *dir, const char *pat
/* record magic file path, regardless of whether it exists as a file */
if ( is_magic )
record_magic_path ( self, dir, path, sz );
-
+
DBGMSG( DBG_KFG, DBG_FLAG(DBG_KFG), ( "KFG: try to load from file '%.*s'\n", (int)sz, path ) );
rc = KDirectoryOpenFileRead ( dir, & cfg_file, "%.*s", ( int ) sz, path );
if ( rc == 0 )
@@ -2492,7 +2460,7 @@ bool load_from_env_variable ( KConfig *self, const KDirectory *dir )
"VDB_CONFIG",
"VDBCONFIG"
};
-
+
int i;
bool loaded = false;
for ( i = 0; ! loaded && i < sizeof env_list / sizeof env_list [ 0 ]; ++ i )
@@ -2520,7 +2488,7 @@ rc_t load_env_variable_rc ( KConfig *self, const KDirectory *dir, bool *loaded )
{
"NCBI_VDB_CONFIG"
};
-
+
int i;
rc_t rc;
@@ -2532,12 +2500,12 @@ rc_t load_env_variable_rc ( KConfig *self, const KDirectory *dir, bool *loaded )
DBGMSG( DBG_KFG, DBG_FLAG(DBG_KFG), ( "KFG: try to load from env. var '%s'\n", env_list[ i ] ) );
if ( eval == NULL || eval [ 0 ] == 0 )
return 0;
-
+
DBGMSG( DBG_KFG, DBG_FLAG(DBG_KFG), ( "KFG: try to load from env. var '%s'\n", eval ) );
rc = KConfigAppendToLoadPath(self, eval);
if ( rc != 0 )
return rc;
-
+
* loaded = load_from_path_list ( self, dir, eval );
if ( * loaded )
DBGMSG( DBG_KFG, DBG_FLAG(DBG_KFG), ( "KFG: found from env. var '%s'\n", eval ) );
@@ -2793,7 +2761,7 @@ void add_predefined_nodes ( KConfig * self, const char * appname )
}
update_node ( self, "vdb/lib/paths/kfg", rc == 0 ? buf : "", true );
- /* Architecture */
+ /* Architecture */
#if ! WINDOWS
if (uname(&name) >= 0)
update_node(self, "kfg/arch/name", name.nodename, true);
@@ -2807,7 +2775,7 @@ void add_predefined_nodes ( KConfig * self, const char * appname )
/* *OS */
#if LINUX
#define OS "linux"
-#elif MAC
+#elif MAC
#define OS "mac"
#elif WINDOWS
#define OS "win"
@@ -2835,7 +2803,7 @@ void add_predefined_nodes ( KConfig * self, const char * appname )
#define BUILD "PROFILE"
#elif _DEBUGGING
#define BUILD "DEBUG"
-#else
+#else
#define BUILD "RELEASE"
#endif
update_node(self, "BUILD", BUILD, true );
@@ -2860,7 +2828,7 @@ void add_predefined_nodes ( KConfig * self, const char * appname )
bytes = appname + bytes - value;
rc = KDirectoryResolvePath ( cwd, true, buf, sizeof buf, "%s", appname);
if ( rc == 0 && bytes != 0 )
- buf [ string_size(buf) - bytes ] = 0;
+ buf [ string_size(buf) - bytes ] = 0;
update_node(self, "APPPATH", rc == 0 ? buf : "", true );
}
@@ -3113,7 +3081,8 @@ extern rc_t ReportKfg ( const ReportFuncs *f, uint32_t indent,
uint32_t configNodesSkipCount, va_list args );
/* "cfg" [ OUT ] - return parameter for mgr
- if ( "local" == true or cfgdir != NULL ) do not initialize G_kfg */
+ if ( "local" == true ) do not initialize G_kfg
+ if ( cfgdir != NULL ) then initialize G_kfg. It is used in unit tests */
static
rc_t KConfigMakeImpl(KConfig ** cfg, const KDirectory * cfgdir, bool local)
{
@@ -3134,7 +3103,8 @@ rc_t KConfigMakeImpl(KConfig ** cfg, const KDirectory * cfgdir, bool local)
KConfig * mgr = NULL;
if ( cfgdir != NULL ) {
- local = true;
+ /* local = true;
+ ALWAYS create and/or return a singleton object. */
}
if ( ! local ) {
@@ -3191,8 +3161,7 @@ rc_t KConfigMakeImpl(KConfig ** cfg, const KDirectory * cfgdir, bool local)
LIB_EXPORT
rc_t CC KConfigMakeLocal(KConfig **cfg, const KDirectory *cfgdir)
{
- return KConfigMake(cfg, cfgdir);
-/* return KConfigMakeImpl(cfg, cfgdir, true); */
+ return KConfigMakeImpl(cfg, cfgdir, true);
}
/* Make
@@ -3215,7 +3184,7 @@ struct KfgConfigNamelist
size_t count;
const char *namelist [ 1 ];
};
-
+
/* Whack
*/
static
@@ -3224,7 +3193,7 @@ rc_t CC KfgConfigNamelistWhack ( KfgConfigNamelist *self )
free ( self );
return 0;
}
-
+
/* Count
*/
static
@@ -3234,7 +3203,7 @@ uint32_t *count )
* count = ( uint32_t ) self -> count;
return 0;
}
-
+
/* Get
*/
static
@@ -3246,7 +3215,7 @@ rc_t CC KfgConfigNamelistGet ( const KfgConfigNamelist *self,
* name = self -> namelist [ idx ];
return 0;
}
-
+
/* Make
*/
static KNamelist_vt_v1 vtKfgConfigNamelist =
@@ -3260,7 +3229,7 @@ static KNamelist_vt_v1 vtKfgConfigNamelist =
KfgConfigNamelistGet
/* end minor version 0 methods */
};
-
+
static
rc_t KfgConfigNamelistMake ( KNamelist **names, uint32_t count )
{
@@ -3272,7 +3241,7 @@ static KNamelist_vt_v1 vtKfgConfigNamelist =
else
{
self -> count = 0;
-
+
rc = KNamelistInit ( & self -> dad,
( const KNamelist_vt* ) & vtKfgConfigNamelist );
if ( rc == 0 )
@@ -3280,13 +3249,13 @@ static KNamelist_vt_v1 vtKfgConfigNamelist =
* names = & self -> dad;
return 0;
}
-
+
free ( self );
}
-
+
return rc;
}
-
+
/* List
* create metadata node listings
*/
@@ -3385,12 +3354,12 @@ static rc_t ReadNodeValueFixed ( const KConfigNode* self, char *buf, size_t bsiz
rc = RC ( rcKFG, rcNode, rcReading, rcBuffer, rcInsufficient );
else
buf[num_read] = '\0';
- }
+ }
return rc;
}
/**********************************************************/
-
+
/* ReadBool
* read a boolean node value
*
@@ -3481,7 +3450,7 @@ LIB_EXPORT rc_t CC KConfigNodeReadI64 ( const KConfigNode *self, int64_t *result
{
/* allow for leading zeros */
char buf [ 256 ];
-
+
rc = ReadNodeValueFixed(self, buf, sizeof(buf));
if (rc == 0)
{
@@ -3520,7 +3489,7 @@ LIB_EXPORT rc_t CC KConfigNodeReadU64 ( const KConfigNode *self, uint64_t* resul
{
/* allow for leading zeros */
char buf [ 256 ];
-
+
rc = ReadNodeValueFixed(self, buf, sizeof(buf));
if (rc == 0)
{
@@ -3598,7 +3567,7 @@ static rc_t ReadNodeValueVariable( const KConfigNode* self, char** buf )
}
else
rc = RC( rcKFG, rcNode, rcReading, rcMemory, rcExhausted );
- }
+ }
return rc;
}
@@ -3687,7 +3656,7 @@ LIB_EXPORT rc_t CC KConfigNodeReadString ( const KConfigNode *self, String** res
}
}
}
- }
+ }
return rc;
}
@@ -3711,7 +3680,7 @@ LIB_EXPORT rc_t CC KConfigRead ( const KConfig * self, const char * path,
}
return rc;
-}
+}
/* this macro wraps a call to KConfigNodeGetXXX in a node-accessing
code to implement the corresponding KConfigGetXXX function */
@@ -3726,14 +3695,14 @@ LIB_EXPORT rc_t CC KConfigRead ( const KConfig * self, const char * path,
if (rc == 0) \
rc = rc2; \
} \
- return rc;
-
+ return rc;
+
/* THESE FUNCTIONS ARE PROTECTED AGAINST BAD "self" AND "path"
BY KConfigOpenNodeRead, BUT THE CONVERSION FUNCTIONS ARE NOT */
LIB_EXPORT rc_t CC KConfigReadBool ( const KConfig* self, const char* path, bool* result )
{
NODE_TO_CONFIG_ACCESSOR(KConfigNodeReadBool);
-}
+}
LIB_EXPORT rc_t CC KConfigReadI64 ( const KConfig* self, const char* path, int64_t* result )
{
NODE_TO_CONFIG_ACCESSOR(KConfigNodeReadI64);
@@ -3883,7 +3852,7 @@ aprintf("decode_ncbi_gap %d\n", __LINE__);
break;
total += num_read;
-
+
if ( num_read < to_read )
{
aprintf("decode_ncbi_gap %d\n", __LINE__);
@@ -3991,7 +3960,42 @@ rc_t _KConfigNncToKGapConfig(const KConfig *self, char *text, KGapConfig *kgc)
return 0;
}
-static rc_t _KConfigFixResolverCgiNode(KConfig *self) {
+LIB_EXPORT rc_t KConfigFixMainResolverCgiNode ( KConfig * self ) {
+ rc_t rc = 0;
+
+ KConfigNode *node = NULL;
+ struct String *result = NULL;
+
+ assert(self);
+
+ if (rc == 0) {
+ rc = KConfigOpenNodeUpdate(self, &node,
+ "/repository/remote/main/CGI/resolver-cgi");
+ }
+
+ if (rc == 0) {
+ rc = KConfigNodeReadString(node, &result);
+ }
+
+ if (rc == 0) {
+ String http;
+ CONST_STRING ( & http,
+ "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" );
+ assert(result);
+ if ( result->size == 0 || StringEqual ( & http, result ) ) {
+ const char https []
+ = "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi";
+ rc = KConfigNodeWrite ( node, https, sizeof https );
+ }
+ }
+
+ free(result);
+
+ KConfigNodeRelease(node);
+
+ return rc;
+}
+LIB_EXPORT rc_t KConfigFixProtectedResolverCgiNode ( KConfig * self ) {
rc_t rc = 0;
KConfigNode *node = NULL;
@@ -4009,11 +4013,14 @@ static rc_t _KConfigFixResolverCgiNode(KConfig *self) {
}
if (rc == 0) {
+ String http;
+ CONST_STRING ( & http,
+ "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" );
assert(result);
- if (result->size == 0) {
- const char buffer[]
- = "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi";
- rc = KConfigNodeWrite(node, buffer, sizeof buffer);
+ if ( result->size == 0 || StringEqual ( & http, result ) ) {
+ const char https []
+ = "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi";
+ rc = KConfigNodeWrite ( node, https, sizeof https );
}
}
@@ -4305,7 +4312,7 @@ aprintf("KConfigImportNgc %d\n", __LINE__);*/
if (rc == 0) {
/*DBGMSG(DBG_KFG, DBG_FLAG(DBG_KFG), ("KConfigImportNgc %d\n", __LINE__));
aprintf("KConfigImportNgc %d\n", __LINE__);*/
- rc = _KConfigFixResolverCgiNode(self);
+ rc = KConfigFixProtectedResolverCgiNode(self);
/*DBGMSG(DBG_KFG, DBG_FLAG(DBG_KFG), ("KConfigImportNgc %d\n", __LINE__));
aprintf("KConfigImportNgc %d\n", __LINE__);*/
}
diff --git a/libs/kfg/default.kfg b/libs/kfg/default.kfg
index c22b53b..516c169 100644
--- a/libs/kfg/default.kfg
+++ b/libs/kfg/default.kfg
@@ -38,16 +38,10 @@
/repository/user/main/public/root = "$(HOME)/ncbi/public"
# Remote access to NCBI's public repository
-/repository/remote/main/CGI/resolver-cgi = "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
-/repository/remote/aux/NCBI/apps/nakmer/volumes/fuseNAKMER = "sadb"
-/repository/remote/aux/NCBI/apps/nannot/volumes/fuseNANNOT = "sadb"
-/repository/remote/aux/NCBI/apps/refseq/volumes/refseq = "refseq"
-/repository/remote/aux/NCBI/apps/sra/volumes/flat = "srapub"
-/repository/remote/aux/NCBI/apps/wgs/volumes/fuseWGS = "wgs"
-/repository/remote/aux/NCBI/root = "http://sra-download.ncbi.nlm.nih.gov"
+/repository/remote/main/CGI/resolver-cgi = "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
# Remote access to NCBI's protected repository
-/repository/remote/protected/CGI/resolver-cgi = "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
+/repository/remote/protected/CGI/resolver-cgi = "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
# Aspera configuration
/tools/ascp/max_rate = "1000m"
diff --git a/libs/kfg/properties.c b/libs/kfg/properties.c
index 5c19f60..57860bf 100644
--- a/libs/kfg/properties.c
+++ b/libs/kfg/properties.c
@@ -220,6 +220,53 @@ LIB_EXPORT rc_t CC KConfig_Set_Http_Proxy_Enabled
}
/* -------------------------------------------------------------------------- */
+/* get/set priority of environmnet vs. configuration for HTTP proxy */
+LIB_EXPORT rc_t CC KConfig_Has_Http_Proxy_Env_Higher_Priority
+ ( const KConfig *self, bool *enabled )
+{
+ rc_t rc = 0;
+
+ if ( self == NULL ) {
+ rc = RC ( rcKFG, rcNode, rcReading, rcSelf, rcNull );
+ }
+ else if ( enabled == NULL ) {
+ rc = RC ( rcKFG, rcNode, rcReading, rcParam, rcNull );
+ }
+ else {
+ String * res = NULL;
+ * enabled = false;
+ rc = KConfigReadString ( self, "/http/proxy/use", &res );
+ if ( rc == 0 ) {
+ String v;
+ CONST_STRING ( & v, "env,kfg" );
+ if ( StringEqual ( res, & v ) ) {
+ * enabled = true;
+ }
+ } else {
+ rc = 0;
+ }
+ free ( res );
+ }
+
+ return rc;
+}
+LIB_EXPORT rc_t CC KConfig_Set_Http_Proxy_Env_Higher_Priority
+ ( KConfig *self, bool enabled )
+{
+ rc_t rc = 0;
+
+ if ( self == NULL ) {
+ rc = RC ( rcKFG, rcNode, rcWriting, rcSelf, rcNull );
+ }
+ else {
+ rc = KConfigWriteString
+ ( self, "/http/proxy/use", enabled ? "env,kfg" : "kfg,env" );
+ }
+
+ return rc;
+}
+
+/* -------------------------------------------------------------------------- */
LIB_EXPORT rc_t CC KConfig_Get_Home( const KConfig *self, char * buffer, size_t buffer_size, size_t * written )
{ return KConfig_Get_Repository_String( self, buffer, buffer_size, written, "HOME" ); }
diff --git a/libs/kfg/repository.c b/libs/kfg/repository.c
index 3f7ade6..8f54404 100644
--- a/libs/kfg/repository.c
+++ b/libs/kfg/repository.c
@@ -518,9 +518,19 @@ LIB_EXPORT rc_t CC KRepositoryAppendToRootHistory( KRepository *self, const char
root[ curr_root_size ] = 0;
rc = append_to_root_history( self, root );
}
+ else
+ {
+ /* we cannot determine the current root, because it is not stored */
+ rc = 0;
+ }
free( root );
}
}
+ else
+ {
+ /* we cannot determine the current root, because it is not stored */
+ rc = 0;
+ }
}
else
rc = append_to_root_history( self, item );
diff --git a/libs/kfs/tar.c b/libs/kfs/tar.c
index d7f058f..e71a351 100644
--- a/libs/kfs/tar.c
+++ b/libs/kfs/tar.c
@@ -849,7 +849,7 @@ static int64_t decode_base64string (const uint8_t* str, size_t len)
* byte is either 0x80 for a positive number or 0xFF for a negative number.
*/
-static int64_t tar_strtoll (const uint8_t * str, size_t len)
+static int64_t tar_strtoll ( const uint8_t * str, size_t len, bool silent )
{
int64_t result = 0;
bool negative = false;
@@ -927,16 +927,17 @@ static int64_t tar_strtoll (const uint8_t * str, size_t len)
TAR_DEBUG (("%s: unknown integer storage type %c%c%c%c%c%c%c%c\n",
temp_buff[0],temp_buff[1],temp_buff[2],temp_buff[3],
temp_buff[4],temp_buff[5],temp_buff[6],temp_buff[7]));
- PLOGMSG (klogErr, (klogErr, "unknown integer storage type $(B0)$(B1)$(B2)$(B3)$(B4)$(B5)$(B6)$(B7)",
+ if ( ! silent ) {
+ PLOGMSG (klogErr, (klogErr, "unknown integer storage type "
+ "$(B0)$(B1)$(B2)$(B3)$(B4)$(B5)$(B6)$(B7)",
"B0=%c,B1=%c,B2=%c,B3=%c,B4=%c,B5=%c,B6=%c,B7=%c",
temp_buff[0],temp_buff[1],temp_buff[2],temp_buff[3],
temp_buff[4],temp_buff[5],temp_buff[6],temp_buff[7]));
+ }
}
return result;
}
-
-
/* ======================================================================
* tar_header_type
* determine most probable tar header block type
@@ -1789,10 +1790,12 @@ uint64_t process_one_entry (KTarState * self, uint64_t offset, uint64_t hard_lim
for (ix = 0; ix< GNU_SPARSES_IN_EXTRA_HEADER; ++ix)
{
rc_t ret;
- of = tar_strtoll((const uint8_t*)current_header.h->sparse.sparse[ix].offset,
- TAR_SIZE_LEN);
- sz = tar_strtoll((const uint8_t*)current_header.h->sparse.sparse[ix].num_bytes,
- TAR_SIZE_LEN);
+ of = tar_strtoll(
+ (const uint8_t*)current_header.h->sparse.sparse[ix].offset,
+ TAR_SIZE_LEN, silent);
+ sz = tar_strtoll(
+ (const uint8_t*)current_header.h->sparse.sparse[ix].num_bytes,
+ TAR_SIZE_LEN, silent);
if (sz == 0)
{
break;
@@ -1848,13 +1851,18 @@ uint64_t process_one_entry (KTarState * self, uint64_t offset, uint64_t hard_lim
*
* this will be wrong if we ever support cpio...
*/
- data_size = (uint64_t)(tar_strtoll((uint8_t*)current_header.h->tar.size,TAR_SIZE_LEN));
+ data_size = (uint64_t) ( tar_strtoll
+ ( (uint8_t*)current_header.h->tar.size,TAR_SIZE_LEN, silent ) );
#if _DEBUGGING && 0
- uid = (tar_strtoll((uint8_t*)current_header.h->tar.uid,TAR_ID_LEN));
- gid = (tar_strtoll((uint8_t*)current_header.h->tar.gid,TAR_ID_LEN));
+ uid = ( tar_strtoll
+ ( (uint8_t*)current_header.h->tar.uid,TAR_ID_LEN, silent ) );
+ gid = ( tar_strtoll
+ ( (uint8_t*)current_header.h->tar.gid,TAR_ID_LEN, silent ) );
#endif
- mtime = (tar_strtoll((uint8_t*)current_header.h->tar.mtime,TAR_TIME_LEN));
- mode = (uint32_t)(tar_strtoll((uint8_t*)current_header.h->tar.mode,TAR_MODE_LEN));
+ mtime = ( tar_strtoll
+ ( (uint8_t*)current_header.h->tar.mtime,TAR_TIME_LEN, silent) );
+ mode = (uint32_t) ( tar_strtoll
+ ( (uint8_t*)current_header.h->tar.mode,TAR_MODE_LEN, silent) );
}
/* -----
@@ -1946,14 +1954,18 @@ uint64_t process_one_entry (KTarState * self, uint64_t offset, uint64_t hard_lim
int32_t ix;
rc_t ret;
- virtual_data_size = (uint64_t)(tar_strtoll((uint8_t*)current_header.h->gnu_89.realsize,TAR_SIZE_LEN));
+ virtual_data_size = (uint64_t)(tar_strtoll(
+ (uint8_t*)current_header.h->gnu_89.realsize,TAR_SIZE_LEN,
+ silent));
for (ix = 0; ix< GNU_SPARSES_IN_OLD_HEADER; ++ix)
{
- uint64_t soffset = tar_strtoll((const uint8_t*)current_header.h->gnu_89.sparse[ix].offset,
- TAR_SIZE_LEN);
- uint64_t ssize = tar_strtoll((const uint8_t*)current_header.h->gnu_89.sparse[ix].num_bytes,
- TAR_SIZE_LEN);
+ uint64_t soffset = tar_strtoll(
+ (const uint8_t*)current_header.h->gnu_89.sparse[ix].offset,
+ TAR_SIZE_LEN, silent);
+ uint64_t ssize = tar_strtoll(
+ (const uint8_t*)current_header.h->gnu_89.sparse[ix].num_bytes,
+ TAR_SIZE_LEN, silent);
if (ssize == 0)
{
break;
diff --git a/libs/klib/rc-idx-tbl.c b/libs/klib/rc-idx-tbl.c
index a851807..1c65462 100644
--- a/libs/klib/rc-idx-tbl.c
+++ b/libs/klib/rc-idx-tbl.c
@@ -30,6 +30,8 @@
#define RC_ENTRY( id, txt ) # id ,
+#define RC_VFIRST( id1, id2 )
+
#define RC_VLAST( id )
#define RC_LAST( id ) NULL
diff --git a/libs/klib/rc-tbl.c b/libs/klib/rc-tbl.c
index e09543f..a8cdef2 100644
--- a/libs/klib/rc-tbl.c
+++ b/libs/klib/rc-tbl.c
@@ -29,6 +29,8 @@
#define RC_ENTRY( id, txt ) txt,
+#define RC_VFIRST( id1, id2 )
+
#define RC_VLAST( id )
#define RC_LAST( id ) NULL
diff --git a/libs/klib/release-vers.h b/libs/klib/release-vers.h
index 052a099..57744be 100644
--- a/libs/klib/release-vers.h
+++ b/libs/klib/release-vers.h
@@ -27,7 +27,7 @@
/* THIS FILE IS NOT GENERATED AUTOMATICALLY! */
/* The Version of current SRA Toolkit Release */
-#define RELEASE_VERS 0x02070000
+#define RELEASE_VERS 0x02080000
/* The Revision of Version of current SRA Toolkit Release */
#define RELEASE_REVISION 0
diff --git a/libs/klib/report-klib.c b/libs/klib/report-klib.c
index af90bd2..1dbacc6 100644
--- a/libs/klib/report-klib.c
+++ b/libs/klib/report-klib.c
@@ -512,11 +512,15 @@ static rc_t ReportEnv(int indent) {
const char tag[] = "Env";
- const char *env_list[] = {
+ const char * env_list [] = {
+ "all_proxy",
+ "ALL_PROXY",
+ "http_proxy",
+ "HTTP_PROXY",
"KLIB_CONFIG",
"VDBCONFIG",
"VDB_CONFIG",
- ENV_KRYPTO_PWFILE
+ ENV_KRYPTO_PWFILE,
};
int i = 0;
diff --git a/libs/klib/status-rc-strings.c b/libs/klib/status-rc-strings.c
index f890b0d..b99962f 100644
--- a/libs/klib/status-rc-strings.c
+++ b/libs/klib/status-rc-strings.c
@@ -30,6 +30,7 @@
#define STRINGIFY(a) #a
#define RC_ENUM(type) const char * PASTE2(type,Strings)[] =
#define RC_ENTRY(a,b) STRINGIFY(a),
+#define RC_VFIRST(a,b)
#define RC_VLAST(a)
#define RC_LAST(a) "INVALID"
diff --git a/libs/klib/status.c b/libs/klib/status.c
index e54b6b9..d068064 100644
--- a/libs/klib/status.c
+++ b/libs/klib/status.c
@@ -110,13 +110,16 @@ rc_t CC KStsDefaultFormatter( void* self, KWrtHandler* writer,
LIB_EXPORT rc_t CC KStsInit ( void )
{
rc_t rc;
-
+
G_sts_level = 0;
rc = KStsHandlerSetStdOut();
+ if ( rc == 0 )
+ rc = KStsLibHandlerSetStdOut ();
+#if 0
if (rc == 0)
rc = KStsLibHandlerSet(NULL, NULL);
-
+#endif
if (rc == 0)
rc = KStsFmtHandlerSetDefault();
diff --git a/libs/klib/text.c b/libs/klib/text.c
index 884146d..bab3f0a 100644
--- a/libs/klib/text.c
+++ b/libs/klib/text.c
@@ -142,6 +142,41 @@ LIB_EXPORT String * CC StringSubstr ( const String *str, String *sub,
return NULL;
}
+/* StringTrim
+ * trims ascii white-space from both ends
+ * returns trimmed string in "trimmed"
+ */
+LIB_EXPORT String * CC StringTrim ( const String * str, String * trimmed )
+{
+ if ( trimmed != NULL )
+ {
+ if ( str == NULL )
+ CONST_STRING ( trimmed, "" );
+ else
+ {
+ const char * addr = str -> addr;
+ size_t i, end, sz = str -> size;
+ uint32_t len = str -> len;
+
+ for ( end = sz; end > 0; -- end )
+ {
+ if ( ! isspace ( addr [ end - 1 ] ) )
+ break;
+ }
+
+ for ( i = 0; i < end; ++ i )
+ {
+ if ( ! isspace ( addr [ i ] ) )
+ break;
+ }
+
+ StringInit ( trimmed, & addr [ i ], end - i, len - ( i + sz - end ) );
+ }
+ }
+
+ return trimmed;
+}
+
/* StringHead
* access the first character
*
diff --git a/libs/klib/writer.c b/libs/klib/writer.c
index b3a3ef6..2223503 100644
--- a/libs/klib/writer.c
+++ b/libs/klib/writer.c
@@ -123,7 +123,8 @@ LIB_EXPORT rc_t CC KWrtInit( const char* appname, uint32_t vers )
memcpy(wrt_app, appname, wrt_app_length);
wrt_app[wrt_app_length] = '\0';
- rc = string_printf ( wrt_vers, sizeof wrt_vers, & wrt_vers_length, "%V", vers );
+ rc = string_printf ( wrt_vers, sizeof wrt_vers, & wrt_vers_length,
+ "%.3V", vers );
assert ( rc == 0 );
rc = KWrtSysInit(&KWrt_DefaultWriterDataStdOut, &KWrt_DefaultWriterDataStdErr);
diff --git a/libs/kns/Makefile b/libs/kns/Makefile
index c1a442c..e6f3067 100644
--- a/libs/kns/Makefile
+++ b/libs/kns/Makefile
@@ -85,7 +85,8 @@ KNS_NO_HTTP_SRC = \
sysendpoint \
syssock \
sysstream \
- sysmgr
+ sysmgr \
+ tls
ifneq (win,$(OS))
KNS_NO_HTTP_SRC += \
diff --git a/libs/kns/http-client.c b/libs/kns/http-client.c
index 96c8a01..90900af 100644
--- a/libs/kns/http-client.c
+++ b/libs/kns/http-client.c
@@ -34,6 +34,7 @@ typedef struct KClientHttpStream KClientHttpStream;
#include <kns/endpoint.h>
#include <kns/socket.h>
#include <kns/stream.h>
+#include <kns/tls.h>
#include <kns/impl.h>
#include <vfs/path.h>
#include <kfs/file.h>
@@ -43,11 +44,12 @@ typedef struct KClientHttpStream KClientHttpStream;
#undef ERR
#endif
-#include <klib/debug.h> /* DBGMSG */
+#include <klib/debug.h>
#include <klib/text.h>
#include <klib/container.h>
#include <klib/out.h>
#include <klib/log.h>
+#include <klib/status.h>
#include <klib/refcount.h>
#include <klib/rc.h>
#include <klib/printf.h>
@@ -69,10 +71,14 @@ typedef struct KClientHttpStream KClientHttpStream;
#include "http-priv.h"
+
#if _DEBUGGING && 0
#include <stdio.h>
#define TRACE( x, ... ) \
fprintf ( stderr, "@@ %s:%d: %s: " x, __FILE__, __LINE__, __func__, __VA_ARGS__ )
+#elif _DEBUGGING && defined _h_klib_status_ && 0
+#define TRACE( x, ... ) \
+ STATUS ( 0, "@@ %s:%d: %s: " x, __FILE__, __LINE__, __func__, __VA_ARGS__ )
#else
#define TRACE( x, ... ) \
( ( void ) 0 )
@@ -130,6 +136,7 @@ struct KClientHttp
bool proxy_default_port;
bool reliable;
+ bool tls;
};
@@ -163,6 +170,7 @@ rc_t KClientHttpClear ( KClientHttp *self )
KClientHttpClose ( self );
KDataBufferWhack ( & self -> hostname_buffer );
+ self -> tls = false;
return 0;
}
@@ -181,81 +189,307 @@ rc_t KClientHttpWhack ( KClientHttp * self )
return 0;
}
+
+typedef struct KEndPointArgsIterator KEndPointArgsIterator;
+struct KEndPointArgsIterator
+{
+ const HttpProxy * proxy;
+ const String * hostname;
+ uint16_t port;
+ uint16_t dflt_proxy_ports [ 2 ];
+ size_t dflt_proxy_ports_idx;
+ bool done;
+};
+
static
-rc_t KClientHttpInitDNSEndpoint ( KClientHttp * self, const String * hostname, uint32_t port, uint16_t dflt_proxy_port )
+void KEndPointArgsIteratorMake ( KEndPointArgsIterator * self,
+ const KNSManager * mgr, const String * hostname, uint16_t port )
{
- rc_t rc = 0;
+ assert ( self );
+ memset ( self, 0, sizeof * self );
+
+ self -> dflt_proxy_ports [ 0 ] = 3128;
+ self -> dflt_proxy_ports [ 1 ] = 8080;
+
+ if ( ! mgr -> http_proxy_only )
+ {
+ self -> hostname = hostname;
+ self -> port = port;
+ }
- const KNSManager * mgr = self -> mgr;
+ if ( mgr -> http_proxy_enabled )
+ self -> proxy = KNSManagerGetHttpProxy ( mgr );
- self -> proxy_default_port = false;
+ if ( self -> hostname == NULL && self -> proxy == NULL )
+ self -> done = true;
+}
- if ( dflt_proxy_port != 0 && mgr -> http_proxy_enabled && mgr -> http_proxy != NULL )
+static
+bool KEndPointArgsIteratorNext ( KEndPointArgsIterator * self,
+ const String ** hostname, uint16_t * port,
+ bool * proxy_default_port, bool * proxy_ep )
+{
+ static const uint16_t dflt_proxy_ports_sz =
+ sizeof self -> dflt_proxy_ports / sizeof self -> dflt_proxy_ports [ 0 ];
+
+ assert ( self != NULL );
+
+ if ( self -> done )
+ return false;
+
+ assert ( hostname && port && proxy_default_port && proxy_ep );
+
+ /* DO NOT WHACK hostname !!! */
+ HttpProxyGet ( self -> proxy, hostname, port );
+
+ * proxy_default_port = false;
+
+ if ( * hostname != NULL )
{
- uint16_t proxy_port = mgr -> http_proxy_port;
- if ( proxy_port == 0 )
+ if ( * port != 0 )
+ self -> dflt_proxy_ports_idx = 0;
+ else
{
- proxy_port = dflt_proxy_port;
- self -> proxy_default_port = true;
- }
+ assert ( self -> dflt_proxy_ports_idx < dflt_proxy_ports_sz );
+ * port = self -> dflt_proxy_ports [ self -> dflt_proxy_ports_idx ];
- rc = KNSManagerInitDNSEndpoint ( mgr, & self -> ep, mgr -> http_proxy, proxy_port );
- if ( rc == 0 )
- {
- self -> proxy_ep = true;
- return 0;
+ if ( ++ self -> dflt_proxy_ports_idx >= dflt_proxy_ports_sz )
+ self -> dflt_proxy_ports_idx = 0;
+
+ * proxy_default_port = true;
}
+
+ if ( self -> dflt_proxy_ports_idx == 0 )
+ self -> proxy = HttpProxyGetNextHttpProxy ( self -> proxy );
+
+ * proxy_ep = true;
+ return true;
}
- self -> proxy_ep = false;
+ self -> done = true;
+ if ( self -> hostname != NULL )
+ {
+ * hostname = self -> hostname;
+ * port = self -> port;
+ * proxy_ep = false;
+ return true;
+ }
- return KNSManagerInitDNSEndpoint ( mgr, & self -> ep, hostname, port );
+ return false;
}
-rc_t KClientHttpOpen ( KClientHttp * self, const String * hostname, uint32_t port )
-{
- rc_t rc = 0;
- KSocket * sock;
- const KNSManager * mgr = self -> mgr;
+static
+rc_t KClientHttpGetLine ( KClientHttp *self, struct timeout_t *tm );
- /* default port list MUST end with 0 to try without proxy */
- uint32_t pp_idx;
- static uint16_t dflt_proxy_ports [] = { 3128, 8080, 0 };
+static
+rc_t KClientHttpGetStatusLine ( KClientHttp *self, timeout_t *tm, String *msg, uint32_t *status, ver_t *version );
+
+static
+rc_t KClientHttpProxyConnect ( KClientHttp * self, const String * hostname, uint32_t port, KSocket * sock,
+ const String * phostname, uint32_t pport )
+{
+ rc_t rc;
- for ( pp_idx = 0; pp_idx < sizeof dflt_proxy_ports / sizeof dflt_proxy_ports [ 0 ]; ++ pp_idx )
+ STATUS ( STAT_GEEK, "%s - extracting stream from socket\n", __func__ );
+ assert ( sock != NULL );
+ rc = KSocketGetStream ( sock, & self -> sock );
+ if ( rc == 0 )
{
- /* if endpoint was not successfully opened on previous attempt */
- if ( ! self -> ep_valid )
+ uint32_t port_save;
+ String hostname_save, hostname_copy;
+
+ size_t len;
+ char buffer [ 4096 ];
+
+ STATUS ( STAT_GEEK, "%s - saving hostname and port\n", __func__ );
+ hostname_save = self -> hostname;
+ port_save = self -> port;
+
+ STATUS ( STAT_GEEK, "%s - saving hostname param\n", __func__ );
+ hostname_copy = * hostname;
+
+ assert ( hostname != NULL );
+ STATUS ( STAT_GEEK, "%s - setting hostname and port to '%S:%u'\n", __func__, phostname, pport );
+ self -> hostname = * phostname;
+ self -> port = pport;
+
+ /* format CONNECT request */
+ rc = string_printf ( buffer, sizeof buffer, & len,
+ "CONNECT %S:%u HTTP/1.1\r\n"
+ "Host: %S:%u\r\n\r\n"
+ , & hostname_copy
+ , port
+ , & hostname_copy
+ , port
+ );
+
+ if ( rc != 0 )
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ), ( "Failed to create proxy request: %R\n", rc ) );
+ else
{
- rc = KClientHttpInitDNSEndpoint ( self, hostname, port, dflt_proxy_ports [ pp_idx ] );
+ size_t sent;
+ timeout_t tm;
+
+ STATUS ( STAT_QA, "%s - created proxy request '%.*s'\n", __func__, ( uint32_t ) len, buffer );
+
+ /* send request and receive a response */
+ STATUS ( STAT_PRG, "%s - sending proxy request\n", __func__ );
+ TimeoutInit ( & tm, self -> write_timeout );
+ rc = KStreamTimedWriteAll ( self -> sock, buffer, len, & sent, & tm );
if ( rc != 0 )
- break;
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ), ( "Failed to send proxy request: %R\n", rc ) );
+ else
+ {
+ String msg;
+ ver_t version;
+ uint32_t status;
+
+ assert ( sent == len );
+
+ STATUS ( STAT_PRG, "%s - reading proxy response status line\n", __func__ );
+ TimeoutInit ( & tm, self -> read_timeout );
+ rc = KClientHttpGetStatusLine ( self, & tm, & msg, & status, & version );
+ if ( rc != 0 )
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ), ( "Failed to read proxy response: %R\n", rc ) );
+ else
+ {
+ if ( ( status / 100 ) != 2 )
+ {
+ rc = RC ( rcNS, rcNoTarg, rcOpening, rcConnection, rcFailed );
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ), ( "Failed to create proxy tunnel: %03u '%S'\n", status, & msg ) );
+ KClientHttpBlockBufferReset ( self );
+ KClientHttpLineBufferReset ( self );
+ }
+ else
+ {
+ STATUS ( STAT_QA, "%s - read proxy response status line: %03u '%S'\n", __func__, status, & msg );
+ do
+ rc = KClientHttpGetLine ( self, & tm );
+ while ( self -> line_valid != 0 );
+ }
+ }
+ }
}
- /* try to connect to endpoint */
- rc = KNSManagerMakeTimedConnection ( mgr, & sock,
- self -> read_timeout, self -> write_timeout, NULL, & self -> ep );
+ STATUS ( STAT_GEEK, "%s - restoring hostname and port\n", __func__ );
+ self -> hostname = hostname_save;
+ self -> port = port_save;
+
+ STATUS ( STAT_GEEK, "%s - releasing socket stream\n", __func__ );
+ KStreamRelease ( self -> sock );
+ self -> sock = NULL;
+ }
+
+ return rc;
+}
+
+
+static
+rc_t KClientHttpOpen ( KClientHttp * self, const String * aHostname, uint32_t aPort )
+{
+ rc_t rc = 0;
+ KSocket * sock = NULL;
+ const String * hostname = NULL;
+ uint16_t port = 0;
+ bool proxy_default_port = false;
+ bool proxy_ep = false;
+ KEndPointArgsIterator it;
+ const KNSManager * mgr = NULL;
+
+ STATUS ( STAT_QA, "%s - opening socket to %S:%u\n", __func__, aHostname, aPort );
+
+ assert ( self != NULL );
+ mgr = self -> mgr;
+
+ KEndPointArgsIteratorMake ( & it, mgr, aHostname, aPort );
+ while ( KEndPointArgsIteratorNext
+ ( & it, & hostname, & port, & proxy_default_port, & proxy_ep ) )
+ {
+ rc = KNSManagerInitDNSEndpoint ( mgr, & self -> ep, hostname, port );
if ( rc == 0 )
{
- /* this is a good endpoint */
- self -> ep_valid = true;
- break;
+ self -> proxy_default_port = proxy_default_port;
+ self -> proxy_ep = proxy_ep;
}
+ if ( rc == 0 )
+ {
+ /* try to establish a connection */
+ rc = KNSManagerMakeTimedConnection ( mgr, & sock,
+ self -> read_timeout, self -> write_timeout, NULL, & self -> ep );
- /* if we did not try a proxy server before, exit loop */
- if ( self -> ep_valid || ! self -> proxy_ep )
- break;
+ /* if we connected to a proxy, try to follow-through to server */
+ if ( proxy_ep && self -> tls && rc == 0 )
+ rc = KClientHttpProxyConnect ( self, aHostname, aPort, sock, hostname, port );
+
+ if ( rc == 0 )
+ {
+ /* this is a good endpoint */
+ self -> ep_valid = true;
+ break;
+ }
+
+ /* here rc is not 0, so we release the socket */
+ KSocketRelease ( sock );
+ sock = NULL;
+ }
}
/* if the connection is open */
if ( rc == 0 )
{
- rc = KSocketGetStream ( sock, & self -> sock );
- KSocketRelease ( sock );
+ STATUS ( STAT_USR, "%s - connected to %S\n", __func__, hostname );
+ if ( self -> tls )
+ {
+ KTLSStream * tls_stream;
+
+ STATUS ( STAT_PRG, "%s - creating TLS wrapper on socket\n", __func__ );
+ rc = KNSManagerMakeTLSStream ( mgr, & tls_stream, sock, aHostname );
+
+ if ( rc != 0 )
+ {
+ if ( ! proxy_ep )
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ), ( "Failed to create TLS stream for '%S'\n", aHostname ) );
+ else
+ {
+ STATUS ( STAT_PRG, "%s - retrying TLS wrapper on socket with proxy hostname\n", __func__ );
+ rc = KNSManagerMakeTLSStream ( mgr, & tls_stream, sock, hostname );
+ if ( rc != 0 )
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ), ( "Failed to create TLS stream for '%S'\n", hostname ) );
+ }
+ }
+
+ KSocketRelease ( sock );
+
+ if ( rc == 0 )
+ {
+ STATUS ( STAT_USR, "%s - verifying CA cert\n", __func__ );
+ rc = KTLSStreamVerifyCACert ( tls_stream );
+ if ( rc != 0 )
+ {
+ LOGERR ( klogErr, rc, "failed to verify CA cert" );
+ STATUS ( STAT_QA, "%s - WARNING: failed to verify CA cert - %R\n", __func__, rc );
+ }
+
+ if ( rc == 0 )
+ {
+ STATUS ( STAT_PRG, "%s - extracting TLS wrapper as stream\n", __func__ );
+ rc = KTLSStreamGetStream ( tls_stream, & self -> sock );
+ }
+
+ KTLSStreamRelease ( tls_stream );
+ }
+ }
+ else
+ {
+ STATUS ( STAT_PRG, "%s - extracting stream from socket\n", __func__ );
+ rc = KSocketGetStream ( sock, & self -> sock );
+ KSocketRelease ( sock );
+ }
if ( rc == 0 )
{
- self -> port = port;
+ STATUS ( STAT_PRG, "%s - setting port number - %d\n", __func__, aPort );
+ self -> port = aPort;
return 0;
}
}
@@ -264,6 +498,7 @@ rc_t KClientHttpOpen ( KClientHttp * self, const String * hostname, uint32_t por
return rc;
}
+
#if _DEBUGGING
/* we need this hook to be able to test the re-connection logic */
static struct KStream * (*ClientHttpReopenCallback) ( void ) = NULL;
@@ -290,12 +525,15 @@ rc_t KClientHttpReopen ( KClientHttp * self )
/* Initialize KClientHttp object */
static
-rc_t KClientHttpInit ( KClientHttp * http, const KDataBuffer *hostname_buffer, KStream * conn, ver_t _vers, const String * _host, uint32_t port )
+rc_t KClientHttpInit ( KClientHttp * http, const KDataBuffer *hostname_buffer, KStream * conn, ver_t _vers, const String * _host, uint32_t port, bool tls )
{
rc_t rc;
if ( port == 0 )
- port = 80;
+ rc = RC ( rcNS, rcNoTarg, rcInitializing, rcParam, rcInvalid );
+
+ /* early setting of TLS property */
+ http -> tls = tls;
/* we accept a NULL connection ( from ) */
if ( conn == NULL )
@@ -321,9 +559,11 @@ rc_t KClientHttpInit ( KClientHttp * http, const KDataBuffer *hostname_buffer, K
( _host -> addr - ( const char* ) hostname_buffer -> base ),
_host -> size );
if ( rc == 0 )
+ {
/* Its safe to assign pointer because we know
that the pointer is within the buffer */
http -> hostname = * _host;
+ }
}
return rc;
@@ -344,15 +584,20 @@ rc_t KClientHttpInit ( KClientHttp * http, const KDataBuffer *hostname_buffer, K
*
* "port" [ IN, DEFAULT ZERO ] - if zero, defaults to standard for scheme
* if non-zero, is taken as explicit port specification
+ *
+ * "reliable" [ IN ] - if true, then trust that the settings are good
+ * and apply extra stubbornness to fail
+ *
+ * "tls" [ IN ] - if true, wrap socket in a TLS encryption protocol
*/
rc_t KNSManagerMakeClientHttpInt ( const KNSManager *self, KClientHttp **_http,
const KDataBuffer *hostname_buffer, KStream *opt_conn,
ver_t vers, int32_t readMillis, int32_t writeMillis,
- const String *host, uint32_t port, bool reliable )
+ const String *host, uint32_t port, bool reliable, bool tls )
{
rc_t rc;
- KClientHttp * http = calloc ( 1, sizeof * http );
+ KClientHttp * http = ( KClientHttp * ) calloc ( 1, sizeof * http );
if ( http == NULL )
rc = RC ( rcNS, rcNoTarg, rcAllocating, rcMemory, rcNull );
else
@@ -386,10 +631,11 @@ rc_t KNSManagerMakeClientHttpInt ( const KNSManager *self, KClientHttp **_http,
text [ host -> size ] = save;
/* init the KClientHttp object */
- rc = KClientHttpInit ( http, hostname_buffer, opt_conn, vers, host, port );
+ rc = KClientHttpInit ( http, hostname_buffer, opt_conn, vers, host, port, tls );
if ( rc == 0 )
{
http -> reliable = reliable;
+
/* assign to OUT http param */
* _http = http;
return 0;
@@ -404,9 +650,10 @@ rc_t KNSManagerMakeClientHttpInt ( const KNSManager *self, KClientHttp **_http,
return rc;
}
-LIB_EXPORT rc_t CC KNSManagerMakeTimedClientHttp ( const KNSManager *self,
+static
+rc_t KNSManagerMakeTimedClientHttpInt ( const KNSManager *self,
KClientHttp **_http, KStream *opt_conn, ver_t vers, int32_t readMillis, int32_t writeMillis,
- const String *host, uint32_t port )
+ const String *host, uint32_t port, uint32_t dflt_port, bool tls )
{
rc_t rc;
@@ -437,11 +684,13 @@ LIB_EXPORT rc_t CC KNSManagerMakeTimedClientHttp ( const KNSManager *self,
String _host;
/* copy hostname with nul termination */
- string_copy ( hostname_buffer . base, ( size_t ) hostname_buffer . elem_count,
+ string_copy ( ( char * ) hostname_buffer . base,
+ ( size_t ) hostname_buffer . elem_count,
host -> addr, host -> size );
/* create copy of host that points into new buffer */
- StringInit ( &_host, hostname_buffer . base, host -> size, host -> len );
+ StringInit ( & _host, ( const char * ) hostname_buffer . base,
+ host -> size, host -> len );
/* limit timeouts */
if ( readMillis < 0 )
@@ -453,9 +702,13 @@ LIB_EXPORT rc_t CC KNSManagerMakeTimedClientHttp ( const KNSManager *self,
else if ( writeMillis > MAX_HTTP_WRITE_LIMIT )
writeMillis = MAX_HTTP_WRITE_LIMIT;
+ /* default port */
+ if ( port == 0 )
+ port = dflt_port;
+
/* initialize http object - will create a new reference to hostname buffer */
rc = KNSManagerMakeClientHttpInt ( self, _http, & hostname_buffer,
- opt_conn, vers, readMillis, writeMillis, &_host, port, false );
+ opt_conn, vers, readMillis, writeMillis, &_host, port, false, tls );
/* release our reference to buffer */
KDataBufferWhack ( & hostname_buffer );
@@ -471,6 +724,14 @@ LIB_EXPORT rc_t CC KNSManagerMakeTimedClientHttp ( const KNSManager *self,
return rc;
}
+
+LIB_EXPORT rc_t CC KNSManagerMakeTimedClientHttp ( const KNSManager *self,
+ KClientHttp ** http, KStream *opt_conn, ver_t vers, int32_t readMillis, int32_t writeMillis,
+ const String *host, uint32_t port )
+{
+ return KNSManagerMakeTimedClientHttpInt ( self, http, opt_conn, vers, readMillis, writeMillis, host, port, 80, false );
+}
+
LIB_EXPORT rc_t CC KNSManagerMakeClientHttp ( const KNSManager *self,
KClientHttp **http, KStream *opt_conn, ver_t vers, const String *host, uint32_t port )
{
@@ -488,6 +749,30 @@ LIB_EXPORT rc_t CC KNSManagerMakeClientHttp ( const KNSManager *self,
self -> http_read_timeout, self -> http_write_timeout, host, port );
}
+LIB_EXPORT rc_t CC KNSManagerMakeTimedClientHttps ( const KNSManager *self,
+ KClientHttp ** https, KStream *opt_conn, ver_t vers, int32_t readMillis, int32_t writeMillis,
+ const String *host, uint32_t port )
+{
+ return KNSManagerMakeTimedClientHttpInt ( self, https, opt_conn, vers, readMillis, writeMillis, host, port, 443, true );
+}
+
+LIB_EXPORT rc_t CC KNSManagerMakeClientHttps ( const KNSManager *self,
+ KClientHttp **https, KStream *opt_conn, ver_t vers, const String *host, uint32_t port )
+{
+ if ( self == NULL )
+ {
+ if ( https == NULL )
+ return RC ( rcNS, rcMgr, rcValidating, rcParam, rcNull );
+
+ * https = NULL;
+
+ return RC ( rcNS, rcMgr, rcValidating, rcSelf, rcNull );
+ }
+
+ return KNSManagerMakeTimedClientHttps ( self, https, opt_conn, vers,
+ self -> http_read_timeout, self -> http_write_timeout, host, port );
+}
+
/* AddRef
* Release
@@ -536,7 +821,7 @@ static
rc_t KClientHttpGetCharFromResponse ( KClientHttp *self, char *ch, struct timeout_t *tm )
{
rc_t rc;
- char * buffer = self -> block_buffer . base;
+ char * buffer = ( char * ) self -> block_buffer . base;
/* check for data in buffer */
if ( KClientHttpBlockBufferIsEmpty ( self ) )
@@ -553,7 +838,7 @@ rc_t KClientHttpGetCharFromResponse ( KClientHttp *self, char *ch, struct timeou
return rc;
/* re-assign new base pointer */
- buffer = self -> block_buffer . base;
+ buffer = ( char * ) self -> block_buffer . base;
}
/* zero out offsets */
@@ -593,7 +878,7 @@ rc_t KClientHttpGetLine ( KClientHttp *self, struct timeout_t *tm )
{
rc_t rc;
- char * buffer = self -> line_buffer . base;
+ char * buffer = ( char * ) self -> line_buffer . base;
size_t bsize = KDataBufferBytes ( & self -> line_buffer );
/* num_valid bytes read starts at 0 */
@@ -634,7 +919,7 @@ rc_t KClientHttpGetLine ( KClientHttp *self, struct timeout_t *tm )
return rc;
/* assign new base pointer */
- buffer = self -> line_buffer . base;
+ buffer = ( char * ) self -> line_buffer . base;
}
/* buffer is not full, insert char into the buffer */
@@ -693,7 +978,7 @@ rc_t KClientHttpAddHeaderString
if ( node == NULL )
{
/* node doesnt exist - allocate memory for a new one */
- node = calloc ( 1, sizeof * node );
+ node = ( KHttpHeader * ) calloc ( 1, sizeof * node );
if ( node == NULL )
rc = RC ( rcNS, rcNoTarg, rcAllocating, rcMemory, rcNull );
else
@@ -704,14 +989,17 @@ rc_t KClientHttpAddHeaderString
if ( rc == 0 )
{
/* copy the string data into storage */
- rc = string_printf ( node -> value_storage . base, bsize, NULL,
+ rc = string_printf ( ( char *) node -> value_storage . base,
+ bsize, NULL,
"%S%S"
, name
, value );
if ( rc == 0 )
{
/* initialize the Strings to point into KHttpHeader node */
- StringInit ( & node -> name, node -> value_storage . base, name -> size, name -> len );
+ StringInit ( & node -> name,
+ ( const char * ) node -> value_storage . base,
+ name -> size, name -> len );
StringInit ( & node -> value, node -> name . addr + name -> size, value -> size, value -> len );
/* insert into tree, sorted by alphabetical order */
@@ -738,7 +1026,7 @@ rc_t KClientHttpAddHeaderString
rc = KDataBufferResize ( & node -> value_storage, cursize + value -> size + 1 + 1 );
if ( rc == 0 )
{
- char *buffer = node -> value_storage . base;
+ char * buffer = ( char * ) node -> value_storage . base;
/* copy string data into buffer */
rc = string_printf ( & buffer [ cursize ], value -> size + 2, NULL,
@@ -768,12 +1056,13 @@ rc_t KClientHttpAddHeaderString
KDataBufferResize ( & node -> value_storage, bsize );
}
/* copy the string data into storage */
- rc = string_printf ( node -> value_storage . base, bsize,
- NULL, "%S%S", name, value );
+ rc = string_printf ( ( char * ) node -> value_storage . base,
+ bsize, NULL, "%S%S", name, value );
if ( rc == 0 )
{
/* initialize the Strings to point into KHttpHeader node */
- StringInit ( & node -> name, node -> value_storage . base,
+ StringInit ( & node -> name,
+ ( const char * ) node -> value_storage . base,
name -> size, name -> len );
StringInit ( & node -> value,
node -> name . addr + name -> size,
@@ -855,8 +1144,8 @@ rc_t KClientHttpGetHeaderLine ( KClientHttp *self, timeout_t *tm, BSTree *hdrs,
else
{
char * sep;
- char * buffer = self -> line_buffer . base;
- char * end = buffer + self -> line_valid;
+ const char * buffer = ( char * ) self -> line_buffer . base;
+ const char * end = buffer + self -> line_valid;
DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS), ("HTTP receive '%s'\n", buffer));
@@ -967,6 +1256,7 @@ rc_t KClientHttpFindHeader ( const BSTree *hdrs, const char *_name, char *buffer
return rc;
}
+static
rc_t KClientHttpGetStatusLine ( KClientHttp *self, timeout_t *tm, String *msg, uint32_t *status, ver_t *version )
{
/* First time reading the response */
@@ -982,8 +1272,8 @@ rc_t KClientHttpGetStatusLine ( KClientHttp *self, timeout_t *tm, String *msg, u
if ( rc == 0 )
{
char * sep;
- char * buffer = self -> line_buffer . base;
- char * end = buffer + self -> line_valid;
+ const char * buffer = ( char * ) self -> line_buffer . base;
+ const char * end = buffer + self -> line_valid;
DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS), ("HTTP receive '%s'\n", buffer));
@@ -999,7 +1289,10 @@ rc_t KClientHttpGetStatusLine ( KClientHttp *self, timeout_t *tm, String *msg, u
{
/* make sure it is http */
if ( strcase_cmp ( "http", 4, buffer, sep - buffer, 4 ) != 0 )
+ {
rc = RC ( rcNS, rcNoTarg, rcParsing, rcNoObj, rcUnsupported );
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS), ("%s: protocol given as '%.*s'\n", __func__, ( uint32_t ) ( sep - buffer ), buffer ));
+ }
else
{
/* move buffer up to version */
@@ -1139,7 +1432,7 @@ rc_t CC KClientHttpStreamTimedRead ( const KClientHttpStream *cself,
}
else
{
- char *buf;
+ const char * buf = NULL;
/* bytes available in buffer */
uint64_t bytes_in_buffer = http -> block_valid - http -> block_read;
@@ -1149,7 +1442,7 @@ rc_t CC KClientHttpStreamTimedRead ( const KClientHttpStream *cself,
num_to_read = bytes_in_buffer;
/* capture base pointer */
- buf = http -> block_buffer . base;
+ buf = ( char * ) http -> block_buffer . base;
/* copy data into the user buffer from the offset of bytes not yet read */
memcpy ( buffer, & buf [ http -> block_read ], num_to_read );
@@ -1225,7 +1518,8 @@ rc_t CC KClientHttpStreamTimedReadChunked ( const KClientHttpStream *cself,
/* convert the hex number containing chunk size to uint64
sep should be pointing at nul byte */
- self -> content_length = strtou64 ( http -> line_buffer . base, & sep, 16 );
+ self -> content_length
+ = strtou64 ( ( const char * ) http -> line_buffer . base, & sep, 16 );
/* TBD - eat spaces here? */
/* check if there was no hex number, or sep isn't pointing to nul byte */
@@ -1331,7 +1625,7 @@ static
rc_t KClientHttpStreamMake ( KClientHttp *self, KStream **sp, const char *strname, size_t content_length, bool size_unknown )
{
rc_t rc;
- KClientHttpStream *s = calloc ( 1, sizeof * s );
+ KClientHttpStream * s = ( KClientHttpStream * ) calloc ( 1, sizeof * s );
if ( s == NULL )
rc = RC ( rcNS, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
else
@@ -1361,7 +1655,7 @@ static
rc_t KClientHttpStreamMakeChunked ( KClientHttp *self, KStream **sp, const char *strname )
{
rc_t rc;
- KClientHttpStream *s = calloc ( 1, sizeof * s );
+ KClientHttpStream * s = ( KClientHttpStream * ) calloc ( 1, sizeof * s );
if ( s == NULL )
rc = RC ( rcNS, rcNoTarg, rcConstructing, rcMemory, rcExhausted );
else
@@ -1499,7 +1793,8 @@ rc_t KClientHttpSendReceiveMsg ( KClientHttp *self, KClientHttpResult **rslt,
if ( rc == 0 )
{
/* create a result object with enough space for msg string + nul */
- KClientHttpResult *result = malloc ( sizeof * result + msg . size + 1 );
+ KClientHttpResult * result = ( KClientHttpResult * )
+ malloc ( sizeof * result + msg . size + 1 );
if ( result == NULL )
rc = RC ( rcNS, rcNoTarg, rcAllocating, rcMemory, rcExhausted );
else
@@ -1560,10 +1855,12 @@ rc_t KClientHttpSendReceiveMsg ( KClientHttp *self, KClientHttpResult **rslt,
}
/* test */
+/*static
void KClientHttpForceSocketClose(const KClientHttp *self)
{
KStreamForceSocketClose(self->sock);
}
+*/
/* AddRef
* Release
@@ -2303,7 +2600,7 @@ rc_t CC KNSManagerMakeClientRequestInt ( const KNSManager *self,
KClientHttp * http;
rc = KNSManagerMakeClientHttpInt ( self, & http, & buf, conn, vers,
- self -> http_read_timeout, self -> http_write_timeout, & block . host, block . port, reliable );
+ self -> http_read_timeout, self -> http_write_timeout, & block . host, block . port, reliable, block . tls );
if ( rc == 0 )
{
rc = KClientHttpMakeRequestInt ( http, req, & block, & buf );
@@ -2763,7 +3060,7 @@ rc_t KClientHttpRequestHandleRedirection ( KClientHttpRequest *self, KClientHttp
http -> ep_valid = false;
/* reinitialize the http from uri */
- rc = KClientHttpInit ( http, &uri, NULL, http -> vers , &b . host, b . port );
+ rc = KClientHttpInit ( http, &uri, NULL, http -> vers , &b . host, b . port, b . tls );
if ( rc == 0 )
{
KClientHttpRequestClear ( self );
@@ -3057,6 +3354,62 @@ rc_t CC KClientHttpRequestPOST_Int ( KClientHttpRequest *self, KClientHttpResult
return rc;
}
+typedef enum {
+ eUPSBegin,
+ eUPSHost,
+ eUPSDone,
+} EUrlParseState;
+static bool GovSiteByHttp ( const char * path ) {
+ if ( path != NULL ) {
+ size_t path_size = string_measure ( path, NULL );
+ size_t size = 0;
+ String http;
+ CONST_STRING ( & http, "http://" );
+ size = http . size;
+
+ /* resolver-cgi is called over http */
+ if ( path_size > size &&
+ strcase_cmp ( path, size, http . addr, size, size ) == 0 )
+ {
+ EUrlParseState state = eUPSBegin;
+ int i = 0;
+ for ( i = 7; i < path_size && state != eUPSDone; ++i ) {
+ switch ( state ) {
+ case eUPSBegin:
+ if ( path [ i ] != '/' ) {
+ state = eUPSHost;
+ }
+ break;
+ case eUPSHost:
+ if ( path [ i ] == '/' ) {
+ state = eUPSDone;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ if ( state == eUPSBegin ) {
+ return false;
+ }
+ else {
+ size_t size = 0;
+ String gov;
+ CONST_STRING ( & gov, ".gov" );
+ size = gov . size;
+ if ( strcase_cmp
+ ( path + i - 5, size, gov . addr, size, size ) == 0 )
+ {
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+}
+
/* POST
* send POST message
* query parameters are sent in URL
@@ -3065,7 +3418,16 @@ rc_t CC KClientHttpRequestPOST_Int ( KClientHttpRequest *self, KClientHttpResult
LIB_EXPORT rc_t CC KClientHttpRequestPOST ( KClientHttpRequest *self, KClientHttpResult **_rslt )
{
KHttpRetrier retrier;
- rc_t rc = KHttpRetrierInit ( & retrier, self -> url_buffer . base, self -> http -> mgr );
+ rc_t rc = 0;
+
+ if ( self == NULL ) {
+ return RC ( rcNS, rcNoTarg, rcUpdating, rcSelf, rcNull );
+ }
+ if ( _rslt == NULL ) {
+ return RC ( rcNS, rcNoTarg, rcUpdating, rcParam, rcNull );
+ }
+
+ rc = KHttpRetrierInit ( & retrier, self -> url_buffer . base, self -> http -> mgr );
if ( rc == 0 )
{
@@ -3076,12 +3438,22 @@ LIB_EXPORT rc_t CC KClientHttpRequestPOST ( KClientHttpRequest *self, KClientHtt
{ /* a non-HTTP problem */
break;
}
+
+ assert ( * _rslt );
+
+ if ( ( * _rslt ) -> status == 403 &&
+ GovSiteByHttp ( self -> url_buffer . base ) )
+ {
+ break;
+ }
+
if ( ! self -> http -> reliable || ! KHttpRetrierWait ( & retrier, ( * _rslt ) -> status ) )
{ /* We are either not configured to retry, or HTTP status is not retriable, or we exhausted
the max number of retries or the total wait time.
rc is 0, but the caller will have to look at _rslt->status to determine success */
break;
}
+
KClientHttpResultRelease ( * _rslt );
}
diff --git a/libs/kns/http-file.c b/libs/kns/http-file.c
index 97ca013..ae41b91 100644
--- a/libs/kns/http-file.c
+++ b/libs/kns/http-file.c
@@ -82,7 +82,7 @@ typedef struct KHttpFile KHttpFile;
( ( void ) 0 )
#endif
-#define USE_CACHE_CONTROL 0
+#define USE_CACHE_CONTROL 1
#define NO_CACHE_LIMIT ( ( uint64_t ) ( 16 * 1024 * 1024 ) )
@@ -228,7 +228,6 @@ otherwise we are going to hit "Apache return HTTP headers twice" bug */
/* tell proxies not to cache if file is above limit */
if ( rc == 0 && self -> no_cache )
rc = KClientHttpRequestSetNoCache ( req );
-#warning "using cache control"
#endif
if ( rc == 0 )
{
@@ -515,7 +514,7 @@ static rc_t KNSManagerVMakeHttpFileInt ( const KNSManager *self,
KClientHttp *http;
rc = KNSManagerMakeClientHttpInt ( self, & http, buf, conn, vers,
- self -> http_read_timeout, self -> http_write_timeout, &block . host, block . port, reliable );
+ self -> http_read_timeout, self -> http_write_timeout, &block . host, block . port, reliable, block . tls );
if ( rc == 0 )
{
KClientHttpRequest *req;
diff --git a/libs/kns/http-priv.h b/libs/kns/http-priv.h
index c9d1460..5019bd6 100644
--- a/libs/kns/http-priv.h
+++ b/libs/kns/http-priv.h
@@ -61,33 +61,7 @@ struct KClientHttp;
struct KClientHttpRequest;
struct KStream;
struct timeout_t;
-
-/*--------------------------------------------------------------------------
- * URLBlock
- * RFC 3986
- */
-typedef enum
-{
- st_NONE,
- st_HTTP,
- st_S3
-} SchemeType;
-
-typedef struct URLBlock URLBlock;
-struct URLBlock
-{
- String scheme;
- String host;
- String path; /* Path includes any parameter portion */
- String query;
- String fragment;
-
- uint32_t port;
-
- SchemeType scheme_type;
-};
-extern void URLBlockInit ( URLBlock *self );
-extern rc_t ParseUrl ( URLBlock * b, const char * url, size_t url_size );
+struct URLBlock;
/*--------------------------------------------------------------------------
* KHttpHeader
@@ -105,34 +79,42 @@ struct KHttpHeader
extern void KHttpHeaderWhack ( BSTNode *n, void *ignore );
extern int64_t CC KHttpHeaderSort ( const BSTNode *na, const BSTNode *nb );
extern int64_t CC KHttpHeaderCmp ( const void *item, const BSTNode *n );
+/*
extern rc_t KHttpGetHeaderLine ( struct KClientHttp *self, struct timeout_t *tm, BSTree *hdrs, bool *blank, bool *close_connection );
extern rc_t KHttpGetStatusLine ( struct KClientHttp *self, struct timeout_t *tm, String *msg, uint32_t *status, ver_t *version );
+*/
/* compatibility for existing code */
+/*
#define KHttpGetHeaderLine KClientHttpGetHeaderLine
#define KHttpGetStatusLine KClientHttpGetStatusLine
+*/
/*--------------------------------------------------------------------------
* KClientHttp
*/
+/*
rc_t KClientHttpOpen ( struct KClientHttp * self, const String * hostname, uint32_t port );
+*/
void KClientHttpClose ( struct KClientHttp * self );
rc_t KClientHttpReopen ( struct KClientHttp * self );
rc_t KNSManagerMakeClientHttpInt ( struct KNSManager const *self, struct KClientHttp **_http,
const KDataBuffer *hostname_buffer, struct KStream *opt_conn,
ver_t vers, int32_t readMillis, int32_t writeMillis,
- const String *host, uint32_t port, bool reliable );
+ const String *host, uint32_t port, bool reliable, bool tls );
/* test */
+/*
void KClientHttpForceSocketClose(const struct KClientHttp *self);
+*/
/*--------------------------------------------------------------------------
* KClientHttpRequest
*/
rc_t KClientHttpMakeRequestInt ( struct KClientHttp const *self,
- struct KClientHttpRequest **req, const URLBlock *block, const KDataBuffer *buf );
+ struct KClientHttpRequest **req, const struct URLBlock *block, const KDataBuffer *buf );
/* exported private functions
diff --git a/libs/kns/http.c b/libs/kns/http.c
index 781dd19..5d9547f 100644
--- a/libs/kns/http.c
+++ b/libs/kns/http.c
@@ -83,9 +83,12 @@ void URLBlockInit ( URLBlock *self )
CONST_STRING ( & self -> query, "" );
CONST_STRING ( & self -> fragment, "" );
- self -> port = 0; /* 0 = DEFAULT 80 for http */
+ self -> port = 0; /* 0 = DEFAULT 80 for http, 443 for https */
self -> scheme_type = st_NONE;
+ self -> tls = false;
+
+ self -> port_dflt = true;
}
/* ParseUrl
@@ -131,27 +134,39 @@ rc_t ParseUrl ( URLBlock * b, const char * url, size_t url_size )
String http;
CONST_STRING ( & http, "http" );
- /* here we assume the scheme will be http */
- b -> scheme_type = st_HTTP;
-
/* assign scheme to the url_block */
StringInit ( & b -> scheme, buf, sep - buf, ( uint32_t ) ( sep - buf ) );
- /* check to make sure it is 'http' */
+ /* here we assume the scheme will be http */
+ b -> port = 80;
+ b -> scheme_type = st_HTTP;
if ( ! StringCaseEqual ( & b -> scheme, & http ) )
{
- /* it is not http, check for s3 */
- String s3;
- CONST_STRING ( & s3, "s3" );
-
- if ( ! StringCaseEqual ( & b -> scheme, & s3 ) )
+ String https;
+ CONST_STRING ( & https, "https" );
+
+ /* check for https */
+ b -> port = 443;
+ b -> scheme_type = st_HTTPS;
+ b -> tls = true;
+ if ( ! StringCaseEqual ( & b -> scheme, & https ) )
{
- b -> scheme_type = st_NONE;
- rc = RC ( rcNS, rcUrl, rcEvaluating, rcName, rcIncorrect );
- PLOGERR ( klogErr ,( klogErr, rc, "Scheme is '$(scheme)'", "scheme=%S", & b -> scheme ) );
- return rc;
+ String s3;
+ CONST_STRING ( & s3, "s3" );
+
+ /* it is not http, check for s3 */
+ b -> port = 80;
+ b -> scheme_type = st_S3;
+ b -> tls = false;
+ if ( ! StringCaseEqual ( & b -> scheme, & s3 ) )
+ {
+ b -> port = 0;
+ b -> scheme_type = st_NONE;
+ rc = RC ( rcNS, rcUrl, rcEvaluating, rcName, rcIncorrect );
+ PLOGERR ( klogErr ,( klogErr, rc, "Scheme is '$(scheme)'", "scheme=%S", & b -> scheme ) );
+ return rc;
+ }
}
- b -> scheme_type = st_S3;
}
/* accept scheme - skip past */
@@ -287,6 +302,8 @@ rc_t ParseUrl ( URLBlock * b, const char * url, size_t url_size )
return rc;
}
+ b -> port_dflt = false;
+
/* assign host to url_block */
StringInit ( & b -> host, buf, sep - buf, ( uint32_t ) ( sep - buf ) );
}
diff --git a/libs/kns/kns_manager-ext.c b/libs/kns/kns_manager-ext.c
index 83ac39c..0d82f7a 100644
--- a/libs/kns/kns_manager-ext.c
+++ b/libs/kns/kns_manager-ext.c
@@ -63,7 +63,7 @@ static rc_t CC KNSManagerNewReleaseVersionImpl(const struct KNSManager *self,
memset(&result, 0, sizeof result);
if (rc == 0) {
rc = KNSManagerMakeRequest(self, &req, 0x01010000, NULL,
- "http://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/sratoolkit.current.version"
+ "https://ftp-trace.ncbi.nlm.nih.gov/sra/sdk/current/sratoolkit.current.version"
);
}
if (rc == 0) {
diff --git a/libs/kns/manager.c b/libs/kns/manager.c
index 0f17553..8fe8e37 100644
--- a/libs/kns/manager.c
+++ b/libs/kns/manager.c
@@ -38,6 +38,8 @@
#include <klib/refcount.h>
#include <klib/rc.h>
+#include <kproc/timeout.h>
+
#include <kns/manager.h>
#include <kns/socket.h>
#include <kns/http.h>
@@ -52,8 +54,10 @@
#include <stdlib.h>
#include <string.h>
+#include <stdio.h> /* fprintf */
+
#ifndef MAX_CONN_LIMIT
-#define MAX_CONN_LIMIT ( 10 * 60 )
+#define MAX_CONN_LIMIT ( 60 * 1000 )
#endif
#ifndef MAX_CONN_READ_LIMIT
@@ -72,6 +76,173 @@ static char kns_manager_user_agent [ 128 ] = "ncbi-vdb";
static atomic_ptr_t kns_singleton;
#endif
+#define RELEASE( type, obj ) do { rc_t rc2 = type##Release ( obj ); \
+ if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while ( false )
+
+struct HttpProxy {
+ const struct String * http_proxy;
+ uint16_t http_proxy_port;
+ struct HttpProxy * next;
+};
+
+const HttpProxy * HttpProxyGetNextHttpProxy ( const HttpProxy * self ) {
+ if ( self == NULL ) {
+ return NULL;
+ }
+ while ( self -> next != NULL ) {
+ self = self -> next;
+ if ( self -> http_proxy != NULL ) {
+ return self;
+ }
+ }
+ return NULL;
+}
+
+/* N.B.: DO NOT WHACK THE RETURNED http_proxy String !!! */
+void HttpProxyGet ( const HttpProxy * self,
+ const String ** http_proxy, uint16_t * http_proxy_port )
+{
+ assert ( http_proxy && http_proxy_port );
+ if ( self != NULL ) {
+ * http_proxy = self -> http_proxy;
+ * http_proxy_port = self -> http_proxy_port;
+ }
+ else {
+ * http_proxy = NULL;
+ * http_proxy_port = 0;
+ }
+}
+
+static HttpProxy * HttpProxyClear ( HttpProxy * self ) {
+ assert ( self );
+
+ StringWhack ( self -> http_proxy );
+
+ self -> http_proxy = NULL;
+ self -> http_proxy_port = 0;
+
+ return self -> next;
+}
+
+static rc_t HttpProxyWhack ( HttpProxy * self, HttpProxy ** next ) {
+ assert ( next );
+ * next = HttpProxyClear ( self );
+
+ memset ( self, 0, sizeof *self );
+
+ free (self );
+
+ return 0;
+}
+
+static
+rc_t HttpProxyGetHttpProxyPath
+ ( const HttpProxy * self, const String ** proxy )
+{
+ rc_t rc = 0;
+
+ if ( proxy == NULL )
+ rc = RC ( rcNS, rcMgr, rcAccessing, rcParam, rcNull );
+ else
+ {
+ if ( self == NULL )
+ rc = RC ( rcNS, rcMgr, rcAccessing, rcSelf, rcNull );
+ else if ( self -> http_proxy != NULL )
+ {
+ return StringCopy ( proxy, self -> http_proxy );
+ }
+
+ * proxy = NULL;
+ }
+
+ return rc;
+}
+
+static
+rc_t HttpProxyAddHttpProxyPath ( HttpProxy * self,
+ const String * proxy, uint16_t proxy_port, KNSManager * mgr )
+{
+ assert ( self );
+
+ if ( proxy == NULL ) {
+ return 0;
+ }
+
+ while ( self -> http_proxy != NULL ) {
+ if ( self -> next == NULL ) {
+ self -> next = calloc ( 1, sizeof * self -> next );
+ if ( self -> next == NULL ) {
+ return RC ( rcNS, rcMgr, rcAllocating, rcMemory, rcExhausted );
+ }
+ }
+ self = self -> next;
+ }
+
+ self -> http_proxy = proxy;
+ self -> http_proxy_port = proxy_port;
+
+ if ( ! mgr -> http_proxy_enabled ) {
+ mgr -> http_proxy_enabled = ( proxy != NULL );
+ }
+
+ return 0;
+}
+
+
+static rc_t KNSManagerHttpProxyClear ( KNSManager * self ) {
+ HttpProxy * p = self -> http_proxy;
+ while ( p ) {
+ p = HttpProxyClear ( p );
+ }
+ return 0;
+}
+
+static rc_t KNSManagerHttpProxyWhack ( const KNSManager * self ) {
+ HttpProxy * p = self -> http_proxy;
+ while ( p ) {
+ HttpProxy * next = NULL;
+ HttpProxyWhack ( p, & next );
+ p = next;
+ }
+ return 0;
+}
+
+static
+rc_t KNSManagerAddHTTPProxyPathImpl ( KNSManager * self,
+ const String * proxy, uint16_t proxy_port )
+{
+ if ( self -> http_proxy == NULL ) {
+ self -> http_proxy = calloc ( 1, sizeof * self -> http_proxy );
+ if ( self -> http_proxy == NULL ) {
+ return RC ( rcNS, rcMgr, rcAllocating, rcMemory, rcExhausted );
+ }
+ }
+
+ return
+ HttpProxyAddHttpProxyPath ( self -> http_proxy, proxy, proxy_port, self );
+}
+
+
+void HttpProxyGetHTTPProxy ( const HttpProxy * self,
+ const struct String ** http_proxy, uint16_t * http_proxy_port )
+{
+ assert ( self && http_proxy && http_proxy_port);
+
+ * http_proxy = self -> http_proxy;
+ * http_proxy_port = self -> http_proxy_port;
+}
+
+const HttpProxy * KNSManagerGetHttpProxy ( const KNSManager * self ) {
+ if ( self == NULL) {
+ return NULL;
+ }
+ if ( self -> http_proxy != NULL ) {
+ return self -> http_proxy;
+ }
+ return HttpProxyGetNextHttpProxy ( self -> http_proxy );
+}
+
+
static
rc_t KNSManagerWhack ( KNSManager * self )
{
@@ -83,8 +254,7 @@ rc_t KNSManagerWhack ( KNSManager * self )
return 0;
#endif
- if ( self -> http_proxy != NULL )
- StringWhack ( self -> http_proxy );
+ KNSManagerHttpProxyWhack ( self );
if ( self -> aws_access_key_id != NULL )
StringWhack ( self -> aws_access_key_id );
@@ -100,6 +270,8 @@ rc_t KNSManagerWhack ( KNSManager * self )
rc = HttpRetrySpecsDestroy ( & self -> retry_specs );
+ KTLSGlobalsWhack ( & self -> tlsg );
+
free ( self );
KNSManagerCleanup ();
@@ -139,54 +311,6 @@ LIB_EXPORT rc_t CC KNSManagerRelease ( const KNSManager *self )
}
static
-void KNSManagerHttpProxyInit ( KNSManager * self, KConfig * kfg )
-{
- const KConfigNode * proxy;
- rc_t rc = KConfigOpenNodeRead ( kfg, & proxy, "http/proxy" );
- if ( rc == 0 )
- {
- const KConfigNode * proxy_path;
- rc = KConfigNodeOpenNodeRead ( proxy, & proxy_path, "path" );
- if ( rc == 0 )
- {
- String * path;
- rc = KConfigNodeReadString ( proxy_path, & path );
- if ( rc == 0 )
- {
- rc = KNSManagerSetHTTPProxyPath ( self, "%S", path );
- if ( rc == 0 )
- {
- const KConfigNode * proxy_enabled;
- rc = KConfigNodeOpenNodeRead ( proxy, & proxy_enabled, "enabled" );
- if ( rc == 0 )
- {
- rc = KConfigNodeReadBool ( proxy_enabled, & self -> http_proxy_enabled );
- KConfigNodeRelease ( proxy_enabled );
- }
- else if ( GetRCState ( rc ) == rcNotFound )
- {
- rc = 0;
- }
-
- if ( rc != 0 )
- {
- KNSManagerSetHTTPProxyPath ( self, NULL );
- assert ( self -> http_proxy_enabled == false );
- }
- }
-
- StringWhack ( path );
- }
-
- KConfigNodeRelease ( proxy_path );
- }
-
- KConfigNodeRelease ( proxy );
- }
-}
-
-
-static
void KNSManagerLoadAWS ( struct KNSManager *self, const KConfig * kfg )
{
rc_t rc;
@@ -260,58 +384,6 @@ void KNSManagerLoadAWS ( struct KNSManager *self, const KConfig * kfg )
}
}
-LIB_EXPORT rc_t CC KNSManagerMakeConfig ( KNSManager **mgrp, KConfig* kfg )
-{
- rc_t rc;
-
- if ( mgrp == NULL )
- rc = RC ( rcNS, rcMgr, rcAllocating, rcParam, rcNull );
- else
- {
- KNSManager * mgr = calloc ( 1, sizeof * mgr );
- if ( mgr == NULL )
- rc = RC ( rcNS, rcMgr, rcAllocating, rcMemory, rcExhausted );
- else
- {
- KRefcountInit ( & mgr -> refcount, 1, "KNSManager", "init", "kns" );
- mgr -> conn_timeout = MAX_CONN_LIMIT;
- mgr -> conn_read_timeout = MAX_CONN_READ_LIMIT;
- mgr -> conn_write_timeout = MAX_CONN_WRITE_LIMIT;
- mgr -> http_read_timeout = MAX_HTTP_READ_LIMIT;
- mgr -> http_write_timeout = MAX_HTTP_WRITE_LIMIT;
- mgr -> maxTotalWaitForReliableURLs_ms = 10 * 60 * 1000; /* 10 min */
- mgr -> maxNumberOfRetriesOnFailureForReliableURLs = 10;
- mgr -> verbose = false;
-
- rc = KNSManagerInit (); /* platform specific init in sysmgr.c ( in unix|win etc. subdir ) */
- if ( rc == 0 )
- {
- /* the manager is not a proper singleton */
- if ( kns_manager_user_agent [ 0 ] == 0 )
- {
- ver_t version = RELEASE_VERS;
- KNSManagerSetUserAgent ( mgr, PKGNAMESTR " ncbi-vdb.%V", version );
- }
-
- rc = HttpRetrySpecsInit ( & mgr -> retry_specs, kfg );
- if ( rc == 0 )
- {
- KNSManagerLoadAWS ( mgr, kfg );
- KNSManagerHttpProxyInit ( mgr, kfg );
- * mgrp = mgr;
- return 0;
- }
- }
-
- free ( mgr );
- }
-
- * mgrp = NULL;
- }
-
- return rc;
-}
-
LIB_EXPORT rc_t CC KNSManagerMake ( KNSManager ** mgrp )
{
rc_t rc;
@@ -400,6 +472,8 @@ LIB_EXPORT bool KNSManagerIsVerbose ( const KNSManager *self )
LIB_EXPORT rc_t CC KNSManagerMakeConnection ( const KNSManager * self,
struct KSocket **conn, struct KEndPoint const *from, struct KEndPoint const *to )
{
+ timeout_t tm;
+
if ( self == NULL )
{
if ( conn == NULL )
@@ -410,8 +484,10 @@ LIB_EXPORT rc_t CC KNSManagerMakeConnection ( const KNSManager * self,
return RC ( rcNS, rcStream, rcConstructing, rcSelf, rcNull );
}
+ TimeoutInit ( & tm, self -> conn_timeout );
+
return KNSManagerMakeRetryTimedConnection ( self, conn,
- self -> conn_timeout, self -> conn_read_timeout, self -> conn_write_timeout, from, to );
+ & tm, self -> conn_read_timeout, self -> conn_write_timeout, from, to );
}
/* MakeTimedConnection
* create a connection-oriented stream
@@ -435,6 +511,8 @@ LIB_EXPORT rc_t CC KNSManagerMakeTimedConnection ( struct KNSManager const * sel
struct KSocket **conn, int32_t readMillis, int32_t writeMillis,
struct KEndPoint const *from, struct KEndPoint const *to )
{
+ timeout_t tm;
+
if ( self == NULL )
{
if ( conn == NULL )
@@ -445,8 +523,10 @@ LIB_EXPORT rc_t CC KNSManagerMakeTimedConnection ( struct KNSManager const * sel
return RC ( rcNS, rcStream, rcConstructing, rcSelf, rcNull );
}
+ TimeoutInit ( & tm, self -> conn_timeout );
+
return KNSManagerMakeRetryTimedConnection ( self, conn,
- self -> conn_timeout, readMillis, writeMillis, from, to );
+ & tm, readMillis, writeMillis, from, to );
}
/* MakeRetryConnection
@@ -464,7 +544,8 @@ LIB_EXPORT rc_t CC KNSManagerMakeTimedConnection ( struct KNSManager const * sel
* both endpoints have to be of type epIP; creates a TCP connection
*/
LIB_EXPORT rc_t CC KNSManagerMakeRetryConnection ( struct KNSManager const * self,
- struct KSocket **conn, int32_t retryTimeout, struct KEndPoint const *from, struct KEndPoint const *to )
+ struct KSocket ** conn, timeout_t * retryTimeout,
+ struct KEndPoint const * from, struct KEndPoint const * to )
{
if ( self == NULL )
{
@@ -488,28 +569,22 @@ LIB_EXPORT rc_t CC KNSManagerMakeRetryConnection ( struct KNSManager const * sel
* for connects, reads and writes respectively.
*/
LIB_EXPORT rc_t CC KNSManagerSetConnectionTimeouts ( KNSManager *self,
- int32_t connectSecs, int32_t readMillis, int32_t writeMillis )
+ int32_t connectMillis, int32_t readMillis, int32_t writeMillis )
{
if ( self == NULL )
return RC ( rcNS, rcMgr, rcUpdating, rcSelf, rcNull );
/* limit values */
- if ( connectSecs < 0 )
- connectSecs = -1;
- else if ( connectSecs > MAX_CONN_LIMIT )
- connectSecs = MAX_CONN_LIMIT;
+ if ( connectMillis < 0 || connectMillis > MAX_CONN_LIMIT )
+ connectMillis = MAX_CONN_LIMIT;
- if ( readMillis < 0 )
- readMillis = -1;
- else if ( readMillis > MAX_CONN_READ_LIMIT )
+ if ( readMillis < 0 || readMillis > MAX_CONN_READ_LIMIT )
readMillis = MAX_CONN_READ_LIMIT;
- if ( writeMillis < 0 )
- writeMillis = -1;
- else if ( writeMillis > MAX_CONN_WRITE_LIMIT )
+ if ( writeMillis < 0 || writeMillis > MAX_CONN_WRITE_LIMIT )
writeMillis = MAX_CONN_WRITE_LIMIT;
- self -> conn_timeout = connectSecs;
+ self -> conn_timeout = connectMillis;
self -> conn_read_timeout = readMillis;
self -> conn_write_timeout = writeMillis;
@@ -531,14 +606,10 @@ LIB_EXPORT rc_t CC KNSManagerSetHTTPTimeouts ( KNSManager *self,
return RC ( rcNS, rcMgr, rcUpdating, rcSelf, rcNull );
/* limit values */
- if ( readMillis < 0 )
- readMillis = -1;
- else if ( readMillis > MAX_HTTP_READ_LIMIT )
+ if ( readMillis < 0 || readMillis > MAX_HTTP_READ_LIMIT )
readMillis = MAX_HTTP_READ_LIMIT;
- if ( writeMillis < 0 )
- writeMillis = -1;
- else if ( writeMillis > MAX_HTTP_WRITE_LIMIT )
+ if ( writeMillis < 0 || writeMillis > MAX_HTTP_WRITE_LIMIT )
writeMillis = MAX_HTTP_WRITE_LIMIT;
self -> http_read_timeout = readMillis;
@@ -547,29 +618,16 @@ LIB_EXPORT rc_t CC KNSManagerSetHTTPTimeouts ( KNSManager *self,
return 0;
}
+
/* GetHTTPProxyPath
* returns path to HTTP proxy server ( if set ) or NULL.
* return status is 0 if the path is valid, non-zero otherwise
*/
LIB_EXPORT rc_t CC KNSManagerGetHTTPProxyPath ( const KNSManager * self, const String ** proxy )
{
- rc_t rc = 0;
-
- if ( proxy == NULL )
- rc = RC ( rcNS, rcMgr, rcAccessing, rcParam, rcNull );
- else
- {
- if ( self == NULL )
- rc = RC ( rcNS, rcMgr, rcAccessing, rcSelf, rcNull );
- else if ( self -> http_proxy != NULL )
- {
- return StringCopy ( proxy, self -> http_proxy );
- }
-
- * proxy = NULL;
- }
-
- return rc;
+ fprintf ( stderr,
+"WARNING : KNSManagerGetHTTPProxyPath IS DEPRECATED AND SHOULD NOT BE USED\n" );
+ return HttpProxyGetHttpProxyPath ( self -> http_proxy, proxy );
}
@@ -592,7 +650,8 @@ LIB_EXPORT rc_t CC KNSManagerSetHTTPProxyPath ( KNSManager * self, const char *
return rc;
}
-LIB_EXPORT rc_t CC KNSManagerVSetHTTPProxyPath ( KNSManager * self, const char * fmt, va_list args )
+static rc_t CC KNSManagerVSetHTTPProxyPathImpl
+ ( KNSManager * self, const char * fmt, va_list args, bool clear )
{
rc_t rc = 0;
@@ -600,58 +659,255 @@ LIB_EXPORT rc_t CC KNSManagerVSetHTTPProxyPath ( KNSManager * self, const char *
rc = RC ( rcNS, rcMgr, rcUpdating, rcSelf, rcNull );
else
{
- uint16_t proxy_port = 0;
- const String * proxy = NULL;
+ if ( clear ) {
+ rc = KNSManagerHttpProxyClear ( self );
+ }
- if ( fmt != NULL && fmt [ 0 ] != 0 )
+ if ( rc == 0 && fmt != NULL && fmt [ 0 ] != 0 )
{
size_t psize;
- char path [ 4096 ];
+ char path [ 4096 * 2 ];
+ const char * p = path;
rc = string_vprintf ( path, sizeof path, & psize, fmt, args );
- if ( rc == 0 && psize != 0 )
- {
- char * colon = string_rchr ( path, psize, ':' );
+ if ( rc == 0 ) {
+ while ( psize != 0 ) {
+ size_t s = psize;
+ uint16_t proxy_port = 0;
+ const String * proxy = NULL;
+ const char * colon = NULL;
+ const char * comma = string_chr ( p, psize, ',' );
+ if ( comma != NULL ) {
+#ifdef MULTIPLE_PROXIES
+ s = comma - p;
+#else
+ rc = RC ( rcNS, rcMgr, rcUpdating, rcPath, rcInvalid );
+ break;
+#endif
+ }
+ colon = string_rchr ( p, s, ':' );
if ( colon != NULL )
{
char * end;
const char * port_spec = colon + 1;
/* it is true that some day we might read symbolic port names... */
long port_num = strtol ( port_spec, & end, 10 );
- if ( port_num <= 0 || port_num >= 0x10000 || end [ 0 ] != 0 )
+ if ( port_num <= 0 || port_num >= 0x10000 ||
+ ( end [ 0 ] != 0 && comma == NULL ) )
rc = RC ( rcNS, rcMgr, rcUpdating, rcPath, rcInvalid );
else
{
proxy_port = ( uint64_t ) port_num;
- psize = colon - path;
+ s = colon - p;
}
}
if ( rc == 0 )
{
String tmp;
- StringInit ( & tmp, path, psize, string_len ( path, psize ) );
+ StringInit ( & tmp, p, s, string_len ( p, s ) );
rc = StringCopy ( & proxy, & tmp );
}
+
+ if ( rc == 0 ) {
+ rc =
+ KNSManagerAddHTTPProxyPathImpl ( self, proxy, proxy_port );
+ }
+ if ( comma == NULL) {
+ psize = 0;
+ }
+ else {
+ s = comma - p + 1;
+ if ( s > psize ) {
+ psize = 0;
+ }
+ else {
+ psize -= s;
+ p += s;
+ }
+ }
+ }
}
}
+ }
+
+ return rc;
+}
+
+LIB_EXPORT rc_t CC KNSManagerVSetHTTPProxyPath ( KNSManager * self, const char * fmt, va_list args )
+{ return KNSManagerVSetHTTPProxyPathImpl (self, fmt, args, true); }
+
+static
+rc_t CC KNSManagerAddHTTPProxyPath ( KNSManager * self, const char * fmt, ... )
+{
+ rc_t rc;
+
+ va_list args;
+ va_start ( args, fmt );
+ rc = KNSManagerVSetHTTPProxyPathImpl ( self, fmt, args, false );
+ va_end ( args );
+
+ return rc;
+}
+
+static
+bool KNSManagerHttpProxyInitFromEnvVar ( KNSManager * self, const char * name )
+{
+ const char * path = getenv ( name );
+ if ( path != NULL ) {
+ if ( KNSManagerAddHTTPProxyPath ( self, path ) != 0 ) {
+ return false;
+ }
+
+ assert ( self -> http_proxy_enabled );
+ return true;
+ }
+
+ return false;
+}
+
+static bool KNSManagerHttpProxyInitFromEnv ( KNSManager * self ) {
+ bool loaded = false;
+
+ const char * env_list [] = {
+ "https_proxy",
+ "HTTPS_PROXY",
+ "all_proxy",
+ "ALL_PROXY",
+ "http_proxy",
+ "HTTP_PROXY",
+ };
+
+ int i = 0;
+ for ( i = 0; i < sizeof env_list / sizeof env_list [ 0 ]; ++ i ) {
+ loaded |= KNSManagerHttpProxyInitFromEnvVar ( self, env_list [i] );
+ }
+
+ return loaded;
+}
+static
+bool KNSManagerHttpProxyInitFromKfg ( KNSManager * self, const KConfig * kfg )
+{
+ bool fromKfg = false;
+
+ const KConfigNode * proxy;
+ rc_t rc = KConfigOpenNodeRead ( kfg, & proxy, "/http/proxy" );
+ if ( rc == 0 )
+ {
+ const KConfigNode * proxy_path;
+ rc = KConfigNodeOpenNodeRead ( proxy, & proxy_path, "path" );
if ( rc == 0 )
{
- if ( self -> http_proxy != NULL )
+ String * path;
+ rc = KConfigNodeReadString ( proxy_path, & path );
+ if ( rc == 0 )
{
- StringWhack ( self -> http_proxy );
- self -> http_proxy_port = 0;
+ rc = KNSManagerAddHTTPProxyPath ( self, "%S", path );
+ if ( rc == 0 ) {
+ fromKfg = true;
+ }
+
+ StringWhack ( path );
}
- self -> http_proxy = proxy;
- self -> http_proxy_enabled = ( proxy != NULL );
- self -> http_proxy_port = proxy_port;
+ KConfigNodeRelease ( proxy_path );
}
+
+ KConfigNodeRelease ( proxy );
}
- return rc;
+ return fromKfg;
+}
+
+static bool StringCmp ( const String * self, const char * val )
+{
+ String v;
+ StringInitCString ( & v, val );
+ return StringEqual ( self, & v );
}
+static rc_t StringRelease ( String * self )
+{
+ StringWhack ( self );
+ return 0;
+}
+
+static
+void KNSManagerHttpProxyInit ( KNSManager * self, const KConfig * kfg )
+{
+ int i = 0;
+
+ int n = 2;
+
+ typedef enum {
+ eEnv,
+ eKfg,
+ } EType;
+
+ EType type [ 2 ] = { eKfg, eEnv };
+
+ rc_t rc = KConfigReadBool
+ ( kfg, "/http/proxy/enabled", & self -> http_proxy_enabled );
+ if ( rc != 0 ) {
+ if ( GetRCState ( rc ) == rcNotFound ) {
+ rc = 0;
+ } else {
+ KNSManagerSetHTTPProxyPath ( self, NULL );
+ assert ( self -> http_proxy_enabled == false );
+ }
+ } else {
+ if ( ! self -> http_proxy_enabled ) {
+ return;
+ }
+ }
+
+ {
+ bool proxy_only = false;
+ rc_t rc = KConfigReadBool ( kfg, "/http/proxy/only", & proxy_only );
+ if ( rc == 0 && proxy_only ) {
+ self -> http_proxy_only = true;
+ }
+ }
+
+ {
+ String * result = NULL;
+ rc = KConfigReadString ( kfg, "/http/proxy/use", & result );
+ if ( rc == 0 ) {
+ if ( StringCmp ( result, "env") ) {
+ n = 1;
+ type [ 0 ] = eEnv;
+ } else if ( StringCmp ( result, "kfg") ) {
+ n = 1;
+ type [ 0 ] = eKfg;
+ } else if ( StringCmp ( result, "none") ) {
+ n = 0;
+ } else if ( StringCmp ( result, "env,kfg") ) {
+ n = 2;
+ type [ 0 ] = eEnv;
+ type [ 1 ] = eKfg;
+ } else if ( StringCmp ( result, "kfg,env") ) {
+ n = 2;
+ type [ 0 ] = eKfg;
+ type [ 1 ] = eEnv;
+ }
+ }
+ RELEASE ( String, result );
+ }
+
+ for ( i = 0; i < n; ++i ) {
+ switch ( type [ i] ) {
+ case eEnv:
+ KNSManagerHttpProxyInitFromEnv ( self );
+ break;
+ case eKfg:
+ KNSManagerHttpProxyInitFromKfg ( self, kfg );
+ break;
+ default:
+ assert ( 0 );
+ break;
+ }
+ }
+}
/* GetHTTPProxyEnabled
* returns true if a non-NULL proxy path exists and user wants to use it
@@ -682,6 +938,63 @@ LIB_EXPORT bool CC KNSManagerSetHTTPProxyEnabled ( KNSManager * self, bool enabl
}
+LIB_EXPORT rc_t CC KNSManagerMakeConfig ( KNSManager **mgrp, KConfig* kfg )
+{
+ rc_t rc;
+
+ if ( mgrp == NULL )
+ rc = RC ( rcNS, rcMgr, rcAllocating, rcParam, rcNull );
+ else
+ {
+ KNSManager * mgr = calloc ( 1, sizeof * mgr );
+ if ( mgr == NULL )
+ rc = RC ( rcNS, rcMgr, rcAllocating, rcMemory, rcExhausted );
+ else
+ {
+ KRefcountInit ( & mgr -> refcount, 1, "KNSManager", "init", "kns" );
+ mgr -> conn_timeout = MAX_CONN_LIMIT;
+ mgr -> conn_read_timeout = MAX_CONN_READ_LIMIT;
+ mgr -> conn_write_timeout = MAX_CONN_WRITE_LIMIT;
+ mgr -> http_read_timeout = MAX_HTTP_READ_LIMIT;
+ mgr -> http_write_timeout = MAX_HTTP_WRITE_LIMIT;
+ mgr -> maxTotalWaitForReliableURLs_ms = 10 * 60 * 1000; /* 10 min */
+ mgr -> maxNumberOfRetriesOnFailureForReliableURLs = 10;
+ mgr -> verbose = false;
+
+ rc = KNSManagerInit (); /* platform specific init in sysmgr.c ( in unix|win etc. subdir ) */
+ if ( rc == 0 )
+ {
+ /* the manager is not a proper singleton */
+ if ( kns_manager_user_agent [ 0 ] == 0 )
+ {
+ ver_t version = RELEASE_VERS;
+ KNSManagerSetUserAgent ( mgr, PKGNAMESTR " ncbi-vdb.%V", version );
+ }
+
+ rc = HttpRetrySpecsInit ( & mgr -> retry_specs, kfg );
+ if ( rc == 0 )
+ {
+ rc = KTLSGlobalsInit ( & mgr -> tlsg, kfg );
+ if ( rc == 0 )
+ {
+ KNSManagerLoadAWS ( mgr, kfg );
+ KNSManagerHttpProxyInit ( mgr, kfg );
+ * mgrp = mgr;
+ return 0;
+ }
+ }
+ }
+
+ free ( mgr );
+ }
+
+ * mgrp = NULL;
+ }
+
+ return rc;
+}
+
+
LIB_EXPORT rc_t CC KNSManagerSetUserAgent ( KNSManager * self, const char * fmt, ... )
{
/* 6/18/14 - don't check "self", since the current implementation
diff --git a/libs/kns/mgr-priv.h b/libs/kns/mgr-priv.h
index e6f754f..3bf479e 100644
--- a/libs/kns/mgr-priv.h
+++ b/libs/kns/mgr-priv.h
@@ -39,6 +39,10 @@
#include <kns/kns-mgr-priv.h>
#endif
+#ifndef _h_libs_kns_tls_priv_
+#include "tls-priv.h"
+#endif
+
#ifdef __cplusplus
extern "C" {
#endif
@@ -51,15 +55,15 @@ struct KNSManager
{
KRefcount refcount;
- struct String const * http_proxy;
-
struct String const *aws_access_key_id;
struct String const *aws_secret_access_key;
struct String const *aws_region;
struct String const *aws_output;
struct HttpRetrySpecs retry_specs;
-
+
+ KTLSGlobals tlsg;
+
int32_t conn_timeout;
int32_t conn_read_timeout;
int32_t conn_write_timeout;
@@ -68,11 +72,12 @@ struct KNSManager
uint32_t maxTotalWaitForReliableURLs_ms;
- uint16_t http_proxy_port;
-
uint8_t maxNumberOfRetriesOnFailureForReliableURLs;
bool http_proxy_enabled; /* TBD - does this need to be static today? */
+ bool http_proxy_only; /* no direct connection - proxy only */
+ HttpProxy * http_proxy;
+
bool verbose;
};
diff --git a/libs/kns/stream-priv.h b/libs/kns/stream-priv.h
index 12575f8..82b7073 100644
--- a/libs/kns/stream-priv.h
+++ b/libs/kns/stream-priv.h
@@ -24,6 +24,12 @@
*
*/
+#ifndef _h_libs_kns_stream_priv_
+#define _h_libs_kns_stream_priv_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
/* VDB.v2 is end of life, and introducing new object types is
dangerous. For our purposes, just call a stream a file */
@@ -38,3 +44,9 @@
#ifndef rcUrl
#define rcUrl rcPath
#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _h_libs_kns_stream_priv_ */
diff --git a/libs/kns/sysmgr.h b/libs/kns/sysmgr.h
index 03928c4..1e45eed 100644
--- a/libs/kns/sysmgr.h
+++ b/libs/kns/sysmgr.h
@@ -38,4 +38,4 @@ void KNSManagerCleanup();
}
#endif
-#endif
\ No newline at end of file
+#endif
diff --git a/interfaces/kproc/timeout.h b/libs/kns/tls-priv.h
similarity index 63%
copy from interfaces/kproc/timeout.h
copy to libs/kns/tls-priv.h
index 738c9c3..767345b 100644
--- a/interfaces/kproc/timeout.h
+++ b/libs/kns/tls-priv.h
@@ -24,45 +24,57 @@
*
*/
-#ifndef _h_kproc_timeout_
-#define _h_kproc_timeout_
+#ifndef _h_libs_kns_tls_priv_
+#define _h_libs_kns_tls_priv_
-#ifndef _h_kproc_extern_
-#include <kproc/extern.h>
+#ifndef _h_kns_tls_
+#include <kns/tls.h>
#endif
-#ifndef _h_klib_defs_
-#include <klib/defs.h>
+#if ! defined ( MBEDTLS_CONFIG_FILE )
+#include <mbedtls/config.h>
+#else
+#include MBEDTLS_CONFIG_FILE
#endif
-#ifndef _h_os_native_
-#include <os-native.h> /* struct timeout_t */
-#endif
+#include <mbedtls/net.h>
+#include <mbedtls/debug.h>
+#include <mbedtls/ssl.h>
+#include <mbedtls/aes.h>
+#include <mbedtls/entropy.h>
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/error.h>
+#include <mbedtls/certs.h>
#ifdef __cplusplus
extern "C" {
#endif
+struct KConfig;
+
/*--------------------------------------------------------------------------
- * system specific timeout object
+ * KTLSGlobals
*/
-typedef struct timeout_t timeout_t;
-
+typedef struct KTLSGlobals KTLSGlobals;
+struct KTLSGlobals
+{
+ mbedtls_x509_crt cacert;
+ mbedtls_ctr_drbg_context ctr_drbg;
+ mbedtls_entropy_context entropy;
+ mbedtls_ssl_config config;
+};
/* Init
- * initialize a timeout in milliseconds
*/
-KPROC_EXTERN rc_t CC TimeoutInit ( timeout_t *tm, uint32_t msec );
+rc_t KTLSGlobalsInit ( KTLSGlobals * tlsg, struct KConfig const * kfg );
-
-/* Prepare
- * ensures that a timeout is prepared with an absolute value
-*/
-KPROC_EXTERN rc_t CC TimeoutPrepare ( timeout_t *tm );
+/* Whack
+ */
+void KTLSGlobalsWhack ( KTLSGlobals * self );
#ifdef __cplusplus
}
#endif
-#endif /* _h_kproc_timeout_ */
+#endif /* _h_libs_kns_tls_priv_ */
diff --git a/libs/kns/tls.c b/libs/kns/tls.c
new file mode 100644
index 0000000..3b50efd
--- /dev/null
+++ b/libs/kns/tls.c
@@ -0,0 +1,1087 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+struct KTLSStream;
+#define KSTREAM_IMPL struct KTLSStream
+
+#include <kns/extern.h>
+#include <kns/manager.h>
+#include <kns/tls.h>
+#include <kns/impl.h>
+#include <kns/socket.h>
+#include <klib/rc.h>
+#include <klib/status.h>
+#include <klib/debug.h>
+#include <klib/log.h>
+#include <klib/text.h>
+#include <klib/namelist.h>
+#include <kproc/timeout.h>
+#include <kfg/config.h>
+
+#include <os-native.h>
+
+#include <sysalloc.h>
+
+#include "mgr-priv.h"
+#include "stream-priv.h"
+
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+
+#if ! defined ( MBEDTLS_CONFIG_FILE )
+#include <mbedtls/config.h>
+#else
+#include MBEDTLS_CONFIG_FILE
+#endif
+
+#include <mbedtls/net.h>
+#include <mbedtls/debug.h>
+#include <mbedtls/ssl.h>
+#include <mbedtls/aes.h>
+#include <mbedtls/entropy.h>
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/error.h>
+#include <mbedtls/certs.h>
+
+static const char ca_crt_ncbi1 [] =
+ "-----BEGIN CERTIFICATE-----\r\n"
+ "MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh\r\n"
+ "MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3\r\n"
+ "d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD\r\n"
+ "QTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT\r\n"
+ "MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j\r\n"
+ "b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG\r\n"
+ "9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsB\r\n"
+ "CSDMAZOnTjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97\r\n"
+ "nh6Vfe63SKMI2tavegw5BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt\r\n"
+ "43C/dxC//AH2hdmoRBBYMql1GNXRor5H4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7P\r\n"
+ "T19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y7vrTC0LUq7dBMtoM1O/4\r\n"
+ "gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQABo2MwYTAO\r\n"
+ "BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbR\r\n"
+ "TLtm8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUw\r\n"
+ "DQYJKoZIhvcNAQEFBQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/Esr\r\n"
+ "hMAtudXH/vTBH1jLuG2cenTnmCmrEbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg\r\n"
+ "06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIttep3Sp+dWOIrWcBAI+0tKIJF\r\n"
+ "PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls\r\n"
+ "YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk\r\n"
+ "CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4wOTAeBggrBgEFBQcD\r\n"
+ "BAYIKwYBBQUHAwEGCCsGAQUFBwMDDBdEaWdpQ2VydCBHbG9iYWwgUm9vdCBDQQ==\r\n"
+ "-----END CERTIFICATE-----\r\n";
+
+static const char ca_crt_ncbi2 [] =
+ "-----BEGIN CERTIFICATE-----\r\n"
+ "MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs\r\n"
+ "MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3\r\n"
+ "d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j\r\n"
+ "ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL\r\n"
+ "MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3\r\n"
+ "LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug\r\n"
+ "RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm\r\n"
+ "+9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW\r\n"
+ "PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM\r\n"
+ "xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB\r\n"
+ "Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3\r\n"
+ "hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg\r\n"
+ "EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF\r\n"
+ "MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA\r\n"
+ "FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec\r\n"
+ "nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z\r\n"
+ "eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF\r\n"
+ "hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2\r\n"
+ "Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe\r\n"
+ "vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep\r\n"
+ "+OkuE6N36B9KMEQwHgYIKwYBBQUHAwQGCCsGAQUFBwMBBggrBgEFBQcDAwwiRGln\r\n"
+ "aUNlcnQgSGlnaCBBc3N1cmFuY2UgRVYgUm9vdCBDQQ==\r\n"
+ "-----END CERTIFICATE-----\r\n";
+
+
+/*--------------------------------------------------------------------------
+ * mbedtls_strerror
+ */
+
+static
+const char * mbedtls_strerror2 ( int err )
+{
+ static char buffer [ 256 ];
+ mbedtls_strerror ( err, buffer, sizeof buffer );
+ return buffer;
+}
+
+/*--------------------------------------------------------------------------
+ * KTLSGlobals
+ */
+
+static
+rc_t tlsg_seed_rng ( KTLSGlobals *self )
+{
+ int ret;
+ const char *pers = "vdb_tls_client";
+ const size_t pers_size = sizeof "vdb_tls_client" - 1;
+
+ STATUS ( STAT_QA, "Seeding the random number generator\n" );
+
+ ret = mbedtls_ctr_drbg_seed ( &self -> ctr_drbg, mbedtls_entropy_func,
+ &self -> entropy, ( const unsigned char * ) pers, pers_size );
+
+ if ( ret != 0 )
+ {
+ rc_t rc = RC ( rcKrypto, rcRng, rcInitializing, rcSeed, rcFailed );
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_ctr_drbg_seed returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ ) );
+ return rc;
+ }
+
+ return 0;
+}
+
+static
+rc_t tlsg_init_certs ( KTLSGlobals *self, const KConfig * kfg )
+{
+ int ret;
+
+ rc_t rc = 0;
+ uint32_t nidx, num_certs = 0;
+
+ static char * node_names [] =
+ {
+ "/tls/ca.crt",
+ "/tls/self-signed"
+ };
+
+ STATUS ( STAT_QA, "Loading CA root certificates\n" );
+
+ for ( nidx = 0; rc == 0 && nidx < sizeof node_names / sizeof node_names [ 0 ]; ++ nidx )
+ {
+ const KConfigNode * ca_crt;
+ const char * node_name = node_names [ nidx ];
+
+ rc = KConfigOpenNodeRead ( kfg, & ca_crt, "%s", node_name );
+ if ( rc != 0 )
+ rc = 0;
+ else
+ {
+ KNamelist * cert_names;
+
+ STATUS ( STAT_PRG, "Listing CA root certificates\n" );
+ rc = KConfigNodeListChildren ( ca_crt, & cert_names );
+ if ( rc != 0 )
+ {
+ rc = ResetRCContext ( rc, rcKrypto, rcToken, rcInitializing );
+ PLOGERR ( klogInt, ( klogInt, rc
+ , "failed to list config node '$(node)'"
+ , "node=%s"
+ , node_name
+ ) );
+ }
+ else
+ {
+ uint32_t count;
+
+ STATUS ( STAT_GEEK, "Counting CA root certificates\n" );
+ rc = KNamelistCount ( cert_names, & count );
+ if ( rc != 0 )
+ {
+ rc = ResetRCContext ( rc, rcKrypto, rcToken, rcInitializing );
+ PLOGERR ( klogInt, ( klogInt, rc
+ , "failed to count names in config node '$(node)'"
+ , "node=%s"
+ , node_name
+ ) );
+ }
+ else
+ {
+ uint32_t i;
+ String * cert_string;
+ const KConfigNode * root_cert;
+
+ STATUS ( STAT_GEEK, "Found %u names in CA root certificates\n", count );
+ STATUS ( STAT_PRG, "Retrieving names in CA root certificates\n" );
+ for ( i = 0; i < count; ++ i )
+ {
+ const char * cert_name;
+
+ STATUS ( STAT_GEEK, "Retrieving name %u in CA root certificates\n", i );
+ rc = KNamelistGet ( cert_names, i, & cert_name );
+ if ( rc != 0 )
+ {
+ rc = ResetRCContext ( rc, rcKrypto, rcToken, rcInitializing );
+ PLOGERR ( klogInt, ( klogInt, rc
+ , "failed to read cert $(idx) in config node '$(node)'"
+ , "node=%s,idx=%u"
+ , node_name
+ , i
+ ) );
+ break;
+ }
+
+ STATUS ( STAT_GEEK, "Retrieving node '%s' from CA root certificates\n", cert_name );
+ rc = KConfigNodeOpenNodeRead ( ca_crt, & root_cert, "%s", cert_name );
+ if ( rc != 0 )
+ {
+ rc = ResetRCContext ( rc, rcKrypto, rcToken, rcInitializing );
+ PLOGERR ( klogInt, ( klogInt, rc
+ , "failed to read node for cert '$(name)' in config node '$(node)'"
+ , "node=%s,name=%s"
+ , node_name
+ , cert_name
+ ) );
+ break;
+ }
+
+ STATUS ( STAT_GEEK, "Retrieving text for node '%s' from CA root certificates\n", cert_name );
+ rc = KConfigNodeReadString ( root_cert, & cert_string );
+ KConfigNodeRelease ( root_cert );
+
+ if ( rc != 0 )
+ {
+ rc = ResetRCContext ( rc, rcKrypto, rcToken, rcInitializing );
+ PLOGERR ( klogInt, ( klogInt, rc
+ , "failed to read node text for cert '$(name)' in config node '$(node)'"
+ , "node=%s,name=%s"
+ , node_name
+ , cert_name
+ ) );
+ break;
+ }
+
+
+ /* these guys take a length, so presumably the string is not NUL terminated.
+ yet, the first thing they do is see if the NUL is included in the length! */
+ STATUS ( STAT_GEEK, "Parsing text for node '%s' from CA root certificates\n", cert_name );
+ ret = mbedtls_x509_crt_parse ( &self -> cacert,
+ ( const unsigned char * ) cert_string -> addr, cert_string -> size + 1 );
+
+ StringWhack ( cert_string );
+
+ if ( ret < 0 )
+ {
+ rc = RC ( rcKrypto, rcToken, rcInitializing, rcEncryption, rcFailed );
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_x509_crt_parse returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ ) );
+ break;
+ }
+
+ ++ num_certs;
+ }
+ }
+
+ KNamelistRelease ( cert_names );
+ }
+
+ KConfigNodeRelease ( ca_crt );
+ }
+ }
+
+#if _DEBUGGING
+ {
+ const KConfigNode * ca_crt;
+ rc_t rc2 = KConfigOpenNodeRead ( kfg, & ca_crt, "/tls/path/ca.crt" );
+ if ( rc2 == 0 )
+ {
+ String * ca_crt_path;
+ STATUS ( STAT_GEEK, "Retrieving path to CA root certificate file\n" );
+ rc2 = KConfigNodeReadString ( ca_crt, & ca_crt_path );
+ if ( rc2 == 0 )
+ {
+ STATUS ( STAT_GEEK, "Parsing text from CA root certificate file '%S'\n", ca_crt_path );
+ ret = mbedtls_x509_crt_parse_file ( &self -> cacert, ca_crt_path -> addr );
+ if ( ret < 0 )
+ {
+ PLOGMSG ( klogWarn, ( klogWarn
+ , "mbedtls_x509_crt_parse_file ( '$(path)' ) returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s,path=%S"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ , ca_crt_path
+ ) );
+ }
+ StringWhack ( ca_crt_path );
+ }
+ KConfigNodeRelease ( ca_crt );
+ }
+ }
+#endif
+
+ if ( num_certs == 0 )
+ {
+ STATUS ( STAT_QA, "Parsing text for default CA root certificates\n" );
+ ret = mbedtls_x509_crt_parse ( &self -> cacert,
+ ( const unsigned char * ) ca_crt_ncbi1, sizeof ca_crt_ncbi1 );
+
+ if ( ret < 0 )
+ {
+ rc = RC ( rcKrypto, rcToken, rcInitializing, rcEncryption, rcFailed );
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_x509_crt_parse returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ ) );
+ }
+ else
+ {
+ num_certs = 1;
+
+ ret = mbedtls_x509_crt_parse ( &self -> cacert,
+ ( const unsigned char * ) ca_crt_ncbi2, sizeof ca_crt_ncbi2 );
+
+ if ( ret >= 0 )
+ ++ num_certs;
+ else
+ {
+ rc = RC ( rcKrypto, rcToken, rcInitializing, rcEncryption, rcFailed );
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_x509_crt_parse returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ ) );
+ }
+ }
+ }
+
+ return num_certs == 0 ? rc : 0;
+}
+
+static
+rc_t tlsg_setup ( KTLSGlobals * self )
+{
+ int ret;
+
+ STATUS ( STAT_QA, "Configuring SSl defaults\n" );
+
+ ret = mbedtls_ssl_config_defaults ( &self -> config,
+ MBEDTLS_SSL_IS_CLIENT,
+ MBEDTLS_SSL_TRANSPORT_STREAM,
+ MBEDTLS_SSL_PRESET_DEFAULT );
+
+ if ( ret != 0 )
+ {
+ rc_t rc = RC ( rcKrypto, rcNoTarg, rcFormatting, rcEncryption, rcFailed );
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_ssl_config_defaults returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ ) );
+ return rc;
+ }
+
+ mbedtls_ssl_conf_authmode( &self -> config, MBEDTLS_SSL_VERIFY_REQUIRED );
+ mbedtls_ssl_conf_ca_chain( &self -> config, &self -> cacert, NULL );
+ mbedtls_ssl_conf_rng( &self -> config, mbedtls_ctr_drbg_random, &self -> ctr_drbg );
+
+ return 0;
+}
+
+/* Init
+ */
+rc_t KTLSGlobalsInit ( KTLSGlobals * tlsg, const KConfig * kfg )
+{
+ rc_t rc;
+
+ mbedtls_x509_crt_init ( &tlsg -> cacert );
+ mbedtls_ctr_drbg_init ( &tlsg -> ctr_drbg );
+ mbedtls_entropy_init ( &tlsg -> entropy );
+ mbedtls_ssl_config_init ( &tlsg -> config );
+
+ rc = tlsg_seed_rng ( tlsg );
+ if ( rc == 0 )
+ {
+ rc = tlsg_init_certs ( tlsg, kfg );
+ if ( rc == 0 )
+ rc = tlsg_setup ( tlsg );
+ }
+
+ return rc;
+}
+
+/* Whack
+ */
+void KTLSGlobalsWhack ( KTLSGlobals * self )
+{
+ mbedtls_ssl_config_free ( &self -> config );
+ mbedtls_entropy_free ( &self -> entropy );
+ mbedtls_ctr_drbg_free ( &self -> ctr_drbg );
+ mbedtls_x509_crt_free ( &self -> cacert );
+
+ memset ( self, 0, sizeof * self );
+}
+
+/*--------------------------------------------------------------------------
+ * KTLSStream
+ */
+
+struct KTLSStream
+{
+ KStream dad;
+
+ /* hold a reference to manager */
+ const KNSManager * mgr;
+
+ /* hold a reference to ciphertext object */
+ KStream * ciphertext;
+
+ /* optional timeout */
+ struct timeout_t * tm;
+
+ /* mbed library specific data */
+ mbedtls_ssl_context ssl;
+
+ /* error returned from ciphertext stream */
+ rc_t rd_rc;
+ rc_t wr_rc;
+};
+
+static
+void KTLSStreamDestroy ( KTLSStream *self )
+{
+ /* tear down all of the stuff created during Make */
+ mbedtls_ssl_close_notify( &self -> ssl ); /* close connection - this might need to be elsewhere */
+ mbedtls_ssl_free ( &self -> ssl );
+
+ /* release the ciphertext object */
+ KStreamRelease ( self -> ciphertext );
+ self -> ciphertext = NULL;
+}
+
+static
+rc_t CC KTLSStreamWhack ( KTLSStream *self )
+{
+ KTLSStreamDestroy ( self );
+
+ /* release the manager */
+ KNSManagerRelease ( self -> mgr );
+ self -> mgr = NULL;
+
+ /* done */
+ free ( self );
+ return 0;
+}
+
+static
+rc_t CC KTLSStreamRead ( const KTLSStream * cself,
+ void * buffer, size_t bsize, size_t * num_read )
+{
+ int ret;
+ rc_t rc = 0;
+ KTLSStream * self = ( KTLSStream * ) cself;
+
+ if ( self -> ciphertext == NULL )
+ {
+ * num_read = 0;
+ return RC ( rcNS, rcSocket, rcReading, rcSocket, rcInvalid );
+ }
+
+ STATUS ( STAT_QA, "Reading from server..." );
+
+ self -> rd_rc = 0;
+
+ while ( 1 )
+ {
+ /* read through TLS library */
+ ret = mbedtls_ssl_read( &self -> ssl, buffer, bsize );
+
+ /* no error */
+ if ( ret >= 0 )
+ {
+ STATUS ( STAT_PRG, "%u bytes read", ret );
+ break;
+ }
+
+ /* detect error at socket level */
+ if ( self -> rd_rc != 0 )
+ {
+ rc = self -> rd_rc;
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_ssl_read returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ ) );
+
+ ret = 0;
+ self -> rd_rc = 0;
+ break;
+ }
+
+ /* this is a TLS error */
+ switch ( ret )
+ {
+ /* The ret is anything other than the following 3, then the ssl context becomes
+ * becomes unusable and should either be freed or call
+ * mbedtls_ssl_session_reset () before a new connection; current connection
+ * must be closed
+ */
+ case MBEDTLS_ERR_SSL_WANT_READ:
+ case MBEDTLS_ERR_SSL_WANT_WRITE:
+ continue; /* TBD - allow the app to check signals */
+ case MBEDTLS_ERR_SSL_CLIENT_RECONNECT:
+ /* can only happen server-side: When this function return MBEDTLS_ERR_SSL_CLIENT_RECONNECT
+ * (which can only happen server-side), it means that a client
+ * is initiating a new connection using the same source port.
+ * You can either treat that as a connection close and wait
+ * for the client to resend a ClientHello, or directly
+ * continue with \c mbedtls_ssl_handshake() with the same
+ * context (as it has beeen reset internally). Either way, you
+ * should make sure this is seen by the application as a new
+ * connection: application state, if any, should be reset, and
+ * most importantly the identity of the client must be checked
+ * again. WARNING: not validating the identity of the client
+ * again, or not transmitting the new identity to the
+ * application layer, would allow authentication bypass!*/
+ rc = RC ( rcNS, rcSocket, rcReading, rcMode, rcUnsupported );
+ break;
+
+ case MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY: /* check - only place their advanced sample client checks for this */
+ break;
+ case MBEDTLS_ERR_NET_CONN_RESET:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcConnection, rcCanceled );
+ break;
+ case MBEDTLS_ERR_SSL_BAD_INPUT_DATA:
+ rc = RC ( rcNS, rcSocket, rcReading, rcParam, rcInvalid );
+ break;
+ case MBEDTLS_ERR_SSL_INTERNAL_ERROR: /* should never happen */
+ case MBEDTLS_ERR_SSL_UNEXPECTED_MESSAGE:
+ default:
+ rc = RC ( rcNS, rcSocket, rcReading, rcError, rcUnexpected );
+ }
+
+ /* invalidate the stream for any further use */
+ KTLSStreamDestroy ( self );
+ ret = 0;
+ break;
+ }
+
+ *num_read = ret;
+ return rc;
+}
+
+static
+rc_t CC KTLSStreamWrite ( KTLSStream * self,
+ const void * buffer, size_t size, size_t * num_writ )
+{
+ int ret;
+ rc_t rc = 0;
+
+ if ( self -> ciphertext == NULL )
+ {
+ * num_writ = 0;
+ return RC ( rcNS, rcSocket, rcWriting, rcSocket, rcInvalid );
+ }
+
+ STATUS ( STAT_PRG, "Writing %zu bytes to to server\n", size );
+
+ self -> wr_rc = 0;
+
+ while ( 1 )
+ {
+ /* write through TLS library
+ * This function will do partial writes in some cases. If the
+ * return value is non-negative but less than length, the
+ * function must be called again with updated arguments:
+ * buf + ret, len - ret (if ret is the return value) until
+ * it returns a value equal to the last 'len' argument.
+ *
+ * We expect to be called through KStreamWriteAll that will
+ * avoid the issue above.
+ */
+ ret = mbedtls_ssl_write ( &self -> ssl, buffer, size );
+
+ /* no error */
+ if ( ret >= 0 )
+ {
+ STATUS ( STAT_PRG, "%u bytes written", ret );
+ break;
+ }
+
+ /* detect error at socket level */
+ if ( self -> wr_rc != 0 )
+ {
+ rc = self -> wr_rc;
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_ssl_write returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ ) );
+ ret = 0;
+ self -> wr_rc = 0;
+ break;
+ }
+
+ /* this is a TLS error */
+ switch ( ret )
+ {
+ case MBEDTLS_ERR_SSL_WANT_READ:
+ case MBEDTLS_ERR_SSL_WANT_WRITE:
+ continue;
+ case MBEDTLS_ERR_SSL_BAD_INPUT_DATA:
+ rc = RC ( rcNS, rcSocket, rcReading, rcParam, rcInvalid );
+ break;
+ default:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcError, rcUnknown );
+ break;
+ }
+
+ ret = 0;
+ break;
+ }
+
+ * num_writ = ret;
+ return rc;
+}
+
+static
+rc_t CC KTLSStreamTimedRead ( const KTLSStream *cself,
+ void * buffer, size_t bsize, size_t * num_read, struct timeout_t * tm )
+{
+ rc_t rc;
+ KTLSStream * self = ( KTLSStream * ) cself;
+
+ self -> tm = tm;
+ rc = KTLSStreamRead ( self, buffer, bsize, num_read );
+ self -> tm = NULL;
+
+ return rc;
+}
+
+static
+rc_t CC KTLSStreamTimedWrite ( KTLSStream *self,
+ const void * buffer, size_t size, size_t * num_writ, struct timeout_t * tm )
+{
+ rc_t rc;
+
+ self -> tm = tm;
+ rc = KTLSStreamWrite ( self, buffer, size, num_writ );
+ self -> tm = NULL;
+
+ return rc;
+}
+
+
+static KStream_vt_v1 vtKTLSStream =
+{
+ 1, 1,
+
+ KTLSStreamWhack,
+ KTLSStreamRead,
+ KTLSStreamWrite,
+ KTLSStreamTimedRead,
+ KTLSStreamTimedWrite
+};
+
+static
+int CC ktls_net_send ( void *ctx, const unsigned char *buf, size_t len )
+{
+ KTLSStream * self = ctx;
+
+ size_t num_writ;
+
+ if ( self -> tm != NULL )
+ self -> wr_rc = KStreamTimedWriteAll ( self -> ciphertext, buf, len, & num_writ, self -> tm );
+ else
+ self -> wr_rc = KStreamWriteAll ( self -> ciphertext, buf, len, & num_writ );
+
+ if ( self -> wr_rc != 0 )
+ {
+ switch ( GetRCObject ( self -> wr_rc ) )
+ {
+ case rcConnection: /* EPIPE && ECONNRESET */
+ if ( GetRCState ( self -> wr_rc ) == rcCanceled ||
+ GetRCState ( self -> wr_rc ) == rcInterrupted )
+ return MBEDTLS_ERR_NET_CONN_RESET;
+ case rcTransfer: /* EINTR */
+ return MBEDTLS_ERR_SSL_WANT_WRITE;
+ default:
+ return MBEDTLS_ERR_NET_SEND_FAILED;
+ }
+ }
+
+ return ( int ) num_writ;
+}
+
+/* called by mbedtls_ssl_fetch_input */
+static
+int CC ktls_net_recv ( void *ctx, unsigned char *buf, size_t len )
+{
+ size_t num_read;
+ KTLSStream * self = ctx;
+
+ if ( self -> tm != NULL )
+ self -> rd_rc = KStreamTimedRead ( self -> ciphertext, buf, len, & num_read, self -> tm );
+ else
+ self -> rd_rc = KStreamRead ( self -> ciphertext, buf, len, & num_read );
+
+ if ( self -> rd_rc != 0 )
+ {
+ /* TBD - discover if the read timed out - possibly return MBEDTLS_ERR_SSL_WANT_READ */
+ switch ( ( int ) GetRCObject ( self -> rd_rc ) )
+ {
+ case rcData:
+ return MBEDTLS_ERR_SSL_TIMEOUT;
+ case rcConnection: /* EPIPE && ECONNRESET */
+ if ( GetRCState ( self -> rd_rc ) == rcCanceled ||
+ GetRCState ( self -> rd_rc ) == rcInterrupted )
+ return MBEDTLS_ERR_NET_CONN_RESET;
+ case rcTransfer: /* EINTR */
+ return MBEDTLS_ERR_SSL_WANT_READ;
+ default:
+ return MBEDTLS_ERR_NET_RECV_FAILED;
+ }
+ }
+
+ return ( int ) num_read;
+}
+
+static
+rc_t ktls_ssl_setup ( KTLSStream *self, const String *host )
+{
+ int ret;
+ const String * hostz;
+ const KTLSGlobals * tlsg;
+
+ STATUS ( STAT_QA, "Setting up SSL/TLS structure" );
+
+ assert ( self -> mgr != NULL );
+ tlsg = & self -> mgr -> tlsg;
+
+ ret = mbedtls_ssl_setup( &self -> ssl, &tlsg -> config );
+ if ( ret != 0 )
+ {
+ rc_t rc = RC ( rcKrypto, rcSocket, rcFormatting, rcEncryption, rcFailed );
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_ssl_setup returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ ) );
+ return rc;
+ }
+
+ /* HEURISTIC - no memory pages will be less than 4K in size.
+ we can safely test for a NUL byte at the end of the address
+ only if we know that this address is not exactly at a potential
+ page boundary. If the NUL byte would be on a potential page boundary,
+ it is not safe to test and we must reallocate.
+
+ Also, if not NUL terminated.
+
+ Finally, this will make Valgrind lose its mind...
+ */
+
+#if ! _DEBUGGING
+ assert ( host != NULL );
+ assert ( host -> addr != NULL );
+
+ if ( ( ( size_t ) ( & host -> addr [ host -> size ] ) & ( 4096 - 1 ) ) != 0 &&
+ host -> addr [ host -> size ] == 0 )
+ {
+ hostz = host;
+ }
+ else
+#endif
+ {
+ rc_t rc = StringCopy ( & hostz, host );
+ if ( rc != 0 )
+ return rc;
+ }
+
+ ret = mbedtls_ssl_set_hostname( &self -> ssl, hostz -> addr );
+
+ if ( hostz != host )
+ StringWhack ( hostz );
+
+ if ( ret != 0 )
+ {
+ rc_t rc = RC ( rcKrypto, rcSocket, rcFormatting, rcEncryption, rcFailed );
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_ssl_set_hostname returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ ) );
+ return rc;
+ }
+
+
+ mbedtls_ssl_set_bio( &self -> ssl, ( void * ) self, ktls_net_send, ktls_net_recv, NULL );
+
+ return 0;
+}
+
+static
+rc_t ktls_handshake ( KTLSStream *self )
+{
+ int ret;
+
+ STATUS ( STAT_QA, "Performing SSL/TLS handshake...\n" );
+
+ ret = mbedtls_ssl_handshake( &self -> ssl );
+ while ( ret != 0 )
+ {
+ if ( ret != MBEDTLS_ERR_SSL_WANT_READ &&
+ ret != MBEDTLS_ERR_SSL_WANT_WRITE )
+ {
+ rc_t rc = RC ( rcKrypto, rcSocket, rcOpening, rcConnection, rcFailed );
+
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_ssl_handshake returned $(ret) ( $(expl) )"
+ , "ret=%d,expl=%s"
+ , ret
+ , mbedtls_strerror2 ( ret )
+ ) );
+
+ if ( ret == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED )
+ {
+ uint32_t flags = mbedtls_ssl_get_verify_result( &self -> ssl );
+ if ( flags != 0 )
+ {
+ char buf [ 4096 ];
+ mbedtls_x509_crt_verify_info ( buf, sizeof( buf ), " !! ", flags );
+
+ PLOGMSG ( klogSys, ( klogSys
+ , "mbedtls_ssl_get_verify_result returned $(flags) ( $(info) )"
+ , "flags=0x%X,info=%s"
+ , flags
+ , buf
+ ) );
+ }
+ }
+
+ return rc;
+ }
+ ret = mbedtls_ssl_handshake( &self -> ssl );
+ }
+
+ return 0;
+}
+
+
+static
+rc_t KTLSStreamMake ( KTLSStream ** objp, const KNSManager * mgr, const KSocket *ciphertext )
+{
+ rc_t rc;
+ KTLSStream * obj;
+
+ STATUS ( STAT_PRG, "%s\n", __func__ );
+
+ obj = calloc ( 1, sizeof * obj );
+ if ( obj == NULL )
+ rc = RC ( rcNS, rcMgr, rcAllocating, rcMemory, rcExhausted );
+ else
+ {
+ /* initialize the stream parent */
+ STATUS ( STAT_PRG, "%s - initializing KStream\n", __func__ );
+ rc = KStreamInit ( & obj -> dad, ( const KStream_vt* ) & vtKTLSStream, "KTLSStream", "", true, true );
+ if ( rc == 0 )
+ {
+ STATUS ( STAT_GEEK, "%s - attaching to KNSManager\n", __func__ );
+ rc = KNSManagerAddRef ( mgr );
+ if ( rc == 0 )
+ {
+ STATUS ( STAT_GEEK, "%s - accessing KStream from socket\n", __func__ );
+ rc = KSocketGetStream ( ciphertext, & obj -> ciphertext );
+ if ( rc == 0 )
+ {
+ obj -> mgr = mgr;
+
+ STATUS ( STAT_PRG, "%s - initializing tls wrapper\n", __func__ );
+ mbedtls_ssl_init ( &obj -> ssl );
+
+ * objp = obj;
+ return 0;
+ }
+
+ KNSManagerRelease ( mgr );
+ }
+ }
+
+ free ( obj );
+ }
+
+ DBGMSG ( DBG_KNS, DBG_FLAG ( DBG_KNS ), ( "Failed to create TLS stream: %R\n", rc ) );
+
+ * objp = NULL;
+ return rc;
+}
+
+
+/* MakeTLSStream
+ * create a TLS wrapper upon an existing KStream
+ *
+ * data read from or written to TLS stream is in plaintext
+ * data from to or wrapped stream is in ciphertext
+ *
+ * "plaintext" [ OUT ] - return parameter for TLS stream
+ *
+ * "ciphertext" [ IN ] - existing stream of ciphertext
+ *
+ * "host" [ IN ] - host dns name
+ */
+LIB_EXPORT rc_t CC KNSManagerMakeTLSStream ( const KNSManager * self,
+ KTLSStream ** plaintext, const KSocket * ciphertext, const String * host )
+{
+ rc_t rc;
+
+ if ( plaintext == NULL )
+ rc = RC ( rcNS, rcMgr, rcAllocating, rcParam, rcNull );
+ else
+ {
+ if ( self == NULL )
+ rc = RC ( rcNS, rcMgr, rcAllocating, rcSelf, rcNull );
+ else if ( ciphertext == NULL )
+ rc = RC ( rcNS, rcMgr, rcAllocating, rcParam, rcNull );
+ else
+ {
+ KTLSStream * ktls;
+
+ /* create KTLSStream object */
+ rc = KTLSStreamMake ( & ktls, self, ciphertext );
+ if ( rc == 0 )
+ {
+ /* perform all initialization possible with information given */
+ rc = ktls_ssl_setup ( ktls, host );
+ if ( rc == 0 )
+ {
+ rc = ktls_handshake ( ktls );
+ if ( rc == 0 )
+ {
+ ktls -> mgr = self;
+ *plaintext = ktls;
+ return 0;
+ }
+ }
+
+ KTLSStreamWhack ( ktls );
+ }
+ }
+
+ * plaintext = NULL;
+ }
+
+ return rc;
+}
+
+
+/* AddRef
+ * Release
+ * ignores NULL references
+ */
+LIB_EXPORT rc_t CC KTLSStreamAddRef ( const KTLSStream * self )
+{
+ /* works because address of self == address of self -> dad,
+ and if self == NULL, & self -> dad == NULL */
+ return KStreamAddRef ( & self -> dad );
+}
+
+LIB_EXPORT rc_t CC KTLSStreamRelease ( const KTLSStream * self )
+{
+ /* works because address of self == address of self -> dad,
+ and if self == NULL, & self -> dad == NULL */
+ return KStreamRelease ( & self -> dad );
+}
+
+/* VerifyCACert
+ */
+LIB_EXPORT rc_t CC KTLSStreamVerifyCACert ( const KTLSStream * self )
+{
+ rc_t rc = 0;
+
+ STATUS ( STAT_QA, "Verifying peer X.509 certificate..." );
+
+ if ( self == NULL )
+ rc = RC ( rcKrypto, rcToken, rcValidating, rcSelf, rcNull );
+ else
+ {
+ uint32_t flags = mbedtls_ssl_get_verify_result( &self -> ssl );
+ if ( flags != 0 )
+ {
+ char buf [ 4096 ];
+ rc_t rc = RC ( rcKrypto, rcToken, rcValidating, rcEncryption, rcFailed );
+
+ mbedtls_x509_crt_verify_info ( buf, sizeof( buf ), " ! ", flags );
+
+ PLOGERR ( klogSys, ( klogSys, rc
+ , "mbedtls_ssl_get_verify_result returned $(flags) ( $(info) )"
+ , "flags=0x%X,info=%s"
+ , flags
+ , buf
+ ) );
+ return rc;
+ }
+ }
+
+ return rc;
+}
+
+
+/* GetStream
+ * reference-counted cast operation
+ * creates a reference to a KStream from a KTLSStream
+ */
+LIB_EXPORT rc_t CC KTLSStreamGetStream ( const KTLSStream * self,
+ struct KStream ** strm )
+{
+ rc_t rc;
+
+ if ( strm == NULL )
+ rc = RC ( rcNS, rcStream, rcValidating, rcParam, rcNull );
+ else
+ {
+ if ( self == NULL )
+ rc = RC ( rcNS, rcStream, rcValidating, rcSelf, rcNull );
+ else if ( self -> dad . vt != ( const KStream_vt* ) & vtKTLSStream )
+ rc = RC (rcNS, rcStream, rcValidating, rcInterface, rcNull );
+ else
+ {
+ /* at this point, the object must be in a fully
+ constructed and validated state. If not, it
+ is an error to access it as a stream. */
+
+ rc = KStreamAddRef ( & self -> dad );
+ if ( rc == 0 )
+ {
+ * strm = & ( ( KTLSStream * ) self ) -> dad;
+ return 0;
+ }
+ }
+
+ * strm = NULL;
+ }
+
+ return rc;
+}
+
diff --git a/libs/kns/unix/syssock.c b/libs/kns/unix/syssock.c
index 39960d8..e3420c3 100644
--- a/libs/kns/unix/syssock.c
+++ b/libs/kns/unix/syssock.c
@@ -40,12 +40,13 @@
#undef ERR
#endif
-#include <klib/debug.h> /* DBGMSG */
+#include <klib/debug.h>
#include <klib/log.h>
#include <klib/out.h>
#include <klib/printf.h>
#include <klib/rc.h>
#include <klib/text.h>
+#include <klib/time.h>
#include <kproc/timeout.h>
@@ -99,7 +100,8 @@ struct KSocket
int32_t write_timeout;
int fd;
- union {
+ union
+ {
struct sockaddr_in v4; /* for ipv4 */
struct sockaddr_in6 v6; /* for ipv6 */
} remote_addr;
@@ -176,167 +178,198 @@ rc_t CC KSocketWhack ( KSocket *self )
}
static
-rc_t HandleErrno ( const char *func_name, unsigned int lineno )
+rc_t KSocketHandleSocknameCall ( int status )
{
- int lerrno;
- rc_t rc = 0;
-
- switch ( lerrno = errno )
+ switch ( status )
{
- case EACCES: /* write permission denied */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcMemory, rcUnauthorized );
- break;
- case EADDRINUSE: /* address is already in use */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcMemory, rcExists );
- break;
- case EADDRNOTAVAIL: /* requested address was not local */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcMemory, rcNotFound );
- break;
- case EAGAIN: /* no more free local ports or insufficient rentries in routing cache */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcNoObj, rcExhausted );
- break;
- case EAFNOSUPPORT: /* address didnt have correct address family in ss_family field */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcName, rcInvalid );
- break;
- case EALREADY: /* socket is non blocking and a previous connection has not yet completed */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcUndefined );
- break;
- case EBADF: /* invalid sock fd */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcInvalid );
- break;
- case ECONNREFUSED: /* remote host refused to allow network connection */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcCanceled );
- break;
- case ECONNRESET: /* connection reset by peer */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcCanceled );
- break;
- case EDESTADDRREQ: /* socket is not connection-mode and no peer address set */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcInvalid );
- break;
- case EFAULT: /* buffer pointer points outside of process's adress space */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcMemory, rcOutofrange );
- break;
- case EINPROGRESS: /* call is in progress */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcUndefined );
- break;
- case EINTR: /* recv interrupted before any data available */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcCanceled );
- break;
- case EINVAL: /* invalid argument */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcParam, rcInvalid );
- break;
- case EISCONN: /* connected already */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcExists );
- break;
- case ELOOP: /* too many symbolic links in resolving addr */
- rc = RC ( rcNS, rcNoTarg, rcResolving, rcLink, rcExcessive );
- break;
- case EMFILE: /* process file table overflow */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcProcess, rcExhausted );
- break;
- case EMSGSIZE: /* msg size too big */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcMessage, rcExcessive );
- break;
- case ENAMETOOLONG: /* addr name is too long */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcName, rcExcessive );
- break;
- case ENETUNREACH: /* network is unreachable */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcNotAvailable );
- break;
- case ENOBUFS: /* output queue for a network connection was full.
- ( wont typically happen in linux. Packets are just silently dropped */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcInterrupted );
- break;
- case ENOENT: /* file does not exist */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcNotFound );
- break;
- case ENOMEM: /* Could not allocate memory */
- rc = RC ( rcNS, rcNoTarg, rcAllocating, rcMemory, rcExhausted );
- break;
- case ENOTCONN: /* socket has not been connected */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcInvalid );
- break;
- case ENOTDIR: /* component of path is not a directory */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcDirEntry, rcInvalid );
- break;
- case ENOTSOCK: /* sock fd does not refer to socket */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcId, rcInvalid );
- break;
- case EOPNOTSUPP: /* bits in flags argument is inappropriate */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcParam, rcInvalid );
- break;
+ case EBADF:
+ return RC ( rcNS, rcSocket, rcIdentifying, rcSocket, rcInvalid );
+ case EFAULT:
+ case EINVAL:
+ return RC ( rcNS, rcSocket, rcIdentifying, rcParam, rcInvalid );
+ case ENOBUFS:
+ return RC ( rcNS, rcSocket, rcIdentifying, rcBuffer, rcExhausted );
+ case ENOTCONN:
+ return RC ( rcNS, rcSocket, rcIdentifying, rcSocket, rcNotOpen );
+ case ENOTSOCK:
+ return RC ( rcNS, rcSocket, rcIdentifying, rcSocket, rcIncorrect );
+ }
+
+ return RC ( rcNS, rcSocket, rcIdentifying, rcError, rcUnknown );
+}
+
+static
+rc_t KSocketHandleSocketCall ( int status )
+{
+ switch ( status )
+ {
+ case EACCES:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcUnauthorized );
+ case EAFNOSUPPORT:
+ case EPROTONOSUPPORT:
+ return RC ( rcNS, rcSocket, rcCreating, rcInterface, rcUnsupported );
+ case EINVAL:
+ return RC ( rcNS, rcSocket, rcCreating, rcInterface, rcUnrecognized );
+ case EMFILE:
+ case ENFILE:
+ return RC ( rcNS, rcSocket, rcCreating, rcFileDesc, rcExhausted );
+ case ENOBUFS:
+ return RC ( rcNS, rcSocket, rcCreating, rcBuffer, rcExhausted );
+ case ENOMEM:
+ return RC ( rcNS, rcSocket, rcCreating, rcMemory, rcExhausted );
+ }
+
+ return RC ( rcNS, rcSocket, rcCreating, rcError, rcUnknown );
+}
+
+static
+rc_t KSocketHandleBindCall ( int status )
+{
+ switch ( status )
+ {
+ case EACCES:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcUnauthorized );
+ case EADDRINUSE:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcBusy );
+ case EADDRNOTAVAIL:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcNotAvailable);
+ case EBADF:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcInvalid );
+ case EFAULT:
+ case EINVAL:
+ return RC ( rcNS, rcSocket, rcCreating, rcParam, rcInvalid );
+ case ELOOP:
+ return RC ( rcNS, rcSocket, rcCreating, rcError, rcCorrupt );
+ case ENAMETOOLONG:
+ return RC ( rcNS, rcSocket, rcCreating, rcName, rcExcessive );
+ case ENOENT:
+ return RC ( rcNS, rcSocket, rcCreating, rcFile, rcNotFound );
+ case ENOMEM:
+ return RC ( rcNS, rcSocket, rcCreating, rcMemory, rcExhausted );
+ case ENOTDIR:
+ return RC ( rcNS, rcSocket, rcCreating, rcName, rcIncorrect );
+ case ENOTSOCK:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcIncorrect );
+ case EROFS:
+ return RC ( rcNS, rcSocket, rcCreating, rcDirectory, rcReadonly );
+ }
+
+ return RC ( rcNS, rcSocket, rcCreating, rcError, rcUnknown );
+}
+
+static
+rc_t KSocketHandleConnectCall ( int status )
+{
+ switch ( status )
+ {
+ case EACCES:
case EPERM:
- rc = RC ( rcNS, rcNoTarg, rcReading, rcMemory, rcUnauthorized );
- break;
- case EPIPE: /* local end has been shut down. Will also receive SIGPIPE or MSG_NOSIGNAL */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcCanceled );
- break;
- case EPROTONOSUPPORT: /* specified protocol is not supported */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcAttr, rcUnsupported );
- break;
- case EROFS: /* socket inode on read only file system */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcNoObj, rcReadonly );
- break;
- case ETIMEDOUT: /* timeout */
- rc = RC ( rcNS, rcNoTarg, rcReading, rcConnection, rcNotAvailable );
- break;
-#if ! defined EAGAIN || ! defined EWOULDBLOCK || EAGAIN != EWOULDBLOCK
- case EWOULDBLOCK:
- rc = RC ( rcNS, rcNoTarg, rcReading, rcCmd, rcBusy );
- break;
-#endif
- default:
- rc = RC ( rcNS, rcNoTarg, rcReading, rcError, rcUnknown );
- PLOGERR (klogErr,
- (klogErr, rc, "unknown system error '$(S)($(E))'",
- "S=%!,E=%d", lerrno, lerrno));
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcUnauthorized );
+ case EADDRINUSE:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcBusy );
+ case EADDRNOTAVAIL:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcNotAvailable );
+ case EAFNOSUPPORT:
+ return RC ( rcNS, rcSocket, rcCreating, rcInterface, rcUnsupported );
+ case EALREADY:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcBusy );
+ case EBADF:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcInvalid );
+ case ECONNREFUSED:
+ return RC ( rcNS, rcSocket, rcCreating, rcConnection, rcFailed );
+ case EFAULT:
+ return RC ( rcNS, rcSocket, rcCreating, rcParam, rcInvalid );
+ case EINPROGRESS:
+ return RC ( rcNS, rcSocket, rcCreating, rcConnection, rcBusy );
+ case EINTR:
+ return 0;
+ case EISCONN:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcBusy );
+ case ENETUNREACH:
+ return RC ( rcNS, rcSocket, rcCreating, rcConnection, rcNotAvailable );
+ case ENOTSOCK:
+ return RC ( rcNS, rcSocket, rcCreating, rcSocket, rcIncorrect );
+ case ETIMEDOUT:
+ return RC ( rcNS, rcSocket, rcCreating, rcTimeout, rcExhausted );
}
-
- if ( rc != 0 )
+
+ return RC ( rcNS, rcSocket, rcCreating, rcError, rcUnknown );
+}
+
+static
+rc_t KSocketHandleAcceptCall ( int status )
+{
+ switch ( status )
{
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%R\n", rc ) );
+ case EWOULDBLOCK:
+ return RC ( rcNS, rcSocket, rcOpening, rcData, rcNotAvailable );
+ case EBADF:
+ return RC ( rcNS, rcSocket, rcOpening, rcSocket, rcInvalid );
+ case ECONNABORTED:
+ return RC ( rcNS, rcSocket, rcOpening, rcConnection, rcCanceled );
+ case EFAULT:
+ case EINVAL:
+ return RC ( rcNS, rcSocket, rcOpening, rcData, rcNotAvailable );
+ case EINTR:
+ return 0;
+ case EMFILE:
+ case ENFILE:
+ return RC ( rcNS, rcSocket, rcOpening, rcFileDesc, rcExhausted );
+ case ENOBUFS:
+ return RC ( rcNS, rcSocket, rcOpening, rcBuffer, rcExhausted );
+ case ENOMEM:
+ return RC ( rcNS, rcSocket, rcOpening, rcMemory, rcExhausted );
+ case ENOTSOCK:
+ return RC ( rcNS, rcSocket, rcOpening, rcSocket, rcIncorrect );
+ case EOPNOTSUPP:
+ return RC ( rcNS, rcSocket, rcOpening, rcSocket, rcUnsupported );
+ case EPROTO:
+ case EPERM: /* LINUX ONLY */
+ return RC ( rcNS, rcSocket, rcOpening, rcConstraint, rcViolated );
}
- return rc;
+ return RC ( rcNS, rcSocket, rcOpening, rcError, rcUnknown );
}
+
+
static rc_t KSocketGetEndpointV6 ( const KSocket * self, KEndPoint * ep, bool remote )
{
rc_t rc = 0;
struct sockaddr_in6 addr;
- socklen_t l = sizeof( addr );
+ socklen_t l = sizeof addr;
int res = 0;
- if ( remote )
+ if ( ! remote )
+ res = getsockname( self -> fd, ( struct sockaddr * )&addr, &l );
+ else if ( ! self -> remote_addr_valid )
+ res = getpeername( self -> fd, ( struct sockaddr * )&addr, &l );
+ else
{
- if ( self -> remote_addr_valid )
- {
- /* the remote part was already recorded through calling accept() */
- memcpy ( ep -> u . ipv6 . addr,
- self -> remote_addr . v6 . sin6_addr . s6_addr,
- sizeof ( ep -> u . ipv6 . addr ) );
- ep->u.ipv6.port = ntohs( self -> remote_addr . v6 . sin6_port );
- ep->type = epIPV6;
- return 0;
- }
- else
- res = getpeername( self -> fd, ( struct sockaddr * )&addr, &l );
+ /* the remote part was already recorded through calling accept() */
+ memcpy ( ep -> u . ipv6 . addr,
+ self -> remote_addr . v6 . sin6_addr . s6_addr,
+ sizeof ( ep -> u . ipv6 . addr ) );
+ ep->u.ipv6.port = ntohs( self -> remote_addr . v6 . sin6_port );
+ ep->type = epIPV6;
+ return 0;
}
- else
- res = getsockname( self -> fd, ( struct sockaddr * )&addr, &l );
- if ( res < 0 )
- rc = HandleErrno ( __func__, __LINE__ );
- else
+ if ( res == 0 )
{
memcpy ( ep -> u . ipv6 . addr,
addr . sin6_addr . s6_addr,
sizeof ( ep -> u . ipv6 . addr ) );
ep->u.ipv6.port = ntohs( addr . sin6_port );
ep->type = epIPV6;
+ return 0;
}
+ rc = KSocketHandleSocknameCall ( errno );
+
+ ep -> type = epInvalid;
+
return rc;
}
@@ -345,32 +378,32 @@ static rc_t KSocketGetEndpointV4 ( const KSocket * self, KEndPoint * ep, bool re
{
rc_t rc = 0;
struct sockaddr_in addr;
- socklen_t l = sizeof( addr );
+ socklen_t l = sizeof addr;
int res = 0;
- if ( remote )
+ if ( ! remote )
+ res = getsockname( self -> fd, ( struct sockaddr * )&addr, &l );
+ else if ( ! self -> remote_addr_valid )
+ res = getpeername( self -> fd, ( struct sockaddr * )&addr, &l );
+ else
{
- if ( self -> remote_addr_valid )
- {
- /* the remote part was already recorded through calling accept() */
- addr.sin_addr.s_addr = self -> remote_addr.v4.sin_addr.s_addr;
- addr.sin_port = self -> remote_addr.v4.sin_port;
- }
- else
- res = getpeername( self -> fd, ( struct sockaddr * )&addr, &l );
+ /* the remote part was already recorded through calling accept() */
+ addr.sin_addr.s_addr = self -> remote_addr.v4.sin_addr.s_addr;
+ addr.sin_port = self -> remote_addr.v4.sin_port;
}
- else
- res = getsockname( self -> fd, ( struct sockaddr * )&addr, &l );
- if ( res < 0 )
- rc = HandleErrno ( __func__, __LINE__ );
- else
+ if ( res == 0 )
{
ep->u.ipv4.addr = ntohl( addr.sin_addr.s_addr );
ep->u.ipv4.port = ntohs( addr.sin_port );
ep->type = epIPV4;
+ return 0;
}
+ rc = KSocketHandleSocknameCall ( errno );
+
+ ep -> type = epInvalid;
+
return rc;
}
@@ -382,15 +415,23 @@ static rc_t KSocketGetEndpoint ( const KSocket * self, KEndPoint * ep, bool remo
rc = RC ( rcNS, rcSocket, rcEvaluating, rcParam, rcNull );
else
{
+ ep -> type = epInvalid;
+
if ( self == NULL )
rc = RC ( rcNS, rcSocket, rcEvaluating, rcSelf, rcNull );
else
{
switch( self->type )
{
- case epIPV6 : rc = KSocketGetEndpointV6( self, ep, remote ); break;
- case epIPV4 : rc = KSocketGetEndpointV4( self, ep, remote ); break;
- default : rc = RC ( rcNS, rcSocket, rcEvaluating, rcFunction, rcUnsupported );
+ case epIPV4:
+ rc = KSocketGetEndpointV4( self, ep, remote );
+ break;
+ case epIPV6:
+ rc = KSocketGetEndpointV6( self, ep, remote );
+ break;
+ default:
+ rc = RC ( rcNS, rcSocket, rcEvaluating, rcFunction, rcUnsupported );
+ break;
}
}
}
@@ -434,28 +475,62 @@ rc_t CC KSocketTimedRead ( const KSocket *self,
/* check for error */
if ( revents < 0 )
{
- rc = HandleErrno ( __func__, __LINE__ );
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedRead socket_wait returned '%s'\n", self, strerror(errno) ) );
+ switch ( errno )
+ {
+ case EFAULT:
+ case EINVAL:
+ rc = RC ( rcNS, rcSocket, rcReading, rcParam, rcInvalid );
+ break;
+ case EINTR:
+ rc = RC ( rcNS, rcSocket, rcReading, rcTransfer, rcInterrupted );
+ break;
+ case ENOMEM:
+ rc = RC ( rcNS, rcSocket, rcReading, rcMemory, rcExhausted );
+ break;
+ default:
+ rc = RC ( rcNS, rcSocket, rcReading, rcError, rcUnknown );
+ break;
+ }
+
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedRead socket_wait returned '%!'\n", self, errno ) );
return rc;
}
+
+ /* no error in errno - check for other potential errors */
if ( ( revents & ( POLLERR | POLLNVAL ) ) != 0 )
{
if ( ( revents & POLLERR ) != 0)
{
int optval = 0;
socklen_t optlen = sizeof optval;
- if ( ( getsockopt ( self -> fd, SOL_SOCKET, SO_ERROR, & optval, & optlen ) == 0 )
- && optval > 0)
+ if ( ( getsockopt ( self -> fd, SOL_SOCKET, SO_ERROR, & optval, & optlen ) == 0 ) && optval > 0 )
{
errno = optval;
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedRead socket_wait/getsockopt returned '%s'\n",
- self, strerror(optval) ) );
- return HandleErrno(__func__, __LINE__);
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedRead socket_wait/getsockopt returned '%!'\n",
+ self, optval ) );
+ switch ( errno )
+ {
+ case EFAULT:
+ case EINVAL:
+ rc = RC ( rcNS, rcSocket, rcReading, rcParam, rcInvalid );
+ break;
+ case EINTR:
+ rc = RC ( rcNS, rcSocket, rcReading, rcTransfer, rcInterrupted );
+ break;
+ case ENOMEM:
+ rc = RC ( rcNS, rcSocket, rcReading, rcMemory, rcExhausted );
+ break;
+ default:
+ rc = RC ( rcNS, rcSocket, rcReading, rcError, rcUnknown );
+ break;
+ }
+
+ return rc;
}
}
DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedRead socket_wait returned POLLERR | POLLNVAL\n", self ) );
- return RC ( rcNS, rcStream, rcReading, rcNoObj, rcUnknown );
+ return RC ( rcNS, rcSocket, rcReading, rcError, rcUnknown );
}
/* check for read availability */
@@ -467,7 +542,39 @@ rc_t CC KSocketTimedRead ( const KSocket *self,
* num_read = count;
return 0;
}
- rc = HandleErrno ( __func__, __LINE__ );
+
+ switch ( errno )
+ {
+ case EWOULDBLOCK:
+ rc = RC ( rcNS, rcSocket, rcReading, rcData, rcNotAvailable );
+ break;
+ case EBADF:
+ rc = RC ( rcNS, rcSocket, rcReading, rcSocket, rcInvalid );
+ break;
+ case ECONNREFUSED:
+ rc = RC ( rcNS, rcSocket, rcReading, rcConnection, rcFailed );
+ break;
+ case EFAULT:
+ case EINVAL:
+ rc = RC ( rcNS, rcSocket, rcReading, rcParam, rcInvalid );
+ break;
+ case EINTR:
+ rc = RC ( rcNS, rcSocket, rcReading, rcTransfer, rcInterrupted );
+ break;
+ case ENOMEM:
+ rc = RC ( rcNS, rcSocket, rcReading, rcMemory, rcExhausted );
+ break;
+ case ENOTCONN:
+ rc = RC ( rcNS, rcSocket, rcReading, rcConnection, rcInvalid );
+ break;
+ case ENOTSOCK:
+ rc = RC ( rcNS, rcSocket, rcReading, rcSocket, rcIncorrect );
+ break;
+ default:
+ rc = RC ( rcNS, rcSocket, rcReading, rcError, rcUnknown );
+ break;
+ }
+
DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedRead recv returned count %d\n", self, count ) );
return rc;
}
@@ -483,9 +590,8 @@ rc_t CC KSocketTimedRead ( const KSocket *self,
/* anything else in revents is an error */
if ( ( revents & ~ POLLIN ) != 0 && errno != 0 )
{
- rc = HandleErrno ( __func__, __LINE__ );
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedRead error '%s'\n", self, strerror ( errno ) ) );
- return rc;
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedRead error '%!'\n", self, errno ) );
+ return RC ( rcNS, rcSocket, rcReading, rcError, rcUnknown );
}
/* finally, call this a timeout */
@@ -532,11 +638,28 @@ rc_t CC KSocketTimedWrite ( KSocket *self,
/* check for error */
if ( revents < 0 )
{
- rc = HandleErrno ( __func__, __LINE__ );
+ switch ( errno )
+ {
+ case EFAULT:
+ case EINVAL:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcParam, rcInvalid );
+ break;
+ case EINTR:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcTransfer, rcInterrupted );
+ break;
+ case ENOMEM:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcMemory, rcExhausted );
+ break;
+ default:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcError, rcUnknown );
+ break;
+ }
+
assert ( rc != 0 );
DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedWrite socket_wait returned '%s'\n", self, strerror ( errno ) ) );
return rc;
}
+
if ( ( revents & ( POLLERR | POLLNVAL ) ) != 0 )
{
DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedWrite socket_wait returned POLLERR | POLLNVAL\n", self ) );
@@ -555,24 +678,69 @@ rc_t CC KSocketTimedWrite ( KSocket *self,
if ( ( revents & ( POLLWRNORM | POLLWRBAND ) ) != 0 )
{
count = send ( self -> fd, buffer, bsize, 0 );
- if ( count >= 0 )
+ if ( count < 0 )
{
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: %d bytes written\n", self, count ) );
- * num_writ = count;
- return 0;
+ switch ( errno )
+ {
+ case EACCES:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcSocket, rcUnauthorized);
+ break;
+ case EWOULDBLOCK:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcData, rcNotAvailable );
+ break;
+ case EBADF:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcSocket, rcInvalid );
+ break;
+ case ECONNRESET:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcConnection, rcCanceled );
+ break;
+ case EINTR:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcTransfer, rcInterrupted );
+ break;
+ case EFAULT:
+ case EINVAL:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcParam, rcInvalid );
+ break;
+ case EMSGSIZE:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcData, rcExcessive );
+ break;
+ case ENOBUFS:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcBuffer, rcExhausted );
+ break;
+ case ENOMEM:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcMemory, rcExhausted );
+ break;
+ case ENOTCONN:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcConnection, rcInvalid );
+ break;
+ case ENOTSOCK:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcSocket, rcIncorrect );
+ break;
+ case EOPNOTSUPP:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcParam, rcIncorrect );
+ break;
+ case EPIPE:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcConnection, rcInterrupted );
+ break;
+ default:
+ rc = RC ( rcNS, rcSocket, rcWriting, rcError, rcUnknown );
+ break;
+ }
+
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedWrite recv returned count %d\n", self, count ) );
+ return rc;
}
- rc = HandleErrno ( __func__, __LINE__ );
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedWrite recv returned count %d\n", self, count ) );
- return rc;
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: %d bytes written\n", self, count ) );
+ * num_writ = count;
+ return 0;
}
/* anything else in revents is an error */
if ( ( revents & ~ POLLOUT ) != 0 && errno != 0 )
{
- rc = HandleErrno ( __func__, __LINE__ );
DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketTimedWrite error '%s'\n", self, strerror ( errno ) ) );
- return rc;
+ return RC ( rcNS, rcSocket, rcWriting, rcError, rcUnknown );
}
/* finally, call this a timeout */
@@ -615,10 +783,41 @@ rc_t KSocketMakePath ( const char * name, char * buf, size_t buf_size )
{
size_t num_writ;
#if 0
+ rc_t rc;
struct passwd* pwd;
+ errno = 0; /* man page claims errno should be set to 0 before call if wanting to check after */
pwd = getpwuid ( geteuid () );
if ( pwd == NULL )
- return HandleErrno ( __func__, __LINE__ );
+ {
+ /* check the context 'rcAccessing' */
+ switch ( errno )
+ {
+ case EBADF:
+ rc = RC ( rcNS, rcFile, rcAccessing, rcFormat, rcInvalid );
+ break;
+ case EINTR:
+ rc = RC ( rcNS, rcFile, rcAccessing, rcConnection, rcInterrupted );
+ break;
+ case EIO:
+ rc = RC ( rcNS, rcFile, rcAccessing, rcNoObj, rcUndefined);
+ break;
+ case EMFILE:
+ case ENFILE:
+ rc = RC ( rcNS, rcFile, rcAccessing, rcFileDesc, rcExhausted );
+ break;
+ case ENOMEM:
+ rc = RC ( rcNS, rcFile, rcAccessing, rcMemory, rcExhausted );
+ break;
+ case ERANGE:
+ rc = RC ( rcNS, rcFile, rcAccessing, rcBuffer, rcInsufficient );
+ break;
+ default:
+ rc = RC ( rcNS, rcFile, rcAccessing, rcError, rcUnknown );
+ break;
+ }
+
+ return rc;
+ }
return string_printf ( buf, buf_size, & num_writ, "%s/.ncbi/%s", pwd -> pw_dir, name );
#else
@@ -631,80 +830,66 @@ rc_t KSocketMakePath ( const char * name, char * buf, size_t buf_size )
}
static
-rc_t KSocketConnectIPv4 ( KSocket *self, int32_t retryTimeout, const KEndPoint *from, const KEndPoint *to )
+rc_t KSocketConnectIPv4 ( KSocket *self, const KEndPoint *from, const KEndPoint *to )
{
rc_t rc = 0;
- uint32_t retry_count = 0;
- struct sockaddr_in ss_from, ss_to;
- memset ( & ss_from, 0, sizeof ss_from );
- if ( from != NULL )
+ /* create the OS socket */
+ self -> fd = socket ( AF_INET, SOCK_STREAM, 0 );
+ if ( self -> fd < 0 )
+ rc = KSocketHandleSocketCall ( errno );
+ else
{
- ss_from . sin_family = AF_INET;
- ss_from . sin_addr . s_addr = htonl ( from -> u . ipv4 . addr );
- ss_from . sin_port = htons ( from -> u . ipv4 . port );
- }
+ /* disable nagle algorithm */
+ int flag = 1;
+ setsockopt ( self -> fd, IPPROTO_TCP, TCP_NODELAY, ( char* ) & flag, sizeof flag );
- memset ( & ss_to, 0, sizeof ss_to );
- ss_to . sin_family = AF_INET;
- ss_to . sin_addr . s_addr = htonl ( to -> u . ipv4 . addr );
- ss_to . sin_port = htons ( to -> u . ipv4 . port );
-
- do
- {
- /* create the OS socket */
- self -> fd = socket ( AF_INET, SOCK_STREAM, 0 );
- if ( self -> fd < 0 )
- rc = HandleErrno ( __func__, __LINE__ );
- else
+ struct sockaddr_in ss;
+ memset ( & ss, 0, sizeof ss );
+ ss . sin_family = AF_INET;
+ if ( from != NULL )
{
- /* disable nagle algorithm */
- int flag = 1;
- setsockopt ( self -> fd, IPPROTO_TCP, TCP_NODELAY, ( char* ) & flag, sizeof flag );
+ ss . sin_addr . s_addr = htonl ( from -> u . ipv4 . addr );
+ ss . sin_port = htons ( from -> u . ipv4 . port );
+ }
- /* bind */
- if ( from != NULL && bind ( self -> fd, ( struct sockaddr* ) & ss_from, sizeof ss_from ) != 0 )
- rc = HandleErrno ( __func__, __LINE__ );
+ /* bind */
+ if ( bind ( self -> fd, ( struct sockaddr* ) & ss, sizeof ss ) != 0 )
+ rc = KSocketHandleBindCall ( errno );
- if ( rc == 0 )
+ if ( rc == 0 )
+ {
+ ss . sin_port = htons ( to -> u . ipv4 . port );
+ ss . sin_addr . s_addr = htonl ( to -> u . ipv4 . addr );
+
+ /* connect */
+ if ( connect ( self -> fd, ( struct sockaddr* ) & ss, sizeof ss ) != 0 )
+#warning "have an issue with EINTR here and other places"
+ rc = KSocketHandleConnectCall ( errno );
+ else
{
- /* connect */
- if ( connect ( self -> fd, ( struct sockaddr* ) & ss_to, sizeof ss_to ) == 0 )
- {
- /* set non-blocking mode */
- flag = fcntl ( self -> fd, F_GETFL );
- fcntl ( self -> fd, F_SETFL, flag | O_NONBLOCK );
- return 0;
- }
- rc = HandleErrno ( __func__, __LINE__ );
+ /* set non-blocking mode */
+ flag = fcntl ( self -> fd, F_GETFL );
+ fcntl ( self -> fd, F_SETFL, flag | O_NONBLOCK );
+ return 0;
}
-
- /* dump socket */
- close ( self -> fd );
- self -> fd = -1;
- }
-
- /* rc != 0 */
- if (retryTimeout < 0 || retry_count < retryTimeout)
- { /* retry */
- sleep ( 1 );
- ++retry_count;
- rc = 0;
}
+
+ /* dump socket */
+ close ( self -> fd );
+ self -> fd = -1;
}
- while (rc == 0);
-
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketConnectIPv4 timed out\n", self ) );
+
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketConnectIPv4 failed - %R\n", self, rc ) );
return rc;
}
static
-rc_t KSocketConnectIPv6 ( KSocket *self, int32_t retryTimeout, const KEndPoint *from, const KEndPoint *to )
+rc_t KSocketConnectIPv6 ( KSocket *self, const KEndPoint *from, const KEndPoint *to )
{
rc_t rc = 0;
- uint32_t retry_count = 0;
struct sockaddr_in6 ss_from, ss_to;
memset ( & ss_from, 0, sizeof ss_from );
@@ -724,105 +909,80 @@ rc_t KSocketConnectIPv6 ( KSocket *self, int32_t retryTimeout, const KEndPoint *
sizeof ( to -> u . ipv6 . addr ) );
ss_to . sin6_port = htons ( to -> u . ipv6 . port );
- do
+ /* create the OS socket */
+ self -> fd = socket ( AF_INET6, SOCK_STREAM, 0 );
+ if ( self -> fd < 0 )
+ rc = KSocketHandleSocketCall ( errno );
+ else
{
- /* create the OS socket */
- self -> fd = socket ( AF_INET6, SOCK_STREAM, 0 );
- if ( self -> fd < 0 )
- rc = HandleErrno ( __func__, __LINE__ );
- else
- {
- /* disable nagle algorithm */
- int flag = 1;
- setsockopt ( self -> fd, IPPROTO_TCP, TCP_NODELAY, ( char* ) & flag, sizeof flag );
+ /* disable nagle algorithm */
+ int flag = 1;
+ setsockopt ( self -> fd, IPPROTO_TCP, TCP_NODELAY, ( char* ) & flag, sizeof flag );
- /* bind */
- if ( from != NULL && bind ( self -> fd, ( struct sockaddr* ) & ss_from, sizeof ss_from ) != 0 )
- rc = HandleErrno ( __func__, __LINE__ );
-
- if ( rc == 0 )
+ /* bind */
+ if ( from != NULL && bind ( self -> fd, ( struct sockaddr* ) & ss_from, sizeof ss_from ) != 0 )
+ rc = KSocketHandleBindCall ( errno );
+
+ if ( rc == 0 )
+ {
+ /* connect */
+ if ( connect ( self -> fd, ( struct sockaddr* ) & ss_to, sizeof ss_to ) != 0 )
+ rc = KSocketHandleConnectCall ( errno );
+ else
{
- /* connect */
- if ( connect ( self -> fd, ( struct sockaddr* ) & ss_to, sizeof ss_to ) == 0 )
- {
- /* set non-blocking mode */
- flag = fcntl ( self -> fd, F_GETFL );
- fcntl ( self -> fd, F_SETFL, flag | O_NONBLOCK );
- return 0;
- }
- rc = HandleErrno ( __func__, __LINE__ );
+ /* set non-blocking mode */
+ flag = fcntl ( self -> fd, F_GETFL );
+ fcntl ( self -> fd, F_SETFL, flag | O_NONBLOCK );
+ return 0;
}
-
- /* dump socket */
- close ( self -> fd );
- self -> fd = -1;
- }
-
- /* rc != 0 */
- if (retryTimeout < 0 || retry_count < retryTimeout)
- { /* retry */
- sleep ( 1 );
- ++retry_count;
- rc = 0;
}
+
+ /* dump socket */
+ close ( self -> fd );
+ self -> fd = -1;
}
- while (rc == 0);
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketConnectIPv6 timed out\n", self ) );
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketConnectIPv6 failed - %R\n", self, rc ) );
return rc;
}
static
-rc_t KSocketConnectIPC ( KSocket *self, int32_t retryTimeout, const KEndPoint *to )
+rc_t KSocketConnectIPC ( KSocket *self, const KEndPoint *to )
{
rc_t rc = 0;
- uint32_t retry_count = 0;
struct sockaddr_un ss_to;
memset ( & ss_to, 0, sizeof ss_to );
ss_to . sun_family = AF_UNIX;
rc = KSocketMakePath ( to -> u . ipc_name, ss_to . sun_path, sizeof ss_to . sun_path );
- do
+ /* create the OS socket */
+ self -> fd = socket ( AF_UNIX, SOCK_STREAM, 0 );
+ if ( self -> fd < 0 )
+ rc = KSocketHandleSocketCall ( errno );
+ else
{
- /* create the OS socket */
- self -> fd = socket ( AF_UNIX, SOCK_STREAM, 0 );
- if ( self -> fd < 0 )
- rc = HandleErrno ( __func__, __LINE__ );
+ /* connect */
+ if ( connect ( self -> fd, ( struct sockaddr* ) & ss_to, sizeof ss_to ) != 0 )
+ rc = KSocketHandleConnectCall ( errno );
else
- {
- /* connect */
- if ( connect ( self -> fd, ( struct sockaddr* ) & ss_to, sizeof ss_to ) == 0 )
- {
- return 0;
- }
- rc = HandleErrno ( __func__, __LINE__ );
-
- /* dump socket */
- close ( self -> fd );
- self -> fd = -1;
- }
-
- /* rc != 0 */
- if (retryTimeout < 0 || retry_count < retryTimeout)
- { /* retry */
- sleep ( 1 );
- ++retry_count;
- rc = 0;
- }
+ return 0;
+
+ /* dump socket */
+ close ( self -> fd );
+ self -> fd = -1;
}
- while (rc == 0);
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketConnectIPC timed out\n", self ) );
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET), ( "%p: KSocketConnectIPC failed - %R\n", self, rc ) );
return rc;
}
KNS_EXTERN rc_t CC KNSManagerMakeRetryTimedConnection ( struct KNSManager const * self,
- struct KSocket **out, int32_t retryTimeout, int32_t readMillis, int32_t writeMillis,
- struct KEndPoint const *from, struct KEndPoint const *to )
+ struct KSocket ** out, timeout_t * retryTimeout, int32_t readMillis, int32_t writeMillis,
+ struct KEndPoint const * from, struct KEndPoint const * to )
{
rc_t rc;
@@ -832,7 +992,7 @@ KNS_EXTERN rc_t CC KNSManagerMakeRetryTimedConnection ( struct KNSManager const
{
if ( self == NULL )
rc = RC ( rcNS, rcStream, rcConstructing, rcSelf, rcNull );
- else if ( to == NULL )
+ else if ( retryTimeout == NULL || to == NULL )
rc = RC ( rcNS, rcStream, rcConstructing, rcParam, rcNull );
else if ( from != NULL && from -> type != to -> type )
rc = RC ( rcNS, rcStream, rcConstructing, rcParam, rcIncorrect );
@@ -849,31 +1009,90 @@ KNS_EXTERN rc_t CC KNSManagerMakeRetryTimedConnection ( struct KNSManager const
rc = KStreamInit ( & conn -> dad, ( const KStream_vt* ) & vtKSocket,
"KSocket", "", true, true );
+
+ /* prepare the timeout */
+ if ( rc == 0 )
+ rc = TimeoutPrepare ( retryTimeout );
+
if ( rc == 0 )
{
+ uint32_t retry;
+
switch ( to -> type )
{
- case epIPV6:
- rc = KSocketConnectIPv6 ( conn, retryTimeout, from, to );
- break;
+ /* ensure the type is correct */
case epIPV4:
- rc = KSocketConnectIPv4 ( conn, retryTimeout, from, to );
- break;
-
+ case epIPV6:
case epIPC:
- rc = KSocketConnectIPC ( conn, retryTimeout, to );
- break;
+
+ /* a retry loop - retry upon a schedule for the alloted time */
+ for ( retry = 0; ; ++ retry )
+ {
+ uint32_t remaining, delay;
+
+ /* try to connect using appropriate protocol */
+ conn -> type = to -> type;
+ switch ( to -> type )
+ {
+ case epIPV4:
+ rc = KSocketConnectIPv4 ( conn, from, to );
+ break;
+
+ case epIPV6:
+ rc = KSocketConnectIPv6 ( conn, from, to );
+ break;
+
+ case epIPC:
+ rc = KSocketConnectIPC ( conn, to );
+ break;
+ }
+
+ /* if connection was successful, return socket */
+ if ( rc == 0 )
+ {
+ * out = conn;
+ return 0;
+ }
+
+ /* check time remaining on timeout ( if any ) */
+ remaining = TimeoutRemaining ( retryTimeout );
+
+ /* break out of loop if no time left */
+ if ( remaining == 0 )
+ break;
+
+ /* apply delay schedule */
+ switch ( retry )
+ {
+ case 0:
+ /* try immediately */
+ continue;
+ case 1:
+ case 2:
+ case 3:
+ /* wait for 100mS between tries */
+ delay = 100;
+ break;
+ default:
+ /* wait for 250mS between tries */
+ delay = 250;
+ }
+
+ /* never wait for more than the remaining timeout */
+ if ( delay > remaining )
+ delay = remaining;
+
+ KSleepMs ( delay );
+ }
+
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_SOCKET),
+ ( "%p: KSocketConnect timed out\n", self ) );
+ break;
default:
rc = RC ( rcNS, rcStream, rcConstructing, rcParam, rcIncorrect );
}
-
- if ( rc == 0 )
- {
- * out = conn;
- return 0;
- }
}
free ( conn );
@@ -897,7 +1116,7 @@ rc_t KNSManagerMakeIPv6Listener ( KSocket *listener, const KEndPoint * ep )
listener -> fd = socket ( AF_INET6, SOCK_STREAM, 0 );
if ( listener -> fd < 0 )
- rc = HandleErrno ( __func__, __LINE__ );
+ rc = KSocketHandleSocketCall ( errno );
else
{
struct sockaddr_in6 ss;
@@ -914,10 +1133,11 @@ rc_t KNSManagerMakeIPv6Listener ( KSocket *listener, const KEndPoint * ep )
ss . sin6_port = htons ( ep -> u . ipv6 . port );
- if ( bind ( listener -> fd, ( struct sockaddr* ) & ss, sizeof ss ) == 0 )
+ if ( bind ( listener -> fd, ( struct sockaddr* ) & ss, sizeof ss ) != 0 )
+ rc = KSocketHandleBindCall ( errno );
+ else
return 0;
- rc = HandleErrno ( __func__, __LINE__ );
-
+
close ( listener -> fd );
listener -> fd = -1;
}
@@ -933,7 +1153,7 @@ rc_t KNSManagerMakeIPv4Listener ( KSocket *listener, const KEndPoint * ep )
listener -> fd = socket ( AF_INET, SOCK_STREAM, 0 );
if ( listener -> fd < 0 )
- rc = HandleErrno ( __func__, __LINE__ );
+ rc = KSocketHandleSocketCall ( errno );
else
{
struct sockaddr_in ss;
@@ -946,9 +1166,10 @@ rc_t KNSManagerMakeIPv4Listener ( KSocket *listener, const KEndPoint * ep )
ss . sin_addr . s_addr = htonl ( ep -> u . ipv4 . addr );
ss . sin_port = htons ( ep -> u . ipv4 . port );
- if ( bind ( listener -> fd, ( struct sockaddr* ) & ss, sizeof ss ) == 0 )
+ if ( bind ( listener -> fd, ( struct sockaddr* ) & ss, sizeof ss ) != 0 )
+ rc = KSocketHandleBindCall ( errno );
+ else
return 0;
- rc = HandleErrno ( __func__, __LINE__ );
close ( listener -> fd );
listener -> fd = -1;
@@ -964,7 +1185,7 @@ rc_t KNSManagerMakeIPCListener ( KSocket *listener, const KEndPoint * ep )
listener -> fd = socket ( AF_UNIX, SOCK_STREAM, 0 );
if ( listener -> fd < 0 )
- rc = HandleErrno ( __func__, __LINE__ );
+ rc = KSocketHandleSocketCall ( errno );
else
{
struct sockaddr_un ss;
@@ -980,7 +1201,7 @@ rc_t KNSManagerMakeIPCListener ( KSocket *listener, const KEndPoint * ep )
{
unlink ( ss . sun_path );
if ( bind ( listener -> fd, ( struct sockaddr* ) & ss, sizeof ss ) != 0 )
- rc = HandleErrno ( __func__, __LINE__ );
+ rc = KSocketHandleBindCall ( errno );
else
{
listener -> path = path;
@@ -1028,6 +1249,7 @@ LIB_EXPORT rc_t CC KNSManagerMakeListener ( const KNSManager *self,
"KSocket", "", true, true );
if ( rc == 0 )
{
+ listener -> type = ep -> type;
switch ( ep -> type )
{
case epIPV6:
@@ -1054,9 +1276,28 @@ LIB_EXPORT rc_t CC KNSManagerMakeListener ( const KNSManager *self,
* out = ( KListener* ) listener;
return 0;
}
-
- rc = HandleErrno ( __func__, __LINE__ );
-
+ else
+ {
+ switch ( errno )
+ {
+ case EADDRINUSE:
+ rc = RC ( rcNS, rcSocket, rcCreating, rcSocket, rcBusy );
+ break;
+ case EBADF:
+ rc = RC ( rcNS, rcSocket, rcCreating, rcSocket, rcInvalid );
+ break;
+ case ENOTSOCK:
+ rc = RC ( rcNS, rcSocket, rcCreating, rcSocket, rcIncorrect );
+ break;
+ case EOPNOTSUPP:
+ rc = RC ( rcNS, rcSocket, rcCreating, rcInterface, rcUnsupported );
+ break;
+ default:
+ rc = RC ( rcNS, rcSocket, rcCreating, rcError, rcUnknown );
+ break;
+ }
+ }
+
if ( listener -> path != NULL )
free ( ( void* ) listener -> path );
}
@@ -1088,9 +1329,11 @@ rc_t KListenerAcceptIPv4 ( KSocket *self, KSocket *conn )
socklen_t len = sizeof conn->remote_addr.v4;
conn -> fd = accept ( self -> fd, ( struct sockaddr * ) & conn->remote_addr.v4, & len );
if ( conn -> fd < 0 )
- return HandleErrno ( __func__, __LINE__ );
+ return KSocketHandleAcceptCall ( errno );
+
if ( len > sizeof conn->remote_addr.v4 )
return RC ( rcNS, rcConnection, rcWaiting, rcBuffer, rcInsufficient );
+
return 0;
}
@@ -1101,9 +1344,11 @@ rc_t KListenerAcceptIPv6 ( KSocket *self, KSocket *conn )
socklen_t len = sizeof conn->remote_addr.v6;
conn -> fd = accept ( self -> fd, ( struct sockaddr * ) & conn->remote_addr.v6, & len );
if ( conn -> fd < 0 )
- return HandleErrno ( __func__, __LINE__ );
+ return KSocketHandleAcceptCall ( errno );
+
if ( len > sizeof conn->remote_addr.v6 )
return RC ( rcNS, rcConnection, rcWaiting, rcBuffer, rcInsufficient );
+
return 0;
}
@@ -1114,9 +1359,11 @@ rc_t KListenerAcceptIPC ( KSocket *self, KSocket *conn )
socklen_t len = sizeof remote;
conn -> fd = accept ( self -> fd, ( struct sockaddr* ) & remote, & len );
if ( conn -> fd < 0 )
- return HandleErrno ( __func__, __LINE__ );
+ return KSocketHandleAcceptCall ( errno );
+
if ( len > sizeof remote )
return RC ( rcNS, rcConnection, rcWaiting, rcBuffer, rcInsufficient );
+
return 0;
}
diff --git a/libs/kproc/procmgr.c b/libs/kproc/procmgr.c
index 6f768b3..22ca7cd 100644
--- a/libs/kproc/procmgr.c
+++ b/libs/kproc/procmgr.c
@@ -31,6 +31,7 @@
#include <kproc/lock.h>
#include <klib/refcount.h>
#include <klib/rc.h>
+#include <atomic.h>
#define rcTask rcCmd
@@ -59,13 +60,14 @@ struct KCleanupTaskQueue
*/
struct KProcMgr
{
- KLock *cleanup_lock;
KCleanupTaskQueue *cleanup;
KRefcount refcount;
};
-static KProcMgr * s_proc_mgr;
+static atomic_ptr_t s_proc_mgr = { NULL };
+static atomic_ptr_t cleanup_lock_ptr = { NULL };
+#define cleanup_lock ( ( KLock * ) cleanup_lock_ptr . ptr )
/* Whack
* tear down proc mgr
@@ -77,19 +79,24 @@ LIB_EXPORT rc_t CC KProcMgrWhack ( void )
{
rc_t rc = 0;
- KProcMgr *self = s_proc_mgr;
- if ( s_proc_mgr != NULL )
- {
- s_proc_mgr = NULL;
+ /* check to see if the singleton was created and
+ try to zero out the static variable */
+ KProcMgr * test, * self = s_proc_mgr . ptr;
+ if ( self != NULL ) do
+ self = atomic_test_and_set_ptr ( & s_proc_mgr, NULL, test = self );
+ while ( self != NULL && self != test );
- rc = KLockAcquire ( self -> cleanup_lock );
+ /* check to see if this thread will be cleaning up on procmgr */
+ if ( self != NULL )
+ {
+ rc = KLockAcquire ( cleanup_lock );
if ( rc == 0 )
{
uint64_t i;
KCleanupTaskQueue *cleanup = self -> cleanup;
self -> cleanup = NULL;
- KLockUnlock ( self -> cleanup_lock );
+ KLockUnlock ( cleanup_lock );
if ( cleanup != NULL )
{
@@ -111,7 +118,6 @@ LIB_EXPORT rc_t CC KProcMgrWhack ( void )
}
}
- KLockRelease ( self -> cleanup_lock );
free ( self );
}
@@ -127,23 +133,37 @@ LIB_EXPORT rc_t CC KProcMgrInit ( void )
{
rc_t rc = 0;
- if ( s_proc_mgr == NULL )
+ if ( s_proc_mgr . ptr == NULL )
{
- KProcMgr *mgr = malloc ( sizeof * s_proc_mgr );
+ KProcMgr * mgr = calloc ( 1, sizeof * mgr );
if ( mgr == NULL )
+ {
rc = RC ( rcPS, rcMgr, rcInitializing, rcMemory, rcExhausted );
+ }
else
{
- rc = KLockMake ( & mgr -> cleanup_lock );
- if ( rc == 0 )
- {
- mgr -> cleanup = NULL;
- KRefcountInit ( & mgr -> refcount, 0, "KProcMgr", "init", "process mgr" );
+ KProcMgr * rslt;
- s_proc_mgr = mgr;
- return 0;
- }
+ mgr -> cleanup = NULL;
+ KRefcountInit ( & mgr -> refcount, 0, "KProcMgr", "init", "process mgr" );
+ rslt = atomic_test_and_set_ptr ( & s_proc_mgr, mgr, NULL );
+ if ( rslt == NULL && s_proc_mgr .ptr == mgr )
+ {
+ KLock* lock;
+ rc = KLockMake ( & lock );
+ if ( rc == 0 )
+ {
+ KLock* lock_rslt = atomic_test_and_set_ptr ( & cleanup_lock_ptr, lock, NULL );
+ if ( lock_rslt == NULL && cleanup_lock_ptr . ptr == lock )
+ {
+ return 0;
+ }
+ KLockRelease ( lock );
+ }
+ s_proc_mgr . ptr = NULL;
+ }
+ /* someone beat us to it */
free ( mgr );
}
}
@@ -164,12 +184,12 @@ LIB_EXPORT rc_t CC KProcMgrMakeSingleton ( KProcMgr ** mgrp )
rc = RC ( rcPS, rcMgr, rcConstructing, rcParam, rcNull );
else
{
- * mgrp = s_proc_mgr;
+ * mgrp = s_proc_mgr . ptr;
- if ( s_proc_mgr == NULL )
+ if ( * mgrp == NULL )
rc = RC ( rcPS, rcMgr, rcConstructing, rcMgr, rcNull );
else
- rc = KProcMgrAddRef ( s_proc_mgr );
+ rc = KProcMgrAddRef ( * mgrp );
}
return rc;
@@ -196,18 +216,31 @@ LIB_EXPORT rc_t CC KProcMgrRelease ( const KProcMgr *self )
{
if ( self != NULL )
{
- switch ( KRefcountDrop ( & self -> refcount, "KProcMgr" ) )
+ if ( KLockAcquire ( cleanup_lock ) == 0 ) /* once created, cleanup_lock does not go away */
{
- case krefWhack:
- return 0;
- case krefNegative:
- return RC ( rcPS, rcMgr, rcReleasing, rcRange, rcExcessive );
+ if ( s_proc_mgr . ptr != NULL )
+ {
+ rc_t rc = KRefcountDrop ( & self -> refcount, "KProcMgr" );
+ KLockUnlock ( cleanup_lock );
+ switch ( rc )
+ {
+ case krefWhack:
+ return 0;
+ case krefNegative:
+ return RC ( rcPS, rcMgr, rcReleasing, rcRange, rcExcessive );
+ }
+ return rc;
+ }
+ else
+ { /* VDB-3067: the singleton may have been destroyed by the main thread exiting; */
+ /* if so, this pointer is dead but we are on a thread that is terminating anyway, do nothing */
+ KLockUnlock ( cleanup_lock );
+ }
}
}
return 0;
}
-
/* AddCleanupTask
* add a task to be performed at process exit time
*
@@ -229,7 +262,7 @@ LIB_EXPORT rc_t CC KProcMgrAddCleanupTask ( KProcMgr *self, KTaskTicket *ticket,
if ( self == NULL )
rc = RC ( rcPS, rcQueue, rcInserting, rcSelf, rcNull );
- else if ( self != s_proc_mgr )
+ else if ( self != s_proc_mgr . ptr )
rc = RC ( rcPS, rcQueue, rcInserting, rcSelf, rcIncorrect );
else if ( task == NULL )
rc = RC ( rcPS, rcQueue, rcInserting, rcTask, rcNull );
@@ -238,7 +271,7 @@ LIB_EXPORT rc_t CC KProcMgrAddCleanupTask ( KProcMgr *self, KTaskTicket *ticket,
rc = KTaskAddRef ( task );
if ( rc == 0 )
{
- rc = KLockAcquire ( self -> cleanup_lock );
+ rc = KLockAcquire ( cleanup_lock );
if ( rc == 0 )
{
const uint64_t extend = 1024;
@@ -344,7 +377,7 @@ LIB_EXPORT rc_t CC KProcMgrAddCleanupTask ( KProcMgr *self, KTaskTicket *ticket,
++ cleanup -> count;
}
- KLockUnlock ( self -> cleanup_lock );
+ KLockUnlock ( cleanup_lock );
}
if ( rc != 0 )
@@ -368,7 +401,7 @@ LIB_EXPORT rc_t CC KProcMgrRemoveCleanupTask ( KProcMgr *self, const KTaskTicket
if ( self == NULL )
rc = RC ( rcPS, rcQueue, rcRemoving, rcSelf, rcNull );
- else if ( self != s_proc_mgr )
+ else if ( self != s_proc_mgr . ptr )
rc = RC ( rcPS, rcQueue, rcRemoving, rcSelf, rcIncorrect );
else if ( ticket == NULL )
rc = RC ( rcPS, rcQueue, rcRemoving, rcId, rcNull );
@@ -380,7 +413,7 @@ LIB_EXPORT rc_t CC KProcMgrRemoveCleanupTask ( KProcMgr *self, const KTaskTicket
idx ^= ( size_t ) self;
/* go into queue */
- rc = KLockAcquire ( self -> cleanup_lock );
+ rc = KLockAcquire ( cleanup_lock );
if ( rc == 0 )
{
KCleanupTaskQueue *cleanup = self -> cleanup;
@@ -401,7 +434,7 @@ LIB_EXPORT rc_t CC KProcMgrRemoveCleanupTask ( KProcMgr *self, const KTaskTicket
}
}
- KLockUnlock ( self -> cleanup_lock );
+ KLockUnlock ( cleanup_lock );
}
if ( rc == 0 )
diff --git a/libs/kproc/unix/systimeout.c b/libs/kproc/unix/systimeout.c
index 35295b7..64c23e5 100644
--- a/libs/kproc/unix/systimeout.c
+++ b/libs/kproc/unix/systimeout.c
@@ -84,3 +84,45 @@ LIB_EXPORT rc_t TimeoutPrepare ( timeout_t *self )
return 0;
}
+
+/* Remaining
+ * ask how many milliseconds remain before timeout expires
+ */
+LIB_EXPORT uint32_t TimeoutRemaining ( timeout_t * self )
+{
+ struct timeval tv;
+ struct timezone tz;
+ int64_t cur_micros, end_micros;
+
+ if ( self == NULL )
+ return 0;
+
+ /* expect timeout to be prepared, but if it isn't
+ prepare it so that use within a loop eventually terminates */
+ if ( ! self -> prepared )
+ {
+ /* prepare an absolute timeout */
+ TimeoutPrepare ( self );
+
+ /* return the entire timeout */
+ return self -> mS;
+ }
+
+ /* current time in seconds and uS */
+ gettimeofday ( & tv, & tz );
+
+ /* convert to uS */
+ cur_micros = tv . tv_sec;
+ cur_micros = cur_micros * 1000 * 1000 + tv . tv_usec;
+
+ /* capture end time in uS */
+ end_micros = self -> ts . tv_sec;
+ end_micros = end_micros * 1000 * 1000 + ( ( self -> ts . tv_nsec + 500 ) / 1000 );
+
+ /* never return negative */
+ if ( cur_micros >= end_micros )
+ return 0;
+
+ /* return positive difference as mS - truncated */
+ return ( end_micros - cur_micros ) / 1000;
+}
diff --git a/libs/ncbi-vdb/Makefile b/libs/ncbi-vdb/Makefile
index b63633f..d654c6a 100644
--- a/libs/ncbi-vdb/Makefile
+++ b/libs/ncbi-vdb/Makefile
@@ -92,6 +92,9 @@ VDB_LIB_CMN = \
ncbi-bam \
vfs \
$(LIBKXML) \
+ mbedx509 \
+ mbedtls \
+ mbedcrypto \
kns \
kfg \
krypto \
diff --git a/libs/ncbi-vdb/libncbi-vdb.vers b/libs/ncbi-vdb/libncbi-vdb.vers
index 24ba9a3..834f262 100644
--- a/libs/ncbi-vdb/libncbi-vdb.vers
+++ b/libs/ncbi-vdb/libncbi-vdb.vers
@@ -1 +1 @@
-2.7.0
+2.8.0
diff --git a/libs/ngs-jni/jni_String.c b/libs/ngs-jni/jni_String.c
index 40ae772..904971d 100644
--- a/libs/ngs-jni/jni_String.c
+++ b/libs/ngs-jni/jni_String.c
@@ -101,7 +101,9 @@ jstring JStringVMake ( ctx_t ctx, JNIEnv * jenv, const char * fmt, va_list args
const char * JStringData ( jstring jself, ctx_t ctx, JNIEnv * jenv )
{
jboolean is_copy;
- return ( ( * jenv ) -> GetStringUTFChars ) ( jenv, jself, & is_copy );
+ if ( jself != NULL )
+ return ( ( * jenv ) -> GetStringUTFChars ) ( jenv, jself, & is_copy );
+ return NULL;
}
diff --git a/libs/ngs/BAM_Alignment.c b/libs/ngs/BAM_Alignment.c
index e85b56c..65618aa 100644
--- a/libs/ngs/BAM_Alignment.c
+++ b/libs/ngs/BAM_Alignment.c
@@ -67,7 +67,7 @@ static void BAM_AlignmentWhack(void * const vp, ctx_t ctx)
{
FUNC_ENTRY(ctx, rcSRA, rcFile, rcAccessing);
BAM_Alignment *const self = (BAM_Alignment *)vp;
-
+
free(self->cur);
NGS_RefcountRelease(&self->provider->dad, ctx);
}
@@ -125,7 +125,7 @@ static bool FindRG(void *const vp, ctx_t ctx, unsigned const ord, BAM_Record_Ext
if (fld->tag[0] == 'R' && fld->tag[1] == 'G' && fld->val_type == 'Z') {
NGS_String **const prslt = vp;
-
+
*prslt = NGS_StringMakeCopy(ctx, fld->value->string, fld->elemcount);
return false; /* done */
}
@@ -139,7 +139,7 @@ static NGS_String * BAM_AlignmentReadGroup(void *const vp, ctx_t ctx)
if (self->cur) {
NGS_String *rslt = NULL;
-
+
BAM_Record_ForEachExtra(self->cur, ctx, FindRG, &rslt);
return rslt;
}
@@ -169,16 +169,16 @@ typedef struct {
static clipped_t const get_clipping(BAM_Record const *rec)
{
clipped_t rslt = { 0, rec->seqlen };
-
+
if (rec->ncigar > 0 && (rec->cigar[0] & 0x0F) == 4) {
unsigned const length = rec->cigar[0] >> 4;
-
+
rslt.start += length;
rslt.length -= length;
}
if (rec->ncigar > 1 && (rec->cigar[rec->ncigar - 1] & 0x0F) == 4) {
unsigned const length = rec->cigar[rec->ncigar - 1] >> 4;
-
+
rslt.length -= length;
}
return rslt;
@@ -225,7 +225,7 @@ static bool BAM_AlignmentIsPrimary(void *const vp, ctx_t ctx)
if (self->cur)
return (self->cur->FLAG & 0x0900) == 0 ? true : false;
-
+
USER_ERROR(xcRowNotFound, "no current row");
return false;
}
@@ -238,7 +238,7 @@ static int64_t BAM_AlignmentAlignmentPosition(void *const vp, ctx_t ctx)
if (self->cur)
return self->cur->POS - 1;
-
+
USER_ERROR(xcRowNotFound, "no current row");
return -1;
}
@@ -248,12 +248,12 @@ static unsigned ComputeRefLen(size_t const count, uint32_t const cigar[])
{
unsigned rslt = 0;
unsigned i;
-
+
for (i = 0; i < count; ++i) {
uint32_t const op = cigar[i];
unsigned const len = op >> 4;
int const code = op & 0x0F;
-
+
switch (code) {
case 0: /* M */
case 2: /* D */
@@ -274,7 +274,7 @@ static uint64_t BAM_AlignmentAlignmentLength(void *const vp, ctx_t ctx)
if (self->cur)
return ComputeRefLen(self->cur->ncigar, self->cur->cigar);
-
+
USER_ERROR(xcRowNotFound, "no current row");
return 0;
}
@@ -287,7 +287,7 @@ static bool BAM_AlignmentIsReversedOrientation(void *const vp, ctx_t ctx)
if (self->cur)
return (self->cur->FLAG & 0x0010) == 0 ? false : true;
-
+
USER_ERROR(xcRowNotFound, "no current row");
return false;
}
@@ -318,7 +318,7 @@ static uint64_t BAM_AlignmentTemplateLength(void *const vp, ctx_t ctx)
if (self->cur)
return self->cur->TLEN;
-
+
USER_ERROR(xcRowNotFound, "no current row");
return 0;
}
@@ -331,14 +331,14 @@ static char const *FormatCIGAR(char *dst, char const OPCODE[], size_t const coun
char *last_out;
char last_code = 0;
uint32_t last_len = 0;
-
+
last_out = dst;
for (i = 0; i < count; ++i) {
uint32_t const op = *--src;
char const code = OPCODE[op & 0x0F];
uint32_t const len1 = op >> 4;
uint32_t len = code == last_code ? ((dst = last_out), last_len + len1) : len1;
-
+
last_len = len;
last_code = code;
last_out = dst;
@@ -364,7 +364,7 @@ static NGS_String *CIGAR(ctx_t ctx, char const OPCODE[], size_t const count, uin
char *const endp = buffer + max;
char const *const rslt = FormatCIGAR(endp, OPCODE, count, cigar);
size_t const len = endp - rslt;
-
+
return NGS_StringMakeCopy(ctx, rslt, len);
}
@@ -377,7 +377,7 @@ static NGS_String *CIGAR_clipped(ctx_t ctx, char const OPCODE[], bool const clip
char const last = cigar[count - 1] & 0x0F;
unsigned const cfirst = (first == 4 || first == 5) ? 1 : 0;
unsigned const clast = (last == 4 || last == 5) ? 1 : 0;
-
+
return CIGAR(ctx, OPCODE, count - cfirst - clast, &cigar[cfirst]);
}
}
@@ -415,7 +415,7 @@ static bool BAM_AlignmentHasMate(void *const vp, ctx_t ctx)
if (self->cur)
return (self->cur->FLAG & 0x0001) == 0 ? false : true;
-
+
USER_ERROR(xcRowNotFound, "no current row");
return false;
}
@@ -450,7 +450,7 @@ static NGS_String * BAM_AlignmentMateReferenceSpec(void *const vp, ctx_t ctx)
if (self->cur)
return self->cur->RNEXT ? NGS_StringMake(ctx, self->cur->RNEXT, strlen(self->cur->RNEXT)) : NULL;
-
+
USER_ERROR(xcRowNotFound, "no current row");
return NULL;
}
@@ -463,7 +463,22 @@ static bool BAM_AlignmentMateIsReversedOrientation(void *const vp, ctx_t ctx)
if (self->cur)
return (self->cur->FLAG & 0x0020) == 0 ? false : true;
-
+
+ USER_ERROR(xcRowNotFound, "no current row");
+ return false;
+}
+
+static bool BAM_AlignmentIsFirst(void *const vp, ctx_t ctx)
+{
+ FUNC_ENTRY(ctx, rcSRA, rcFile, rcAccessing);
+ BAM_Alignment *const self = (BAM_Alignment *)vp;
+
+ if (self->cur) {
+ bool const isMated = (self->cur->FLAG & 0x001) == 0 ? false : true;
+ bool const isFirst = (self->cur->FLAG & 0x040) == 0 ? false : true;
+ bool const isLast = (self->cur->FLAG & 0x080) == 0 ? false : true;
+ return (isMated && isFirst && !isLast);
+ }
USER_ERROR(xcRowNotFound, "no current row");
return false;
}
@@ -477,13 +492,13 @@ static bool ShouldSkip(BAM_Alignment const *const self)
{
if (!self->cur->RNAME) /* not aligned */
return true;
-
+
if ((self->cur->FLAG & 0x0900) == 0 && !self->primary) /* is primary and don't want primary */
return true;
-
+
if ((self->cur->FLAG & 0x0900) != 0 && !self->secondary) /* is secondary and don't want secondary */
return true;
-
+
return false;
}
@@ -579,7 +594,7 @@ static NGS_Alignment_vt const vt =
BAM_AlignmentFragmentGetQualities,
BAM_AlignmentFragmentNext
},
-
+
BAM_AlignmentAlignmentId,
BAM_AlignmentReferenceSpec,
BAM_AlignmentMappingQuality,
@@ -602,6 +617,7 @@ static NGS_Alignment_vt const vt =
BAM_AlignmentMateAlignment,
BAM_AlignmentMateReferenceSpec,
BAM_AlignmentMateIsReversedOrientation,
+ BAM_AlignmentIsFirst,
/* Iterator */
BAM_AlignmentIteratorNext
@@ -626,7 +642,7 @@ struct NGS_Alignment *BAM_AlignmentMake(ctx_t ctx, bool const primary, bool cons
void *self = calloc(1, sizeof(BAM_Alignment));
if (self) {
NGS_Alignment *const super = &((BAM_Alignment *)self)->super;
-
+
TRY(NGS_AlignmentInit(ctx, super, &vt, "BAM_Alignment", name)) {
TRY(BAM_AlignmentInit(self, ctx, primary, secondary, provider_f, provider)) {
return self;
diff --git a/libs/ngs/CSRA1_Alignment.c b/libs/ngs/CSRA1_Alignment.c
index 92c3b9a..98cdcba 100644
--- a/libs/ngs/CSRA1_Alignment.c
+++ b/libs/ngs/CSRA1_Alignment.c
@@ -859,6 +859,26 @@ bool CSRA1_AlignmentGetMateIsReversedOrientation( CSRA1_Alignment* self, ctx_t c
return NGS_CursorGetBool ( GetCursor ( self ), ctx, self -> cur_row, align_MATE_REF_ORIENTATION);
}
+bool CSRA1_AlignmentIsFirst( CSRA1_Alignment* self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+ int32_t seq_read_id;
+
+ assert ( self );
+ if ( ! self -> seen_first )
+ {
+ USER_ERROR ( xcIteratorUninitialized, "Alignment accessed before a call to AlignmentIteratorNext()" );
+ return false;
+ }
+
+ TRY ( seq_read_id = NGS_CursorGetInt64 ( GetCursor ( self ), ctx, self -> cur_row, align_SEQ_READ_ID ) )
+ {
+ return seq_read_id == 1;
+ }
+ return false;
+}
+
+
/*--------------------------------------------------------------------------
* NGS_AlignmentIterator
*/
@@ -1134,6 +1154,7 @@ static NGS_Alignment_vt CSRA1_Alignment_vt_inst =
CSRA1_AlignmentGetMateAlignment,
CSRA1_AlignmentGetMateReferenceSpec,
CSRA1_AlignmentGetMateIsReversedOrientation,
+ CSRA1_AlignmentIsFirst,
/* Iterator */
CSRA1_AlignmentIteratorNext
diff --git a/libs/ngs/CSRA1_Read.c b/libs/ngs/CSRA1_Read.c
index 59f6d93..4290962 100644
--- a/libs/ngs/CSRA1_Read.c
+++ b/libs/ngs/CSRA1_Read.c
@@ -48,7 +48,7 @@
#include <sysalloc.h>
#ifndef min
-# define min(a,b) ( (a) < (b) ? (a) : (b) )
+# define min(a,b) ( (a) < (b) ? (a) : (b) )
#endif
/*--------------------------------------------------------------------------
@@ -78,7 +78,7 @@ static NGS_Read_vt CSRA1_Read_vt_inst =
CSRA1_FragmentIsAligned,
SRA_FragmentNext
},
-
+
/* NGS_Read */
SRA_ReadGetId,
SRA_ReadGetName,
@@ -90,7 +90,7 @@ static NGS_Read_vt CSRA1_Read_vt_inst =
CSRA1_ReadFragIsAligned,
SRA_ReadIteratorNext,
SRA_ReadIteratorGetCount,
-};
+};
/* Init
*/
@@ -108,48 +108,13 @@ void CSRA1_ReadInit ( ctx_t ctx, SRA_Read * self, const char *clsname, const cha
TRY ( self -> run_name = NGS_StringDuplicate ( run_name, ctx ) )
{
self -> wants_full = true;
- self -> wants_partial = true;
- self -> wants_unaligned = true;
+ self -> wants_partial = true;
+ self -> wants_unaligned = true;
}
}
}
}
-/* Whack
- */
-static
-void CSRA1_ReadIteratorInitFragment ( SRA_Read * self, ctx_t ctx )
-{
- const void * base;
- uint32_t elem_bits, boff, row_len;
-
- /* read from READ_TYPE must succeed */
- TRY ( NGS_CursorCellDataDirect ( self -> curs, ctx, self -> cur_row, seq_READ_TYPE, & elem_bits, & base, & boff, & row_len ) )
- {
- assert ( elem_bits == 8 );
- assert ( boff == 0 );
- self -> READ_TYPE = base;
-
- TRY ( NGS_CursorCellDataDirect ( self -> curs, ctx, self -> cur_row, seq_READ_LEN, & elem_bits, & base, & boff, & row_len ) )
- {
- uint32_t i;
-
- assert ( elem_bits == 32 );
- assert ( boff == 0 );
- self -> READ_LEN = base;
- self -> frag_max = row_len;
-
- /* naked hackery to quickly scan types */
- assert ( READ_TYPE_TECHNICAL == 0 );
- assert ( READ_TYPE_BIOLOGICAL == 1 );
-
- /* NB - should also be taking READ_FILTER into account */
- for ( i = 0; i < row_len; ++ i )
- self -> bio_frags += self -> READ_TYPE [ i ] & READ_TYPE_BIOLOGICAL;
- }
- }
-}
-
/* Release
* release reference
*/
@@ -176,7 +141,7 @@ void CSRA1_ReadIteratorInit ( ctx_t ctx,
bool wants_unaligned )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
-
+
if ( cself == NULL )
INTERNAL_ERROR ( xcParamNull, "bad object reference" );
else
@@ -204,12 +169,12 @@ NGS_Read * CSRA1_ReadIteratorMake ( ctx_t ctx,
bool wants_unaligned )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
-
+
CSRA1_Read * cref;
SRA_Read * ref;
-
+
assert ( curs != NULL );
-
+
cref = calloc ( 1, sizeof * cref );
if ( cref == NULL )
SYSTEM_ERROR ( xcNoMemory, "allocating CSRA1_ReadIterator on '%.*s'", NGS_StringSize ( run_name, ctx ), NGS_StringData ( run_name, ctx ) );
@@ -225,7 +190,7 @@ NGS_Read * CSRA1_ReadIteratorMake ( ctx_t ctx,
TRY ( CSRA1_ReadIteratorInit ( ctx, cref, "CSRA1_ReadIterator", instname, run_name, wants_full, wants_partial, wants_unaligned ) )
{
ref = & cref -> dad;
-
+
ref -> curs = NGS_CursorDuplicate ( curs, ctx );
TRY ( NGS_CursorGetRowRange ( ref -> curs, ctx, & ref -> cur_row, & ref -> row_count ) )
{
@@ -235,10 +200,10 @@ NGS_Read * CSRA1_ReadIteratorMake ( ctx_t ctx,
CSRA1_ReadRelease ( cref, ctx );
return NULL;
}
-
+
free ( cref );
}
-
+
return NULL;
}
@@ -254,12 +219,12 @@ NGS_Read * CSRA1_ReadIteratorMakeRange ( ctx_t ctx,
bool wants_unaligned )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
-
+
CSRA1_Read * cref;
SRA_Read * ref;
-
+
assert ( curs != NULL );
-
+
cref = calloc ( 1, sizeof * ref );
if ( cref == NULL )
SYSTEM_ERROR ( xcNoMemory, "allocating CSRA1_ReadIterator on '%.*s'", NGS_StringSize ( run_name, ctx ), NGS_StringData ( run_name, ctx ) );
@@ -275,7 +240,7 @@ NGS_Read * CSRA1_ReadIteratorMakeRange ( ctx_t ctx,
TRY ( CSRA1_ReadIteratorInit ( ctx, cref, "CSRA1_ReadIterator", instname, run_name, wants_full, wants_partial, wants_unaligned ) )
{
ref = & cref -> dad;
-
+
ref -> curs = NGS_CursorDuplicate ( curs, ctx );
TRY ( NGS_CursorGetRowRange ( ref -> curs, ctx, & ref -> cur_row, & ref -> row_count ) )
{
@@ -286,10 +251,10 @@ NGS_Read * CSRA1_ReadIteratorMakeRange ( ctx_t ctx,
CSRA1_ReadRelease ( cref, ctx );
return NULL;
}
-
+
free ( cref );
}
-
+
return NULL;
}
@@ -306,7 +271,7 @@ NGS_Read * CSRA1_ReadIteratorMakeReadGroup ( ctx_t ctx,
bool wants_unaligned )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
-
+
TRY ( CSRA1_Read * cref = (CSRA1_Read*) CSRA1_ReadIteratorMakeRange ( ctx,
curs,
run_name,
@@ -349,21 +314,21 @@ NGS_Read * CSRA1_ReadMake ( ctx_t ctx, const NGS_Cursor * curs, int64_t readId,
const char *instname = "";
#endif
ref = & cref -> dad;
-
+
TRY ( CSRA1_ReadInit ( ctx, ref, "CSRA1_Read", instname, run_name ) )
{
uint64_t row_count = NGS_CursorGetRowCount ( curs, ctx );
-
+
/* validate the requested rowId and seek to it */
if ( readId <= 0 || (uint64_t)readId > row_count )
{
INTERNAL_ERROR ( xcCursorAccessFailed, "rowId ( %li ) out of range for %.*s", readId, NGS_StringSize ( run_name, ctx ), NGS_StringData ( run_name, ctx ) );
- }
+ }
else
{
ref -> curs = NGS_CursorDuplicate ( curs, ctx );
ref -> cur_row = readId;
- TRY ( CSRA1_ReadIteratorInitFragment ( ref, ctx ) )
+ TRY ( SRA_ReadIteratorInitFragment ( cref , ctx ) )
{
ref -> row_max = readId + 1;
ref -> row_count = 1;
@@ -371,7 +336,7 @@ NGS_Read * CSRA1_ReadMake ( ctx_t ctx, const NGS_Cursor * curs, int64_t readId,
return & ref -> dad;
}
}
-
+
CSRA1_ReadRelease ( cref, ctx );
return NULL;
}
@@ -385,35 +350,35 @@ bool CSRA1_FragmentIsAligned ( CSRA1_Read * cself, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcAccessing );
const SRA_Read * self;
-
+
assert ( cself != NULL );
-
+
self = & cself -> dad;
-
+
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to nextRead()" );
return false;
}
-
+
if ( self -> cur_row >= self -> row_max )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
return false;
}
-
+
if ( ! self -> seen_first_frag )
{
USER_ERROR ( xcIteratorUninitialized, "Fragment accessed before a call to nextFragment()" );
return false;
}
-
+
if ( self -> frag_idx >= self -> frag_max )
{
USER_ERROR ( xcCursorExhausted, "No more fragments available" );
return false;
}
-
+
{
const void * base;
uint32_t elem_bits, boff, row_len;
@@ -422,33 +387,33 @@ bool CSRA1_FragmentIsAligned ( CSRA1_Read * cself, ctx_t ctx )
CLEAR();
return false;
}
-
+
{
const int64_t * orig = base;
assert(elem_bits == 64);
assert(boff == 0);
-
+
return orig[self -> frag_idx] != 0;
}
}
-
+
}
bool CSRA1_ReadFragIsAligned ( CSRA1_Read * cself, ctx_t ctx, uint32_t frag_idx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcAccessing );
const SRA_Read * self;
-
+
assert ( cself != NULL );
-
+
self = & cself -> dad;
-
+
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to nextRead()" );
return false;
}
-
+
if ( self -> cur_row >= self -> row_max )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
@@ -460,7 +425,7 @@ bool CSRA1_ReadFragIsAligned ( CSRA1_Read * cself, ctx_t ctx, uint32_t frag_idx
USER_ERROR ( xcIntegerOutOfBounds, "bad fragment index" );
return false;
}
-
+
{
const void * base;
uint32_t elem_bits, boff, row_len;
@@ -474,23 +439,25 @@ bool CSRA1_ReadFragIsAligned ( CSRA1_Read * cself, ctx_t ctx, uint32_t frag_idx
assert ( row_len == self -> frag_max );
/* technically, we do not expect technical reads (fragments) within CSRA1,
- but it is correct to check for this possibility */
+ but it is correct to check for this possibility
+ same applies to 0-length biological fragments (VDB-3132)
+ */
if ( self -> bio_frags == self -> frag_max )
return orig [ frag_idx ] != 0;
for ( idx = bidx = 0; idx < row_len; ++ idx )
{
- if ( ( self -> READ_TYPE [ idx ] & READ_TYPE_BIOLOGICAL ) != 0 )
+ if ( ( self -> READ_TYPE [ idx ] & READ_TYPE_BIOLOGICAL ) != 0 && self -> READ_LEN [ idx ] != 0 )
{
if ( bidx == frag_idx )
return orig [ idx ] != 0;
-
+
++ bidx;
}
}
}
}
-
+
CLEAR();
return false;
}
diff --git a/libs/ngs/CSRA1_Reference.c b/libs/ngs/CSRA1_Reference.c
index b9611e7..9442e6e 100644
--- a/libs/ngs/CSRA1_Reference.c
+++ b/libs/ngs/CSRA1_Reference.c
@@ -41,6 +41,8 @@ typedef struct CSRA1_Reference CSRA1_Reference;
#include "CSRA1_ReferenceWindow.h"
#include "CSRA1_Pileup.h"
+#include "VByteBlob.h"
+
#include <kfc/ctx.h>
#include <kfc/rsrc.h>
#include <kfc/except.h>
@@ -54,6 +56,8 @@ typedef struct CSRA1_Reference CSRA1_Reference;
#include <vdb/cursor.h>
#include <vdb/schema.h>
#include <vdb/vdb-priv.h>
+#include <vdb/blob.h>
+#include "../vdb/blob-priv.h"
#include <stddef.h>
#include <assert.h>
@@ -87,7 +91,7 @@ static NGS_Reference_vt CSRA1_Reference_vt_inst =
{
/* NGS_Refcount */
{ CSRA1_ReferenceWhack },
-
+
/* NGS_Reference */
CSRA1_ReferenceGetCommonName,
CSRA1_ReferenceGetCanonicalName,
@@ -102,26 +106,26 @@ static NGS_Reference_vt CSRA1_Reference_vt_inst =
CSRA1_ReferenceGetPileups,
CSRA1_ReferenceGetPileupSlice,
CSRA1_ReferenceGetStatistics,
-
+
/* NGS_ReferenceIterator */
CSRA1_ReferenceIteratorNext,
};
struct CSRA1_Reference
{
- NGS_Reference dad;
-
+ NGS_Reference dad;
+
uint32_t chunk_size;
-
+
int64_t first_row;
int64_t last_row; /* inclusive */
const struct VDatabase * db; /* pointer to the opened db, cannot be NULL */
const struct NGS_Cursor * curs; /* can be NULL if created for an empty iterator */
uint64_t align_id_offset;
uint64_t cur_length; /* size of current reference in bases (0 = not yet counted) */
-
+
int64_t iteration_row_last; /* 0 = not iterating */
-
+
bool seen_first;
};
@@ -140,10 +144,10 @@ int64_t CSRA1_Reference_GetLastRowId ( const struct NGS_Reference * self, ctx_t
/* Init
*/
static
-void CSRA1_ReferenceInit ( ctx_t ctx,
+void CSRA1_ReferenceInit ( ctx_t ctx,
CSRA1_Reference * ref,
NGS_ReadCollection * coll,
- const char *clsname,
+ const char *clsname,
const char *instname,
uint64_t align_id_offset )
{
@@ -171,7 +175,7 @@ void CSRA1_ReferenceWhack ( CSRA1_Reference * self, ctx_t ctx )
VDatabaseRelease ( self -> db );
self -> db = NULL;
-
+
NGS_ReferenceWhack ( & self -> dad, ctx );
}
@@ -183,23 +187,23 @@ NGS_String * CSRA1_ReferenceGetCommonName ( CSRA1_Reference * self, ctx_t ctx )
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return NULL;
+ return NULL;
}
-
+
return NGS_CursorGetString ( self -> curs, ctx, self -> first_row, reference_NAME );
}
NGS_String * CSRA1_ReferenceGetCanonicalName ( CSRA1_Reference * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
assert ( self != NULL );
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return NULL;
+ return NULL;
}
-
+
return NGS_CursorGetString ( self -> curs, ctx, self -> first_row, reference_SEQ_ID);
}
@@ -208,7 +212,7 @@ bool CSRA1_ReferenceGetIsCircular ( const CSRA1_Reference * self, ctx_t ctx )
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
assert ( self );
-
+
if ( self -> curs == NULL )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
@@ -217,9 +221,9 @@ bool CSRA1_ReferenceGetIsCircular ( const CSRA1_Reference * self, ctx_t ctx )
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return false;
+ return false;
}
-
+
/* if current row is valid, read data */
if ( self -> first_row <= self -> last_row )
{
@@ -233,7 +237,7 @@ static
uint64_t CountRows ( NGS_Cursor const * curs, ctx_t ctx, uint32_t colIdx, const void* value, uint32_t value_size, int64_t firstRow, uint64_t end_row)
{ /* count consecutive rows having the same value in column # colIdx as in firstRow, starting from and including firstRow */
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
const void* last_value = value;
uint64_t cur_row = (uint64_t)firstRow + 1;
while (cur_row < end_row)
@@ -250,7 +254,7 @@ uint64_t CountRows ( NGS_Cursor const * curs, ctx_t ctx, uint32_t colIdx, const
last_value = base;
}
-
+
++ cur_row;
}
return cur_row - firstRow;
@@ -269,23 +273,23 @@ uint64_t CSRA1_ReferenceGetLength ( CSRA1_Reference * self, ctx_t ctx )
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return 0;
+ return 0;
}
-
+
if ( self -> cur_length == 0 ) /* not yet calculated */
- {
- self -> cur_length = self -> chunk_size * ( self -> last_row - self -> first_row ) +
- NGS_CursorGetUInt32 ( self -> curs,
- ctx,
- self -> last_row,
+ {
+ self -> cur_length = self -> chunk_size * ( self -> last_row - self -> first_row ) +
+ NGS_CursorGetUInt32 ( self -> curs,
+ ctx,
+ self -> last_row,
reference_SEQ_LEN );
}
-
+
return self -> cur_length;
}
struct NGS_String * CSRA1_ReferenceGetBases ( CSRA1_Reference * self, ctx_t ctx, uint64_t offset, uint64_t size )
-{
+{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
assert ( self );
@@ -297,9 +301,9 @@ struct NGS_String * CSRA1_ReferenceGetBases ( CSRA1_Reference * self, ctx_t ctx,
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return NULL;
+ return NULL;
}
-
+
{
uint64_t totalBases = CSRA1_ReferenceGetLength ( self, ctx );
if ( offset >= totalBases )
@@ -307,13 +311,13 @@ struct NGS_String * CSRA1_ReferenceGetBases ( CSRA1_Reference * self, ctx_t ctx,
return NGS_StringMake ( ctx, "", 0 );
}
else
- {
+ {
uint64_t basesToReturn = totalBases - offset;
char* data;
-
+
if (size != (size_t)-1 && basesToReturn > size)
basesToReturn = size;
-
+
data = (char*) malloc ( basesToReturn );
if ( data == NULL )
{
@@ -325,11 +329,16 @@ struct NGS_String * CSRA1_ReferenceGetBases ( CSRA1_Reference * self, ctx_t ctx,
size_t cur_offset = 0;
while ( cur_offset < basesToReturn )
{
- /* we will potentially ask for more than available in the current chunk;
- CSRA1_ReferenceGetChunkSize will return only as much as is available in the chunk */
- NGS_String* chunk = CSRA1_ReferenceGetChunk ( self, ctx, offset + cur_offset, basesToReturn - cur_offset );
- cur_offset += string_copy(data + cur_offset, basesToReturn - cur_offset,
- NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
+ /* we will potentially ask for more than available in the current blob;
+ CSRA1_ReferenceGetChunk will return only as much as is available in the blob */
+ NGS_String* chunk;
+ ON_FAIL ( chunk = CSRA1_ReferenceGetChunk ( self, ctx, offset + cur_offset, basesToReturn - cur_offset ) )
+ {
+ free ( data );
+ return NULL;
+ }
+ cur_offset += string_copy ( data + cur_offset, basesToReturn - cur_offset,
+ NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
NGS_StringRelease ( chunk, ctx );
}
return NGS_StringMakeOwned ( ctx, data, basesToReturn );
@@ -339,40 +348,65 @@ struct NGS_String * CSRA1_ReferenceGetBases ( CSRA1_Reference * self, ctx_t ctx,
}
struct NGS_String * CSRA1_ReferenceGetChunk ( CSRA1_Reference * self, ctx_t ctx, uint64_t offset, uint64_t size )
-{
+{ /* return maximum available contiguous bases starting from the offset */
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+ NGS_String* ret = NULL;
assert ( self );
if ( self -> curs == NULL )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
- return NULL;
}
- if ( ! self -> seen_first )
+ else if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return NULL;
}
-
- if ( offset >= CSRA1_ReferenceGetLength ( self, ctx ) )
+ else if ( offset >= CSRA1_ReferenceGetLength ( self, ctx ) )
{
- return NGS_StringMake ( ctx, "", 0 );
+ ret = NGS_StringMake ( ctx, "", 0 );
}
else
{
- const NGS_String* read = NGS_CursorGetString ( self -> curs, ctx, self -> first_row + offset / self -> chunk_size, reference_READ);
- NGS_String* ret;
- if ( size == (size_t)-1 )
- ret = NGS_StringSubstrOffset ( read, ctx, offset % self -> chunk_size );
+ uint64_t totalBases = CSRA1_ReferenceGetLength ( self, ctx );
+ if ( offset >= totalBases )
+ {
+ return NGS_StringMake ( ctx, "", 0 );
+ }
else
- ret = NGS_StringSubstrOffsetSize ( read, ctx, offset % self -> chunk_size, size );
- NGS_StringRelease ( read, ctx );
- return ret;
+ {
+ int64_t rowId = self -> first_row + offset / self -> chunk_size;
+ TRY ( const VBlob* blob = NGS_CursorGetVBlob ( self -> curs, ctx, rowId, reference_READ ) )
+ {
+ rc_t rc;
+ const void* data;
+ uint64_t cont_size;
+ TRY ( VByteBlob_ContiguousChunk ( blob, ctx, rowId, &data, &cont_size, true ) ) /* stop at a repeated row */
+ {
+ uint64_t offsetInBlob = offset % self -> chunk_size;
+ if ( size == (uint64_t)-1 || offsetInBlob + size > cont_size )
+ {
+ size = cont_size - offsetInBlob;
+ }
+ if ( offset + size > totalBases )
+ { /* when requested more bases than there are in the reference, be careful not to return a part of the next reference sitting in the same blob */
+ size = totalBases - offset;
+ }
+ ret = NGS_StringMakeCopy ( ctx, (const char*)data + offsetInBlob, size ); /* have to make a copy since otherwise would have to hold on to the entire blob, with no idea when to release it */
+ }
+
+ rc = VBlobRelease ( (VBlob*) blob );
+ if ( rc != 0 )
+ {
+ INTERNAL_ERROR ( xcUnexpected, "VBlobRelease() rc = %R", rc );
+ }
+ }
+ }
}
+ return ret;
}
struct NGS_Alignment* CSRA1_ReferenceGetAlignment ( CSRA1_Reference * self, ctx_t ctx, const char * alignmentIdStr )
-{
+{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
assert ( self );
@@ -384,7 +418,7 @@ struct NGS_Alignment* CSRA1_ReferenceGetAlignment ( CSRA1_Reference * self, ctx_
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return NULL;
+ return NULL;
}
{
@@ -394,9 +428,9 @@ struct NGS_Alignment* CSRA1_ReferenceGetAlignment ( CSRA1_Reference * self, ctx_
{
TRY ( NGS_String * commonName = CSRA1_ReferenceGetCommonName ( self, ctx ) )
{
- if ( string_cmp( NGS_StringData ( spec, ctx ),
+ if ( string_cmp( NGS_StringData ( spec, ctx ),
NGS_StringSize ( spec, ctx ),
- NGS_StringData ( commonName, ctx ),
+ NGS_StringData ( commonName, ctx ),
NGS_StringSize ( commonName, ctx ),
(uint32_t)NGS_StringSize ( spec, ctx ) ) == 0 )
{
@@ -404,12 +438,12 @@ struct NGS_Alignment* CSRA1_ReferenceGetAlignment ( CSRA1_Reference * self, ctx_
NGS_StringRelease ( commonName, ctx );
return ref;
}
-
- USER_ERROR ( xcWrongReference,
- "Requested alignment is on a wrong reference: reference '%.*s', alignment has '%.*s'",
+
+ USER_ERROR ( xcWrongReference,
+ "Requested alignment is on a wrong reference: reference '%.*s', alignment has '%.*s'",
NGS_StringSize ( commonName, ctx ), NGS_StringData ( commonName, ctx ),
NGS_StringSize ( spec, ctx ), NGS_StringData ( spec, ctx ) );
-
+
NGS_StringRelease ( commonName, ctx );
}
NGS_StringRelease ( spec, ctx );
@@ -421,7 +455,7 @@ struct NGS_Alignment* CSRA1_ReferenceGetAlignment ( CSRA1_Reference * self, ctx_
}
struct NGS_Alignment* CSRA1_ReferenceGetAlignments ( CSRA1_Reference * self, ctx_t ctx, bool wants_primary, bool wants_secondary, uint32_t filters, int32_t map_qual )
-{
+{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
assert ( self );
@@ -433,7 +467,7 @@ struct NGS_Alignment* CSRA1_ReferenceGetAlignments ( CSRA1_Reference * self, ctx
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return NULL;
+ return NULL;
}
{
@@ -444,26 +478,26 @@ struct NGS_Alignment* CSRA1_ReferenceGetAlignments ( CSRA1_Reference * self, ctx
/* wants_with_window does not matter, this is not a slice */
filters &= ~ NGS_AlignmentFilterBits_start_within_window;
- return CSRA1_ReferenceWindowMake ( ctx,
- self -> dad . coll,
+ return CSRA1_ReferenceWindowMake ( ctx,
+ self -> dad . coll,
self -> curs,
circular,
ref_len,
self -> chunk_size,
- self -> first_row,
- self -> first_row,
- self -> last_row + 1,
+ self -> first_row,
+ self -> first_row,
+ self -> last_row + 1,
0,
0,
- wants_primary,
+ wants_primary,
wants_secondary,
- filters,
+ filters,
map_qual,
self -> align_id_offset );
}
}
}
-
+
return NULL;
}
@@ -513,16 +547,16 @@ uint64_t CSRA1_ReferenceGetAlignmentCount ( const CSRA1_Reference * self, ctx_t
}
/*
- Calculate starting reference chunk to cover alignments overlapping with the slice;
+ Calculate starting reference chunk to cover alignments overlapping with the slice;
separately for primary and secondary alignments
*/
static
-void LoadOverlaps ( CSRA1_Reference * self,
- ctx_t ctx,
+void LoadOverlaps ( CSRA1_Reference * self,
+ ctx_t ctx,
uint32_t chunk_size,
- uint64_t offset,
- int64_t * primary_begin,
- int64_t * secondary_begin)
+ uint64_t offset,
+ int64_t * primary_begin,
+ int64_t * secondary_begin)
{
int64_t first_row = self -> first_row + offset / chunk_size;
uint32_t primary_len;
@@ -530,34 +564,34 @@ void LoadOverlaps ( CSRA1_Reference * self,
int32_t primary_pos;
int32_t secondary_pos;
uint32_t offset_in_chunk = offset % chunk_size;
-
+
{ /*OVERLAP_REF_LEN*/
const void* base;
uint32_t elem_bits, boff, row_len;
ON_FAIL ( NGS_CursorCellDataDirect ( self -> curs, ctx, first_row, reference_OVERLAP_REF_LEN, & elem_bits, & base, & boff, & row_len ) )
{ /* no overlap columns, apply 10-chunk lookback */
CLEAR ();
- if ( first_row > 11 )
+ if ( first_row > 11 )
{
- *primary_begin =
+ *primary_begin =
*secondary_begin = first_row - 10;
}
else
{
- *primary_begin =
+ *primary_begin =
*secondary_begin = 1;
}
return;
}
-
+
assert ( elem_bits == 32 );
assert ( boff == 0 );
assert ( row_len == 3 );
-
+
primary_len = ( (const uint32_t*)base ) [0];
secondary_len = ( (const uint32_t*)base ) [1];
}
-
+
if (primary_len == 0 && secondary_len == 0)
{
*primary_begin = *secondary_begin = first_row;
@@ -568,14 +602,14 @@ void LoadOverlaps ( CSRA1_Reference * self,
uint32_t elem_bits, boff, row_len;
ON_FAIL( NGS_CursorCellDataDirect ( self -> curs, ctx, first_row, reference_OVERLAP_REF_POS, & elem_bits, & base, & boff, & row_len ) )
return;
-
+
assert ( elem_bits == 32 );
assert ( boff == 0 );
assert ( row_len == 3 );
-
+
primary_pos = ( (const int32_t*)base ) [0];
secondary_pos = ( (const int32_t*)base ) [1];
-
+
if ( primary_len == 0 || primary_len < offset_in_chunk )
{
* primary_begin = first_row;
@@ -584,7 +618,7 @@ void LoadOverlaps ( CSRA1_Reference * self,
{
* primary_begin = self -> first_row + primary_pos / chunk_size;
}
-
+
if ( secondary_len == 0 || secondary_len < offset_in_chunk )
{
* secondary_begin = first_row;
@@ -594,18 +628,18 @@ void LoadOverlaps ( CSRA1_Reference * self,
* secondary_begin = self -> first_row + secondary_pos / chunk_size;
}
}
-}
+}
-struct NGS_Alignment* CSRA1_ReferenceGetAlignmentSlice ( CSRA1_Reference * self,
- ctx_t ctx,
- uint64_t offset,
- uint64_t size,
- bool wants_primary,
+struct NGS_Alignment* CSRA1_ReferenceGetAlignmentSlice ( CSRA1_Reference * self,
+ ctx_t ctx,
+ uint64_t offset,
+ uint64_t size,
+ bool wants_primary,
bool wants_secondary,
uint32_t filters,
int32_t map_qual )
{
- FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
assert ( self );
if ( self -> curs == NULL )
@@ -616,9 +650,9 @@ struct NGS_Alignment* CSRA1_ReferenceGetAlignmentSlice ( CSRA1_Reference * self,
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return NULL;
+ return NULL;
}
-
+
if ( size == 0 )
{
NGS_Alignment* ret = NGS_AlignmentMakeNull ( ctx, "", 0 );
@@ -631,20 +665,20 @@ struct NGS_Alignment* CSRA1_ReferenceGetAlignmentSlice ( CSRA1_Reference * self,
TRY ( uint64_t ref_len = CSRA1_ReferenceGetLength ( self, ctx ) )
{
if ( circular )
- { /* for a circular reference, always look at the whole of it
+ { /* for a circular reference, always look at the whole of it
(to account for alignments that start near the end and overlap with the first chunk) */
- return CSRA1_ReferenceWindowMake ( ctx,
- self -> dad . coll,
+ return CSRA1_ReferenceWindowMake ( ctx,
+ self -> dad . coll,
self -> curs,
true, /* circular */
ref_len,
self -> chunk_size,
self->first_row, /*primary_begin*/
self->first_row, /*secondary_begin*/
- self -> last_row + 1,
+ self -> last_row + 1,
offset,
size,
- wants_primary,
+ wants_primary,
wants_secondary,
filters,
map_qual,
@@ -662,19 +696,19 @@ struct NGS_Alignment* CSRA1_ReferenceGetAlignmentSlice ( CSRA1_Reference * self,
int64_t end = self -> first_row + ( offset + size - 1 ) / self -> chunk_size + 1;
if ( end > self -> last_row )
end = self -> last_row + 1;
-
- return CSRA1_ReferenceWindowMake ( ctx,
- self -> dad . coll,
+
+ return CSRA1_ReferenceWindowMake ( ctx,
+ self -> dad . coll,
self -> curs,
false,
ref_len,
self -> chunk_size,
- primary_begin,
+ primary_begin,
secondary_begin,
- end,
+ end,
offset,
size,
- wants_primary,
+ wants_primary,
wants_secondary,
filters,
map_qual,
@@ -685,10 +719,10 @@ struct NGS_Alignment* CSRA1_ReferenceGetAlignmentSlice ( CSRA1_Reference * self,
}
return NULL;
}
-}
+}
struct NGS_Pileup* CSRA1_ReferenceGetPileups ( CSRA1_Reference * self, ctx_t ctx, bool wants_primary, bool wants_secondary, uint32_t filters, int32_t map_qual )
-{
+{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
assert ( self );
@@ -700,16 +734,16 @@ struct NGS_Pileup* CSRA1_ReferenceGetPileups ( CSRA1_Reference * self, ctx_t ctx
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return NULL;
+ return NULL;
}
- return CSRA1_PileupIteratorMake ( ctx,
+ return CSRA1_PileupIteratorMake ( ctx,
& self -> dad,
- self -> db,
+ self -> db,
self -> curs,
CSRA1_Reference_GetFirstRowId ( (NGS_Reference const*)self, ctx ),
CSRA1_Reference_GetLastRowId ( (NGS_Reference const*)self, ctx ),
- wants_primary,
+ wants_primary,
wants_secondary,
filters,
map_qual );
@@ -728,18 +762,18 @@ static struct NGS_Pileup* CSRA1_ReferenceGetPileupSlice ( CSRA1_Reference * self
if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Reference accessed before a call to ReferenceIteratorNext()" );
- return NULL;
+ return NULL;
}
-
- return CSRA1_PileupIteratorMakeSlice ( ctx,
+
+ return CSRA1_PileupIteratorMakeSlice ( ctx,
& self -> dad,
- self -> db,
+ self -> db,
self -> curs,
CSRA1_Reference_GetFirstRowId ( (NGS_Reference const*)self, ctx ),
CSRA1_Reference_GetLastRowId ( (NGS_Reference const*)self, ctx ),
offset,
size,
- wants_primary,
+ wants_primary,
wants_secondary,
filters,
map_qual );
@@ -790,14 +824,14 @@ bool CSRA1_ReferenceFind ( NGS_Cursor const * curs, ctx_t ctx, const char * spec
uint64_t total_row_count;
size_t spec_size = string_size ( spec );
-
+
TRY ( NGS_CursorGetRowRange ( curs, ctx, & cur_row, & total_row_count ) )
{
const void * prev_NAME_base = NULL;
const void * prev_SEQ_ID_base = NULL;
end_row = cur_row + total_row_count;
while ( cur_row < end_row )
- {
+ {
const void * base;
uint32_t elem_bits, boff, row_len;
@@ -805,7 +839,7 @@ bool CSRA1_ReferenceFind ( NGS_Cursor const * curs, ctx_t ctx, const char * spec
ON_FAIL ( NGS_CursorCellDataDirect ( curs, ctx, cur_row, reference_NAME, & elem_bits, & base, & boff, & row_len ) )
return false;
- /* if the value has not changed, the base ptr will not be updated */
+ /* if the value has not changed, the base ptr will not be updated */
if ( prev_NAME_base != base )
{
if ( ( size_t ) row_len == spec_size )
@@ -828,7 +862,7 @@ bool CSRA1_ReferenceFind ( NGS_Cursor const * curs, ctx_t ctx, const char * spec
ON_FAIL ( NGS_CursorCellDataDirect ( curs, ctx, cur_row, reference_SEQ_ID, & elem_bits, & base, & boff, & row_len ) )
return false;
- /* if the value has not changed, the base ptr will not be updated */
+ /* if the value has not changed, the base ptr will not be updated */
if ( prev_SEQ_ID_base != base )
{
if ( ( size_t ) row_len == spec_size )
@@ -851,14 +885,14 @@ bool CSRA1_ReferenceFind ( NGS_Cursor const * curs, ctx_t ctx, const char * spec
}
}
}
-
+
return false;
}
-NGS_Reference * CSRA1_ReferenceMake ( ctx_t ctx,
+NGS_Reference * CSRA1_ReferenceMake ( ctx_t ctx,
struct NGS_ReadCollection * coll,
const struct VDatabase * db,
- const struct NGS_Cursor * curs,
+ const struct NGS_Cursor * curs,
const char * spec,
uint64_t align_id_offset )
{
@@ -866,7 +900,7 @@ NGS_Reference * CSRA1_ReferenceMake ( ctx_t ctx,
assert ( coll != NULL );
assert ( curs != NULL );
-
+
{
TRY ( NGS_String * collName = NGS_ReadCollectionGetName ( coll, ctx ) )
{
@@ -892,13 +926,13 @@ NGS_Reference * CSRA1_ReferenceMake ( ctx_t ctx,
TRY ( CSRA1_ReferenceInit ( ctx, ref, coll, "CSRA1_Reference", instname, align_id_offset ) )
{
uint64_t rowCount;
-
+
ref -> curs = NGS_CursorDuplicate ( curs, ctx );
ref -> db = db;
VDatabaseAddRef ( ref -> db );
-
+
/* find requested name */
if ( CSRA1_ReferenceFind ( ref -> curs, ctx, spec, & ref -> first_row, & rowCount ) )
{
@@ -910,8 +944,8 @@ NGS_Reference * CSRA1_ReferenceMake ( ctx_t ctx,
NGS_StringRelease ( collName, ctx );
return ( NGS_Reference * ) ref;
}
- }
-
+ }
+
INTERNAL_ERROR ( xcRowNotFound, "Reference not found ( NAME = %s )", spec );
CSRA1_ReferenceWhack ( ref, ctx );
}
@@ -931,7 +965,7 @@ NGS_Reference * CSRA1_ReferenceMake ( ctx_t ctx,
/* Make
*/
-NGS_Reference * CSRA1_ReferenceIteratorMake ( ctx_t ctx,
+NGS_Reference * CSRA1_ReferenceIteratorMake ( ctx_t ctx,
struct NGS_ReadCollection * coll,
const struct VDatabase * db,
const struct NGS_Cursor * curs,
@@ -942,7 +976,7 @@ NGS_Reference * CSRA1_ReferenceIteratorMake ( ctx_t ctx,
assert ( coll != NULL );
assert ( db != NULL );
assert ( curs != NULL );
-
+
{
TRY ( NGS_String * collName = NGS_ReadCollectionGetName ( coll, ctx ) )
{
@@ -967,12 +1001,12 @@ NGS_Reference * CSRA1_ReferenceIteratorMake ( ctx_t ctx,
TRY ( CSRA1_ReferenceInit ( ctx, ref, coll, "CSRA1_Reference", instname, align_id_offset ) )
{
uint64_t row_count;
-
+
ref -> curs = NGS_CursorDuplicate ( curs, ctx );
ref -> db = db;
VDatabaseAddRef ( ref -> db );
-
+
TRY ( NGS_CursorGetRowRange ( ref -> curs, ctx, & ref -> first_row, & row_count ) )
{
TRY ( ref -> chunk_size = NGS_CursorGetUInt32 ( ref -> curs, ctx, ref -> first_row, reference_MAX_SEQ_LEN ) )
@@ -1001,12 +1035,12 @@ NGS_Reference * CSRA1_ReferenceIteratorMake ( ctx_t ctx,
bool CSRA1_ReferenceIteratorNext ( CSRA1_Reference * self, ctx_t ctx )
{
assert ( self != NULL );
-
+
if ( self -> curs == NULL || self -> first_row > self -> iteration_row_last)
return false; /* iteration over or not initialized */
self -> cur_length = 0;
-
+
if ( self -> seen_first )
{ /* skip to the next reference */
self -> first_row = self -> last_row + 1;
@@ -1020,20 +1054,20 @@ bool CSRA1_ReferenceIteratorNext ( CSRA1_Reference * self, ctx_t ctx )
{ /* first reference */
self -> seen_first = true;
}
-
+
{ /* update self -> last_row */
const void * refNameBase = NULL;
uint32_t nameLength;
{ /* get the new reference's name */
uint32_t elem_bits, boff;
- ON_FAIL ( NGS_CursorCellDataDirect ( self -> curs, ctx, self -> first_row, reference_NAME,
+ ON_FAIL ( NGS_CursorCellDataDirect ( self -> curs, ctx, self -> first_row, reference_NAME,
& elem_bits, & refNameBase, & boff, & nameLength ) )
return false;
assert ( elem_bits == 8 );
assert ( boff == 0 );
}
-
+
{ /* use index on reference name if available */
uint64_t rowCount;
rc_t rc = 1; /* != 0 in case the following TRY fails */
@@ -1052,17 +1086,17 @@ bool CSRA1_ReferenceIteratorNext ( CSRA1_Reference * self, ctx_t ctx )
free ( key );
}
}
-
+
CLEAR();
-
+
if ( rc != 0 )
{ /* index is not available, do a table scan */
rowCount = CountRows ( self -> curs, ctx, reference_NAME, refNameBase, nameLength, self -> first_row, self -> iteration_row_last );
}
-
+
self -> last_row = self -> first_row + rowCount - 1;
}
}
-
+
return true;
}
diff --git a/libs/ngs/CSRA1_ReferenceWindow.c b/libs/ngs/CSRA1_ReferenceWindow.c
index 3ef6f87..71a1786 100644
--- a/libs/ngs/CSRA1_ReferenceWindow.c
+++ b/libs/ngs/CSRA1_ReferenceWindow.c
@@ -57,7 +57,7 @@ typedef struct CSRA1_ReferenceWindow CSRA1_ReferenceWindow;
#include <limits.h>
#ifndef min
-# define min(a,b) ( (a) < (b) ? (a) : (b) )
+# define min(a,b) ( (a) < (b) ? (a) : (b) )
#endif
/*--------------------------------------------------------------------------
@@ -70,7 +70,7 @@ enum
};
struct AlignmentInfo
{
- int64_t id;
+ int64_t id;
/* sort order */
int64_t pos; /* asc */
uint64_t len; /* desc */
@@ -81,25 +81,25 @@ typedef struct AlignmentInfo AlignmentInfo;
struct CSRA1_ReferenceWindow
{
- NGS_Refcount dad;
+ NGS_Refcount dad;
NGS_ReadCollection * coll;
-
+
const NGS_Cursor * reference_curs;
-
+
bool circular;
bool primary;
bool secondary;
uint32_t filters; /* uses NGS_AlignmentFilterBits from NGS_Alignment.h */
int32_t map_qual;
-
+
uint32_t chunk_size;
uint64_t ref_length; /* total reference length in bases */
uint64_t id_offset;
-
+
/* remaining range of chunks in the reference table */
int64_t ref_begin;
int64_t ref_end;
-
+
/* for use in a slice iterator: */
/* slice (0, 0) = all */
uint64_t slice_offset;
@@ -107,12 +107,12 @@ struct CSRA1_ReferenceWindow
/* starting chunks for primary/secondary tables */
int64_t ref_primary_begin;
int64_t ref_secondary_begin;
-
+
/* false - not positioned on any chunk */
bool seen_first;
-
+
/* alignments against current chunk, sorted in canonical order */
- AlignmentInfo* align_info;
+ AlignmentInfo* align_info;
size_t align_info_cur;
size_t align_info_total;
NGS_Alignment* cur_align; /* cached current alignment, corresponds to align_info_cur */
@@ -149,7 +149,7 @@ static
void CSRA1_ReferenceWindowWhack ( CSRA1_ReferenceWindow * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcDestroying );
-
+
NGS_AlignmentRelease ( self -> cur_align, ctx );
free ( self -> align_info );
NGS_CursorRelease ( self -> reference_curs, ctx );
@@ -158,20 +158,20 @@ void CSRA1_ReferenceWindowWhack ( CSRA1_ReferenceWindow * self, ctx_t ctx )
static
NGS_Alignment* GetAlignment ( CSRA1_ReferenceWindow* self, ctx_t ctx )
-{
- if ( self -> seen_first &&
+{
+ if ( self -> seen_first &&
( self -> circular || self -> ref_begin < self ->ref_end ) && /* for circular references, all chunks are loaded at once */
self -> align_info_cur < self -> align_info_total )
- {
+ {
if ( self -> cur_align == NULL )
{
TRY ( NGS_String * run_name = NGS_ReadCollectionGetName ( self -> coll, ctx ) )
{
- TRY ( const NGS_String * id = NGS_IdMake ( ctx,
- run_name,
- self -> align_info [ self -> align_info_cur ] . cat == Primary ?
+ TRY ( const NGS_String * id = NGS_IdMake ( ctx,
+ run_name,
+ self -> align_info [ self -> align_info_cur ] . cat == Primary ?
NGSObject_PrimaryAlignment:
- NGSObject_SecondaryAlignment,
+ NGSObject_SecondaryAlignment,
self -> align_info [ self -> align_info_cur ] . id /* + self -> id_offset ? */) )
{
self -> cur_align = NGS_ReadCollectionGetAlignment ( self -> coll, ctx, NGS_StringData ( id, ctx ) );
@@ -187,11 +187,11 @@ NGS_Alignment* GetAlignment ( CSRA1_ReferenceWindow* self, ctx_t ctx )
}
-static
+static
NGS_String* CSRA1_FragmentGetId ( CSRA1_ReferenceWindow * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_FragmentGetId ( (NGS_Fragment*)ref, ctx );
@@ -199,7 +199,7 @@ NGS_String* CSRA1_FragmentGetId ( CSRA1_ReferenceWindow * self, ctx_t ctx )
return NULL;
}
-static
+static
struct NGS_String * CSRA1_FragmentGetSequence ( CSRA1_ReferenceWindow * self, ctx_t ctx, uint64_t offset, uint64_t length )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
@@ -210,7 +210,7 @@ struct NGS_String * CSRA1_FragmentGetSequence ( CSRA1_ReferenceWindow * self, ct
return NULL;
}
-static
+static
struct NGS_String * CSRA1_FragmentGetQualities ( CSRA1_ReferenceWindow * self, ctx_t ctx, uint64_t offset, uint64_t length )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
@@ -221,7 +221,7 @@ struct NGS_String * CSRA1_FragmentGetQualities ( CSRA1_ReferenceWindow * self, c
return NULL;
}
-static
+static
bool CSRA1_FragmentIsPaired ( CSRA1_ReferenceWindow * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
@@ -232,14 +232,14 @@ bool CSRA1_FragmentIsPaired ( CSRA1_ReferenceWindow * self, ctx_t ctx )
return false;
}
-static
+static
bool CSRA1_FragmentIsAligned ( CSRA1_ReferenceWindow * self, ctx_t ctx )
{
assert ( self != NULL );
return true;
}
-static
+static
bool CSRA1_FragmentNext ( CSRA1_ReferenceWindow * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
@@ -263,7 +263,7 @@ static
struct NGS_String* CSRA1_ReferenceWindowGetReferenceSpec( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetReferenceSpec ( ref, ctx );
@@ -275,7 +275,7 @@ static
int CSRA1_ReferenceWindowGetMappingQuality( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetMappingQuality ( ref, ctx );
@@ -287,7 +287,7 @@ static
INSDC_read_filter CSRA1_ReferenceWindowGetReadFilter( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetReadFilter ( ref, ctx );
@@ -299,7 +299,7 @@ static
struct NGS_String* CSRA1_ReferenceWindowGetReferenceBases( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetReferenceBases ( ref, ctx );
@@ -311,7 +311,7 @@ static
struct NGS_String* CSRA1_ReferenceWindowGetReadGroup( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetReadGroup ( ref, ctx );
@@ -323,7 +323,7 @@ static
NGS_String * CSRA1_ReferenceWindowGetReadId( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetReadId ( ref, ctx );
@@ -335,7 +335,7 @@ static
struct NGS_String* CSRA1_ReferenceWindowGetClippedFragmentBases( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetClippedFragmentBases ( ref, ctx );
@@ -347,7 +347,7 @@ static
struct NGS_String* CSRA1_ReferenceWindowGetClippedFragmentQualities( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetClippedFragmentQualities ( ref, ctx );
@@ -359,7 +359,7 @@ static
struct NGS_String* CSRA1_ReferenceWindowGetAlignedFragmentBases( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetAlignedFragmentBases ( ref, ctx );
@@ -371,7 +371,7 @@ static
bool CSRA1_ReferenceWindowIsPrimary( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentIsPrimary ( ref, ctx );
@@ -383,7 +383,7 @@ static
int64_t CSRA1_ReferenceWindowGetAlignmentPosition( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetAlignmentPosition ( ref, ctx );
@@ -395,7 +395,7 @@ static
uint64_t CSRA1_ReferenceWindowGetReferencePositionProjectionRange( CSRA1_ReferenceWindow* self, ctx_t ctx, int64_t ref_pos )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetReferencePositionProjectionRange ( ref, ctx, ref_pos );
@@ -407,7 +407,7 @@ static
uint64_t CSRA1_ReferenceWindowGetAlignmentLength( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetAlignmentLength ( ref, ctx );
@@ -419,7 +419,7 @@ static
bool CSRA1_ReferenceWindowGetIsReversedOrientation( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetIsReversedOrientation ( ref, ctx );
@@ -431,7 +431,7 @@ static
int CSRA1_ReferenceWindowGetSoftClip( CSRA1_ReferenceWindow* self, ctx_t ctx, bool left )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetSoftClip ( ref, ctx, left );
@@ -443,7 +443,7 @@ static
uint64_t CSRA1_ReferenceWindowGetTemplateLength( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetTemplateLength ( ref, ctx );
@@ -455,7 +455,7 @@ static
struct NGS_String* CSRA1_ReferenceWindowGetShortCigar( CSRA1_ReferenceWindow* self, ctx_t ctx, bool clipped )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetShortCigar ( ref, ctx, clipped );
@@ -467,7 +467,7 @@ static
struct NGS_String* CSRA1_ReferenceWindowGetLongCigar( CSRA1_ReferenceWindow* self, ctx_t ctx, bool clipped )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetLongCigar ( ref, ctx, clipped );
@@ -479,7 +479,7 @@ static
char CSRA1_ReferenceWindowGetRNAOrientation( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetRNAOrientation ( ref, ctx );
@@ -491,7 +491,7 @@ static
bool CSRA1_ReferenceWindowHasMate( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentHasMate ( ref, ctx );
@@ -504,7 +504,7 @@ static
struct NGS_String* CSRA1_ReferenceWindowGetMateAlignmentId( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetMateAlignmentId ( ref, ctx );
@@ -516,7 +516,7 @@ static
CSRA1_ReferenceWindow* CSRA1_ReferenceWindowGetMateAlignment( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return ( CSRA1_ReferenceWindow * ) NGS_AlignmentGetMateAlignment( ref, ctx );
@@ -528,7 +528,7 @@ static
struct NGS_String* CSRA1_ReferenceWindowGetMateReferenceSpec( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetMateReferenceSpec ( ref, ctx );
@@ -540,7 +540,7 @@ static
bool CSRA1_ReferenceWindowGetMateIsReversedOrientation( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
{
return NGS_AlignmentGetMateIsReversedOrientation ( ref, ctx );
@@ -548,6 +548,18 @@ bool CSRA1_ReferenceWindowGetMateIsReversedOrientation( CSRA1_ReferenceWindow* s
return false;
}
+static
+bool CSRA1_ReferenceWindowIsFirst( CSRA1_ReferenceWindow* self, ctx_t ctx )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+
+ TRY ( NGS_Alignment* ref = GetAlignment ( self, ctx ) )
+ {
+ return NGS_AlignmentIsFirst ( ref, ctx );
+ }
+ return false;
+}
+
/*--------------------------------------------------------------------------
* Iterator
*/
@@ -556,19 +568,19 @@ int64_t AlignmentSort ( const void * p_a, const void * p_b, void *data )
{
const struct AlignmentInfo* a = ( const struct AlignmentInfo * ) p_a;
const struct AlignmentInfo* b = ( const struct AlignmentInfo * ) p_b;
-
+
if ( a -> pos < b -> pos )
return -1;
else if ( a -> pos > b -> pos )
return 1;
-
+
/* cannot use uint64_t - uint64_t because of possible overflow */
if ( a -> len < b -> len ) return 1;
if ( a -> len > b -> len ) return -1;
-
+
if ( a -> cat != b -> cat )
return (int64_t) a -> cat - (int64_t) b -> cat;
-
+
/* sort by mapq in reverse order */
if ( a -> mapq != b -> mapq )
return (int64_t) b -> mapq - (int64_t) a -> mapq;
@@ -581,21 +593,21 @@ static
void LoadAlignmentInfo ( CSRA1_ReferenceWindow* self, ctx_t ctx, size_t* idx, int64_t id, bool primary, int64_t offset, uint64_t size )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
- TRY ( NGS_Alignment* al = CSRA1_AlignmentMake ( ctx,
- ( struct CSRA1_ReadCollection * ) self -> coll,
- id,
- "", 0,
- primary,
+
+ TRY ( NGS_Alignment* al = CSRA1_AlignmentMake ( ctx,
+ ( struct CSRA1_ReadCollection * ) self -> coll,
+ id,
+ "", 0,
+ primary,
self -> id_offset ) )
{
int64_t pos = NGS_AlignmentGetAlignmentPosition ( al, ctx );
int64_t len = (int64_t) NGS_AlignmentGetAlignmentLength ( al, ctx );
-
+
if ( ! CSRA1_ReferenceWindowFilterStartWithinWindow ( self ) || pos >= offset )
{
bool overlaps = true;
-
+
if ( size > 0 )
{ /* a slice*/
int64_t end_slice = offset + (int64_t)size;
@@ -605,11 +617,11 @@ void LoadAlignmentInfo ( CSRA1_ReferenceWindow* self, ctx_t ctx, size_t* idx, in
}
if ( ! CSRA1_ReferenceWindowFilterStartWithinWindow ( self ) &&
! CSRA1_ReferenceWindowFilterNoWraparound ( self ) &&
- pos + len >= (int64_t) self -> ref_length )
+ pos + len >= (int64_t) self -> ref_length )
{ /* account for possible carryover on a circular reference */
pos -= self -> ref_length;
}
- overlaps = pos < end_slice && ( pos + len > offset );
+ overlaps = pos < end_slice && ( pos + len > offset );
}
/* use single-pass loop as a sort of sanctimonious goto mechanism */
@@ -678,7 +690,7 @@ void LoadAlignmentInfo ( CSRA1_ReferenceWindow* self, ctx_t ctx, size_t* idx, in
break;
}
}
-
+
NGS_AlignmentRelease ( al, ctx );
}
CATCH ( xcSecondaryAlignmentMissingPrimary )
@@ -692,13 +704,13 @@ void LoadAlignmentIndex ( CSRA1_ReferenceWindow* self, ctx_t ctx, int64_t row_id
{
const void * base;
uint32_t elem_bits, boff, row_len;
- TRY ( NGS_CursorCellDataDirect ( self -> reference_curs,
+ TRY ( NGS_CursorCellDataDirect ( self -> reference_curs,
ctx,
- row_id,
- id_col_idx,
- & elem_bits,
- & base,
- & boff,
+ row_id,
+ id_col_idx,
+ & elem_bits,
+ & base,
+ & boff,
& row_len ) )
{
assert ( elem_bits == 64 );
@@ -713,7 +725,7 @@ int64_t AlignmentSortCircular ( const void * p_a, const void * p_b, void *data )
{
const struct AlignmentInfo* a = ( const struct AlignmentInfo * ) p_a;
const struct AlignmentInfo* b = ( const struct AlignmentInfo * ) p_b;
-
+
uint64_t total = *(uint64_t*)data;
int64_t a_start = a -> pos;
int64_t b_start = b -> pos;
@@ -725,19 +737,19 @@ int64_t AlignmentSortCircular ( const void * p_a, const void * p_b, void *data )
{
b_start -= total;
}
-
+
if ( a_start < b_start )
return -1;
else if ( a_start > b_start )
return 1;
-
+
/* cannot use uint64_t - uint64_t because of possible overflow */
if ( a -> len < b -> len ) return 1;
if ( a -> len > b -> len ) return -1;
if ( a -> cat != b -> cat )
return (int64_t) a -> cat - (int64_t) b -> cat;
-
+
/* sort by mapq in reverse order */
if ( a -> mapq != b -> mapq )
return (int64_t) b -> mapq - (int64_t) a -> mapq;
@@ -756,15 +768,15 @@ void LoadAlignments ( CSRA1_ReferenceWindow* self, ctx_t ctx, int64_t chunk_row_
uint32_t total_added = 0;
if ( self -> primary && self -> ref_primary_begin <= chunk_row_id )
- {
- ON_FAIL ( LoadAlignmentIndex ( self, ctx, chunk_row_id, reference_PRIMARY_ALIGNMENT_IDS, & primary_idx, & primary_idx_end ) )
+ {
+ ON_FAIL ( LoadAlignmentIndex ( self, ctx, chunk_row_id, reference_PRIMARY_ALIGNMENT_IDS, & primary_idx, & primary_idx_end ) )
return;
- }
+ }
if ( self -> secondary && self -> ref_secondary_begin <= chunk_row_id )
- {
- ON_FAIL ( LoadAlignmentIndex ( self, ctx, chunk_row_id, reference_SECONDARY_ALIGNMENT_IDS, & secondary_idx, & secondary_idx_end ) )
- {
+ {
+ ON_FAIL ( LoadAlignmentIndex ( self, ctx, chunk_row_id, reference_SECONDARY_ALIGNMENT_IDS, & secondary_idx, & secondary_idx_end ) )
+ {
if ( GetRCObject ( ctx -> rc ) == rcColumn && GetRCState ( ctx -> rc ) == rcNotFound )
{ /* SECONDARY_ALIGNMENT_IDS is missing; no problem */
self -> secondary = false; /* do not try anymore */
@@ -775,13 +787,13 @@ void LoadAlignments ( CSRA1_ReferenceWindow* self, ctx_t ctx, int64_t chunk_row_
return;
}
}
- }
+ }
total_added = primary_idx_end + secondary_idx_end;
if ( total_added > 0 )
- {
+ {
self -> align_info = realloc ( self -> align_info, ( self -> align_info_total + total_added ) * sizeof ( * self -> align_info ) );
- if ( self -> align_info == NULL )
+ if ( self -> align_info == NULL )
{
SYSTEM_ERROR ( xcNoMemory, "allocating CSRA1_ReferenceWindow chunk" );
return;
@@ -791,12 +803,12 @@ void LoadAlignments ( CSRA1_ReferenceWindow* self, ctx_t ctx, int64_t chunk_row_
uint32_t i;
for ( i = 0; i < primary_idx_end; ++i )
{
- ON_FAIL ( LoadAlignmentInfo( self, ctx, & self -> align_info_total, primary_idx [ i ], true, offset, size ) )
+ ON_FAIL ( LoadAlignmentInfo( self, ctx, & self -> align_info_total, primary_idx [ i ], true, offset, size ) )
return;
}
for ( i = 0; i < secondary_idx_end; ++i )
{
- ON_FAIL ( LoadAlignmentInfo( self, ctx, & self -> align_info_total, secondary_idx [ i ] + self -> id_offset, false, offset, size ) )
+ ON_FAIL ( LoadAlignmentInfo( self, ctx, & self -> align_info_total, secondary_idx [ i ] + self -> id_offset, false, offset, size ) )
return;
}
}
@@ -810,10 +822,10 @@ bool LoadFirstCircular ( CSRA1_ReferenceWindow* self, ctx_t ctx )
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
int64_t last_chunk = self -> ref_end - 1;
assert ( self );
-
+
self -> align_info_total = 0;
- /* for windows on circular references, self->ref_begin and and self->ref_end - 1
+ /* for windows on circular references, self->ref_begin and and self->ref_end - 1
are the rowId's of the first and last chunk of the reference, regardless of slicing */
if ( ! CSRA1_ReferenceWindowFilterNoWraparound ( self ) && self -> ref_begin < last_chunk )
{ /* load the last chunk of the reference, to cover possible overlaps into the first chunk */
@@ -823,16 +835,16 @@ bool LoadFirstCircular ( CSRA1_ReferenceWindow* self, ctx_t ctx )
return false;
}
else if ( self -> slice_offset < self -> chunk_size )
- { /* loading possible overlaps with a slice inside the first chunk */
+ { /* loading possible overlaps with a slice inside the first chunk */
ON_FAIL ( LoadAlignments ( self, ctx, last_chunk, self -> slice_offset, self -> chunk_size - self -> slice_offset ) )
return false;
}
/* target slice is not in the first chunk, no need to look for overlaps from the end of the reference */
}
-
+
ON_FAIL ( LoadAlignments ( self, ctx, self -> ref_begin, self -> slice_offset, self -> slice_size ) )
return false;
-
+
if ( self -> align_info_total > 0 )
{
ksort ( self -> align_info, self -> align_info_total, sizeof ( * self -> align_info ), AlignmentSortCircular, & self -> ref_length );
@@ -846,56 +858,56 @@ static
bool LoadNextChunk ( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
assert ( self );
-
+
self -> align_info_total = 0;
while ( self -> ref_begin < self -> ref_end )
{
ON_FAIL ( LoadAlignments ( self, ctx, self -> ref_begin, self -> slice_offset, self -> slice_size ) )
return false;
-
+
if ( self -> align_info_total > 0 )
{
ksort ( self -> align_info, self -> align_info_total, sizeof ( * self -> align_info ), AlignmentSort, NULL );
self -> align_info_cur = 0;
-
+
return true;
}
/* this chunk had no alignments - move to the next one */
++ self -> ref_begin;
}
-
+
return false;
-}
-
+}
+
static
bool CSRA1_ReferenceWindowIteratorNext ( CSRA1_ReferenceWindow* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
if ( ! self -> seen_first )
{ /* first call - position on the first alignment */
self -> seen_first = true;
if ( self -> circular )
- {
+ {
return LoadFirstCircular ( self, ctx );
}
}
- else
+ else
{
/* clear cached alignment*/
NGS_AlignmentRelease ( self -> cur_align, ctx );
self -> cur_align = NULL;
-
+
++ self -> align_info_cur;
if ( self -> align_info_cur < self -> align_info_total )
return true;
-
+
++ self -> ref_begin;
}
-
+
return LoadNextChunk ( self, ctx );
}
@@ -913,8 +925,8 @@ static NGS_Alignment_vt CSRA1_ReferenceWindow_vt_inst =
CSRA1_FragmentIsPaired,
CSRA1_FragmentIsAligned,
CSRA1_FragmentNext
- },
-
+ },
+
CSRA1_ReferenceWindowGetAlignmentId,
CSRA1_ReferenceWindowGetReferenceSpec,
CSRA1_ReferenceWindowGetMappingQuality,
@@ -940,14 +952,15 @@ static NGS_Alignment_vt CSRA1_ReferenceWindow_vt_inst =
CSRA1_ReferenceWindowGetMateAlignment,
CSRA1_ReferenceWindowGetMateReferenceSpec,
CSRA1_ReferenceWindowGetMateIsReversedOrientation,
+ CSRA1_ReferenceWindowIsFirst,
/* Iterator */
CSRA1_ReferenceWindowIteratorNext
};
static
-void CSRA1_ReferenceWindowInit ( CSRA1_ReferenceWindow * ref,
- ctx_t ctx,
+void CSRA1_ReferenceWindowInit ( CSRA1_ReferenceWindow * ref,
+ ctx_t ctx,
NGS_ReadCollection * coll,
const struct NGS_Cursor* curs,
bool circular,
@@ -965,7 +978,7 @@ void CSRA1_ReferenceWindowInit ( CSRA1_ReferenceWindow * ref,
uint64_t id_offset )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
-
+
TRY ( NGS_AlignmentInit ( ctx, ref, & CSRA1_ReferenceWindow_vt_inst, "CSRA1_ReferenceWindow", "" ) )
{
TRY ( ref -> coll = (NGS_ReadCollection *) NGS_RefcountDuplicate ( & coll -> dad, ctx ) )
@@ -988,12 +1001,12 @@ void CSRA1_ReferenceWindowInit ( CSRA1_ReferenceWindow * ref,
ref -> slice_size = size;
}
}
-}
+}
/* MakeCommon
* makes a common alignment from VCursor
*/
-NGS_Alignment * CSRA1_ReferenceWindowMake ( ctx_t ctx,
+NGS_Alignment * CSRA1_ReferenceWindowMake ( ctx_t ctx,
struct NGS_ReadCollection * coll,
const struct NGS_Cursor* curs,
bool circular,
@@ -1021,23 +1034,23 @@ NGS_Alignment * CSRA1_ReferenceWindowMake ( ctx_t ctx,
SYSTEM_ERROR ( xcNoMemory, "allocating CSRA1_ReferenceWindow" );
else
{
- TRY ( CSRA1_ReferenceWindowInit ( ref,
- ctx,
- coll,
- curs,
+ TRY ( CSRA1_ReferenceWindowInit ( ref,
+ ctx,
+ coll,
+ curs,
circular,
ref_length,
chunk_size,
primary_begin_row,
secondary_begin_row,
- end_row,
+ end_row,
offset,
- size,
- primary,
+ size,
+ primary,
secondary,
filters,
map_qual,
- id_offset ) )
+ id_offset ) )
{
return ( NGS_Alignment * ) ref;
}
diff --git a/libs/ngs/EBI_ReferenceSequence.c b/libs/ngs/EBI_ReferenceSequence.c
index 367fc42..8dd1a71 100644
--- a/libs/ngs/EBI_ReferenceSequence.c
+++ b/libs/ngs/EBI_ReferenceSequence.c
@@ -68,7 +68,7 @@ static NGS_ReferenceSequence_vt EBI_ReferenceSequence_vt_inst =
{
/* NGS_Refcount */
{ EBI_ReferenceSequenceWhack }
-
+
/* NGS_ReferenceSequence */
,EBI_ReferenceSequenceGetCanonicalName
,EBI_ReferenceSequenceGetIsCircular
@@ -107,9 +107,9 @@ void EBI_ReferenceSequenceWhack ( EBI_ReferenceSequence * self, ctx_t ctx )
/* Init
*/
static
-void EBI_ReferenceSequenceInit ( ctx_t ctx,
+void EBI_ReferenceSequenceInit ( ctx_t ctx,
EBI_ReferenceSequence * ref,
- const char *clsname,
+ const char *clsname,
const char *instname )
{
FUNC_ENTRY ( ctx, rcNS, rcTable, rcOpening );
@@ -142,10 +142,14 @@ static bool is_md5 ( const char * spec )
return i == char_count;
}
+/* TBD - WHAT IS THIS? IT TAKES A CONTEXT AND RETURNS AN RC!!
+ THIS CODE CAN'T WORK AS INTENDED.
+*/
static rc_t NGS_ReferenceSequenceComposeEBIUrl ( ctx_t ctx, const char * spec, bool ismd5, char* url, size_t url_size )
{
+ /* TBD - obtain these from configuration */
char const url_templ_md5[] = "http://www.ebi.ac.uk/ena/cram/md5/%s";
- char const url_templ_acc[] = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nucleotide&rettype=fasta&id=%s";
+ char const url_templ_acc[] = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=nucleotide&rettype=fasta&id=%s";
size_t num_written = 0;
rc_t rc = string_printf ( url, url_size, & num_written, ismd5 ? url_templ_md5 : url_templ_acc, spec );
@@ -340,9 +344,9 @@ NGS_ReferenceSequence * NGS_ReferenceSequenceMakeEBI ( ctx_t ctx, const char * s
NGS_String * EBI_ReferenceSequenceGetCanonicalName ( EBI_ReferenceSequence * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcNS, rcDoc, rcReading );
-
+
assert ( self != NULL );
-
+
return NGS_StringDuplicate ( self -> ebi_ref_spec, ctx );
}
@@ -351,7 +355,7 @@ bool EBI_ReferenceSequenceGetIsCircular ( const EBI_ReferenceSequence * self, ct
FUNC_ENTRY ( ctx, rcNS, rcDoc, rcReading );
assert ( self );
-
+
return false;
}
@@ -365,11 +369,11 @@ uint64_t EBI_ReferenceSequenceGetLength ( EBI_ReferenceSequence * self, ctx_t ct
}
struct NGS_String * EBI_ReferenceSequenceGetBases ( EBI_ReferenceSequence * self, ctx_t ctx, uint64_t offset, uint64_t size )
-{
+{
FUNC_ENTRY ( ctx, rcNS, rcDoc, rcReading );
assert ( self );
-
+
{
uint64_t totalBases = EBI_ReferenceSequenceGetLength ( self, ctx );
if ( offset >= totalBases )
@@ -377,29 +381,20 @@ struct NGS_String * EBI_ReferenceSequenceGetBases ( EBI_ReferenceSequence * self
return NGS_StringMake ( ctx, "", 0 );
}
else
- {
+ {
uint64_t basesToReturn = totalBases - offset;
char* data;
-
+
if (size != (size_t)-1 && basesToReturn > size)
basesToReturn = size;
-
- data = (char*) malloc ( basesToReturn );
- if ( data == NULL )
- {
- SYSTEM_ERROR ( xcNoMemory, "allocating %lu bases", basesToReturn );
- return NGS_StringMake ( ctx, "", 0 );
- }
- else
- {
- return NGS_StringMakeCopy ( ctx, (const char*) self -> buf_ref_data + offset, basesToReturn );
- }
+
+ return NGS_StringMakeCopy ( ctx, (const char*) self -> buf_ref_data + offset, basesToReturn );
}
}
}
struct NGS_String * EBI_ReferenceSequenceGetChunk ( EBI_ReferenceSequence * self, ctx_t ctx, uint64_t offset, uint64_t size )
-{
+{
FUNC_ENTRY ( ctx, rcNS, rcDoc, rcReading );
assert ( self );
diff --git a/libs/ngs/Makefile b/libs/ngs/Makefile
index 72380ff..9fa6100 100644
--- a/libs/ngs/Makefile
+++ b/libs/ngs/Makefile
@@ -114,6 +114,7 @@ NGS_SRC = \
NGS_ErrBlock \
NGS_FragmentBlob \
NGS_FragmentBlobIterator \
+ VByteBlob \
NGS_OBJ = \
$(addsuffix .$(LOBX),$(NGS_SRC))
diff --git a/libs/ngs/NGS_Alignment.c b/libs/ngs/NGS_Alignment.c
index 2a80205..1bd0b2a 100644
--- a/libs/ngs/NGS_Alignment.c
+++ b/libs/ngs/NGS_Alignment.c
@@ -51,7 +51,7 @@
#define Self( obj ) \
( ( NGS_Alignment* ) ( obj ) )
-
+
static NGS_String_v1 * ITF_Alignment_v1_get_id ( const NGS_Alignment_v1 * self, NGS_ErrBlock_v1 * err )
{
HYBRID_FUNC_ENTRY ( rcSRA, rcRefcount, rcAccessing );
@@ -766,6 +766,20 @@ bool NGS_AlignmentGetMateIsReversedOrientation( NGS_Alignment* self, ctx_t ctx )
return false;
}
+bool NGS_AlignmentIsFirst ( NGS_Alignment* self, ctx_t ctx )
+{
+ if ( self == NULL )
+ {
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcAccessing );
+ INTERNAL_ERROR ( xcSelfNull, "NGS_AlignmentIsFirst failed" );
+ }
+ else
+ {
+ return VT ( self, isFirst ) ( self, ctx );
+ }
+ return false;
+}
+
/*--------------------------------------------------------------------------
* NGS_AlignmentIterator
*/
@@ -793,7 +807,7 @@ void NullAlignmentWhack ( NGS_Alignment * self, ctx_t ctx )
{
}
-static
+static
struct NGS_String * NullAlignment_FragmentToString ( NGS_Alignment * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
@@ -801,7 +815,7 @@ struct NGS_String * NullAlignment_FragmentToString ( NGS_Alignment * self, ctx_t
return 0;
}
-static
+static
struct NGS_String * NullAlignment_FragmentOffsetLenToString ( NGS_Alignment * self, ctx_t ctx, uint64_t offset, uint64_t length )
{
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
@@ -809,7 +823,7 @@ struct NGS_String * NullAlignment_FragmentOffsetLenToString ( NGS_Alignment * se
return 0;
}
-static
+static
bool NullAlignment_FragmentToBool ( NGS_Alignment * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
@@ -818,14 +832,14 @@ bool NullAlignment_FragmentToBool ( NGS_Alignment * self, ctx_t ctx )
}
-static int64_t NullAlignment_toI64 ( NGS_ALIGNMENT* self, ctx_t ctx )
+static int64_t NullAlignment_toI64 ( NGS_ALIGNMENT* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
INTERNAL_ERROR ( xcSelfNull, "NULL Alignment accessed" );
return 0;
}
-static struct NGS_String* NullAlignment_toString ( NGS_ALIGNMENT* self, ctx_t ctx )
+static struct NGS_String* NullAlignment_toString ( NGS_ALIGNMENT* self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
INTERNAL_ERROR ( xcSelfNull, "NULL Alignment accessed" );
@@ -914,7 +928,7 @@ static NGS_Alignment_vt NullAlignment_vt_inst =
NullAlignment_FragmentToBool,
NullAlignment_FragmentToBool
},
-
+
NullAlignment_toString, /* getId */
NullAlignment_toString, /* getReferenceSpec */
NullAlignment_toInt, /* getMappingQuality */
@@ -940,6 +954,7 @@ static NGS_Alignment_vt NullAlignment_vt_inst =
NullAlignment_toAlignment, /* getMateAlignment */
NullAlignment_toString, /* getMateReferenceSpec */
NullAlignment_toBool, /* getMateIsReversedOrientation */
+ NullAlignment_toBool, /* isFirst */
/* Iterator */
NullAlignment_noNext /* next */
diff --git a/libs/ngs/NGS_Alignment.h b/libs/ngs/NGS_Alignment.h
index 5d2c9b1..c06fd71 100644
--- a/libs/ngs/NGS_Alignment.h
+++ b/libs/ngs/NGS_Alignment.h
@@ -140,10 +140,12 @@ struct NGS_String* NGS_AlignmentGetMateReferenceSpec( NGS_Alignment* self, ctx_t
bool NGS_AlignmentGetMateIsReversedOrientation( NGS_Alignment* self, ctx_t ctx );
+bool NGS_AlignmentIsFirst ( NGS_Alignment* self, ctx_t ctx );
+
/*--------------------------------------------------------------------------
* NGS_AlignmentIterator
*/
-
+
bool NGS_AlignmentIteratorNext ( NGS_Alignment* self, ctx_t ctx );
/*--------------------------------------------------------------------------
@@ -185,6 +187,7 @@ struct NGS_Alignment_vt
NGS_ALIGNMENT* ( * getMateAlignment ) ( NGS_ALIGNMENT* self, ctx_t ctx );
struct NGS_String* ( * getMateReferenceSpec ) ( NGS_ALIGNMENT* self, ctx_t ctx );
bool ( * getMateIsReversedOrientation ) ( NGS_ALIGNMENT* self, ctx_t ctx );
+ bool ( * isFirst ) ( NGS_ALIGNMENT* self, ctx_t ctx );
/* Iterator */
bool ( * next ) ( NGS_ALIGNMENT* self, ctx_t ctx );
diff --git a/libs/ngs/NGS_Cursor.c b/libs/ngs/NGS_Cursor.c
index ef2c046..b8fdcef 100644
--- a/libs/ngs/NGS_Cursor.c
+++ b/libs/ngs/NGS_Cursor.c
@@ -216,7 +216,7 @@ const NGS_Cursor* NGS_CursorMakeDb ( ctx_t ctx,
rc_t rc = VDatabaseOpenTableRead ( db, & table, "%s", tableName );
if ( rc != 0 )
{
- INTERNAL_ERROR ( xcTableOpenFailed, "%.*s.%s", NGS_StringSize ( run_name, ctx ), NGS_StringData ( run_name, ctx ), tableName );
+ INTERNAL_ERROR ( xcTableOpenFailed, "%.*s.%s rc = %R", NGS_StringSize ( run_name, ctx ), NGS_StringData ( run_name, ctx ), tableName, rc );
return NULL;
}
@@ -573,3 +573,39 @@ uint32_t NGS_CursorGetColumnIndex ( const NGS_Cursor * self, ctx_t ctx, uint32_t
return self -> col_idx [ column_id ];
}
+/* GetVBlob
+ */
+const struct VBlob* NGS_CursorGetVBlob ( const NGS_Cursor * self, ctx_t ctx, int64_t rowId, uint32_t column_id )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
+
+ rc_t rc = VCursorSetRowId ( self -> curs, rowId );
+ if ( rc != 0 )
+ {
+ INTERNAL_ERROR ( xcUnexpected, "VCursorSetRowId() rc = %R", rc );
+ }
+ else
+ {
+ rc = VCursorOpenRow ( self -> curs );
+ if ( rc != 0 )
+ {
+ INTERNAL_ERROR ( xcUnexpected, "VCursorOpenRow() rc = %R", rc );
+ }
+ else
+ {
+ const struct VBlob* ret;
+ rc = VCursorGetBlob ( self -> curs, & ret, NGS_CursorGetColumnIndex ( self, ctx, column_id ) );
+ if ( rc != 0 || FAILED () )
+ {
+ VCursorCloseRow ( self -> curs );
+ INTERNAL_ERROR ( xcUnexpected, "VCursorGetBlob(READ) rc = %R", rc );
+ }
+ else
+ {
+ VCursorCloseRow ( self -> curs );
+ return ret;
+ }
+ }
+ }
+ return NULL;
+}
diff --git a/libs/ngs/NGS_Cursor.h b/libs/ngs/NGS_Cursor.h
index 850bd88..0e3ba32 100644
--- a/libs/ngs/NGS_Cursor.h
+++ b/libs/ngs/NGS_Cursor.h
@@ -37,8 +37,10 @@ typedef struct NGS_Cursor NGS_Cursor;
extern "C" {
#endif
-struct VTable;
struct VDatabase;
+struct VTable;
+struct VCursor;
+struct VBlob;
struct NGS_String;
/*--------------------------------------------------------------------------
@@ -50,11 +52,11 @@ struct NGS_String;
/* Make
*/
const NGS_Cursor* NGS_CursorMake ( ctx_t ctx, const struct VTable* table, const char * col_specs[], uint32_t num_cols );
-const NGS_Cursor* NGS_CursorMakeDb ( ctx_t ctx,
- const struct VDatabase* db,
- const struct NGS_String* run_name,
- const char* tableName,
- const char * col_specs[],
+const NGS_Cursor* NGS_CursorMakeDb ( ctx_t ctx,
+ const struct VDatabase* db,
+ const struct NGS_String* run_name,
+ const char* tableName,
+ const char * col_specs[],
uint32_t num_cols );
/* Release
@@ -70,41 +72,41 @@ const NGS_Cursor * NGS_CursorDuplicate ( const NGS_Cursor * self, ctx_t ctx );
/* CellDataDirect
* Adds requested column if necessary and calls VCursorCellDataDirect
*/
-void NGS_CursorCellDataDirect ( const NGS_Cursor *self,
+void NGS_CursorCellDataDirect ( const NGS_Cursor *self,
ctx_t ctx,
int64_t rowId,
- uint32_t colIdx,
- uint32_t *elem_bits,
+ uint32_t colIdx,
+ uint32_t *elem_bits,
const void **base,
- uint32_t *boff,
+ uint32_t *boff,
uint32_t *row_len );
/* GetString
-*/
+*/
struct NGS_String * NGS_CursorGetString ( const NGS_Cursor * self, ctx_t ctx, int64_t rowId, uint32_t colIdx );
/* GetInt64
-*/
+*/
int64_t NGS_CursorGetInt64 ( const NGS_Cursor * self, ctx_t ctx, int64_t rowId, uint32_t colIdx );
/* GetUInt64
-*/
+*/
uint64_t NGS_CursorGetUInt64 ( const NGS_Cursor * self, ctx_t ctx, int64_t rowId, uint32_t colIdx );
/* GetInt32
-*/
+*/
int32_t NGS_CursorGetInt32 ( const NGS_Cursor * self, ctx_t ctx, int64_t rowId, uint32_t colIdx );
/* GetUInt32
-*/
+*/
uint32_t NGS_CursorGetUInt32 ( const NGS_Cursor * self, ctx_t ctx, int64_t rowId, uint32_t colIdx );
/* GetBool
-*/
+*/
bool NGS_CursorGetBool ( const NGS_Cursor * self, ctx_t ctx, int64_t rowId, uint32_t colIdx );
/* GetChar
-*/
+*/
char NGS_CursorGetChar ( const NGS_Cursor * self, ctx_t ctx, int64_t rowId, uint32_t colIdx );
/* GetRowCount
@@ -117,16 +119,20 @@ void NGS_CursorGetRowRange ( const NGS_Cursor * self, ctx_t ctx, int64_t* first,
/* GetTable
*/
-const struct VTable* NGS_CursorGetTable ( const NGS_Cursor * self, ctx_t ctx );
+const struct VTable* NGS_CursorGetTable ( const NGS_Cursor * self, ctx_t ctx );
/* GetVCursor
*/
-const struct VCursor* NGS_CursorGetVCursor ( const NGS_Cursor * self );
+const struct VCursor* NGS_CursorGetVCursor ( const NGS_Cursor * self );
/* GetColumnIndex
*/
uint32_t NGS_CursorGetColumnIndex ( const NGS_Cursor * self, ctx_t ctx, uint32_t column_id );
+/* GetVBlob
+ */
+const struct VBlob* NGS_CursorGetVBlob ( const NGS_Cursor * self, ctx_t ctx, int64_t rowId, uint32_t column_id );
+
#ifdef __cplusplus
}
#endif
diff --git a/libs/ngs/NGS_FragmentBlob.c b/libs/ngs/NGS_FragmentBlob.c
index 40f3fbc..43ec0e7 100644
--- a/libs/ngs/NGS_FragmentBlob.c
+++ b/libs/ngs/NGS_FragmentBlob.c
@@ -43,6 +43,7 @@
#include "NGS_Id.h"
#include "NGS_Cursor.h"
#include "SRA_Read.h"
+#include "VByteBlob.h"
struct NGS_FragmentBlob
{
@@ -105,82 +106,17 @@ NGS_FragmentBlobMake ( ctx_t ctx, const NGS_String* run, const struct NGS_Cursor
{
TRY ( ret -> run = NGS_StringDuplicate ( run, ctx ) )
{
- const struct VCursor* vcurs = NGS_CursorGetVCursor ( curs );
- rc_t rc = VCursorSetRowId ( vcurs, rowId );
- if ( rc != 0 )
+ TRY ( ret -> blob_READ = NGS_CursorGetVBlob ( curs, ctx, rowId, seq_READ ) )
{
- INTERNAL_ERROR ( xcUnexpected, "VCursorSetRowId() rc = %R", rc );
- }
- else
- {
- rc = VCursorOpenRow ( vcurs );
- if ( rc != 0 )
+ TRY ( ret -> blob_READ_LEN = NGS_CursorGetVBlob ( curs, ctx, rowId, seq_READ_LEN ) )
{
- INTERNAL_ERROR ( xcUnexpected, "VCursorOpenRow() rc = %R", rc );
- }
- else
- {
- rc = VCursorGetBlob ( vcurs, & ret -> blob_READ, NGS_CursorGetColumnIndex ( curs, ctx, seq_READ ) );
- if ( rc != 0 )
+ TRY ( ret -> blob_READ_TYPE = NGS_CursorGetVBlob ( curs, ctx, rowId, seq_READ_TYPE ) );
{
- VCursorCloseRow ( vcurs );
- INTERNAL_ERROR ( xcUnexpected, "VCursorGetBlob(READ) rc = %R", rc );
- }
- else
- {
- rc = VCursorGetBlob ( vcurs, & ret -> blob_READ_LEN, NGS_CursorGetColumnIndex ( curs, ctx, seq_READ_LEN ) );
- if ( rc != 0 )
- {
- VCursorCloseRow ( vcurs );
- INTERNAL_ERROR ( xcUnexpected, "VCursorGetBlob(READ_LEN) rc = %R", rc );
- }
- else
+ ret -> rowId = rowId;
+ ret -> curs = NGS_CursorDuplicate ( curs, ctx );
+ TRY ( VByteBlob_ContiguousChunk ( ret -> blob_READ, ctx, ret -> rowId, &ret -> data, &ret -> size, false ) )
{
- rc = VCursorGetBlob ( vcurs, & ret -> blob_READ_TYPE, NGS_CursorGetColumnIndex ( curs, ctx, seq_READ_TYPE ) );
- if ( rc != 0 )
- {
- VCursorCloseRow ( vcurs );
- INTERNAL_ERROR ( xcUnexpected, "VCursorGetBlob(READ_TYPE) rc = %R", rc );
- }
- else
- {
- rc = VCursorCloseRow ( vcurs );
- if ( rc != 0 )
- {
- INTERNAL_ERROR ( xcUnexpected, "VCursorCloseRow() rc = %R", rc );
- }
- else
- {
- ret -> rowId = rowId;
- ret -> curs = NGS_CursorDuplicate ( curs, ctx );
- {
- /* calculate data/size of the blob (sub-VBlob) starting at rowId */
- uint32_t frag_type_elem_bits;
- const void *frag_type_base;
- uint32_t frag_type_boff;
- uint32_t frag_type_row_len;
- rc_t rc = VBlobCellData ( ret -> blob_READ,
- ret -> rowId,
- & frag_type_elem_bits,
- & frag_type_base,
- & frag_type_boff,
- & frag_type_row_len );
- if ( rc != 0 )
- {
- INTERNAL_ERROR ( xcUnexpected, "VBlobCellData() rc = %R", rc );
- }
- else
- {
- assert( frag_type_elem_bits == 8 );
- assert( frag_type_boff == 0 );
- ret -> data = frag_type_base;
- ret -> size = BlobBufferBytes ( ret -> blob_READ ) -
- ( (const uint8_t*)( ret -> data ) - (const uint8_t*)( ret -> blob_READ -> data . base ) );
- return ret;
- }
- }
- }
- }
+ return ret;
}
}
}
@@ -428,7 +364,9 @@ NGS_FragmentBlobInfoByOffset ( const struct NGS_FragmentBlob * self, ctx_t ctx,
elem_count_t length = PageMapIteratorDataLength ( &pmIt );
elem_count_t offset = PageMapIteratorDataOffset ( &pmIt );
row_count_t repeat = PageMapIteratorRepeatCount ( &pmIt );
-
+/*TODO: with the updated design:
+ assert ( repeat == 1 );
+ */
if ( offsetInBases < offset + length * repeat )
{
while ( repeat > 1 )
diff --git a/libs/ngs/NGS_ReadCollection.c b/libs/ngs/NGS_ReadCollection.c
index f563717..7f70e63 100644
--- a/libs/ngs/NGS_ReadCollection.c
+++ b/libs/ngs/NGS_ReadCollection.c
@@ -618,7 +618,7 @@ NGS_ReadCollection * NGS_ReadCollectionMake ( ctx_t ctx, const char * spec )
{
return NGS_ReadCollectionMakeVTable ( ctx, tbl, spec );
}
- INTERNAL_ERROR ( xcUnimplemented, "Cannot open accession '%s' as an SRA table.", spec );
+ USER_ERROR ( xcTableOpenFailed, "Cannot open accession '%s' as an SRA table.", spec );
}
}
else
@@ -629,11 +629,11 @@ NGS_ReadCollection * NGS_ReadCollectionMake ( ctx_t ctx, const char * spec )
KConfigMakeRepositoryMgrRead ( kfg, & repoMgr ) != 0 ||
KRepositoryMgrHasRemoteAccess ( repoMgr ) )
{
- INTERNAL_ERROR ( xcUnimplemented, "Cannot open accession '%s'.", spec );
+ USER_ERROR ( xcTableOpenFailed, "Cannot open accession '%s', rc = %R", spec, rc );
}
else
{
- INTERNAL_ERROR ( xcUnimplemented, "Cannot open accession '%s'. Note: remote access is disabled in the configuration.", spec );
+ USER_ERROR ( xcTableOpenFailed, "Cannot open accession '%s', rc = %R. Note: remote access is disabled in the configuration.", spec, rc );
}
KRepositoryMgrRelease ( repoMgr );
KConfigRelease ( kfg );
diff --git a/libs/ngs/NGS_Statistics.c b/libs/ngs/NGS_Statistics.c
index fc7f796..b04dd8a 100644
--- a/libs/ngs/NGS_Statistics.c
+++ b/libs/ngs/NGS_Statistics.c
@@ -122,6 +122,7 @@ NGS_String_v1 * NGS_Statistics_v1_next_path ( const NGS_Statistics_v1 * self, NG
ON_FAIL ( bool more = NGS_StatisticsNextPath( Self ( self ), ctx, path, & new_path ) )
{
NGS_ErrBlockThrow ( err, ctx );
+ return NULL;
}
if ( more )
diff --git a/libs/ngs/SRA_Read.c b/libs/ngs/SRA_Read.c
index 9671cdd..32cef46 100644
--- a/libs/ngs/SRA_Read.c
+++ b/libs/ngs/SRA_Read.c
@@ -49,7 +49,7 @@
#include <sysalloc.h>
#ifndef min
-# define min(a,b) ( (a) < (b) ? (a) : (b) )
+# define min(a,b) ( (a) < (b) ? (a) : (b) )
#endif
/*--------------------------------------------------------------------------
@@ -84,7 +84,7 @@ static NGS_Read_vt NGS_Read_vt_inst =
SRA_FragmentIsAligned,
SRA_FragmentNext
},
-
+
/* NGS_Read */
SRA_ReadGetId,
SRA_ReadGetName,
@@ -96,7 +96,7 @@ static NGS_Read_vt NGS_Read_vt_inst =
SRA_ReadFragIsAligned,
SRA_ReadIteratorNext,
SRA_ReadIteratorGetCount,
-};
+};
/* Init
*/
@@ -114,21 +114,21 @@ void SRA_ReadInit ( ctx_t ctx, SRA_Read * self, const char *clsname, const char
TRY ( self -> run_name = NGS_StringDuplicate ( run_name, ctx ) )
{
self -> wants_full = true;
- self -> wants_partial = true;
- self -> wants_unaligned = true;
+ self -> wants_partial = true;
+ self -> wants_unaligned = true;
}
}
}
}
static
-void SRA_ReadIteratorInit ( ctx_t ctx,
- SRA_Read * self,
- const char *clsname,
- const char *instname,
+void SRA_ReadIteratorInit ( ctx_t ctx,
+ SRA_Read * self,
+ const char *clsname,
+ const char *instname,
const NGS_String * run_name,
- bool wants_full,
- bool wants_partial,
+ bool wants_full,
+ bool wants_partial,
bool wants_unaligned )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
@@ -142,8 +142,8 @@ void SRA_ReadIteratorInit ( ctx_t ctx,
TRY ( self -> run_name = NGS_StringDuplicate ( run_name, ctx ) )
{
self -> wants_full = wants_full;
- self -> wants_partial = wants_partial;
- self -> wants_unaligned = wants_unaligned;
+ self -> wants_partial = wants_partial;
+ self -> wants_unaligned = wants_unaligned;
}
}
}
@@ -154,7 +154,7 @@ void SRA_ReadIteratorInit ( ctx_t ctx,
void SRA_ReadWhack ( SRA_Read * self, ctx_t ctx )
{
NGS_CursorRelease ( self -> curs, ctx );
-
+
NGS_StringRelease ( self -> group_name, ctx );
NGS_StringRelease ( self -> run_name, ctx );
}
@@ -178,7 +178,6 @@ SRA_Read * SRA_ReadDuplicate ( const SRA_Read * self, ctx_t ctx )
return NGS_RefcountDuplicate ( NGS_ReadToRefcount ( & self -> dad ), ctx );
}
-static
void SRA_ReadIteratorInitFragment ( SRA_Read * self, ctx_t ctx )
{
const void * base;
@@ -206,7 +205,12 @@ void SRA_ReadIteratorInitFragment ( SRA_Read * self, ctx_t ctx )
/* NB - should also be taking READ_FILTER into account */
for ( i = 0; i < row_len; ++ i )
- self -> bio_frags += self -> READ_TYPE [ i ] & READ_TYPE_BIOLOGICAL;
+ {
+ if ( self -> READ_LEN [ i ] != 0 )
+ {
+ self -> bio_frags += self -> READ_TYPE [ i ] & READ_TYPE_BIOLOGICAL;
+ }
+ }
}
}
}
@@ -234,12 +238,12 @@ NGS_Read * SRA_ReadMake ( ctx_t ctx, const NGS_Cursor * curs, int64_t readId, co
TRY ( SRA_ReadInit ( ctx, ref, "SRA_Read", instname, run_name ) )
{
uint64_t row_count = NGS_CursorGetRowCount ( curs, ctx );
-
+
/* validate the requested rowId and seek to it */
if ( readId <= 0 || (uint64_t)readId > row_count )
{
INTERNAL_ERROR ( xcCursorAccessFailed, "rowId ( %li ) out of range for %.*s", readId, NGS_StringSize ( run_name, ctx ), NGS_StringData ( run_name, ctx ) );
- }
+ }
else
{
ref -> curs = NGS_CursorDuplicate ( curs, ctx );
@@ -252,7 +256,7 @@ NGS_Read * SRA_ReadMake ( ctx_t ctx, const NGS_Cursor * curs, int64_t readId, co
return & ref -> dad;
}
}
-
+
SRA_ReadRelease ( ref, ctx );
return NULL;
}
@@ -267,10 +271,10 @@ NGS_Read * SRA_ReadMake ( ctx_t ctx, const NGS_Cursor * curs, int64_t readId, co
NGS_String * SRA_ReadGetId ( SRA_Read * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcAccessing );
-
+
assert ( self != NULL );
-
- if ( ! self -> seen_first )
+
+ if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to ReadIteratorNext()" );
return NULL;
@@ -279,7 +283,7 @@ NGS_String * SRA_ReadGetId ( SRA_Read * self, ctx_t ctx )
/* if current row is valid, read data */
if ( self -> cur_row < self -> row_max )
return NGS_IdMake ( ctx, self -> run_name, NGSObject_Read, self -> cur_row );
-
+
USER_ERROR ( xcCursorExhausted, "No more rows available" );
return NULL;
}
@@ -289,10 +293,10 @@ NGS_String * SRA_ReadGetId ( SRA_Read * self, ctx_t ctx )
NGS_String * SRA_ReadGetName ( SRA_Read * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
assert ( self != NULL );
-
- if ( ! self -> seen_first )
+
+ if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to ReadIteratorNext()" );
return NULL;
@@ -301,14 +305,14 @@ NGS_String * SRA_ReadGetName ( SRA_Read * self, ctx_t ctx )
{
NGS_String * ret;
ON_FAIL ( ret = NGS_CursorGetString( self -> curs, ctx, self -> cur_row, seq_NAME ) )
- {
- if ( GetRCObject ( ctx -> rc ) == rcColumn && GetRCState ( ctx -> rc ) == rcNotFound )
+ {
+ if ( GetRCObject ( ctx -> rc ) == (int)rcColumn && GetRCState ( ctx -> rc ) == rcNotFound )
{ /* no NAME column; synthesize a read name based on run_name and row_id */
CLEAR ();
ret = NGS_IdMake ( ctx, self -> run_name, NGSObject_Read, self -> cur_row );
}
}
-
+
return ret;
}
}
@@ -318,15 +322,15 @@ NGS_String * SRA_ReadGetName ( SRA_Read * self, ctx_t ctx )
NGS_String * SRA_ReadGetReadGroup ( SRA_Read * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
assert ( self != NULL );
-
- if ( ! self -> seen_first )
+
+ if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to ReadIteratorNext()" );
return NULL;
}
-
+
return NGS_CursorGetString ( self -> curs, ctx, self -> cur_row, seq_GROUP );
}
@@ -335,8 +339,8 @@ enum NGS_ReadCategory SRA_ReadGetCategory ( const SRA_Read * self, ctx_t ctx )
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcAccessing );
assert ( self != NULL );
-
- if ( ! self -> seen_first )
+
+ if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to ReadIteratorNext()" );
return NGS_ReadCategory_unaligned;
@@ -351,7 +355,7 @@ enum NGS_ReadCategory SRA_ReadGetCategory ( const SRA_Read * self, ctx_t ctx )
CLEAR();
return NGS_ReadCategory_unaligned;
}
-
+
{
uint32_t i;
bool seen_aligned = false;
@@ -375,7 +379,7 @@ enum NGS_ReadCategory SRA_ReadGetCategory ( const SRA_Read * self, ctx_t ctx )
{
return seen_unaligned ? NGS_ReadCategory_partiallyAligned : NGS_ReadCategory_fullyAligned;
}
- else
+ else
{ /* no aligned fragments */
return NGS_ReadCategory_unaligned;
}
@@ -393,8 +397,8 @@ NGS_String * SRA_ReadGetSequence ( SRA_Read * self, ctx_t ctx, uint64_t offset,
NGS_String * seq;
assert ( self != NULL );
-
- if ( ! self -> seen_first )
+
+ if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to ReadIteratorNext()" );
return NULL;
@@ -416,7 +420,7 @@ NGS_String * SRA_ReadGetSequence ( SRA_Read * self, ctx_t ctx, uint64_t offset,
/* GetReadQualities
* GetReadSubQualities
*/
-static
+static
NGS_String * GetReadQualities ( SRA_Read * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
@@ -450,7 +454,7 @@ NGS_String * GetReadQualities ( SRA_Read * self, ctx_t ctx )
free ( copy );
}
}
-
+
if ( ! FAILED () )
{
return new_data;
@@ -465,15 +469,15 @@ NGS_String * SRA_ReadGetQualities ( SRA_Read * self, ctx_t ctx, uint64_t offset,
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
NGS_String * qual;
-
+
assert ( self != NULL );
-
- if ( ! self -> seen_first )
+
+ if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to ReadIteratorNext()" );
return NULL;
}
-
+
TRY ( qual = GetReadQualities ( self, ctx ) )
{
NGS_String * sub;
@@ -492,21 +496,21 @@ NGS_String * SRA_ReadGetQualities ( SRA_Read * self, ctx_t ctx, uint64_t offset,
uint32_t SRA_ReadNumFragments ( SRA_Read * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
assert ( self != NULL );
-
- if ( ! self -> seen_first )
+
+ if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to ReadIteratorNext()" );
return 0;
- }
-
+ }
+
if ( self -> cur_row >= self -> row_max )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
return false;
}
-
+
return self -> bio_frags;
}
@@ -515,15 +519,15 @@ uint32_t SRA_ReadNumFragments ( SRA_Read * self, ctx_t ctx )
bool SRA_ReadFragIsAligned ( SRA_Read * self, ctx_t ctx, uint32_t frag_idx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
assert ( self != NULL );
-
- if ( ! self -> seen_first )
+
+ if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to ReadIteratorNext()" );
return 0;
- }
-
+ }
+
if ( self -> cur_row >= self -> row_max )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
@@ -535,7 +539,7 @@ bool SRA_ReadFragIsAligned ( SRA_Read * self, ctx_t ctx, uint32_t frag_idx )
USER_ERROR ( xcIntegerOutOfBounds, "bad fragment index" );
return false;
}
-
+
return false;
}
@@ -545,11 +549,11 @@ bool SRA_ReadFragIsAligned ( SRA_Read * self, ctx_t ctx, uint32_t frag_idx )
/* Make
*/
-NGS_Read * SRA_ReadIteratorMake ( ctx_t ctx,
- const NGS_Cursor * curs,
- const NGS_String * run_name,
- bool wants_full,
- bool wants_partial,
+NGS_Read * SRA_ReadIteratorMake ( ctx_t ctx,
+ const NGS_Cursor * curs,
+ const NGS_String * run_name,
+ bool wants_full,
+ bool wants_partial,
bool wants_unaligned )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
@@ -590,13 +594,13 @@ NGS_Read * SRA_ReadIteratorMake ( ctx_t ctx,
/* MakeRange
*/
-NGS_Read * SRA_ReadIteratorMakeRange ( ctx_t ctx,
- const NGS_Cursor * curs,
- const NGS_String * run_name,
- uint64_t first,
- uint64_t count,
- bool wants_full,
- bool wants_partial,
+NGS_Read * SRA_ReadIteratorMakeRange ( ctx_t ctx,
+ const NGS_Cursor * curs,
+ const NGS_String * run_name,
+ uint64_t first,
+ uint64_t count,
+ bool wants_full,
+ bool wants_partial,
bool wants_unaligned )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
@@ -638,25 +642,25 @@ NGS_Read * SRA_ReadIteratorMakeRange ( ctx_t ctx,
/* MakeReadGroup
*/
-NGS_Read * SRA_ReadIteratorMakeReadGroup ( ctx_t ctx,
- const NGS_Cursor * curs,
- const NGS_String * run_name,
- const NGS_String * group_name,
- uint64_t first,
- uint64_t count,
- bool wants_full,
- bool wants_partial,
+NGS_Read * SRA_ReadIteratorMakeReadGroup ( ctx_t ctx,
+ const NGS_Cursor * curs,
+ const NGS_String * run_name,
+ const NGS_String * group_name,
+ uint64_t first,
+ uint64_t count,
+ bool wants_full,
+ bool wants_partial,
bool wants_unaligned )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
-
- TRY ( SRA_Read * ref = (SRA_Read*) SRA_ReadIteratorMakeRange ( ctx,
- curs,
- run_name,
- first,
- count,
- wants_full,
- wants_partial,
+
+ TRY ( SRA_Read * ref = (SRA_Read*) SRA_ReadIteratorMakeRange ( ctx,
+ curs,
+ run_name,
+ first,
+ count,
+ wants_full,
+ wants_partial,
wants_unaligned ) )
{
TRY ( ref -> group_name = NGS_StringDuplicate ( group_name, ctx ) )
@@ -679,17 +683,17 @@ bool SRA_ReadIteratorNext ( SRA_Read * self, ctx_t ctx )
/* reset fragment */
self -> seen_first_frag = false;
self -> seen_last_frag = false;
-
+
self -> cur_frag = 0;
self -> bio_frags = 0;
self -> frag_idx = 0;
self -> frag_max = 0;
self -> frag_start = 0;
self -> frag_len = 0;
-
+
self -> READ_TYPE = NULL;
self -> READ_LEN = NULL;
-
+
if ( self -> seen_first )
{ /* move to next row */
++ self -> cur_row;
@@ -698,17 +702,17 @@ bool SRA_ReadIteratorNext ( SRA_Read * self, ctx_t ctx )
{
self -> seen_first = true;
}
-
+
while ( self -> cur_row < self -> row_max )
{
/* work the category filter */
- if ( ! self -> wants_full ||
+ if ( ! self -> wants_full ||
! self -> wants_partial ||
! self -> wants_unaligned )
{
ON_FAIL ( enum NGS_ReadCategory cat = SRA_ReadGetCategory ( self, ctx ) )
return false;
-
+
if ( ( cat == NGS_ReadCategory_fullyAligned && ! self -> wants_full )
||
( cat == NGS_ReadCategory_partiallyAligned && ! self -> wants_partial )
@@ -716,21 +720,21 @@ bool SRA_ReadIteratorNext ( SRA_Read * self, ctx_t ctx )
( cat == NGS_ReadCategory_unaligned && ! self -> wants_unaligned ) )
{
++ self -> cur_row;
- continue;
+ continue;
}
}
-
+
/* work the read group filter if required */
if ( self -> group_name != NULL )
{
uint32_t size;
-
+
ON_FAIL ( NGS_String* group = NGS_CursorGetString ( self -> curs, ctx, self -> cur_row, seq_GROUP ) )
return false;
-
+
size = ( uint32_t ) NGS_StringSize ( group, ctx );
- if ( string_cmp ( NGS_StringData ( self -> group_name, ctx ),
- NGS_StringSize ( self -> group_name, ctx ),
+ if ( string_cmp ( NGS_StringData ( self -> group_name, ctx ),
+ NGS_StringSize ( self -> group_name, ctx ),
NGS_StringData ( group, ctx ),
size,
size ) != 0 )
@@ -746,10 +750,10 @@ bool SRA_ReadIteratorNext ( SRA_Read * self, ctx_t ctx )
{
return true;
}
-
+
break;
}
-
+
return false;
}
@@ -759,7 +763,7 @@ bool SRA_ReadIteratorNext ( SRA_Read * self, ctx_t ctx )
uint64_t SRA_ReadIteratorGetCount ( const SRA_Read * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
-
+
assert ( self != NULL );
return self -> row_count;
}
@@ -767,14 +771,14 @@ uint64_t SRA_ReadIteratorGetCount ( const SRA_Read * self, ctx_t ctx )
NGS_String * SRA_FragmentGetId ( SRA_Read * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
-
+
assert ( self != NULL );
- if ( ! self -> seen_first_frag )
+ if ( ! self -> seen_first_frag )
{
USER_ERROR ( xcIteratorUninitialized, "Fragment accessed before a call to FragmentIteratorNext()" );
return NULL;
}
- if ( self -> seen_last_frag )
+ if ( self -> seen_last_frag )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
return NULL;
@@ -787,19 +791,19 @@ static
NGS_String * GetFragmentString ( const SRA_Read * self, ctx_t ctx, NGS_String * str )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
assert ( self != NULL );
- if ( ! self -> seen_first_frag )
+ if ( ! self -> seen_first_frag )
{
USER_ERROR ( xcIteratorUninitialized, "Fragment accessed before a call to FragmentIteratorNext()" );
return NULL;
}
- if ( self -> seen_last_frag )
+ if ( self -> seen_last_frag )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
return NULL;
}
-
+
if ( self -> cur_row < self -> row_max )
{
TRY ( NGS_String * frag = NGS_StringSubstrOffsetSize ( str, ctx, self -> frag_start, self -> frag_len ) )
@@ -815,19 +819,19 @@ struct NGS_String * SRA_FragmentGetSequence ( SRA_Read * self, ctx_t ctx, uint64
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
NGS_String * ret = NULL;
-
+
assert ( self != NULL );
- if ( ! self -> seen_first_frag )
+ if ( ! self -> seen_first_frag )
{
USER_ERROR ( xcIteratorUninitialized, "Fragment accessed before a call to FragmentIteratorNext()" );
return NULL;
}
- if ( self -> seen_last_frag )
+ if ( self -> seen_last_frag )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
return NULL;
- }
-
+ }
+
{
TRY ( NGS_String * read = NGS_CursorGetString ( self -> curs, ctx, self -> cur_row, seq_READ ) )
{
@@ -846,19 +850,19 @@ struct NGS_String * SRA_FragmentGetQualities ( SRA_Read * self, ctx_t ctx, uint6
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
NGS_String * ret = NULL;
-
+
assert ( self != NULL );
- if ( ! self -> seen_first_frag )
+ if ( ! self -> seen_first_frag )
{
USER_ERROR ( xcIteratorUninitialized, "Fragment accessed before a call to FragmentIteratorNext()" );
return NULL;
}
- if ( self -> seen_last_frag )
+ if ( self -> seen_last_frag )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
return NULL;
}
-
+
{
TRY ( NGS_String * readQual = GetReadQualities ( self, ctx ) )
{
@@ -876,14 +880,14 @@ struct NGS_String * SRA_FragmentGetQualities ( SRA_Read * self, ctx_t ctx, uint6
bool SRA_FragmentIsPaired ( SRA_Read * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcAccessing );
-
+
assert ( self != NULL );
- if ( ! self -> seen_first_frag )
+ if ( ! self -> seen_first_frag )
{
USER_ERROR ( xcIteratorUninitialized, "Fragment accessed before a call to FragmentIteratorNext()" );
return false;
}
- if ( self -> seen_last_frag )
+ if ( self -> seen_last_frag )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
return false;
@@ -903,33 +907,36 @@ bool SRA_FragmentNext ( SRA_Read * self, ctx_t ctx )
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcConstructing );
assert ( self != NULL );
-
- if ( ! self -> seen_first )
+
+ if ( ! self -> seen_first )
{
USER_ERROR ( xcIteratorUninitialized, "Read accessed before a call to ReadIteratorNext()" );
return false;
}
-
+
if ( self -> seen_first_frag )
{ /* move to next fragment */
++ self -> cur_frag;
++ self -> frag_idx;
}
- /* advance to next biological fragment */
+ /* advance to next non-empty biological fragment */
for ( self -> seen_first_frag = true; self -> frag_idx < self -> frag_max; ++ self -> frag_idx )
{
- /* get coordinates */
- self -> frag_start += self -> frag_len;
- self -> frag_len = self -> READ_LEN [ self -> frag_idx ];
+ if ( self -> READ_LEN [ self -> frag_idx ] != 0 )
+ {
+ /* get coordinates */
+ self -> frag_start += self -> frag_len;
+ self -> frag_len = self -> READ_LEN [ self -> frag_idx ];
- /* test for biological fragment */
- assert ( READ_TYPE_TECHNICAL == 0 );
- assert ( READ_TYPE_BIOLOGICAL == 1 );
- if ( ( self -> READ_TYPE [ self -> frag_idx ] & READ_TYPE_BIOLOGICAL ) != 0 )
- return true;
+ /* test for biological fragment */
+ assert ( READ_TYPE_TECHNICAL == 0 );
+ assert ( READ_TYPE_BIOLOGICAL == 1 );
+ if ( ( self -> READ_TYPE [ self -> frag_idx ] & READ_TYPE_BIOLOGICAL ) != 0 )
+ return true;
+ }
}
-
+
self -> seen_last_frag = true;
return false;
}
diff --git a/libs/ngs/SRA_Read.h b/libs/ngs/SRA_Read.h
index d8bbc61..e5fc797 100644
--- a/libs/ngs/SRA_Read.h
+++ b/libs/ngs/SRA_Read.h
@@ -52,7 +52,7 @@ struct NGS_Cursor;
struct NGS_String;
enum SequenceTableColumn
-{ // keep in sync with sequence_col_specs in SRA_Read.c
+{ // keep in sync with sequence_col_specs in SRA_Read.c
seq_READ,
seq_READ_TYPE,
seq_QUALITY,
@@ -63,7 +63,7 @@ enum SequenceTableColumn
seq_SPOT_COUNT,
seq_NUM_COLS
-}; // keep in sync with sequence_col_specs in SRA_Read.c
+}; // keep in sync with sequence_col_specs in SRA_Read.c
extern const char * sequence_col_specs [];
@@ -72,18 +72,18 @@ extern const char * sequence_col_specs [];
*/
struct SRA_Read
{
- NGS_Read dad;
-
+ NGS_Read dad;
+
struct NGS_String * run_name;
struct NGS_String * group_name; /* if not NULL, only return reads from this read group */
-
+
int64_t cur_row;
int64_t row_max;
uint64_t row_count;
const INSDC_read_type * READ_TYPE;
const INSDC_coord_len * READ_LEN;
-
+
struct NGS_Cursor const * curs;
uint32_t cur_frag;
@@ -99,7 +99,7 @@ struct SRA_Read
/* read filtering criteria */
bool wants_full;
- bool wants_partial;
+ bool wants_partial;
bool wants_unaligned;
};
@@ -121,7 +121,7 @@ struct NGS_String * SRA_ReadGetQualities ( NGS_READ * self, ctx_t ctx, uint
uint32_t SRA_ReadNumFragments ( NGS_READ * self, ctx_t ctx );
bool SRA_ReadFragIsAligned ( NGS_READ * self, ctx_t ctx, uint32_t frag_idx );
bool SRA_ReadIteratorNext ( NGS_READ * self, ctx_t ctx );
-uint64_t SRA_ReadIteratorGetCount ( const NGS_READ * self, ctx_t ctx );
+uint64_t SRA_ReadIteratorGetCount ( const NGS_READ * self, ctx_t ctx );
/* Make
* a single read
@@ -130,36 +130,40 @@ struct NGS_Read * SRA_ReadMake ( ctx_t ctx, const struct NGS_Cursor * curs, int6
/* IteratorMake
*/
-struct NGS_Read * SRA_ReadIteratorMake ( ctx_t ctx,
- const struct NGS_Cursor * curs,
- const struct NGS_String * run_name,
- bool wants_full,
- bool wants_partial,
+struct NGS_Read * SRA_ReadIteratorMake ( ctx_t ctx,
+ const struct NGS_Cursor * curs,
+ const struct NGS_String * run_name,
+ bool wants_full,
+ bool wants_partial,
bool wants_unaligned );
+/* IteratorInitFragment
+ */
+void SRA_ReadIteratorInitFragment ( NGS_READ * self, ctx_t ctx );
+
/* IteratorMakeRange
* all reads in the specified range of rowIds
*/
-struct NGS_Read * SRA_ReadIteratorMakeRange ( ctx_t ctx,
- const struct NGS_Cursor * curs,
- const struct NGS_String * run_name,
- uint64_t first,
- uint64_t count,
- bool wants_full,
- bool wants_partial,
+struct NGS_Read * SRA_ReadIteratorMakeRange ( ctx_t ctx,
+ const struct NGS_Cursor * curs,
+ const struct NGS_String * run_name,
+ uint64_t first,
+ uint64_t count,
+ bool wants_full,
+ bool wants_partial,
bool wants_unaligned );
/* IteratorMakeReadGroup
* within the specified range of rowIds, will only return reads belonging to the specified read group (groupName)
*/
-struct NGS_Read * SRA_ReadIteratorMakeReadGroup ( ctx_t ctx,
- const struct NGS_Cursor * curs,
- const struct NGS_String * run_name,
- const struct NGS_String * group_name,
- uint64_t first,
- uint64_t count,
- bool wants_full,
- bool wants_partial,
+struct NGS_Read * SRA_ReadIteratorMakeReadGroup ( ctx_t ctx,
+ const struct NGS_Cursor * curs,
+ const struct NGS_String * run_name,
+ const struct NGS_String * group_name,
+ uint64_t first,
+ uint64_t count,
+ bool wants_full,
+ bool wants_partial,
bool wants_unaligned );
#ifdef __cplusplus
diff --git a/libs/ngs/SRA_ReferenceSequence.c b/libs/ngs/SRA_ReferenceSequence.c
index 699855e..c36501a 100644
--- a/libs/ngs/SRA_ReferenceSequence.c
+++ b/libs/ngs/SRA_ReferenceSequence.c
@@ -71,7 +71,7 @@ static NGS_ReferenceSequence_vt SRA_ReferenceSequence_vt_inst =
{
/* NGS_Refcount */
{ SRA_ReferenceSequenceWhack },
-
+
/* NGS_ReferenceSequence */
SRA_ReferenceSequenceGetCanonicalName,
SRA_ReferenceSequenceGetIsCircular,
@@ -89,7 +89,7 @@ struct SRA_ReferenceSequence
const struct NGS_Cursor * curs;
uint32_t chunk_size;
-
+
int64_t first_row;
int64_t last_row; /* inclusive */
uint64_t cur_length; /* size of current reference in bases (0 = not yet counted) */
@@ -143,9 +143,9 @@ void SRA_ReferenceSequenceWhack ( SRA_ReferenceSequence * self, ctx_t ctx )
/* Init
*/
static
-void SRA_ReferenceSequenceInit ( ctx_t ctx,
+void SRA_ReferenceSequenceInit ( ctx_t ctx,
SRA_ReferenceSequence * ref,
- const char *clsname,
+ const char *clsname,
const char *instname )
{
FUNC_ENTRY ( ctx, rcSRA, rcTable, rcOpening );
@@ -203,7 +203,7 @@ NGS_ReferenceSequence * NGS_ReferenceSequenceMakeSRA ( ctx_t ctx, const char * s
size_t pref_size = sizeof ( REF_PREFIX ) - 1;
if ( string_match ( REF_PREFIX, pref_size, ts_buff, string_size ( ts_buff ), (uint32_t)pref_size, NULL ) != pref_size )
{
- INTERNAL_ERROR ( xcUnimplemented, "Cannot open accession '%s' as a reference table.", spec );
+ USER_ERROR ( xcTableOpenFailed, "Cannot open accession '%s' as a reference table.", spec );
}
else
{
@@ -246,9 +246,9 @@ int64_t SRA_ReferenceSequence_GetLastRowId ( const struct NGS_ReferenceSequence
NGS_String * SRA_ReferenceSequenceGetCanonicalName ( SRA_ReferenceSequence * self, ctx_t ctx )
{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
-
+
assert ( self != NULL );
-
+
return NGS_CursorGetString ( self -> curs, ctx, self -> first_row, reference_SEQ_ID);
}
@@ -257,7 +257,7 @@ bool SRA_ReferenceSequenceGetIsCircular ( const SRA_ReferenceSequence * self, ct
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
assert ( self );
-
+
if ( self -> curs == NULL )
{
USER_ERROR ( xcCursorExhausted, "No more rows available" );
@@ -283,21 +283,21 @@ uint64_t SRA_ReferenceSequenceGetLength ( SRA_ReferenceSequence * self, ctx_t ct
USER_ERROR ( xcCursorExhausted, "No more rows available" );
return 0;
}
-
+
if ( self -> cur_length == 0 ) /* not yet calculated */
- {
- self -> cur_length = self -> chunk_size * ( self -> last_row - self -> first_row ) +
- NGS_CursorGetUInt32 ( self -> curs,
- ctx,
- self -> last_row,
+ {
+ self -> cur_length = self -> chunk_size * ( self -> last_row - self -> first_row ) +
+ NGS_CursorGetUInt32 ( self -> curs,
+ ctx,
+ self -> last_row,
reference_SEQ_LEN );
}
-
+
return self -> cur_length;
}
struct NGS_String * SRA_ReferenceSequenceGetBases ( SRA_ReferenceSequence * self, ctx_t ctx, uint64_t offset, uint64_t size )
-{
+{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
assert ( self );
@@ -306,7 +306,7 @@ struct NGS_String * SRA_ReferenceSequenceGetBases ( SRA_ReferenceSequence * self
USER_ERROR ( xcCursorExhausted, "No more rows available" );
return NULL;
}
-
+
{
uint64_t totalBases = SRA_ReferenceSequenceGetLength ( self, ctx );
if ( offset >= totalBases )
@@ -314,13 +314,13 @@ struct NGS_String * SRA_ReferenceSequenceGetBases ( SRA_ReferenceSequence * self
return NGS_StringMake ( ctx, "", 0 );
}
else
- {
+ {
uint64_t basesToReturn = totalBases - offset;
char* data;
-
+
if (size != (size_t)-1 && basesToReturn > size)
basesToReturn = size;
-
+
data = (char*) malloc ( basesToReturn );
if ( data == NULL )
{
@@ -332,10 +332,10 @@ struct NGS_String * SRA_ReferenceSequenceGetBases ( SRA_ReferenceSequence * self
size_t cur_offset = 0;
while ( cur_offset < basesToReturn )
{
- /* we will potentially ask for more than available in the current chunk;
+ /* we will potentially ask for more than available in the current chunk;
SRA_ReferenceSequenceGetChunkSize will return only as much as is available in the chunk */
NGS_String* chunk = SRA_ReferenceSequenceGetChunk ( self, ctx, offset + cur_offset, basesToReturn - cur_offset );
- cur_offset += string_copy(data + cur_offset, basesToReturn - cur_offset,
+ cur_offset += string_copy(data + cur_offset, basesToReturn - cur_offset,
NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
NGS_StringRelease ( chunk, ctx );
}
@@ -346,7 +346,7 @@ struct NGS_String * SRA_ReferenceSequenceGetBases ( SRA_ReferenceSequence * self
}
struct NGS_String * SRA_ReferenceSequenceGetChunk ( SRA_ReferenceSequence * self, ctx_t ctx, uint64_t offset, uint64_t size )
-{
+{
FUNC_ENTRY ( ctx, rcSRA, rcCursor, rcReading );
assert ( self );
diff --git a/libs/ngs/SRA_Statistics.c b/libs/ngs/SRA_Statistics.c
index 6e62ec8..88d4a3a 100644
--- a/libs/ngs/SRA_Statistics.c
+++ b/libs/ngs/SRA_Statistics.c
@@ -166,21 +166,24 @@ uint32_t SRA_StatisticsGetValueType ( const SRA_Statistics * self, ctx_t ctx, co
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
assert ( self );
- assert ( path );
-
+
+ if ( path == NULL )
+ INTERNAL_ERROR ( xcParamNull, "path is NULL" );
+ else
{
DictionaryEntry * node = ( DictionaryEntry * )
BSTreeFind ( & self -> dictionary, ( const void * ) path, DictionaryEntryFind );
if ( node == NULL )
{
INTERNAL_ERROR ( xcUnexpected, "dictionary item '%s' is not found", path );
- return NGS_StatisticValueType_Undefined;
}
else
{
return node -> type;
}
}
+
+ return NGS_StatisticValueType_Undefined;
}
NGS_String* SRA_StatisticsGetAsString ( const SRA_Statistics * self, ctx_t ctx, const char * path )
@@ -188,8 +191,10 @@ NGS_String* SRA_StatisticsGetAsString ( const SRA_Statistics * self, ctx_t ctx,
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
assert ( self );
- assert ( path );
+ if ( path == NULL )
+ INTERNAL_ERROR ( xcParamNull, "path is NULL" );
+ else
{
DictionaryEntry * node = ( DictionaryEntry * )
BSTreeFind ( & self -> dictionary, ( const void * ) path, DictionaryEntryFind );
@@ -236,7 +241,7 @@ NGS_String* SRA_StatisticsGetAsString ( const SRA_Statistics * self, ctx_t ctx,
}
}
- return 0;
+ return NULL;
}
static int64_t NGS_StringToI64( const NGS_String * str, ctx_t ctx )
@@ -280,8 +285,10 @@ int64_t SRA_StatisticsGetAsI64 ( const SRA_Statistics * self, ctx_t ctx, const c
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
assert ( self );
- assert ( path );
+ if ( path == NULL )
+ INTERNAL_ERROR ( xcParamNull, "path is NULL" );
+ else
{
DictionaryEntry * node = ( DictionaryEntry * )
BSTreeFind ( & self -> dictionary, ( const void * ) path, DictionaryEntryFind );
@@ -372,8 +379,10 @@ uint64_t SRA_StatisticsGetAsU64 ( const SRA_Statistics * self, ctx_t ctx, const
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
assert ( self );
- assert ( path );
+ if ( path == NULL )
+ INTERNAL_ERROR ( xcParamNull, "path is NULL" );
+ else
{
DictionaryEntry * node = ( DictionaryEntry * )
BSTreeFind ( & self -> dictionary, ( const void * ) path, DictionaryEntryFind );
@@ -451,8 +460,10 @@ double SRA_StatisticsGetAsDouble ( const SRA_Statistics * self, ctx_t ctx, const
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
assert ( self );
- assert ( path );
+ if ( path == NULL )
+ INTERNAL_ERROR ( xcParamNull, "path is NULL" );
+ else
{
DictionaryEntry * node = ( DictionaryEntry * )
BSTreeFind ( & self -> dictionary, ( const void * ) path, DictionaryEntryFind );
@@ -490,12 +501,13 @@ double SRA_StatisticsGetAsDouble ( const SRA_Statistics * self, ctx_t ctx, const
bool SRA_StatisticsNextPath ( const SRA_Statistics * self, ctx_t ctx, const char * path, const char** next )
{
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
- const DictionaryEntry * node;
+ const DictionaryEntry * node = NULL;
assert ( self );
- assert ( path );
- if ( path[0] == 0 )
+ if ( path == NULL )
+ INTERNAL_ERROR ( xcParamNull, "path is NULL" );
+ else if ( path[0] == 0 )
{
node = ( const DictionaryEntry * ) BSTreeFirst ( & self -> dictionary );
}
@@ -555,8 +567,10 @@ void SRA_StatisticsAddString ( SRA_Statistics * self, ctx_t ctx, const char * pa
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
assert ( self );
- assert ( path );
+ if ( path == NULL )
+ INTERNAL_ERROR ( xcParamNull, "path is NULL" );
+ else
{
TRY ( DictionaryEntry * node = MakeNode ( self, ctx, path ) )
{
@@ -571,8 +585,10 @@ void SRA_StatisticsAddI64 ( SRA_Statistics * self, ctx_t ctx, const char * path,
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
assert ( self );
- assert ( path );
+ if ( path == NULL )
+ INTERNAL_ERROR ( xcParamNull, "path is NULL" );
+ else
{
TRY ( DictionaryEntry * node = MakeNode ( self, ctx, path ) )
{
@@ -587,8 +603,10 @@ void SRA_StatisticsAddU64 ( SRA_Statistics * self, ctx_t ctx, const char * path,
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
assert ( self );
- assert ( path );
+ if ( path == NULL )
+ INTERNAL_ERROR ( xcParamNull, "path is NULL" );
+ else
{
TRY ( DictionaryEntry * node = MakeNode ( self, ctx, path ) )
{
@@ -603,9 +621,10 @@ void SRA_StatisticsAddDouble ( SRA_Statistics * self, ctx_t ctx, const char * pa
FUNC_ENTRY ( ctx, rcSRA, rcDatabase, rcAccessing );
assert ( self );
- assert ( path );
- if ( isnan ( value ) )
+ if ( path == NULL )
+ INTERNAL_ERROR ( xcParamNull, "path is NULL" );
+ else if ( isnan ( value ) )
{
INTERNAL_ERROR ( xcUnexpected, "NAN is not supported" );
}
diff --git a/libs/ngs/VByteBlob.c b/libs/ngs/VByteBlob.c
new file mode 100644
index 0000000..80e5e79
--- /dev/null
+++ b/libs/ngs/VByteBlob.c
@@ -0,0 +1,117 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include "VByteBlob.h"
+
+#include <kfc/except.h>
+#include <kfc/xc.h>
+
+#include <vdb/blob.h>
+#include <../libs/vdb/blob-priv.h>
+#include <../libs/vdb/page-map.h>
+
+/* Calculate the biggest available contiguous data portion of the blob:
+* starts at rowId, ends before a repeated value or at the end of the blob
+*/
+void
+VByteBlob_ContiguousChunk ( const VBlob* p_blob, ctx_t ctx, int64_t rowId, const void** p_data, uint64_t* p_size, bool p_stopAtRepeat )
+{
+ FUNC_ENTRY ( ctx, rcSRA, rcBlob, rcAccessing );
+
+ assert ( p_blob );
+ assert ( p_data );
+ assert ( p_size );
+
+ {
+ uint32_t elem_bits;
+ const void *base;
+ uint32_t boff;
+ uint32_t row_len;
+ rc_t rc = VBlobCellData ( p_blob,
+ rowId,
+ & elem_bits,
+ & base,
+ & boff,
+ & row_len );
+ if ( rc != 0 )
+ {
+ INTERNAL_ERROR ( xcUnexpected, "VBlobCellData() rc = %R", rc );
+ }
+ else
+ {
+ rc_t rc;
+ int64_t first;
+ uint64_t count;
+
+ assert( elem_bits == 8 );
+ assert( boff == 0 );
+ *p_data = base;
+ *p_size = 0;
+
+ rc = VBlobIdRange ( p_blob, & first, & count );
+ if ( rc != 0 )
+ {
+ INTERNAL_ERROR ( xcUnexpected, "VBlobIdRange() rc = %R", rc );
+ }
+ else if ( p_stopAtRepeat )
+ {
+ PageMapIterator pmIt;
+
+ assert ( rowId >= first && rowId < first + (int64_t)count );
+
+ if ( rowId - first + 1 < (int64_t)count ) /* more rows in the blob */
+ { /* *p_size is the size of value on rowId. Increase size to include subsequent rows, until we see a repeat or the blob ends */
+ rc = PageMapNewIterator ( (const PageMap*)p_blob->pm, &pmIt, rowId - first, count - ( rowId - first ) ); /* here, rowId is relative to the blob */
+ if ( rc != 0 )
+ {
+ INTERNAL_ERROR ( xcUnexpected, "PageMapNewIterator() rc = %R", rc );
+ }
+ else
+ {
+ do
+ {
+ *p_size += PageMapIteratorDataLength ( &pmIt );
+ if ( PageMapIteratorRepeatCount ( &pmIt ) > 1 )
+ {
+ break;
+ }
+ }
+ while ( PageMapIteratorNext ( &pmIt ) );
+ }
+ }
+ else
+ {
+ *p_size = row_len;
+ }
+ }
+ else
+ { /* set the size to include the rest of the blob's data */
+ *p_size = BlobBufferBytes ( p_blob ) - ( (const uint8_t*)( base ) - (const uint8_t*)( p_blob -> data . base ) );
+ }
+ }
+ }
+}
+
diff --git a/libs/kns/sysmgr.h b/libs/ngs/VByteBlob.h
similarity index 76%
copy from libs/kns/sysmgr.h
copy to libs/ngs/VByteBlob.h
index 03928c4..e9d623b 100644
--- a/libs/kns/sysmgr.h
+++ b/libs/ngs/VByteBlob.h
@@ -24,18 +24,24 @@
*
*/
-#ifndef _h_sysmgr_
-#define _h_sysmgr_
+#ifndef _h_vbyteblob_
+#define _h_vbyteblob_
+
+#include <kfc/ctx.h>
+
+struct VBlob;
#ifdef __cplusplus
extern "C" {
#endif
-rc_t KNSManagerInit();
-void KNSManagerCleanup();
+/* Calculate the biggest available contiguous data portion of the blob:
+* starts at rowId, ends before a repeated value or at the end of the blob
+*/
+void VByteBlob_ContiguousChunk ( const struct VBlob* blob, ctx_t ctx, int64_t rowId, const void** data, uint64_t* size, bool stopAtRepeat );
#ifdef __cplusplus
}
#endif
-#endif
\ No newline at end of file
+#endif /* _h_vbyteblob_ */
diff --git a/libs/sra/sramgr-cmn.c b/libs/sra/sramgr-cmn.c
index 14d0c8b..6754190 100644
--- a/libs/sra/sramgr-cmn.c
+++ b/libs/sra/sramgr-cmn.c
@@ -269,7 +269,7 @@ LIB_EXPORT rc_t CC SRAMgrResolve( const SRAMgr *self, const char* acc, char* buf
if (rc == 0)
{
const VPath* resolved;
- rc = VResolverQuery ( (const struct VResolver*)self->_pmgr, eProtocolHttp, path, &resolved, NULL, NULL );
+ rc = VResolverQuery ( (const struct VResolver*)self->_pmgr, 0, path, &resolved, NULL, NULL );
if (rc == 0)
{
rc = VPathReadPath ( resolved, buf, buf_size, NULL );
diff --git a/libs/svdb/svdb.c b/libs/svdb/svdb.c
index 44e46be..2aeac9d 100644
--- a/libs/svdb/svdb.c
+++ b/libs/svdb/svdb.c
@@ -454,7 +454,7 @@ MOD_EXPORT const char * CC svdb_accession_2_path( const char * accession )
else
{
const VPath * local;
- rc = VResolverQuery ( resolver, eProtocolHttp, vpath, &local, NULL, NULL );
+ rc = VResolverQuery ( resolver, 0, vpath, &local, NULL, NULL );
if ( rc != 0 )
log_and_err( rc, "VResolverQuery() failed in svdb_accession_2_path()" );
else
diff --git a/libs/vdb/database-cmn.c b/libs/vdb/database-cmn.c
index 0823127..16b6a89 100644
--- a/libs/vdb/database-cmn.c
+++ b/libs/vdb/database-cmn.c
@@ -360,7 +360,7 @@ LIB_EXPORT rc_t CC VDBManagerVOpenDBRead ( const VDBManager *self,
{
/* otherwise, ask resolver to find a local path,
or get a remote path and optional place to cache data */
- rc = VResolverQuery ( resolver, eProtocolHttp, orig, & plocal, & premote, & pcache );
+ rc = VResolverQuery ( resolver, 0, orig, & plocal, & premote, & pcache );
if ( rc != 0 && GetRCState ( rc ) == rcNotFound )
{
rc = VPathAddRef ( orig );
@@ -430,7 +430,7 @@ LIB_EXPORT rc_t CC VDBManagerVOpenDBRead ( const VDBManager *self,
KLogLevelSet ( klogFatal );
assert ( premote == NULL );
assert ( pcache == NULL );
- rc2 = VResolverQuery ( resolver, eProtocolHttp, orig, NULL, & premote, & pcache );
+ rc2 = VResolverQuery ( resolver, 0, orig, NULL, & premote, & pcache );
assert ( ( rc2 == 0 ) ||
( rc2 != 0 && premote == NULL ) );
diff --git a/libs/vdb/database-depend.c b/libs/vdb/database-depend.c
index 72e8cfc..137b8c2 100644
--- a/libs/vdb/database-depend.c
+++ b/libs/vdb/database-depend.c
@@ -533,7 +533,7 @@ static rc_t FindRef(Ctx* ctx, const char* seqId, Resolved* resolved,
}
if (rc == 0) {
- rc = VResolverRemote(ctx->resolver, eProtocolHttp, acc, &remote);
+ rc = VResolverRemote(ctx->resolver, 0, acc, &remote);
if (rc == 0) {
rc = VPathMakeString(remote, &resolved->remote);
if (rc == 0) {
diff --git a/libs/vdb/dbmgr-cmn.c b/libs/vdb/dbmgr-cmn.c
index a423566..d3f3c5f 100644
--- a/libs/vdb/dbmgr-cmn.c
+++ b/libs/vdb/dbmgr-cmn.c
@@ -915,3 +915,29 @@ LIB_EXPORT rc_t CC VDBManagerSetCacheRoot ( const struct VDBManager * self,
}
return rc;
}
+
+
+LIB_EXPORT rc_t CC VDBManagerDeleteCacheOlderThan ( const VDBManager * self,
+ uint32_t days )
+{
+ rc_t rc;
+ if ( self == NULL )
+ rc = RC ( rcVDB, rcMgr, rcSelecting, rcSelf, rcNull );
+ else
+ {
+ const KDBManager * kbd = NULL;
+ rc = VDBManagerGetKDBManagerRead ( self, & kbd );
+ if ( rc == 0 )
+ {
+ VFSManager * vfs = NULL;
+ rc = KDBManagerGetVFSManager ( kbd, & vfs );
+ if ( rc == 0 )
+ {
+ rc = VFSManagerDeleteCacheOlderThan ( vfs, days );
+ VFSManagerRelease ( vfs );
+ }
+ KDBManagerRelease ( kbd );
+ }
+ }
+ return rc;
+}
diff --git a/libs/vdb/libvdb.vers.h b/libs/vdb/libvdb.vers.h
index 28af961..aa0b7cd 100644
--- a/libs/vdb/libvdb.vers.h
+++ b/libs/vdb/libvdb.vers.h
@@ -24,4 +24,4 @@
*
*/
-#define LIBVDB_VERS 0x02070014
+#define LIBVDB_VERS 0x02070015
diff --git a/libs/vdb/prod-cmn.c b/libs/vdb/prod-cmn.c
index b80c003..a77c8a8 100644
--- a/libs/vdb/prod-cmn.c
+++ b/libs/vdb/prod-cmn.c
@@ -508,7 +508,108 @@ rc_t VFunctionProdCallNDRowFunc(
/* TODO: enable in next release */
#define PAGEMAP_PRE_EXPANDING_SINGLE_ROW_FIX 0
static
-rc_t VFunctionProdCallRowFunc( VFunctionProd *self, VBlob **prslt, int64_t row_id,
+rc_t VFunctionProdCallNullaryRowFunc(VFunctionProd *self,
+ VBlob **prslt,
+ int64_t row_id,
+ uint32_t row_count,
+ const VXformInfo *info)
+{
+ rc_t rc = 0;
+ KDataBuffer scratch;
+ VRowData args[1];
+ VRowResult rslt;
+
+ memset(&scratch, 0, sizeof(scratch));
+ memset(&args[0], 0, sizeof(args[0]));
+
+ rslt.data = &scratch;
+ rslt.elem_count = 0;
+ rslt.elem_bits = scratch.elem_bits = VTypedescSizeof(&self->dad.desc);
+
+ rc = self->u.rf(self->fself, info, row_id, &rslt, 0, args);
+ if (rc == 0) {
+ VBlob *blob = NULL;
+
+#if PROD_NAME
+ rc = VBlobNew ( &blob, -INT64_MAX - 1, INT64_MAX, self->dad.name );
+#else
+ rc = VBlobNew ( &blob, -INT64_MAX - 1, INT64_MAX, "VFunctionProdCallDetRowFunc" );
+#endif
+ if (rc == 0) {
+ blob->byte_order = vboNative;
+ assert(rslt.elem_count <= UINT32_MAX);
+ KDataBufferSub(rslt.data, &blob->data, 0, rslt.elem_count);
+ if ( rslt.data != & scratch )
+ KDataBufferWhack(rslt.data);
+ rc = PageMapNewSingle(&blob->pm, UINT32_MAX, (uint32_t)rslt.elem_count);
+ if (rc == 0)
+ *prslt = blob;
+ else
+ vblob_release(blob, NULL);
+ }
+ }
+ KDataBufferWhack(&scratch);
+ return rc;
+}
+
+static bool computeWindow(uint32_t *const pwindow, int64_t const start_id, int64_t const stop_id, int64_t const row_id, uint32_t const max_blob_regroup)
+{
+ int64_t window = stop_id - start_id + 1;
+ bool window_resized = false;
+
+ /*** from previous fetch **/
+
+ /** detect sequentual io ***/
+ if (row_id == stop_id + 1)
+ {
+ if (window > max_blob_regroup)
+ {
+ window = max_blob_regroup;
+ window_resized = true;
+ }
+ else if (row_id % (4 * window) == 1)
+ {
+ if (window < max_blob_regroup)
+ {
+ if (4 * window <= max_blob_regroup)
+ window *= 4;
+ else
+ window = max_blob_regroup;
+ window_resized = true;
+ }
+ /* we know that row_id lands on the first row of the new window */
+ }
+ }
+ else
+ {
+ /* random access - use tiny blob window */
+ window = 1;
+ window_resized = true;
+ }
+ assert(window <= UINT32_MAX);
+ *pwindow = window;
+ return window_resized;
+}
+
+#if UNIT_TEST_VDB_3058
+static void UnitTest_VDB_3058(void)
+{
+/* test for case of integer overflow to zero causing divide-by-zero error in computeWindow */
+ uint32_t max_blob_regroup = 1024;
+ uint32_t window = 0;
+ int64_t start_id = 1;
+ int64_t stop_id = UINT32_MAX / 4 + 1;
+ int64_t row_id = stop_id + 1;
+ bool window_resized = computeWindow(&window, start_id, stop_id, row_id, max_blob_regroup);
+
+ /* this output isn't important, it's just to make sure that
+ * the compiler doesn't optimize out the relevant computations */
+ printf("window: %u\nwindow_resized: %s\n", window, window_resized ? "true" : "false");
+}
+#endif
+
+static
+rc_t VFunctionProdCallRowFunc1( VFunctionProd *self, VBlob **prslt, int64_t row_id,
uint32_t row_count, const VXformInfo *info, Vector *args,int64_t param_start_id,int64_t param_stop_id)
{
rc_t rc;
@@ -524,79 +625,27 @@ rc_t VFunctionProdCallRowFunc( VFunctionProd *self, VBlob **prslt, int64_t row_i
uint32_t window;
uint32_t min_row_count=UINT32_MAX; /* will increase row_count due to larger common repeat count of parameters */
int64_t row_id_max=0;
- uint32_t MAX_BLOB_REGROUP; /** max rows in blob for regrouping ***/
+ uint32_t max_blob_regroup; /** max rows in blob for regrouping ***/
bool function_failed = false;
bool window_resized = false;
- if (argc == 0) {
- memset(&scratch, 0, sizeof(scratch));
- memset(&args_os[0], 0, sizeof(args_os[0]));
- rslt.data = &scratch;
- rslt.elem_count = 0;
- rslt.elem_bits = scratch.elem_bits = VTypedescSizeof(&self->dad.desc);
-
- rc = self->u.rf(self->fself, info, row_id, &rslt, 0, args_os);
- if (rc == 0) {
-#if PROD_NAME
- rc = VBlobNew ( &blob, -INT64_MAX - 1, INT64_MAX, self->dad.name );
-#else
- rc = VBlobNew ( &blob, -INT64_MAX - 1, INT64_MAX, "VFunctionProdCallDetRowFunc" );
-#endif
- if (rc == 0) {
- blob->byte_order = vboNative;
- assert(rslt.elem_count <= UINT32_MAX);
- KDataBufferSub(rslt.data, &blob->data, 0, rslt.elem_count);
- if ( rslt.data != & scratch )
- KDataBufferWhack(rslt.data);
- rc = PageMapNewSingle(&blob->pm, UINT32_MAX, (uint32_t)rslt.elem_count);
- if (rc == 0)
- *prslt = blob;
- else
- vblob_release(blob, NULL);
- }
- }
- KDataBufferWhack(&scratch);
- return rc;
- }
-
if(self->curs->cache_curs && self->curs->cache_col_active){
/*** since cache_cursor exist, trying to avoid prefetching data which is in cache cursor ***/
row_id_max = self->curs->cache_empty_end;
- MAX_BLOB_REGROUP=256;
+ max_blob_regroup=256;
} else {
- MAX_BLOB_REGROUP=1024;
+ max_blob_regroup=1024;
}
if(self->dad.sub == vftRowFast)
{
- window = MAX_BLOB_REGROUP;
+ window = max_blob_regroup;
}
else
{
- /*** from previous fetch **/
- window = self->stop_id - self->start_id + 1;
+ window_resized = computeWindow(&window, self->start_id, self->stop_id, row_id, max_blob_regroup);
+ }
- /** detect sequentual io ***/
- if ( row_id == self->stop_id + 1 )
- {
- if( row_id % ( 4 * window ) == 1 )
- {
- if ( window < MAX_BLOB_REGROUP )
- {
- if ( 4 * window <= MAX_BLOB_REGROUP )
- window *= 4;
- else
- window = MAX_BLOB_REGROUP;
- window_resized = true;
- }
- /* we know that row_id lands on the first row of the new window */
- }
- }
- else
- {
- /* random access - use tiny blob window */
- window = 1;
- }
- }
+ assert ( 0 < window && window <= max_blob_regroup );
if(window == 1)
{
@@ -807,6 +856,18 @@ rc_t VFunctionProdCallRowFunc( VFunctionProd *self, VBlob **prslt, int64_t row_i
}
static
+rc_t VFunctionProdCallRowFunc( VFunctionProd *self, VBlob **prslt, int64_t row_id,
+ uint32_t row_count, const VXformInfo *info, Vector *args,int64_t param_start_id,int64_t param_stop_id)
+{
+ uint32_t const argc = VectorLength(args);
+
+ if (argc == 0)
+ return VFunctionProdCallNullaryRowFunc(self, prslt, row_id, row_count, info);
+ else
+ return VFunctionProdCallRowFunc1(self, prslt, row_id, row_count, info, args, param_start_id, param_stop_id);
+}
+
+static
rc_t VFunctionProdCallArrayFunc( VFunctionProd *self, VBlob **prslt,
int64_t id, const VXformInfo *info, Vector *args ) {
VBlob *rslt = 0;
diff --git a/test/vxf/Makefile b/libs/vdb/test/Makefile
similarity index 76%
copy from test/vxf/Makefile
copy to libs/vdb/test/Makefile
index 064aab3..dbd648c 100644
--- a/test/vxf/Makefile
+++ b/libs/vdb/test/Makefile
@@ -22,41 +22,30 @@
#
# ===========================================================================
-default: runtests
-TOP ?= $(abspath ../..)
+default: runtests
-MODULE = test/vxf
+TOP ?= $(abspath ../../..)
+MODULE = libs/vdb/test
-TEST_TOOLS = \
- wb-test-vxf
+TEST_TOOLS=\
+ test-prod-cmn
include $(TOP)/build/Makefile.env
$(TEST_TOOLS): makedirs
@ $(MAKE_CMD) $(TEST_BINDIR)/$@
-.PHONY: $(TEST_TOOLS)
-
clean: stdclean
-#-------------------------------------------------------------------------------
-# white-box test
-#
-TEST_SRC = \
- wb-test-vxf \
- wb-irzip-impl
-
-TEST_OBJ = \
- $(addsuffix .$(OBJX),$(TEST_SRC))
-
-TEST_LIBS = \
- -skapp \
- -sktst \
- -sncbi-vdb \
- -sxml2 \
- -sm
-
-$(TEST_BINDIR)/wb-test-vxf: $(TEST_OBJ)
- $(LP) --exe -o $@ $^ $(TEST_LIBS)
+LIB=\
+ -skapp \
+ -sncbi-vdb
+
+VDB_DIR=../
+
+OBJ=$(addsuffix .$(OBJX),test-prod-cmn)
+
+$(TEST_BINDIR)/test-prod-cmn: $(OBJ) $(addprefix $(VDB_DIR),prod-cmn.c)
+ $(LD) --exe -o $@ $(OBJ) $(LIB)
diff --git a/libs/vdb/test/test-prod-cmn.c b/libs/vdb/test/test-prod-cmn.c
new file mode 100644
index 0000000..7d46374
--- /dev/null
+++ b/libs/vdb/test/test-prod-cmn.c
@@ -0,0 +1,10 @@
+#include <stdio.h>
+
+#define UNIT_TEST_VDB_3058 (1)
+#include "../prod-cmn.c"
+
+int main(int argc, char *argv[])
+{
+ UnitTest_VDB_3058();
+ return 0;
+}
diff --git a/libs/vdb/wdatabase.c b/libs/vdb/wdatabase.c
index d597aec..f330c74 100644
--- a/libs/vdb/wdatabase.c
+++ b/libs/vdb/wdatabase.c
@@ -417,7 +417,7 @@ LIB_EXPORT rc_t CC VDatabaseVOpenDBUpdate ( VDatabase *self, VDatabase **dbp,
rc = RC ( rcVDB, rcDatabase, rcOpening, rcDatabase, rcReadonly );
else
{
- rc = VDatabaseMake ( dbp, NULL, self, self -> schema );
+ rc = VDatabaseMake ( dbp, self -> mgr, self, self -> schema );
if ( rc == 0 )
{
VDatabase *db = * dbp;
diff --git a/libs/vfs/manager.c b/libs/vfs/manager.c
index 70a5e4b..96963d3 100644
--- a/libs/vfs/manager.c
+++ b/libs/vfs/manager.c
@@ -27,12 +27,12 @@
#include <vfs/extern.h>
#include "path-priv.h"
-#include "resolver-priv.h" /* VResolverGetProjectId */
+#include "resolver-priv.h"
#include <sra/srapath.h>
#include <vfs/manager.h>
-#include <vfs/manager-priv.h> /* VFSManagerMakeFromKfg */
+#include <vfs/manager-priv.h>
#include <vfs/path.h>
#include <vfs/path-priv.h>
#include <vfs/resolver.h>
@@ -59,20 +59,22 @@
#include <kfs/quickmount.h>
#include <kfs/cacheteefile.h>
#include <kfs/lockfile.h>
+#include <kfs/defs.h>
#include <kns/http.h>
-#include <kns/kns-mgr-priv.h> /* KNSManagerMakeReliableHttpFile */
+#include <kns/kns-mgr-priv.h>
#include <kns/manager.h>
#include <kxml/xml.h>
-#include <klib/debug.h> /* DBGMSG */
+#include <klib/debug.h>
#include <klib/log.h>
#include <klib/printf.h>
#include <klib/rc.h>
#include <klib/refcount.h>
#include <klib/namelist.h>
#include <klib/vector.h>
+#include <klib/time.h>
#include <strtol.h>
@@ -84,7 +86,7 @@
#include <assert.h>
-#ifdef _DEBUGGING
+#if _DEBUGGING
#define MGR_DEBUG(msg) DBGMSG(DBG_VFS,DBG_FLAG(DBG_VFS_MGR), msg)
#else
#define MGR_DEBUG(msg)
@@ -125,6 +127,8 @@ struct VFSManager
struct KKeyStore* keystore;
KRefcount refcount;
+
+ VRemoteProtocols protocols;
};
static const char kfsmanager_classname [] = "VFSManager";
@@ -522,7 +526,7 @@ static rc_t VFSManagerResolvePathResolver (const VFSManager * self,
if (not_done && ((flags & vfsmgr_rflag_no_acc_remote) == 0))
{
- rc = VResolverRemote (self->resolver, eProtocolHttp,
+ rc = VResolverRemote (self->resolver, self -> protocols,
in_path, (const VPath **)out_path);
}
}
@@ -626,6 +630,7 @@ static rc_t VFSManagerResolvePathInt (const VFSManager * self,
/* these are considered fully resolved already */
case vpuri_http:
+ case vpuri_https:
case vpuri_ftp:
rc = VPathAddRef (in_path);
if (rc == 0)
@@ -1245,6 +1250,7 @@ LIB_EXPORT rc_t CC VFSManagerOpenFileRead ( const VFSManager *self,
break;
case vpuri_http:
+ case vpuri_https:
case vpuri_ftp:
rc = VFSManagerOpenCurlFile ( self, f, path );
break;
@@ -1285,6 +1291,7 @@ rc_t CC VFSManagerOpenDirectoryUpdateDirectoryRelative (const VFSManager *self,
switch ( uri_type )
{
case vpuri_http :
+ case vpuri_https:
case vpuri_ftp :
return RC( rcVFS, rcMgr, rcOpening, rcParam, rcWrongType );
@@ -1433,8 +1440,16 @@ rc_t VFSManagerOpenDirectoryReadHttp (const VFSManager *self,
sizeof extension - 1,
extension, sizeof extension - 1, sizeof extension - 1 ) != 0 )
{
- PLOGERR ( klogErr, ( klogErr, rc, "error with http open '$(U)'",
- "U=%S:%S", & path -> scheme, & s ) );
+ const String * p = NULL;
+ rc_t rc = VPathMakeString ( path, & p );
+ if ( rc == 0 ) {
+ PLOGERR ( klogErr, ( klogErr, rc, "error with http open '$(path)'",
+ "path=%S", p ) );
+ free ( ( void * ) p );
+ } else {
+ PLOGERR ( klogErr, ( klogErr, rc, "error with http open '$(scheme):$(path)'",
+ "scheme=%S,path=%S", & path -> scheme, s ) );
+ }
}
}
else
@@ -1510,7 +1525,7 @@ rc_t VFSManagerOpenDirectoryReadHttpResolved (const VFSManager *self,
if ( high_reliability )
{
PLOGERR ( klogErr, ( klogErr, rc, "error with http open '$(U)'",
- "U=%s", uri->addr ) );
+ "U=%S", uri ) );
}
}
else
@@ -1823,6 +1838,7 @@ rc_t VFSManagerOpenDirectoryReadDirectoryRelativeInt (const VFSManager *self,
break;
case vpuri_http:
+ case vpuri_https:
case vpuri_ftp:
rc = VFSManagerOpenDirectoryReadHttp ( self, dir, d, path, force_decrypt );
break;
@@ -1888,6 +1904,7 @@ rc_t CC VFSManagerOpenDirectoryReadDecryptRemote (const VFSManager *self,
switch ( VPathGetUri_t ( path ) )
{
case vpuri_http:
+ case vpuri_https:
case vpuri_ftp:
rc = VFSManagerOpenDirectoryReadHttpResolved ( self, d, path, cache, true );
break;
@@ -2256,6 +2273,80 @@ LIB_EXPORT rc_t CC VFSManagerRemove ( const VFSManager *self, bool force,
return rc;
}
+/* RemoteProtocols
+ */
+LIB_EXPORT VRemoteProtocols CC VRemoteProtocolsParse ( const String * protos )
+{
+ VRemoteProtocols parsed_protos = 0;
+
+ bool have_proto [ eProtocolMask + 1 ];
+
+ size_t i, end;
+ const char * start;
+ String http, https, fasp;
+
+ CONST_STRING ( & http, "http" );
+ CONST_STRING ( & https, "https" );
+ CONST_STRING ( & fasp, "fasp" );
+
+ end = protos -> size;
+ start = protos -> addr;
+
+ memset ( have_proto, 0, sizeof have_proto );
+
+ for ( i = end; i > 0; )
+ {
+ -- i;
+ if ( i == 0 || start [ i ] == ',' )
+ {
+ VRemoteProtocols parsed_proto = 0;
+
+ /* beginning of protocol string is either 0 or 1 past the comma */
+ size_t begin = ( i == 0 ) ? 0 : i + 1;
+
+ /* capture single protocol string */
+ String proto;
+ StringInit ( & proto, & start [ begin ], end - begin, string_len ( & start [ begin ], end - begin ) );
+
+ /* trim white space */
+ StringTrim ( & proto, & proto );
+
+ /* compare against known protocols */
+ if ( StringCaseEqual ( & http, & proto ) )
+ parsed_proto = eProtocolHttp;
+ else if ( StringCaseEqual ( & https, & proto ) )
+ parsed_proto = eProtocolHttps;
+ else if ( StringCaseEqual ( & fasp, & proto ) )
+ parsed_proto = eProtocolFasp;
+
+ if ( parsed_proto != eProtocolNone && ! have_proto [ parsed_proto ] )
+ {
+ parsed_protos <<= 3;
+ parsed_protos |= parsed_proto;
+ have_proto [ parsed_proto ] = true;
+ }
+
+ end = i;
+ }
+ }
+
+ return parsed_protos;
+}
+
+void KConfigReadRemoteProtocols ( const KConfig * self, VRemoteProtocols * remote_protos )
+{
+ String * protos;
+ rc_t rc = KConfigReadString ( self, "/name-resolver/remote-protocols", & protos );
+ if ( rc == 0 )
+ {
+ VRemoteProtocols parsed_protos = VRemoteProtocolsParse ( protos );
+ if ( parsed_protos != 0 )
+ * remote_protos = parsed_protos;
+
+ StringWhack ( protos );
+ }
+}
+
/* Make
*/
LIB_EXPORT rc_t CC VFSManagerMake ( VFSManager ** pmanager )
@@ -2292,6 +2383,9 @@ LIB_EXPORT rc_t CC VFSManagerMakeFromKfg ( struct VFSManager ** pmanager,
KRefcountInit (& obj -> refcount, 1,
kfsmanager_classname, "init", "singleton" );
+ /* hard-coded default */
+ obj -> protocols = eProtocolHttpHttps;
+
rc = KDirectoryNativeDir ( & obj -> cwd );
if ( rc == 0 )
{
@@ -2305,6 +2399,9 @@ LIB_EXPORT rc_t CC VFSManagerMakeFromKfg ( struct VFSManager ** pmanager,
}
if ( rc == 0 )
{
+ /* look for remote protocols in configuration */
+ KConfigReadRemoteProtocols ( obj -> cfg, & obj -> protocols );
+
rc = KCipherManagerMake ( & obj -> cipher );
if ( rc == 0 )
{
@@ -2326,7 +2423,7 @@ LIB_EXPORT rc_t CC VFSManagerMakeFromKfg ( struct VFSManager ** pmanager,
}
*pmanager = singleton = obj;
- DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_MGR), ("%s(%p)\n", __FUNCTION__, cfg));
+ DBGMSG(DBG_KNS, DBG_FLAG(DBG_KNS_MGR), ("%s(%p)\n", __FUNCTION__, cfg));
return 0;
}
}
@@ -2377,6 +2474,8 @@ LIB_EXPORT rc_t CC VFSManagerGetResolver ( const VFSManager * self, struct VReso
{
if ( self == NULL )
rc = RC (rcVFS, rcMgr, rcAccessing, rcSelf, rcNull);
+ else if ( self -> resolver == NULL )
+ rc = RC ( rcVFS, rcMgr, rcAccessing, rcResolver, rcNull );
else
{
rc = VResolverAddRef ( self -> resolver );
@@ -2944,7 +3043,7 @@ static rc_t VFSManagerResolveAcc( const VFSManager * self,
assert (local_cache);
#if 1
- rc = VResolverQuery ( self -> resolver, eProtocolHttp, source, & local, & remote, local_cache );
+ rc = VResolverQuery ( self -> resolver, self -> protocols, source, & local, & remote, local_cache );
if ( rc == 0 )
{
assert ( local != NULL || remote != NULL );
@@ -2958,7 +3057,7 @@ static rc_t VFSManagerResolveAcc( const VFSManager * self,
if ( GetRCState( rc ) == rcNotFound )
{
/* if not found localy, try to find it remotely */
- rc = VResolverRemote ( self->resolver, eProtocolHttp,
+ rc = VResolverRemote ( self->resolver, self -> protocols,
source, (const VPath **)path_to_build, remote_file );
if ( rc == 0 && remote_file != NULL && local_cache != NULL )
{
@@ -3108,6 +3207,7 @@ LIB_EXPORT rc_t CC VFSManagerResolveSpec ( const VFSManager * self,
break;
case vpuri_http : /* !! fall through !! */
+ case vpuri_https:
case vpuri_ftp : rc = VFSManagerResolveRemote( self, &temp, path_to_build, remote_file, local_cache );
break;
@@ -3288,7 +3388,7 @@ LIB_EXPORT rc_t CC VFSManagerSetCacheRoot ( const VFSManager * self,
rc = KRepositoryName( repo, repo_name, sizeof repo_name, &repo_name_len );
if ( rc == 0 )
{
- char new_root[ 1024 ];
+ char new_root[ 4096 ];
size_t num_writ;
repo_name[ repo_name_len ] = 0;
rc = string_printf( new_root, sizeof new_root, &num_writ, indirect_root, repo_name );
@@ -3324,3 +3424,96 @@ LIB_EXPORT rc_t CC VFSManagerSetCacheRoot ( const VFSManager * self,
}
return rc;
}
+
+
+static rc_t inspect_file( KDirectory * dir, KTime_t date, const char * path )
+{
+ KTime_t file_date;
+ rc_t rc = KDirectoryDate ( dir, &file_date, "%s", path );
+ if ( rc == 0 )
+ {
+ if ( file_date < date )
+ KDirectoryRemove ( dir, false, "%s", path );
+ }
+ return rc;
+}
+
+
+static rc_t inspect_dir( KDirectory * dir, KTime_t date, const char * path )
+{
+ KNamelist * itemlist;
+ rc_t rc = KDirectoryList( dir, &itemlist, NULL, NULL, "%s", path );
+ if ( rc == 0 )
+ {
+ uint32_t count, idx;
+ rc = KNamelistCount ( itemlist, &count );
+ for ( idx = 0; rc == 0 && idx < count; idx++ )
+ {
+ const char * item;
+ rc = KNamelistGet ( itemlist, idx, &item );
+ {
+ char item_path[ 4096 ];
+ size_t num_writ;
+ rc = string_printf ( item_path, sizeof item_path, &num_writ, "%s/%s", path, item );
+ if ( rc == 0 )
+ {
+ uint32_t pathtype = KDirectoryPathType( dir, "%s", item_path );
+ switch( pathtype )
+ {
+ case kptFile : rc = inspect_file( dir, date, item_path ); break;
+ case kptDir : rc = inspect_dir( dir, date, item_path ); break; /* recursion! */
+ default : break;
+ }
+ }
+ }
+ }
+ KNamelistRelease( itemlist );
+ }
+ return rc;
+}
+
+
+LIB_EXPORT rc_t CC VFSManagerDeleteCacheOlderThan ( const VFSManager * self,
+ uint32_t days )
+{
+ rc_t rc;
+ if ( self == NULL )
+ rc = RC ( rcVFS, rcMgr, rcSelecting, rcSelf, rcNull );
+ else if ( self -> cfg == NULL )
+ rc = RC ( rcVFS, rcMgr, rcSelecting, rcItem, rcNull );
+ else
+ {
+ /* loop through the user-repositories to get the root property */
+ const KRepositoryMgr * repo_mgr;
+ rc = KConfigMakeRepositoryMgrRead ( self -> cfg, &repo_mgr );
+ if ( rc == 0 )
+ {
+ KRepositoryVector user_repos;
+ rc = KRepositoryMgrUserRepositories ( repo_mgr, &user_repos );
+ if ( rc == 0 )
+ {
+ uint32_t start = VectorStart( &user_repos );
+ uint32_t count = VectorLength( &user_repos );
+ uint32_t idx;
+ for ( idx = 0; rc == 0 && idx < count; ++idx )
+ {
+ KRepository * repo = VectorGet ( &user_repos, idx + start );
+ if ( repo != NULL )
+ {
+ char path[ 4096 ];
+ size_t root_size;
+ rc = KRepositoryRoot ( repo, path, sizeof path, &root_size );
+ if ( rc == 0 )
+ {
+ KTime_t date = KTimeStamp() - ( days * 60 * 60 * 24 );
+ rc = inspect_dir( self->cwd, date, path );
+ }
+ }
+ }
+ KRepositoryVectorWhack ( &user_repos );
+ }
+ KRepositoryMgrRelease ( repo_mgr );
+ }
+ }
+ return rc;
+}
diff --git a/libs/vfs/path-priv.h b/libs/vfs/path-priv.h
index d05d226..902842b 100644
--- a/libs/vfs/path-priv.h
+++ b/libs/vfs/path-priv.h
@@ -126,6 +126,7 @@ typedef enum eVPUri_t
vpuri_file,
vpuri_ncbi_acc,
vpuri_http,
+ vpuri_https,
vpuri_ftp,
vpuri_ncbi_legrefseq,
vpuri_ncbi_obj, /* for dbGaP objects */
diff --git a/libs/vfs/path.c b/libs/vfs/path.c
index 559c978..240a289 100644
--- a/libs/vfs/path.c
+++ b/libs/vfs/path.c
@@ -251,6 +251,18 @@ void VPathCaptureScheme ( VPath * self, const char * uri, size_t start, size_t e
break;
}
break;
+ case 5:
+ /* 5 character schemes */
+ switch ( uri [ 0 ] )
+ {
+ case 'h':
+ case 'H':
+ /* https */
+ if ( strcase_cmp ( scheme + 1, 4, "https" + 1, 4, 4 ) == 0 )
+ self -> scheme_type = vpuri_https;
+ break;
+ }
+ break;
case 8:
/* 8 character schemes starting with "ncbi-" */
@@ -3507,7 +3519,7 @@ rc_t LegacyVPathResolveAccession ( VPath ** new_path, const VPath * path )
{
rc = VResolverLocal ( resolver, path, ( const VPath** ) new_path );
if ( GetRCState ( rc ) == rcNotFound )
- rc = VResolverRemote ( resolver, eProtocolHttp, path, ( const VPath** ) new_path );
+ rc = VResolverRemote ( resolver, 0, path, ( const VPath** ) new_path );
VResolverRelease ( resolver );
}
@@ -3590,6 +3602,7 @@ LIB_EXPORT rc_t CC LegacyVPathMakeDirectoryRelative ( VPath ** new_path,
break;
case vpuri_http:
+ case vpuri_https:
case vpuri_ftp:
case vpuri_fasp:
/* calling code would know how to handle these */
diff --git a/libs/vfs/resolver-priv.h b/libs/vfs/resolver-priv.h
index abd7f45..c82a135 100644
--- a/libs/vfs/resolver-priv.h
+++ b/libs/vfs/resolver-priv.h
@@ -234,6 +234,9 @@ rc_t VResolverRemoteResolve ( const VResolver *self,
const struct VPath ** path, const struct VPath **mapping,
const struct KFile ** opt_file_rtn, bool refseq_ctx, bool is_oid );
+
+void KConfigReadRemoteProtocols ( struct KConfig const * self, VRemoteProtocols * remote_protos );
+
#ifdef __cplusplus
}
#endif
diff --git a/libs/vfs/resolver.c b/libs/vfs/resolver.c
index c920814..a157ea3 100644
--- a/libs/vfs/resolver.c
+++ b/libs/vfs/resolver.c
@@ -82,18 +82,18 @@
#define ALLOW_AUX_REPOSITORIES 0
#define NAME_SERVICE_MAJ_VERS_ 1
-#define NAME_SERVICE_MIN_VERS_ 1
+#define NAME_SERVICE_MIN_VERS_ 2
#define ONE_DOT_ONE 0x01010000
static uint32_t NAME_SERVICE_MAJ_VERS = NAME_SERVICE_MAJ_VERS_;
-static uint32_t NAME_SERVICE_MIN_VERS = NAME_SERVICE_MAJ_VERS_;
+static uint32_t NAME_SERVICE_MIN_VERS = NAME_SERVICE_MIN_VERS_;
static uint32_t NAME_SERVICE_VERS
- = NAME_SERVICE_MAJ_VERS_ << 24 | NAME_SERVICE_MAJ_VERS_ << 16;
+ = NAME_SERVICE_MAJ_VERS_ << 24 | NAME_SERVICE_MIN_VERS_ << 16;
static void VFSManagerSetNameResolverVersion(uint32_t maj, uint32_t min) {
NAME_SERVICE_MAJ_VERS = maj;
NAME_SERVICE_MIN_VERS = min;
NAME_SERVICE_VERS
- = NAME_SERVICE_MAJ_VERS_ << 24 | NAME_SERVICE_MAJ_VERS_ << 16;
+ = NAME_SERVICE_MAJ_VERS_ << 24 | NAME_SERVICE_MIN_VERS_ << 16;
}
void VFSManagerSetNameResolverVersion3_0(void)
{ VFSManagerSetNameResolverVersion(3, 0); }
@@ -655,10 +655,11 @@ rc_t VPathCheckFromNamesCGI ( const VPath * path, const String *ticket, const VP
if ( ! path -> from_uri )
return RC ( rcVFS, rcResolver, rcResolving, rcMessage, rcCorrupt );
- /* can only be http or fasp */
+ /* can only be http, https or fasp */
switch ( path -> scheme_type )
{
case vpuri_http:
+ case vpuri_https:
case vpuri_fasp:
break;
default:
@@ -1221,6 +1222,7 @@ typedef enum {
vBad,
v1_0,
v1_1,
+ v1_2,
v2,
v3,
} TVersion;
@@ -1235,6 +1237,7 @@ rc_t VResolverAlgParseResolverCGIResponse ( const KDataBuffer *result,
{
const char V1_0[] = "#1.0";
const char V1_1[] = "#1.1";
+ const char V1_2[] = "#1.2";
const char V2 [] = "#2.0";
const char V3 [] = "#3.0";
struct {
@@ -1245,6 +1248,7 @@ rc_t VResolverAlgParseResolverCGIResponse ( const KDataBuffer *result,
const VPath **mapping, const String *acc, const String *ticket);
} version[] = {
{V1_1, sizeof V1_1 - 1, v1_1, VResolverAlgParseResolverCGIResponse_1_1},
+ {V1_2, sizeof V1_2 - 1, v1_2, VResolverAlgParseResolverCGIResponse_1_1},
{V3 , sizeof V3 - 1, v3 , VResolverAlgParseResolverCGIResponse_3_0},
{V1_0, sizeof V1_0 - 1, v1_0, VResolverAlgParseResolverCGIResponse_1_0},
{V2 , sizeof V2 - 1, v2 , VResolverAlgParseResolverCGIResponse_2_0},
@@ -1306,6 +1310,25 @@ rc_t VResolverAlgParseResolverCGIResponse ( const KDataBuffer *result,
}
}
+
+#ifdef VDB_3162
+#error 1
+/* test-only code to emulate 403 response while calling names.cgi */
+static bool TEST_VDB_3162 = false;
+void TESTING_VDB_3162 ( void ) {
+ TEST_VDB_3162 = true;
+}
+static uint32_t TESTING_VDB_3162_CODE ( rc_t rc, uint32_t code ) {
+ if ( rc == 0 && TEST_VDB_3162 ) {
+ TEST_VDB_3162 = false;
+ return 403;
+ } else {
+ return code;
+ }
+}
+#endif
+
+
/* RemoteProtectedResolve
* use NCBI CGI to resolve accession into URL
*/
@@ -1319,7 +1342,7 @@ rc_t VResolverAlgRemoteProtectedResolve( const VResolverAlg *self,
assert(path);
DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS), ("names.cgi = %S\n", self -> root));
- rc = KNSManagerMakeReliableClientRequest ( kns, & req, 0x01000000, NULL, self -> root -> addr );
+ rc = KNSManagerMakeReliableClientRequest ( kns, & req, 0x01010000, NULL, self -> root -> addr );
if ( rc == 0 )
{
/* build up POST information: */
@@ -1343,34 +1366,53 @@ rc_t VResolverAlgRemoteProtectedResolve( const VResolverAlg *self,
rc = KHttpRequestAddPostParam ( req, "tic=%S", self -> ticket );
}
- if (NAME_SERVICE_VERS >= ONE_DOT_ONE) { /* SRA-1690 */
- if ( rc == 0 ) {
- const char *val;
- switch ( protocols ) {
- case eProtocolHttp:
- val = "http";
- break;
- case eProtocolFasp:
- val = "fasp";
- break;
- case eProtocolFaspHttp:
- val = "fasp,http";
- break;
- case eProtocolHttpFasp:
- val = "http,fasp";
- break;
- default:
- val = NULL;
- rc = RC(
- rcVFS, rcResolver, rcResolving, rcParam, rcInvalid);
- }
+ if ( rc == 0 && NAME_SERVICE_VERS >= ONE_DOT_ONE )
+ {
+ uint32_t i;
+ const char * prefs [ eProtocolMaxPref ];
+ const char * seps [ eProtocolMaxPref ];
+ VRemoteProtocols protos = protocols;
- if ( rc == 0 ) {
- DBGMSG(DBG_VFS, DBG_FLAG(DBG_VFS),
- (" accept-proto = %s\n", val));
- rc = KHttpRequestAddPostParam(req, "accept-proto=%s", val);
+ prefs [ 0 ] = seps [ 0 ] = NULL;
+ prefs [ 1 ] = prefs [ 2 ] = seps [ 1 ] = seps [ 2 ] = "";
+
+ for ( i = 0; protos != 0 && i < sizeof prefs / sizeof prefs [ 0 ]; protos >>= 3 )
+ {
+ /* 1.1 protocols */
+ switch ( protos & eProtocolMask )
+ {
+ case eProtocolHttp:
+ prefs [ i ] = "http";
+ seps [ i ++ ] = ",";
+ break;
+ case eProtocolFasp:
+ prefs [ i ] = "fasp";
+ seps [ i ++ ] = ",";
+ break;
+ default:
+ if ( NAME_SERVICE_VERS > ONE_DOT_ONE )
+ {
+ /* 1.2 protocols */
+ switch ( protos & eProtocolMask )
+ {
+ case eProtocolHttps:
+ prefs [ i ] = "https";
+ seps [ i ++ ] = ",";
+ break;
+ }
+ }
}
}
+
+ if ( prefs [ 0 ] == NULL )
+ rc = RC ( rcVFS, rcResolver, rcResolving, rcParam, rcInvalid );
+
+ else
+ {
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS ),
+ (" accept-proto = %s%s%s%s%s\n", prefs [ 0 ], seps [ 1 ], prefs [ 1 ], seps [ 2 ], prefs [ 2 ] ) );
+ rc = KHttpRequestAddPostParam ( req, "accept-proto=%s%s%s%s%s", prefs [ 0 ], seps [ 1 ], prefs [ 1 ], seps [ 2 ], prefs [ 2 ] );
+ }
}
if ( rc == 0 )
@@ -1383,6 +1425,15 @@ rc_t VResolverAlgRemoteProtectedResolve( const VResolverAlg *self,
uint32_t code;
rc = KHttpResultStatus ( rslt, &code, NULL, 0, NULL );
+
+#ifdef VDB_3162
+ if ( TEST_VDB_3162 ) {
+ code = TESTING_VDB_3162_CODE ( rc, code );
+ }
+#endif
+
+ DBGMSG
+ ( DBG_VFS, DBG_FLAG ( DBG_VFS ), ( " Code = %d\n", code ) );
if ( code == 200 )
{
KStream *response;
@@ -1435,13 +1486,22 @@ rc_t VResolverAlgRemoteProtectedResolve( const VResolverAlg *self,
KStreamRelease ( response );
}
- } else if ( code == 404 ) { /* HTTP/1.1 400 Bad Request -
- resolver CGI was not found */
- rc = RC ( rcVFS, rcResolver, rcResolving, rcConnection,
- rcNotFound );
- } else { /* Something completely unexpected */
- rc = RC ( rcVFS, rcResolver, rcResolving, rcConnection,
- rcUnexpected );
+ }
+ else if ( code == 403 ) {
+ /* HTTP/1.1 403 Forbidden
+ - resolver CGI was called over http insted of https */
+ rc = RC ( rcVFS, rcResolver, rcResolving,
+ rcConnection, rcUnauthorized );
+ }
+ else if ( code == 404 )
+ {
+ /* HTTP/1.1 404 Bad Request - resolver CGI was not found */
+ rc = RC ( rcVFS, rcResolver, rcResolving, rcConnection, rcNotFound );
+ }
+ else
+ {
+ /* Something completely unexpected */
+ rc = RC ( rcVFS, rcResolver, rcResolving, rcConnection, rcUnexpected );
}
KHttpResultRelease ( rslt );
}
@@ -1459,6 +1519,87 @@ rc_t VResolverAlgRemoteProtectedResolve( const VResolverAlg *self,
return rc;
}
+/* If resolver-cgi is on government site and is called over http:
+ fix it to https */
+static
+rc_t VResolverAlgFixHTTPSOnlyStandard ( VResolverAlg * self, bool * fixed )
+{
+ rc_t rc = 0;
+
+ const String * root = NULL;
+
+ assert ( self && fixed );
+
+ * fixed = false;
+
+ root = self -> root;
+
+ if ( root != NULL ) {
+ size_t size = 0;
+ String http;
+ CONST_STRING ( & http, "http://" );
+ size = http . size;
+
+ /* resolver-cgi is called over http */
+ if ( root -> size > size &&
+ strcase_cmp ( root -> addr, size, http . addr, size, size ) == 0 )
+ {
+ VPath * path = NULL;
+ rc = VPathMakeFmt ( & path, "%S", root );
+ if ( rc == 0 ) {
+ String host;
+ rc = VPathGetHost ( path, & host );
+ if ( rc == 0 ) {
+ String gov;
+ CONST_STRING ( & gov, ".gov" );
+ size = gov . size;
+
+ /* If resolver-cgi is on government site */
+ if ( host . size > size &&
+ strcase_cmp ( host . addr + host . size - size,
+ size, gov . addr, size, size ) == 0 )
+ {
+ size_t newLen = root -> len + 2;
+ String * tmp = malloc ( sizeof * tmp + newLen );
+ if ( tmp == NULL ) {
+ rc = RC ( rcVFS, rcResolver,
+ rcResolving, rcMemory, rcExhausted );
+ }
+ else
+ {
+ /* capture all of root past "http" */
+ String remainder;
+ StringSubstr ( root, & remainder, 4, 0 );
+
+ /* prepare "tmp" to point to buffer space */
+ tmp -> addr = ( char * ) ( tmp + 1 );
+
+ /* print into buffer */
+ rc = string_printf ( ( char * ) tmp -> addr,
+ newLen, & tmp -> size, "https%S", & remainder );
+ if ( rc != 0 )
+ free ( tmp );
+ else
+ {
+ tmp -> len = root -> len + 1;
+ rc = VectorAppend ( & self -> vols, NULL, tmp );
+ if ( rc == 0 )
+ {
+ self -> root = tmp;
+ * fixed = true;
+ }
+ }
+ }
+ }
+ }
+ }
+ VPathRelease ( path );
+ }
+ }
+
+ return rc;
+}
+
/* RemoteResolve
* resolve an accession into a VPath or not found
*
@@ -1490,8 +1631,29 @@ rc_t VResolverAlgRemoteResolve ( const VResolverAlg *self,
#endif
)
{
- rc = VResolverAlgRemoteProtectedResolve ( self,
- kns, protocols, & tok -> acc, path, mapping, legacy_wgs_refseq );
+ bool done = false;
+ int i = 0;
+ for ( i = 0; i < 2 && ! done; ++i ) {
+ rc = VResolverAlgRemoteProtectedResolve ( self, kns,
+ protocols, & tok -> acc, path, mapping, legacy_wgs_refseq );
+ if ( rc == SILENT_RC (
+ rcVFS, rcResolver, rcResolving, rcConnection, rcUnauthorized ) )
+ { /* resolver-cgi is called over http instead of https:
+ fix it */
+ bool fixed = false;
+ rc_t r2 = VResolverAlgFixHTTPSOnlyStandard
+ ( ( VResolverAlg * ) self, & fixed );
+ if ( ! fixed || r2 != 0 ) {
+ if ( r2 != 0 ) {
+ rc = r2;
+ }
+ done = true;
+ }
+ }
+ else {
+ done = true;
+ }
+ }
if (rc == 0 && path != NULL && *path != NULL &&
opt_file_rtn != NULL && *opt_file_rtn == NULL &&
@@ -1703,7 +1865,7 @@ struct VResolver
uint32_t num_app_vols [ appCount ];
/* preferred protocols preferences. Default: HTTP */
- VRemoteProtocols protocols;
+ VRemoteProtocols protocols, dflt_protocols;
/** projectId of protected user repository;
0 when repository is not user protected */
@@ -2250,7 +2412,7 @@ LIB_EXPORT
rc_t CC VResolverLocal ( const VResolver * self,
const VPath * accession, const VPath ** path )
{
- rc_t rc = VResolverQuery ( self, eProtocolHttp, accession, path, NULL, NULL );
+ rc_t rc = VResolverQuery ( self, self -> protocols, accession, path, NULL, NULL );
if ( rc == 0 )
{
switch ( accession -> path_type )
@@ -2782,7 +2944,7 @@ LIB_EXPORT
rc_t CC VResolverCache ( const VResolver * self,
const VPath * url, const VPath ** path, uint64_t file_size )
{
- return VResolverQuery ( self, eProtocolHttp, url, NULL, NULL, path );
+ return VResolverQuery ( self, self -> protocols, url, NULL, NULL, path );
}
/* QueryOID
@@ -3238,6 +3400,9 @@ rc_t VResolverQueryInt ( const VResolver * self, VRemoteProtocols protocols,
rc = RC ( rcVFS, rcResolver, rcResolving, rcParam, rcNull );
else
{
+ if ( protocols == 0 )
+ protocols = self -> protocols;
+
if ( local != NULL )
{
* local = NULL;
@@ -3247,10 +3412,19 @@ rc_t VResolverQueryInt ( const VResolver * self, VRemoteProtocols protocols,
if ( remote != NULL )
{
+ VRemoteProtocols remote_protos;
+
* remote = NULL;
- if ( protocols >= eProtocolLastDefined )
- return RC ( rcVFS, rcResolver, rcResolving, rcParam, rcInvalid );
+ if ( protocols > VRemoteProtocolsMake3 ( eProtocolMax, eProtocolMax, eProtocolMax ) )
+ return RC ( rcVFS, rcResolver, rcUpdating, rcParam, rcInvalid );
+
+ for ( remote_protos = protocols; remote_protos != 0; remote_protos >>= 3 )
+ {
+ VRemoteProtocols proto = remote_protos & eProtocolMask;
+ if ( proto == eProtocolNone || proto > eProtocolMax )
+ return RC ( rcVFS, rcResolver, rcUpdating, rcParam, rcInvalid );
+ }
if ( atomic32_read ( & enable_remote ) == vrAlwaysDisable )
remote = NULL;
@@ -3271,6 +3445,15 @@ rc_t VResolverQueryInt ( const VResolver * self, VRemoteProtocols protocols,
rc = RC ( rcVFS, rcResolver, rcResolving, rcPath, rcNotFound );
else
{
+ uint32_t i;
+
+ /* record requested protocols */
+ bool has_proto [ eProtocolMask + 1 ];
+ memset ( has_proto, 0, sizeof has_proto );
+
+ for ( i = 0; i < eProtocolMaxPref; ++ i )
+ has_proto [ ( ( protocols >> ( i * 3 ) ) & eProtocolMask ) ] = true;
+
switch ( query -> scheme_type )
{
case vpuri_none:
@@ -3281,23 +3464,21 @@ rc_t VResolverQueryInt ( const VResolver * self, VRemoteProtocols protocols,
break;
case vpuri_http:
- switch ( protocols )
- {
- case eProtocolHttp:
- case eProtocolFaspHttp:
- case eProtocolHttpFasp:
+ /* check for all that allow http */
+ if ( has_proto [ eProtocolHttp ] )
+ return VResolverQueryURL ( self, protocols, query, remote, cache );
+ return RC ( rcVFS, rcResolver, rcResolving, rcPath, rcIncorrect );
+
+ case vpuri_https:
+ /* check for all that allow https */
+ if ( has_proto [ eProtocolHttps ] )
return VResolverQueryURL ( self, protocols, query, remote, cache );
- }
return RC ( rcVFS, rcResolver, rcResolving, rcPath, rcIncorrect );
case vpuri_fasp:
- switch ( protocols )
- {
- case eProtocolFasp:
- case eProtocolFaspHttp:
- case eProtocolHttpFasp:
+ /* check for all that allow fasp */
+ if ( has_proto [ eProtocolFasp ] )
return VResolverQueryURL ( self, protocols, query, remote, cache );
- }
return RC ( rcVFS, rcResolver, rcResolving, rcPath, rcIncorrect );
default:
@@ -4074,7 +4255,7 @@ rc_t VResolverForceRemoteRefseq ( VResolver *self )
return 0;
/* create one from hard-coded constants */
- StringInitCString ( & local_root, "http://ftp-trace.ncbi.nlm.nih.gov/sra" );
+ StringInitCString ( & local_root, "https://ftp-trace.ncbi.nlm.nih.gov/sra" );
rc = StringCopy ( & root, & local_root );
if ( rc == 0 )
{
@@ -4138,7 +4319,7 @@ rc_t VResolverForceRemoteProtected ( VResolver *self )
/* create one from hard-coded constants */
String cgi_root;
- StringInitCString ( & cgi_root, "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" );
+ StringInitCString ( & cgi_root, "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" );
rc = StringCopy ( & root, & cgi_root );
if ( rc == 0 )
{
@@ -4494,28 +4675,40 @@ static rc_t VResolverLoad(VResolver *self, const KRepository *protected,
VectorReorder ( & self -> remote, VResolverAlgSort, NULL );
}
- self -> protocols = eProtocolHttp;
-
return rc;
}
LIB_EXPORT
-rc_t CC VResolverProtocols ( VResolver * self,
- VRemoteProtocols protocols )
+rc_t CC VResolverProtocols ( VResolver * self, VRemoteProtocols protocols )
{
+ VRemoteProtocols remote_protos;
+
if ( self == NULL )
return RC ( rcVFS, rcResolver, rcUpdating, rcSelf, rcNull );
- if ( protocols >= eProtocolLastDefined )
- return RC ( rcVFS, rcResolver, rcUpdating, rcParam, rcInvalid );
+ if ( protocols == 0)
+ self -> protocols = self -> dflt_protocols;
+ else
+ {
+ if ( protocols > VRemoteProtocolsMake3 ( eProtocolMax, eProtocolMax, eProtocolMax ) )
+ return RC ( rcVFS, rcResolver, rcUpdating, rcParam, rcInvalid );
- self -> protocols = protocols;
+ for ( remote_protos = protocols; remote_protos != 0; remote_protos >>= 3 )
+ {
+ VRemoteProtocols proto = remote_protos & eProtocolMask;
+ if ( proto == eProtocolNone || proto > eProtocolMax )
+ return RC ( rcVFS, rcResolver, rcUpdating, rcParam, rcInvalid );
+ }
+
+ self -> protocols = protocols;
+ }
return 0;
}
-rc_t VResolverGetProjectId ( const VResolver * self, uint32_t * projectId ) {
+rc_t VResolverGetProjectId ( const VResolver * self, uint32_t * projectId )
+{
if ( self == NULL )
return RC ( rcVFS, rcResolver, rcAccessing, rcSelf, rcNull );
else if ( projectId == NULL )
@@ -4556,17 +4749,27 @@ rc_t VResolverMake ( VResolver ** objp, const KDirectory *wd,
KRefcountInit ( & obj -> refcount, 1, "VResolver", "make", "resolver" );
- if (mgr != NULL) {
- rc_t rc = VFSManagerGetKNSMgr(mgr, &kns);
- if (rc != 0) {
+ if ( mgr != NULL )
+ {
+ rc_t rc = VFSManagerGetKNSMgr ( mgr, & kns );
+ if ( rc != 0 )
+ {
rc = 0;
kns = NULL;
}
}
+
+ /* set up protocols */
+ obj -> dflt_protocols = eProtocolHttpHttps;
+ if ( kfg != NULL )
+ KConfigReadRemoteProtocols ( kfg, & obj -> dflt_protocols );
+
+ obj -> protocols = obj -> dflt_protocols;
+
rc = VResolverLoad ( obj, protected, kfg, kns );
- KNSManagerRelease(kns);
+ KNSManagerRelease ( kns );
kns = NULL;
KRepositoryProjectId ( protected, & obj -> projectId );
diff --git a/libs/vxf/entrez.c b/libs/vxf/entrez.c
index 28fa030..57811c2 100644
--- a/libs/vxf/entrez.c
+++ b/libs/vxf/entrez.c
@@ -94,7 +94,7 @@ static uint32_t vrowdata_get32( const VRowData * src, uint32_t * dst )
return RC ( rcXF, rcFunction, rcReading, rcType, rcIncorrect );
}
-const char * entrez_server = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
+const char * entrez_server = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
static rc_t CC perform_entrez_fetch ( void *self,
const VXformInfo *info, int64_t row_id, VRowResult *rslt,
diff --git a/libs/vxf/fzip.c b/libs/vxf/fzip.c
index 784bec2..8cb36a3 100644
--- a/libs/vxf/fzip.c
+++ b/libs/vxf/fzip.c
@@ -81,7 +81,7 @@ static rc_t invoke_zlib(void *dst, uint64_t *dsize, const void *src, uint32_t ss
break;
}
zr = deflateEnd(&s);
- if (zr != Z_OK)
+ if ( zr != Z_OK && s.total_out != 0 )
rc = RC(rcXF, rcFunction, rcExecuting, rcSelf, rcUnexpected);
if (rc == 0) {
assert(s.total_out <= UINT32_MAX);
diff --git a/libs/vxf/irzip.c b/libs/vxf/irzip.c
index 2ef4995..d9b99ee 100644
--- a/libs/vxf/irzip.c
+++ b/libs/vxf/irzip.c
@@ -89,9 +89,9 @@ static rc_t zlib_compress(szbuf *dst, const void *src, size_t ssize, int32_t str
break;
}
zr = deflateEnd(&s);
- if (zr != Z_OK)
+ if ( zr != Z_OK && s.total_out != 0 )
rc = RC(rcVDB, rcFunction, rcExecuting, rcSelf, rcUnexpected);
- if (rc == 0) {
+ if ( rc == 0 ) {
dst->used = (uint32_t)s.total_out;
}
return rc;
diff --git a/libs/vxf/izip.c b/libs/vxf/izip.c
index 33e1306..566ba28 100644
--- a/libs/vxf/izip.c
+++ b/libs/vxf/izip.c
@@ -217,7 +217,7 @@ static rc_t zlib_compress(szbuf *dst, const void *src, uint32_t ssize, int32_t s
break;
}
zr = deflateEnd(&s);
- if (zr != Z_OK)
+ if ( zr != Z_OK && s.total_out != 0 )
rc = RC(rcVDB, rcFunction, rcExecuting, rcSelf, rcUnexpected);
if (rc == 0) {
assert(s.total_out <= UINT32_MAX);
diff --git a/libs/xfs/path.c b/libs/xfs/path.c
index 446b763..cb5116b 100644
--- a/libs/xfs/path.c
+++ b/libs/xfs/path.c
@@ -327,7 +327,7 @@ _StrCAdd (
XFS_CAN ( Str )
XFS_CA ( StrLen, 0 )
- TheStr = malloc ( sizeof ( char ) * ( StrLen + 1 ) );
+ TheStr = calloc ( StrLen + 1, sizeof ( char ) );
if ( TheStr == NULL ) {
return XFS_RC ( rcExhausted );
}
@@ -507,7 +507,7 @@ _PathCompile ( struct XFSPath * self, const char ** Str )
}
}
- Path = malloc ( sizeof ( char ) * ( StrSize + 1 ) );
+ Path = calloc ( StrSize + 1, sizeof ( char ) );
if ( Path == NULL ) {
return XFS_RC ( rcExhausted );
}
@@ -580,12 +580,13 @@ _PathParse (
)
{
rc_t RCt;
- const char * Bg, * Cr;
+ const char * Bg, * Cr, * En;
bool Abs;
RCt = 0;
Bg = NULL;
Cr = NULL;
+ En = NULL;
Abs = false;
XFS_CAN ( self )
@@ -602,7 +603,8 @@ _PathParse (
if ( RCt == 0 ) {
Bg = Path;
Cr = Path;
- while ( * Cr != 0 ) {
+ En = Bg + string_size ( Path );
+ while ( Cr < En ) {
if ( * Cr == '/' ) {
if ( 0 < Cr - Bg ) {
RCt = _StrCAdd ( self -> tokens, Bg, Cr - Bg );
@@ -611,7 +613,7 @@ _PathParse (
}
}
- while ( * Cr != 0 ) {
+ while ( Cr < En ) {
if ( * Cr != '/' ) {
Bg = Cr;
break;
@@ -1079,7 +1081,7 @@ XFSPathAppendPath (
);
if ( RCt == 0 ) {
NewLen = strlen ( TheSelf -> orig ) + strlen ( Path -> orig );
- NewOrig = malloc ( sizeof ( char ) * ( 1 + NewLen ) );
+ NewOrig = calloc ( 1 + NewLen, sizeof ( char ) );
if ( NewOrig == NULL ) {
RCt = XFS_RC ( rcExhausted );
}
diff --git a/libs/xfs/unix/operations.c b/libs/xfs/unix/operations.c
index 9391300..daadbc9 100644
--- a/libs/xfs/unix/operations.c
+++ b/libs/xfs/unix/operations.c
@@ -658,10 +658,10 @@ XFS_FUSE_getattr ( const char * ThePath, struct stat * TheStat )
Node = NULL;
Type = kxfsNotFound;
- pLogMsg ( klogDebug, "POLL(!): [$(path)]", "path=%s", ThePath );
pLogMsg ( klogDebug, "GETATTR(Fuse): [$(path)]", "path=%s", ThePath );
if ( ThePath == NULL || TheStat == NULL ) {
+pLogMsg ( klogDebug, "GETATTR(Fuse,cont): [$(path)] [INVALID]", "path=%s", ThePath );
return EINVAL * - 1;
}
memset ( TheStat, 0, sizeof ( struct stat ) );
@@ -671,6 +671,7 @@ XFS_FUSE_getattr ( const char * ThePath, struct stat * TheStat )
if ( Type == kxfsNotFound ) {
XFSNodeRelease ( Node );
+pLogMsg ( klogDebug, "GETATTR(Fuse,cont): [$(path)] [NotFound]", "path=%s", ThePath );
return ENOENT * - 1;
}
@@ -679,6 +680,8 @@ XFS_FUSE_getattr ( const char * ThePath, struct stat * TheStat )
XFSNodeRelease ( Node );
}
+ pLogMsg ( klogDebug, "GETATTR(Fuse,cont): [$(path)] [$(rc)]", "path=%s,rc=%d", ThePath, RCt );
+
return XFS_FUSE_rc_to_errno ( RCt ) * - 1;
} /* XFS_FUSE_getattr() */
@@ -1709,6 +1712,8 @@ XFS_FUSE_access ( const char * ThePath, int Access )
XFSNodeRelease ( Node );
}
+ pLogMsg ( klogDebug, "ACCESS(Fuse,cont): [$(path)] RC[$(rc)]", "path=%s,rc=%d", ThePath, RCt );
+
return RCt == 0 ? 0 : - 1;
} /* XFS_FUSE_access() */
@@ -1970,7 +1975,7 @@ XFS_FUSE_poll (
unsigned * TheReventsp
)
{
- pLogMsg ( klogDebug, "POLL(!): [$(path)]", "path=%s", ThePath );
+ pLogMsg ( klogDebug, "POLL(*): [$(path)]", "path=%s", ThePath );
return -EPERM;
} /* XFS_FUSE_poll() */
diff --git a/libs/xfs/xgap.c b/libs/xfs/xgap.c
index 31d9b7d..8ccbc44 100644
--- a/libs/xfs/xgap.c
+++ b/libs/xfs/xgap.c
@@ -825,7 +825,7 @@ XFSGapProjectLocateAccession (
if ( RCt == 0 ) {
RCt = VResolverQuery (
self -> resolver,
- eProtocolHttp,
+ 0,
Query,
NULL,
Remote,
@@ -874,7 +874,7 @@ XFSGapProjectLocateObject (
if ( RCt == 0 ) {
RCt = VResolverQuery (
self -> resolver,
- eProtocolHttp,
+ 0,
Query,
NULL,
Remote,
diff --git a/libs/xfs/xhttp.c b/libs/xfs/xhttp.c
index 5760430..e3e2298 100644
--- a/libs/xfs/xhttp.c
+++ b/libs/xfs/xhttp.c
@@ -56,25 +56,15 @@
|| Lyrics:
|| That file contains interface to SRA archive at
||
- || http://ftp-trace.ncbi.nlm.nih.gov/sra
- ||
- || That archive contains folders and files. For example, that
- || directory contains 'DRR000001' subdirectory :
- ||
- || (http://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/reads/ByRun/sra/DRR/DRR000)
- ||
- || DRR000001/ 26-May-2014 07:22 -
- || DRR000002/ 26-May-2014 08:00 -
- || DRR000003/ 26-May-2014 08:07 -
- || DRR000004/ 26-May-2014 08:07 -
+ || https://sra-download.ncbi.nlm.nih.gov/sra
||
|| If we will enter 'DRR000001' subdirectory, we will file here :
||
- || (http://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/reads/ByRun/sra/DRR/DRR000/DRR000001/)
+ || (https://sra-download.ncbi.nlm.nih.gov/srapub/DRR000001/)
||
|| DRR000001.sra 26-May-2014 07:22 569M
||
- || So, code could looks little weird, parsing and servint that format
+ || So, code could looks little weird, parsing and serving that format
||
\*||*/
diff --git a/test/vxf/Makefile b/test/align/Makefile
similarity index 82%
copy from test/vxf/Makefile
copy to test/align/Makefile
index 064aab3..c245e5e 100644
--- a/test/vxf/Makefile
+++ b/test/align/Makefile
@@ -22,41 +22,35 @@
#
# ===========================================================================
+
default: runtests
TOP ?= $(abspath ../..)
-
-MODULE = test/vxf
+MODULE = test/align
TEST_TOOLS = \
- wb-test-vxf
+ test-load-index \
include $(TOP)/build/Makefile.env
$(TEST_TOOLS): makedirs
@ $(MAKE_CMD) $(TEST_BINDIR)/$@
-.PHONY: $(TEST_TOOLS)
-
clean: stdclean
-
#-------------------------------------------------------------------------------
-# white-box test
+# test-load-index
#
-TEST_SRC = \
- wb-test-vxf \
- wb-irzip-impl
-
-TEST_OBJ = \
- $(addsuffix .$(OBJX),$(TEST_SRC))
-
-TEST_LIBS = \
- -skapp \
- -sktst \
- -sncbi-vdb \
- -sxml2 \
- -sm
-
-$(TEST_BINDIR)/wb-test-vxf: $(TEST_OBJ)
- $(LP) --exe -o $@ $^ $(TEST_LIBS)
+TEST_INDEX_SRC = \
+ indextest
+
+TEST_INDEX_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_INDEX_SRC))
+
+TEST_INDEX_LIB = \
+ -skapp \
+ -sktst \
+ -sncbi-vdb
+
+$(TEST_BINDIR)/test-load-index: $(TEST_INDEX_OBJ)
+ $(LP) --exe -o $@ $^ $(TEST_INDEX_LIB)
diff --git a/test/align/indextest.cpp b/test/align/indextest.cpp
new file mode 100644
index 0000000..01e3b67
--- /dev/null
+++ b/test/align/indextest.cpp
@@ -0,0 +1,113 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+/**
+* Unit tests for the Loader module
+*/
+#include <ktst/unit_test.hpp>
+
+#include <klib/printf.h>
+
+#include <stdexcept>
+#include <string>
+
+extern "C" {
+#include <klib/rc.h>
+#include <align/bam.h>
+}
+
+using namespace std;
+
+TEST_SUITE(IndexTestSuite);
+
+class LoaderFixture
+{
+ BAMFile const *bam;
+
+ static std::string BAM_FILE_NAME(void) {
+ return std::string("/panfs/pan1/sra-test/bam/VDB-3148.bam");
+ }
+ static std::string INDEX_FILE_NAME(void) {
+ return BAM_FILE_NAME() + ".bai";
+ }
+public:
+ LoaderFixture()
+ {
+ rc_t const rc = BAMFileMake(&bam, BAM_FILE_NAME().c_str());
+ if (rc != 0)
+ throw std::runtime_error("can't open " + BAM_FILE_NAME());
+ }
+ ~LoaderFixture()
+ {
+ BAMFileRelease(bam);
+ }
+ void testIndex(void) const {
+ rc_t const expected_rc = SILENT_RC(rcAlign, rcIndex, rcReading, rcData, rcExcessive);
+ rc_t const rc = BAMFileOpenIndex(bam, INDEX_FILE_NAME().c_str());
+ if (rc == 0)
+ throw std::runtime_error("Index open was supposed to fail");
+ if (rc != expected_rc)
+ throw std::runtime_error("Index open did not fail with the expected result code; perhaps the test file is missing?");
+ }
+};
+
+FIXTURE_TEST_CASE ( LoadIndex, LoaderFixture )
+{
+ testIndex();
+}
+
+//////////////////////////////////////////// Main
+#include <kapp/args.h>
+#include <klib/out.h>
+#include <kfg/config.h>
+
+extern "C"
+{
+
+ver_t CC KAppVersion ( void )
+{
+ return 0x1000000;
+}
+
+const char UsageDefaultName[] = "test-loader";
+
+rc_t CC UsageSummary (const char * progname)
+{
+ return KOutMsg ( "Usage:\n" "\t%s [options]\n\n", progname );
+}
+
+rc_t CC Usage( const Args* args )
+{
+ return 0;
+}
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ return IndexTestSuite(argc, argv);
+}
+
+}
+
diff --git a/test/cc/asm-test.cpp b/test/cc/asm-test.cpp
index 246ba05..7cba437 100644
--- a/test/cc/asm-test.cpp
+++ b/test/cc/asm-test.cpp
@@ -33,6 +33,7 @@
#include <ktst/unit_test.hpp>
+#include <atomic.h>
#include <atomic32.h>
#include <arch-impl.h>
@@ -430,6 +431,20 @@ TEST_CASE(a64_read_and_add_even_false)
REQUIRE_EQ ( ( int64_t ) atomic64_read ( & v ), ( int64_t ) 1 );
}
+TEST_CASE(a64_test_and_set_ptr_success)
+{
+ atomic_ptr_t v = { (void*)0 };
+ REQUIRE_EQ ( (void*)0, (void*) atomic_test_and_set_ptr ( & v, (void*)1, (void*)0 ) ); // v.ptr == 0; replace and return the old value
+ REQUIRE_EQ ( (void*)1, (void*) v . ptr );
+}
+
+TEST_CASE(a64_test_and_set_ptr_failure)
+{
+ atomic_ptr_t v = { (void*)0 };
+ REQUIRE_EQ ( (void*)0, (void*) atomic_test_and_set_ptr ( & v, (void*)1, (void*)2 ) ); // v.ptr != 2, not changed
+ REQUIRE_EQ ( (void*)0, (void*) v . ptr ); // did not change
+}
+
#endif
//////////////////////////////////////////// Main
diff --git a/test/kdb/Makefile b/test/kdb/Makefile
index 5337dde..c4ebcd4 100644
--- a/test/kdb/Makefile
+++ b/test/kdb/Makefile
@@ -31,7 +31,8 @@ MODULE = test/kdb
TEST_TOOLS = \
test-kdb \
- test-wkdb
+ test-wkdb \
+ test-rowset
include $(TOP)/build/Makefile.env
@@ -64,6 +65,26 @@ TEST_BTREE_LIB = \
$(BINDIR)/test-btree: $(TEST_BTREE_OBJ)
$(LD) --exe -o $@ $^ $(TEST_BTREE_LIB)
+
+#-------------------------------------------------------------------------------
+# test rowset
+#
+TEST_ROWSET_SRC = \
+ rowsettest
+
+TEST_ROWSET_OBJ = \
+ $(addsuffix .$(OBJX),$(TEST_ROWSET_SRC))
+
+TEST_ROWSET_LIB = \
+ -skapp \
+ -sktst \
+ -sncbi-wvdb
+
+$(TEST_BINDIR)/test-rowset: $(TEST_ROWSET_OBJ)
+ $(LP) --exe -o $@ $^ $(TEST_ROWSET_LIB)
+
+vg_rowset: std
+ $(VALGRIND) $(TEST_BINDIR)/test-rowset
#-------------------------------------------------------------------------------
# test-kdb
diff --git a/test/kdb/remote_open_test.cpp b/test/kdb/remote_open_test.cpp
index e947a67..31f8043 100644
--- a/test/kdb/remote_open_test.cpp
+++ b/test/kdb/remote_open_test.cpp
@@ -155,7 +155,7 @@ public:
{
m_configName = name+".kfg";
std::ofstream f(m_configName.c_str());
- f << "repository/remote/main/CGI/resolver-cgi = \"http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi\"\n"
+ f << "repository/remote/main/CGI/resolver-cgi = \"https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi\"\n"
<< "repository/user/main/public/root=\"./root\"\n"
<< "repository/user/main/public/apps/sra/volumes/sraFlat=\"sra\"\n";
f.close();
@@ -188,7 +188,7 @@ public:
if (VFSManagerMakePath ( m_vfsmgr, &accession, p_accession.c_str() ))
throw logic_error ( "RemoteDBFixture::Resolve: VFSManagerMakePath failed" );
- if (VResolverQuery( resolver, eProtocolHttp, accession, NULL, &m_path, &m_cache))
+ if (VResolverQuery( resolver, 0, accession, NULL, &m_path, &m_cache))
throw logic_error ( "RemoteDBFixture::Resolve: VResolverQuery failed" );
//cout << ToString(m_path) << endl;
diff --git a/test/kdb/rowsettest.cpp b/test/kdb/rowsettest.cpp
new file mode 100644
index 0000000..788686f
--- /dev/null
+++ b/test/kdb/rowsettest.cpp
@@ -0,0 +1,819 @@
+/*===========================================================================
+ *
+ * PUBLIC DOMAIN NOTICE
+ * National Center for Biotechnology Information
+ *
+ * This software/database is a "United States Government Work" under the
+ * terms of the United States Copyright Act. It was written as part of
+ * the author's official duties as a United States Government employee and
+ * thus cannot be copyrighted. This software/database is freely available
+ * to the public for use. The National Library of Medicine and the U.S.
+ * Government have not placed any restriction on its use or reproduction.
+ *
+ * Although all reasonable efforts have been taken to ensure the accuracy
+ * and reliability of the software and data, the NLM and the U.S.
+ * Government do not and cannot warrant the performance or results that
+ * may be obtained by using this software or data. The NLM and the U.S.
+ * Government disclaim all warranties, express or implied, including
+ * warranties of performance, merchantability or fitness for any particular
+ * purpose.
+ *
+ * Please cite the author in any work or product based on this material.
+ *
+ * ===========================================================================
+ *
+ */
+
+// allow g++ to find INT64_MAX in stdint.h
+#define __STDC_LIMIT_MACROS
+
+#include <ktst/unit_test.hpp>
+
+#include <klib/log.h>
+#include <klib/out.h>
+#include <klib/rc.h>
+#include <kfc/ctx.h>
+#include <kfc/except.h>
+#include <kfc/xc.h>
+#include <kfs/directory.h>
+#include <kdb/manager.h>
+#include <kdb/table.h>
+#include <kdb/rowset.h>
+
+#include <stdlib.h>
+#include <time.h>
+
+#include <vector>
+#include <set>
+#include <algorithm>
+#include <string>
+
+#define ENTRY \
+ HYBRID_FUNC_ENTRY ( rcDB, rcVector, rcValidating ); \
+ m_ctx = ctx
+
+#define EXIT \
+ REQUIRE ( ! FAILED () ); \
+ Release()
+
+////// additional REQUIRE macros
+
+#define REQUIRE_FAILED() ( REQUIRE ( FAILED () ), CLEAR() )
+
+#define REQUIRE_EXPR(expr) \
+ (expr); \
+ REQUIRE ( !FAILED () )
+
+#define REQUIRE_EXPR_FAILED(expr) \
+ (expr); \
+ REQUIRE_FAILED()
+
+#define THROW_IF_FAILED(msg) \
+ if ( m_ctx -> rc != 0 ) \
+ throw std :: logic_error ( (msg) );
+
+TEST_SUITE(KRowSetTestSuite);
+
+void vector_inserter ( ctx_t ctx, int64_t row_id, void *data )
+{
+ std::vector<int64_t> * rows = (std::vector<int64_t> *) data;
+ rows->push_back ( row_id );
+}
+
+class RowSetFixture
+{
+public:
+ const KCtx* m_ctx; // points into the test case's local memory
+ KDirectory* m_wd;
+ KDBManager* m_mgr;
+ KTable * m_table;
+ std::string m_tableName;
+
+ RowSetFixture()
+ : m_ctx(NULL)
+ , m_table(NULL)
+ , m_tableName("")
+ {
+ THROW_ON_RC ( KDirectoryNativeDir ( & m_wd ) );
+ THROW_ON_RC ( KDBManagerMakeUpdate ( & m_mgr, m_wd ) );
+ }
+
+ virtual ~RowSetFixture()
+ {
+ if ( m_table != NULL && m_tableName != "" )
+ CloseRemoveTable();
+
+ KDBManagerRelease ( m_mgr );
+ KDirectoryRelease ( m_wd );
+ }
+
+ const KTable * GetTable ( const char * name )
+ {
+ if ( m_tableName . empty() )
+ m_tableName = name;
+
+ if ( m_tableName != name )
+ throw std :: invalid_argument ( "name does not match a name of already create table" );
+
+ if ( m_table == NULL )
+ CreateTable ();
+
+ return m_table;
+ }
+
+ void CloseRemoveTable ()
+ {
+ KTableRelease ( m_table );
+ m_table = NULL;
+
+ KDirectoryRemove ( m_wd, true, m_tableName . c_str () );
+ m_tableName = "";
+ }
+
+ int64_t GenerateId ( int64_t range_start, int64_t range_count )
+ {
+ if ( range_start == -1 )
+ range_start = 0;
+
+ if ( range_count == -1 || range_start + range_count < 0 )
+ range_count = INT64_MAX - range_start;
+
+ int64_t generated_id = ((int64_t)rand() << 32) | rand();
+ generated_id &= INT64_MAX; // make sure it is positive
+
+ generated_id = generated_id % range_count;
+ generated_id += range_start;
+
+ return generated_id;
+ }
+
+ /*
+ * "density" - can is a number between -1 and 8.
+ * -1 means that most probably each row will go to a separate leaf
+ * 0 means that rows might go to same leaves, but those leaves should still not be dense (and store data in array of ranges, not in bitmap)
+ * from 1 to 8 change density level:
+ * - 1 means that every (up to) 8k rows go to a each leaf
+ * - 8 means that every (up to) 65k rows (maximum rows per leaf) will go to each leaf
+ */
+ std::set<int64_t> InsertRandomRows ( KRowSet * rowset, int num_rows, int density )
+ {
+ int64_t range_start = 0;
+ int64_t range_count;
+ std::set<int64_t> inserted_rows_set;
+
+ assert ( num_rows > 0 );
+ assert ( density >= -1 );
+ assert ( density <= 8 );
+
+ if ( density == -1 )
+ range_count = -1;
+ else if ( density == 0 )
+ range_count = ((num_rows - 1) / 5 + 1) * 65536;
+ else
+ {
+ assert ( num_rows > 8 );
+ range_count = ((num_rows - 1) / (8192 * density) + 1) * 65536;
+ }
+
+ for ( int i = 0; i < num_rows; ++i )
+ {
+ int64_t row_id = GenerateId ( range_start, range_count );
+ if ( inserted_rows_set.find( row_id ) == inserted_rows_set.end() )
+ {
+ KRowSetAddRowId ( rowset, m_ctx, row_id );
+ THROW_IF_FAILED ( "Failed to insert a row" );
+ inserted_rows_set.insert( row_id );
+ }
+ else
+ --i;
+ }
+
+ return inserted_rows_set;
+ }
+
+ void RunChecks ( const KRowSet * rowset, std::set<int64_t> & inserted_rows_set )
+ {
+ RunChecksInt ( rowset, inserted_rows_set );
+ }
+
+ std::set<int64_t> SetIntersection ( const std::set<int64_t>& set1, const std::set<int64_t>& set2 )
+ {
+ std::set<int64_t> result;
+ std::set_intersection(
+ set1.begin(), set1.end(),
+ set2.begin(), set2.end(),
+ std::inserter(result, result.begin())
+ );
+
+ return result;
+ }
+
+ std::set<int64_t> SetUnion ( const std::set<int64_t>& set1, const std::set<int64_t>& set2 )
+ {
+ std::set<int64_t> result;
+ std::set_union(
+ set1.begin(), set1.end(),
+ set2.begin(), set2.end(),
+ std::inserter(result, result.begin())
+ );
+
+ return result;
+ }
+
+ virtual void Release()
+ {
+ if (m_ctx != 0)
+ {
+
+ m_ctx = 0; // a pointer into the caller's local memory
+ }
+ }
+private:
+ void RunChecksInt ( const KRowSet * rowset, std::set<int64_t> & inserted_rows_set )
+ {
+ std::vector<int64_t> inserted_rows;
+ std::vector<int64_t> returned_rows;
+ uint64_t num_rows;
+
+ KRowSetVisit ( rowset, m_ctx, vector_inserter, (void *)&returned_rows );
+ THROW_IF_FAILED ( "Failed to iterate over rowset" );
+
+ for (std::set<int64_t>::iterator it = inserted_rows_set.begin(); it != inserted_rows_set.end(); ++it)
+ {
+ bool has_row = KRowSetHasRowId ( rowset, m_ctx, *it );
+ THROW_IF_FAILED ( "Failed to check a row in rowset" );
+ if ( ! has_row )
+ FAIL("Row is not found in rowset");
+ inserted_rows . push_back ( *it );
+ }
+
+ num_rows = KRowSetGetNumRowIds( rowset, m_ctx );
+ THROW_IF_FAILED ( "Failed to get number of row ids in rowset" );
+/*
+ // useful in debugging
+ KOutMsg("Inserted rows: %lu, returned rows: %lu\n", inserted_rows.size(), returned_rows.size() );
+ KOutMsg("inserted:\n");
+ for ( int i = 0; i < inserted_rows.size(); ++i )
+ {
+ KOutMsg ( "%d, ", inserted_rows[i] );
+ }
+ KOutMsg("\n");
+ KOutMsg("returned:\n");
+ for ( int i = 0; i < returned_rows.size(); ++i )
+ {
+ KOutMsg ( "%d, ", returned_rows[i] );
+ }
+ KOutMsg("\n");
+*/
+
+ if ( inserted_rows.size() != returned_rows.size() )
+ FAIL("inserted_rows.size() != returned_rows.size()");
+ if ( num_rows != returned_rows.size() )
+ FAIL("num_rows != returned_rows.size()");
+ if ( inserted_rows != returned_rows )
+ FAIL("inserted_rows != returned_rows");
+ }
+
+ void CreateTable ()
+ {
+ THROW_ON_RC ( KDBManagerCreateTable ( m_mgr, & m_table, kcmInit + kcmMD5, m_tableName . c_str() ) );
+ }
+};
+
+
+FIXTURE_TEST_CASE ( KRowSetScatteredRows, RowSetFixture )
+{
+ ENTRY;
+ KRowSet * rowset;
+ std::set<int64_t> inserted_rows_set;
+ bool has_row;
+
+ REQUIRE_EXPR ( rowset = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ REQUIRE_EXPR ( has_row = KRowSetHasRowId ( rowset, ctx, 1 ) );
+ REQUIRE ( ! has_row );
+
+ inserted_rows_set = InsertRandomRows ( rowset, 10000, -1 );
+ RunChecks ( rowset, inserted_rows_set );
+
+ REQUIRE_EXPR ( KRowSetRelease( rowset, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( KRowSetDenseRows, RowSetFixture )
+{
+ ENTRY;
+ KRowSet * rowset;
+ std::set<int64_t> inserted_rows_set;
+
+ REQUIRE_EXPR ( rowset = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ inserted_rows_set = InsertRandomRows ( rowset, 10000, 1 );
+ RunChecks ( rowset, inserted_rows_set );
+
+ REQUIRE_EXPR ( KRowSetRelease( rowset, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( KRowSetSerialRows, RowSetFixture )
+{
+ ENTRY;
+ KRowSet * rowset;
+ std::set<int64_t> inserted_rows_set;
+
+ REQUIRE_EXPR ( rowset = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ for ( int i = 0; i < 20000; ++i )
+ {
+ int64_t row_id = 50000 + i; // row ids will only go to first two leaves
+ REQUIRE_EXPR ( KRowSetAddRowId ( rowset, ctx, row_id) );
+ inserted_rows_set.insert( row_id );
+ }
+ RunChecks ( rowset, inserted_rows_set );
+
+ REQUIRE_EXPR ( KRowSetRelease( rowset, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( KRowSetRowRanges, RowSetFixture )
+{
+ ENTRY;
+ KRowSet * rowset;
+ std::set<int64_t> inserted_rows_set;
+
+ int64_t row_ids[] = { 0, 5, 1, 6, 20, 10, 55, 60, 65, 70, 75, 80, 85, 999, 2001 };
+ uint64_t counts[] = { 1, 1, 4, 4, 10, 10, 1, 1, 1, 1, 1, 1, 1, 1000, 1000 };
+
+ REQUIRE_EXPR ( rowset = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ for ( size_t i = 0; i < sizeof row_ids / sizeof row_ids[0]; ++i )
+ {
+ int64_t row_id = row_ids[i];
+ uint64_t count = counts[i];
+
+ REQUIRE_EXPR ( KRowSetAddRowIdRange ( rowset, ctx, row_id, count ) );
+ for ( uint64_t j = 0; j < count; ++j )
+ inserted_rows_set.insert( row_id + j );
+ }
+ RunChecks ( rowset, inserted_rows_set );
+
+ REQUIRE_EXPR ( KRowSetRelease( rowset, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( KRowSetRowRangesOverlapDuplicates, RowSetFixture )
+{
+ ENTRY;
+ KRowSet * rowset;
+ std::set<int64_t> inserted_rows_set;
+
+ int64_t row_ids[] = { 5, 10 };
+ uint64_t counts[] = { 1, 5 };
+
+ int64_t overlap_row_ids[] = { 0, 5, 5, 2, 9, 9, 14 };
+ uint64_t overlap_counts[] = { 6, 1, 2, 6, 2, 10, 2 };
+
+ REQUIRE_EXPR ( rowset = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ for ( size_t i = 0; i < sizeof row_ids / sizeof row_ids[0]; ++i )
+ {
+ int64_t row_id = row_ids[i];
+ uint64_t count = counts[i];
+
+ REQUIRE_EXPR ( KRowSetAddRowIdRange ( rowset, ctx, row_id, count ) );
+ for ( uint64_t j = 0; j < count; ++j )
+ inserted_rows_set.insert( row_id + j );
+ }
+
+ for ( size_t i = 0; i < sizeof overlap_row_ids / sizeof overlap_row_ids[0]; ++i )
+ {
+ int64_t row_id = overlap_row_ids[i];
+ uint64_t count = overlap_counts[i];
+
+ REQUIRE_EXPR ( KRowSetAddRowIdRange ( rowset, ctx, row_id, count ) );
+ for ( uint64_t j = 0; j < count; ++j )
+ inserted_rows_set.insert( row_id + j );
+ }
+ RunChecks ( rowset, inserted_rows_set );
+
+ REQUIRE_EXPR ( KRowSetRelease( rowset, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( KRowSetRowRangesDenseOverlapDuplicates, RowSetFixture )
+{
+ ENTRY;
+ KRowSet * rowset;
+ std::set<int64_t> inserted_rows_set;
+
+ int64_t row_ids[] = { 0, 5, 10, 15, 20, 25, 30, 35, 40, 1000 };
+ uint64_t counts[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
+
+ int64_t overlap_row_ids[] = { 500 };
+ uint64_t overlap_counts[] = { 1000 };
+
+ REQUIRE_EXPR ( rowset = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ for ( size_t i = 0; i < sizeof row_ids / sizeof row_ids[0]; ++i )
+ {
+ int64_t row_id = row_ids[i];
+ uint64_t count = counts[i];
+
+ REQUIRE_EXPR ( KRowSetAddRowIdRange ( rowset, ctx, row_id, count ) );
+ for ( uint64_t j = 0; j < count; ++j )
+ inserted_rows_set.insert( row_id + j );
+ }
+
+ for ( size_t i = 0; i < sizeof overlap_row_ids / sizeof overlap_row_ids[0]; ++i )
+ {
+ int64_t row_id = overlap_row_ids[i];
+ uint64_t count = overlap_counts[i];
+
+ REQUIRE_EXPR ( KRowSetAddRowIdRange ( rowset, ctx, row_id, count ) );
+ for ( uint64_t j = 0; j < count; ++j )
+ inserted_rows_set.insert( row_id + j );
+
+ }
+ RunChecks ( rowset, inserted_rows_set );
+
+ REQUIRE_EXPR ( KRowSetRelease( rowset, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( KRowSetIterator, RowSetFixture )
+{
+ ENTRY;
+
+ KRowSet * rowset;
+
+ int64_t row_id_inserted = 55;
+
+ REQUIRE_EXPR ( rowset = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+ REQUIRE_EXPR ( KRowSetAddRowIdRange ( rowset, ctx, row_id_inserted, 1 ) );
+
+ uint64_t num_rows;
+ REQUIRE_EXPR ( num_rows = KRowSetGetNumRowIds ( rowset, ctx ) );
+ REQUIRE_EQ ( num_rows, (uint64_t)1 );
+
+ int64_t row_id_retrieved;
+ KRowSetIterator * it;
+ REQUIRE_EXPR ( it = KRowSetMakeIterator ( rowset, ctx ) );
+ REQUIRE_EXPR ( KRowSetRelease( rowset, ctx ) );
+ rowset = NULL;
+
+ REQUIRE ( KRowSetIteratorIsValid ( it ) );
+ REQUIRE_EXPR ( row_id_retrieved = KRowSetIteratorGetRowId ( it, ctx ) );
+ REQUIRE_EQ ( row_id_inserted, row_id_retrieved );
+
+ bool has_row;
+ REQUIRE_EXPR ( has_row = KRowSetIteratorNext ( it, ctx ) );
+ REQUIRE ( ! has_row );
+ REQUIRE ( ! KRowSetIteratorIsValid ( it ) );
+ REQUIRE_EXPR_FAILED ( KRowSetIteratorGetRowId ( it, ctx ) );
+ REQUIRE_EXPR_FAILED ( KRowSetIteratorNext ( it, ctx ) );
+
+// // move forward out of boundaries and then move back
+// for ( int i = 0; i < move_out_boundaries; ++i )
+// {
+// REQUIRE_EXPR_FAILED ( KRowSetIteratorNext( it, ctx ) );
+// REQUIRE ( !KRowSetIteratorIsValid ( it ) );
+// }
+// for ( int i = move_out_boundaries - 1; i >= 0; --i )
+// {
+// rc_t rc = KRowSetIteratorPrev( it );
+// if ( i == 0 )
+// {
+// REQUIRE_RC ( rc );
+// }
+// else
+// {
+// REQUIRE_RC_FAIL ( rc );
+// REQUIRE ( !KRowSetIteratorIsValid ( it ) );
+// }
+// }
+// REQUIRE ( KRowSetIteratorIsValid ( it ) );
+// REQUIRE_RC ( KRowSetIteratorRowId ( it, &row_id_retrieved ) );
+// REQUIRE_EQ ( row_id_inserted, row_id_retrieved );
+//
+// // move backward out of boundaries and then move back
+// for ( int i = 0; i < move_out_boundaries; ++i )
+// {
+// REQUIRE_RC_FAIL ( KRowSetIteratorPrev( it ) );
+// REQUIRE ( !KRowSetIteratorIsValid ( it ) );
+// }
+// for ( int i = move_out_boundaries - 1; i >= 0; --i )
+// {
+// rc_t rc = KRowSetIteratorNext( it );
+// if ( i == 0 )
+// {
+// REQUIRE_RC ( rc );
+// }
+// else
+// {
+// REQUIRE_RC_FAIL ( rc );
+// REQUIRE ( !KRowSetIteratorIsValid ( it ) );
+// }
+// }
+// REQUIRE ( KRowSetIteratorIsValid ( it ) );
+// REQUIRE_RC ( KRowSetIteratorRowId ( it, &row_id_retrieved ) );
+// REQUIRE_EQ ( row_id_inserted, row_id_retrieved );
+
+ REQUIRE_EXPR ( KRowSetIteratorRelease ( it, ctx ) );
+ EXIT;
+}
+//
+//FIXTURE_TEST_CASE ( KRowSetIteratorMoveForwardAndBackward, RowSetFixture )
+//{
+// KRowSet * rowset;
+//
+// int64_t row_ids[] = { 0, 5, 10, 15, 20, 25, 30, 35, 40, 1000, 100000 };
+//
+// rowset = KTableMakeRowSet ( NULL, ctx );
+// for ( int i = 0; i < sizeof row_ids / sizeof row_ids[0]; ++i )
+// {
+// int64_t row_id = row_ids[i];
+// bool inserted;
+//
+// REQUIRE_RC ( KRowSetAddRowId ( rowset, row_id, &inserted ) );
+// REQUIRE ( inserted );
+// }
+//
+// uint64_t num_rows;
+// REQUIRE_RC ( KRowSetGetNumRowIds ( rowset, &num_rows ) );
+// REQUIRE_EQ ( num_rows, (uint64_t)sizeof row_ids / sizeof row_ids[0] );
+//
+// KRowSetIterator * it;
+// REQUIRE_RC ( KRowSetMakeIterator ( rowset, &it ) );
+// REQUIRE_RC ( KRowSetRelease( rowset ) );
+// rowset = NULL;
+//
+// // move forward
+// for ( int i = 0; i < sizeof row_ids / sizeof row_ids[0]; ++i )
+// {
+// int64_t row_id;
+//
+// if ( i != 0 )
+// REQUIRE_RC ( KRowSetIteratorNext( it ) );
+//
+// REQUIRE ( KRowSetIteratorIsValid ( it ) );
+// REQUIRE_RC ( KRowSetIteratorRowId ( it, &row_id ) );
+// REQUIRE_EQ ( row_id, row_ids[i] );
+// }
+//
+// // move backward
+// for ( int i = sizeof row_ids / sizeof row_ids[0] - 1; i >= 0; --i )
+// {
+// int64_t row_id;
+//
+// if ( i != sizeof row_ids / sizeof row_ids[0] - 1 )
+// REQUIRE_RC ( KRowSetIteratorPrev( it ) );
+//
+// REQUIRE ( KRowSetIteratorIsValid ( it ) );
+// REQUIRE_RC ( KRowSetIteratorRowId ( it, &row_id ) );
+// REQUIRE_EQ ( row_id, row_ids[i] );
+// }
+//
+// REQUIRE_RC ( KRowSetIteratorRelease ( it ) );
+//}
+
+FIXTURE_TEST_CASE ( KRowSetIteratorOverEmptySet, RowSetFixture )
+{
+ ENTRY;
+ KRowSet * rowset;
+ REQUIRE_EXPR ( rowset = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ KRowSetIterator * it;
+ REQUIRE_EXPR ( it = KRowSetMakeIterator ( rowset, ctx ) );
+ REQUIRE ( !KRowSetIteratorIsValid ( it ) );
+
+ REQUIRE_EXPR ( KRowSetIteratorRelease( it, ctx ) );
+ REQUIRE_EXPR ( KRowSetRelease( rowset, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( KRowSetIntersectSimpleTest, RowSetFixture )
+{
+ ENTRY;
+ KRowSet * rowset1;
+ KRowSet * rowset2;
+ std::set<int64_t> inserted_rows_set1;
+ std::set<int64_t> inserted_rows_set2;
+
+ REQUIRE_EXPR ( rowset1 = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+ REQUIRE_EXPR ( rowset2 = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ for ( int i = 0; i < 5000; ++i )
+ {
+ int64_t row_id = GenerateId ( -1, -1 );
+ if ( inserted_rows_set1.find( row_id ) == inserted_rows_set1.end() )
+ {
+ REQUIRE_EXPR ( KRowSetAddRowId ( rowset1, ctx, row_id) );
+ inserted_rows_set1.insert( row_id );
+ // just don't insert a single row to a second rowset, but otherwise keep them equal
+ if ( i != 0 )
+ {
+ REQUIRE_EXPR ( KRowSetAddRowId ( rowset2, ctx, row_id) );
+ inserted_rows_set2.insert( row_id );
+ }
+ }
+ else
+ --i;
+ }
+
+ KRowSet * rowset_result;
+ REQUIRE_EXPR ( rowset_result = KRowSetIntersect ( ctx, rowset1, rowset2 ) );
+
+ RunChecks ( rowset1, inserted_rows_set1 );
+ RunChecks ( rowset2, inserted_rows_set2 );
+ RunChecks ( rowset_result, inserted_rows_set2 );
+
+ REQUIRE_EXPR ( KRowSetRelease( rowset1, ctx ) );
+ REQUIRE_EXPR ( KRowSetRelease( rowset2, ctx ) );
+ REQUIRE_EXPR ( KRowSetRelease( rowset_result, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( KRowSetIntersectNormalTest, RowSetFixture )
+{
+ ENTRY;
+ // check all 4 combinations of dense/scattered leaves
+ for ( int i = 0; i < 4; ++i )
+ {
+ KRowSet * rowset1;
+ KRowSet * rowset2;
+ KRowSet * rowset_result;
+ std::set<int64_t> inserted_rows_set1;
+ std::set<int64_t> inserted_rows_set2;
+ std::set<int64_t> intersection_set;
+
+ REQUIRE_EXPR ( rowset1 = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+ REQUIRE_EXPR ( rowset2 = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ inserted_rows_set1 = InsertRandomRows ( rowset1, 5000, (i & 1) ? 8 : 0 );
+ inserted_rows_set2 = InsertRandomRows ( rowset2, 5000, (i & 2) ? 8 : 0 );
+
+ RunChecks ( rowset1, inserted_rows_set1 );
+ RunChecks ( rowset2, inserted_rows_set2 );
+
+ REQUIRE_EXPR ( rowset_result = KRowSetIntersect ( ctx, rowset1, rowset2 ) );
+ intersection_set = SetIntersection ( inserted_rows_set1, inserted_rows_set2 );
+
+ RunChecks ( rowset_result, intersection_set );
+ RunChecks ( rowset1, inserted_rows_set1 );
+ RunChecks ( rowset2, inserted_rows_set2 );
+ REQUIRE_EXPR ( KRowSetRelease( rowset_result, ctx ) );
+
+ REQUIRE_EXPR ( rowset_result = KRowSetIntersect ( ctx, rowset2, rowset1 ) );
+ RunChecks ( rowset_result, intersection_set );
+ RunChecks ( rowset1, inserted_rows_set1 );
+ RunChecks ( rowset2, inserted_rows_set2 );
+
+ REQUIRE_EXPR ( KRowSetRelease( rowset1, ctx ) );
+ REQUIRE_EXPR ( KRowSetRelease( rowset2, ctx ) );
+ REQUIRE_EXPR ( KRowSetRelease( rowset_result, ctx ) );
+ }
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( KRowSetUnionSimpleTest, RowSetFixture )
+{
+ ENTRY;
+ KRowSet * rowset1;
+ KRowSet * rowset2;
+ std::set<int64_t> inserted_rows_set1;
+ std::set<int64_t> inserted_rows_set2;
+
+ REQUIRE_EXPR ( rowset1 = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+ REQUIRE_EXPR ( rowset2 = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ for ( int i = 0; i < 5000; ++i )
+ {
+ int64_t row_id = GenerateId ( -1, -1 );
+ if ( inserted_rows_set1.find( row_id ) == inserted_rows_set1.end() )
+ {
+ REQUIRE_EXPR ( KRowSetAddRowId ( rowset1, ctx, row_id) );
+ inserted_rows_set1.insert( row_id );
+ // just don't insert a single row to a second rowset, but otherwise keep them equal
+ if ( i != 0 )
+ {
+ REQUIRE_EXPR ( KRowSetAddRowId ( rowset2, ctx, row_id) );
+ inserted_rows_set2.insert( row_id );
+ }
+ }
+ else
+ --i;
+ }
+
+ KRowSet * rowset_result;
+ REQUIRE_EXPR ( rowset_result = KRowSetUnion ( ctx, rowset1, rowset2 ) );
+
+ RunChecks ( rowset1, inserted_rows_set1 );
+ RunChecks ( rowset2, inserted_rows_set2 );
+ RunChecks ( rowset_result, inserted_rows_set1 );
+
+ REQUIRE_EXPR ( KRowSetRelease( rowset1, ctx ) );
+ REQUIRE_EXPR ( KRowSetRelease( rowset2, ctx ) );
+ REQUIRE_EXPR ( KRowSetRelease( rowset_result, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( KRowSetUnionNormalTest, RowSetFixture )
+{
+ ENTRY;
+ // check all 4 combinations of dense/scattered leaves
+ for ( int i = 0; i < 4; ++i )
+ {
+ KRowSet * rowset1;
+ KRowSet * rowset2;
+ KRowSet * rowset_result;
+ std::set<int64_t> inserted_rows_set1;
+ std::set<int64_t> inserted_rows_set2;
+ std::set<int64_t> union_set;
+
+ REQUIRE_EXPR ( rowset1 = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+ REQUIRE_EXPR ( rowset2 = KTableMakeRowSet ( GetTable ( GetName() ), ctx ) );
+
+ inserted_rows_set1 = InsertRandomRows ( rowset1, 5000, (i & 1) ? 8 : 0 );
+ inserted_rows_set2 = InsertRandomRows ( rowset2, 5000, (i & 2) ? 8 : 0 );
+
+ RunChecks ( rowset1, inserted_rows_set1 );
+ RunChecks ( rowset2, inserted_rows_set2 );
+
+ REQUIRE_EXPR ( rowset_result = KRowSetUnion ( ctx, rowset1, rowset2 ) );
+ union_set = SetUnion ( inserted_rows_set1, inserted_rows_set2 );
+
+ RunChecks ( rowset_result, union_set );
+ RunChecks ( rowset1, inserted_rows_set1 );
+ RunChecks ( rowset2, inserted_rows_set2 );
+ REQUIRE_EXPR ( KRowSetRelease( rowset_result, ctx ) );
+
+ REQUIRE_EXPR ( rowset_result = KRowSetUnion ( ctx, rowset2, rowset1 ) );
+ RunChecks ( rowset_result, union_set );
+ RunChecks ( rowset1, inserted_rows_set1 );
+ RunChecks ( rowset2, inserted_rows_set2 );
+
+ REQUIRE_EXPR ( KRowSetRelease( rowset1, ctx ) );
+ REQUIRE_EXPR ( KRowSetRelease( rowset2, ctx ) );
+ REQUIRE_EXPR ( KRowSetRelease( rowset_result, ctx ) );
+ }
+ EXIT;
+}
+
+//////////////////////////////////////////// Main
+extern "C"
+{
+
+#include <kapp/main.h>
+#include <kapp/args.h>
+
+ver_t CC KAppVersion ( void )
+{
+ return 0;
+}
+
+
+const char UsageDefaultName[] = "test-rowset";
+
+rc_t CC UsageSummary ( const char *progname )
+{
+ return KOutMsg ( "\n"
+ "Usage:\n"
+ " %s [Options] <target>\n"
+ "\n"
+ "Summary:\n"
+ " test the rowset.\n"
+ , progname
+ );
+}
+
+rc_t CC Usage ( const Args *args )
+{
+ const char * progname = UsageDefaultName;
+ const char * fullpath = UsageDefaultName;
+ rc_t rc;
+
+ if (args == NULL)
+ rc = RC (rcApp, rcArgv, rcAccessing, rcSelf, rcNull);
+ else
+ rc = ArgsProgram (args, &fullpath, &progname);
+ if (rc)
+ progname = fullpath = UsageDefaultName;
+
+ UsageSummary (progname);
+
+ KOutMsg ("Options:\n");
+
+ HelpOptionsStandard ();
+
+ HelpVersion (fullpath, KAppVersion());
+
+ return rc;
+}
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ srand ( time(NULL) );
+ return KRowSetTestSuite(argc, argv);
+}
+
+}
+
diff --git a/test/kfg/flat-sra-kfg.cpp b/test/kfg/flat-sra-kfg.cpp
index d647931..163da47 100644
--- a/test/kfg/flat-sra-kfg.cpp
+++ b/test/kfg/flat-sra-kfg.cpp
@@ -24,6 +24,7 @@
*
*/
+#include <kapp/args.h> /* ArgsMakeAndHandle */
#include <kfg/config.h> /* KConfig */
#include <kfs/directory.h> /* KDirectory */
#include <kfs/file.h> /* KFileRelease */
@@ -45,7 +46,8 @@ using ncbi::NK::TestCase;
using std::cerr;
using std::string;
-TEST_SUITE(flatSraKfgTestSuite);
+static rc_t argsHandler(int argc, char* argv[]);
+TEST_SUITE_WITH_ARGS_HANDLER(flatSraKfgTestSuite, argsHandler);
static KNSManager * kns = NULL;
@@ -79,7 +81,7 @@ public:
const KFile * f = NULL;
if ( expectedShort . size () ) {
REQUIRE_RC ( VResolverQuery
- ( resolver, eProtocolHttp, queryShort, NULL, & remote, NULL ) );
+ ( resolver, 0, queryShort, NULL, & remote, NULL ) );
compare ( remote, expectedShort );
RELEASE ( VPath, remote );
REQUIRE_RC ( KNSManagerMakeReliableHttpFile
@@ -87,12 +89,12 @@ public:
RELEASE ( KFile, f );
} else {
REQUIRE_RC_FAIL ( VResolverQuery
- ( resolver, eProtocolHttp, queryShort, NULL, & remote, NULL ) );
+ ( resolver, 0, queryShort, NULL, & remote, NULL ) );
}
if ( expectedLong . size () ) {
REQUIRE_RC ( VResolverQuery
- ( resolver, eProtocolHttp, queryLong, NULL, & remote, NULL ) );
+ ( resolver, 0, queryLong, NULL, & remote, NULL ) );
compare ( remote, expectedLong );
RELEASE ( VPath, remote );
REQUIRE_RC ( KNSManagerMakeReliableHttpFile
@@ -100,7 +102,7 @@ public:
RELEASE ( KFile, f );
} else {
REQUIRE_RC_FAIL ( VResolverQuery
- ( resolver, eProtocolHttp, queryLong, NULL, & remote, NULL ) );
+ ( resolver, 0, queryLong, NULL, & remote, NULL ) );
}
@@ -118,16 +120,16 @@ public:
static const char cgiPath[] = "/repository/remote/main/CGI/resolver-cgi";
static const char goodCgi[]
- = "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi";
+ = "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi";
static const char badCgi[]
- = "http://XXX.ncbi.nlm.nih.gov/Traces/names/names.cgi";
+ = "https://XXX.ncbi.nlm.nih.gov/Traces/names/names.cgi";
#ifdef ALL
TEST_CASE(test_sra) {
const string newShort
("http://sra-download.ncbi.nlm.nih.gov/srapub/SRR000001");
const string newLong
("http://sra-download.ncbi.nlm.nih.gov/srapub/SRR1000254");
- const string oldShort("http://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/"
+ const string oldShort("https://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/"
"reads/ByRun/sra/SRR/SRR000/SRR000001/SRR000001.sra");
rc_t rc = 0;
@@ -151,7 +153,7 @@ TEST_CASE(test_sra) {
REQUIRE_RC ( VFSManagerMakeAccPath ( mgr, & queryLong, "SRR1000254" ) );
// const char rootPath [] = "/repository/remote/aux/NCBI/root";
-// const char newRoot[] = "http://sra-download.ncbi.nlm.nih.gov";
+// const char newRoot[] = "https://sra-download.ncbi.nlm.nih.gov";
// fail using incomplete configuration
Test(this, mgr, cfg, queryShort, queryLong, "incomplete-old");
@@ -166,7 +168,7 @@ TEST_CASE(test_sra) {
/* aux repositories are ignored : VDB-3090
// old aux configuration cannot resolve long accession
- const char oldRoot [] = "http://ftp-trace.ncbi.nlm.nih.gov/sra";
+ const char oldRoot [] = "https://ftp-trace.ncbi.nlm.nih.gov/sra";
REQUIRE_RC ( KConfigWriteString ( cfg, rootPath, oldRoot ) );
Test(this, mgr, cfg, queryShort, queryLong, "aux-old", oldShort);
*/
@@ -249,7 +251,7 @@ public:
REQUIRE_RC ( VFSManagerMakeResolver ( mgr, & resolver, cfg ) );
// fail to resolve using empty config
REQUIRE_RC_FAIL ( VResolverQuery
- ( resolver, eProtocolHttp, query, NULL, & remote, NULL ) );
+ ( resolver, 0, query, NULL, & remote, NULL ) );
RELEASE ( VResolver, resolver );
// resolve using good cgi
@@ -257,7 +259,7 @@ public:
REQUIRE_RC ( VFSManagerMakeResolver ( mgr, & resolver, cfg ) );
REQUIRE_RC ( VResolverQuery
- ( resolver, eProtocolHttp, query, NULL, & remoteCgi, NULL ) );
+ ( resolver, 0, query, NULL, & remoteCgi, NULL ) );
if ( ! expected ) {
cerr << "\nTO FIX !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! :\n "
<< "\tVResolverQuery(CGI, " << acc << ") SHOULD FAIL !!!!!!!!!!!\n";
@@ -269,14 +271,14 @@ cerr << "\nTO FIX !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! :\n "
REQUIRE_RC ( KConfigWriteString ( cfg, cgiPath, badCgi ) );
REQUIRE_RC ( VFSManagerMakeResolver ( mgr, & resolver, cfg ) );
REQUIRE_RC_FAIL ( VResolverQuery
- ( resolver, eProtocolHttp, query, NULL, & remote, NULL ) );
+ ( resolver, 0, query, NULL, & remote, NULL ) );
RELEASE ( VResolver, resolver );
// resolve using aux configuration
REQUIRE_RC ( KConfigWriteString ( cfg, name, value ) );
REQUIRE_RC ( VFSManagerMakeResolver ( mgr, & resolver, cfg ) );
REQUIRE_RC ( VResolverQuery
- ( resolver, eProtocolHttp, query, NULL, & remote, NULL ) );
+ ( resolver, 0, query, NULL, & remote, NULL ) );
if ( ! expected ) {
cerr << "TO FIX !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! :\n "
<< "\tVResolverQuery(AUX, " << acc << ") SHOULD FAIL !!!!!!!!!!!\n";
@@ -359,7 +361,7 @@ TEST_CASE(test_nannot) {
TEST_CASE(test_AAAB01_1) {
Fixture fixture(this, "refseq AAAB01.1", "AAAB01.1",
"/repository/remote/aux/NCBI/apps/refseq/volumes/refseq", "refseq",
- "http://ftp-trace.ncbi.nlm.nih.gov/sra/refseq/AAAB01" );
+ "https://ftp-trace.ncbi.nlm.nih.gov/sra/refseq/AAAB01" );
}
#endif
#ifdef ALL
@@ -395,12 +397,22 @@ TEST_CASE(test_WGS_AAAB01) {
TEST_CASE(test_WGS_AAAB01_1) {
Fixture fixture(this, "WGS AAAB01_1", "AAAB01.1",
"/repository/remote/aux/NCBI/apps/wgs/volumes/fuseWGS", "wgs",
- "http://ftp-trace.ncbi.nlm.nih.gov/sra/wgs/AA/AB/AAAB01.1" );
+ "https://ftp-trace.ncbi.nlm.nih.gov/sra/wgs/AA/AB/AAAB01.1" );
}
TEST_CASE(test_WGS_AAAB01_9) {
Fixture fixture(this, "WGS AAAB01_2", "AAAB01.9",
"/repository/remote/aux/NCBI/apps/wgs/volumes/fuseWGS", "wgs", NULL );
}*/
+
+static rc_t argsHandler(int argc, char * argv[]) {
+ Args * args = NULL;
+ rc_t rc = ArgsMakeAndHandle(&args, argc, argv, 0, NULL, 0);
+ ArgsWhack(args);
+ return rc;
+}
+rc_t CC Usage ( const Args * args ) { return 0; }
+const char UsageDefaultName [] = "flat-sra-kfg";
+rc_t CC UsageSummary ( const char * prog_name ) { return 0; }
extern "C" {
ver_t CC KAppVersion ( void ) { return 0; }
rc_t CC KMain ( int argc, char *argv [] ) {
diff --git a/test/kfg/kfgtest.cpp b/test/kfg/kfgtest.cpp
index 9529e93..bb37b1c 100644
--- a/test/kfg/kfgtest.cpp
+++ b/test/kfg/kfgtest.cpp
@@ -923,7 +923,7 @@ FIXTURE_TEST_CASE(KConfigImportNgc_Basic, KfgFixture)
// .../root = "repos/ngc/" ; creates the dir
//
// also, creates the following node if it is not present
- // /repository/remote/protected/CGI/resolver-cgi="http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
+ // /repository/remote/protected/CGI/resolver-cgi="https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
{
REQUIRE_RC(KConfigNodeReadVPath(GetNode("/repository/user/protected/dbGaP-2956/encryption-key-path"), &path));
VPathReadPath(path, buf, sizeof(buf), &num_read);
@@ -965,7 +965,7 @@ FIXTURE_TEST_CASE(KConfigImportNgc_Basic, KfgFixture)
const String* str;
REQUIRE_RC(KConfigNodeReadVPath(GetNode("/repository/remote/protected/CGI/resolver-cgi"), &path));
REQUIRE_RC(VPathMakeString(path, &str));
- REQUIRE_EQ(string(str->addr), string("http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"));
+ REQUIRE_EQ(string(str->addr), string("https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"));
StringWhack(str);
}
diff --git a/test/kfs/cacheteetest.cpp b/test/kfs/cacheteetest.cpp
index a124d8e..be2eab0 100644
--- a/test/kfs/cacheteetest.cpp
+++ b/test/kfs/cacheteetest.cpp
@@ -292,7 +292,6 @@ TEST_CASE( CacheTee_Read )
REQUIRE_RC( KDirectoryRelease( dir ) );
}
-
TEST_CASE( CacheTee_Promoting )
{
KOutMsg( "Test: CacheTee_Promoting\n" );
@@ -338,7 +337,6 @@ TEST_CASE( CacheTee_Promoting )
REQUIRE_RC( KDirectoryRelease( dir ) );
}
-
static rc_t cache_access( int tid, int num_threads, const KFile * origfile, const KFile * cacheteefile )
{
rc_t rc;
@@ -409,8 +407,6 @@ static rc_t CC thread_func( const KThread *self, void *data )
return cache_access( td->tid, td->num_threads, td->origfile, td->cacheteefile );
}
-// TODO: fix, this does not work on Windows
-#if !defined(WINDOWS) && !defined(_WIN32)
TEST_CASE( CacheTee_Multiple_Users_Multiple_Inst )
{
KOutMsg( "Test: CacheTee_Multiple_Users_Multiple_Inst\n" );
@@ -482,6 +478,8 @@ TEST_CASE( CacheTee_Multiple_Users_Single_Inst )
REQUIRE_RC( KDirectoryRelease( dir ) );
}
+// TODO: fix, this does not work on Windows
+#if !defined(WINDOWS) && !defined(_WIN32)
TEST_CASE( CacheTee_ReadOnly )
{
KOutMsg( "Test: CacheTee_ReadOnly\n" );
@@ -500,9 +498,8 @@ TEST_CASE( CacheTee_ReadOnly )
REQUIRE_RC( read_partial( tee, 100, 100 ) );
REQUIRE_RC( KFileRelease( tee ) );
- /* switch the cache-file to read-only */
REQUIRE_RC( KDirectorySetAccess ( dir, false, 0, 0222, "%s", CACHEFILE1 ) );
-
+
/* make a second cache-tee and read all from it... */
REQUIRE_RC( KDirectoryMakeCacheTee ( dir, &tee, org, BLOCKSIZE, "%s", CACHEFILE ) );
REQUIRE_RC( read_all( tee, 1024 * 32 ) );
@@ -523,7 +520,9 @@ TEST_CASE( CacheTee_ReadOnly )
REQUIRE_RC( KFileRelease( cache ) );
REQUIRE_RC( KFileRelease( org ) );
REQUIRE_RC( KDirectoryRelease( dir ) );
+
}
+#endif
TEST_CASE( CacheTee_Multiple_Users_with_Promoting )
{
@@ -562,7 +561,6 @@ TEST_CASE( CacheTee_Multiple_Users_with_Promoting )
REQUIRE_RC( KFileRelease( org ) );
REQUIRE_RC( KDirectoryRelease( dir ) );
}
-#endif
//////////////////////////////////////////// Main
extern "C"
@@ -598,6 +596,7 @@ rc_t CC KMain ( int argc, char *argv [] )
rc = CacheTeeTests( argc, argv );
finish_cachetee_tests();
}
+ KOutMsg( "and the result is: %d", rc );
return rc;
}
diff --git a/test/kns/Makefile b/test/kns/Makefile
index 4a4f574..6c809f3 100644
--- a/test/kns/Makefile
+++ b/test/kns/Makefile
@@ -31,11 +31,36 @@ MODULE = test/kns
TEST_TOOLS = \
KNSManagerSingletonTest \
test-kns \
+ test-proxy \
test-http \
test-http-dropconn \
include $(TOP)/build/Makefile.env
+EXT_TOOLS = \
+ test-proxy-with-env
+
+ALL_TOOLS = \
+ $(INT_TOOLS) \
+ $(EXT_TOOLS)
+
+#-------------------------------------------------------------------------------
+# outer targets
+#
+all std: makedirs
+ @ $(MAKE_CMD) $(TARGDIR)/$@
+
+$(ALL_TOOLS): makedirs
+ @ $(MAKE_CMD) $(BINDIR)/$@
+
+.PHONY: all std $(ALL_TOOLS)
+
+#-------------------------------------------------------------------------------
+# std
+#
+$(TARGDIR)/std: \
+ $(addprefix $(BINDIR)/,$(EXT_TOOLS))
+
$(TEST_TOOLS): makedirs
@ $(MAKE_CMD) $(TEST_BINDIR)/$@
@@ -89,6 +114,19 @@ $(TEST_BINDIR)/test-kns: $(KNSTEST_OBJ)
kns: test-kns
$(TEST_BINDIR)/test-kns #-l=all
+
+#----------------------------------------------------------------
+# test-proxy
+#
+PROXYTEST_SRC = \
+ http-client \
+ test-proxy \
+
+PROXYTEST_OBJ = \
+ $(addsuffix .$(OBJX),$(PROXYTEST_SRC))
+
+$(TEST_BINDIR)/test-proxy: $(PROXYTEST_OBJ)
+ $(LP) --exe -o $@ $^ $(KNSTEST_LIB)
#----------------------------------------------------------------
# test-http
@@ -120,3 +158,15 @@ $(TEST_BINDIR)/test-http-dropconn: $(HTTP_DROPCONN_TEST_OBJ)
dropconn: test-http-dropconn
$(TEST_BINDIR)/test-http-dropconn # -l=all
+#----------------------------------------------------------------
+# test-proxy-with-env
+#
+TOOL_SRC = \
+ http-client \
+ test-proxy-with-env \
+
+TOOL_OBJ = \
+ $(addsuffix .$(OBJX),$(TOOL_SRC))
+
+$(BINDIR)/test-proxy-with-env: $(TOOL_OBJ)
+ $(LP) --exe -o $@ $^ -L$(VDB_LIBDIR) -L$(VDB_LIBDIR)/../ilib $(KNSTEST_LIB)
diff --git a/test/kns/TestProxy.hpp b/test/kns/TestProxy.hpp
new file mode 100644
index 0000000..f0394f8
--- /dev/null
+++ b/test/kns/TestProxy.hpp
@@ -0,0 +1,227 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <kfg/config.h> /* KConfigRelease */
+#include <kns/kns-mgr-priv.h> /* KNSManagerMakeConfig */
+#include <kns/manager.h> /* KNSManagerRelease */
+#include <ktst/unit_test_suite.hpp> // TestCase
+
+#include "test-proxy.h" // KNSManagerMakeKEndPointArgsIterator
+
+#define RELEASE( type, obj ) do { rc_t rc2 = type##Release ( obj ); \
+ if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while ( false )
+
+struct C { // Configuration Helper
+ C ( const std::string & aPath, const std::string & aValue )
+ : path ( aPath )
+ , value ( aValue )
+ , next ( NULL)
+ {}
+
+ ~ C ( void )
+ {
+ delete next;
+ next = NULL;
+ }
+
+ void add ( const std::string & path, const std::string & value )
+ {
+ C * p = this;
+ while ( p -> next != NULL) {
+ p = p -> next;
+ }
+
+ p -> next = new C ( path, value );
+ }
+
+ bool contains ( const std::string & name, const std::string & aValue ) const
+ {
+ const C * c = this;
+ while ( c ) {
+ if ( name == c -> path && aValue == c -> value ) {
+ return true;
+ }
+ c = c -> next;
+ }
+
+ return false;
+ }
+
+ const std::string path;
+ const std::string value;
+
+ C * next;
+};
+
+struct E { // Expected Proxy Values
+ E ( const std::string & aPath, uint16_t aPort )
+ : path ( aPath )
+ , port ( aPort )
+ , next ( NULL )
+ {}
+
+ ~ E ( void )
+ {
+ delete next;
+ next = NULL;
+ }
+
+ void add ( const std::string & path, uint16_t port )
+ {
+ E * p = this;
+ while ( p -> next != NULL) {
+ p = p -> next;
+ }
+
+ p -> next = new E ( path, port );
+ }
+
+ const std::string path;
+ const uint16_t port;
+
+ E * next;
+};
+
+class CKConfig {
+ KConfig * _self;
+
+public:
+ CKConfig ( const C * c = NULL ) : _self (NULL) {
+ rc_t rc = KConfigMake ( & _self, NULL );
+ if ( rc != 0 ) {
+ throw rc;
+ }
+
+ while ( c ) {
+ rc_t rc = KConfigWriteString
+ ( _self, c -> path . c_str (), c -> value . c_str () );
+ if ( rc != 0 ) {
+ throw rc;
+ }
+
+ c = c -> next;
+ }
+ }
+
+ ~CKConfig ( void ) {
+ KConfigRelease ( _self );
+ _self = NULL;
+ }
+
+ KConfig * get ( void ) const {
+ return _self;
+ }
+};
+
+class TestProxy : private ncbi::NK::TestCase {
+ static bool _StringEqual ( const String * a, const String * b ) {
+ assert ( a && b );
+ return ( ( a ) -> len == ( b ) -> len &&
+ memcmp ( ( a ) -> addr, ( b ) -> addr, ( a ) -> len ) == 0 );
+ }
+ TestCase * _dad;
+ CKConfig _kfg;
+ void testProxies ( const KNSManager * mgr, const E * e ) {
+ const HttpProxy * p = KNSManagerGetHttpProxy ( mgr );
+ if ( p != NULL && e == NULL) {
+ REQUIRE ( ! p );
+ }
+ while ( e ) {
+ const String * http_proxy = NULL;
+ uint16_t http_proxy_port = 0;
+ HttpProxyGet ( p, & http_proxy, & http_proxy_port );
+ REQUIRE ( http_proxy );
+ std::string proxy ( http_proxy -> addr, http_proxy -> len );
+ REQUIRE_EQ ( e -> path, proxy );
+ REQUIRE_EQ ( e -> port, http_proxy_port );
+ e = e -> next;
+ p = HttpProxyGetNextHttpProxy ( p );
+ REQUIRE ( ( e && p ) || ( ! e && ! p ) );
+ }
+ }
+ void testEndPoints ( const KNSManager * mgr, const E * e, const C * c ) {
+ String aHost;
+ CONST_STRING ( & aHost, "a.host.domain" );
+ uint16_t aPort = 8976;
+ struct KEndPointArgsIterator * i
+ = KNSManagerMakeKEndPointArgsIterator ( mgr, & aHost, aPort );
+ REQUIRE ( i );
+ const String * hostname = NULL;
+ uint16_t port = ~ 0;
+ bool proxy_default_port = true;
+ bool proxy_ep = false;
+ while ( e ) {
+ REQUIRE ( KEndPointArgsIterator_Next ( i,
+ & hostname, & port, & proxy_default_port, & proxy_ep ) );
+ REQUIRE ( proxy_ep );
+ String host;
+ StringInit ( & host,
+ e -> path. c_str (), e -> path. size (), e -> path. size () );
+ REQUIRE ( StringEqual ( hostname, & host ) );
+ if ( e -> port != 0 ) {
+ REQUIRE_EQ ( port, e -> port );
+ REQUIRE ( ! proxy_default_port );
+ } else {
+ REQUIRE_EQ ( static_cast < int> ( port), 3128 );
+ REQUIRE ( proxy_default_port );
+ REQUIRE ( KEndPointArgsIterator_Next ( i, & hostname,
+ & port, & proxy_default_port, & proxy_ep ) );
+ REQUIRE ( proxy_ep );
+ REQUIRE ( StringEqual ( hostname, & host ) );
+ REQUIRE_EQ ( static_cast < int> ( port ), 8080 );
+ REQUIRE ( proxy_default_port );
+ }
+ e = e -> next;
+ }
+ if ( c == NULL || ! c -> contains ( "/http/proxy/only", "true" ) ) {
+ REQUIRE ( KEndPointArgsIterator_Next ( i, & hostname,
+ & port, & proxy_default_port, & proxy_ep ) );
+ REQUIRE ( ! proxy_ep );
+ REQUIRE ( _StringEqual ( hostname, & aHost ) );
+ REQUIRE_EQ ( port, aPort );
+ REQUIRE ( ! proxy_default_port );
+ }
+ REQUIRE ( ! KEndPointArgsIterator_Next ( i,
+ & hostname, & port, & proxy_default_port, & proxy_ep ) );
+ }
+public:
+ TestProxy
+ ( TestCase * dad, const C * c = NULL, const E * e = NULL )
+ : TestCase ( dad -> GetName () ), _dad ( dad ), _kfg ( c )
+ {
+ rc_t rc = 0;
+ KNSManager * mgr = NULL;
+ REQUIRE_RC ( KNSManagerMakeConfig ( & mgr, _kfg . get () ) );
+ testProxies ( mgr, e );
+ testEndPoints ( mgr, e, c );
+ RELEASE ( KNSManager, mgr );
+ REQUIRE_RC ( rc );
+ }
+ ~ TestProxy ( void )
+ { assert ( _dad ); _dad -> ErrorCounterAdd ( GetErrorCounter () ); }
+};
+
+////////////////////////////////////////////////////////////////////////////////
diff --git a/test/kns/http-client.c b/test/kns/http-client.c
new file mode 100644
index 0000000..d69b292
--- /dev/null
+++ b/test/kns/http-client.c
@@ -0,0 +1,17 @@
+#include "../../libs/kns/http-client.c"
+
+const struct KEndPointArgsIterator * KNSManagerMakeKEndPointArgsIterator
+ ( const KNSManager * self, const String * hostname, uint32_t port )
+{
+ static struct KEndPointArgsIterator i;
+ KEndPointArgsIteratorMake ( & i, self, hostname, port );
+ return & i;
+}
+
+bool KEndPointArgsIterator_Next ( KEndPointArgsIterator * self,
+ const String ** hostname, uint16_t * port,
+ bool * proxy_default_port, bool * proxy_ep )
+{
+ return KEndPointArgsIteratorNext
+ ( self, hostname, port, proxy_default_port, proxy_ep);
+}
diff --git a/test/kns/httptest.cpp b/test/kns/httptest.cpp
index e805eef..33891ad 100644
--- a/test/kns/httptest.cpp
+++ b/test/kns/httptest.cpp
@@ -794,8 +794,9 @@ TEST_CASE(ContentLength) {
/* calling good cgi returns 200 and resolved path */
REQUIRE_RC ( KNSManagerMakeReliableClientRequest ( kns, & req, 0x01000000,
- NULL, "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" ) );
+ NULL, "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" ) );
REQUIRE_RC ( KHttpRequestAddPostParam ( req, "acc=AAAB01" ) );
+ REQUIRE_RC ( KHttpRequestAddPostParam ( req, "accept-proto=http" ) );
REQUIRE_RC ( KHttpRequestPOST ( req, & rslt ) );
REQUIRE_RC ( KClientHttpResultStatus ( rslt, & code, NULL, 0, NULL ) );
REQUIRE_EQ ( code, 200u );
@@ -812,7 +813,7 @@ TEST_CASE(ContentLength) {
/* calling non-existing cgi returns 404 */
REQUIRE_RC ( KNSManagerMakeReliableClientRequest ( kns, & req, 0x01000000,
- NULL, "http://www.ncbi.nlm.nih.gov/Traces/names/bad.cgi" ) );
+ NULL, "https://www.ncbi.nlm.nih.gov/Traces/names/bad.cgi" ) );
REQUIRE_RC ( KHttpRequestAddPostParam ( req, "acc=AAAB01" ) );
REQUIRE_RC ( KHttpRequestPOST ( req, & rslt ) );
REQUIRE_RC ( KClientHttpResultStatus ( rslt, & code, NULL, 0, NULL ) );
diff --git a/test/kns/knstest.cpp b/test/kns/knstest.cpp
index 564cae3..7c635e6 100644
--- a/test/kns/knstest.cpp
+++ b/test/kns/knstest.cpp
@@ -35,6 +35,8 @@
#include <klib/printf.h>
+#include <kproc/timeout.h>
+
#include <kns/manager.h>
#include <kns/endpoint.h>
#include <kns/stream.h>
@@ -44,6 +46,8 @@
#include <kfs/directory.h>
#include <kfs/file.h>
+#include <kapp/args.h> /* ArgsMakeAndHandle */
+
#include <kproc/thread.h>
#include <kproc/timeout.h>
@@ -55,7 +59,8 @@
#include <algorithm>
#include <sstream>
-TEST_SUITE(KnsTestSuite);
+static rc_t argsHandler(int argc, char* argv[]);
+TEST_SUITE_WITH_ARGS_HANDLER(KnsTestSuite, argsHandler);
using namespace std;
using namespace ncbi::NK;
@@ -179,8 +184,10 @@ FIXTURE_TEST_CASE(MakeIPCConnection_NoListener, KnsManagerFixture)
{
CONST_STRING(&name, "socket");
REQUIRE_RC(KNSManagerInitIPCEndpoint(mgr, &ep, &name));
+ timeout_t tm;
+ TimeoutInit ( & tm, 0 );
KSocket* socket;
- REQUIRE_RC_FAIL(KNSManagerMakeRetryConnection(mgr, &socket, 0, NULL, &ep)); /* no server; no retries */
+ REQUIRE_RC_FAIL(KNSManagerMakeRetryConnection(mgr, &socket, &tm, NULL, &ep)); /* no server; no retries */
REQUIRE_NULL(socket);
}
@@ -349,8 +356,11 @@ public:
KStream* MakeStream( int32_t p_retryTimeout )
{
+ timeout_t tm;
+ TimeoutInit ( & tm, p_retryTimeout );
+
KSocket* socket;
- THROW_ON_RC ( KNSManagerMakeRetryConnection(mgr, &socket, p_retryTimeout, NULL, &ep) );
+ THROW_ON_RC ( KNSManagerMakeRetryConnection(mgr, &socket, &tm, NULL, &ep) );
if (socket == 0)
throw logic_error ( "MakeStream: KStreamRelease failed" );
@@ -380,7 +390,7 @@ PROCESS_FIXTURE_TEST_CASE(IPCEndpoint_Basic, SocketFixture, 0, 5)
{ // client runs in a child process
string content = GetName();
- KStream* stream = MakeStream ( 50 ); /* this might make some retries while the server is setting up */
+ KStream* stream = MakeStream ( 50 * 1000 ); /* this might make some retries while the server is setting up */
LOG(LogLevel::e_message, "client '" << GetName() << "' after KNSMakeConnection" << endl);
REQUIRE_RC(KStreamWrite(stream, content.c_str(), content.length(), &num));
@@ -397,12 +407,12 @@ PROCESS_FIXTURE_TEST_CASE(IPCEndpoint_Basic, SocketFixture, 0, 5)
PROCESS_FIXTURE_TEST_CASE(IPCEndpoint_MultipleListeners, SocketFixture, 0, 100)
{ // client runs in a child process
- KStream* stream = MakeStream ( 50 ); /* this might make some retries while the server is setting up */
+ KStream* stream = MakeStream ( 50 * 1000 ); /* this might make some retries while the server is setting up */
LOG(LogLevel::e_message, "client '" << GetName() << "' after KNSMakeConnection1" << endl);
TestEnv::Sleep(1); // on Windows 32, when the two calls to KNSManagerMakeConnection follow too closely, sometimes things get messed up
- KStream* stream2 = MakeStream ( 5 ); /* should work from the first try now*/
+ KStream* stream2 = MakeStream ( 5 * 1000 ); /* should work from the first try now*/
LOG(LogLevel::e_message, "client '" << GetName() << "' after KNSMakeConnection2" << endl);
string content = string(GetName())+"_1";
@@ -431,7 +441,7 @@ PROCESS_FIXTURE_TEST_CASE(IPCEndpoint_ReadAll, SocketFixture, 0, 5)
{ // call ReadAll requesting more bytes than available, see it return only what is available
string content = GetName();
- KStream* stream = MakeStream ( 5 );
+ KStream* stream = MakeStream ( 5 * 1000 );
LOG(LogLevel::e_message, "client '" << GetName() << "' after KNSMakeConnection" << endl);
REQUIRE_RC(KStreamWrite(stream, content.c_str(), content.length(), &num));
@@ -477,7 +487,7 @@ public:
if (KNSManagerInitIPCEndpoint(mgr, &ep, &name) != 0)
throw logic_error ( string("TimedReadSocketFixture: SetupClient(") + p_content + "), KNSManagerInitIPCEndpoint failed" );
- m_stream = MakeStream ( 5 );
+ m_stream = MakeStream ( 5 * 1000 );
LOG(LogLevel::e_message, "client '" << p_content << "' after KNSMakeConnection" << endl);
}
void SetupClient(const string& p_content, uint32_t p_timeoutMs)
@@ -624,14 +634,17 @@ public:
if (KNSManagerInitIPCEndpoint(mgr, &ep, &name) != 0)
throw logic_error ( string("TimedConnection_ReadSocketFixture: SetupClient(") + p_content + "), KNSManagerInitIPCEndpoint failed" );
- m_stream = MakeStreamTimed( 5, p_readMillis, p_writeMillis );
+ m_stream = MakeStreamTimed( 5 * 1000, p_readMillis, p_writeMillis );
LOG(LogLevel::e_message, "client '" << p_content << "' after KNSMakeConnection" << endl);
}
KStream* MakeStreamTimed( int32_t p_retryTimeout, int32_t p_readMillis, int32_t p_writeMillis )
{
+ timeout_t tm;
+ TimeoutInit ( & tm, p_retryTimeout );
+
KSocket* socket;
- THROW_ON_RC ( KNSManagerMakeRetryTimedConnection(mgr, &socket, p_retryTimeout, p_readMillis, p_writeMillis, NULL, &ep) );
+ THROW_ON_RC ( KNSManagerMakeRetryTimedConnection(mgr, &socket, &tm, p_readMillis, p_writeMillis, NULL, &ep) );
if (socket == 0)
throw logic_error ( "MakeStreamTimed: KStreamRelease failed" );
@@ -716,7 +729,7 @@ PROCESS_FIXTURE_TEST_CASE(TimedConnection_ReadOverride_0_Timeout, TimedConnectio
}
PROCESS_FIXTURE_TEST_CASE(TimedConnection_SettingsOverride_0_Timeout, TimedConnection_ReadSocketFixture, 0, 20)
{ // 2.2.2 time out immediately when the server has not yet responded
- REQUIRE_RC(KNSManagerSetConnectionTimeouts(mgr, 5, 0, 0)); // override default setting (long time-out) to "no wait"
+ REQUIRE_RC(KNSManagerSetConnectionTimeouts(mgr, 5000, 0, 0)); // override default setting (long time-out) to "no wait"
string content = GetName();
TimedReadSocketFixture::SetupClient(content);
@@ -829,8 +842,8 @@ public:
if (KNSManagerInitIPCEndpoint(mgr, &ep, &name) != 0)
throw logic_error ( string("TimedWriteSocketFixture: SetupClient(") + p_name + "), KNSManagerInitIPCEndpoint failed" );
- m_data = MakeStream ( 5 );
- m_control = MakeStream ( 5 );
+ m_data = MakeStream ( 5 * 1000 );
+ m_control = MakeStream ( 5 * 1000 );
// identify data/control channels to the server
WriteMessage(m_data, "data");
@@ -1034,6 +1047,14 @@ PROCESS_FIXTURE_TEST_CASE(TimedWrite_NULL_Timeout, TimedWriteSocketFixture, 0, 2
// TODO: KStreamWriteExactly, KStreamTimedWriteExactly
//////////////////////////////////////////// Main
+
+static rc_t argsHandler(int argc, char * argv[]) {
+ Args * args = NULL;
+ rc_t rc = ArgsMakeAndHandle(&args, argc, argv, 0, NULL, 0);
+ ArgsWhack(args);
+ return rc;
+}
+
extern "C"
{
diff --git a/test/kns/test-proxy-with-env.cpp b/test/kns/test-proxy-with-env.cpp
new file mode 100644
index 0000000..1c67958
--- /dev/null
+++ b/test/kns/test-proxy-with-env.cpp
@@ -0,0 +1,112 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <kfs/directory.h> /* KDirectoryRelease */
+#include <ktst/unit_test.hpp> // TEST_SUITE
+
+#include "TestProxy.hpp" // TestProxy
+
+#include <fstream> // ifstream
+#include <sstream> // ostringstream
+
+using std :: ifstream;
+using std :: istringstream;
+using std :: ostringstream;
+using std :: string;
+
+TEST_SUITE ( ProxyWithEnvTestSuite );
+
+TEST_CASE ( TEST_PROXY_FROM_ENV ) {
+ ncbi :: NK :: TestRunner * t ( ncbi :: NK :: GetTestSuite () );
+ assert ( t );
+ REQUIRE_GT ( t -> argc, 1 );
+ assert ( t -> argv );
+ assert ( t -> argv [ 1 ] );
+ const char * testcase ( t -> argv [ 1 ] );
+ REQUIRE ( testcase );
+ C * c = NULL;
+ {
+ ostringstream s;
+ s << testcase << "/config";
+ ifstream myfile ( s . str () . c_str () );
+ string line;
+ while ( getline ( myfile, line) ) {
+ istringstream s ( line );
+ string path, value;
+ s >> path >> value;
+ if ( c == NULL ) {
+ c = new C ( path, value );
+ } else {
+ c -> add ( path, value );
+ }
+ }
+ }
+ E * e = NULL;
+ {
+ ostringstream s;
+ s << testcase << "/expected";
+ ifstream myfile ( s . str () . c_str () );
+ string line;
+ while ( getline ( myfile, line) ) {
+ istringstream s ( line );
+ string host;
+ int port;
+ s >> host >> port;
+ if ( e == NULL ) {
+ e = new E ( host, port );
+ } else {
+ e -> add ( host, port );
+ }
+ }
+ }
+ TestProxy ( this, c, e );
+ delete e;
+ e = NULL;
+ delete c;
+ c = NULL;
+}
+
+extern "C" {
+ ver_t CC KAppVersion ( void ) { return 0; }
+
+#define TODO -1
+
+ rc_t CC KMain ( int argc, char * argv [] ) {
+
+#if 0
+const char name[] = "http_proxy"; char *e= getenv(name); ostringstream s;
+s << "getenv("<<name<<")="<<(e==NULL?"NULL":e)<<"\n";std::cerr <<s.str();
+#endif
+
+ if ( argc == 0 ) {
+ return TODO;
+ }
+
+ KConfigDisableUserSettings ();
+
+ return ProxyWithEnvTestSuite ( argc, argv );
+ }
+}
diff --git a/test/kns/test-proxy.cpp b/test/kns/test-proxy.cpp
new file mode 100644
index 0000000..40a51ad
--- /dev/null
+++ b/test/kns/test-proxy.cpp
@@ -0,0 +1,263 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+
+#include <ktst/unit_test.hpp> // TEST_SUITE
+
+#include "TestProxy.hpp" // TestProxy
+
+#include <fstream> // ifstream
+#include <sstream> // ostringstream
+
+using std :: ostringstream;
+using std :: string;
+
+class TestRunner : private ncbi :: NK :: TestCase {
+ TestCase * _dad;
+
+public:
+ TestRunner ( TestCase * dad, const string & testcase )
+ : TestCase ( dad -> GetName () )
+ , _dad ( dad )
+ {
+ ostringstream cmd;
+
+ ostringstream s;
+ s << "test-proxy/" << testcase << "/environment";
+ std :: ifstream myfile ( s . str () . c_str () );
+ string line;
+ while ( getline ( myfile, line) ) {
+ cmd << "export " << line << "; ";
+ }
+
+ ncbi :: NK :: TestRunner * t ( ncbi :: NK :: GetTestSuite () );
+ assert ( t );
+ assert ( t -> argv );
+ assert ( t -> argv [ 0 ] );
+ const char * testBin ( t -> argv [ 0 ] );
+ const char * file = strrchr ( testBin, '/' );
+ cmd << string ( testBin, file - testBin )
+ << "/../bin/test-proxy-with-env test-proxy/" << testcase;
+
+//std::cerr<<cmd . str ()<<"\n";
+
+ REQUIRE_EQ ( system ( cmd . str (). c_str () ), 0 );
+ }
+
+ ~ TestRunner ( void )
+ {
+ assert ( _dad );
+ _dad -> ErrorCounterAdd ( GetErrorCounter () );
+ }
+};
+
+TEST_SUITE ( ProxyTestSuite );
+
+TEST_CASE ( TEST_NOPROXY ) { // no proxy
+ TestProxy ( this );
+}
+
+TEST_CASE ( TEST_PROXY_1PATH ) { // 1 proxy with port
+ C c ( "/http/proxy/path", "proxy.gov:7678");
+ E e ( "proxy.gov", 7678 );
+ TestProxy ( this, & c, & e );
+}
+
+TEST_CASE ( TEST_PROXY_1PATH_NoPort ) { // 1 proxy without port
+ C c ( "/http/proxy/path", "proxy.gov");
+ E e ( "proxy.gov", 0 );
+ TestProxy ( this, & c, & e );
+}
+
+#ifdef MULTIPLE_PROXIES
+TEST_CASE ( TEST_PROXY_2PATH ) { // 2 proxies with port
+ C c ( "/http/proxy/path", "proxy.gov:7678,proxy2.org:768");
+ E e ( "proxy.gov", 7678 );
+ e . add ( "proxy2.org", 768 );
+ TestProxy ( this, & c, & e );
+}
+TEST_CASE ( TEST_PROXY_2PATH_NoPort ) { // 2 proxies with/without port
+ C c ( "/http/proxy/path", "proxy.gov:7678,proxy2.org");
+ E e ( "proxy.gov", 7678 );
+ e . add ( "proxy2.org", 0 );
+ TestProxy ( this, & c, & e );
+}
+// 2 proxies from config: proxy only, no direct access
+TEST_CASE ( TEST_PROXY_onlyWithoutEnv ) {
+ C c (
+ "/http/proxy/path", "port.config.proxy.gov:678,no-port.config.proxy2.org");
+ c . add ( "/http/proxy/only", "true" );
+ E e ( "port.config.proxy.gov", 678 );
+ e . add ( "no-port.config.proxy2.org", 0 );
+ TestProxy ( this, & c, & e );
+}
+#endif
+
+// 1 proxy from config: proxy only, no direct access
+TEST_CASE ( TEST_PROXY_onlyWithoutEnv ) {
+ C c ( "/http/proxy/path", "port.config.proxy.gov:678" );
+ c . add ( "/http/proxy/only", "true" );
+ E e ( "port.config.proxy.gov", 678 );
+ TestProxy ( this, & c, & e );
+}
+
+// error in proxy configuration
+TEST_CASE ( TEST_PROXY_kfgErr ) {
+ C c ( "/http/proxy/path", "port.config.proxy.go:v678" );
+ TestProxy ( this, & c );
+}
+
+#ifndef MULTIPLE_PROXIES
+// comma in proxy configuration
+TEST_CASE ( TEST_PROXY_commaErr ) {
+ C c ( "/http/proxy/path",
+ "port.config.proxy.gov:678,port.config.proxy.gov:67" );
+ TestProxy ( this, & c );
+}
+#endif
+
+TEST_CASE ( TEST_PROXY_NOPROXY_NoENV ) { // no proxy from kfg, no environment
+ TestRunner ( this, "none" );
+}
+
+TEST_CASE ( TEST_PROXY_1PATH_NoENV ) { // 1 proxy from kfg, no environment
+ TestRunner ( this, "no-env" );
+}
+
+// 1 proxy without port, no environment
+TEST_CASE ( TEST_PROXY_1PATH_NoPort_NoENV ) {
+ TestRunner ( this, "no-port-no-env" );
+}
+
+#ifdef MULTIPLE_PROXIES
+TEST_CASE ( TEST_PROXY_2PATH_NoENV ) { // 2 proxies with port, no environment
+ TestRunner ( this, "2-path-no-env" );
+}
+// 2 proxies with/without port, no environment
+TEST_CASE ( TEST_PROXY_2PATH_NoPort_NoENV ) {
+ TestRunner ( this, "2-path-no-port-no-env" );
+}
+#endif
+
+TEST_CASE ( TEST_PROXY_http_proxy ) { // http_proxy from environment
+ TestRunner ( this, "http_proxy" );
+}
+
+#ifdef MULTIPLE_PROXIES // 2 proxies from config, 3 environment variables
+#endif
+TEST_CASE ( TEST_PROXY_all ) { // 1 proxy from config, 3 environment variables
+ TestRunner ( this, "all" );
+}
+
+#ifdef MULTIPLE_PROXIES
+// 2 proxies from config, 3 environment variables : ignore config
+#endif
+// 1 proxy from config, 3 environment variables : ignore config
+TEST_CASE ( TEST_PROXY_env_only ) {
+ TestRunner ( this, "env-only" );
+}
+
+// 1 proxy from config, 3 environment variables : ignore environment
+TEST_CASE ( TEST_PROXY_kfg_only ) {
+ TestRunner ( this, "kfg-only" );
+}
+
+#ifdef MULTIPLE_PROXIES
+// 2 proxies from config, 3 environment variables : config, then environment
+#endif
+// 1 proxy from config, 3 environment variables : config, then environment
+TEST_CASE ( TEST_PROXY_kfg_env ) {
+ TestRunner ( this, "kfg-env" );
+}
+
+#ifdef MULTIPLE_PROXIES
+// 2 proxies from config, 3 environment variables : environment, then config
+#endif
+// 1 proxy from config, 3 environment variables : environment, then config
+TEST_CASE ( TEST_PROXY_env_kfg ) {
+ TestRunner ( this, "env-kfg" );
+}
+
+#ifdef MULTIPLE_PROXIES
+// 2 proxies from config, 3 environment variables : ignore all
+#endif
+// 1 proxy from config, 3 environment variables : ignore all
+TEST_CASE ( TEST_PROXY_ignore ) {
+ TestRunner ( this, "ignore" );
+}
+
+#ifdef MULTIPLE_PROXIES
+// 2 proxies from config, 3 environment variables : proxy/enabled = true
+#endif
+// 1 proxy from config, 3 environment variables : proxy/enabled = true
+TEST_CASE ( TEST_PROXY_enabled ) {
+ TestRunner ( this, "enabled" );
+}
+
+#ifdef MULTIPLE_PROXIES
+// 2 proxies from config, 3 environment variables : proxy/enabled = false
+#endif
+// 1 proxy from config, 3 environment variables : proxy/enabled = false
+TEST_CASE ( TEST_PROXY_disabled ) {
+ TestRunner ( this, "disabled" );
+}
+
+#ifdef MULTIPLE_PROXIES
+// 2 proxies from config, 3 environment variables : proxy only, no direct access
+#endif
+// 1 proxy from config, 3 environment variables : proxy only, no direct access
+TEST_CASE ( TEST_PROXY_onlyWithEnv ) {
+ TestRunner ( this, "proxy-only" );
+}
+
+#ifdef MULTIPLE_PROXIES
+// 2 proxies from config, 3 environment variables : proxy-only=false
+#endif
+// 1 proxy from config, 3 environment variables : proxy-only=false
+TEST_CASE ( TEST_PROXY_notOnlyWithEnv ) {
+ TestRunner ( this, "proxy-not-only" );
+}
+
+// error in congiguration and environment
+TEST_CASE ( TEST_PROXY_error ) {
+ TestRunner ( this, "error" );
+}
+
+#ifndef MULTIPLE_PROXIES
+// comma in proxy configuration
+TEST_CASE ( TEST_PROXY_ENV_commaErr ) {
+ TestRunner ( this, "comma-error" );
+}
+#endif
+
+extern "C" {
+ ver_t CC KAppVersion ( void ) { return 0; }
+ rc_t CC KMain ( int argc, char * argv [] ) {
+ KConfigDisableUserSettings ();
+ return ProxyTestSuite ( argc, argv );
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
diff --git a/test/kns/test-proxy.h b/test/kns/test-proxy.h
new file mode 100644
index 0000000..c741593
--- /dev/null
+++ b/test/kns/test-proxy.h
@@ -0,0 +1,7 @@
+extern "C" {
+ struct KEndPointArgsIterator * KNSManagerMakeKEndPointArgsIterator
+ ( const KNSManager * self, const String * hostname, uint32_t port );
+ bool KEndPointArgsIterator_Next ( KEndPointArgsIterator * self,
+ const String ** hostname, uint16_t * port,
+ bool * proxy_default_port, bool * proxy_ep );
+}
diff --git a/test/kns/test-proxy/2-path-no-env/config b/test/kns/test-proxy/2-path-no-env/config
new file mode 100644
index 0000000..2ad12c4
--- /dev/null
+++ b/test/kns/test-proxy/2-path-no-env/config
@@ -0,0 +1 @@
+/http/proxy/path proxy.gov:7678,proxy2.org:768
diff --git a/test/kns/test-proxy/2-path-no-env/expected b/test/kns/test-proxy/2-path-no-env/expected
new file mode 100644
index 0000000..66ef788
--- /dev/null
+++ b/test/kns/test-proxy/2-path-no-env/expected
@@ -0,0 +1,2 @@
+proxy.gov 7678
+proxy2.org 768
diff --git a/test/kns/test-proxy/2-path-no-port-no-env/config b/test/kns/test-proxy/2-path-no-port-no-env/config
new file mode 100644
index 0000000..a8dc289
--- /dev/null
+++ b/test/kns/test-proxy/2-path-no-port-no-env/config
@@ -0,0 +1 @@
+/http/proxy/path proxy.gov:7678,proxy2.org
diff --git a/test/kns/test-proxy/2-path-no-port-no-env/expected b/test/kns/test-proxy/2-path-no-port-no-env/expected
new file mode 100644
index 0000000..00e19a0
--- /dev/null
+++ b/test/kns/test-proxy/2-path-no-port-no-env/expected
@@ -0,0 +1,2 @@
+proxy.gov 7678
+proxy2.org 0
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/config b/test/kns/test-proxy/MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/config
new file mode 100644
index 0000000..f817da5
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/config
@@ -0,0 +1 @@
+/http/proxy/path port.config.proxy.gov:678,no-port.config.proxy2.org
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/environment b/test/kns/test-proxy/MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/environment
new file mode 100644
index 0000000..2d1acba
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234,no-port.http.proxy
+HTTP_PROXY=no-port.HTTP.proxy,port.HTTP.proxy:567
+all_proxy=no-port.all.proxy,port.all.proxy:89
+ALL_PROXY=port.ALL.proxy:9,no-port.ALL.proxy
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/expected b/test/kns/test-proxy/MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/expected
new file mode 100644
index 0000000..cf902ec
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/all.MULTIPLE_PROXIES/expected
@@ -0,0 +1,10 @@
+port.config.proxy.gov 678
+no-port.config.proxy2.org 0
+port.http.proxy 1234
+no-port.http.proxy 0
+no-port.HTTP.proxy 0
+port.HTTP.proxy 567
+no-port.all.proxy 0
+port.all.proxy 89
+port.ALL.proxy 9
+no-port.ALL.proxy 0
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/disabled.MULTIPLE_PROXIES/config b/test/kns/test-proxy/MULTIPLE_PROXIES/disabled.MULTIPLE_PROXIES/config
new file mode 100644
index 0000000..a8ef730
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/disabled.MULTIPLE_PROXIES/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678,no-port.config.proxy2.org
+/http/proxy/enabled false
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/disabled.MULTIPLE_PROXIES/environment b/test/kns/test-proxy/MULTIPLE_PROXIES/disabled.MULTIPLE_PROXIES/environment
new file mode 100644
index 0000000..2d1acba
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/disabled.MULTIPLE_PROXIES/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234,no-port.http.proxy
+HTTP_PROXY=no-port.HTTP.proxy,port.HTTP.proxy:567
+all_proxy=no-port.all.proxy,port.all.proxy:89
+ALL_PROXY=port.ALL.proxy:9,no-port.ALL.proxy
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/disabled.MULTIPLE_PROXIES/expected b/test/kns/test-proxy/MULTIPLE_PROXIES/disabled.MULTIPLE_PROXIES/expected
new file mode 100644
index 0000000..e69de29
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/enabled.MULTIPLE_PROXIES/config b/test/kns/test-proxy/MULTIPLE_PROXIES/enabled.MULTIPLE_PROXIES/config
new file mode 100644
index 0000000..4ecdb8d
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/enabled.MULTIPLE_PROXIES/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678,no-port.config.proxy2.org
+/http/proxy/enabled true
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/enabled.MULTIPLE_PROXIES/environment b/test/kns/test-proxy/MULTIPLE_PROXIES/enabled.MULTIPLE_PROXIES/environment
new file mode 100644
index 0000000..2d1acba
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/enabled.MULTIPLE_PROXIES/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234,no-port.http.proxy
+HTTP_PROXY=no-port.HTTP.proxy,port.HTTP.proxy:567
+all_proxy=no-port.all.proxy,port.all.proxy:89
+ALL_PROXY=port.ALL.proxy:9,no-port.ALL.proxy
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/enabled.MULTIPLE_PROXIES/expected b/test/kns/test-proxy/MULTIPLE_PROXIES/enabled.MULTIPLE_PROXIES/expected
new file mode 100644
index 0000000..cf902ec
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/enabled.MULTIPLE_PROXIES/expected
@@ -0,0 +1,10 @@
+port.config.proxy.gov 678
+no-port.config.proxy2.org 0
+port.http.proxy 1234
+no-port.http.proxy 0
+no-port.HTTP.proxy 0
+port.HTTP.proxy 567
+no-port.all.proxy 0
+port.all.proxy 89
+port.ALL.proxy 9
+no-port.ALL.proxy 0
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/env-kfg.MULTIPLE_PROXIES/config b/test/kns/test-proxy/MULTIPLE_PROXIES/env-kfg.MULTIPLE_PROXIES/config
new file mode 100644
index 0000000..441f697
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/env-kfg.MULTIPLE_PROXIES/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678,no-port.config.proxy2.org
+/http/proxy/use env,kfg
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/env-kfg.MULTIPLE_PROXIES/environment b/test/kns/test-proxy/MULTIPLE_PROXIES/env-kfg.MULTIPLE_PROXIES/environment
new file mode 100644
index 0000000..2d1acba
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/env-kfg.MULTIPLE_PROXIES/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234,no-port.http.proxy
+HTTP_PROXY=no-port.HTTP.proxy,port.HTTP.proxy:567
+all_proxy=no-port.all.proxy,port.all.proxy:89
+ALL_PROXY=port.ALL.proxy:9,no-port.ALL.proxy
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/env-kfg.MULTIPLE_PROXIES/expected b/test/kns/test-proxy/MULTIPLE_PROXIES/env-kfg.MULTIPLE_PROXIES/expected
new file mode 100644
index 0000000..517bb51
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/env-kfg.MULTIPLE_PROXIES/expected
@@ -0,0 +1,10 @@
+port.http.proxy 1234
+no-port.http.proxy 0
+no-port.HTTP.proxy 0
+port.HTTP.proxy 567
+no-port.all.proxy 0
+port.all.proxy 89
+port.ALL.proxy 9
+no-port.ALL.proxy 0
+port.config.proxy.gov 678
+no-port.config.proxy2.org 0
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/env-only.MULTIPLE_PROXIES/config b/test/kns/test-proxy/MULTIPLE_PROXIES/env-only.MULTIPLE_PROXIES/config
new file mode 100644
index 0000000..cd47172
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/env-only.MULTIPLE_PROXIES/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678,no-port.config.proxy2.org
+/http/proxy/use env
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/env-only.MULTIPLE_PROXIES/environment b/test/kns/test-proxy/MULTIPLE_PROXIES/env-only.MULTIPLE_PROXIES/environment
new file mode 100644
index 0000000..2d1acba
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/env-only.MULTIPLE_PROXIES/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234,no-port.http.proxy
+HTTP_PROXY=no-port.HTTP.proxy,port.HTTP.proxy:567
+all_proxy=no-port.all.proxy,port.all.proxy:89
+ALL_PROXY=port.ALL.proxy:9,no-port.ALL.proxy
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/env-only.MULTIPLE_PROXIES/expected b/test/kns/test-proxy/MULTIPLE_PROXIES/env-only.MULTIPLE_PROXIES/expected
new file mode 100644
index 0000000..4c8213d
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/env-only.MULTIPLE_PROXIES/expected
@@ -0,0 +1,8 @@
+port.http.proxy 1234
+no-port.http.proxy 0
+no-port.HTTP.proxy 0
+port.HTTP.proxy 567
+no-port.all.proxy 0
+port.all.proxy 89
+port.ALL.proxy 9
+no-port.ALL.proxy 0
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/ignore.MULTIPLE_PROXIES/config b/test/kns/test-proxy/MULTIPLE_PROXIES/ignore.MULTIPLE_PROXIES/config
new file mode 100644
index 0000000..acbb179
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/ignore.MULTIPLE_PROXIES/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678,no-port.config.proxy2.org
+/http/proxy/use none
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/ignore.MULTIPLE_PROXIES/environment b/test/kns/test-proxy/MULTIPLE_PROXIES/ignore.MULTIPLE_PROXIES/environment
new file mode 100644
index 0000000..2d1acba
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/ignore.MULTIPLE_PROXIES/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234,no-port.http.proxy
+HTTP_PROXY=no-port.HTTP.proxy,port.HTTP.proxy:567
+all_proxy=no-port.all.proxy,port.all.proxy:89
+ALL_PROXY=port.ALL.proxy:9,no-port.ALL.proxy
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/ignore.MULTIPLE_PROXIES/expected b/test/kns/test-proxy/MULTIPLE_PROXIES/ignore.MULTIPLE_PROXIES/expected
new file mode 100644
index 0000000..e69de29
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/kfg-env.MULTIPLE_PROXIES/config b/test/kns/test-proxy/MULTIPLE_PROXIES/kfg-env.MULTIPLE_PROXIES/config
new file mode 100644
index 0000000..885472e
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/kfg-env.MULTIPLE_PROXIES/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678,no-port.config.proxy2.org
+/http/proxy/use kfg,env
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/kfg-env.MULTIPLE_PROXIES/environment b/test/kns/test-proxy/MULTIPLE_PROXIES/kfg-env.MULTIPLE_PROXIES/environment
new file mode 100644
index 0000000..2d1acba
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/kfg-env.MULTIPLE_PROXIES/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234,no-port.http.proxy
+HTTP_PROXY=no-port.HTTP.proxy,port.HTTP.proxy:567
+all_proxy=no-port.all.proxy,port.all.proxy:89
+ALL_PROXY=port.ALL.proxy:9,no-port.ALL.proxy
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/kfg-env.MULTIPLE_PROXIES/expected b/test/kns/test-proxy/MULTIPLE_PROXIES/kfg-env.MULTIPLE_PROXIES/expected
new file mode 100644
index 0000000..cf902ec
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/kfg-env.MULTIPLE_PROXIES/expected
@@ -0,0 +1,10 @@
+port.config.proxy.gov 678
+no-port.config.proxy2.org 0
+port.http.proxy 1234
+no-port.http.proxy 0
+no-port.HTTP.proxy 0
+port.HTTP.proxy 567
+no-port.all.proxy 0
+port.all.proxy 89
+port.ALL.proxy 9
+no-port.ALL.proxy 0
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-not-only.MULTIPLE_PROXIES/config b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-not-only.MULTIPLE_PROXIES/config
new file mode 100644
index 0000000..d82b7f5
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-not-only.MULTIPLE_PROXIES/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678,no-port.config.proxy2.org
+/http/proxy/only false
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-not-only.MULTIPLE_PROXIES/environment b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-not-only.MULTIPLE_PROXIES/environment
new file mode 100644
index 0000000..2d1acba
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-not-only.MULTIPLE_PROXIES/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234,no-port.http.proxy
+HTTP_PROXY=no-port.HTTP.proxy,port.HTTP.proxy:567
+all_proxy=no-port.all.proxy,port.all.proxy:89
+ALL_PROXY=port.ALL.proxy:9,no-port.ALL.proxy
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-not-only.MULTIPLE_PROXIES/expected b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-not-only.MULTIPLE_PROXIES/expected
new file mode 100644
index 0000000..cf902ec
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-not-only.MULTIPLE_PROXIES/expected
@@ -0,0 +1,10 @@
+port.config.proxy.gov 678
+no-port.config.proxy2.org 0
+port.http.proxy 1234
+no-port.http.proxy 0
+no-port.HTTP.proxy 0
+port.HTTP.proxy 567
+no-port.all.proxy 0
+port.all.proxy 89
+port.ALL.proxy 9
+no-port.ALL.proxy 0
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-only.MILTIPLE_PROXIES/config b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-only.MILTIPLE_PROXIES/config
new file mode 100644
index 0000000..f4072e5
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-only.MILTIPLE_PROXIES/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678,no-port.config.proxy2.org
+/http/proxy/only true
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-only.MILTIPLE_PROXIES/environment b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-only.MILTIPLE_PROXIES/environment
new file mode 100644
index 0000000..2d1acba
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-only.MILTIPLE_PROXIES/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234,no-port.http.proxy
+HTTP_PROXY=no-port.HTTP.proxy,port.HTTP.proxy:567
+all_proxy=no-port.all.proxy,port.all.proxy:89
+ALL_PROXY=port.ALL.proxy:9,no-port.ALL.proxy
diff --git a/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-only.MILTIPLE_PROXIES/expected b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-only.MILTIPLE_PROXIES/expected
new file mode 100644
index 0000000..cf902ec
--- /dev/null
+++ b/test/kns/test-proxy/MULTIPLE_PROXIES/proxy-only.MILTIPLE_PROXIES/expected
@@ -0,0 +1,10 @@
+port.config.proxy.gov 678
+no-port.config.proxy2.org 0
+port.http.proxy 1234
+no-port.http.proxy 0
+no-port.HTTP.proxy 0
+port.HTTP.proxy 567
+no-port.all.proxy 0
+port.all.proxy 89
+port.ALL.proxy 9
+no-port.ALL.proxy 0
diff --git a/test/kns/test-proxy/all/config b/test/kns/test-proxy/all/config
new file mode 100644
index 0000000..f9c2ef3
--- /dev/null
+++ b/test/kns/test-proxy/all/config
@@ -0,0 +1 @@
+/http/proxy/path no-port.config.proxy2.org
diff --git a/test/kns/test-proxy/all/environment b/test/kns/test-proxy/all/environment
new file mode 100644
index 0000000..43e6af5
--- /dev/null
+++ b/test/kns/test-proxy/all/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234
+HTTP_PROXY=no-port.HTTP.proxy
+all_proxy=no-port.all.proxy
+ALL_PROXY=port.ALL.proxy:9
diff --git a/test/kns/test-proxy/all/expected b/test/kns/test-proxy/all/expected
new file mode 100644
index 0000000..0032912
--- /dev/null
+++ b/test/kns/test-proxy/all/expected
@@ -0,0 +1,5 @@
+no-port.config.proxy2.org 0
+no-port.all.proxy 0
+port.ALL.proxy 9
+port.http.proxy 1234
+no-port.HTTP.proxy 0
diff --git a/test/kns/test-proxy/comma-error/environment b/test/kns/test-proxy/comma-error/environment
new file mode 100644
index 0000000..4123a89
--- /dev/null
+++ b/test/kns/test-proxy/comma-error/environment
@@ -0,0 +1 @@
+http_proxy=port.http.proxy:1234,port.http.proxy:123
diff --git a/test/kns/test-proxy/comma-error/expected b/test/kns/test-proxy/comma-error/expected
new file mode 100644
index 0000000..e69de29
diff --git a/test/kns/test-proxy/disabled/config b/test/kns/test-proxy/disabled/config
new file mode 100644
index 0000000..469da64
--- /dev/null
+++ b/test/kns/test-proxy/disabled/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678
+/http/proxy/enabled false
diff --git a/test/kns/test-proxy/disabled/environment b/test/kns/test-proxy/disabled/environment
new file mode 100644
index 0000000..43e6af5
--- /dev/null
+++ b/test/kns/test-proxy/disabled/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234
+HTTP_PROXY=no-port.HTTP.proxy
+all_proxy=no-port.all.proxy
+ALL_PROXY=port.ALL.proxy:9
diff --git a/test/kns/test-proxy/disabled/expected b/test/kns/test-proxy/disabled/expected
new file mode 100644
index 0000000..e69de29
diff --git a/test/kns/test-proxy/enabled/config b/test/kns/test-proxy/enabled/config
new file mode 100644
index 0000000..f6a15db
--- /dev/null
+++ b/test/kns/test-proxy/enabled/config
@@ -0,0 +1,2 @@
+/http/proxy/path no-port.config.proxy2.org
+/http/proxy/enabled true
diff --git a/test/kns/test-proxy/enabled/environment b/test/kns/test-proxy/enabled/environment
new file mode 100644
index 0000000..0669d2f
--- /dev/null
+++ b/test/kns/test-proxy/enabled/environment
@@ -0,0 +1,4 @@
+http_proxy=no-port.http.proxy
+HTTP_PROXY=port.HTTP.proxy:567
+all_proxy=port.all.proxy:89
+ALL_PROXY=no-port.ALL.proxy
diff --git a/test/kns/test-proxy/enabled/expected b/test/kns/test-proxy/enabled/expected
new file mode 100644
index 0000000..81c1f8c
--- /dev/null
+++ b/test/kns/test-proxy/enabled/expected
@@ -0,0 +1,5 @@
+no-port.config.proxy2.org 0
+port.all.proxy 89
+no-port.ALL.proxy 0
+no-port.http.proxy 0
+port.HTTP.proxy 567
diff --git a/test/kns/test-proxy/env-kfg/config b/test/kns/test-proxy/env-kfg/config
new file mode 100644
index 0000000..f29770e
--- /dev/null
+++ b/test/kns/test-proxy/env-kfg/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678
+/http/proxy/use env,kfg
diff --git a/test/kns/test-proxy/env-kfg/environment b/test/kns/test-proxy/env-kfg/environment
new file mode 100644
index 0000000..43e6af5
--- /dev/null
+++ b/test/kns/test-proxy/env-kfg/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234
+HTTP_PROXY=no-port.HTTP.proxy
+all_proxy=no-port.all.proxy
+ALL_PROXY=port.ALL.proxy:9
diff --git a/test/kns/test-proxy/env-kfg/expected b/test/kns/test-proxy/env-kfg/expected
new file mode 100644
index 0000000..d1c9aeb
--- /dev/null
+++ b/test/kns/test-proxy/env-kfg/expected
@@ -0,0 +1,5 @@
+no-port.all.proxy 0
+port.ALL.proxy 9
+port.http.proxy 1234
+no-port.HTTP.proxy 0
+port.config.proxy.gov 678
diff --git a/test/kns/test-proxy/env-only/config b/test/kns/test-proxy/env-only/config
new file mode 100644
index 0000000..ca70ec4
--- /dev/null
+++ b/test/kns/test-proxy/env-only/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678
+/http/proxy/use env
diff --git a/test/kns/test-proxy/env-only/environment b/test/kns/test-proxy/env-only/environment
new file mode 100644
index 0000000..0669d2f
--- /dev/null
+++ b/test/kns/test-proxy/env-only/environment
@@ -0,0 +1,4 @@
+http_proxy=no-port.http.proxy
+HTTP_PROXY=port.HTTP.proxy:567
+all_proxy=port.all.proxy:89
+ALL_PROXY=no-port.ALL.proxy
diff --git a/test/kns/test-proxy/env-only/expected b/test/kns/test-proxy/env-only/expected
new file mode 100644
index 0000000..d153dfb
--- /dev/null
+++ b/test/kns/test-proxy/env-only/expected
@@ -0,0 +1,4 @@
+port.all.proxy 89
+no-port.ALL.proxy 0
+no-port.http.proxy 0
+port.HTTP.proxy 567
diff --git a/test/kns/test-proxy/error/config b/test/kns/test-proxy/error/config
new file mode 100644
index 0000000..f2351ca
--- /dev/null
+++ b/test/kns/test-proxy/error/config
@@ -0,0 +1 @@
+/http/proxy/path no-port.config.proxy2:org
diff --git a/test/kns/test-proxy/error/environment b/test/kns/test-proxy/error/environment
new file mode 100644
index 0000000..e9b3a52
--- /dev/null
+++ b/test/kns/test-proxy/error/environment
@@ -0,0 +1 @@
+http_proxy=port.http.prox:y1234
diff --git a/test/kns/test-proxy/error/expected b/test/kns/test-proxy/error/expected
new file mode 100644
index 0000000..e69de29
diff --git a/test/kns/test-proxy/http_proxy/environment b/test/kns/test-proxy/http_proxy/environment
new file mode 100644
index 0000000..fc650ec
--- /dev/null
+++ b/test/kns/test-proxy/http_proxy/environment
@@ -0,0 +1 @@
+http_proxy=http.proxy:1234
diff --git a/test/kns/test-proxy/http_proxy/expected b/test/kns/test-proxy/http_proxy/expected
new file mode 100644
index 0000000..33af3c4
--- /dev/null
+++ b/test/kns/test-proxy/http_proxy/expected
@@ -0,0 +1 @@
+http.proxy 1234
diff --git a/test/kns/test-proxy/ignore/config b/test/kns/test-proxy/ignore/config
new file mode 100644
index 0000000..09fa173
--- /dev/null
+++ b/test/kns/test-proxy/ignore/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678
+/http/proxy/use none
diff --git a/test/kns/test-proxy/ignore/environment b/test/kns/test-proxy/ignore/environment
new file mode 100644
index 0000000..43e6af5
--- /dev/null
+++ b/test/kns/test-proxy/ignore/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234
+HTTP_PROXY=no-port.HTTP.proxy
+all_proxy=no-port.all.proxy
+ALL_PROXY=port.ALL.proxy:9
diff --git a/test/kns/test-proxy/ignore/expected b/test/kns/test-proxy/ignore/expected
new file mode 100644
index 0000000..e69de29
diff --git a/test/kns/test-proxy/kfg-env/config b/test/kns/test-proxy/kfg-env/config
new file mode 100644
index 0000000..ce5350f
--- /dev/null
+++ b/test/kns/test-proxy/kfg-env/config
@@ -0,0 +1,2 @@
+/http/proxy/path no-port.config.proxy2.org
+/http/proxy/use kfg,env
diff --git a/test/kns/test-proxy/kfg-env/environment b/test/kns/test-proxy/kfg-env/environment
new file mode 100644
index 0000000..43e6af5
--- /dev/null
+++ b/test/kns/test-proxy/kfg-env/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234
+HTTP_PROXY=no-port.HTTP.proxy
+all_proxy=no-port.all.proxy
+ALL_PROXY=port.ALL.proxy:9
diff --git a/test/kns/test-proxy/kfg-env/expected b/test/kns/test-proxy/kfg-env/expected
new file mode 100644
index 0000000..0032912
--- /dev/null
+++ b/test/kns/test-proxy/kfg-env/expected
@@ -0,0 +1,5 @@
+no-port.config.proxy2.org 0
+no-port.all.proxy 0
+port.ALL.proxy 9
+port.http.proxy 1234
+no-port.HTTP.proxy 0
diff --git a/test/kns/test-proxy/kfg-only/config b/test/kns/test-proxy/kfg-only/config
new file mode 100644
index 0000000..dcf2a5b
--- /dev/null
+++ b/test/kns/test-proxy/kfg-only/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678
+/http/proxy/use kfg
diff --git a/test/kns/test-proxy/kfg-only/environment b/test/kns/test-proxy/kfg-only/environment
new file mode 100644
index 0000000..43e6af5
--- /dev/null
+++ b/test/kns/test-proxy/kfg-only/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234
+HTTP_PROXY=no-port.HTTP.proxy
+all_proxy=no-port.all.proxy
+ALL_PROXY=port.ALL.proxy:9
diff --git a/test/kns/test-proxy/kfg-only/expected b/test/kns/test-proxy/kfg-only/expected
new file mode 100644
index 0000000..59670de
--- /dev/null
+++ b/test/kns/test-proxy/kfg-only/expected
@@ -0,0 +1 @@
+port.config.proxy.gov 678
diff --git a/test/kns/test-proxy/no-env/config b/test/kns/test-proxy/no-env/config
new file mode 100644
index 0000000..8c4a115
--- /dev/null
+++ b/test/kns/test-proxy/no-env/config
@@ -0,0 +1 @@
+/http/proxy/path proxy.gov:7678
diff --git a/test/kns/test-proxy/no-env/expected b/test/kns/test-proxy/no-env/expected
new file mode 100644
index 0000000..ac57896
--- /dev/null
+++ b/test/kns/test-proxy/no-env/expected
@@ -0,0 +1 @@
+proxy.gov 7678
\ No newline at end of file
diff --git a/test/kns/test-proxy/no-port-no-env/config b/test/kns/test-proxy/no-port-no-env/config
new file mode 100644
index 0000000..d85f020
--- /dev/null
+++ b/test/kns/test-proxy/no-port-no-env/config
@@ -0,0 +1 @@
+/http/proxy/path proxy.gov
diff --git a/test/kns/test-proxy/no-port-no-env/expected b/test/kns/test-proxy/no-port-no-env/expected
new file mode 100644
index 0000000..a6f13ab
--- /dev/null
+++ b/test/kns/test-proxy/no-port-no-env/expected
@@ -0,0 +1 @@
+proxy.gov 0
\ No newline at end of file
diff --git a/test/kns/test-proxy/proxy-not-only/config b/test/kns/test-proxy/proxy-not-only/config
new file mode 100644
index 0000000..41ca345
--- /dev/null
+++ b/test/kns/test-proxy/proxy-not-only/config
@@ -0,0 +1,2 @@
+/http/proxy/path no-port.config.proxy2.org
+/http/proxy/only false
diff --git a/test/kns/test-proxy/proxy-not-only/environment b/test/kns/test-proxy/proxy-not-only/environment
new file mode 100644
index 0000000..0669d2f
--- /dev/null
+++ b/test/kns/test-proxy/proxy-not-only/environment
@@ -0,0 +1,4 @@
+http_proxy=no-port.http.proxy
+HTTP_PROXY=port.HTTP.proxy:567
+all_proxy=port.all.proxy:89
+ALL_PROXY=no-port.ALL.proxy
diff --git a/test/kns/test-proxy/proxy-not-only/expected b/test/kns/test-proxy/proxy-not-only/expected
new file mode 100644
index 0000000..81c1f8c
--- /dev/null
+++ b/test/kns/test-proxy/proxy-not-only/expected
@@ -0,0 +1,5 @@
+no-port.config.proxy2.org 0
+port.all.proxy 89
+no-port.ALL.proxy 0
+no-port.http.proxy 0
+port.HTTP.proxy 567
diff --git a/test/kns/test-proxy/proxy-only/config b/test/kns/test-proxy/proxy-only/config
new file mode 100644
index 0000000..204cdd1
--- /dev/null
+++ b/test/kns/test-proxy/proxy-only/config
@@ -0,0 +1,2 @@
+/http/proxy/path port.config.proxy.gov:678
+/http/proxy/only true
diff --git a/test/kns/test-proxy/proxy-only/environment b/test/kns/test-proxy/proxy-only/environment
new file mode 100644
index 0000000..43e6af5
--- /dev/null
+++ b/test/kns/test-proxy/proxy-only/environment
@@ -0,0 +1,4 @@
+http_proxy=port.http.proxy:1234
+HTTP_PROXY=no-port.HTTP.proxy
+all_proxy=no-port.all.proxy
+ALL_PROXY=port.ALL.proxy:9
diff --git a/test/kns/test-proxy/proxy-only/expected b/test/kns/test-proxy/proxy-only/expected
new file mode 100644
index 0000000..bcb6093
--- /dev/null
+++ b/test/kns/test-proxy/proxy-only/expected
@@ -0,0 +1,5 @@
+port.config.proxy.gov 678
+no-port.all.proxy 0
+port.ALL.proxy 9
+port.http.proxy 1234
+no-port.HTTP.proxy 0
diff --git a/test/kproc/kproctest.cpp b/test/kproc/kproctest.cpp
index 5527c30..c934da9 100644
--- a/test/kproc/kproctest.cpp
+++ b/test/kproc/kproctest.cpp
@@ -523,6 +523,117 @@ TEST_CASE( KCondition_MakeRelease )
REQUIRE_RC(KConditionRelease(cond));
}
+class KConditionFixture
+{
+public:
+ KConditionFixture()
+ : threadRc(0),
+ thread(0),
+ lock(0),
+ is_signaled(false),
+ do_broadcast(false)
+ {
+ if (KLockMake(&lock) != 0)
+ throw logic_error("KConditionFixture: KLockMake failed");
+ if (KConditionMake(&cond) != 0)
+ throw logic_error("KConditionFixture: KConditionMake failed");
+ }
+ ~KConditionFixture()
+ {
+ if (thread != 0)
+ {
+ if (KThreadWait(thread, NULL) != 0)
+ throw logic_error("~KConditionFixture: KThreadWait failed");
+ if (threadRc != 0)
+ throw logic_error("~KConditionFixture: thread failed, threadRc != 0");
+ if (KThreadRelease(thread) != 0)
+ throw logic_error("~KConditionFixture: KThreadRelease failed");
+ }
+ if (KLockRelease((const KLock*)lock) != 0)
+ throw logic_error("~KConditionFixture: KLockRelease failed");
+ if (KConditionRelease(cond) != 0)
+ throw logic_error("~KConditionFixture: KConditionRelease failed");
+ }
+
+protected:
+ class Thread {
+ public:
+ // danger - this should be an extern "C" function
+ // with CC calling convention on Windows
+ static rc_t KCondition_ThreadFn ( const KThread *thread, void *data )
+ {
+ KConditionFixture* self = (KConditionFixture*)data;
+
+ LOG(LogLevel::e_message, "KCondition_ThreadFn: sleeping" << endl);
+ TestEnv::SleepMs(300);
+ LOG(LogLevel::e_message, "KCondition_ThreadFn: signaling condition" << endl);
+ self->is_signaled = true;
+ if (!self->do_broadcast)
+ self->threadRc = KConditionSignal(self->cond);
+ else
+ self->threadRc = KConditionBroadcast(self->cond);
+
+ LOG(LogLevel::e_message, "KCondition_ThreadFn: exiting" << endl);
+ return 0;
+ }
+ };
+
+ rc_t StartThread()
+ {
+ LOG(LogLevel::e_message, "StartThread: starting thread" << endl);
+
+ threadRc = 0;
+ rc_t rc = KThreadMake(&thread, Thread::KCondition_ThreadFn, this);
+ return rc;
+ }
+
+public:
+ rc_t threadRc;
+ KThread* thread;
+ timeout_t tm;
+ KLock* lock;
+ KCondition* cond;
+ bool is_signaled;
+ bool do_broadcast;
+};
+
+FIXTURE_TEST_CASE( KCondition_TimedWait_Timeout, KConditionFixture )
+{
+ REQUIRE_RC(KLockAcquire(lock));
+ REQUIRE_RC(TimeoutInit(&tm, 100));
+ REQUIRE_RC(KConditionSignal(cond)); // signaling before waiting should not do anything
+ REQUIRE_EQ(KConditionTimedWait(cond, lock, &tm), RC ( rcPS, rcCondition, rcWaiting, rcTimeout, rcExhausted )); // timed out
+
+ REQUIRE_RC(KLockUnlock(lock));
+}
+
+FIXTURE_TEST_CASE( KCondition_TimedWait_Signaled, KConditionFixture )
+{
+ is_signaled = false;
+
+ REQUIRE_RC(KLockAcquire(lock));
+
+ REQUIRE_RC(StartThread());
+ REQUIRE_RC(KConditionWait(cond, lock));
+ REQUIRE(is_signaled == true);
+
+ REQUIRE_RC(KLockUnlock(lock));
+}
+
+FIXTURE_TEST_CASE( KCondition_TimedWait_Signaled_Broadcast, KConditionFixture )
+{
+ is_signaled = false;
+ do_broadcast = true;
+
+ REQUIRE_RC(KLockAcquire(lock));
+
+ REQUIRE_RC(StartThread());
+ REQUIRE_RC(KConditionWait(cond, lock));
+ REQUIRE(is_signaled == true);
+
+ REQUIRE_RC(KLockUnlock(lock));
+}
+
///////////////////////// KQueue
TEST_CASE( KQueue_NULL )
{
diff --git a/test/krypto/test-aes-ciphers.c b/test/krypto/test-aes-ciphers.c
index 06d3712..7c582bd 100644
--- a/test/krypto/test-aes-ciphers.c
+++ b/test/krypto/test-aes-ciphers.c
@@ -59,7 +59,7 @@
#include <stdlib.h>
/*
- http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf
+ https://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf
http://www.inconteam.com/software-development/41-encryption/55-aes-test-vectors
*/
diff --git a/test/krypto/test-modes.c b/test/krypto/test-modes.c
index 5b26026..0fb13eb 100644
--- a/test/krypto/test-modes.c
+++ b/test/krypto/test-modes.c
@@ -42,7 +42,7 @@
static int num_errors = 0;
/*
- http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf
+ https://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf
http://www.inconteam.com/software-development/41-encryption/55-aes-test-vectors
*/
diff --git a/test/ngs-c++/CSRA1test.cpp b/test/ngs-c++/CSRA1test.cpp
index 917a13f..88d1e4e 100644
--- a/test/ngs-c++/CSRA1test.cpp
+++ b/test/ngs-c++/CSRA1test.cpp
@@ -48,7 +48,7 @@ public:
static const char* CSRA1_WithCircularRef;
static const char* CSRA1_WithCircularRef2;
static const char* CSRA1_SingleFragmentPerSpot;
-
+
public:
CSRA1_Fixture()
{
@@ -59,11 +59,11 @@ public:
ngs :: ReadIterator getReads ( ngs :: Read :: ReadCategory cat = ngs :: Read :: all )
{
- return NgsFixture :: getReads ( CSRA1_PrimaryOnly, cat );
+ return NgsFixture :: getReads ( CSRA1_PrimaryOnly, cat );
}
ngs :: Read getRead ( const ngs :: String& p_id)
{
- return NgsFixture :: getRead ( CSRA1_PrimaryOnly, p_id );
+ return NgsFixture :: getRead ( CSRA1_PrimaryOnly, p_id );
}
ngs :: Read getFragment (const ngs :: String& p_readId, uint32_t p_fragIdx)
{
@@ -71,15 +71,15 @@ public:
}
ngs :: Reference getReference ( const char* spec )
{
- return NgsFixture :: getReference ( CSRA1_PrimaryOnly, spec );
+ return NgsFixture :: getReference ( CSRA1_PrimaryOnly, spec );
}
bool hasReference ( const char* spec )
{
- return NgsFixture :: hasReference ( CSRA1_PrimaryOnly, spec );
+ return NgsFixture :: hasReference ( CSRA1_PrimaryOnly, spec );
}
ngs :: ReferenceIterator getReferences ()
{
- return NgsFixture :: getReferences ( CSRA1_PrimaryOnly );
+ return NgsFixture :: getReferences ( CSRA1_PrimaryOnly );
}
ngs :: AlignmentIterator getAlignments( ngs :: Alignment :: AlignmentCategory category = ngs :: Alignment :: all )
{
@@ -101,7 +101,7 @@ const char* CSRA1_Fixture::CSRA1_SingleFragmentPerSpot = "SRR2096940";
FIXTURE_TEST_CASE(CSRA1_Read_ReadId, CSRA1_Fixture)
{
- REQUIRE_EQ( string ( CSRA1_PrimaryOnly ) + ".R.1", getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadId() . toString () );
+ REQUIRE_EQ( string ( CSRA1_PrimaryOnly ) + ".R.1", getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadId() . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Read_ReadName, CSRA1_Fixture)
@@ -116,7 +116,7 @@ FIXTURE_TEST_CASE(CSRA1_Read_ReadGroup, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_Read_getNumFragments, CSRA1_Fixture)
{
- REQUIRE_EQ( ( uint32_t ) 2, getRead ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1" ) . getNumFragments() );
+ REQUIRE_EQ( ( uint32_t ) 2, getRead ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1" ) . getNumFragments() );
}
FIXTURE_TEST_CASE(CSRA1_Read_ReadCategory_FullyAligned, CSRA1_Fixture)
@@ -130,48 +130,48 @@ FIXTURE_TEST_CASE(CSRA1_Read_ReadCategory_PartiallyAligned, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_Read_getReadBases, CSRA1_Fixture)
{
- ngs :: String bases = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadBases () . toString ();
+ ngs :: String bases = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadBases () . toString ();
ngs :: String expected("ACTCGACATTCTGCCTTCGACCTATCTTTCTCCTCTCCCAGTCATCGCCCAGTAGAATTACCAGGCAATGAACCAGGGCC"
"TTCCATCCCAACGGCACAGCAAAGGTATGATACCTGAGATGTTGCGGGATGGTGGGTTTGTGAGGAGATGGCCACGCAGG"
"CAAGGTCTTTTGGAATGGTTCACTGTTGGAGTGAACCCATAT");
- REQUIRE_EQ( expected, bases );
+ REQUIRE_EQ( expected, bases );
}
FIXTURE_TEST_CASE(CSRA1_Read_getSubReadBases_Offset, CSRA1_Fixture)
{
- ngs :: String bases = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadBases ( 160 ) . toString( );
+ ngs :: String bases = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadBases ( 160 ) . toString( );
ngs :: String expected("CAAGGTCTTTTGGAATGGTTCACTGTTGGAGTGAACCCATAT");
- REQUIRE_EQ( expected, bases );
+ REQUIRE_EQ( expected, bases );
}
FIXTURE_TEST_CASE(CSRA1_Read_getSubReadBases_OffsetLength, CSRA1_Fixture)
{
- ngs :: String bases = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadBases ( 160, 10 ) . toString ();
+ ngs :: String bases = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadBases ( 160, 10 ) . toString ();
ngs :: String expected("CAAGGTCTTT");
- REQUIRE_EQ( expected, bases );
+ REQUIRE_EQ( expected, bases );
}
FIXTURE_TEST_CASE(CSRA1_Read_getReadQualities, CSRA1_Fixture)
{
- ngs :: String quals = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadQualities() . toString ();
+ ngs :: String quals = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadQualities() . toString ();
ngs :: String expected("@@CDDBDFFBFHFIEEFGIGGHIEHIGIGGFGEGAFDHIIIIIGGGDFHII;=BF at FEHGIEEH?AHHFHFFFFDC5'5="
"?CC?ADCD at AC??9BDDCDB<@@@DDADDFFHGHIIDHFFHDEFEHIIGHIIDGGGFHIJIGAGHAH=;DGEGEEEDDDB"
"<ABBD;ACDDDCBCCCDD at CCDDDCDCDBDD@ACC>A@?>C3");
- REQUIRE_EQ( expected, quals );
+ REQUIRE_EQ( expected, quals );
}
FIXTURE_TEST_CASE(CSRA1_Read_getSubReadQualities_Offset, CSRA1_Fixture)
{
- ngs :: String quals = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadQualities( 160 ) . toString ();
+ ngs :: String quals = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadQualities( 160 ) . toString ();
ngs :: String expected("<ABBD;ACDDDCBCCCDD at CCDDDCDCDBDD@ACC>A@?>C3");
- REQUIRE_EQ( expected, quals );
+ REQUIRE_EQ( expected, quals );
}
FIXTURE_TEST_CASE(CSRA1_Read_getSubReadQualities_OffsetLength, CSRA1_Fixture)
{
- ngs :: String quals = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadQualities( 160, 10 ) . toString ();
+ ngs :: String quals = getRead ( string ( CSRA1_PrimaryOnly ) + ".R.1" ) . getReadQualities( 160, 10 ) . toString ();
ngs :: String expected("<ABBD;ACDD");
- REQUIRE_EQ( expected, quals );
+ REQUIRE_EQ( expected, quals );
}
FIXTURE_TEST_CASE(CSRA1_Read_fragmentIsAligned_MultiFragmentsPerPartiallyAlignedSpot, CSRA1_Fixture)
@@ -191,51 +191,51 @@ FIXTURE_TEST_CASE(CSRA1_Read_fragmentIsAligned_MultiFragmentsPerPartiallyAligned
//TODO: error cases (?)
FIXTURE_TEST_CASE(CSRA1_ReadIterator_NoReadBeforeNext, CSRA1_Fixture)
{
- ngs :: ReadIterator readIt = getReads ();
+ ngs :: ReadIterator readIt = getReads ();
// accessing the read through an iterator before a call to nextRead() throws
REQUIRE_THROW ( readIt . getReadId() );
}
FIXTURE_TEST_CASE(CSRA1_ReadIterator_Open_All, CSRA1_Fixture)
{
- ngs :: ReadIterator readIt = getReads ();
+ ngs :: ReadIterator readIt = getReads ();
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", readIt . getReadId() . toString () );
}
#if SHOW_UNIMPLEMENTED
FIXTURE_TEST_CASE(CSRA1_ReadIterator_Open_Filtered, CSRA1_Fixture)
{
- ngs :: ReadIterator readIt = getReads ( ngs :: Read :: partiallyAligned );
+ ngs :: ReadIterator readIt = getReads ( ngs :: Read :: partiallyAligned );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.3", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.3", readIt . getReadId() . toString () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.5", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.5", readIt . getReadId() . toString () );
}
#endif
FIXTURE_TEST_CASE(CSRA1_ReadIterator_Next, CSRA1_Fixture)
{
- ngs :: ReadIterator readIt = getReads ();
+ ngs :: ReadIterator readIt = getReads ();
REQUIRE( readIt . nextRead () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.2", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.2", readIt . getReadId() . toString () );
}
FIXTURE_TEST_CASE(CSRA1_ReadIterator_End, CSRA1_Fixture)
{
- ngs :: ReadIterator readIt = getReads ();
+ ngs :: ReadIterator readIt = getReads ();
ngs :: String lastId;
while (readIt . nextRead ())
{
lastId = readIt . getReadId() . toString ();
}
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.2280633", lastId );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.2280633", lastId );
}
FIXTURE_TEST_CASE(CSRA1_ReadIterator_BeyondEnd, CSRA1_Fixture)
{
- ngs :: ReadIterator readIt = getReads ();
+ ngs :: ReadIterator readIt = getReads ();
while (readIt . nextRead ())
{
}
@@ -244,24 +244,24 @@ FIXTURE_TEST_CASE(CSRA1_ReadIterator_BeyondEnd, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_ReadIterator_Range, CSRA1_Fixture)
{
- ngs :: ReadIterator readIt = ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getReadRange ( 10, 5 );
+ ngs :: ReadIterator readIt = ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getReadRange ( 10, 5 );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.10", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.10", readIt . getReadId() . toString () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.11", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.11", readIt . getReadId() . toString () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.12", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.12", readIt . getReadId() . toString () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.13", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.13", readIt . getReadId() . toString () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.14", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.14", readIt . getReadId() . toString () );
REQUIRE( ! readIt . nextRead () );
}
FIXTURE_TEST_CASE(CSRA1_ReadIterator_EmptyRange, CSRA1_Fixture)
{
ngs :: ReadCollection run = ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly );
- ngs :: ReadIterator readIt = run . getReadRange ( run . getReadCount() + 1, 5 );
+ ngs :: ReadIterator readIt = run . getReadRange ( run . getReadCount() + 1, 5 );
REQUIRE_THROW ( readIt . getReadId(); );
REQUIRE ( ! readIt . nextRead () );
}
@@ -284,37 +284,37 @@ FIXTURE_TEST_CASE(CSRA1_Fragment_Id, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_Fragment_getFragmentBases, CSRA1_Fixture)
{
ngs :: String expected("AAGGTATGATACCTGAGATGTTGCGGGATGGTGGGTTTGTGAGGAGATGGCCACGCAGGCAAGGTCTTTTGGAATGGTTCACTGTTGGAGTGAACCCATAT");
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentBases () . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentBases () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Fragment_getSubFragmentBases_Offset, CSRA1_Fixture)
{
ngs :: String expected("ACTGTTGGAGTGAACCCATAT");
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentBases ( 80 ) . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentBases ( 80 ) . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Fragment_getSubFragmentBases_OffsetLength, CSRA1_Fixture)
{
ngs :: String expected("ACTGTT");
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentBases ( 80, 6 ) . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentBases ( 80, 6 ) . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Fragment_getFragmentQualities, CSRA1_Fixture)
{
ngs :: String expected("@@@DDADDFFHGHIIDHFFHDEFEHIIGHIIDGGGFHIJIGAGHAH=;DGEGEEEDDDB<ABBD;ACDDDCBCCCDD at CCDDDCDCDBDD@ACC>A@?>C3");
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentQualities () . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentQualities () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Fragment_getSubFragmentQualities_Offset, CSRA1_Fixture)
{
ngs :: String expected("DDDCDCDBDD at ACC>A@?>C3");
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentQualities ( 80 ) . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentQualities ( 80 ) . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Fragment_getSubFragmentQualities_OffsetLength, CSRA1_Fixture)
{
ngs :: String expected("DDDCDC");
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentQualities ( 80, 6 ) . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".R.1", 2 ) . getFragmentQualities ( 80, 6 ) . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Fragment_isPaired_MultiFragmentsPerAlignedSpot, CSRA1_Fixture)
@@ -350,24 +350,24 @@ FIXTURE_TEST_CASE(CSRA1_Fragment_isPaired_SingleFragmentPerSpot, CSRA1_Fixture)
//TODO: getPileupRange
//TODO: error cases
FIXTURE_TEST_CASE(CSRA1_Reference_CommonName, CSRA1_Fixture)
-{ // SRR1199225
- const char* name = "supercont2.1";
- REQUIRE_EQ( ngs :: String ( name ), getReference ( name ). getCommonName () );
+{ // SRR1199225
+ const char* name = "supercont2.1";
+ REQUIRE_EQ( ngs :: String ( name ), getReference ( name ). getCommonName () );
}
FIXTURE_TEST_CASE(CSRA1_Reference_CanonicalName, CSRA1_Fixture)
{
- const char* name = "chr7";
- const char* canoName = "NC_000007.13";
- REQUIRE_EQ( ngs :: String ( canoName ), ncbi :: NGS :: openReadCollection ( "SRR821492" ) . getReference ( name ). getCanonicalName () );
+ const char* name = "chr7";
+ const char* canoName = "NC_000007.13";
+ REQUIRE_EQ( ngs :: String ( canoName ), ncbi :: NGS :: openReadCollection ( "SRR821492" ) . getReference ( name ). getCanonicalName () );
}
FIXTURE_TEST_CASE(CSRA1_Reference_IsCircular_No, CSRA1_Fixture)
{
- REQUIRE( ! getReference ( "supercont2.1" ). getIsCircular () );
+ REQUIRE( ! getReference ( "supercont2.1" ). getIsCircular () );
}
FIXTURE_TEST_CASE(CSRA1_Reference_IsCircular_Yes, CSRA1_Fixture)
{
- REQUIRE( ncbi :: NGS :: openReadCollection ( "SRR821492" ) . getReference ( "chrM" ) . getIsCircular () );
+ REQUIRE( ncbi :: NGS :: openReadCollection ( "SRR821492" ) . getReference ( "chrM" ) . getIsCircular () );
}
static uint64_t CSRA1_ReferenceLength = (uint64_t)2291499l;
@@ -386,7 +386,7 @@ FIXTURE_TEST_CASE(CSRA1_Reference_GetLength_VDB_2832, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceBases_All, CSRA1_Fixture)
{
- ngs :: String str = getReference ( "supercont2.1" ). getReferenceBases ( 0 );
+ ngs :: String str = getReference ( "supercont2.1" ). getReferenceBases ( 0 );
REQUIRE_EQ( (size_t)CSRA1_ReferenceLength, str . size () );
REQUIRE_EQ( string("GAATTCT"), str . substr (0, 7) );
REQUIRE_EQ( string("ATCTG"), str . substr (str.size() - 5, 5) );
@@ -394,14 +394,14 @@ FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceBases_All, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceBases_Offset_1, CSRA1_Fixture)
{ // offset points into the first chunk of the reference
- ngs :: String str = getReference ( "supercont2.1" ). getReferenceBases ( 1000 );
+ ngs :: String str = getReference ( "supercont2.1" ). getReferenceBases ( 1000 );
REQUIRE_EQ( CSRA1_ReferenceLength - 1000, (uint64_t) str . size () );
REQUIRE_EQ( string("TCCATTC"), str . substr (0, 7) );
REQUIRE_EQ( string("ATCTG"), str . substr (str.size() - 5, 5) );
}
FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceBases_Offset_2, CSRA1_Fixture)
{ // offset points into the second chunk of the reference
- ngs :: String str = getReference ( "supercont2.1" ). getReferenceBases ( 7000 );
+ ngs :: String str = getReference ( "supercont2.1" ). getReferenceBases ( 7000 );
REQUIRE_EQ( CSRA1_ReferenceLength - 7000, (uint64_t) str . size () );
REQUIRE_EQ( string("CCTGTCC"), str . substr (0, 7) );
REQUIRE_EQ( string("ATCTG"), str . substr (str.size() - 5, 5) );
@@ -409,102 +409,89 @@ FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceBases_Offset_2, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceBases_OffsetLength_1, CSRA1_Fixture)
{ // offset points into the first chunk of the reference
- ngs :: String str = getReference ( "supercont2.1" ). getReferenceBases ( 2000, 10 );
+ ngs :: String str = getReference ( "supercont2.1" ). getReferenceBases ( 2000, 10 );
REQUIRE_EQ( ngs::String ( "GGGCAAATGA" ), str );
}
FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceBases_OffsetLength_2, CSRA1_Fixture)
{ // offset points into the second chunk of the reference
ngs :: Reference ref = getReference ( "supercont2.1" );
- ngs :: String str = ref . getReferenceBases ( 9000, 10 );
+ ngs :: String str = ref . getReferenceBases ( 9000, 10 );
REQUIRE_EQ( ngs::String ( "GCGCTATGAC" ), str );
}
FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_All, CSRA1_Fixture)
{
ngs :: Reference ref = getReference ( "supercont2.1" );
- ngs :: StringRef str = ref . getReferenceChunk ( 0 );
- REQUIRE_EQ( (size_t)5000, str . size () );
+ ngs :: StringRef str = ref . getReferenceChunk ( 0 );
+ REQUIRE_EQ( (size_t)20000, str . size () );
REQUIRE_EQ( string("GAATTCT"), str . substr (0, 7) . toString () );
- REQUIRE_EQ( string("CTAGG"), str . substr (str.size() - 5, 5) . toString () );
+ REQUIRE_EQ( string("CATCA"), str . substr (str.size() - 5, 5) . toString () );
}
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Offset_1, CSRA1_Fixture)
-{ // offset points into the first chunk of the reference
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Offset, CSRA1_Fixture)
+{
ngs :: Reference ref = getReference ( "supercont2.1" );
- ngs :: StringRef str = ref . getReferenceChunk ( 1000 );
- REQUIRE_EQ( (size_t)4000, str . size () );
+ ngs :: StringRef str = ref . getReferenceChunk ( 1000 );
+ REQUIRE_EQ( (size_t)19000, str . size () );
REQUIRE_EQ( string("TCCATTC"), str . substr (0, 7) . toString () );
- REQUIRE_EQ( string("CTAGG"), str . substr (str.size() - 5, 5) . toString () );
-}
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Offset_2, CSRA1_Fixture)
-{ // offset points into the second chunk of the reference
- ngs :: Reference ref = getReference ( "supercont2.1" );
- ngs :: StringRef str = ref . getReferenceChunk ( 7000 );
- REQUIRE_EQ( (size_t)3000, str . size () );
- REQUIRE_EQ( string("CCTGTCC"), str . substr (0, 7) . toString () );
- REQUIRE_EQ( string("AATCT"), str . substr (str.size() - 5, 5) . toString () );
+ REQUIRE_EQ( string("CATCA"), str . substr (str.size() - 5, 5) . toString () );
}
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_1, CSRA1_Fixture)
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength, CSRA1_Fixture)
{ // offset points into the first chunk of the reference
ngs :: Reference ref = getReference ( "supercont2.1" );
REQUIRE_EQ( ngs::String ( "GGGCAAATGA" ), ref . getReferenceChunk ( 2000, 10 ) . toString ());
}
-FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_2, CSRA1_Fixture)
-{ // offset points into the second chunk of the reference
- ngs :: Reference ref = getReference ( "supercont2.1" );
- REQUIRE_EQ( ngs::String ( "GCGCTATGAC" ), ref . getReferenceChunk ( 9000, 10 ) . toString () );
-}
FIXTURE_TEST_CASE(CSRA1_Reference_GetAlignment, CSRA1_Fixture)
{
- ngs :: Alignment al = getReference ( "supercont2.1" ). getAlignment( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" );
- REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1", al . getAlignmentId () . toString () );
+ ngs :: Alignment al = getReference ( "supercont2.1" ). getAlignment( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" );
+ REQUIRE_EQ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1", al . getAlignmentId () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Reference_GetAlignments, CSRA1_Fixture)
{
- ngs :: AlignmentIterator al = getReference ( "supercont2.1" ). getAlignments ( ngs :: Alignment :: all );
+ ngs :: AlignmentIterator al = getReference ( "supercont2.1" ). getAlignments ( ngs :: Alignment :: all );
}
///// ReferenceIterator
FIXTURE_TEST_CASE(CSRA1_ReferenceIterator_NoObjectBeforeNextOpen, CSRA1_Fixture)
{
- ngs :: ReferenceIterator refIt = getReferences();
+ ngs :: ReferenceIterator refIt = getReferences();
// dereferencing the iterator without calling Next() throws
- REQUIRE_THROW ( refIt . getCommonName() );
+ REQUIRE_THROW ( refIt . getCommonName() );
}
FIXTURE_TEST_CASE(CSRA1_ReferenceIterator_Open, CSRA1_Fixture)
{
- ngs :: ReferenceIterator refIt = getReferences();
+ ngs :: ReferenceIterator refIt = getReferences();
REQUIRE( refIt . nextReference () );
- REQUIRE_EQ( string("supercont2.1"), refIt . getCommonName() );
+ REQUIRE_EQ( string("supercont2.1"), refIt . getCommonName() );
}
FIXTURE_TEST_CASE(CSRA1_ReferenceIterator_Next, CSRA1_Fixture)
{
- ngs :: ReferenceIterator refIt = getReferences();
+ ngs :: ReferenceIterator refIt = getReferences();
REQUIRE( refIt . nextReference () );
REQUIRE( refIt . nextReference () );
- REQUIRE_EQ( string("supercont2.2"), refIt . getCommonName() );
+ REQUIRE_EQ( string("supercont2.2"), refIt . getCommonName() );
}
FIXTURE_TEST_CASE(CSRA1_ReferenceIterator_End, CSRA1_Fixture)
{
- ngs :: ReferenceIterator refIt = getReferences();
+ ngs :: ReferenceIterator refIt = getReferences();
string lastName;
while (refIt . nextReference ())
{
lastName = refIt . getCommonName();
//cout << " lastName " << refIt . getLength () << endl;
}
- REQUIRE_EQ( string("supercont2.14"), lastName );
+ REQUIRE_EQ( string("supercont2.14"), lastName );
}
FIXTURE_TEST_CASE(CSRA1_ReferenceIterator_BeyondEnd, CSRA1_Fixture)
{
- ngs :: ReferenceIterator refIt = getReferences();
+ ngs :: ReferenceIterator refIt = getReferences();
while (refIt . nextReference ())
{
}
@@ -518,85 +505,87 @@ FIXTURE_TEST_CASE(CSRA1_ReferenceIterator_BeyondEnd, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_Alignment_getAlignmentId, CSRA1_Fixture)
{
- REQUIRE_EQ( ngs :: String ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ),
+ REQUIRE_EQ( ngs :: String ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getAlignmentId() . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getReferenceSpec, CSRA1_Fixture)
{
- REQUIRE_EQ( string("supercont2.1"),
+ REQUIRE_EQ( string("supercont2.1"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getReferenceSpec () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getMappingQuality, CSRA1_Fixture)
{
- REQUIRE_EQ( 60,
+ REQUIRE_EQ( 60,
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getMappingQuality () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getReferenceBases, CSRA1_Fixture)
{
- REQUIRE_EQ( string("ACTCGACATTCTGTCTTCGACCTATCTTTCTCCTCTCCCAGTCATCGCCCAGTAGAATTACCAGGCAATGAACCACGGCCTTTCATCCCAACGGCACAGCA"),
+ REQUIRE_EQ( string("ACTCGACATTCTGTCTTCGACCTATCTTTCTCCTCTCCCAGTCATCGCCCAGTAGAATTACCAGGCAATGAACCACGGCCTTTCATCCCAACGGCACAGCA"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getReferenceBases () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getReadGroup, CSRA1_Fixture)
{
- REQUIRE_EQ( string("C1ELY.6"),
+ REQUIRE_EQ( string("C1ELY.6"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getReadGroup () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getReadGroup_Empty, CSRA1_Fixture)
{
- REQUIRE_EQ( string(""),
+ REQUIRE_EQ( string(""),
ncbi :: NGS :: openReadCollection ( CSRA1_NoReadGroups ) . getAlignment ( ngs :: String ( CSRA1_NoReadGroups ) + ".PA.1" ) . getReadGroup () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getReadId, CSRA1_Fixture)
{
- REQUIRE_EQ( string ( CSRA1_PrimaryOnly ) + ".R.165753",
+ REQUIRE_EQ( string ( CSRA1_PrimaryOnly ) + ".R.165753",
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.5" ) . getReadId () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getFragmentId, CSRA1_Fixture)
{
- REQUIRE_EQ( string ( CSRA1_PrimaryOnly ) + ".FA0.1",
+ REQUIRE_EQ( string ( CSRA1_PrimaryOnly ) + ".FA0.1",
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getFragmentId () . toString ());
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getFragmentBases, CSRA1_Fixture)
{
- REQUIRE_EQ( string("TGGATGCTCTGGAAAATCTGAAAAGTGGTGTTTGTAAGGTTTGCTGGCTGCCCATATACCACATGGATGATGGGGCTTTCCATTTTAATGTTGAAGGAGGA"),
+ REQUIRE_EQ( string("TGGATGCTCTGGAAAATCTGAAAAGTGGTGTTTGTAAGGTTTGCTGGCTGCCCATATACCACATGGATGATGGGGCTTTCCATTTTAATGTTGAAGGAGGA"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.4" ) . getFragmentBases () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getFragmentQualities, CSRA1_Fixture)
{
- REQUIRE_EQ( string("######AA>55;5(;63;;3@;A9??;6..73CDCIDA>DCB>@B=;@B?;;ADAB<DD?1*>@C9:EC?2++A3+F4EEB<E>EEIEDC2?C:;AB+==1"),
+ REQUIRE_EQ( string("######AA>55;5(;63;;3@;A9??;6..73CDCIDA>DCB>@B=;@B?;;ADAB<DD?1*>@C9:EC?2++A3+F4EEB<E>EEIEDC2?C:;AB+==1"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.4" ) . getFragmentQualities () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getClippedFragmentBases, CSRA1_Fixture)
{
- REQUIRE_EQ( string("CTTCAACATTAAAATGGAAAGCCCCATCATCCATGTGGTATATGGGCAGCCAGCAAACCTTACAAACACCACTTTTCAGATTTTCCAGAGCATCCA"),
+ REQUIRE_EQ( string("CTTCAACATTAAAATGGAAAGCCCCATCATCCATGTGGTATATGGGCAGCCAGCAAACCTTACAAACACCACTTTTCAGATTTTCCAGAGCATCCA"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.4" ) . getClippedFragmentBases () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getClippedFragmentQualities, CSRA1_Fixture)
{
- REQUIRE_EQ( string("#AA>55;5(;63;;3@;A9??;6..73CDCIDA>DCB>@B=;@B?;;ADAB<DD?1*>@C9:EC?2++A3+F4EEB<E>EEIEDC2?C:;AB+==1"),
+ REQUIRE_EQ( string("#AA>55;5(;63;;3@;A9??;6..73CDCIDA>DCB>@B=;@B?;;ADAB<DD?1*>@C9:EC?2++A3+F4EEB<E>EEIEDC2?C:;AB+==1"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.4" ) . getClippedFragmentQualities () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getAlignedFragmentBases, CSRA1_Fixture)
{
- REQUIRE_EQ( string("ATATGGGTTCACTCCAACAGTGAACCATTCCAAAAGACCTTGCCTGCGTGGCCATCTCCTCACAAACCCACCATCCCGCAACATCTCAGGTATCATACCTT"),
+ REQUIRE_EQ( string("AAGGTATGATACCTGAGATGTTGCGGGATGGTGGGTTTGTGAGGAGATGGCCACGCAGGCAAGGTCTTTTGGAATGGTTCACTGTTGGAGTGAACCCATAT"),
+ ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.2" ) . getFragmentBases () . toString () );
+ REQUIRE_EQ( string("ATATGGGTTCACTCCAACAGTGAACCATTCCAAAAGACCTTGCCTGCGTGGCCATCTCCTCACAAACCCACCATCCCGCAACATCTCAGGTATCATACCTT"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.2" ) . getAlignedFragmentBases () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getAlignmentCategory, CSRA1_Fixture)
{
- REQUIRE_EQ( ngs :: Alignment :: primaryAlignment,
+ REQUIRE_EQ( ngs :: Alignment :: primaryAlignment,
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getAlignmentCategory() );
}
@@ -633,7 +622,7 @@ FIXTURE_TEST_CASE(CSRA1_Alignment_getReferencePositionProjectionRangeD, CSRA1_Fi
FIXTURE_TEST_CASE(CSRA1_Alignment_getAlignmentLength, CSRA1_Fixture)
{
- REQUIRE_EQ( (uint64_t)101, //REF_LEN or length(RAW_READ)
+ REQUIRE_EQ( (uint64_t)101, //REF_LEN or length(RAW_READ)
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getAlignmentLength () );
}
@@ -647,52 +636,52 @@ FIXTURE_TEST_CASE(CSRA1_Alignment_getIsReversedOrientation_True, CSRA1_Fixture)
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getSoftClip_None, CSRA1_Fixture)
-{
+{
REQUIRE_EQ( 0, ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getSoftClip ( ngs :: Alignment :: clipLeft ) );
REQUIRE_EQ( 0, ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getSoftClip ( ngs :: Alignment :: clipRight ) );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getSoftClip_Left, CSRA1_Fixture)
-{
+{
REQUIRE_EQ( 5, ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.4" ) . getSoftClip ( ngs :: Alignment :: clipLeft ) );
REQUIRE_EQ( 0, ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.4" ) . getSoftClip ( ngs :: Alignment :: clipRight ) );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getSoftClip_Right, CSRA1_Fixture)
-{
+{
REQUIRE_EQ( 0, ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.10" ) . getSoftClip ( ngs :: Alignment :: clipLeft ) );
REQUIRE_EQ( 13, ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.10" ) . getSoftClip ( ngs :: Alignment :: clipRight ) );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getTemplateLength, CSRA1_Fixture)
{
- REQUIRE_EQ( (uint64_t)201,
+ REQUIRE_EQ( (uint64_t)201,
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getTemplateLength () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getShortCigar_Unclipped, CSRA1_Fixture)
{
- REQUIRE_EQ( string("5S96M"),
+ REQUIRE_EQ( string("5S96M"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.4" ) . getShortCigar ( false ) . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getShortCigar_Clipped, CSRA1_Fixture)
{
- REQUIRE_EQ( string("96M"),
+ REQUIRE_EQ( string("96M"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.4" ) . getShortCigar ( true ) . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getLongCigar_Unclipped, CSRA1_Fixture)
{
- REQUIRE_EQ( string("5S1X8=1X39=1X46="),
+ REQUIRE_EQ( string("5S1X8=1X39=1X46="),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.4" ) . getLongCigar ( false ) . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getLongCigar_Clipped, CSRA1_Fixture)
{
- REQUIRE_EQ( string("1X8=1X39=1X46="),
+ REQUIRE_EQ( string("1X8=1X39=1X46="),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.4" ) . getLongCigar ( true ) . toString () );
}
FIXTURE_TEST_CASE(CSRA1_Alignment_getRNAOrientation_Missing, CSRA1_Fixture)
{
- REQUIRE_EQ( '?',
+ REQUIRE_EQ( '?',
ncbi :: NGS :: openReadCollection ( CSRA1_NoReadGroups ) . getAlignment ( ngs :: String ( CSRA1_NoReadGroups ) + ".PA.1" ) . getRNAOrientation() );
}
@@ -712,7 +701,7 @@ FIXTURE_TEST_CASE(CSRA1_Alignment_hasMate_Secondary, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_Alignment_getMateAlignmentId_Primary, CSRA1_Fixture)
{
- REQUIRE_EQ( string ( CSRA1_PrimaryOnly ) + ".PA.2",
+ REQUIRE_EQ( string ( CSRA1_PrimaryOnly ) + ".PA.2",
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getMateAlignmentId () . toString () );
}
@@ -744,7 +733,7 @@ FIXTURE_TEST_CASE(CSRA1_Alignment_getMateAlignment_SecondaryThrows, CSRA1_Fixtur
FIXTURE_TEST_CASE(CSRA1_Alignment_getMateReferenceSpec, CSRA1_Fixture)
{
- REQUIRE_EQ( string("supercont2.1"),
+ REQUIRE_EQ( string("supercont2.1"),
ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.1" ) . getMateReferenceSpec () );
}
@@ -765,7 +754,7 @@ FIXTURE_TEST_CASE(CSRA1_Alignment_isPaired_MultiFragmentsPerSpot, CSRA1_Fixture)
alignment = readCollection . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.2" );
REQUIRE_EQ( true, alignment.isPaired() );
-
+
// has unaligned mate
alignment = readCollection . getAlignment ( ngs :: String ( CSRA1_PrimaryOnly ) + ".PA.6" );
REQUIRE_EQ( true, alignment.isPaired() );
@@ -784,23 +773,23 @@ FIXTURE_TEST_CASE(CSRA1_Alignment_isPaired_SingleFragmentPerSpot, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_AlignmentIterator_NoAlignmentBeforeNext, CSRA1_Fixture)
{
- ngs :: AlignmentIterator it = getAlignments ();
- REQUIRE_THROW( it . getAlignmentId () );
+ ngs :: AlignmentIterator it = getAlignments ();
+ REQUIRE_THROW( it . getAlignmentId () );
}
FIXTURE_TEST_CASE(CSRA1_AlignmentIterator_PrimaryOnOpen, CSRA1_Fixture)
{
- ngs :: AlignmentIterator it = getAlignments ();
+ ngs :: AlignmentIterator it = getAlignments ();
REQUIRE( it . nextAlignment () );
- REQUIRE_EQ( ngs :: String ( CSRA1_WithSecondary ) + ".PA.1", it . getAlignmentId () . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_WithSecondary ) + ".PA.1", it . getAlignmentId () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_AlignmentIterator_Next, CSRA1_Fixture)
{
- ngs :: AlignmentIterator it = getAlignments ();
+ ngs :: AlignmentIterator it = getAlignments ();
REQUIRE( it . nextAlignment () );
REQUIRE( it . nextAlignment () );
- REQUIRE_EQ( ngs :: String ( CSRA1_WithSecondary ) + ".PA.2", it . getAlignmentId () . toString () );
+ REQUIRE_EQ( ngs :: String ( CSRA1_WithSecondary ) + ".PA.2", it . getAlignmentId () . toString () );
}
FIXTURE_TEST_CASE(CSRA1_AlignmentIterator_Next_Switch_To_Secondary, CSRA1_Fixture)
@@ -814,17 +803,17 @@ FIXTURE_TEST_CASE(CSRA1_AlignmentIterator_Next_Switch_To_Secondary, CSRA1_Fixtur
FIXTURE_TEST_CASE(CSRA1_AlignmentIterator_BeyondEnd, CSRA1_Fixture)
{
- ngs :: AlignmentIterator it = getAlignments ();
+ ngs :: AlignmentIterator it = getAlignments ();
while ( it . nextAlignment() )
{
}
- REQUIRE_THROW ( it . getAlignmentId (); );
+ REQUIRE_THROW ( it . getAlignmentId (); );
}
FIXTURE_TEST_CASE(CSRA1_AlignmentIterator_Empty, CSRA1_Fixture)
{
// the next line should not throw
- ngs :: AlignmentIterator it = ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignments ( ngs :: Alignment :: secondaryAlignment );
+ ngs :: AlignmentIterator it = ncbi :: NGS :: openReadCollection ( CSRA1_PrimaryOnly ) . getAlignments ( ngs :: Alignment :: secondaryAlignment );
REQUIRE_THROW ( it . getAlignmentId(); );
}
@@ -858,7 +847,7 @@ FIXTURE_TEST_CASE(CSRA1_AlignmentIterator_Secondary, CSRA1_Fixture)
}
FIXTURE_TEST_CASE(CSRA1_AlignmentIterator_All, CSRA1_Fixture)
-{
+{
ngs :: AlignmentIterator it = getAlignments ( ngs :: Alignment :: all );
REQUIRE( it . nextAlignment() );
size_t count = 1;
@@ -884,9 +873,9 @@ FIXTURE_TEST_CASE(CSRA1_AlignmentIterator_All, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_ReferenceWindow, CSRA1_Fixture)
{
ngs :: AlignmentIterator it = NgsFixture :: getReference ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" )
- . getAlignments ( ngs :: Alignment :: all );
+ . getAlignments ( ngs :: Alignment :: all );
REQUIRE ( it . nextAlignment () );
-
+
// the first 2 secondary alignments' locations on the list: #34, #61
size_t count = 1;
while ( it . nextAlignment() )
@@ -908,8 +897,8 @@ FIXTURE_TEST_CASE(CSRA1_ReferenceWindow, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_ReferenceWindow_Slice, CSRA1_Fixture)
{
ngs :: AlignmentIterator it = NgsFixture :: getReference ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" )
- . getAlignmentSlice ( 516000, 100000 );
-
+ . getAlignmentSlice ( 516000, 100000 );
+
REQUIRE ( it . nextAlignment () );
REQUIRE_EQ ( ngs :: String ( CSRA1_WithSecondary ) + ".PA.33", it. getAlignmentId () . toString () );
REQUIRE ( it . nextAlignment () );
@@ -918,7 +907,7 @@ FIXTURE_TEST_CASE(CSRA1_ReferenceWindow_Slice, CSRA1_Fixture)
REQUIRE_EQ ( ngs :: String ( CSRA1_WithSecondary ) + ".SA.169", it. getAlignmentId () . toString () );
REQUIRE ( it . nextAlignment () );
REQUIRE_EQ ( ngs :: String ( CSRA1_WithSecondary ) + ".PA.35", it. getAlignmentId () . toString () );
-
+
REQUIRE ( ! it . nextAlignment () );
}
@@ -927,7 +916,7 @@ FIXTURE_TEST_CASE(CSRA1_ReferenceWindow_Slice_Filtered_Start_Within_Slice, CSRA1
ngs :: Reference ref = ncbi :: NGS :: openReadCollection ( CSRA1_WithCircularRef ) . getReference ( "NC_012920.1" );
int64_t sliceStart = 1000;
ngs :: AlignmentIterator it = ref . getFilteredAlignmentSlice ( sliceStart, 200, ngs :: Alignment :: all, ngs :: Alignment :: startWithinSlice, 0 );
-
+
REQUIRE ( it . nextAlignment () );
REQUIRE_LE ( sliceStart, it.getAlignmentPosition() );
}
@@ -936,7 +925,7 @@ FIXTURE_TEST_CASE(CSRA1_ReferenceWindow_Slice_Unfiltered_Start_Before_Slice, CSR
ngs :: Reference ref = ncbi :: NGS :: openReadCollection ( CSRA1_WithCircularRef ) . getReference ( "NC_012920.1" );
int64_t sliceStart = 1000;
ngs :: AlignmentIterator it = ref . getFilteredAlignmentSlice ( sliceStart, 200, ngs :: Alignment :: all, (ngs::Alignment::AlignmentFilter)0, 0 );
-
+
REQUIRE ( it . nextAlignment () );
REQUIRE_GT ( sliceStart, it.getAlignmentPosition() );
}
@@ -946,13 +935,13 @@ FIXTURE_TEST_CASE(CSRA1_ReferenceWindow_Slice_Filtered_Wraparound, CSRA1_Fixture
ngs :: Reference ref = ncbi :: NGS :: openReadCollection ( CSRA1_WithCircularRef2 ) . getReference ( "chrM" );
int64_t sliceStart = 5;
ngs :: AlignmentIterator it = ref . getFilteredAlignmentSlice ( sliceStart, 100, ngs :: Alignment :: all, (ngs::Alignment::AlignmentFilter)0, 0 );
-
+
REQUIRE ( it . nextAlignment () );
-
+
// the first returned alignment starts before the start of the circular reference, overlaps with slice
int64_t pos = it.getAlignmentPosition();
REQUIRE_LT ( sliceStart + 100, pos );
-
+
// check for overlap with the slice
uint64_t refLen = ref . getLength();
pos -= ( int64_t ) refLen;
@@ -966,9 +955,9 @@ FIXTURE_TEST_CASE(CSRA1_ReferenceWindow_Slice_Filtered_Wraparound_StartWithinSli
ngs :: Reference ref = ncbi :: NGS :: openReadCollection ( CSRA1_WithCircularRef2 ) . getReference ( "chrM" );
int64_t sliceStart = 5;
ngs :: AlignmentIterator it = ref . getFilteredAlignmentSlice ( sliceStart, 100, ngs :: Alignment :: all, ngs :: Alignment :: startWithinSlice, 0 );
-
+
REQUIRE ( it . nextAlignment () );
-
+
// the first returned alignment starts inside the slice
int64_t pos = it.getAlignmentPosition();
REQUIRE_LE ( sliceStart, pos );
@@ -981,9 +970,9 @@ FIXTURE_TEST_CASE(CSRA1_ReferenceWindow_Slice_Filtered_NoWraparound, CSRA1_Fixtu
ngs :: Reference ref = ncbi :: NGS :: openReadCollection ( CSRA1_WithCircularRef2 ) . getReference ( "chrM" );
int64_t sliceStart = 5;
ngs :: AlignmentIterator it = ref . getFilteredAlignmentSlice ( sliceStart, 100, ngs :: Alignment :: all, ngs :: Alignment :: noWraparound, 0 );
-
+
REQUIRE ( it . nextAlignment () );
-
+
// the first returned sliceStart starts outside the slice but does not wrap around
REQUIRE_GT ( sliceStart, it.getAlignmentPosition() );
}
@@ -994,9 +983,9 @@ FIXTURE_TEST_CASE(CSRA1_ReferenceWindow_Slice_Filtered_NoWraparound_StartWithinS
ngs :: Reference ref = ncbi :: NGS :: openReadCollection ( CSRA1_WithCircularRef2 ) . getReference ( "chrM" );
int64_t sliceStart = 5;
ngs :: AlignmentIterator it = ref . getFilteredAlignmentSlice ( sliceStart, 100, ngs :: Alignment :: all, (ngs::Alignment::AlignmentFilter) (ngs :: Alignment :: noWraparound | ngs :: Alignment :: startWithinSlice), 0 );
-
+
REQUIRE ( it . nextAlignment () );
-
+
// the first returned alignment starts inside the slice
int64_t pos = it.getAlignmentPosition();
REQUIRE_LE ( sliceStart, pos );
@@ -1065,7 +1054,7 @@ FIXTURE_TEST_CASE(CSRA1_ReadGroup_GetStatistics, CSRA1_Fixture)
{
ngs :: ReadGroup rg = ncbi :: NGS :: openReadCollection ( CSRA1_WithGroups ) . getReadGroup ( "GS57510-FS3-L03" );
ngs :: Statistics stats = rg . getStatistics ();
-
+
REQUIRE_EQ ( (uint64_t)34164461870, stats . getAsU64 ( "BASE_COUNT" ) );
REQUIRE_EQ ( (uint64_t)34164461870, stats . getAsU64 ( "BIO_BASE_COUNT" ) );
REQUIRE_EQ ( (uint64_t)488063741, stats . getAsU64 ( "SPOT_COUNT" ) );
@@ -1075,7 +1064,7 @@ FIXTURE_TEST_CASE(CSRA1_ReadGroup_GetStatistics, CSRA1_Fixture)
/////TODO: ReadGroupIterator
FIXTURE_TEST_CASE(CSRA1_ReadGroupIterator_AfterNext, CSRA1_Fixture)
-{
+{
ngs :: ReadGroupIterator rg = ncbi :: NGS :: openReadCollection ( "ERR730927" ) . getReadGroups ();
REQUIRE ( rg . nextReadGroup () );
REQUIRE_EQ ( string ( "1#72" ), rg . getName () );
diff --git a/test/ngs-c++/Makefile b/test/ngs-c++/Makefile
index 122da2f..fc5fabf 100644
--- a/test/ngs-c++/Makefile
+++ b/test/ngs-c++/Makefile
@@ -40,8 +40,8 @@ include $(TOP)/build/Makefile.env
ifeq (,$(NGS_LIBDIR))
$(TEST_TOOLS):
- @true
-
+ @true
+
runtests:
@ echo "NGS C++ tests are not run since NGS library does not appear to be installed"
@@ -59,12 +59,12 @@ endif
$(TEST_TOOLS): makedirs
@ $(MAKE_CMD) $(TEST_BINDIR)/$@
-.PHONY: $(TEST_TOOLS)
+.PHONY: $(TEST_TOOLS)
clean: stdclean
#-------------------------------------------------------------------------------
-# common defs
+# common defs
#
INCDIRS += -I$(NGS_INCDIR)
@@ -85,9 +85,12 @@ TEST_NGS_SRA_CPP_SRC = \
TEST_NGS_SRA_CPP_OBJ = \
$(addsuffix .$(OBJX),$(TEST_NGS_SRA_CPP_SRC))
-$(TEST_BINDIR)/test-ngs_sra-c++: $(TEST_NGS_SRA_CPP_OBJ)
+$(TEST_BINDIR)/test-ngs_sra-c++: $(TEST_NGS_SRA_CPP_OBJ)
$(LP) --exe -o $@ $^ $(TEST_NGS_CPP_LIB)
+sra: test-ngs_sra-c++
+ $(TEST_BINDIR)/$^
+
#-------------------------------------------------------------------------------
# test-ngs_sradb-c++
#
@@ -97,9 +100,12 @@ TEST_NGS_SRADB_CPP_SRC = \
TEST_NGS_SRADB_CPP_OBJ = \
$(addsuffix .$(OBJX),$(TEST_NGS_SRADB_CPP_SRC))
-$(TEST_BINDIR)/test-ngs_sradb-c++: $(TEST_NGS_SRADB_CPP_OBJ)
+$(TEST_BINDIR)/test-ngs_sradb-c++: $(TEST_NGS_SRADB_CPP_OBJ)
$(LP) --exe -o $@ $^ $(TEST_NGS_CPP_LIB)
+sradb: test-ngs_sradb-c++
+ $(TEST_BINDIR)/$^
+
#-------------------------------------------------------------------------------
# test-ngs_csra1-c++
#
@@ -109,9 +115,12 @@ TEST_NGS_CSRA1_CPP_SRC = \
TEST_NGS_CSRA1_CPP_OBJ = \
$(addsuffix .$(OBJX),$(TEST_NGS_CSRA1_CPP_SRC))
-$(TEST_BINDIR)/test-ngs_csra1-c++: $(TEST_NGS_CSRA1_CPP_OBJ)
+$(TEST_BINDIR)/test-ngs_csra1-c++: $(TEST_NGS_CSRA1_CPP_OBJ)
$(LP) --exe -o $@ $^ $(TEST_NGS_CPP_LIB)
+csra1: test-ngs_csra1-c++
+ $(TEST_BINDIR)/$^
+
#-------------------------------------------------------------------------------
# test-ngs_csra1pileup-c++
#
@@ -121,7 +130,10 @@ TEST_NGS_CSRA1PILEUP_CPP_SRC = \
TEST_NGS_CSRA1PILEUP_CPP_OBJ = \
$(addsuffix .$(OBJX),$(TEST_NGS_CSRA1PILEUP_CPP_SRC))
-$(TEST_BINDIR)/test-ngs_csra1pileup-c++: $(TEST_NGS_CSRA1PILEUP_CPP_OBJ)
+$(TEST_BINDIR)/test-ngs_csra1pileup-c++: $(TEST_NGS_CSRA1PILEUP_CPP_OBJ)
$(LP) --exe -o $@ $^ $(TEST_NGS_CPP_LIB)
+pileup: test-ngs_csra1pileup-c++
+ $(TEST_BINDIR)/$^
+
endif
diff --git a/test/ngs-c++/SRAtest.cpp b/test/ngs-c++/SRAtest.cpp
index 69857a1..26a9cfc 100644
--- a/test/ngs-c++/SRAtest.cpp
+++ b/test/ngs-c++/SRAtest.cpp
@@ -39,7 +39,7 @@ class SRAFixture : public NgsFixture
{
public:
static const char* SRA_Accession;
-
+
public:
SRAFixture()
{
@@ -50,15 +50,15 @@ public:
ngs :: ReadIterator getReads ( ngs :: Read :: ReadCategory cat = ngs :: Read :: all )
{
- return NgsFixture :: getReads ( SRA_Accession, cat );
+ return NgsFixture :: getReads ( SRA_Accession, cat );
}
ngs :: Read getRead ( const ngs :: String& p_id)
{
- return NgsFixture :: getRead ( SRA_Accession, p_id );
+ return NgsFixture :: getRead ( SRA_Accession, p_id );
}
ngs :: Reference getReference ( const char* spec )
{
- return NgsFixture :: getReference ( SRA_Accession, spec );
+ return NgsFixture :: getReference ( SRA_Accession, spec );
}
ngs :: Fragment getFragment ( const ngs :: String& p_readId, uint32_t p_fragIdx )
{
@@ -81,7 +81,7 @@ TEST_CASE(SRA_String_StringRefScope)
p = str_ref.data();
size = str_ref.size();
}
-
+
// data under p are still available for the current implementation
for (size_t i = 0; i < size; ++i)
@@ -114,21 +114,21 @@ FIXTURE_TEST_CASE(SRA_ReadCollection_GetName, SRAFixture)
FIXTURE_TEST_CASE(SRA_ReadCollection_GetReads, SRAFixture)
{
- ngs :: ReadIterator readIt = ncbi :: NGS :: openReadCollection ( SRA_Accession ) . getReads ( ngs :: Read :: all );
-}
+ ngs :: ReadIterator readIt = ncbi :: NGS :: openReadCollection ( SRA_Accession ) . getReads ( ngs :: Read :: all );
+}
FIXTURE_TEST_CASE(SRA_ReadCollection_GetReads_Filtered, SRAFixture)
{
- ngs :: ReadIterator readIt = ncbi :: NGS :: openReadCollection ( SRA_Accession ) . getReads ( ngs :: Read :: unaligned );
-}
+ ngs :: ReadIterator readIt = ncbi :: NGS :: openReadCollection ( SRA_Accession ) . getReads ( ngs :: Read :: unaligned );
+}
FIXTURE_TEST_CASE(SRA_ReadCollection_GetReadRange, SRAFixture)
{
- ngs :: ReadIterator readIt = ncbi :: NGS :: openReadCollection ( SRA_Accession ) . getReadRange ( 100, 200 );
-}
+ ngs :: ReadIterator readIt = ncbi :: NGS :: openReadCollection ( SRA_Accession ) . getReadRange ( 100, 200 );
+}
FIXTURE_TEST_CASE(SRA_ReadCollection_GetRead, SRAFixture)
{
- ngs :: Read read = getRead ( string ( SRA_Accession ) + ".R.1" );
+ ngs :: Read read = getRead ( string ( SRA_Accession ) + ".R.1" );
}
FIXTURE_TEST_CASE(SRA_ReadCollection_GetRead_Failed, SRAFixture)
@@ -139,8 +139,8 @@ FIXTURE_TEST_CASE(SRA_ReadCollection_GetRead_Failed, SRAFixture)
FIXTURE_TEST_CASE(SRA_ReadCollection_ReadCount, SRAFixture)
{
ngs :: ReadCollection run = ncbi :: NGS :: openReadCollection ( SRA_Accession );
- REQUIRE_EQ( (uint64_t)470985, run . getReadCount() );
-}
+ REQUIRE_EQ( (uint64_t)470985, run . getReadCount() );
+}
FIXTURE_TEST_CASE(SRA_ReadCollection_GetReference_Failed, SRAFixture)
{
@@ -151,7 +151,7 @@ FIXTURE_TEST_CASE(SRA_ReadCollection_GetReferences, SRAFixture)
{
ngs :: ReferenceIterator refIt = NgsFixture :: getReferences ( SRA_Accession ); // always an empty iterator
}
-
+
FIXTURE_TEST_CASE(SRA_ReadCollection_GetAlignment, SRAFixture)
{
ngs :: ReadCollection run = ncbi :: NGS :: openReadCollection ( SRA_Accession );
@@ -193,70 +193,70 @@ FIXTURE_TEST_CASE(SRA_ReadCollection_GetAlignmentRangeFiltered, SRAFixture)
//TODO: error cases (?)
FIXTURE_TEST_CASE(SRA_ReadIterator_NoReadBeforeNext, SRAFixture)
{
- ngs :: ReadIterator readIt = getReads ();
+ ngs :: ReadIterator readIt = getReads ();
// accessing the read through an iterator before a call to nextRead() throws
REQUIRE_THROW ( readIt . getReadId() );
}
FIXTURE_TEST_CASE(SRA_ReadIterator_Open_All, SRAFixture)
{
- ngs :: ReadIterator readIt = getReads ();
+ ngs :: ReadIterator readIt = getReads ();
REQUIRE( readIt . nextRead () );
ngs :: StringRef sref = readIt . getReadId();
ngs :: String id = sref . toString ();
- REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.1", id );
+ REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.1", id );
}
#if SHOW_UNIMPLEMENTED
FIXTURE_TEST_CASE(SRA_ReadIterator_Open_Filtered, SRAFixture)
{
- ngs :: ReadIterator readIt = getReads ( ngs :: Read :: partiallyAligned );
+ ngs :: ReadIterator readIt = getReads ( ngs :: Read :: partiallyAligned );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( (int64_t)5, readIt . getReadId() );
+ REQUIRE_EQ( (int64_t)5, readIt . getReadId() );
}
#endif
FIXTURE_TEST_CASE(SRA_ReadIterator_Next, SRAFixture)
{
- ngs :: ReadIterator readIt = getReads ();
+ ngs :: ReadIterator readIt = getReads ();
REQUIRE( readIt . nextRead () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.2", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.2", readIt . getReadId() . toString () );
}
FIXTURE_TEST_CASE(SRA_ReadIterator_End, SRAFixture)
{
- ngs :: ReadIterator readIt = getReads ();
+ ngs :: ReadIterator readIt = getReads ();
ngs :: String lastId;
while (readIt . nextRead ())
{
lastId = readIt . getReadId() . toString ();
}
- REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.470985", lastId );
+ REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.470985", lastId );
}
FIXTURE_TEST_CASE(SRA_ReadIterator_BeyondEnd, SRAFixture)
{
- ngs :: ReadIterator readIt = getReads ();
+ ngs :: ReadIterator readIt = getReads ();
while (readIt . nextRead ())
{
}
- REQUIRE_THROW ( readIt . getReadId() );
+ REQUIRE_THROW ( readIt . getReadId() );
}
FIXTURE_TEST_CASE(SRA_ReadIterator_Range, SRAFixture)
{
- ngs :: ReadIterator readIt = ncbi :: NGS :: openReadCollection ( SRA_Accession ) . getReadRange ( 10, 5 );
+ ngs :: ReadIterator readIt = ncbi :: NGS :: openReadCollection ( SRA_Accession ) . getReadRange ( 10, 5 );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.10", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.10", readIt . getReadId() . toString () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.11", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.11", readIt . getReadId() . toString () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.12", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.12", readIt . getReadId() . toString () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.13", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.13", readIt . getReadId() . toString () );
REQUIRE( readIt . nextRead () );
- REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.14", readIt . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.14", readIt . getReadId() . toString () );
REQUIRE( ! readIt . nextRead () );
}
@@ -265,12 +265,12 @@ FIXTURE_TEST_CASE(SRA_ReadIterator_Range, SRAFixture)
FIXTURE_TEST_CASE(SRA_Read_ReadId, SRAFixture)
{
- REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.1", getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadId() . toString () );
+ REQUIRE_EQ( ngs :: String ( SRA_Accession ) + ".R.1", getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadId() . toString () );
}
FIXTURE_TEST_CASE(SRA_Read_getNumFragments, SRAFixture)
{
- REQUIRE_EQ( ( uint32_t ) 2, getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getNumFragments() );
+ REQUIRE_EQ( ( uint32_t ) 2, getRead ( ngs :: String ( SRA_Accession ) + ".R.2" ) . getNumFragments() );
}
FIXTURE_TEST_CASE(SRA_Read_ReadName, SRAFixture)
@@ -290,63 +290,63 @@ FIXTURE_TEST_CASE(SRA_Read_ReadCategory, SRAFixture)
FIXTURE_TEST_CASE(SRA_Read_getReadBases, SRAFixture)
{
- ngs :: String bases = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadBases () . toString ();
- ngs :: String expected(
+ ngs :: String bases = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadBases () . toString ();
+ ngs :: String expected(
"TCAGATTCTCCTAGCCTACATCCGTACGAGTTAGCGTGGGATTACGAGGTGCACACCATTTCATTCCGTACGGGTAAATT"
"TTTGTATTTTTAGCAGACGGCAGGGTTTCACCATGGTTGACCAACGTACTAATCTTGAACTCCTGACCTCAAGTGATTTG"
"CCTGCCTTCAGCCTCCCAAAGTGACTGGGTATTACAGATGTGAGCGAGTTTGTGCCCAAGCCTTATAAGTAAATTTATAA"
"ATTTACATAATTTAAATGACTTATGCTTAGCGAAATAGGGTAAG"
);
- REQUIRE_EQ( expected, bases );
+ REQUIRE_EQ( expected, bases );
}
FIXTURE_TEST_CASE(SRA_Read_getReadBases_Offset, SRAFixture)
{
- ngs :: String bases = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadBases ( 200 ) . toString( );
+ ngs :: String bases = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadBases ( 200 ) . toString( );
ngs :: String expected(
"TGAGCGAGTTTGTGCCCAAGCCTTATAAGTAAATTTATAAATTTACATAATTTAAATGACTTATGCTTAGCGAAATAGGGTAAG"
);
- REQUIRE_EQ( expected, bases );
+ REQUIRE_EQ( expected, bases );
}
FIXTURE_TEST_CASE(SRA_Read_getReadBases_OffsetLength, SRAFixture)
{
- ngs :: String bases = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadBases ( 200, 6 ) . toString ();
+ ngs :: String bases = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadBases ( 200, 6 ) . toString ();
ngs :: String expected("TGAGCG");
- REQUIRE_EQ( expected, bases );
+ REQUIRE_EQ( expected, bases );
}
FIXTURE_TEST_CASE(SRA_Read_getReadQualities, SRAFixture)
{
- ngs :: String quals = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadQualities() . toString ();
+ ngs :: String quals = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadQualities() . toString ();
ngs :: String expected(
"=<8<85)9=9/3-8?68<7=8<3657747==49==+;FB2;A;5:'*>69<:74)9.;C?+;<B<B;(<';FA/;C>*GC"
"8/%9<=GC8.#=2:5:16D==<EA2EA.;5=44<;2C=5;@73&<<2;5;6+9<?776+:24'26:7,<9A;=:;0C>*6"
"?7<<C=D=<52?:9CA2CA23<2<;3CA12:A<9414<7<<6;99<2/=9#<;9B at 27.;=6>:77>:1<A>+CA138?<"
")C at 2166:A<B?->:%<<9<;33<;6?9;<;4=:%<$CA1+1%1");
- REQUIRE_EQ( expected, quals );
+ REQUIRE_EQ( expected, quals );
}
FIXTURE_TEST_CASE(SRA_Read_getReadQualities_Offset, SRAFixture)
{
- ngs :: String quals = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadQualities( 200 ) . toString ();
+ ngs :: String quals = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadQualities( 200 ) . toString ();
ngs :: String expected(
"<6;99<2/=9#<;9B at 27.;=6>:77>:1<A>+CA138?<)C at 2166:A<B?->:%<<9<;33<;6?9;<;4=:%<$CA1+1%1"
);
- REQUIRE_EQ( expected, quals );
+ REQUIRE_EQ( expected, quals );
}
FIXTURE_TEST_CASE(SRA_Read_getReadQualities_OffsetLength, SRAFixture)
{
- ngs :: String quals = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadQualities( 200, 10 ) . toString ();
+ ngs :: String quals = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" ) . getReadQualities( 200, 10 ) . toString ();
ngs :: String expected("<6;99<2/=9");
- REQUIRE_EQ( expected, quals );
+ REQUIRE_EQ( expected, quals );
}
///// FragmentIterator
FIXTURE_TEST_CASE(SRA_FragmentIterator_Next_and_Beyond, SRAFixture)
{
- ngs :: Read read = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" );
+ ngs :: Read read = getRead ( ngs :: String ( SRA_Accession ) + ".R.2" );
// 2 fragments total
REQUIRE ( read . nextFragment () ); // still on first fragment
REQUIRE ( read . nextFragment () );
@@ -359,13 +359,13 @@ FIXTURE_TEST_CASE(SRA_FragmentIterator_Next_and_Beyond, SRAFixture)
FIXTURE_TEST_CASE(SRA_Fragment_Id, SRAFixture)
{
- ngs :: Read read = getRead ( ngs :: String ( SRA_Accession ) + ".R.1" );
+ ngs :: Read read = getRead ( ngs :: String ( SRA_Accession ) + ".R.2" );
REQUIRE ( read . nextFragment () );
- REQUIRE_EQ ( string ( SRA_Accession ) + ".FR0.1", read . getFragmentId() . toString () );
-
+ REQUIRE_EQ ( string ( SRA_Accession ) + ".FR0.2", read . getFragmentId() . toString () );
+
REQUIRE ( read . nextFragment () );
- REQUIRE_EQ ( string ( SRA_Accession ) + ".FR1.1", read . getFragmentId() . toString () );
+ REQUIRE_EQ ( string ( SRA_Accession ) + ".FR1.2", read . getFragmentId() . toString () );
}
FIXTURE_TEST_CASE(SRA_Fragment_getFragmentBases, SRAFixture)
@@ -374,21 +374,21 @@ FIXTURE_TEST_CASE(SRA_Fragment_getFragmentBases, SRAFixture)
"ACAGACTCAACCTGCATAATAAATAACATTGAAACTTAGTTTCCTTCTTGGGCTTTCGGTGAGAAAACATAAGTTAAAAC"
"TGAGCGGGCTGGCAAGGCN"
);
- REQUIRE_EQ( expected,
- getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentBases () . toString ()
- );
+ REQUIRE_EQ( expected,
+ getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentBases () . toString ()
+ );
}
FIXTURE_TEST_CASE(SRA_Fragment_getSubFragmentBases_Offset, SRAFixture)
{
ngs :: String expected("GGCAAGGCN");
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentBases ( 90 ) . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentBases ( 90 ) . toString () );
}
FIXTURE_TEST_CASE(SRA_Fragment_getSubFragmentBases_OffsetLength, SRAFixture)
{
ngs :: String expected("GGCAAG");
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentBases ( 90, 6 ) . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentBases ( 90, 6 ) . toString () );
}
FIXTURE_TEST_CASE(SRA_Fragment_getFragmentQualities, SRAFixture)
@@ -397,19 +397,19 @@ FIXTURE_TEST_CASE(SRA_Fragment_getFragmentQualities, SRAFixture)
"<=::=<8=D=C<<<<<<A=;CA1<=7<;A<;CA1<@:<9>;&>7;4<>7CA0<C at 0:<5<;:<CA7+:7<@9<B=CA7+<"
"<99<:B?.<;:2<A<A;:!"
);
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentQualities () . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentQualities () . toString () );
}
FIXTURE_TEST_CASE(SRA_Fragment_getSubFragmentQualities_Offset, SRAFixture)
{
ngs :: String expected(":2<A<A;:!");
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentQualities ( 90 ) . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentQualities ( 90 ) . toString () );
}
FIXTURE_TEST_CASE(SRA_Fragment_getSubFragmentQualities_OffsetLength, SRAFixture)
{
ngs :: String expected(":2<A<A");
- REQUIRE_EQ( expected, getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentQualities ( 90, 6 ) . toString () );
+ REQUIRE_EQ( expected, getFragment ( ngs :: String ( SRA_Accession ) + ".R.2", 2 ) . getFragmentQualities ( 90, 6 ) . toString () );
}
/////ReferenceIterator
diff --git a/test/ngs-java/ngs_test_CSRA1.java b/test/ngs-java/ngs_test_CSRA1.java
index 069a51e..e38d055 100644
--- a/test/ngs-java/ngs_test_CSRA1.java
+++ b/test/ngs-java/ngs_test_CSRA1.java
@@ -32,6 +32,8 @@ import static org.junit.Assert.assertFalse;
import org.junit.Test;
import ngs.ReadCollection;
+import ngs.ReadGroupIterator;
+import ngs.Statistics;
import ngs.Alignment;
import ngs.ErrorMsg;
@@ -857,4 +859,105 @@ public class ngs_test_CSRA1 {
assertEquals ( name, r . getReadGroup () );
*/
}
+
+ @Test
+ public void ReadCollection_openReadCollection_null() throws ngs.ErrorMsg
+ {
+ try
+ {
+ NGS.openReadCollection(null);
+ fail();
+ }
+ catch ( ngs.ErrorMsg e ) {}
+ }
+
+ @Test
+ public void ReadCollection_setAppVersion_null() throws ngs.ErrorMsg
+ {
+ NGS.setAppVersionString(null);
+ }
+
+ @Test
+ public void ReadCollection_openReferenceSequence_null() throws ngs.ErrorMsg
+ {
+ try
+ {
+ NGS.openReferenceSequence(null);
+ fail();
+ }
+ catch ( ngs.ErrorMsg e ) {}
+ }
+
+ @Test
+ public void ReadCollection_isValid_null() throws ngs.ErrorMsg
+ {
+ try
+ {
+ NGS.isValid(null);
+ fail();
+ }
+ catch ( NullPointerException e ) {}
+ }
+
+ @Test
+ public void ReadCollection_hasReadGroup_null() throws ngs.ErrorMsg
+ {
+ try
+ {
+ ReadCollection run = NGS.openReadCollection(PrimaryOnly);
+ run.hasReadGroup(null);
+ }
+ catch ( ngs.ErrorMsg e )
+ {
+ fail();
+ }
+ }
+
+ @Test
+ public void ReadCollection_hasReference_null() throws ngs.ErrorMsg
+ {
+ try
+ {
+ ReadCollection run = NGS.openReadCollection(PrimaryOnly);
+ run.hasReference(null);
+ }
+ catch ( ngs.ErrorMsg e )
+ {
+ fail();
+ }
+ }
+
+ @Test
+ public void ReadCollection_getReference_null() throws ngs.ErrorMsg
+ {
+ try
+ {
+ ReadCollection run = NGS.openReadCollection(PrimaryOnly);
+ run.getReference(null);
+ fail();
+ }
+ catch ( ngs.ErrorMsg e ) {}
+ }
+
+ @Test
+ public void ReadCollection_getAlignment_null() throws ngs.ErrorMsg
+ {
+ try
+ {
+ ReadCollection run = NGS.openReadCollection(PrimaryOnly);
+ run.getAlignment(null);
+ fail();
+ }
+ catch ( ngs.ErrorMsg e ) {}
+ }
+
+ @Test
+ public void ReadCollection_ReadGroup_Statistics_getValueType_null() throws ngs.ErrorMsg
+ {
+ ReadCollection run = NGS.openReadCollection(PrimaryOnly);
+ ReadGroupIterator rgIt = run.getReadGroups();
+ rgIt.nextReadGroup();
+ Statistics st = rgIt.getStatistics();
+ st.getValueType(null);
+ }
}
diff --git a/test/ngs/ngstest.cpp b/test/ngs/ngstest.cpp
index 9c05b7a..0f11578 100644
--- a/test/ngs/ngstest.cpp
+++ b/test/ngs/ngstest.cpp
@@ -33,6 +33,8 @@
#include "ngs_c_fixture.hpp"
+#include <kfc/xc.h>
+
#include <SRA_ReadGroupInfo.h>
#include <SRA_Statistics.h>
#include <NGS_Cursor.h>
@@ -47,6 +49,7 @@
#include <vdb/table.h>
#include <vdb/database.h>
+#include <vdb/blob.h>
#include <stdexcept>
#include <cstring>
@@ -768,7 +771,9 @@ TEST_CASE(NGS_Statistics_ConversionString_TrailingSpace)
TEST_CASE(NGS_FailedToOpen)
{
HYBRID_FUNC_ENTRY ( rcSRA, rcRow, rcAccessing );
+
NGS_ReadCollectionMake ( ctx, BAD_ACCESSION);
+ REQUIRE ( ctx_xc_isa ( ctx, xcTableOpenFailed ) );
KConfig* kfg;
REQUIRE_RC ( KConfigMakeLocal ( &kfg, NULL ) );
@@ -776,13 +781,16 @@ TEST_CASE(NGS_FailedToOpen)
REQUIRE_RC ( KConfigMakeRepositoryMgrRead ( kfg, &repoMgr ) );
if ( KRepositoryMgrHasRemoteAccess ( repoMgr ) )
{
- REQUIRE_EQ ( string ( "Cannot open accession '" BAD_ACCESSION "'"),
- string ( WHAT () ) );
+ string startsWith = "Cannot open accession '" BAD_ACCESSION "', rc = RC";
+ REQUIRE_EQ ( startsWith, string ( WHAT () ) . substr ( 0, startsWith . size () ) );
}
else
{
- REQUIRE_EQ ( string ( "Cannot open accession '" BAD_ACCESSION "'. Note: remote access is disabled in the configuration"),
- string ( WHAT () ) );
+ string startsWith = "Cannot open accession '" BAD_ACCESSION "', rc = RC";
+ string what = WHAT ();
+ REQUIRE_EQ ( startsWith, what . substr ( 0, startsWith . size () ) );
+ string endsWith = "Note: remote access is disabled in the configuration";
+ REQUIRE_EQ ( endsWith, what . substr ( what . size() - endsWith . size () ) );
}
REQUIRE_FAILED ();
REQUIRE_RC ( KRepositoryMgrRelease ( repoMgr ) );
@@ -808,52 +816,111 @@ TEST_CASE(NGS_OpenBySysPath)
//////////////////////////////////////////// NGS_Cursor
-TEST_CASE ( NGS_Cursor_GetColumnIndex_adds_column)
+class NGSCursorFixture : public NGS_C_Fixture
{
- HYBRID_FUNC_ENTRY ( rcSRA, rcRow, rcAccessing );
+public:
+ NGSCursorFixture()
+ : m_cursor ( 0 )
+ {
+ }
- const VDBManager * mgr = ctx -> rsrc -> vdb;
- REQUIRE_NOT_NULL ( mgr );
+ ~NGSCursorFixture()
+ {
+ }
- const VDatabase *db;
- REQUIRE_RC ( VDBManagerOpenDBRead ( mgr, & db, NULL, "%s", SRADB_Accession_WithBamHeader ) );
+ virtual void Release()
+ {
+ if (m_ctx != 0)
+ {
+ NGS_CursorRelease ( m_cursor, m_ctx );
+ }
+ NGS_C_Fixture :: Release ();
+ }
- VTable* tbl;
- REQUIRE_RC ( VDatabaseOpenTableRead ( db, (const VTable**)&tbl, "SEQUENCE" ) );
+ void MakeCursor ( const string& p_acc, const string& p_table, const char * col_specs[], uint32_t num_cols )
+ {
+ const VDatabase *db;
+ THROW_ON_RC ( VDBManagerOpenDBRead ( m_ctx -> rsrc -> vdb, & db, NULL, "%s", p_acc . c_str () ) );
- const NGS_Cursor* curs = NGS_CursorMake ( ctx, tbl, sequence_col_specs, seq_NUM_COLS ); // this will add the first column (READ) to the cursor
- REQUIRE ( ! FAILED () );
+ VTable* tbl;
+ THROW_ON_RC ( VDatabaseOpenTableRead ( db, (const VTable**)&tbl, p_table . c_str () ) );
- REQUIRE_NE ( (uint32_t)0, NGS_CursorGetColumnIndex ( curs, ctx, seq_READ_LEN ) ); // this should add the column we are requesting to the cursor
+ m_cursor = NGS_CursorMake ( m_ctx, tbl, col_specs, num_cols ); // this will add the first column to the cursor
- NGS_CursorRelease ( curs, ctx );
- REQUIRE ( ! FAILED () );
+ THROW_ON_RC ( VTableRelease ( tbl ) );
+ THROW_ON_RC ( VDatabaseRelease ( db ) );
+ }
+
+ const NGS_Cursor* m_cursor;
+};
+
+FIXTURE_TEST_CASE ( NGS_CursorMakeDB_fails, NGSCursorFixture )
+{
+ ENTRY;
+ const VDatabase *db;
+ THROW_ON_RC ( VDBManagerOpenDBRead ( m_ctx -> rsrc -> vdb, & db, NULL, "%s", SRADB_Accession_WithBamHeader ) );
+
+ NGS_String* runName = NGS_StringMake ( ctx, SRADB_Accession_WithBamHeader, string_size ( SRADB_Accession_WithBamHeader ) );
+ m_cursor = NGS_CursorMakeDb ( m_ctx, db, runName, "bad table", sequence_col_specs, seq_NUM_COLS );
+ REQUIRE ( FAILED () );
+ REQUIRE_NULL ( m_cursor );
+ // make sure RC is reported
+ REQUIRE ( string ( WHAT () ) . find ( "rc =" ) != string :: npos);
+ CLEAR ();
- REQUIRE_RC ( VTableRelease ( tbl ) );
+ NGS_StringRelease ( runName, ctx );
REQUIRE_RC ( VDatabaseRelease ( db ) );
+ EXIT;
}
-TEST_CASE ( NGS_Cursor_Leak_when_Make_fails )
+
+FIXTURE_TEST_CASE ( NGS_Cursor_GetColumnIndex_adds_column, NGSCursorFixture )
{
- HYBRID_FUNC_ENTRY ( rcSRA, rcRow, rcAccessing );
+ ENTRY;
+ MakeCursor ( SRADB_Accession_WithBamHeader, "SEQUENCE", sequence_col_specs, seq_NUM_COLS ); // this will add the first column (READ) to the cursor
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( m_cursor );
- const VDBManager * mgr = ctx -> rsrc -> vdb;
- REQUIRE_NOT_NULL ( mgr );
+ REQUIRE_NE ( (uint32_t)0, NGS_CursorGetColumnIndex ( m_cursor, ctx, seq_READ_LEN ) ); // this should add the column we are requesting to the cursor
- const VDatabase *db;
- REQUIRE_RC ( VDBManagerOpenDBRead ( mgr, & db, NULL, "%s", SRADB_Accession_WithBamHeader ) );
+ EXIT;
+}
- VTable* tbl;
- REQUIRE_RC ( VDatabaseOpenTableRead ( db, (const VTable**)&tbl, "SEQUENCE" ) );
+FIXTURE_TEST_CASE ( NGS_Cursor_Leak_when_Make_fails, NGSCursorFixture )
+{ // use valgrind to detect the absence of a leak
+ ENTRY;
const char * bogus_col_specs [] = { "not a column at all!" };
- REQUIRE_NULL ( NGS_CursorMake ( ctx, tbl, bogus_col_specs, 1 ) );
+ MakeCursor ( SRADB_Accession_WithBamHeader, "SEQUENCE", bogus_col_specs, 1 );
REQUIRE ( FAILED () );
+ REQUIRE_NULL ( m_cursor );
CLEAR();
- REQUIRE_RC ( VTableRelease ( tbl ) );
- REQUIRE_RC ( VDatabaseRelease ( db ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE ( NGS_Cursor_GetVBlob, NGSCursorFixture )
+{
+ ENTRY;
+ MakeCursor ( SRADB_Accession_WithBamHeader, "SEQUENCE", sequence_col_specs, seq_NUM_COLS ); // this will add the first column (READ) to the cursor
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( m_cursor );
+
+ const int64_t rowId = 10;
+ const struct VBlob* blob = NGS_CursorGetVBlob ( m_cursor, ctx, rowId, seq_READ );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( blob );
+
+ int64_t first;
+ uint64_t count;
+ REQUIRE_RC ( VBlobIdRange ( blob, &first, &count ) );
+ REQUIRE_LE ( first, rowId );
+ REQUIRE_GT ( (int64_t)( first + count ), rowId );
+
+ REQUIRE_RC ( VBlobRelease ( blob ) );
+
+ EXIT;
}
//////////////////////////////////////////// Main
diff --git a/test/ngs/ngstest_csra1.cpp b/test/ngs/ngstest_csra1.cpp
index 00b6072..643c651 100644
--- a/test/ngs/ngstest_csra1.cpp
+++ b/test/ngs/ngstest_csra1.cpp
@@ -500,6 +500,19 @@ FIXTURE_TEST_CASE(CSRA1_NGS_AlignmentIsPrimary_No, CSRA1_Fixture)
EXIT;
}
+FIXTURE_TEST_CASE(CSRA1_NGS_AlignmentIsFirst_Yes, CSRA1_Fixture)
+{
+ ENTRY_GET_ALIGN( CSRA1_PrimaryOnly, 1 );
+ REQUIRE( NGS_AlignmentIsFirst ( m_align, ctx ) );
+ EXIT;
+}
+FIXTURE_TEST_CASE(CSRA1_NGS_AlignmentIsFirst_No, CSRA1_Fixture)
+{
+ ENTRY_GET_ALIGN( CSRA1_PrimaryOnly, 2 );
+ REQUIRE( ! NGS_AlignmentIsFirst ( m_align, ctx ) );
+ EXIT;
+}
+
FIXTURE_TEST_CASE(CSRA1_NGS_AlignmentGetAlignmentPosition, CSRA1_Fixture)
{
ENTRY_GET_ALIGN( CSRA1_PrimaryOnly, 1 );
@@ -737,7 +750,6 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetBases_Full, CSRA1_Fixture)
REQUIRE_EQ ( NGS_ReferenceGetLength ( m_ref, ctx ), ( uint64_t ) bases. size () );
EXIT;
}
-
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetBases, CSRA1_Fixture)
{
ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
@@ -897,6 +909,7 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_FullReference_NoW
EXIT;
}
+// NGS_ReferenceGetChunk
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetChunk_Empty, CSRA1_Fixture)
{ // offset beyond the end of reference
ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
@@ -904,6 +917,109 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetChunk_Empty, CSRA1_Fixture)
EXIT;
}
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_All, CSRA1_Fixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 0, (size_t)-1 );
+ REQUIRE_EQ( (size_t)20000, NGS_StringSize ( chunk, ctx ) );
+
+ string str = string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
+ REQUIRE_EQ( string("GAATTCT"), str . substr (0, 7) );
+ REQUIRE_EQ( string("CATCA"), str . substr ( str.size() - 5, 5 ) );
+
+ NGS_StringRelease ( chunk, ctx );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Offset_1, CSRA1_Fixture)
+{ // offset points into the first blob of REFERENCE.READ
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 1000, (size_t)-1 );
+ REQUIRE_EQ( (size_t)19000, NGS_StringSize ( chunk, ctx ) ); // first blob's size is 20000
+
+ string str = string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
+ REQUIRE_EQ( string("TCCATTC"), str . substr (0, 7) );
+ REQUIRE_EQ( string("CATCA"), str . substr (str.size() - 5, 5) );
+
+ NGS_StringRelease ( chunk, ctx );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Offset_2, CSRA1_Fixture)
+{ // offset points into the second blob of REFERENCE.READ
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 22000, (size_t)-1 );
+ REQUIRE_EQ( (size_t)3000, NGS_StringSize ( chunk, ctx ) ); // second blob's size is 5000
+
+ string str = string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) );
+ REQUIRE_EQ( string("CTCAGAT"), str . substr (0, 7) );
+ REQUIRE_EQ( string("TATTC"), str . substr (str.size() - 5, 5) );
+
+ NGS_StringRelease ( chunk, ctx );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_1, CSRA1_Fixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 2000, 10 );
+ REQUIRE_EQ( string ( "GGGCAAATGA" ), string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) ) );
+ NGS_StringRelease ( chunk, ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_2, CSRA1_Fixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 20020, 10 );
+ REQUIRE_EQ( string ( "ACATGACGGA" ), string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) ) );
+ NGS_StringRelease ( chunk, ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_OffsetLength_ReturnShorter, CSRA1_Fixture)
+{
+ ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
+
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, 19995, 200 ); // only 5 bases left in this blob
+ REQUIRE_EQ( string ( "CATCA" ), string ( NGS_StringData ( chunk, ctx ), NGS_StringSize ( chunk, ctx ) ) );
+ NGS_StringRelease ( chunk, ctx );
+
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(CSRA1_Reference_GetReferenceChunk_Chunks_Vs_Blobs, CSRA1_Fixture)
+{
+ ENTRY_GET_REF( "SRR600094", "NC_000022.10" );
+
+ NGS_String * full = NGS_ReferenceGetBases ( m_ref, ctx, 0, (uint64_t)-1 );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( full );
+ REQUIRE_EQ ( (uint64_t)NGS_ReferenceGetLength ( m_ref, ctx ), (uint64_t)NGS_StringSize ( full, ctx ) );
+
+ size_t offset=0;
+ while ( offset < NGS_StringSize ( full, ctx ) )
+ {
+ NGS_String * chunk = NGS_ReferenceGetChunk ( m_ref, ctx, offset, 5000 );
+ REQUIRE ( ! FAILED () );
+ REQUIRE_NOT_NULL ( chunk );
+ size_t chunkSize = NGS_StringSize ( chunk, ctx );
+ REQUIRE_EQ ( string ( NGS_StringData ( full, ctx ) + offset, chunkSize ), string ( NGS_StringData ( chunk, ctx ), chunkSize ) );
+ NGS_StringRelease ( chunk, ctx );
+ offset += chunkSize;
+ }
+
+ NGS_StringRelease ( full, ctx );
+
+ EXIT;
+}
+
+//
+
FIXTURE_TEST_CASE(CSRA1_NGS_Reference_SharedCursor, CSRA1_Fixture)
{
ENTRY_GET_REF( CSRA1_PrimaryOnly, "supercont2.1" );
@@ -1242,8 +1358,7 @@ const char UsageDefaultName[] = "test-ngs_csra1";
rc_t CC KMain ( int argc, char *argv [] )
{
- rc_t m_coll=NgsCsra1TestSuite(argc, argv);
- return m_coll;
+ return NgsCsra1TestSuite(argc, argv);
}
}
diff --git a/test/ngs/ngstest_csra1_refwin.cpp b/test/ngs/ngstest_csra1_refwin.cpp
index 1abd39c..f403ce0 100644
--- a/test/ngs/ngstest_csra1_refwin.cpp
+++ b/test/ngs/ngstest_csra1_refwin.cpp
@@ -69,7 +69,7 @@ public:
}
NGS_C_Fixture :: Release ();
}
-
+
void GetAlignment(uint64_t id)
{
stringstream str;
@@ -84,21 +84,21 @@ public:
throw logic_error ("premature end of iterator");
if ( m_ctx -> rc != 0 )
throw logic_error ("NGS_AlignmentIteratorNext failed");
-
+
if (print)
cout << toString ( NGS_AlignmentGetAlignmentId ( m_align, m_ctx ), m_ctx, true ) << endl;
}
}
void PrintAlignment ()
{
- cout << "id=" << toString ( NGS_AlignmentGetAlignmentId ( m_align, m_ctx ), m_ctx, true )
- << " pos=" << NGS_AlignmentGetAlignmentPosition ( m_align, m_ctx )
- << " len=" << NGS_AlignmentGetAlignmentLength ( m_align, m_ctx )
+ cout << "id=" << toString ( NGS_AlignmentGetAlignmentId ( m_align, m_ctx ), m_ctx, true )
+ << " pos=" << NGS_AlignmentGetAlignmentPosition ( m_align, m_ctx )
+ << " len=" << NGS_AlignmentGetAlignmentLength ( m_align, m_ctx )
<< ( NGS_AlignmentIsPrimary ( m_align, m_ctx ) ? " PRI" : " SEC" )
<< " mapq=" << NGS_AlignmentGetMappingQuality( m_align, m_ctx )
- << endl;
+ << endl;
}
-
+
bool NextId (const string& p_expected)
{
if ( ! NGS_AlignmentIteratorNext ( m_align, m_ctx ) || m_ctx -> rc != 0 )
@@ -114,7 +114,7 @@ public:
}
return m_ctx -> rc == 0;
}
-
+
bool VerifyOrder()
{
uint64_t prevPos = 0;
@@ -132,7 +132,7 @@ public:
PrintAlignment();
return false;
}
- if ( prevPos == newPos )
+ if ( prevPos == newPos )
{
if ( prevLen < newLen )
{
@@ -163,7 +163,7 @@ public:
}
return true;
}
-
+
NGS_Alignment* m_align;
};
@@ -171,52 +171,54 @@ public:
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_NoAccessBeforeNext, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_PrimaryOnly, "supercont2.1" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
- REQUIRE_NULL ( NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
+
+ REQUIRE_NULL ( NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
REQUIRE_FAILED ();
- REQUIRE_NULL ( NGS_AlignmentGetReferenceSpec ( m_align, ctx ) );
+ REQUIRE_NULL ( NGS_AlignmentGetReferenceSpec ( m_align, ctx ) );
REQUIRE_FAILED ();
- NGS_AlignmentGetMappingQuality ( m_align, ctx );
+ NGS_AlignmentGetMappingQuality ( m_align, ctx );
REQUIRE_FAILED ();
- REQUIRE_NULL ( NGS_AlignmentGetReadGroup ( m_align, ctx ) );
+ REQUIRE_NULL ( NGS_AlignmentGetReadGroup ( m_align, ctx ) );
REQUIRE_FAILED ();
- REQUIRE_NULL ( NGS_AlignmentGetReadId ( m_align, ctx ) );
+ REQUIRE_NULL ( NGS_AlignmentGetReadId ( m_align, ctx ) );
REQUIRE_FAILED ();
- REQUIRE_NULL ( NGS_AlignmentGetClippedFragmentBases ( m_align, ctx ) );
+ REQUIRE_NULL ( NGS_AlignmentGetClippedFragmentBases ( m_align, ctx ) );
REQUIRE_FAILED ();
- REQUIRE_NULL ( NGS_AlignmentGetClippedFragmentQualities ( m_align, ctx ) );
+ REQUIRE_NULL ( NGS_AlignmentGetClippedFragmentQualities ( m_align, ctx ) );
REQUIRE_FAILED ();
- NGS_AlignmentIsPrimary ( m_align, ctx );
+ NGS_AlignmentIsPrimary ( m_align, ctx );
REQUIRE_FAILED ();
- NGS_AlignmentGetAlignmentPosition ( m_align, ctx );
+ NGS_AlignmentGetAlignmentPosition ( m_align, ctx );
REQUIRE_FAILED ();
- NGS_AlignmentGetAlignmentLength ( m_align, ctx );
+ NGS_AlignmentGetAlignmentLength ( m_align, ctx );
REQUIRE_FAILED ();
- NGS_AlignmentGetIsReversedOrientation ( m_align, ctx );
+ NGS_AlignmentGetIsReversedOrientation ( m_align, ctx );
REQUIRE_FAILED ();
- NGS_AlignmentGetSoftClip ( m_align, ctx, true );
+ NGS_AlignmentGetSoftClip ( m_align, ctx, true );
REQUIRE_FAILED ();
- NGS_AlignmentGetTemplateLength ( m_align, ctx );
+ NGS_AlignmentGetTemplateLength ( m_align, ctx );
REQUIRE_FAILED ();
- REQUIRE_NULL ( NGS_AlignmentGetShortCigar ( m_align, ctx, true ) );
+ REQUIRE_NULL ( NGS_AlignmentGetShortCigar ( m_align, ctx, true ) );
REQUIRE_FAILED ();
- REQUIRE_NULL ( NGS_AlignmentGetLongCigar ( m_align, ctx, true ) );
+ REQUIRE_NULL ( NGS_AlignmentGetLongCigar ( m_align, ctx, true ) );
+ REQUIRE_FAILED ();
+ NGS_AlignmentIsFirst ( m_align, ctx );
REQUIRE_FAILED ();
- REQUIRE ( ! NGS_AlignmentHasMate ( m_align, ctx ) );
+ REQUIRE ( ! NGS_AlignmentHasMate ( m_align, ctx ) );
REQUIRE ( ! FAILED () ); /* HasMate does not throw */
-
- REQUIRE_NULL ( NGS_AlignmentGetMateAlignmentId ( m_align, ctx ) );
+
+ REQUIRE_NULL ( NGS_AlignmentGetMateAlignmentId ( m_align, ctx ) );
+ REQUIRE_FAILED ();
+ REQUIRE_NULL ( NGS_AlignmentGetMateAlignment ( m_align, ctx ) );
REQUIRE_FAILED ();
- REQUIRE_NULL ( NGS_AlignmentGetMateAlignment ( m_align, ctx ) );
+ REQUIRE_NULL ( NGS_AlignmentGetMateReferenceSpec ( m_align, ctx ) );
REQUIRE_FAILED ();
- REQUIRE_NULL ( NGS_AlignmentGetMateReferenceSpec ( m_align, ctx ) );
+ NGS_AlignmentGetMateIsReversedOrientation ( m_align, ctx );
REQUIRE_FAILED ();
- NGS_AlignmentGetMateIsReversedOrientation ( m_align, ctx );
- REQUIRE_FAILED ();
-
+
// fragment functions
REQUIRE_NULL ( NGS_FragmentGetId ( (NGS_Fragment*)m_align, ctx ) );
REQUIRE_FAILED ();
@@ -224,96 +226,99 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_NoAccessBeforeNext, CSRA1_Fixture)
REQUIRE_FAILED ();
REQUIRE_NULL ( NGS_FragmentGetQualities ( (NGS_Fragment*)m_align, ctx, 2, 5 ) );
REQUIRE_FAILED ();
-
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_AccessAfterNext, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_PrimaryOnly, "supercont2.1" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
+
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
- REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".PA.1", NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
+ REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".PA.1", NGS_AlignmentGetAlignmentId ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
- REQUIRE_STRING ( "supercont2.1", NGS_AlignmentGetReferenceSpec ( m_align, ctx ) );
+ REQUIRE_STRING ( "supercont2.1", NGS_AlignmentGetReferenceSpec ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_EQ ( 60, NGS_AlignmentGetMappingQuality ( m_align, ctx ) );
+
+ REQUIRE_EQ ( 60, NGS_AlignmentGetMappingQuality ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( "C1ELY.6", NGS_AlignmentGetReadGroup ( m_align, ctx ) );
+
+ REQUIRE_STRING ( "C1ELY.6", NGS_AlignmentGetReadGroup ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".R.1", NGS_AlignmentGetReadId ( m_align, ctx ) );
+
+ REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".R.1", NGS_AlignmentGetReadId ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( "ACTCGACATTCTGCCTTCGACCTATCTTTCTCCTCTCCCAGTCATCGCCCAGTAGAATTACCAGGCAATGAACCAGGGCCTTCCATCCCAACGGCACAGCA",
+
+ REQUIRE_STRING ( "ACTCGACATTCTGCCTTCGACCTATCTTTCTCCTCTCCCAGTCATCGCCCAGTAGAATTACCAGGCAATGAACCAGGGCCTTCCATCCCAACGGCACAGCA",
NGS_AlignmentGetClippedFragmentBases ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( "@@CDDBDFFBFHFIEEFGIGGHIEHIGIGGFGEGAFDHIIIIIGGGDFHII;=BF at FEHGIEEH?AHHFHFFFFDC5'5=?CC?ADCD at AC??9BDDCDB<",
+
+ REQUIRE_STRING ( "@@CDDBDFFBFHFIEEFGIGGHIEHIGIGGFGEGAFDHIIIIIGGGDFHII;=BF at FEHGIEEH?AHHFHFFFFDC5'5=?CC?ADCD at AC??9BDDCDB<",
NGS_AlignmentGetClippedFragmentQualities ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE ( NGS_AlignmentIsPrimary ( m_align, ctx ) );
+
+ REQUIRE ( NGS_AlignmentIsPrimary ( m_align, ctx ) );
+ REQUIRE ( ! FAILED () );
+
+ REQUIRE_EQ ( (int64_t)85, NGS_AlignmentGetAlignmentPosition ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_EQ ( (int64_t)85, NGS_AlignmentGetAlignmentPosition ( m_align, ctx ) );
+
+ REQUIRE_EQ ( (uint64_t)101, NGS_AlignmentGetAlignmentLength ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_EQ ( (uint64_t)101, NGS_AlignmentGetAlignmentLength ( m_align, ctx ) );
+
+ REQUIRE ( ! NGS_AlignmentGetIsReversedOrientation ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE ( ! NGS_AlignmentGetIsReversedOrientation ( m_align, ctx ) );
+
+ REQUIRE_EQ ( (int)0, NGS_AlignmentGetSoftClip ( m_align, ctx, true ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_EQ ( (int)0, NGS_AlignmentGetSoftClip ( m_align, ctx, true ) );
+
+ REQUIRE_EQ ( (uint64_t)201, NGS_AlignmentGetTemplateLength ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_EQ ( (uint64_t)201, NGS_AlignmentGetTemplateLength ( m_align, ctx ) );
+
+ REQUIRE_STRING ( "101M", NGS_AlignmentGetShortCigar ( m_align, ctx, true ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( "101M", NGS_AlignmentGetShortCigar ( m_align, ctx, true ) );
+
+ REQUIRE_STRING ( "13=1X61=1X6=1X18=", NGS_AlignmentGetLongCigar ( m_align, ctx, true ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( "13=1X61=1X6=1X18=", NGS_AlignmentGetLongCigar ( m_align, ctx, true ) );
+
+ REQUIRE ( NGS_AlignmentHasMate ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
- REQUIRE ( NGS_AlignmentHasMate ( m_align, ctx ) );
- REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".PA.2", NGS_AlignmentGetMateAlignmentId ( m_align, ctx ) );
+ REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".PA.2", NGS_AlignmentGetMateAlignmentId ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
+
NGS_Alignment * mate = NGS_AlignmentGetMateAlignment ( m_align, ctx );
- REQUIRE_NOT_NULL ( mate );
+ REQUIRE_NOT_NULL ( mate );
REQUIRE ( ! FAILED () );
NGS_RefcountRelease ( NGS_AlignmentToRefcount ( mate ) , ctx );
REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( "supercont2.1", NGS_AlignmentGetMateReferenceSpec ( m_align, ctx ) );
+
+ REQUIRE_STRING ( "supercont2.1", NGS_AlignmentGetMateReferenceSpec ( m_align, ctx ) );
+ REQUIRE ( ! FAILED () );
+
+ REQUIRE ( NGS_AlignmentGetMateIsReversedOrientation ( m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
- REQUIRE ( NGS_AlignmentGetMateIsReversedOrientation ( m_align, ctx ) );
+
+ NGS_AlignmentIsFirst ( m_align, ctx );
REQUIRE ( ! FAILED () );
-
+
// fragment functions
- REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".FA0.1", NGS_FragmentGetId ( (NGS_Fragment*)m_align, ctx ) );
+ REQUIRE_STRING ( string ( CSRA1_PrimaryOnly ) + ".FA0.1", NGS_FragmentGetId ( (NGS_Fragment*)m_align, ctx ) );
REQUIRE ( ! FAILED () );
-
+
REQUIRE_STRING ( "ACTCG", NGS_FragmentGetSequence ( (NGS_Fragment*)m_align, ctx, 0, 5 ) );
REQUIRE ( ! FAILED () );
-
+
REQUIRE_STRING ( "@@CDD", NGS_FragmentGetQualities ( (NGS_Fragment*)m_align, ctx, 0, 5 ) );
REQUIRE ( ! FAILED () );
-
+
REQUIRE ( ! NGS_FragmentIteratorNext ( (NGS_Fragment*)m_align, ctx ) );
REQUIRE_FAILED ();
-
+
EXIT;
}
@@ -322,65 +327,65 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_AccessAfterNext, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Primary, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.1") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.2") );
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
+
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.1") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.2") );
// etc
-
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Primary_Only, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
+
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
// skip to where a secondary alignment which would show up, between primary alignments 34 and 35
SkipAlignments( 33 );
-
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.34") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.35") );
-
+
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.34") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.35") );
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Secondary, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, false, true );
-
-
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.169") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.170") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.171") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.172") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.173") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.174") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.175") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.176") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.177") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.178") );
-
+
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, false, true );
+
+
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.169") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.170") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.171") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.172") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.173") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.174") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.175") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.176") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.177") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.178") );
+
REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_PrimaryAndSecondary, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, true );
+
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, true );
// skip to where a secondary alignment is expected, between primary alignments 34 and 35
SkipAlignments( 33 );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.34") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.169") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.35") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.34") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.169") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.35") );
EXIT;
}
@@ -388,36 +393,36 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_PrimaryAndSecondary, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Sort, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, true );
+
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, true );
// secondary 170 should go before primary 62
SkipAlignments( 61 ); /*skip over 60 primary and 1 secondary */
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.61") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.170") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.62") );
-
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.61") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.170") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.62") );
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_FullPrimary_Order, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, false );
REQUIRE ( VerifyOrder() );
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_FullSecondary_Order, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, false, true );
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, false, true );
REQUIRE ( VerifyOrder() );
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_FullAll_Order, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, true );
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, true );
REQUIRE ( VerifyOrder() );
EXIT;
}
@@ -425,23 +430,23 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_FullAll_Order, CSRA1_Fixture)
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Empty, CSRA1_Fixture)
{
ENTRY_GET_REF( "SRR960954", "gi|296100371|ref|NC_014121.1|" );
-
- m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 0, 20000, false, false );
+
+ m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 0, 20000, false, false );
REQUIRE ( ! FAILED () );
-
+
REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_OutOfBounds, CSRA1_Fixture)
{
ENTRY_GET_REF( "SRR960954", "gi|296100371|ref|NC_014121.1|" );
-
- m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, NGS_ReferenceGetLength ( m_ref, ctx ), 1, true, true );
+
+ m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, NGS_ReferenceGetLength ( m_ref, ctx ), 1, true, true );
REQUIRE ( ! FAILED () );
-
+
REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
+
EXIT;
}
@@ -449,19 +454,19 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
- m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 517000, 100000, true, true );
+ m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 517000, 100000, true, true );
REQUIRE ( ! FAILED () );
-
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.34") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.169") );
- REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.35") );
-
+
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.34") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".SA.169") );
+ REQUIRE ( NextId ( string ( CSRA1_WithSecondary ) + ".PA.35") );
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Order, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
- m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 517000, 100000, true, true );
+ m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 517000, 100000, true, true );
REQUIRE ( VerifyOrder() );
EXIT;
}
@@ -471,11 +476,11 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_NoSecondary, CSRA1_Fixture)
const char* accesssion = "SRR644545";
ENTRY_GET_REF ( accesssion, "NC_000004.11" );
- m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 0, 100000, true, true );
+ m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 0, 100000, true, true );
REQUIRE ( ! FAILED () );
-
- REQUIRE ( NextId ( string ( accesssion ) + ".PA.1") );
-
+
+ REQUIRE ( NextId ( string ( accesssion ) + ".PA.1") );
+
EXIT;
}
@@ -483,19 +488,19 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Overlap_Primary, CSRA1_Fixture
{
ENTRY_GET_REF ( CSRA1_PrimaryOnly, "supercont2.1" );
- m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 10000, 100, true, true );
+ m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 10000, 100, true, true );
REQUIRE ( ! FAILED () );
-
+
// 2 primary alignments start in the previous chunk and overlap with the chunk starting at 10000
- REQUIRE ( NextId ( string ( CSRA1_PrimaryOnly ) + ".PA.96") );
- REQUIRE ( NextId ( string ( CSRA1_PrimaryOnly ) + ".PA.97") );
-
+ REQUIRE ( NextId ( string ( CSRA1_PrimaryOnly ) + ".PA.96") );
+ REQUIRE ( NextId ( string ( CSRA1_PrimaryOnly ) + ".PA.97") );
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Overlap_Primary_Order, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_PrimaryOnly, "supercont2.1" );
- m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 10000, 100, true, true );
+ m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 10000, 100, true, true );
REQUIRE ( VerifyOrder() );
EXIT;
}
@@ -504,12 +509,12 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Overlap_Secondary, CSRA1_Fixtu
{
ENTRY_GET_REF ( "SRR960954", "gi|296100371|ref|NC_014121.1|" );
- m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 4020000, 100, true, true );
+ m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 4020000, 100, true, true );
REQUIRE ( ! FAILED () );
-
+
// a secondary alignment starts in the previous chunk and overlaps with the chunk starting at 4020000
- REQUIRE ( NextId ( string ( "SRR960954.SA.1384") ) );
-
+ REQUIRE ( NextId ( string ( "SRR960954.SA.1384") ) );
+
EXIT;
}
@@ -517,26 +522,26 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Overlap_PrimaryAndSecondary, C
{
ENTRY_GET_REF ( "ERR225922", "2" );
- m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 203895000, 100, true, true );
+ m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 203895000, 100, true, true );
REQUIRE ( ! FAILED () );
-
+
// several primary and secondary alignments start in the previous chunk and overlap with the chunk starting at 203895000
// pos, len, mapq
- REQUIRE ( NextId ( string ( "ERR225922.PA.7023") ) );
- REQUIRE ( NextId ( string ( "ERR225922.PA.7024") ) );
- REQUIRE ( NextId ( string ( "ERR225922.PA.7025") ) );
- REQUIRE ( NextId ( string ( "ERR225922.PA.7026") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7023") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7024") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7025") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7026") ) );
REQUIRE ( NextId ( string ( "ERR225922.PA.7028") ) ); // 203894959, 76, 60 (higher MAPQ than 7027, same pos/len)
REQUIRE ( NextId ( string ( "ERR225922.PA.7027") ) ); // 203894959, 76, 60
REQUIRE ( NextId ( string ( "ERR225922.PA.7029") ) ); // 203894961, 76, 60 (same as 45681 but primary)
REQUIRE ( NextId ( string ( "ERR225922.SA.45681") ) ); // 203894961, 76, 60 (same as 7029 but secondary; id=45521+160 )
- REQUIRE ( NextId ( string ( "ERR225922.PA.7030") ) );
- REQUIRE ( NextId ( string ( "ERR225922.PA.7031") ) );
- REQUIRE ( NextId ( string ( "ERR225922.PA.7032") ) );
- REQUIRE ( NextId ( string ( "ERR225922.PA.7033") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7030") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7031") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7032") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7033") ) );
REQUIRE ( NextId ( string ( "ERR225922.SA.45682") ) ); // (id=45,521+161)
- REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, m_ctx ) );
-
+ REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, m_ctx ) );
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Overlap_PrimaryAndSecondary_Cutoff, CSRA1_Fixture)
@@ -545,13 +550,13 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Overlap_PrimaryAndSecondary_Cu
m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 203895050, 100, true, true ); // increase offset by 50 bases
REQUIRE ( ! FAILED () );
-
+
// compared to the test above, alignments that do not reach 50+ bases into the chunk are excluded
- REQUIRE ( NextId ( string ( "ERR225922.PA.7032") ) );
- REQUIRE ( NextId ( string ( "ERR225922.PA.7033") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7032") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7033") ) );
REQUIRE ( NextId ( string ( "ERR225922.SA.45682") ) ); // 45,521+161
- REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, m_ctx ) );
-
+ REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, m_ctx ) );
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Overlap_PrimaryAndSecondary_FullCutoff, CSRA1_Fixture)
@@ -560,10 +565,10 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Overlap_PrimaryAndSecondary_Fu
m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 203895150, 100, true, true ); // increase offset to exclude all overlaps
REQUIRE ( ! FAILED () );
-
+
// none of the alignments reach 150+ bases into the chunk
- REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, m_ctx ) );
-
+ REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, m_ctx ) );
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Filtered, CSRA1_Fixture)
@@ -579,17 +584,17 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Filtered, CSRA1_Fixture)
m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, 203894961, 100, wants_primary, wants_secondary, filters, no_map_qual ); // only the ones that start within the window
REQUIRE ( ! FAILED () );
-
+
// only the alignments starting inside the slice (203894961)
REQUIRE ( NextId ( string ( "ERR225922.PA.7029") ) ); // 203894961, 76, 60 (same as 45681 but primary)
REQUIRE ( NextId ( string ( "ERR225922.SA.45681") ) ); // 203894961, 76, 60 (same as 7029 but secondary; id=45521+160 )
- REQUIRE ( NextId ( string ( "ERR225922.PA.7030") ) );
- REQUIRE ( NextId ( string ( "ERR225922.PA.7031") ) );
- REQUIRE ( NextId ( string ( "ERR225922.PA.7032") ) );
- REQUIRE ( NextId ( string ( "ERR225922.PA.7033") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7030") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7031") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7032") ) );
+ REQUIRE ( NextId ( string ( "ERR225922.PA.7033") ) );
REQUIRE ( NextId ( string ( "ERR225922.SA.45682") ) ); // (id=45,521+161)
- REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, m_ctx ) );
-
+ REQUIRE ( ! NGS_AlignmentIteratorNext ( m_align, m_ctx ) );
+
EXIT;
}
@@ -597,15 +602,15 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Filtered, CSRA1_Fixture)
// FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Overlap_LookbackDiffersForPrimaryAndSecondary, CSRA1_Fixture)
// FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Slice_Overlap_DefaultLookback, CSRA1_Fixture)
-// circular reference
+// circular reference
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Circular, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithCircularReference, "chrM" );
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, true );
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, true );
REQUIRE ( ! FAILED () );
-
+
// chrM is 16569 bases long // pos, len, mapq
REQUIRE ( NextId ( string ( CSRA1_WithCircularReference ) + ".PA.4511855081") ); // 16528, 42, 0 // last base 16528+42-1 = 16569 (mod 16569 = #0 in 0-based coord)
REQUIRE ( NextId ( string ( CSRA1_WithCircularReference ) + ".PA.4511855600") ); // 16529, 41, 66
@@ -614,9 +619,9 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Circular, CSRA1_Fixture)
REQUIRE ( NextId ( string ( CSRA1_WithCircularReference ) + ".PA.4511855597") ); // 16529, 41, 4
REQUIRE ( NextId ( string ( CSRA1_WithCircularReference ) + ".PA.4511855596") ); // 16529, 41, 0
REQUIRE ( NextId ( string ( CSRA1_WithCircularReference ) + ".PA.4511855599") ); // 16529, 41, 0
-
+
// ...
-
+
// TODO: make sure alignments do not reappear at the end of iteration!
EXIT;
}
@@ -625,7 +630,7 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Circular_Slice, CSRA1_Fixture)
{
ENTRY_GET_REF ( CSRA1_WithCircularReference, "chrM" );
- m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 1, 100, true, true );
+ m_align = NGS_ReferenceGetAlignmentSlice ( m_ref, ctx, 1, 100, true, true );
REQUIRE ( ! FAILED () );
// pos, len, mapq
REQUIRE ( NextId ( string ( CSRA1_WithCircularReference ) + ".PA.4511856243") ); // -39, 41, 56
@@ -633,9 +638,9 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_Circular_Slice, CSRA1_Fixture)
REQUIRE ( NextId ( string ( CSRA1_WithCircularReference ) + ".PA.4511856242") ); // -39, 41, 0
REQUIRE ( NextId ( string ( CSRA1_WithCircularReference ) + ".PA.4511857002") ); // -38, 40, 74
REQUIRE ( NextId ( string ( CSRA1_WithCircularReference ) + ".PA.4511857001") ); // -38, 40, 70
-
+
// ...
-
+
EXIT;
}
@@ -647,34 +652,34 @@ uint64_t SliceLength = 100;
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_Wraparound, CSRA1_Fixture )
{ // wraparound alignments overlapping with a slice
ENTRY_GET_REF( CSRA1_WithCircularReference, "chrM" );
-
+
const bool wants_primary = true;
const bool wants_secondary = true;
#pragma message ( "filtering did not work prior to today's changes. I have set filter bits to allow this test to pass. the test should be fixed so that filters are set to the two desired flags." )
const uint32_t no_filters = NGS_AlignmentFilterBits_pass_bad | NGS_AlignmentFilterBits_pass_dups;
const int32_t no_map_qual = 0;
- m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, SliceOffset, SliceLength, wants_primary, wants_secondary, no_filters, no_map_qual );
+ m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, SliceOffset, SliceLength, wants_primary, wants_secondary, no_filters, no_map_qual );
REQUIRE ( ! FAILED () && m_align );
REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
+
// the first returned alignment starts before the start of the circular reference, overlaps with slice
int64_t pos = NGS_AlignmentGetAlignmentPosition ( m_align, ctx );
REQUIRE_LT ( ( int64_t ) ( SliceOffset + SliceLength ), pos );
-
+
// check for overlap with the slice
uint64_t refLen = NGS_ReferenceGetLength ( m_ref, ctx );
- pos -= ( int64_t ) refLen;
+ pos -= ( int64_t ) refLen;
REQUIRE_GT ( ( int64_t ) 0, pos );
REQUIRE_GE ( pos + NGS_AlignmentGetAlignmentLength( m_align, ctx ), SliceOffset );
-
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_Wraparound_StartInWindow, CSRA1_Fixture )
{ // when StartInWindow filter is specified, it removes wraparound alignments as well
ENTRY_GET_REF( CSRA1_WithCircularReference, "chrM" );
-
+
const bool wants_primary = true;
const bool wants_secondary = true;
#pragma message ( "filtering did not work prior to today's changes. I have set filter bits to allow this test to pass. the test should be fixed so that filters are set to the two desired flags." )
@@ -682,22 +687,22 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_Wraparound_StartI
NGS_AlignmentFilterBits_start_within_window;
const int32_t no_map_qual = 0;
- m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, SliceOffset, SliceLength, wants_primary, wants_secondary, filters, no_map_qual );
+ m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, SliceOffset, SliceLength, wants_primary, wants_secondary, filters, no_map_qual );
REQUIRE ( ! FAILED () && m_align );
REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
+
// the first returned alignment starts inside the slice
int64_t pos = NGS_AlignmentGetAlignmentPosition ( m_align, ctx );
REQUIRE_LE ( ( int64_t ) SliceOffset, pos );
REQUIRE_LT ( pos, ( int64_t ) ( SliceOffset + SliceLength ) );
-
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_NoWraparound, CSRA1_Fixture )
{ // only removes wraparound alignments, not the ones starting before the slice
ENTRY_GET_REF( CSRA1_WithCircularReference, "chrM" );
-
+
const bool wants_primary = true;
const bool wants_secondary = true;
#pragma message ( "filtering did not work prior to today's changes. I have set filter bits to allow this test to pass. the test should be fixed so that filters are set to the two desired flags." )
@@ -705,20 +710,20 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_NoWraparound, CSR
NGS_AlignmentFilterBits_no_wraparound;
const int32_t no_map_qual = 0;
- m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, SliceOffset, SliceLength, wants_primary, wants_secondary, filters, no_map_qual );
+ m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, SliceOffset, SliceLength, wants_primary, wants_secondary, filters, no_map_qual );
REQUIRE ( ! FAILED () && m_align );
REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
+
// the first returned alignment starts outside the slice but does not wrap around
REQUIRE_GT ( ( int64_t ) SliceOffset, NGS_AlignmentGetAlignmentPosition ( m_align, ctx ) );
-
+
EXIT;
}
FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_NoWraparound_StartInWindow, CSRA1_Fixture )
{ // when StartInWindow filter is specified, it removes wraparound alignments as well
ENTRY_GET_REF( CSRA1_WithCircularReference, "chrM" );
-
+
const bool wants_primary = true;
const bool wants_secondary = true;
#pragma message ( "filtering did not work prior to today's changes. I have set filter bits to allow this test to pass. the test should be fixed so that filters are set to the two desired flags." )
@@ -726,15 +731,15 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceGetFilteredAlignmentSlice_NoWraparound_Star
NGS_AlignmentFilterBits_no_wraparound | NGS_AlignmentFilterBits_start_within_window;
const int32_t no_map_qual = 0;
- m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, SliceOffset, SliceLength, wants_primary, wants_secondary, filters, no_map_qual );
+ m_align = NGS_ReferenceGetFilteredAlignmentSlice ( m_ref, ctx, SliceOffset, SliceLength, wants_primary, wants_secondary, filters, no_map_qual );
REQUIRE ( ! FAILED () && m_align );
REQUIRE ( NGS_AlignmentIteratorNext ( m_align, ctx ) );
-
+
// the first returned alignment starts inside the slice
int64_t pos = NGS_AlignmentGetAlignmentPosition ( m_align, ctx );
REQUIRE_LE ( ( int64_t ) SliceOffset, pos );
REQUIRE_LT ( pos, ( int64_t ) ( SliceOffset + SliceLength ) );
-
+
EXIT;
}
@@ -748,8 +753,8 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_PrintEmAll_1, CSRA1_Fixture)
// ENTRY_GET_REF ( CSRA1_WithSecondary, "gi|169794206|ref|NC_010410.1|" );
// ENTRY_GET_REF ( CSRA1_PrimaryOnly, "supercont2.1" );
ENTRY_GET_REF ( CSRA1_WithCircularReference, "chrM" );
-
- m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, true );
+
+ m_align = NGS_ReferenceGetAlignments ( m_ref, ctx, true, true );
uint64_t prevPos = 0;
uint64_t prevLen = 0;
@@ -757,16 +762,16 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_PrintEmAll_1, CSRA1_Fixture)
int prevMapq = 0;
while ( NGS_AlignmentIteratorNext ( m_align, ctx ) )
{
-
+
uint64_t newPos = NGS_AlignmentGetAlignmentPosition ( m_align, m_ctx );
uint64_t newLen = NGS_AlignmentGetAlignmentLength ( m_align, m_ctx );
bool newIsPrimary = NGS_AlignmentIsPrimary ( m_align, m_ctx );
int newMapq = NGS_AlignmentGetMappingQuality( m_align, m_ctx );
-
+
PrintAlignment ();
-
+
/* REQUIRE_LE (prevPos, newPos);
- if ( prevPos == newPos )
+ if ( prevPos == newPos )
{
REQUIRE_GE ( prevLen, newLen );
if ( prevLen == newLen )
@@ -783,7 +788,7 @@ FIXTURE_TEST_CASE(CSRA1_NGS_ReferenceWindow_PrintEmAll_1, CSRA1_Fixture)
prevIsPrimary = newIsPrimary;
prevMapq = newMapq;
}
-
+
EXIT;
}
#endif
diff --git a/test/ngs/ngstest_reference.cpp b/test/ngs/ngstest_reference.cpp
index fd3f5d7..54dfa00 100644
--- a/test/ngs/ngstest_reference.cpp
+++ b/test/ngs/ngstest_reference.cpp
@@ -29,12 +29,16 @@
*/
#include "ngs_c_fixture.hpp"
+
+#include <string.h>
+
#include <ktst/unit_test.hpp>
+
+#include <kfc/xc.h>
+
#include "NGS_ReferenceSequence.h"
#include "NGS_String.h"
-#include <string.h>
-
using namespace std;
using namespace ncbi::NK;
@@ -56,20 +60,41 @@ FIXTURE_TEST_CASE(SRA_Reference_Open_FailsOnNonReference, NGS_C_Fixture)
ENTRY;
const char* SRA_Accession = "SRR000001";
REQUIRE_NULL ( NGS_ReferenceSequenceMake ( ctx, SRA_Accession ) );
+ REQUIRE ( ctx_xc_isa ( ctx, xcTableOpenFailed ) );
REQUIRE_FAILED ();
EXIT;
}
FIXTURE_TEST_CASE(EBI_Reference_Open_EBI_MD5, NGS_C_Fixture)
{
+/* The following request should success it order to this test to work:
+http://www.ebi.ac.uk/ena/cram/md5/ffd6aeffb54ade3d28ec7644afada2e9
+Otherwise CALL_TO_EBI_RESOLVER_FAILS is set
+and this test is expected to fail */
+ const bool CALL_TO_EBI_RESOLVER_FAILS = false;
+
ENTRY;
const char* EBI_Accession = "ffd6aeffb54ade3d28ec7644afada2e9";
+
+ if ( CALL_TO_EBI_RESOLVER_FAILS ) {
+ }
+
NGS_ReferenceSequence * ref = NGS_ReferenceSequenceMake ( ctx, EBI_Accession );
+
+ if ( CALL_TO_EBI_RESOLVER_FAILS ) {
+ REQUIRE ( FAILED () );
+ REQUIRE_NULL ( ref );
+ LOG(ncbi::NK::LogLevel::e_error,
+ "CANNOT TEST EBI ACCESSION BECAUSE THEIR SITE DOES NOT RESPOND!\n");
+ LOG(ncbi::NK::LogLevel::e_error, "NOW EXPECTING AN ERROR MESSAGE ...");
+ return;
+ }
+
REQUIRE ( ! FAILED () );
REQUIRE_NOT_NULL ( ref );
NGS_String* bases = NGS_ReferenceSequenceGetBases ( ref, ctx, 4, 16 );
-
+
REQUIRE ( strcmp (NGS_StringData ( bases, ctx ), "CTTTCTGACCGAAATT") == 0 );
REQUIRE_EQ ( NGS_ReferenceSequenceGetLength(ref, ctx), (uint64_t)784333 );
diff --git a/test/ngs/ngstest_sra.cpp b/test/ngs/ngstest_sra.cpp
index 87f6687..f8371f4 100644
--- a/test/ngs/ngstest_sra.cpp
+++ b/test/ngs/ngstest_sra.cpp
@@ -412,7 +412,7 @@ FIXTURE_TEST_CASE(SRA_NGS_ReadNumFragments, SRA_Fixture)
{
ENTRY_GET_READ(SRA_Accession, 1);
uint32_t num = NGS_ReadNumFragments ( m_read, ctx );
- REQUIRE_EQ ( (uint32_t)2, num );
+ REQUIRE_EQ ( (uint32_t)1, num ); // VDB-3132: only count non-empty fragments
EXIT;
}
@@ -486,15 +486,26 @@ FIXTURE_TEST_CASE(SRA_NGS_FragmentGetQualities_Full, SRA_Fixture)
// Iteration over Fragments
FIXTURE_TEST_CASE(SRA_NGS_FragmentIteratorNext, SRA_Fixture)
{
- ENTRY_GET_READ(SRA_Accession, 1); // calls NGS_FragmentIteratorNext
+ ENTRY_GET_READ(SRA_Accession, 2); // calls NGS_FragmentIteratorNext
- REQUIRE_STRING ( string ( SRA_Accession ) + ".FR0.1", NGS_FragmentGetId ( (NGS_Fragment*)m_read, ctx ) );
+ REQUIRE_STRING ( string ( SRA_Accession ) + ".FR0.2", NGS_FragmentGetId ( (NGS_Fragment*)m_read, ctx ) );
REQUIRE ( NGS_FragmentIteratorNext ( (NGS_Fragment*)m_read, ctx ) );
REQUIRE ( ! FAILED () );
- REQUIRE_STRING ( string ( SRA_Accession ) + ".FR1.1", NGS_FragmentGetId ( (NGS_Fragment*)m_read, ctx ) );
+ REQUIRE_STRING ( string ( SRA_Accession ) + ".FR1.2", NGS_FragmentGetId ( (NGS_Fragment*)m_read, ctx ) );
+
+ REQUIRE ( ! NGS_FragmentIteratorNext ( (NGS_Fragment*)m_read, ctx ) );
+ EXIT;
+}
+
+FIXTURE_TEST_CASE(SRA_NGS_FragmentIteratorNext_SkipEmpty, SRA_Fixture)
+{
+ ENTRY_GET_READ(SRA_Accession, 1); // calls NGS_FragmentIteratorNext
+
+ REQUIRE_STRING ( string ( SRA_Accession ) + ".FR0.1", NGS_FragmentGetId ( (NGS_Fragment*)m_read, ctx ) );
+ // VDB-3132: skip empty FR1.1
REQUIRE ( ! NGS_FragmentIteratorNext ( (NGS_Fragment*)m_read, ctx ) );
EXIT;
@@ -519,10 +530,8 @@ FIXTURE_TEST_CASE(SRA_NGS_FragmentIterator_NullRead, SRA_Fixture)
FIXTURE_TEST_CASE(SRA_NGS_FragmentIteratorNext_BeyondEnd, SRA_Fixture)
{
- ENTRY_GET_READ(SRA_Accession, 1);
-
- REQUIRE ( NGS_FragmentIteratorNext ( (NGS_Fragment*)m_read, ctx ) );
- REQUIRE ( ! FAILED () );
+ ENTRY_GET_READ(SRA_Accession, 1); // calls NGS_FragmentIteratorNext
+ // row 1 contains only 1 non-empty bio fragment
REQUIRE ( ! NGS_FragmentIteratorNext ( (NGS_Fragment*)m_read, ctx ) );
REQUIRE ( ! FAILED () );
@@ -585,12 +594,7 @@ FIXTURE_TEST_CASE(SRA_ReadCollection_GetReads_Next, SRA_Fixture)
REQUIRE ( ! FAILED () );
REQUIRE_STRING ( string ( SRA_Accession ) + ".FR0.1", NGS_FragmentGetId ( (NGS_Fragment*)m_read, ctx ) );
-
- REQUIRE ( NGS_FragmentIteratorNext ( (NGS_Fragment*)m_read, ctx ) );
- REQUIRE ( ! FAILED () );
-
- REQUIRE_STRING ( string ( SRA_Accession ) + ".FR1.1", NGS_FragmentGetId ( (NGS_Fragment*)m_read, ctx ) );
-
+ // the second bio read is empty and not reported
REQUIRE ( ! NGS_FragmentIteratorNext ( (NGS_Fragment*)m_read, ctx ) );
REQUIRE ( ! FAILED () );
}
@@ -682,9 +686,10 @@ FIXTURE_TEST_CASE(SRA_ReadCollection_GetReads_NumFragments, SRA_Fixture)
m_read = NGS_ReadCollectionGetReads ( m_coll, ctx, true, true, true );
+ REQUIRE ( NGS_ReadIteratorNext ( m_read, ctx ) ); // skip row 1 which has only 1 non-empty bio fragment
REQUIRE ( NGS_ReadIteratorNext ( m_read, ctx ) );
- REQUIRE_STRING ( ReadId ( 1 ), NGS_ReadGetReadId ( m_read, ctx ) );
+ REQUIRE_STRING ( ReadId ( 2 ), NGS_ReadGetReadId ( m_read, ctx ) );
REQUIRE ( ! FAILED () );
REQUIRE_EQ ( (uint32_t)2, NGS_ReadNumFragments ( m_read, ctx ) );
@@ -1026,6 +1031,7 @@ FIXTURE_TEST_CASE(SRA_GetFragmentBlobs, SRA_Fixture)
REQUIRE_EQ ( (uint64_t)1080, NGS_FragmentBlobSize ( blob, ctx ) );
+ NGS_FragmentBlobRelease ( blob, ctx );
NGS_FragmentBlobIteratorRelease ( blobIt, ctx );
EXIT;
diff --git a/test/only-remote-repository.kfg b/test/only-remote-repository.kfg
index 2e7ef51..b5e95a1 100644
--- a/test/only-remote-repository.kfg
+++ b/test/only-remote-repository.kfg
@@ -1,2 +1,2 @@
-/repository/remote/main/CGI/resolver-cgi = "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
+/repository/remote/main/CGI/resolver-cgi = "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
/repository/user/main/public/cache-enabled = "false"
diff --git a/test/vdb/.gitignore b/test/vdb/.gitignore
new file mode 100644
index 0000000..65eef93
--- /dev/null
+++ b/test/vdb/.gitignore
@@ -0,0 +1 @@
+db
diff --git a/test/vdb/Makefile b/test/vdb/Makefile
index 7a0aaf9..a2c2848 100644
--- a/test/vdb/Makefile
+++ b/test/vdb/Makefile
@@ -35,7 +35,8 @@ TEST_TOOLS = \
test-sparse-col-rw \
test-sparse-col-rd \
test-blob-val \
- test-VDB-3060
+ test-VDB-3060 \
+ test-VDB-3061
include $(TOP)/build/Makefile.env
@@ -216,10 +217,30 @@ VDB_3060_OBJ = \
VDB_3060_LIB = \
-skapp \
-sktst \
- -sncbi-vdb \
+ -sncbi-vdb
$(TEST_BINDIR)/test-VDB-3060: $(VDB_3060_OBJ)
$(LP) --exe -o $@ $^ $(VDB_3060_LIB)
VDB-3060: test-VDB-3060
$(TEST_BINDIR)/test-VDB-3060
+
+#-------------------------------------------------------------------------------
+# VDB-3061
+#
+VDB_3061_SRC = \
+ test-VDB-3061
+
+VDB_3061_OBJ = \
+ $(addsuffix .$(OBJX),$(VDB_3061_SRC))
+
+VDB_3061_LIB = \
+ -skapp \
+ -sktst \
+ -sncbi-vdb
+
+$(TEST_BINDIR)/test-VDB-3061: $(VDB_3061_OBJ)
+ $(LP) --exe -o $@ $^ $(VDB_3061_LIB)
+
+VDB-3061: test-VDB-3061
+ $(TEST_BINDIR)/test-VDB-3061
diff --git a/test/vdb/test-VDB-3060.cpp b/test/vdb/test-VDB-3060.cpp
index d3ac905..67a5e4b 100644
--- a/test/vdb/test-VDB-3060.cpp
+++ b/test/vdb/test-VDB-3060.cpp
@@ -23,6 +23,8 @@
// ===========================================================================
#include <vdb/manager.h> // VDBManager
+#include <kdb/manager.h> // KDBManager
+#include <kdb/kdb-priv.h>
#include <vdb/vdb-priv.h>
#include <ktst/unit_test.hpp> // TEST_CASE
@@ -219,7 +221,7 @@ TEST_CASE( GetCacheRoot_3 )
VPathRelease( vpath );
}
-TEST_CASE( tow_managers )
+TEST_CASE( two_managers )
{
const VDBManager * vdb_mgr2 = NULL;
VPath const * vpath1 = NULL;
@@ -272,6 +274,35 @@ TEST_CASE( tow_managers )
if ( vdb_mgr2 != NULL ) VDBManagerRelease ( vdb_mgr2 );
}
+TEST_CASE( root_tmp )
+{
+ std::cout << "testing root-tmp" << std::endl;
+
+ const KDBManager * kdb_mgr = NULL;
+ rc_t rc = VDBManagerGetKDBManagerRead( vdb_mgr, &kdb_mgr) ;
+ if ( rc != 0 )
+ FAIL( "FAIL: VDBManagerGetKDBManagerRead() failed" );
+
+ VFSManager * vfs_mgr_1 = NULL;
+ rc = KDBManagerGetVFSManager( kdb_mgr, &vfs_mgr_1 );
+ if ( rc != 0 )
+ FAIL( "FAIL: KDBManagerGetVFSManager() failed" );
+
+ VPath * vpath = NULL;
+ rc = VFSManagerMakeSysPath( vfs_mgr_1, &vpath, "/tmp1" );
+ if ( rc != 0 )
+ FAIL( "FAIL: VFSManagerMakeSysPath() failed" );
+
+ rc = VDBManagerSetCacheRoot( vdb_mgr, vpath );
+ if ( rc != 0 )
+ FAIL( "FAIL: VDBManagerSetCacheRoot( mgr, vpath ) failed" );
+
+ if ( vpath != NULL ) VPathRelease( vpath );
+ if ( vfs_mgr_1 != NULL ) VFSManagerRelease ( vfs_mgr_1 );
+ if ( kdb_mgr != NULL ) KDBManagerRelease ( kdb_mgr );
+
+}
+
char * org_home;
const char HomeSub[] = "test_root_history";
char new_home[ 1024 ];
diff --git a/test/vdb/test-VDB-3061.cpp b/test/vdb/test-VDB-3061.cpp
new file mode 100644
index 0000000..fe577cd
--- /dev/null
+++ b/test/vdb/test-VDB-3061.cpp
@@ -0,0 +1,249 @@
+// ===========================================================================
+//
+// PUBLIC DOMAIN NOTICE
+// National Center for Biotechnology Information
+//
+// This software/database is a "United States Government Work" under the
+// terms of the United States Copyright Act. It was written as part of
+// the author's official duties as a United States Government employee and
+// thus cannot be copyrighted. This software/database is freely available
+// to the public for use. The National Library of Medicine and the U.S.
+// Government have not placed any restriction on its use or reproduction.
+//
+// Although all reasonable efforts have been taken to ensure the accuracy
+// and reliability of the software and data, the NLM and the U.S.
+// Government do not and cannot warrant the performance or results that
+// may be obtained by using this software or data. The NLM and the U.S.
+// Government disclaim all warranties, express or implied, including
+// warranties of performance, merchantability or fitness for any particular
+// purpose.
+//
+// Please cite the author in any work or product based on this material.
+//
+// ===========================================================================
+
+#include <vdb/manager.h> // VDBManager
+#include <vdb/vdb-priv.h>
+
+#include <ktst/unit_test.hpp> // TEST_CASE
+#include <vfs/path.h>
+#include <vfs/manager.h>
+#include <klib/text.h>
+#include <klib/out.h>
+#include <klib/printf.h>
+#include <klib/time.h>
+#include <kfs/directory.h>
+#include <kfs/file.h>
+#include <kfg/config.h>
+
+#include <sysalloc.h>
+#include <cstdlib>
+#include <stdexcept>
+
+using namespace std;
+
+TEST_SUITE( VDB_3061 )
+
+char new_home[ 4096 ];
+
+static rc_t create_cache_file( KDirectory * dir, const char * path, const char * sub, const char * name, int64_t age )
+{
+ KFile * f;
+ rc_t rc = KDirectoryCreateFile( dir, &f, false, 0775, kcmInit, "%s/ncbi/%s/%s", path, sub, name );
+ if ( rc == 0 )
+ {
+ KFileRelease( f );
+
+ KTime_t date = KTimeStamp() - age;
+ rc = KDirectorySetDate ( dir, false, date, "%s/ncbi/%s/%s", path, sub, name );
+ }
+ return rc;
+}
+
+static rc_t create_repo_dirs( KDirectory * dir, const char * path, const char * sub, uint32_t age )
+{
+ rc_t rc = KDirectoryCreateDir( dir, 0775, kcmInit | kcmParents, "%s/ncbi/%s/files", path, sub );
+ if ( rc == 0 )
+ rc = KDirectoryCreateDir( dir, 0775, kcmInit | kcmParents, "%s/ncbi/%s/nannot", path, sub );
+ if ( rc == 0 )
+ rc = KDirectoryCreateDir( dir, 0775, kcmInit | kcmParents, "%s/ncbi/%s/refseq", path, sub );
+ if ( rc == 0 )
+ rc = KDirectoryCreateDir( dir, 0775, kcmInit | kcmParents, "%s/ncbi/%s/sra", path, sub );
+ if ( rc == 0 )
+ rc = KDirectoryCreateDir( dir, 0775, kcmInit | kcmParents, "%s/ncbi/%s/wgs", path, sub );
+ if ( rc == 0 )
+ rc = create_cache_file( dir, path, sub, "/sra/file1.txt", 60 * 60 * 24 * age );
+ return rc;
+}
+
+
+TEST_CASE( CLEAR_CACHE )
+{
+ /* create a repository-structure equivalent to the config-values, with 3 files in it */
+ KDirectory * dir;
+ REQUIRE_RC( KDirectoryNativeDir( &dir ) );
+ REQUIRE_RC( create_repo_dirs( dir, new_home, "public", 10 ) ); /* 10 days old */
+ REQUIRE_RC( create_repo_dirs( dir, new_home, "dbGaP-2956", 4 ) ); /* 4 days old */
+ REQUIRE_RC( create_repo_dirs( dir, new_home, "dbGaP-4831", 5 ) ); /* 5 days old*/
+ REQUIRE_RC( KDirectoryRelease ( dir ) );
+
+ /* we run the new function VDBManagerDeleteCacheOlderThan() with a 5-day threshold */
+ const VDBManager * vdb_mgr;
+ REQUIRE_RC( VDBManagerMakeRead( &vdb_mgr, NULL ) );
+ REQUIRE_RC( VDBManagerDeleteCacheOlderThan ( vdb_mgr, 5 ) );
+ REQUIRE_RC( VDBManagerRelease ( vdb_mgr ) );
+
+ /* now the 10 day old one should have disappeared, the other 2 are still there */
+ REQUIRE_RC( KDirectoryNativeDir( &dir ) );
+ uint32_t pt1 = KDirectoryPathType( dir, "%s/ncbi/public/sra/file1.txt", new_home );
+ REQUIRE_EQ( pt1, (uint32_t)kptNotFound );
+ uint32_t pt2 = KDirectoryPathType( dir, "%s/ncbi/dbGaP-2956/sra/file1.txt", new_home );
+ REQUIRE_EQ( pt2, (uint32_t)kptFile );
+ uint32_t pt3 = KDirectoryPathType( dir, "%s/ncbi/dbGaP-4831/sra/file1.txt", new_home );
+ REQUIRE_EQ( pt3, (uint32_t)kptFile );
+ REQUIRE_RC( KDirectoryRelease ( dir ) );
+}
+
+static rc_t write_root( KConfig *cfg, const char * base, const char * cat, const char * sub_cat )
+{
+ char key[ 256 ];
+ size_t num_writ;
+ rc_t rc = string_printf ( key, sizeof key, &num_writ, "/repository/user/%s/%s/root", cat, sub_cat );
+ if ( rc == 0 )
+ {
+ char value[ 256 ];
+ rc = string_printf ( value, sizeof value, &num_writ, "%s/ncbi/%s", base, sub_cat );
+ if ( rc == 0 )
+ rc = KConfigWriteString( cfg, key, value );
+ }
+ return rc;
+}
+
+
+static rc_t write_dflt_path( KConfig *cfg, const char * base )
+{
+ char value[ 256 ];
+ size_t num_writ;
+ rc_t rc = string_printf ( value, sizeof value, &num_writ, "%s/ncbi", base );
+ if ( rc == 0 )
+ rc = KConfigWriteString( cfg, "/repository/user/default-path", value );
+ return rc;
+}
+
+#ifdef WINDOWS
+static char * convert_sys_path( const char * sys_path )
+{
+ char * res = NULL;
+ VFSManager * vfs_mgr;
+ rc_t rc = VFSManagerMake ( &vfs_mgr );
+ if ( rc == 0 )
+ {
+ VPath * p;
+ rc = VFSManagerMakeSysPath ( vfs_mgr, &p, sys_path );
+ if ( rc == 0 )
+ {
+ const String * S;
+ rc = VPathMakeString( p, &S );
+ if ( rc == 0 )
+ res = string_dup ( S->addr, S->size );
+ VPathRelease( p );
+ }
+ VFSManagerRelease( vfs_mgr );
+ }
+ return res;
+}
+#endif
+
+static rc_t create_test_config( KConfig **cfg, const char * base )
+{
+ rc_t rc = KConfigMake ( cfg, NULL );
+ if ( rc == 0 )
+ {
+#ifdef WINDOWS
+ const char * cfg_base = convert_sys_path( base );
+#else
+ const char * cfg_base = base;
+#endif
+ if ( cfg_base != NULL )
+ {
+ rc = write_root( *cfg, cfg_base, "main", "public" );
+ if ( rc == 0 )
+ rc = write_root( *cfg, cfg_base, "protected", "dbGaP-2956" );
+ if ( rc == 0 )
+ rc = write_root( *cfg, cfg_base, "protected", "dbGaP-4831" );
+ if ( rc == 0 )
+ rc = write_dflt_path( *cfg, cfg_base );
+#ifdef WINDOWS
+ free( ( void * ) cfg_base );
+#endif
+ }
+ }
+ return rc;
+}
+
+char * org_home;
+char new_home_buffer[ 4096 ]; /* buffer for putenv has to stay alive! */
+
+static rc_t prepare_test( KConfig **cfg, const char * sub )
+{
+ size_t num_writ;
+#ifdef WINDOWS
+ org_home = getenv ( "USERPROFILE" );
+ rc_t rc = string_printf ( new_home, sizeof new_home, &num_writ, "%s\\%s", org_home, sub );
+#else
+ org_home = getenv( "HOME" );
+ rc_t rc = string_printf ( new_home, sizeof new_home, &num_writ, "%s/%s", org_home, sub );
+#endif
+ if ( rc == 0 )
+#ifdef WINDOWS
+ rc = string_printf ( new_home_buffer, sizeof new_home_buffer, &num_writ, "HOME=%s", new_home );
+#else
+ rc = string_printf ( new_home_buffer, sizeof new_home_buffer, &num_writ, "USERPROFILE=%s", new_home );
+#endif
+ if ( rc == 0 )
+ rc = putenv( new_home_buffer );
+ if ( rc == 0 )
+ rc = create_test_config( cfg, new_home );
+ return rc;
+}
+
+void finish_test( const char * sub )
+{
+ /* clear the temp. home-directory */
+ KDirectory * dir;
+ rc_t rc = KDirectoryNativeDir( &dir );
+ if ( rc == 0 )
+ {
+#ifdef WINDOWS
+ rc = KDirectoryRemove( dir, true, "%s/%s", org_home, sub );
+#else
+ rc = KDirectoryRemove( dir, true, "%s\\%s", org_home, sub );
+#endif
+ KDirectoryRelease( dir );
+ }
+}
+
+//////////////////////////////////////////// Main
+extern "C"
+{
+
+#include <kapp/args.h>
+
+ver_t CC KAppVersion ( void ) { return 0x1000000; }
+rc_t CC UsageSummary ( const char * progname ) { return 0; }
+rc_t CC Usage ( const Args * args ) { return 0; }
+const char UsageDefaultName[] = "test-VDB-3060";
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ const char HomeSub[] = "test_root_history";
+ KConfig *cfg;
+ rc_t rc = prepare_test( &cfg, HomeSub );
+ if ( rc == 0 )
+ rc = VDB_3061( argc, argv );
+ finish_test( HomeSub );
+ KConfigRelease ( cfg );
+ return rc;
+}
+
+}
diff --git a/test/vdb/test-wvdb.cpp b/test/vdb/test-wvdb.cpp
index 3d70580..8e6ef6e 100644
--- a/test/vdb/test-wvdb.cpp
+++ b/test/vdb/test-wvdb.cpp
@@ -53,30 +53,54 @@ class WVDB_Fixture
{
public:
WVDB_Fixture()
+ : m_db ( 0 )
{
- THROW_ON_RC ( KDirectoryNativeDir ( & m_wd ) );
- THROW_ON_RC ( VDBManagerMakeUpdate ( & m_mgr, m_wd ) );
}
~WVDB_Fixture()
{
- VDBManagerRelease ( m_mgr );
+ if ( m_db )
+ {
+ VDatabaseRelease ( m_db );
+ }
RemoveDatabase();
- KDirectoryRelease ( m_wd );
}
- void RemoveDatabase()
+
+ void RemoveDatabase ()
{
if ( ! m_databaseName . empty () )
{
- KDirectoryRemove ( m_wd, true, m_databaseName . c_str () );
+ KDirectory* wd;
+ KDirectoryNativeDir ( & wd );
+ KDirectoryRemove ( wd, true, m_databaseName . c_str () );
+ KDirectoryRelease ( wd );
}
}
- KDirectory* m_wd;
- VDBManager* m_mgr;
+ void MakeDatabase ( const string& p_schemaText, const string& p_schemaSpec )
+ {
+ RemoveDatabase();
+
+ VDBManager* mgr;
+ THROW_ON_RC ( VDBManagerMakeUpdate ( & mgr, NULL ) );
+ VSchema* schema;
+ THROW_ON_RC ( VDBManagerMakeSchema ( mgr, & schema ) );
+ THROW_ON_RC ( VSchemaParseText(schema, NULL, p_schemaText . c_str(), p_schemaText . size () ) );
+
+ THROW_ON_RC ( VDBManagerCreateDB ( mgr,
+ & m_db,
+ schema,
+ p_schemaSpec . c_str (),
+ kcmInit + kcmMD5,
+ "%s",
+ m_databaseName . c_str () ) );
+ THROW_ON_RC ( VSchemaRelease ( schema ) );
+ THROW_ON_RC ( VDBManagerRelease ( mgr ) );
+ }
+
string m_databaseName;
+ VDatabase* m_db;
};
-
// this test case is not very useful but is here as a blueprint for other write-side tests
FIXTURE_TEST_CASE ( BlobCorruptOnCommit, WVDB_Fixture)
{
@@ -107,32 +131,13 @@ FIXTURE_TEST_CASE ( BlobCorruptOnCommit, WVDB_Fixture)
"};\n"
;
const char * schemaSpec = "db";
-
- // MakeDatabase
- VDatabase* db;
- {
- VDBManager* mgr;
- REQUIRE_RC ( VDBManagerMakeUpdate ( & mgr, NULL ) );
- VSchema* schema;
- REQUIRE_RC ( VDBManagerMakeSchema ( mgr, & schema ) );
- REQUIRE_RC ( VSchemaParseText(schema, NULL, schemaText . c_str(), schemaText . size () ) );
-
- REQUIRE_RC ( VDBManagerCreateDB ( mgr,
- & db,
- schema,
- schemaSpec,
- kcmInit + kcmMD5,
- "%s",
- m_databaseName . c_str () ) );
- REQUIRE_RC ( VSchemaRelease ( schema ) );
- REQUIRE_RC ( VDBManagerRelease ( mgr ) );
- }
+ MakeDatabase ( schemaText, schemaSpec );
// MakeCursor
VCursor* cursor;
{
VTable* table;
- REQUIRE_RC ( VDatabaseCreateTable ( db, & table, "REFERENCE", kcmCreate | kcmMD5, "%s", "REFERENCE" ) );
+ REQUIRE_RC ( VDatabaseCreateTable ( m_db, & table, "REFERENCE", kcmCreate | kcmMD5, "%s", "REFERENCE" ) );
REQUIRE_RC ( VTableCreateCursorWrite ( table, & cursor, kcmInsert ) );
REQUIRE_RC ( VTableRelease ( table ) );
}
@@ -159,8 +164,6 @@ FIXTURE_TEST_CASE ( BlobCorruptOnCommit, WVDB_Fixture)
REQUIRE_RC ( VCursorCommit ( cursor ) ); // this returns rcVDB,rcBlob,rcValidating,rcBlob,rcCorrupt if the schema does not support
// writing to the LABEL column from the code
REQUIRE_RC ( VCursorRelease ( cursor ) );
-
- REQUIRE_RC ( VDatabaseRelease ( db ) );
}
FIXTURE_TEST_CASE ( ColumnOpenMetadata, WVDB_Fixture )
@@ -174,22 +177,11 @@ FIXTURE_TEST_CASE ( ColumnOpenMetadata, WVDB_Fixture )
const char* TableName = "TABLE1";
const char* ColumnName = "column1";
- VDatabase* db;
- {
- VSchema* schema;
- REQUIRE_RC ( VDBManagerMakeSchema ( m_mgr, & schema ) );
- REQUIRE_RC ( VSchemaParseText ( schema, NULL, schemaText . c_str (), schemaText . size () ) );
-
- REQUIRE_RC ( VDBManagerCreateDB ( m_mgr,
- & db,
- schema,
- "root_database",
- kcmInit + kcmMD5,
- "%s",
- m_databaseName . c_str () ) );
+ MakeDatabase ( schemaText, "root_database" );
+ {
VTable* table;
- REQUIRE_RC ( VDatabaseCreateTable ( db , & table, TableName, kcmInit + kcmMD5, TableName ) );
+ REQUIRE_RC ( VDatabaseCreateTable ( m_db , & table, TableName, kcmInit + kcmMD5, TableName ) );
VCursor* cursor;
REQUIRE_RC ( VTableCreateCursorWrite ( table, & cursor, kcmInsert ) );
@@ -212,12 +204,11 @@ FIXTURE_TEST_CASE ( ColumnOpenMetadata, WVDB_Fixture )
REQUIRE_RC ( VCursorRelease ( cursor ) );
REQUIRE_RC ( VTableRelease ( table ) );
- REQUIRE_RC ( VSchemaRelease ( schema ) );
}
// update the column's metadata without re-opening the database
{
VTable* table;
- REQUIRE_RC ( VDatabaseOpenTableUpdate ( db , & table, TableName ) );
+ REQUIRE_RC ( VDatabaseOpenTableUpdate ( m_db , & table, TableName ) );
KTable* ktbl;
REQUIRE_RC ( VTableOpenKTableUpdate ( table, & ktbl ) );
KColumn* col;
@@ -237,7 +228,6 @@ FIXTURE_TEST_CASE ( ColumnOpenMetadata, WVDB_Fixture )
REQUIRE_RC ( KColumnRelease ( col ) );
REQUIRE_RC ( KTableRelease ( ktbl ) );
REQUIRE_RC ( VTableRelease ( table ) );
- REQUIRE_RC ( VDatabaseRelease ( db ) );
}
}
@@ -253,22 +243,10 @@ FIXTURE_TEST_CASE ( VTableDropColumn_PhysicalColumn, WVDB_Fixture )
const char* ColumnName1 = "column1";
const char* ColumnName2 = "column2";
- VDatabase* db;
+ MakeDatabase ( schemaText, "root_database" );
{
- VSchema* schema;
- REQUIRE_RC ( VDBManagerMakeSchema ( m_mgr, & schema ) );
- REQUIRE_RC ( VSchemaParseText ( schema, NULL, schemaText . c_str (), schemaText . size () ) );
-
- REQUIRE_RC ( VDBManagerCreateDB ( m_mgr,
- & db,
- schema,
- "root_database",
- kcmInit + kcmMD5,
- "%s",
- m_databaseName . c_str () ) );
-
VTable* table;
- REQUIRE_RC ( VDatabaseCreateTable ( db , & table, TableName, kcmInit + kcmMD5, TableName ) );
+ REQUIRE_RC ( VDatabaseCreateTable ( m_db , & table, TableName, kcmInit + kcmMD5, TableName ) );
VCursor* cursor;
REQUIRE_RC ( VTableCreateCursorWrite ( table, & cursor, kcmInsert ) );
@@ -295,19 +273,18 @@ FIXTURE_TEST_CASE ( VTableDropColumn_PhysicalColumn, WVDB_Fixture )
REQUIRE_RC ( VCursorRelease ( cursor ) );
REQUIRE_RC ( VTableRelease ( table ) );
- REQUIRE_RC ( VSchemaRelease ( schema ) );
}
// drop column1
{
VTable* table;
- REQUIRE_RC ( VDatabaseOpenTableUpdate ( db , & table, TableName ) );
+ REQUIRE_RC ( VDatabaseOpenTableUpdate ( m_db , & table, TableName ) );
REQUIRE_RC ( VTableDropColumn ( table, ColumnName1 ) );
REQUIRE_RC ( VTableRelease ( table ) );
}
// finally, check resulted db
{
VTable* table;
- REQUIRE_RC ( VDatabaseOpenTableUpdate ( db , & table, TableName ) );
+ REQUIRE_RC ( VDatabaseOpenTableUpdate ( m_db , & table, TableName ) );
const VCursor* cursor;
REQUIRE_RC ( VTableCreateCursorRead ( (const VTable*) table, & cursor ) );
uint32_t column_idx1;
@@ -323,7 +300,51 @@ FIXTURE_TEST_CASE ( VTableDropColumn_PhysicalColumn, WVDB_Fixture )
REQUIRE_RC ( VCursorRelease ( cursor ) );
REQUIRE_RC ( VTableRelease ( table ) );
}
- REQUIRE_RC ( VDatabaseRelease ( db ) );
+}
+
+FIXTURE_TEST_CASE ( CreateTableInNestedDatabase, WVDB_Fixture )
+{ // VDB-1617: VDatabaseOpenTableRead inside a nested database segfaults
+ m_databaseName = ScratchDir + GetName();
+ string schemaText =
+ "table table1 #1.0.0 { column ascii column1; };"
+ "database database0 #1 { table table1 #1 TABLE1; } ;"
+ "database db #1 { database database0 #1 SUBDB; } ;" ;
+
+ // Create the database and the table
+ MakeDatabase ( schemaText, "db" );
+ { // make nested database and a table in it
+ VDatabase* subdb;
+ REQUIRE_RC ( VDatabaseCreateDB ( m_db, & subdb, "SUBDB", kcmInit + kcmMD5, "SUBDB" ) );
+
+ VTable *tbl;
+ REQUIRE_RC ( VDatabaseCreateTable ( subdb, & tbl, "TABLE1", kcmInit + kcmMD5, "TABLE1" ) );
+
+ REQUIRE_RC ( VTableRelease ( tbl ) );
+ REQUIRE_RC ( VDatabaseRelease ( subdb ) );
+ }
+ REQUIRE_RC ( VDatabaseRelease ( m_db ) );
+
+ // Re-open the database, try to open the table
+ {
+ VDBManager * mgr;
+ VDatabase * subdb;
+ const VTable * tbl;
+
+ REQUIRE_RC ( VDBManagerMakeUpdate ( & mgr, NULL ) );
+ REQUIRE_RC ( VDBManagerOpenDBUpdate ( mgr,
+ & m_db,
+ NULL,
+ "%s",
+ m_databaseName . c_str () ) );
+ REQUIRE_RC ( VDatabaseOpenDBUpdate ( m_db, & subdb, "SUBDB" ) );
+
+ // open the nested database and a table in it
+ REQUIRE_RC ( VDatabaseOpenTableRead ( subdb, & tbl, "%s", "TABLE1" ) ); // segfault no more...
+ REQUIRE_RC ( VTableRelease ( tbl ) );
+
+ REQUIRE_RC ( VDatabaseRelease ( subdb ) );
+ REQUIRE_RC ( VDBManagerRelease ( mgr ) );
+ }
}
FIXTURE_TEST_CASE ( VTableDropColumn_MetadataColumn_VDB_2735, WVDB_Fixture )
@@ -338,22 +359,10 @@ FIXTURE_TEST_CASE ( VTableDropColumn_MetadataColumn_VDB_2735, WVDB_Fixture )
const char* ColumnName1 = "column1";
const char* ColumnName2 = "column2";
- VDatabase* db;
+ MakeDatabase ( schemaText, "root_database" );
{
- VSchema* schema;
- REQUIRE_RC ( VDBManagerMakeSchema ( m_mgr, & schema ) );
- REQUIRE_RC ( VSchemaParseText ( schema, NULL, schemaText . c_str (), schemaText . size () ) );
-
- REQUIRE_RC ( VDBManagerCreateDB ( m_mgr,
- & db,
- schema,
- "root_database",
- kcmInit + kcmMD5,
- "%s",
- m_databaseName . c_str () ) );
-
VTable* table;
- REQUIRE_RC ( VDatabaseCreateTable ( db , & table, TableName, kcmInit + kcmMD5, TableName ) );
+ REQUIRE_RC ( VDatabaseCreateTable ( m_db , & table, TableName, kcmInit + kcmMD5, TableName ) );
VCursor* cursor;
REQUIRE_RC ( VTableCreateCursorWrite ( table, & cursor, kcmInsert ) );
@@ -380,19 +389,18 @@ FIXTURE_TEST_CASE ( VTableDropColumn_MetadataColumn_VDB_2735, WVDB_Fixture )
REQUIRE_RC ( VCursorRelease ( cursor ) );
REQUIRE_RC ( VTableRelease ( table ) );
- REQUIRE_RC ( VSchemaRelease ( schema ) );
}
// drop column1
{
VTable* table;
- REQUIRE_RC ( VDatabaseOpenTableUpdate ( db , & table, TableName ) );
+ REQUIRE_RC ( VDatabaseOpenTableUpdate (m_db , & table, TableName ) );
REQUIRE_RC ( VTableDropColumn ( table, ColumnName1 ) );
REQUIRE_RC ( VTableRelease ( table ) );
}
// finally, check resulted db
{
VTable* table;
- REQUIRE_RC ( VDatabaseOpenTableUpdate ( db , & table, TableName ) );
+ REQUIRE_RC ( VDatabaseOpenTableUpdate ( m_db , & table, TableName ) );
const VCursor* cursor;
REQUIRE_RC ( VTableCreateCursorRead ( (const VTable*) table, & cursor ) );
uint32_t column_idx1;
@@ -408,7 +416,6 @@ FIXTURE_TEST_CASE ( VTableDropColumn_MetadataColumn_VDB_2735, WVDB_Fixture )
REQUIRE_RC ( VCursorRelease ( cursor ) );
REQUIRE_RC ( VTableRelease ( table ) );
}
- REQUIRE_RC ( VDatabaseRelease ( db ) );
}
FIXTURE_TEST_CASE ( VCursor_FindNextRowIdDirect, WVDB_Fixture )
@@ -422,22 +429,10 @@ FIXTURE_TEST_CASE ( VCursor_FindNextRowIdDirect, WVDB_Fixture )
const char* TableName = "TABLE1";
const char* ColumnName = "column1";
+ MakeDatabase ( schemaText, "root_database" );
{
- VDatabase* db;
- VSchema* schema;
- REQUIRE_RC ( VDBManagerMakeSchema ( m_mgr, & schema ) );
- REQUIRE_RC ( VSchemaParseText ( schema, NULL, schemaText . c_str (), schemaText . size () ) );
-
- REQUIRE_RC ( VDBManagerCreateDB ( m_mgr,
- & db,
- schema,
- "root_database",
- kcmInit + kcmMD5,
- "%s",
- m_databaseName . c_str () ) );
-
VTable* table;
- REQUIRE_RC ( VDatabaseCreateTable ( db , & table, TableName, kcmInit + kcmMD5, TableName ) );
+ REQUIRE_RC ( VDatabaseCreateTable ( m_db , & table, TableName, kcmInit + kcmMD5, TableName ) );
VCursor* cursor;
REQUIRE_RC ( VTableCreateCursorWrite ( table, & cursor, kcmInsert ) );
@@ -461,15 +456,16 @@ FIXTURE_TEST_CASE ( VCursor_FindNextRowIdDirect, WVDB_Fixture )
REQUIRE_RC ( VCursorRelease ( cursor ) );
REQUIRE_RC ( VTableReindex ( table ) );
REQUIRE_RC ( VTableRelease ( table ) );
- REQUIRE_RC ( VSchemaRelease ( schema ) );
- REQUIRE_RC ( VDatabaseRelease ( db ) );
}
+ REQUIRE_RC ( VDatabaseRelease ( m_db ) );
+
{ // reopen
- const VDatabase* db;
- VDBManagerOpenDBRead ( m_mgr, & db, NULL, m_databaseName . c_str () );
+ VDBManager * mgr;
+ REQUIRE_RC ( VDBManagerMakeUpdate ( & mgr, NULL ) );
+ VDBManagerOpenDBRead ( mgr, (const VDatabase**)& m_db, NULL, m_databaseName . c_str () );
const VTable* table;
- REQUIRE_RC ( VDatabaseOpenTableRead ( db , & table, TableName ) );
+ REQUIRE_RC ( VDatabaseOpenTableRead ( m_db , & table, TableName ) );
const VCursor* cursor;
REQUIRE_RC ( VTableCreateCursorRead ( table, & cursor ) );
@@ -486,7 +482,6 @@ FIXTURE_TEST_CASE ( VCursor_FindNextRowIdDirect, WVDB_Fixture )
REQUIRE_RC ( VCursorRelease ( cursor ) );
REQUIRE_RC ( VTableRelease ( table ) );
- REQUIRE_RC ( VDatabaseRelease ( db ) );
}
}
diff --git a/test/vfs/Makefile b/test/vfs/Makefile
index c849c6a..9ac91a2 100644
--- a/test/vfs/Makefile
+++ b/test/vfs/Makefile
@@ -29,6 +29,7 @@ TOP ?= $(abspath ../..)
MODULE = test/vfs
TEST_TOOLS = \
+ redirect-rejected-names-cgi-http-to-https \
test-caching \
test-vfsmanager \
test-path-c \
@@ -81,6 +82,16 @@ $(TEST_BINDIR)/test-path: $(PATHTEST_OBJ)
$(LP) --exe -o $@ $^ $(PATHTEST_LIB)
#----------------------------------------------------------------
+HTTPS_SRC = \
+ redirect-rejected-names-cgi-http-to-https
+
+HTTPS_OBJ = \
+ $(addsuffix .$(OBJX),$(HTTPS_SRC))
+
+$(TEST_BINDIR)/redirect-rejected-names-cgi-http-to-https: $(HTTPS_OBJ)
+ $(LP) --exe -o $@ $^ $(PATHTEST_LIB)
+
+#----------------------------------------------------------------
# test-caching
CACHING_SRC = \
diff --git a/test/vfs/redirect-rejected-names-cgi-http-to-https.cpp b/test/vfs/redirect-rejected-names-cgi-http-to-https.cpp
new file mode 100644
index 0000000..3f494f0
--- /dev/null
+++ b/test/vfs/redirect-rejected-names-cgi-http-to-https.cpp
@@ -0,0 +1,189 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ==============================================================================
+*
+*/
+
+#include <kfg/config.h> /* KConfig */
+#include <kfs/directory.h> /* KDirectory */
+#include <klib/debug.h> /* KDbgSetString */
+#include <klib/rc.h>
+#include <kns/http.h> /* KClientHttpRequest */
+#include <kns/manager.h> /* KNSManager */
+#include <ktst/unit_test.hpp>
+#include <vfs/manager.h> /* VFSManager */
+#include <vfs/manager-priv.h> /* VFSManagerMakeFromKfg */
+#include <vfs/path.h> /* VPath */
+#include <vfs/resolver.h> /* VResolver */
+
+#include "../../libs/kns/http-priv.h" /* KNSManagerMakeClientHttpInt */
+
+#define RELEASE(type, obj) do { rc_t rc2 = type##Release(obj); \
+ if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while (false)
+
+extern "C" { void TESTING_VDB_3162 ( void ); }
+
+using std::cerr;
+
+TEST_SUITE(VResolverTestSuite);
+
+typedef enum {
+ e200,
+ e403,
+} EForbidden;
+
+class Test : protected ncbi :: NK :: TestCase {
+ TestCase * _dad;
+public:
+ Test ( const std :: string & description, TestCase * dad,
+ EForbidden forbidden, const char * name, const char * url,
+ bool fail = false )
+ : TestCase ( name ), _dad ( dad )
+ {
+ rc_t rc = 0;
+
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS ), ( "TESTING %s\n", name ) );
+
+ KDirectory * native = NULL;
+ REQUIRE_RC ( KDirectoryNativeDir ( & native ) );
+
+ const KDirectory * dir = NULL;
+ REQUIRE_RC ( KDirectoryOpenDirRead ( native,
+ &dir, false, "redirect-rejected-names-cgi-http-to-https" ) );
+
+ KConfig * cfg = NULL;
+ REQUIRE_RC ( KConfigMake ( & cfg, dir ) );
+ REQUIRE_RC ( KConfigWriteString
+ ( cfg, "repository/remote/main/CGI/resolver-cgi", url ) );
+
+ VFSManager * vfs = NULL;
+ REQUIRE_RC ( VFSManagerMakeFromKfg ( & vfs, cfg ) );
+
+ VResolver * resolver = NULL;
+ REQUIRE_RC ( VFSManagerGetResolver ( vfs, & resolver ) );
+
+ VPath * query = NULL;
+ REQUIRE_RC ( VFSManagerMakePath ( vfs, & query, "SRR000001" ) );
+
+ const VPath * remote = NULL;
+
+ if ( forbidden == e403 ) {
+#ifdef VDB_3162
+ TESTING_VDB_3162 ();
+#endif
+ }
+
+#ifdef VDB_3162
+ if ( fail )
+ REQUIRE_RC_FAIL
+ ( VResolverQuery ( resolver, 0, query, 0, & remote, 0 ) );
+ else
+#endif
+ {
+ KNSManager * mgr = NULL;
+ REQUIRE_RC ( VFSManagerGetKNSMgr ( vfs, & mgr ) );
+ KClientHttpRequest * req = NULL;
+ if ( KNSManagerMakeClientRequest
+ ( mgr, & req, 0x01010000, NULL, url )
+ == SILENT_RC
+ ( rcNS, rcNoTarg, rcValidating, rcConnection, rcNotFound ) )
+ {
+ std::cerr << "Skipped test of not found " << url << "\n";
+ } else {
+ REQUIRE_RC
+ ( VResolverQuery ( resolver, 0, query, 0, & remote, 0 ) );
+ }
+ RELEASE ( KClientHttpRequest, req );
+ RELEASE ( KNSManager, mgr );
+ }
+ RELEASE ( VPath, query );
+
+ RELEASE ( VPath, remote );
+
+ RELEASE ( VResolver, resolver );
+
+ RELEASE ( VFSManager, vfs );
+
+ RELEASE ( KConfig, cfg );
+
+ RELEASE ( KDirectory, dir );
+
+ RELEASE ( KDirectory, native );
+
+ REQUIRE ( ! rc );
+ }
+ ~Test ( void ) {
+ assert( _dad );
+ _dad -> ErrorCounterAdd ( GetErrorCounter () );
+ }
+};
+
+TEST_CASE(TEST) {
+#ifdef VDB_3162
+#else
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS ),
+ ( "Simulation 403 in VFS library was DISABLED\n\n" ) );
+#endif
+ Test ( "Called over HTTP: retry over HTTPS after 403", this, e403 , "HTTP",
+ "http://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" );
+ Test ( "Called over htTP: retry over https after 403", this, e403 , "htTP",
+ "hTtP://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" );
+ Test ( "Called over http to GOV: retry over HTTPS after 403",
+ this, e403 , "http-gOv",
+ "http://www.ncbi.nlm.nih.GoV/Traces/names/names.cgi" );
+
+ Test ( "Called over HTTPS: got 200", this, e200, "HTTPS",
+ "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" );
+ Test ( "Called over httPS: got 200", this, e200, "httPS",
+ "httPS://www.ncbi.nlm.nih.gov/Traces/names/names.cgi" );
+ Test ( "Called over HTTPS to Gov: got 200", this, e200, "https to Gov",
+ "https://www.ncbi.nlm.nih.GoV/Traces/names/names.cgi" );
+
+ Test (
+ "Called over HTTP: fail after 403 - not retrying non-government sites",
+ this, e403, "HTTP, not government",
+ "http://www/Traces/names/names.cgi", true );
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS ), ( "\n" ) );
+ Test (
+ "Called over htTP: fail after 403 - not retrying non-government sites",
+ this, e403, "htTP, not government",
+ "htTP://www/Traces/names/names.cgi", true );
+
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS ), ( "\n" ) );
+ Test ( "Called over HTTPS: fail after 403", this, e403, "403 by HTTPS",
+ "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi", true );
+ DBGMSG ( DBG_VFS, DBG_FLAG ( DBG_VFS ), ( "\n" ) );
+ Test ( "Called over httPS: fail after 403", this, e403, "403 by httPS",
+ "httPS://www.ncbi.nlm.nih.gov/Traces/names/names.cgi", true );
+}
+
+extern "C" {
+ ver_t CC KAppVersion ( void ) { return 0; }
+ rc_t CC UsageSummary ( const char * progname) { return 0; }
+ rc_t CC Usage ( const struct Args * args ) { return 0; }
+ const char UsageDefaultName[] = "redirect-rejected-names-cgi-http-to-https";
+ rc_t CC KMain ( int argc, char *argv [] ) {
+ assert ( ! KDbgSetString ( "VFS" ) );
+ return VResolverTestSuite ( argc, argv );
+ }
+}
diff --git a/test/vfs/redirect-rejected-names-cgi-http-to-https/incomplete.kfg b/test/vfs/redirect-rejected-names-cgi-http-to-https/incomplete.kfg
new file mode 100644
index 0000000..21731b3
--- /dev/null
+++ b/test/vfs/redirect-rejected-names-cgi-http-to-https/incomplete.kfg
@@ -0,0 +1 @@
+foo = "bar"
\ No newline at end of file
diff --git a/test/vxf/Makefile b/test/vxf/Makefile
index 064aab3..dbba80c 100644
--- a/test/vxf/Makefile
+++ b/test/vxf/Makefile
@@ -29,8 +29,9 @@ TOP ?= $(abspath ../..)
MODULE = test/vxf
TEST_TOOLS = \
- wb-test-vxf
-
+ wb-test-vxf \
+ vdb-2915-test
+
include $(TOP)/build/Makefile.env
$(TEST_TOOLS): makedirs
@@ -60,3 +61,23 @@ TEST_LIBS = \
$(TEST_BINDIR)/wb-test-vxf: $(TEST_OBJ)
$(LP) --exe -o $@ $^ $(TEST_LIBS)
+
+#-------------------------------------------------------------------------------
+# vdb-2915
+#
+VDB_2915_SRC = \
+ VDB-2915
+
+VDB_2915_OBJ = \
+ $(addsuffix .$(OBJX),$(VDB_2915_SRC))
+
+VDB_2915_LIBS = \
+ -skapp \
+ -sktst \
+ -sncbi-wvdb \
+ -sxml2 \
+ -sm
+
+$(TEST_BINDIR)/vdb-2915-test: $(VDB_2915_OBJ)
+ $(LP) --exe -o $@ $^ $(VDB_2915_LIBS)
+
diff --git a/test/vxf/VDB-2915.cpp b/test/vxf/VDB-2915.cpp
new file mode 100644
index 0000000..fe4276a
--- /dev/null
+++ b/test/vxf/VDB-2915.cpp
@@ -0,0 +1,508 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* ===========================================================================
+*
+*/
+#include <vdb/manager.h>
+#include <vdb/table.h>
+#include <vdb/cursor.h>
+#include <vdb/schema.h>
+#include <vdb/vdb-priv.h>
+
+#include <kfg/config.h>
+#include <vfs/manager.h>
+#include <vfs/manager-priv.h>
+
+#include <kdb/table.h>
+
+#include <ktst/unit_test.hpp>
+#include <klib/out.h>
+#include <klib/rc.h>
+
+#include <sstream>
+#include <cstdlib>
+#include <algorithm>
+#include <iterator>
+#include <sysalloc.h>
+
+using namespace std;
+
+TEST_SUITE( VDB_2915_TEST_SUITE );
+
+/* --------------------------------------------------------------------------
+There was a problem in the irzip, izip and fzip encoding :
+
+If the data was too random, the library was not able to load the data.
+
+The reason for that was a copy-paste error which has now been fixed.
+
+This test creates a temporary output-tabe, and tries to load random
+data into it. This has to succeed!
+--------------------------------------------------------------------------*/
+
+const string ScratchDir = "./";
+
+/*
+typedef union
+{
+ float f;
+ struct
+ {
+ unsigned int mantisa : 23;
+ unsigned int exponent : 8;
+ unsigned int sign : 1;
+ } parts;
+} double_cast;
+
+static float random_float( void )
+{
+ double_cast x;
+ x.parts.mantisa = rand() & 0x7FFFFF;
+ x.parts.exponent = rand() & 0xFF;
+ x.parts.sign = rand() & 0x1;
+ return x.f;
+}
+
+static void fill_random_float_buffer( float * buffer, size_t value_count )
+{
+ for ( size_t i = 0; i < value_count; ++i )
+ buffer[ i ] = random_float();
+}
+*/
+
+static int32_t rand_int32()
+{
+ int32_t res = rand();
+ if ( rand() & 1 ) res *= -1;
+ return res;
+}
+
+/*
+static int64_t rand_int64()
+{
+ int64_t res = rand();
+ res *= rand();
+ if ( rand() & 1 ) res *= -1;
+ return res;
+}
+
+static uint64_t rand_uint64()
+{
+ uint64_t res = rand();
+ res *= rand();
+ res &= 0x0FFFFFFFFFFFFFFF; // JOJOBA: kinda strange :)
+ // res &= 0x7FFFFFFFFFFFFFFF;
+ return res;
+}
+*/
+
+class WVDB_Fixture
+{
+ KDirectory * _wd;
+ KConfig * _cfg;
+public:
+ WVDB_Fixture() : _wd ( NULL ), _cfg ( NULL ), m_remove( false )
+ , m_tableName( "RANDOM_DATA" ), m_table( 0 ), m_cursor( 0 )
+ {
+ THROW_ON_RC( KDirectoryNativeDir( &_wd ) );
+
+ const KDirectory * dir = NULL;
+ THROW_ON_RC
+ ( KDirectoryOpenDirRead ( _wd, &dir, false, "local_config" ) );
+
+ THROW_ON_RC( KConfigMake ( &_cfg, dir ) );
+
+ KConfigPrint( _cfg, 0 ); /* for test */
+
+ THROW_ON_RC( KDirectoryRelease( dir ) );
+ }
+
+ ~WVDB_Fixture()
+ {
+ if ( m_cursor ) { VCursorRelease( m_cursor ); }
+ if ( m_table ) { VTableRelease( m_table ); }
+ if ( m_remove ) { RemoveTable(); }
+ KConfigRelease( _cfg );
+ KDirectoryRelease( _wd );
+ }
+
+ void RemoveTable()
+ {
+ if ( ! m_tableName . empty () )
+ {
+ KDirectoryRemove( _wd, true, m_tableName.c_str() );
+ }
+ }
+
+ void MakeTable( const string& p_schemaText, const string& p_schemaSpec )
+ {
+ RemoveTable();
+
+ struct VFSManager * vfs_mgr;
+ THROW_ON_RC( VFSManagerMakeFromKfg ( &vfs_mgr, _cfg ) );
+
+ VDBManager * mgr;
+ THROW_ON_RC
+ ( VDBManagerMakeUpdateWithVFSManager ( &mgr, _wd, vfs_mgr ) );
+
+ VSchema *schema;
+ THROW_ON_RC( VDBManagerMakeSchema ( mgr, &schema ) );
+ THROW_ON_RC( VSchemaParseText( schema, NULL, p_schemaText.c_str(), p_schemaText.size() ) );
+ THROW_ON_RC( VDBManagerCreateTable( mgr,
+ &m_table,
+ schema,
+ p_schemaSpec.c_str(),
+ kcmInit + kcmMD5,
+ "%s",
+ m_tableName.c_str() ) );
+ THROW_ON_RC( VTableCreateCursorWrite( m_table, &m_cursor, kcmInsert ) );
+ THROW_ON_RC( VSchemaRelease( schema ) );
+ THROW_ON_RC( VDBManagerRelease( mgr ) );
+ THROW_ON_RC( VFSManagerRelease( vfs_mgr ) );
+ }
+
+ void OpenTable()
+ {
+ VDBManager * mgr;
+ THROW_ON_RC( VDBManagerMakeUpdate ( &mgr, NULL ) );
+ THROW_ON_RC( VDBManagerOpenTableRead( mgr,
+ ( const VTable ** )&m_table,
+ NULL,
+ "%s",
+ m_tableName.c_str() ) );
+ THROW_ON_RC( VTableCreateCursorRead( m_table, ( const VCursor ** )&m_cursor ) );
+ THROW_ON_RC( VDBManagerRelease( mgr ) );
+ }
+
+ bool m_remove;
+ string m_tableName;
+ VTable * m_table;
+ VCursor * m_cursor;
+};
+
+/*
+ we have to exclude 64-bit tests, because the write-lib is broken for irzip
+ if feed with random data
+*/
+
+FIXTURE_TEST_CASE ( LOAD_RANDOM_DATA, WVDB_Fixture )
+{
+ RemoveTable();
+
+ string schemaText = "version 1;"
+ "include 'vdb/vdb.vschema';"
+ "table A_TABLE #1.0"
+ "{"
+ " column < U8 > izip_encoding CU1;"
+ " column U8 RU1;"
+ " column < U16 > izip_encoding CU2;"
+ " column U16 RU2;"
+ " column < U32 > izip_encoding CU3;"
+ " column U32 RU3;"
+ " column < U64 > izip_encoding CU4;"
+ " column U64 RU4;"
+
+ " column < I8 > izip_encoding CI1;"
+ " column I8 RI1;"
+ " column < I16 > izip_encoding CI2;"
+ " column I16 RI2;"
+ " column < I32 > izip_encoding CI3;"
+ " column I32 RI3;"
+ " column < I64 > izip_encoding CI4;"
+ " column I64 RI4;"
+ "};";
+
+ REQUIRE_RC( KOutMsg( "creating table '%s'\n", m_tableName.c_str() ) );
+
+ MakeTable ( schemaText, "A_TABLE" );
+
+ uint32_t cu1, cu2, cu3 /*, cu4 */;
+ uint32_t ru1, ru2, ru3 /*, ru4 */;
+
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &cu1, "%s", "CU1" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &cu2, "%s", "CU2" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &cu3, "%s", "CU3" ) );
+ /* REQUIRE_RC ( VCursorAddColumn ( m_cursor, &cu4, "%s", "CU4" ) ); */
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ru1, "%s", "RU1" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ru2, "%s", "RU2" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ru3, "%s", "RU3" ) );
+ /* REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ru4, "%s", "RU4" ) ); */
+
+ uint32_t ci1, ci2, ci3 /*, ci4 */;
+ uint32_t ri1, ri2, ri3 /*, ri4 */;
+
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ci1, "%s", "CI1" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ci2, "%s", "CI2" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ci3, "%s", "CI3" ) );
+ /* REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ci4, "%s", "CI4" ) ); */
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ri1, "%s", "RI1" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ri2, "%s", "RI2" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ri3, "%s", "RI3" ) );
+ /* REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ri4, "%s", "RI4" ) ); */
+
+ REQUIRE_RC ( VCursorOpen ( m_cursor ) );
+
+ srand( 123456 );
+
+ size_t i, j;
+ for ( i = 1; i <= 512; ++i )
+ {
+ REQUIRE_RC ( VCursorOpenRow ( m_cursor ) );
+
+ {
+ uint8_t b[ 1024 ];
+ for ( j = 0; j < 1024; ++j ) b[ j ] = rand() & 0xFF;
+ REQUIRE_RC ( VCursorWrite ( m_cursor, cu1, 8, b, 0, 1024 ) );
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ru1, 8, b, 0, 1024 ) );
+ }
+
+ {
+ int8_t b[ 1024 ];
+ for ( j = 0; j < 1024; ++j ) b[ j ] = rand() & 0xFF;
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ci1, 8, b, 0, 1024 ) );
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ri1, 8, b, 0, 1024 ) );
+ }
+
+ {
+ uint16_t b[ 1024 ];
+ for ( j = 0; j < 1024; ++j ) b[ j ] = rand() & 0xFFFF;
+ REQUIRE_RC ( VCursorWrite ( m_cursor, cu2, 16, b, 0, 1024 ) );
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ru2, 16, b, 0, 1024 ) );
+ }
+
+ {
+ int16_t b[ 1024 ];
+ for ( j = 0; j < 1024; ++j ) b[ j ] = rand() & 0xFFFF;
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ci2, 16, b, 0, 1024 ) );
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ri2, 16, b, 0, 1024 ) );
+ }
+
+ {
+ uint32_t b[ 1024 ];
+ for ( j = 0; j < 1024; ++j ) b[ j ] = rand();
+ REQUIRE_RC ( VCursorWrite ( m_cursor, cu3, 32, b, 0, 1024 ) );
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ru3, 32, b, 0, 1024 ) );
+ }
+
+ {
+ int32_t b[ 1024 ];
+ for ( j = 0; j < 1024; ++j ) b[ j ] = rand_int32();
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ci3, 32, b, 0, 1024 ) );
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ri3, 32, b, 0, 1024 ) );
+ }
+
+ /*
+ {
+ uint64_t b[ 1024 ];
+ for ( j = 0; j < 1024; ++j ) b[ j ] = rand_uint64();
+ REQUIRE_RC ( VCursorWrite ( m_cursor, cu4, 64, b, 0, 1024 ) );
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ru4, 64, b, 0, 1024 ) );
+ }
+
+ {
+ int64_t b[ 1024 ];
+ for ( j = 0; j < 1024; ++j ) b[ j ] = rand_int64();
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ci4, 64, b, 0, 1024 ) );
+ REQUIRE_RC ( VCursorWrite ( m_cursor, ri4, 64, b, 0, 1024 ) );
+ }
+ */
+
+ REQUIRE_RC ( VCursorCommitRow ( m_cursor ) );
+ REQUIRE_RC ( VCursorCloseRow ( m_cursor ) );
+ }
+ REQUIRE_RC ( VCursorCommit ( m_cursor ) );
+}
+
+FIXTURE_TEST_CASE ( CHECK_RANDOM_DATA, WVDB_Fixture )
+{
+ OpenTable();
+
+ REQUIRE_RC( KOutMsg( "verify table '%s'\n", m_tableName.c_str() ) );
+
+ uint32_t cu1, cu2, cu3 /*, cu4 */;
+ uint32_t ru1, ru2, ru3 /*, ru4 */;
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &cu1, "%s", "CU1" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &cu2, "%s", "CU2" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &cu3, "%s", "CU3" ) );
+ /* REQUIRE_RC ( VCursorAddColumn ( m_cursor, &cu4, "%s", "CU4" ) ); */
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ru1, "%s", "RU1" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ru2, "%s", "RU2" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ru3, "%s", "RU3" ) );
+ /* REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ru4, "%s", "RU4" ) ); */
+
+ uint32_t ci1, ci2, ci3 /*, ci4 */;
+ uint32_t ri1, ri2, ri3 /*, ri4 */;
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ci1, "%s", "CI1" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ci2, "%s", "CI2" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ci3, "%s", "CI3" ) );
+ /* REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ci4, "%s", "CI4" ) ); */
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ri1, "%s", "RI1" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ri2, "%s", "RI2" ) );
+ REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ri3, "%s", "RI3" ) );
+ /* REQUIRE_RC ( VCursorAddColumn ( m_cursor, &ri4, "%s", "RI4" ) ); */
+
+ REQUIRE_RC ( VCursorOpen ( m_cursor ) );
+
+ for ( int64_t i = 1; i <= 512; ++i )
+ {
+ uint32_t c_bits, r_bits, c_boff, r_boff, c_len, r_len;
+ {
+ uint8_t * c;
+ uint8_t * r;
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, cu1, &c_bits, ( const void** )&c, &c_boff, &c_len ) );
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ru1, &r_bits, ( const void** )&r, &r_boff, &r_len ) );
+ REQUIRE_EQ ( c_bits, ( uint32_t )8 );
+ REQUIRE_EQ ( r_bits, ( uint32_t )8 );
+ REQUIRE_EQ ( c_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( r_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( c_len, ( uint32_t )1024 );
+ REQUIRE_EQ ( r_len, ( uint32_t )1024 );
+ REQUIRE ( std::equal( c, c + 1024, r ) );
+ }
+
+ {
+ int8_t * c;
+ int8_t * r;
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ci1, &c_bits, ( const void** )&c, &c_boff, &c_len ) );
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ri1, &r_bits, ( const void** )&r, &r_boff, &r_len ) );
+ REQUIRE_EQ ( c_bits, ( uint32_t )8 );
+ REQUIRE_EQ ( r_bits, ( uint32_t )8 );
+ REQUIRE_EQ ( c_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( r_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( c_len, ( uint32_t )1024 );
+ REQUIRE_EQ ( r_len, ( uint32_t )1024 );
+ REQUIRE ( std::equal( c, c + 1024, r ) );
+ }
+
+ {
+ uint16_t * c;
+ uint16_t * r;
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, cu2, &c_bits, ( const void** )&c, &c_boff, &c_len ) );
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ru2, &r_bits, ( const void** )&r, &r_boff, &r_len ) );
+ REQUIRE_EQ ( c_bits, ( uint32_t )16 );
+ REQUIRE_EQ ( r_bits, ( uint32_t )16 );
+ REQUIRE_EQ ( c_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( r_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( c_len, ( uint32_t )1024 );
+ REQUIRE_EQ ( r_len, ( uint32_t )1024 );
+ REQUIRE ( std::equal( c, c + 1024, r ) );
+ }
+
+ {
+ int16_t * c;
+ int16_t * r;
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ci2, &c_bits, ( const void** )&c, &c_boff, &c_len ) );
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ri2, &r_bits, ( const void** )&r, &r_boff, &r_len ) );
+ REQUIRE_EQ ( c_bits, ( uint32_t )16 );
+ REQUIRE_EQ ( r_bits, ( uint32_t )16 );
+ REQUIRE_EQ ( c_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( r_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( c_len, ( uint32_t )1024 );
+ REQUIRE_EQ ( r_len, ( uint32_t )1024 );
+ REQUIRE ( std::equal( c, c + 1024, r ) );
+ }
+
+ {
+ uint32_t * c;
+ uint32_t * r;
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, cu3, &c_bits, ( const void** )&c, &c_boff, &c_len ) );
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ru3, &r_bits, ( const void** )&r, &r_boff, &r_len ) );
+ REQUIRE_EQ ( c_bits, ( uint32_t )32 );
+ REQUIRE_EQ ( r_bits, ( uint32_t )32 );
+ REQUIRE_EQ ( c_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( r_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( c_len, ( uint32_t )1024 );
+ REQUIRE_EQ ( r_len, ( uint32_t )1024 );
+ REQUIRE ( std::equal( c, c + 1024, r ) );
+ }
+
+ {
+ int32_t * c;
+ int32_t * r;
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ci3, &c_bits, ( const void** )&c, &c_boff, &c_len ) );
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ri3, &r_bits, ( const void** )&r, &r_boff, &r_len ) );
+ REQUIRE_EQ ( c_bits, ( uint32_t )32 );
+ REQUIRE_EQ ( r_bits, ( uint32_t )32 );
+ REQUIRE_EQ ( c_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( r_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( c_len, ( uint32_t )1024 );
+ REQUIRE_EQ ( r_len, ( uint32_t )1024 );
+ REQUIRE ( std::equal( c, c + 1024, r ) );
+ }
+
+ /*
+ {
+ uint64_t * c;
+ uint64_t * r;
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, cu4, &c_bits, ( const void** )&c, &c_boff, &c_len ) );
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ru4, &r_bits, ( const void** )&r, &r_boff, &r_len ) );
+ REQUIRE_EQ ( c_bits, ( uint32_t )64 );
+ REQUIRE_EQ ( r_bits, ( uint32_t )64 );
+ REQUIRE_EQ ( c_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( r_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( c_len, ( uint32_t )1024 );
+ REQUIRE_EQ ( r_len, ( uint32_t )1024 );
+ REQUIRE ( std::equal( c, c + 1024, r ) );
+ }
+
+ {
+ int64_t * c;
+ int64_t * r;
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ci4, &c_bits, ( const void** )&c, &c_boff, &c_len ) );
+ REQUIRE_RC ( VCursorCellDataDirect ( m_cursor, i, ri4, &r_bits, ( const void** )&r, &r_boff, &r_len ) );
+ REQUIRE_EQ ( c_bits, ( uint32_t )64 );
+ REQUIRE_EQ ( r_bits, ( uint32_t )64 );
+ REQUIRE_EQ ( c_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( r_boff, ( uint32_t )0 );
+ REQUIRE_EQ ( c_len, ( uint32_t )1024 );
+ REQUIRE_EQ ( r_len, ( uint32_t )1024 );
+ REQUIRE ( std::equal( c, c + 1024, r ) );
+ }
+ */
+ }
+
+ m_remove = true;
+}
+
+
+//////////////////////////////////////////// Main
+extern "C"
+{
+
+#include <kapp/args.h>
+#include <kfg/config.h>
+
+ver_t CC KAppVersion ( void ) { return 0x1000000; }
+rc_t CC UsageSummary ( const char * progname ) { return 0; }
+rc_t CC Usage ( const Args * args ){ return 0; }
+
+const char UsageDefaultName[] = "vdb_2915_testwb-test-vxf";
+
+rc_t CC KMain ( int argc, char *argv [] )
+{
+ KConfigDisableUserSettings();
+ rc_t rc = VDB_2915_TEST_SUITE( argc, argv );
+ return rc;
+}
+
+}
diff --git a/test/vxf/local_config/local.kfg b/test/vxf/local_config/local.kfg
new file mode 100644
index 0000000..9c77995
--- /dev/null
+++ b/test/vxf/local_config/local.kfg
@@ -0,0 +1,2 @@
+/vdb/schema/paths="$(PWD)/../../interfaces"
+/repository/remote/main/CGI/resolver-cgi = "https://www.ncbi.nlm.nih.gov/Traces/names/names.cgi"
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/ncbi-vdb.git
More information about the debian-med-commit
mailing list