[med-svn] [Git][med-team/ncbi-vdb][upstream] New upstream version 2.9.3+dfsg
Andreas Tille
gitlab at salsa.debian.org
Fri Jan 11 22:21:12 GMT 2019
Andreas Tille pushed to branch upstream at Debian Med / ncbi-vdb
Commits:
4e78839f by Andreas Tille at 2019-01-11T22:11:22Z
New upstream version 2.9.3+dfsg
- - - - -
14 changed files:
- CHANGES.md
- build/Makefile.vers
- libs/blast/blast-mgr.c
- libs/kdb/libkdb.vers.h
- libs/klib/release-vers.h
- libs/kns/http-file.c
- libs/kns/tls-priv.h
- libs/kns/tls.c
- libs/ncbi-vdb/libncbi-vdb.vers
- libs/vdb/cursor-table.c
- libs/vdb/libvdb.vers.h
- test/kns/Makefile
- + test/kns/test200for-whole-file.cpp
- test/vdb/test-tablecursor.cpp
Changes:
=====================================
CHANGES.md
=====================================
@@ -1,6 +1,13 @@
# NCBI External Developer Release:
+## NCBI VDB 2.9.3
+**October 17, 2018**
+ **kns**: added possibility to skip server's certificate validation
+ **kns**: expect to receive HTTP status 200 when sending range-request that includes the whole file
+ **vdb**: fixed a bug in accessing pagemap process request for cursors which do not have pagemap thread running
+
+
## NCBI VDB 2.9.2-1
**July 31, 2018**
**vdb**: restored lost exported attribute on several entries in the private API
=====================================
build/Makefile.vers
=====================================
@@ -23,4 +23,4 @@
# ===========================================================================
# NCBI-VDB and library version
-VERSION = 2.9.2
+VERSION = 2.9.3
=====================================
libs/blast/blast-mgr.c
=====================================
@@ -50,7 +50,7 @@
#include <stdio.h> /* fprintf */
#include <string.h> /* memset */
-#define TOOLKIT "sratoolkit2_9_2"
+#define TOOLKIT "sratoolkit2_9_3"
/******************************************************************************/
=====================================
libs/kdb/libkdb.vers.h
=====================================
@@ -24,4 +24,4 @@
*
*/
-#define LIBKDB_VERS 0x0207001B
+#define LIBKDB_VERS 0x0207001C
=====================================
libs/klib/release-vers.h
=====================================
@@ -28,7 +28,7 @@
/* Version of current SRA Toolkit Release */
-#define RELEASE_VERS 0x02090002
+#define RELEASE_VERS 0x02090003
/* Type of Version of current SRA Toolkit Release is one of:
=====================================
libs/kns/http-file.c
=====================================
@@ -250,32 +250,55 @@ otherwise we are going to hit "Apache return HTTP headers twice" bug */
TRACE ( "KClientHttpResultStatus ( rslt, & http_status, NULL, 0, NULL ); failed: rc=%u\n", rc );
else
{
+ uint64_t start_pos = 0;
+ size_t result_size = 0;
+ bool have_size = false;
switch ( * http_status )
{
+ case 200:
+ /* We requested Bytes Range but got 200:
+ accept it what the whole file was requested */
+ have_size = KClientHttpResultSize ( rslt,
+ &result_size );
+ if ( pos != 0 || ! have_size
+ || result_size > bsize )
+ {
+ rc = RC ( rcNS, rcFile, rcReading,
+ rcData, rcUnexpected );
+ TRACE ( "KClientHttpResultStatus ( rslt, "
+"& http_status, NULL, 0, NULL ); unexpected status=%d\n", * http_status );
+ break;
+ }
+ /* no break here, now read the file */
case 206:
- {
- uint64_t start_pos;
- size_t result_size;
-
/* don't need retries now */
proxy_retries = 0;
/* extract actual amount being returned by server */
- rc = KClientHttpResultRange ( rslt, &start_pos, &result_size );
- if ( rc != 0 || start_pos != pos || result_size != bsize )
- {
- if ( rc != 0 )
- TRACE ( "KClientHttpResultRange ( rslt, & start_pos, & result_size ); failed: rc=%u\n", rc );
- else
+ if ( * http_status == 206 ) {
+ /* get result range when 206 was returned,
+ we got it already when status == 200 */
+ rc = KClientHttpResultRange ( rslt,
+ &start_pos, &result_size );
+ if ( rc != 0 || start_pos != pos
+ || result_size != bsize )
{
- if ( start_pos != pos )
- TRACE ( "KClientHttpResultRange ( rslt, & start_pos, & result_size ); failed: start_pos=%lu != pos=%lu\n", start_pos, pos );
- if ( result_size != bsize )
- TRACE ( "KClientHttpResultRange ( rslt, & start_pos, & result_size ); failed: result_size=%lu != bsize=%lu\n", result_size, bsize );
+ if ( rc != 0 )
+ TRACE ( "KClientHttpResultRange ( rslt, & start_pos, & result_size ); failed: rc=%u\n", rc );
+ else
+ {
+ if ( start_pos != pos )
+ TRACE ( "KClientHttpResultRange ( rslt, & start_pos, & result_size ); failed: start_pos=%lu != pos=%lu\n", start_pos, pos );
+ if ( result_size != bsize )
+ TRACE ( "KClientHttpResultRange ( rslt, & start_pos, & result_size ); failed: result_size=%lu != bsize=%lu\n", result_size, bsize );
+ }
+ break;
}
}
- else
{
+ /* read the response for partial file requests
+ or when the whole file was returned */
+
KStream *response;
rc = KClientHttpResultGetInputStream ( rslt, &response );
@@ -318,7 +341,6 @@ otherwise we are going to hit "Apache return HTTP headers twice" bug */
}
}
break;
- }
case 403:
case 404:
=====================================
libs/kns/tls-priv.h
=====================================
@@ -64,6 +64,8 @@ struct KTLSGlobals
mbedtls_ssl_config config;
bool allow_all_certs;
+
+ bool safe_to_modify_ssl_config; /* needed for SetAllowAllCerts () */
};
/* Init
=====================================
libs/kns/tls.c
=====================================
@@ -457,10 +457,23 @@ rc_t tlsg_setup ( KTLSGlobals * self )
return rc;
}
- vdb_mbedtls_ssl_conf_authmode( &self -> config, MBEDTLS_SSL_VERIFY_REQUIRED );
+ /* turn off certificate validation when self -> allow_all_certs == true */
+ vdb_mbedtls_ssl_conf_authmode( &self -> config,
+ self -> allow_all_certs ? MBEDTLS_SSL_VERIFY_OPTIONAL
+ : MBEDTLS_SSL_VERIFY_REQUIRED );
+
vdb_mbedtls_ssl_conf_ca_chain( &self -> config, &self -> cacert, NULL );
vdb_mbedtls_ssl_conf_rng( &self -> config, vdb_mbedtls_ctr_drbg_random, &self -> ctr_drbg );
+ /* We need that to be sure that we are free to call
+ * vdb_mbedtls_ssl_conf_authmode () next time when
+ * KNSManagerSetAllowAllCerts () will be called
+ *
+ * Because smart special design we do not need to add
+ * special code to deinitialize that variable.
+ */
+ self -> safe_to_modify_ssl_config = true;
+
return 0;
}
@@ -553,6 +566,21 @@ LIB_EXPORT rc_t CC KNSManagerSetAllowAllCerts ( KNSManager *self, bool allow_all
else
{
self -> tlsg . allow_all_certs = allow_all_certs;
+ /*
+ * We are acting from supposition that at some particular
+ * moments there should be called initlialisation of
+ * TLS configurations, which will be reflected at next
+ * handshake
+ */
+ if ( self -> tlsg . safe_to_modify_ssl_config ) {
+ vdb_mbedtls_ssl_conf_authmode(
+ &self -> tlsg . config,
+ ( self -> tlsg . allow_all_certs
+ ? MBEDTLS_SSL_VERIFY_OPTIONAL
+ : MBEDTLS_SSL_VERIFY_REQUIRED
+ )
+ );
+ }
}
return rc;
=====================================
libs/ncbi-vdb/libncbi-vdb.vers
=====================================
@@ -1 +1 @@
-2.9.2
+2.9.3
=====================================
libs/vdb/cursor-table.c
=====================================
@@ -2201,7 +2201,7 @@ uint64_t CC VTableCursorGetCacheCapacity(const VCURSOR_IMPL *self)
const PageMapProcessRequest* VTableCursorPageMapProcessRequest(const struct VCURSOR_IMPL *self)
{
assert ( self != NULL );
- return self->pagemap_thread ? &self->pmpr : NULL;
+ return & self -> pmpr;
}
const struct VTable * VTableCursorGetTable ( const struct VCURSOR_IMPL * self )
=====================================
libs/vdb/libvdb.vers.h
=====================================
@@ -24,4 +24,4 @@
*
*/
-#define LIBVDB_VERS 0x0207001B
+#define LIBVDB_VERS 0x0207001C
=====================================
test/kns/Makefile
=====================================
@@ -31,6 +31,7 @@ TOP ?= $(abspath ../..)
MODULE = test/kns
TEST_TOOLS = \
+ test200for-whole-file \
test-kns \
test-http-dropconn \
KNSManagerSingletonTest \
@@ -126,6 +127,18 @@ $(TEST_BINDIR)/test-kns: $(KNSTEST_OBJ)
kns: test-kns
$(TEST_BINDIR)/test-kns #-l=all
+#----------------------------------------------------------------
+# test200for-whole-file
+#
+K2_SRC = \
+ test200for-whole-file \
+
+K2_OBJ = \
+ $(addsuffix .$(OBJX),$(K2_SRC))
+
+$(TEST_BINDIR)/test200for-whole-file: $(K2_OBJ)
+ $(LP) --exe -o $@ $^ $(KNSTEST_LIB)
+
#----------------------------------------------------------------
# test-proxy
#
=====================================
test/kns/test200for-whole-file.cpp
=====================================
@@ -0,0 +1,139 @@
+/*===========================================================================
+*
+* PUBLIC DOMAIN NOTICE
+* National Center for Biotechnology Information
+*
+* This software/database is a "United States Government Work" under the
+* terms of the United States Copyright Act. It was written as part of
+* the author's official duties as a United States Government employee and
+* thus cannot be copyrighted. This software/database is freely available
+* to the public for use. The National Library of Medicine and the U.S.
+* Government have not placed any restriction on its use or reproduction.
+*
+* Although all reasonable efforts have been taken to ensure the accuracy
+* and reliability of the software and data, the NLM and the U.S.
+* Government do not and cannot warrant the performance or results that
+* may be obtained by using this software or data. The NLM and the U.S.
+* Government disclaim all warranties, express or implied, including
+* warranties of performance, merchantability or fitness for any particular
+* purpose.
+*
+* Please cite the author in any work or product based on this material.
+*
+* =========================================================================== */
+
+#include <kapp/args.h> /* ArgsMakeAndHandle */
+#include <kfg/kfg-priv.h> /* KConfigMakeEmpty */
+#include <kfs/file.h> /* KFileRelease */
+#include <klib/debug.h> /* KDbgSetString */
+#include <kns/http.h> /* KNSManagerMakeHttpFile */
+#include <kns/manager.h> /* KNSManagerRelease */
+#include <ktst/unit_test.hpp> /* TEST_SUITE_WITH_ARGS_HANDLER */
+
+#define RELEASE( type, obj ) do { rc_t rc2 = type##Release ( obj ); \
+ if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while ( false )
+
+static rc_t argsHandler ( int argc, char * argv [] )
+{ return ArgsMakeAndHandle ( NULL, argc, argv, 0, NULL, 0 ); }
+
+TEST_SUITE_WITH_ARGS_HANDLER ( T200FOR_WHOLE_FILE, argsHandler )
+
+TEST_CASE ( Test_206) {
+ KNSManager * mgr = NULL;
+ REQUIRE_RC ( KNSManagerMake ( & mgr ) );
+
+ const KFile * file = NULL;
+ REQUIRE_RC ( KNSManagerMakeHttpFile ( mgr, & file, NULL, 0x01010000,
+ "https://sra-download.ncbi.nlm.nih.gov/traces/refseq/KC702174.1" ) );
+
+ uint64_t size = 0;
+ REQUIRE_RC ( KFileSize ( file, & size ) );
+
+ void * buffer = malloc ( size );
+ REQUIRE_NOT_NULL ( buffer );
+
+ size_t num_read = 0;
+
+ // read incomplete file: expect 206 response code
+ size_t bsize = size - 1;
+ REQUIRE_RC ( KFileRead ( file, 0, buffer, bsize, & num_read ) );
+ REQUIRE_EQ ( num_read, bsize );
+
+ // read the whole: storage.googleapis.com returns 200
+ bsize = size;
+ REQUIRE_RC ( KFileRead ( file, 0, buffer, bsize, & num_read ) );
+ REQUIRE_EQ ( num_read, bsize );
+
+ // request more that file size: expect exact file size
+ REQUIRE_RC ( KFileRead ( file, 0, buffer, size * 2, & num_read ) );
+ REQUIRE_EQ ( num_read, static_cast < size_t > ( size ) );
+
+ free ( buffer );
+
+ REQUIRE_RC ( KFileRelease ( file ) );
+
+ REQUIRE_RC ( KNSManagerRelease ( mgr ) );
+}
+
+TEST_CASE ( Test_200 ) {
+ KNSManager * mgr = NULL;
+ REQUIRE_RC ( KNSManagerMake ( & mgr ) );
+
+ const KFile * file = NULL;
+ REQUIRE_RC ( KNSManagerMakeHttpFile ( mgr, & file, NULL, 0x01010000,
+ "https://storage.googleapis.com/yan-blastdb/2018-09-12-08-33-02/fuse.xml"
+ ) );
+
+ uint64_t size = 0;
+ REQUIRE_RC ( KFileSize ( file, & size ) );
+
+ void * buffer = malloc ( size );
+ REQUIRE_NOT_NULL ( buffer );
+
+ size_t num_read = 0;
+
+ // read incomplete file: expect 206 response code
+ size_t bsize = size - 1;
+ REQUIRE_RC ( KFileRead ( file, 0, buffer, bsize, & num_read ) );
+ REQUIRE_EQ ( num_read, bsize );
+
+ // read the whole: storage.googleapis.com returns 200
+ bsize = size;
+ REQUIRE_RC ( KFileRead ( file, 0, buffer, bsize, & num_read ) );
+ REQUIRE_EQ ( num_read, bsize );
+
+ // request more that file size: expect exact file size
+ REQUIRE_RC ( KFileRead ( file, 0, buffer, size * 2, & num_read ) );
+ REQUIRE_EQ ( num_read, static_cast < size_t > ( size ) );
+
+ free ( buffer );
+
+ REQUIRE_RC ( KFileRelease ( file ) );
+
+ REQUIRE_RC ( KNSManagerRelease ( mgr ) );
+}
+
+extern "C" {
+ const char UsageDefaultName[] = "test200for-whole-file";
+ rc_t CC UsageSummary ( const char * progname) { return 0; }
+ rc_t CC Usage ( const struct Args * args ) { return 0; }
+ ver_t CC KAppVersion ( void ) { return 0; }
+
+ rc_t CC KMain ( int argc, char * argv [] ) { if (
+0 ) assert ( ! KDbgSetString ( "KNS-HTTP" ) );
+ KConfigDisableUserSettings ();
+
+ KConfig * kfg = NULL;
+ rc_t rc = KConfigMakeEmpty ( & kfg );
+ // turn off certificate validation to download from storage.googleapis.com
+ if ( rc == 0 )
+ rc = KConfigWriteString ( kfg, "/tls/allow-all-certs", "true" );
+
+ if ( rc == 0 )
+ rc = T200FOR_WHOLE_FILE ( argc, argv );
+
+ RELEASE ( KConfig, kfg );
+
+ return rc;
+ }
+}
=====================================
test/vdb/test-tablecursor.cpp
=====================================
@@ -550,7 +550,9 @@ FIXTURE_TEST_CASE( VTableCursor_PageMapProcessRequest, TableCursorFixture )
// per non-cached cursor (see VTableCreateCachedCursorReadImpl() in cursor-table.c)
REQUIRE_RC ( VCursorLaunchPagemapThread ( (VCursor*)m_cur ) );
const PageMapProcessRequest * req = VCursorPageMapProcessRequest ( m_cur );
- REQUIRE_NULL ( req );
+ // VDB-3652: VCursorPageMapProcessRequest returns a non-NULL value even if the thread is not running
+ REQUIRE_NOT_NULL ( req );
+ REQUIRE_NULL ( req -> pm );
}
FIXTURE_TEST_CASE( VTableCursor_CacheActive, TableCursorFixture )
View it on GitLab: https://salsa.debian.org/med-team/ncbi-vdb/commit/4e78839f93dbedde5bd47bba97abf5b15c8788b4
--
View it on GitLab: https://salsa.debian.org/med-team/ncbi-vdb/commit/4e78839f93dbedde5bd47bba97abf5b15c8788b4
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20190111/d576c8f1/attachment-0001.html>
More information about the debian-med-commit
mailing list