[med-svn] [Git][med-team/libmaus2][upstream] New upstream version 2.0.713+ds
Andreas Tille
gitlab at salsa.debian.org
Fri Mar 27 08:49:12 GMT 2020
Andreas Tille pushed to branch upstream at Debian Med / libmaus2
Commits:
ab8fa5e2 by Andreas Tille at 2020-03-27T08:37:59+01:00
New upstream version 2.0.713+ds
- - - - -
8 changed files:
- ChangeLog
- configure.ac
- src/Makefile.am
- src/libmaus2/LibMausConfig.hpp.in
- src/libmaus2/bambam/Scram.c
- src/libmaus2/bambam/parallel/BlockMergeControl.hpp
- src/libmaus2/bambam/parallel/ScramCramEncoding.hpp
- src/libmaus2/geometry/RangeSet.hpp
Changes:
=====================================
ChangeLog
=====================================
@@ -1,3 +1,10 @@
+libmaus2 (2.0.713-1) unstable; urgency=medium
+
+ * Add new interface for setting CRAM compression profile in parallel CRAM encoding
+ * Clean up RangeSet interface
+
+ -- German Tischler-Höhle <germant at miltenyibiotec.de> Wed, 25 Mar 2020 17:38:36 +0100
+
libmaus2 (2.0.712-1) unstable; urgency=medium
* Add experimental archive mode for cram writing
=====================================
configure.ac
=====================================
@@ -1,5 +1,5 @@
-AC_INIT(libmaus2,2.0.712,[germant at miltnyibiotec.de],[libmaus2],[https://github.com/gt1/libmaus2])
-LIBRARY_VERSION=2:712:0
+AC_INIT(libmaus2,2.0.713,[germant at miltnyibiotec.de],[libmaus2],[https://github.com/gt1/libmaus2])
+LIBRARY_VERSION=2:713:0
AC_MSG_NOTICE([Configuring for source in directory ${srcdir}])
AC_CANONICAL_SYSTEM
AC_CANONICAL_HOST
@@ -2327,6 +2327,8 @@ IOLIBHAVENEWCRAMINTERFACE=no
LIBMAUS2_HAVE_IO_NEW_CRAM_INTERFACE=
LIBMAUS2_HAVE_IO_LIB_CRAM_ENCODER_GET_FD=
LIBMAUS2_HAVE_IO_LIB_BAMBAM_H=
+LIBMAUS2_HAVE_IO_LIB_CRAM_SET_OPTION=
+LIBMAUS2_IO_LIB_HAVE_CRAM_OPT_PROFILE=
AC_ARG_WITH([io_lib],
[AS_HELP_STRING([--with-io_lib@<:@=PATH@:>@], [support for io_lib library @<:@default=no@:>@])],
@@ -2399,8 +2401,16 @@ if test "${have_dynamic_loading}" = "yes" -a "${getpagesize}" = "yes" -a "${posi
AC_CHECK_FUNC(cram_enque_compression_block,cram_enque_compression_block=yes,cram_enque_compression_block=no)
AC_CHECK_FUNC(cram_process_work_package,cram_process_work_package=yes,cram_process_work_package=no)
AC_CHECK_FUNC(cram_encoder_get_fd,cram_encoder_get_fd=yes,cram_encoder_get_fd=no)
+ AC_CHECK_FUNC(cram_set_option,cram_set_option=yes,cram_set_option=no)
+ AC_CHECK_DECL([CRAM_OPT_PROFILE], [LIBMAUS2_IO_LIB_HAVE_CRAM_OPT_PROFILE=yes], [LIBMAUS2_IO_LIB_HAVE_CRAM_OPT_PROFILE=no], [#include <io_lib/scram.h>])
AC_LANG_POP
+ if test "${LIBMAUS2_IO_LIB_HAVE_CRAM_OPT_PROFILE}" = "yes" ; then
+ LIBMAUS2_IO_LIB_HAVE_CRAM_OPT_PROFILE="#define LIBMAUS2_IO_LIB_HAVE_CRAM_OPT_PROFILE"
+ else
+ LIBMAUS2_IO_LIB_HAVE_CRAM_OPT_PROFILE=
+ fi
+
if test "${io_lib_scram_h}" != "yes" ; then
AC_MSG_ERROR([--with-io_lib is set to ${with_io_lib}, but io_lib headers are not available])
fi
@@ -2603,6 +2613,9 @@ if test "${have_dynamic_loading}" = "yes" -a "${getpagesize}" = "yes" -a "${posi
if test $io_lib_cram_bambam_h = "yes" ; then
LIBMAUS2_HAVE_IO_LIB_BAMBAM_H="#define LIBMAUS2_HAVE_IO_LIB_BAMBAM_H"
fi
+ if test $cram_set_option = "yes" ; then
+ LIBMAUS2_HAVE_IO_LIB_CRAM_SET_OPTION="#define LIBMAUS2_HAVE_IO_LIB_CRAM_SET_OPTION"
+ fi
CPPFLAGS="${CPPFLAGSSAVE}"
LDFLAGS="${LDFLAGSSAVE}"
@@ -3292,4 +3305,6 @@ AC_SUBST([LIBMAUS2_HAVE_MMAP_HUGEPAGES])
AC_SUBST([LIBMAUS2_HAVE_SYS_PRCTL_H])
AC_SUBST([LIBMAUS2_HAVE_IO_LIB_CRAM_ENCODER_GET_FD])
AC_SUBST([LIBMAUS2_HAVE_IO_LIB_BAMBAM_H])
+AC_SUBST([LIBMAUS2_HAVE_IO_LIB_CRAM_SET_OPTION])
+AC_SUBST([LIBMAUS2_IO_LIB_HAVE_CRAM_OPT_PROFILE])
AC_OUTPUT(Makefile src/Makefile libmaus2.pc libmaus2irods.pc ubuntu.sh src/libmaus2/LibMausConfig.hpp test/Makefile)
=====================================
src/Makefile.am
=====================================
@@ -2502,13 +2502,13 @@ testgnutls_LDFLAGS = ${AM_LDFLAGS} @GNUTLSLIBS@
testgnutls_CXXFLAGS = ${AM_CXXFLAGS} @WARNCXXFLAGS@
testparallelbamsorting_SOURCES = test/testparallelbamsorting.cpp
-testparallelbamsorting_CPPFLAGS = ${AM_CPPFLAGS} @SNAPPYCPPFLAGS@ @IOLIBCPPFLAGS@
+testparallelbamsorting_CPPFLAGS = ${AM_CPPFLAGS} @SNAPPYCPPFLAGS@
testparallelbamsorting_LDFLAGS = ${AM_LDFLAGS} @SNAPPYLDFLAGS@
testparallelbamsorting_LDADD = libmaus2.la libmaus2_simd_align_128.la libmaus2_simd_align_256.la @SNAPPYLIBS@ ${DLLIB}
testparallelbamsorting_CXXFLAGS = ${AM_CXXFLAGS} @WARNCXXFLAGS@
testparallelbamvalidation_SOURCES = test/testparallelbamvalidation.cpp
-testparallelbamvalidation_CPPFLAGS = ${AM_CPPFLAGS} @SNAPPYCPPFLAGS@ @IOLIBCPPFLAGS@
+testparallelbamvalidation_CPPFLAGS = ${AM_CPPFLAGS} @SNAPPYCPPFLAGS@
testparallelbamvalidation_LDFLAGS = ${AM_LDFLAGS} @SNAPPYLDFLAGS@
testparallelbamvalidation_LDADD = libmaus2.la libmaus2_simd_align_128.la libmaus2_simd_align_256.la @SNAPPYLIBS@ ${DLLIB}
testparallelbamvalidation_CXXFLAGS = ${AM_CXXFLAGS} @WARNCXXFLAGS@
@@ -2542,11 +2542,11 @@ testmd5_CXXFLAGS = ${AM_CXXFLAGS} @WARNCXXFLAGS@
testparallelbamrewrite_SOURCES = test/testparallelbamrewrite.cpp
testparallelbamrewrite_LDADD = libmaus2.la libmaus2_simd_align_128.la libmaus2_simd_align_256.la
-testparallelbamrewrite_CXXFLAGS = ${AM_CXXFLAGS} @WARNCXXFLAGS@ @IOLIBCPPFLAGS@
+testparallelbamrewrite_CXXFLAGS = ${AM_CXXFLAGS} @WARNCXXFLAGS@
testparallelbamblocksort_SOURCES = test/testparallelbamblocksort.cpp
testparallelbamblocksort_LDADD = libmaus2.la libmaus2_simd_align_128.la libmaus2_simd_align_256.la @SNAPPYLIBS@ ${DLLIB} @GMPLIBS@
-testparallelbamblocksort_CPPFLAGS = ${AM_CPPFLAGS} ${CPPFLAGS} @SNAPPYCPPFLAGS@ @IOLIBCPPFLAGS@
+testparallelbamblocksort_CPPFLAGS = ${AM_CPPFLAGS} ${CPPFLAGS} @SNAPPYCPPFLAGS@
testparallelbamblocksort_CXXFLAGS = ${AM_CXXFLAGS} @WARNCXXFLAGS@
testparallelbamblocksort_LDFLAGS = ${AM_LDFLAGS} ${LDFLAGS} @SNAPPYLDFLAGS@ @GMPLDFLAGS@
=====================================
src/libmaus2/LibMausConfig.hpp.in
=====================================
@@ -113,6 +113,8 @@
@LIBMAUS2_HAVE_IO_NEW_CRAM_INTERFACE@
@LIBMAUS2_HAVE_IO_LIB_CRAM_ENCODER_GET_FD@
@LIBMAUS2_HAVE_IO_LIB_BAMBAM_H@
+ at LIBMAUS2_HAVE_IO_LIB_CRAM_SET_OPTION@
+ at LIBMAUS2_IO_LIB_HAVE_CRAM_OPT_PROFILE@
/**
* flag constants
=====================================
src/libmaus2/bambam/Scram.c
=====================================
@@ -781,31 +781,18 @@ void scram_cram_deallocate_encoder(void *context)
cram_deallocate_encoder(context);
}
-#if defined(LIBMAUS2_HAVE_IO_LIB_CRAM_ENCODER_GET_FD)
-int scram_cram_set_archive_mode(void * context)
+#if defined(LIBMAUS2_HAVE_IO_LIB_CRAM_ENCODER_GET_FD) && defined(LIBMAUS2_HAVE_IO_LIB_CRAM_SET_OPTION) && defined(LIBMAUS2_IO_LIB_HAVE_CRAM_OPT_PROFILE)
+int scram_cram_set_cram_profile(void * context, char const * profile)
{
cram_fd * fd = cram_encoder_get_fd(context);
int r = 0;
- fprintf(stderr,"[V] enabling cram archive mode\n");
-
- int const use_bz2 = 1;
- int const use_lzma = 1;
- int const s_opt = 100000;
- int const bases_per_slice = s_opt * 500;
-
- if ( use_bz2 )
- cram_set_option(fd, CRAM_OPT_USE_BZIP2, use_bz2);
- if ( use_lzma )
- cram_set_option(fd, CRAM_OPT_USE_LZMA, use_lzma);
-
- cram_set_option(fd, CRAM_OPT_SEQS_PER_SLICE, s_opt);
- cram_set_option(fd, CRAM_OPT_BASES_PER_SLICE, bases_per_slice);
+ r = cram_set_option(fd, CRAM_OPT_PROFILE, profile);
return r;
}
#else
-int scram_cram_set_archive_mode(void * context)
+int scram_cram_set_cram_profile(void * context, char const * profile)
{
int r = 1;
=====================================
src/libmaus2/bambam/parallel/BlockMergeControl.hpp
=====================================
@@ -331,7 +331,7 @@ namespace libmaus2
libmaus2::timing::RealTimeClock rtc;
- bool cram_archive_mode;
+ std::string cramprofile;
void enqueHeader()
{
@@ -371,8 +371,8 @@ namespace libmaus2
lme.finish();
throw lme;
}
- if ( cram_archive_mode )
- cram_set_archive_mode(cramsupport.context);
+ if ( cramprofile.size() )
+ cram_set_profile(cramsupport.context,cramprofile);
break;
}
case output_format_bam:
@@ -434,9 +434,9 @@ namespace libmaus2
return PcramEncoder->cram_allocate_encoder(userdata,header,headerlength,writefunc);
}
- int cram_set_archive_mode(void *userdata)
+ int cram_set_profile(void *userdata, std::string const & cramprofile)
{
- return PcramEncoder->cram_set_archive_mode(userdata);
+ return PcramEncoder->cram_set_profile(userdata,cramprofile);
}
void cram_deallocate_encoder(void * context)
@@ -539,24 +539,8 @@ namespace libmaus2
cramsupport(STP.getNumThreads()),
bamindex(rbamindex),
bamnumericalindex(rbamnumericalindex),
- cram_archive_mode(false)
+ cramprofile(crammode)
{
- if ( crammode.size() )
- {
- if ( crammode == "archive" )
- {
- cram_archive_mode = 1;
- }
- else
- {
- libmaus2::exception::LibMausException lme;
- lme.getStream() << "[E] unknown CRAM compression mode " << crammode << std::endl;
- lme.finish();
- throw lme;
- }
-
- }
-
for ( std::vector<libmaus2::bambam::parallel::GenericInputControlStreamInfo>::size_type i = 0; i < in.size(); ++i )
{
libmaus2::bambam::parallel::GenericInputSingleData::unique_ptr_type tptr(
=====================================
src/libmaus2/bambam/parallel/ScramCramEncoding.hpp
=====================================
@@ -45,8 +45,8 @@ namespace libmaus2
libmaus2::util::DynamicLibraryFunction<enque_func_t>::unique_ptr_type Penque_func;
typedef int (*dispatch_func_t)(void *);
libmaus2::util::DynamicLibraryFunction<dispatch_func_t>::unique_ptr_type Pdispatch_func;
- typedef int (*set_archive_mode_func_t)(void *);
- libmaus2::util::DynamicLibraryFunction<set_archive_mode_func_t>::unique_ptr_type Pset_archive_mode_func;
+ typedef int (*set_cram_profile_func_t)(void *, char const *);
+ libmaus2::util::DynamicLibraryFunction<set_cram_profile_func_t>::unique_ptr_type Pset_cram_profile_func;
#endif
ScramCramEncoding()
@@ -89,10 +89,11 @@ namespace libmaus2
new libmaus2::util::DynamicLibraryFunction<dispatch_func_t>(scramlib,"scram_cram_process_work_package")
);
Pdispatch_func = UNIQUE_PTR_MOVE(Tdispatch_func);
- libmaus2::util::DynamicLibraryFunction<set_archive_mode_func_t>::unique_ptr_type Tset_archive_mode_func(
- new libmaus2::util::DynamicLibraryFunction<dispatch_func_t>(scramlib,"scram_cram_set_archive_mode")
+
+ libmaus2::util::DynamicLibraryFunction<set_cram_profile_func_t>::unique_ptr_type Tset_cram_profile_func(
+ new libmaus2::util::DynamicLibraryFunction<set_cram_profile_func_t>(scramlib,"scram_cram_set_cram_profile")
);
- Pset_archive_mode_func = UNIQUE_PTR_MOVE(Tset_archive_mode_func);
+ Pset_cram_profile_func = UNIQUE_PTR_MOVE(Tset_cram_profile_func);
#endif
}
@@ -108,13 +109,13 @@ namespace libmaus2
#endif
}
- int cram_set_archive_mode(void * context)
+ int cram_set_profile(void * context, std::string const & profile)
{
#if defined(LIBMAUS2_HAVE_DL_FUNCS)
- return Pset_archive_mode_func->func(context);
+ return Pset_cram_profile_func->func(context,profile.c_str());
#else
libmaus2::exception::LibMausException lme;
- lme.getStream() << "ScramCramEncoding::cram_set_archive_mode: no support for new CRAM encoding interface" << std::endl;
+ lme.getStream() << "ScramCramEncoding::scram_cram_set_cram_profile: no support for new CRAM encoding interface" << std::endl;
lme.finish();
throw lme;
#endif
=====================================
src/libmaus2/geometry/RangeSet.hpp
=====================================
@@ -24,7 +24,6 @@
#include <deque>
#include <cassert>
#include <libmaus2/math/IntegerInterval.hpp>
-#include <libmaus2/util/SimpleQueue.hpp>
namespace libmaus2
{
@@ -496,93 +495,126 @@ namespace libmaus2
uint64_t search(
element_type const & elem,
- libmaus2::util::SimpleQueue<search_q_element> & todo
+ libmaus2::autoarray::AutoArray<search_q_element> & S
) const
{
+ uint64_t S_o = 0;
uint64_t o = 0;
- todo.clear();
-
// if element is not empty
if ( elem.getTo() > elem.getFrom() )
// push it
- todo.push_back(search_q_element(0,k,elem.getFrom(),elem.getTo(),0,0,0,0));
+ // todo.push_back(search_q_element(k /* level */,search_upair(elem.getFrom(),elem.getTo())));
+ // todo.push_back(search_q_element(0/*visit*/,k,elem.getFrom(),elem.getTo(),0,0,0,0));
+ S.push(S_o,search_q_element(0/*visit*/,k,elem.getFrom(),elem.getTo(),0,0,0,0));
+
+ #if 0
+ std::cerr << std::string(80,'-') << std::endl;
+ #endif
// while we have intervals to be checked
- while ( ! todo.empty() )
+ while ( S_o )
{
+ #if 0
+ for ( uint64_t i = 0; i < S.size(); ++i )
+ std::cerr << std::string(i,' ') << S[i].toString() << std::endl;
+ std::cerr << S.size() << "\t" << S.back().toString() << std::endl;
+ #endif
+
// get next interval
- search_q_element q = todo.front();
- todo.pop_front();
+ search_q_element q = S[--S_o];
- // level
- int const level = k-q.first;
+ if ( q.visit == 0 )
+ {
+ // level
+ int const level = k-q.level;
- // base bin for level
- uint64_t const basebin = (1ull << level)-1;
- // div/split size for level
- uint64_t const div = (1ull<<(q.first+1));
+ // base bin for level
+ uint64_t const basebin = (1ull << level)-1;
+ // div/split size for level
+ uint64_t const div = (1ull<<(q.level+1));
- uint64_t const mask = (q.first >= 0) ? (static_cast<uint64_t>(1ull)<<q.first) : static_cast<uint64_t>(0);
+ uint64_t const mask = (q.level >= 0) ? (static_cast<uint64_t>(1ull)<<q.level) : static_cast<uint64_t>(0);
- // interval
- uint64_t const low = q.second.first;
- uint64_t const high = q.second.second;
- // bin for low end
- uint64_t const bin = basebin + low/div;
+ // interval
+ uint64_t const low = q.from;
+ uint64_t const high = q.to;
+ // bin for low end
+ uint64_t const bin = basebin + low/div;
- // see if we have anything in bin matching low end
- if ( bins.find(bin) != bins.end() )
- {
- // get list of entries in bin
- std::vector<element_type> const & V = bins.find(bin)->second;
- // compute intersections between elements in bin and query
- for ( uint64_t i = 0; i < V.size(); ++i )
+ // see if we have anything in bin matching low end
+ if ( bins.find(bin) != bins.end() )
{
- libmaus2::math::IntegerInterval<uint64_t> I0(elem.getFrom(),elem.getTo()-1);
- libmaus2::math::IntegerInterval<uint64_t> I1(V[i].getFrom(),V[i].getTo()-1);
- if ( !I0.intersection(I1).isEmpty() )
- ++o;
+ // get list of entries in bin
+ std::vector<element_type> const & V = bins.find(bin)->second;
+ // compute intersections between elements in bin and query
+ for ( uint64_t i = 0; i < V.size(); ++i )
+ {
+ libmaus2::math::IntegerInterval<uint64_t> I0(elem.getFrom(),elem.getTo()-1);
+ libmaus2::math::IntegerInterval<uint64_t> I1(V[i].getFrom(),V[i].getTo()-1);
+ if ( !I0.intersection(I1).isEmpty() )
+ o += 1;
+ }
}
- }
- // std::cerr << level << "," << q.first << "," << q.second.first << "," << q.second.second << "," << basebin << "," << div << "," << bin << std::endl;
+ // std::cerr << level << "," << q.first << "," << q.second.first << "," << q.second.second << "," << basebin << "," << div << "," << bin << std::endl;
- // query interval should not be empty
- bool const hlok = high>low;
+ // query interval should not be empty
+ bool const hlok = high>low;
- if ( ! hlok )
- {
- libmaus2::exception::LibMausException lme;
- lme.getStream() << "[E] RangeSet<>::search(): interval [" << elem.getFrom() << "," << elem.getTo() << ") len " << len << " high=" << high << " low=" << low << std::endl;
- lme.finish();
- throw lme;
- }
+ if ( ! hlok )
+ {
+ libmaus2::exception::LibMausException lme;
+ lme.getStream() << "[E] RangeSet<>::search(): interval [" << elem.getFrom() << "," << elem.getTo() << ") len " << len << " high=" << high << " low=" << low << std::endl;
+ lme.finish();
+ throw lme;
+ }
+ // do we cross the bin split bound?
+ bool const cross = ((high-1)&mask) != (low & mask);
- // do we cross the bin split bound?
- bool const cross = ((high-1)&mask) != (low & mask);
+ if ( cross )
+ {
+ uint64_t const mid = ((low >> q.level)+1)<<q.level;
- if ( cross )
- {
- uint64_t const mid = ((low >> q.first)+1)<<q.first;
+ assert ( low < mid );
+ assert ( mid < high );
- // std::cerr << "low=" << low << ",mid=" << mid << ",high=" << high << std::endl;
+ S.push(S_o,search_q_element(1,q.level,q.from,q.to,low,mid,high,cross));
+ }
+ else
+ {
+ S.push(S_o,search_q_element(1,q.level,q.from,q.to,low,0,high,cross));
+ }
+ }
+ else if ( q.visit == 1 )
+ {
+ S.push(S_o,search_q_element(q.visit+1,q.level,q.from,q.to,q.low,q.mid,q.high,q.cross));
- bool const ok = (mid > low) && (high > mid);
- assert ( ok );
+ if ( !q.cross )
+ {
+ if ( q.level >= 0 )
+ S.push(S_o,search_q_element(0,q.level-1,q.low,q.high,0,0,0,0));
+ }
- todo.push_back(search_q_element(0,q.level-1,low,mid,0,0,0,0));
- todo.push_back(search_q_element(0,q.level-1,mid,high,0,0,0,0));
}
- else if ( q.first >= 0 )
+ else if ( q.visit == 2 )
{
- todo.push_back(search_q_element(0,q.level-1,low,high,0,0,0,0));
+ S.push(S_o,search_q_element(q.visit+1,q.level,q.from,q.to,q.low,q.mid,q.high,q.cross));
+
+ if ( q.cross )
+ {
+ S.push(S_o,search_q_element(0,q.level-1,q.low,q.mid,0,0,0,0));
+ }
+
}
- else
+ else if ( q.visit == 3 )
{
- // std::cerr << "base " << low << "," << high << " bin " << bin << std::endl;
+ if ( q.cross )
+ {
+ S.push(S_o,search_q_element(0,q.level-1,q.mid,q.high,0,0,0,0));
+ }
}
}
View it on GitLab: https://salsa.debian.org/med-team/libmaus2/-/commit/ab8fa5e26bb32ebb3fff940a9c56b37ce24ef78e
--
View it on GitLab: https://salsa.debian.org/med-team/libmaus2/-/commit/ab8fa5e26bb32ebb3fff940a9c56b37ce24ef78e
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20200327/70b65577/attachment-0001.html>
More information about the debian-med-commit
mailing list