[med-svn] [Git][med-team/btllib][upstream] New upstream version 1.7.5+dfsg
Michael R. Crusoe (@crusoe)
gitlab at salsa.debian.org
Mon Jan 13 10:14:37 GMT 2025
Michael R. Crusoe pushed to branch upstream at Debian Med / btllib
Commits:
129b1b14 by Michael R. Crusoe at 2025-01-13T11:04:45+01:00
New upstream version 1.7.5+dfsg
- - - - -
17 changed files:
- .clang-tidy
- README.md
- include/btllib/counting_bloom_filter-inl.hpp
- include/btllib/counting_bloom_filter.hpp
- include/btllib/mi_bloom_filter-inl.hpp
- include/btllib/nthash_kmer.hpp
- include/btllib/util.hpp
- meson.build
- recipes/mi_bloom_filter.cpp
- src/btllib/status.cpp
- src/btllib/util.cpp
- tests/counting_bloom_filter.cpp
- tests/large.bam
- tests/python/test_calc_phred_avg.py
- tests/util.cpp
- wrappers/python/btllib.py
- wrappers/python/btllib_wrap.cxx
Changes:
=====================================
.clang-tidy
=====================================
@@ -54,6 +54,24 @@
-altera-id-dependent-backward-branch,
-clang-diagnostic-unused-command-line-argument,
-clang-diagnostic-unneeded-internal-declaration,
+ -readability-convert-member-functions-to-static,
+ -misc-unused-parameters,
+ -misc-include-cleaner,
+ -performance-avoid-endl,
+ -misc-use-anonymous-namespace,
+ -performance-enum-size,
+ -cppcoreguidelines-avoid-const-or-ref-data-members,
+ -cppcoreguidelines-noexcept-move-operations,
+ -clang-analyzer-core.UndefinedBinaryOperatorResult,
+ -misc-header-include-cycle,
+ -misc-const-correctness,
+ -hicpp-noexcept-move,
+ -cppcoreguidelines-avoid-do-while,
+ -readability-static-accessed-through-instance,
+ -performance-noexcept-move-constructor,
+ -hicpp-use-emplace,
+ -readability-avoid-nested-conditional-operator,
+ -modernize-use-emplace,
-readability-identifier-length',
WarningsAsErrors: '*',
CheckOptions: [
@@ -66,4 +84,4 @@
{ key: readability-identifier-naming.GlobalConstantPointerCase, value: UPPER_CASE },
{ key: readability-identifier-naming.ConstexprVariableCase, value: UPPER_CASE },
]
-}
\ No newline at end of file
+}
=====================================
README.md
=====================================
@@ -30,7 +30,8 @@ Using the library
---
- Run time dependencies:
* SAMtools for reading SAM, BAM, and CRAM files.
- * gzip, tar, pigz, bzip2, xz, lrzip, zip, and/or 7zip for compressing/decompressing files. Not all of these are necessary, only the ones whose compressions you'll be using.
+ * gzip, tar, pigz, bzip2, xz, lrzip, zip, and/or 7zip for compressing/decompressing files. Not all of these are necessary, only the ones whose compressions you'll be using.
+ * Note that lrzip is not available on the btllib conda osx-arm64 build
* wget for downloading sequences from a URL.
- Building C++ code (`$PREFIX` is the path where btllib is installed):
* Link your code with `$PREFIX/lib/libbtllib.a` (pass `-L $PREFIX/lib -l btllib` flags to the compiler).
@@ -69,7 +70,7 @@ For btllib developers
The following are all the available `ninja` commands which can be run within `build` directory:
- `ninja clang-format` formats the whitespace in code (requires clang-format 8+).
-- `ninja wrap` wraps C++ code for Python (requires SWIG 4.0+).
+- `ninja wrap` wraps C++ code for Python (requires SWIG ≥4.0 and <4.3).
- `ninja clang-tidy` runs clang-tidy on C++ code and makes sure it passes (requires clang-tidy 8+).
- `ninja` builds the tests and wrapper libraries / makes sure they compile.
- `ninja test` runs the tests.
@@ -85,6 +86,7 @@ Credits
- Components:
- [Hamid Mohamadi](https://github.com/mohamadi) and [Parham Kazemi](https://github.com/parham-k) for [ntHash](https://github.com/bcgsc/ntHash)
- [Justin Chu](https://github.com/JustinChu) for [MIBloomFilter](https://github.com/bcgsc/btl_bloomfilter)
+ - [Johnathan Wong](https://github.com/jwcodee) for [aaHash](https://github.com/bcgsc/btllib)
- Included dependencies:
- [Chase Geigle](https://github.com/skystrife) for [cpptoml](https://github.com/skystrife/cpptoml)
- Simon Gog, Timo Beller, Alistair Moffat, and Matthias Petri for [sdsl-lite](https://github.com/simongog/sdsl-lite)
=====================================
include/btllib/counting_bloom_filter-inl.hpp
=====================================
@@ -56,13 +56,12 @@ inline CountingBloomFilter<T>::CountingBloomFilter(size_t bytes,
*/
template<typename T>
inline void
-CountingBloomFilter<T>::insert(const uint64_t* hashes, T min_val)
+CountingBloomFilter<T>::set(const uint64_t* hashes, T min_val, T new_val)
{
// Update flag to track if increment is done on at least one counter
bool update_done = false;
- T new_val, tmp_min_val;
+ T tmp_min_val;
while (true) {
- new_val = min_val + 1;
for (size_t i = 0; i < hash_num; ++i) {
tmp_min_val = min_val;
update_done |= array[hashes[i] % array_size].compare_exchange_strong(
@@ -80,59 +79,25 @@ CountingBloomFilter<T>::insert(const uint64_t* hashes, T min_val)
template<typename T>
inline void
-CountingBloomFilter<T>::insert(const uint64_t* hashes)
+CountingBloomFilter<T>::insert(const uint64_t* hashes, T n)
{
- contains_insert(hashes);
+ contains_insert(hashes, n);
}
template<typename T>
inline void
CountingBloomFilter<T>::remove(const uint64_t* hashes)
{
- // Update flag to track if increment is done on at least one counter
- bool update_done = false;
T min_val = contains(hashes);
- T new_val, tmp_min_val;
- while (true) {
- new_val = min_val - 1;
- for (size_t i = 0; i < hash_num; ++i) {
- tmp_min_val = min_val;
- update_done |= array[hashes[i] % array_size].compare_exchange_strong(
- tmp_min_val, new_val);
- }
- if (update_done) {
- break;
- }
- min_val = contains(hashes);
- if (min_val == std::numeric_limits<T>::max()) {
- break;
- }
- }
+ set(hashes, min_val, min_val > 1 ? min_val - 1 : 0);
}
template<typename T>
void
CountingBloomFilter<T>::clear(const uint64_t* hashes)
{
- // Update flag to track if increment is done on at least one counter
- bool update_done = false;
T min_val = contains(hashes);
- T new_val, tmp_min_val;
- while (true) {
- new_val = 0;
- for (size_t i = 0; i < hash_num; ++i) {
- tmp_min_val = min_val;
- update_done |= array[hashes[i] % array_size].compare_exchange_strong(
- tmp_min_val, new_val);
- }
- if (update_done) {
- break;
- }
- min_val = contains(hashes);
- if (min_val == std::numeric_limits<T>::max()) {
- break;
- }
- }
+ set(hashes, min_val, 0);
}
template<typename T>
@@ -151,23 +116,23 @@ CountingBloomFilter<T>::contains(const uint64_t* hashes) const
template<typename T>
inline T
-CountingBloomFilter<T>::contains_insert(const uint64_t* hashes)
+CountingBloomFilter<T>::contains_insert(const uint64_t* hashes, T n)
{
const auto count = contains(hashes);
- if (count < std::numeric_limits<T>::max()) {
- insert(hashes, count);
+ if (count <= std::numeric_limits<T>::max() - n) {
+ set(hashes, count, count + n);
}
return count;
}
template<typename T>
inline T
-CountingBloomFilter<T>::insert_contains(const uint64_t* hashes)
+CountingBloomFilter<T>::insert_contains(const uint64_t* hashes, T n)
{
const auto count = contains(hashes);
- if (count < std::numeric_limits<T>::max()) {
- insert(hashes, count);
- return count + 1;
+ if (count <= std::numeric_limits<T>::max() + n) {
+ set(hashes, count, count + n);
+ return count + n;
}
return std::numeric_limits<T>::max();
}
@@ -179,7 +144,7 @@ CountingBloomFilter<T>::insert_thresh_contains(const uint64_t* hashes,
{
const auto count = contains(hashes);
if (count < threshold) {
- insert(hashes, count);
+ set(hashes, count, count + 1);
return count + 1;
}
return count;
@@ -192,7 +157,7 @@ CountingBloomFilter<T>::contains_insert_thresh(const uint64_t* hashes,
{
const auto count = contains(hashes);
if (count < threshold) {
- insert(hashes, count);
+ set(hashes, count, count + 1);
}
return count;
}
=====================================
include/btllib/counting_bloom_filter.hpp
=====================================
@@ -75,15 +75,20 @@ public:
*
* @param hashes Integer array of the element's hash values. Array size should
* equal the hash_num argument used when the Bloom filter was constructed.
+ * @param n Increment value
*/
- void insert(const uint64_t* hashes);
+ void insert(const uint64_t* hashes, T n = 1);
/**
* Insert an element.
*
* @param hashes Integer vector of the element's hash values.
+ * @param n Increment value
*/
- void insert(const std::vector<uint64_t>& hashes) { insert(hashes.data()); }
+ void insert(const std::vector<uint64_t>& hashes, T n = 1)
+ {
+ insert(hashes.data(), n);
+ }
/**
* Delete an element.
@@ -142,21 +147,23 @@ public:
*
* @param hashes Integer array of the element's hash values. Array size should
* equal the hash_num argument used when the Bloom filter was constructed.
+ * @param n Increment value
*
* @return The count of the queried element before insertion.
*/
- T contains_insert(const uint64_t* hashes);
+ T contains_insert(const uint64_t* hashes, T n = 1);
/**
* Get the count of an element and then increment the count.
*
* @param hashes Integer vector of the element's hash values.
+ * @param n Increment value
*
* @return The count of the queried element before insertion.
*/
- T contains_insert(const std::vector<uint64_t>& hashes)
+ T contains_insert(const std::vector<uint64_t>& hashes, T n = 1)
{
- return contains_insert(hashes.data());
+ return contains_insert(hashes.data(), n);
}
/**
@@ -165,21 +172,23 @@ public:
* @param hashes Integer array of the element's hash values. Array size
* should equal the hash_num argument used when the Bloom filter was
* constructed.
+ * @param n Increment value
*
* @return The count of the queried element after insertion.
*/
- T insert_contains(const uint64_t* hashes);
+ T insert_contains(const uint64_t* hashes, T n = 1);
/**
* Increment an element's count and then return the count.
*
* @param hashes Integer vector of the element's hash values.
+ * @param n Increment value
*
* @return The count of the queried element after insertion.
*/
- T insert_contains(const std::vector<uint64_t>& hashes)
+ T insert_contains(const std::vector<uint64_t>& hashes, T n = 1)
{
- return insert_contains(hashes.data());
+ return insert_contains(hashes.data(), n);
}
/**
@@ -280,7 +289,7 @@ public:
private:
CountingBloomFilter(const std::shared_ptr<BloomFilterInitializer>& bfi);
- void insert(const uint64_t* hashes, T min_val);
+ void set(const uint64_t* hashes, T min_val, T new_val);
friend class KmerCountingBloomFilter<T>;
@@ -346,17 +355,22 @@ public:
*
* @param hashes Integer array of the k-mer's hash values. Array size should
* equal the hash_num argument used when the Bloom filter was constructed.
+ * @param n Increment value
*/
- void insert(const uint64_t* hashes) { counting_bloom_filter.insert(hashes); }
+ void insert(const uint64_t* hashes, T n = 1)
+ {
+ counting_bloom_filter.insert(hashes, n);
+ }
/**
* Insert a k-mer into the filter.
*
* @param hashes Integer vector of the k-mer's hash values.
+ * @param n Increment value
*/
- void insert(const std::vector<uint64_t>& hashes)
+ void insert(const std::vector<uint64_t>& hashes, T n = 1)
{
- counting_bloom_filter.insert(hashes);
+ counting_bloom_filter.insert(hashes, n);
}
/**
@@ -499,24 +513,26 @@ public:
*
* @param hashes Integer array of the k-mers's hash values. Array size should
* equal the hash_num argument used when the Bloom filter was constructed.
+ * @param n Increment value
*
* @return The count of the queried k-mer before insertion.
*/
- T contains_insert(const uint64_t* hashes)
+ T contains_insert(const uint64_t* hashes, T n = 1)
{
- return counting_bloom_filter.contains_insert(hashes);
+ return counting_bloom_filter.contains_insert(hashes, n);
}
/**
* Get the count of a k-mer and then increment the count.
*
* @param hashes Integer vector of the k-mer's hash values.
+ * @param n Increment value
*
* @return The count of the queried k-mer before insertion.
*/
- T contains_insert(const std::vector<uint64_t>& hashes)
+ T contains_insert(const std::vector<uint64_t>& hashes, T n = 1)
{
- return counting_bloom_filter.contains_insert(hashes);
+ return counting_bloom_filter.contains_insert(hashes, n);
}
/**
@@ -547,24 +563,26 @@ public:
* @param hashes Integer array of the k-mer's hash values. Array size
* should equal the hash_num argument used when the Bloom filter was
* constructed.
+ * @param n Increment value
*
* @return The count of the queried k-mer after insertion.
*/
- T insert_contains(const uint64_t* hashes)
+ T insert_contains(const uint64_t* hashes, T n = 1)
{
- return counting_bloom_filter.insert_contains(hashes);
+ return counting_bloom_filter.insert_contains(hashes, n);
}
/**
* Increment a k-mer's count and then return the count.
*
* @param hashes Integer vector of the k-mer's hash values.
+ * @param n Increment value
*
* @return The count of the queried k-mer after insertion.
*/
- T insert_contains(const std::vector<uint64_t>& hashes)
+ T insert_contains(const std::vector<uint64_t>& hashes, T n = 1)
{
- return counting_bloom_filter.insert_contains(hashes);
+ return counting_bloom_filter.insert_contains(hashes, n);
}
/**
=====================================
include/btllib/mi_bloom_filter-inl.hpp
=====================================
@@ -190,7 +190,7 @@ MIBloomFilter<T>::insert_id(const uint64_t* hashes, const T& id)
{
assert(bv_insertion_completed && !id_insertion_completed);
- uint rand = std::rand(); // NOLINT
+ uint32_t rand = std::rand(); // NOLINT
for (unsigned i = 0; i < hash_num; ++i) {
uint64_t rank = get_rank_pos(hashes[i]);
uint16_t count = ++counts_array[rank];
=====================================
include/btllib/nthash_kmer.hpp
=====================================
@@ -493,7 +493,7 @@ private:
bool has_n = true;
while (pos <= seq_len - k + 1 && has_n) {
has_n = false;
- for (unsigned i = 0; i < k; i++) {
+ for (unsigned i = 0; i < k && pos <= seq_len - k + 1; i++) {
if (SEED_TAB[(unsigned char)seq[pos + k - i - 1]] == SEED_N) {
pos += k - i;
has_n = true;
=====================================
include/btllib/util.hpp
=====================================
@@ -111,6 +111,19 @@ get_basename(const std::string& path);
std::string
get_dirname(const std::string& path);
+/**
+ * Calculate the sum of the phred scores of a string.
+ *
+ * @param qual The quality string to calculate the sum from.
+ * @param start_pos The start position of the substring. Defaults to 0.
+ * @param len The length of the substring. Defaults to 0. If 0, the whole string
+ * is used.
+ *
+ * @return The sum of the phred scores of the substring.
+ */
+double
+sum_phred(const std::string& qual, size_t start_pos = 0, size_t len = 0);
+
/**
* Calculate the average phred score of a string,
* depending on the start position and length.
=====================================
meson.build
=====================================
@@ -1,5 +1,5 @@
project('btllib', 'cpp',
- version : '1.7.0',
+ version : '1.7.5',
license : 'GPL3',
default_options : [ 'cpp_std=c++17', 'warning_level=3', 'werror=true', 'b_coverage=true' ],
meson_version : '>= 0.60.0')
=====================================
recipes/mi_bloom_filter.cpp
=====================================
@@ -252,15 +252,15 @@ main(int argc, char* argv[])
btllib::SeqReader::Flag::SHORT_MODE,
DEFAULT_SEQ_READER_THREADS);
#pragma omp parallel default(none) shared(ids, \
- id_counter, \
- mi_bf, \
- mi_bf_stage, \
- hash_num, \
- kmer_size, \
- by_file, \
- spaced_seed_set, \
- spaced_seeds, \
- reader)
+ id_counter, \
+ mi_bf, \
+ mi_bf_stage, \
+ hash_num, \
+ kmer_size, \
+ by_file, \
+ spaced_seed_set, \
+ spaced_seeds, \
+ reader)
try {
for (const auto record : reader) {
#pragma omp critical
=====================================
src/btllib/status.cpp
=====================================
@@ -31,25 +31,25 @@ get_time()
void
log_info(const std::string& msg)
{
- std::cerr << ('[' + get_time() + "]" + PRINT_COLOR_INFO + "[INFO] " +
- PRINT_COLOR_END + msg + '\n')
- << std::flush;
+ std::string info_msg = "[" + get_time() + "]" + PRINT_COLOR_INFO + "[INFO] " +
+ PRINT_COLOR_END + msg;
+ std::cerr << info_msg << std::endl;
}
void
log_warning(const std::string& msg)
{
- std::cerr << ('[' + get_time() + "]" + PRINT_COLOR_WARNING + "[WARNING] " +
- PRINT_COLOR_END + msg + '\n')
- << std::flush;
+ std::string warning_msg = "[" + get_time() + "]" + PRINT_COLOR_WARNING +
+ "[WARNING] " + PRINT_COLOR_END + msg;
+ std::cerr << warning_msg << std::endl;
}
void
log_error(const std::string& msg)
{
- std::cerr << ('[' + get_time() + "]" + PRINT_COLOR_ERROR + "[ERROR] " +
- PRINT_COLOR_END + msg + '\n')
- << std::flush;
+ std::string error_msg = "[" + get_time() + "]" + PRINT_COLOR_ERROR +
+ "[ERROR] " + PRINT_COLOR_END + msg;
+ std::cerr << error_msg << std::endl;
}
void
@@ -111,4 +111,4 @@ check_file_accessibility(const std::string& filepath)
btllib::check_error(ret != 0, get_strerror() + ": " + filepath);
}
-} // namespace btllib
\ No newline at end of file
+} // namespace btllib
=====================================
src/btllib/util.cpp
=====================================
@@ -3,7 +3,9 @@
#include "btllib/status.hpp"
#include <algorithm>
+#include <cmath>
#include <condition_variable>
+#include <cstdlib>
#include <cstring>
#include <mutex>
#include <string>
@@ -158,6 +160,23 @@ get_basename(const std::string& path)
return path.substr(p + 1);
}
+double
+sum_phred(const std::string& qual, const size_t start_pos, size_t len)
+{
+ double phred_sum = 0;
+ static constexpr double PHRED_OFFSET = 33.0;
+ for (size_t i = start_pos; i < start_pos + len; ++i) {
+ // Convert ASCII character to Phred score
+ int phred_score = (int)(qual.at(i) - PHRED_OFFSET);
+
+ // Delog the Phred score: 10^(-Q/10)
+ double delog_phred = pow(10.0, -phred_score / 10.0);
+
+ phred_sum += delog_phred;
+ }
+ return phred_sum;
+}
+
double
calc_phred_avg(const std::string& qual, const size_t start_pos, size_t len)
{
@@ -170,14 +189,9 @@ calc_phred_avg(const std::string& qual, const size_t start_pos, size_t len)
std::exit(EXIT_FAILURE); // NOLINT(concurrency-mt-unsafe)
}
- size_t phred_sum = 0;
-
- for (size_t i = start_pos; i < start_pos + len; ++i) {
- phred_sum += (size_t)qual.at(i);
- }
+ double phred_sum = sum_phred(qual, start_pos, len);
- static constexpr double PHRED_OFFSET = 33.0;
- return ((double)phred_sum / (double)len) - PHRED_OFFSET;
+ return -10 * log10(phred_sum / len);
}
void
@@ -194,4 +208,4 @@ Barrier::wait()
}
}
-} // namespace btllib
\ No newline at end of file
+} // namespace btllib
=====================================
tests/counting_bloom_filter.cpp
=====================================
@@ -215,5 +215,19 @@ main()
TEST_ASSERT_EQ(cbf.contains(hashes), 0);
}
+ {
+ std::cerr << "Testing CBF element initialization" << std::endl;
+ std::vector<uint64_t> hashes = { 0x47c80ef7eab,
+ 0x8b4a469ef6,
+ 0x32e7ab5203 };
+ btllib::CountingBloomFilter8 cbf(64, hashes.size());
+ cbf.insert(hashes, 2);
+ TEST_ASSERT_EQ(cbf.contains(hashes), 2);
+ cbf.insert(hashes, 5);
+ TEST_ASSERT_EQ(cbf.contains(hashes), 7);
+ cbf.insert(hashes);
+ TEST_ASSERT_EQ(cbf.contains(hashes), 8);
+ }
+
return 0;
}
\ No newline at end of file
=====================================
tests/large.bam
=====================================
Binary files a/tests/large.bam and b/tests/large.bam differ
=====================================
tests/python/test_calc_phred_avg.py
=====================================
@@ -7,9 +7,9 @@ class CalcPhredAvgTests(unittest.TestCase):
def test_calc_phred_avg(self):
qual = "$$%%)*0)'%%&$$%&$&'''*)(((((()55561--.12356577-++**++,////.*))((()+))**010/..--+**++*+++)++++78883"
self.assertAlmostEqual(
- btllib.calc_phred_avg(qual, 0, 10), 6.4, places=3)
- self.assertAlmostEqual(btllib.calc_phred_avg(qual), 10.949, places=3)
+ btllib.calc_phred_avg(qual, 0, 10), 5.34264, places=3)
+ self.assertAlmostEqual(btllib.calc_phred_avg(qual), 8.54327, places=3)
self.assertAlmostEqual(
- btllib.calc_phred_avg(qual, 0, 4), 3.5, places=3)
+ btllib.calc_phred_avg(qual, 0, 4), 3.47128, places=3)
self.assertAlmostEqual(
- btllib.calc_phred_avg(qual, 5, 20), 6.15, places=3)
+ btllib.calc_phred_avg(qual, 5, 20), 5.48923, places=3)
=====================================
tests/util.cpp
=====================================
@@ -32,10 +32,10 @@ main()
double avg1 = btllib::calc_phred_avg(qual);
double avg2 = btllib::calc_phred_avg(qual, 0, 4);
double avg3 = btllib::calc_phred_avg(qual, 5, 20);
- TEST_ASSERT_LT(std::abs(avg - 6.4), 1e-4);
- TEST_ASSERT_LT(std::abs(avg1 - 10.949), 1e-4);
- TEST_ASSERT_LT(std::abs(avg2 - 3.5), 1e-4);
- TEST_ASSERT_LT(std::abs(avg3 - 6.15), 1e-4);
+ TEST_ASSERT_LT(std::abs(avg - 5.34264), 1e-4);
+ TEST_ASSERT_LT(std::abs(avg1 - 8.54327), 1e-4);
+ TEST_ASSERT_LT(std::abs(avg2 - 3.47128), 1e-4);
+ TEST_ASSERT_LT(std::abs(avg3 - 5.48923), 1e-4);
return 0;
-}
\ No newline at end of file
+}
=====================================
wrappers/python/btllib.py
=====================================
@@ -1,5 +1,5 @@
# This file was automatically generated by SWIG (https://www.swig.org).
-# Version 4.1.1
+# Version 4.2.1
#
# Do not make changes to this file unless you know what you are doing - modify
# the SWIG interface file instead.
=====================================
wrappers/python/btllib_wrap.cxx
=====================================
The diff for this file was not included because it is too large.
View it on GitLab: https://salsa.debian.org/med-team/btllib/-/commit/129b1b146b4f235230f9084c431e077600a85b21
--
View it on GitLab: https://salsa.debian.org/med-team/btllib/-/commit/129b1b146b4f235230f9084c431e077600a85b21
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20250113/703e2f14/attachment-0001.htm>
More information about the debian-med-commit
mailing list