[med-svn] [Git][med-team/libmmmulti][upstream] New upstream version 0.1+git20210125.d8fb6c9
Andreas Tille (@tille)
gitlab at salsa.debian.org
Wed Oct 28 08:30:27 GMT 2026
Andreas Tille pushed to branch upstream at Debian Med / libmmmulti
Commits:
6a937831 by Andreas Tille at 2026-10-28T09:28:17+01:00
New upstream version 0.1+git20210125.d8fb6c9
- - - - -
7 changed files:
- + .gitmodules
- CMakeLists.txt
- README.md
- src/main.cpp
- src/mmiitree.hpp
- src/mmmultimap.hpp
- src/mmmultiset.hpp
Changes:
=====================================
.gitmodules
=====================================
@@ -0,0 +1,25 @@
+[submodule "deps/mio"]
+ path = deps/mio
+ url = https://github.com/mandreyel/mio.git
+ branch = master
+[submodule "deps/args"]
+ path = deps/args
+ url = https://github.com/Taywee/args.git
+[submodule "deps/DYNAMIC"]
+ path = deps/DYNAMIC
+ url = https://github.com/vgteam/DYNAMIC.git
+[submodule "deps/sdsl-lite"]
+ path = deps/sdsl-lite
+ url = https://github.com/vgteam/sdsl-lite.git
+[submodule "deps/ips4o"]
+ path = deps/ips4o
+ url = https://github.com/ips4o/ips4o.git
+[submodule "deps/atomic_queue"]
+ path = deps/atomic_queue
+ url = https://github.com/max0x7ba/atomic_queue.git
+[submodule "deps/paryfor"]
+ path = deps/paryfor
+ url = https://github.com/ekg/paryfor.git
+[submodule "deps/hopscotch-map"]
+ path = deps/hopscotch-map
+ url = https://github.com/Tessil/hopscotch-map.git
=====================================
CMakeLists.txt
=====================================
@@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.1)
# Project's name
project(mmmulti)
# We build using c++14
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
@@ -13,6 +13,9 @@ find_package(Threads REQUIRED)
# Use all standard-compliant optimizations
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -mcx16 -g")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -mcx16 -g")
+# for debugging, unset and run with ASAN_OPTIONS=detect_leaks=1:symbolize=1 LSAN_OPTIONS=verbosity=1:log_threads=1
+#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O -mcx16 -g -fsanitize=address")
+#set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O -mcx16 -g -fsanitize=address")
# Set the output folder where your program will be created
set(CMAKE_BINARY_DIR ${CMAKE_SOURCE_DIR}/bin)
@@ -37,9 +40,8 @@ include(${CMAKE_ROOT}/Modules/ExternalProject.cmake)
# sdsl-lite (full build using its cmake config)
ExternalProject_Add(sdsl-lite
- GIT_REPOSITORY "https://github.com/simongog/sdsl-lite.git"
- GIT_TAG "ddb0fbbc33bb183baa616f17eb48e261ac2a3672"
- CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_DIR} # TODO ADD static build flag
+ SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/sdsl-lite"
+ CMAKE_ARGS "${CMAKE_ARGS};-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
UPDATE_COMMAND ""
INSTALL_COMMAND "")
ExternalProject_Get_property(sdsl-lite INSTALL_DIR)
@@ -50,45 +52,39 @@ set(sdsl-lite-divsufsort_LIB "${INSTALL_DIR}/src/sdsl-lite-build/external/libdiv
# DYNAMIC (full build using its cmake config)
ExternalProject_Add(dynamic
- GIT_REPOSITORY "https://github.com/vgteam/DYNAMIC.git"
- GIT_TAG "615d8be5276bcd4c5a3d8e31679b4f8e81b2eefc"
- # we don't actually install dynamic... it's header only
- #CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_DIR}
+ SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/DYNAMIC"
UPDATE_COMMAND ""
INSTALL_COMMAND ""
BUILD_COMMAND ""
CONFIGURE_COMMAND "")
-ExternalProject_Get_property(dynamic INSTALL_DIR)
-set(dynamic_INCLUDE "${INSTALL_DIR}/src/dynamic/include")
+ExternalProject_Get_property(dynamic SOURCE_DIR)
+set(dynamic_INCLUDE "${SOURCE_DIR}/include")
-# In-place Parallel Super Scalar Samplesort (IPS⁴o), header only
-ExternalProject_Add(ips4o
- GIT_REPOSITORY "https://github.com/SaschaWitt/ips4o.git"
- GIT_TAG "a34d7d40c0f1279510e35e0dc2c69637b3c5d0b6"
- #GIT_REPOSITORY "https://github.com/ekg/ips4o.git"
- #GIT_TAG "f7eccc5a26928b5c0772922f7b76072414ef1801"
+ExternalProject_Add(hopscotch_map
+ SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/hopscotch-map"
UPDATE_COMMAND ""
INSTALL_COMMAND ""
BUILD_COMMAND ""
CONFIGURE_COMMAND "")
-ExternalProject_Get_property(ips4o SOURCE_DIR)
-set(ips4o_INCLUDE "${SOURCE_DIR}")
+ExternalProject_Get_property(hopscotch_map SOURCE_DIR)
+set(hopscotch_map_INCLUDE "${SOURCE_DIR}/include")
+
+# In-place Parallel Super Scalar Samplesort (IPS⁴o), header only
+add_subdirectory(${CMAKE_SOURCE_DIR}/deps/ips4o)
# atomic queue
ExternalProject_Add(atomicqueue
- GIT_REPOSITORY "https://github.com/max0x7ba/atomic_queue.git"
- GIT_TAG "430f732da0889b090705ad00ce15d4463fe7b536"
+ SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/atomic_queue/include/atomic_queue"
UPDATE_COMMAND ""
INSTALL_COMMAND ""
BUILD_COMMAND ""
CONFIGURE_COMMAND "")
ExternalProject_Get_property(atomicqueue SOURCE_DIR)
-set(atomicqueue_INCLUDE "${SOURCE_DIR}")
+set(atomicqueue_INCLUDE "${SOURCE_DIR}/")
# paryfor parallel_for
ExternalProject_Add(paryfor
- GIT_REPOSITORY "https://github.com/ekg/paryfor"
- GIT_TAG "509b28a092f732a068e2908bb9e359a8562cd32f"
+ SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/paryfor"
UPDATE_COMMAND ""
INSTALL_COMMAND ""
BUILD_COMMAND ""
@@ -97,13 +93,9 @@ ExternalProject_Get_property(paryfor SOURCE_DIR)
set(paryfor_INCLUDE "${SOURCE_DIR}")
# taywee's C++ args library, header only
-ExternalProject_Add(tayweeargs
- GIT_REPOSITORY "https://github.com/Taywee/args.git"
- GIT_TAG "3de44ec671db452cc0c4ef86399b108939768abb"
- UPDATE_COMMAND ""
- INSTALL_COMMAND "")
-ExternalProject_Get_property(tayweeargs SOURCE_DIR)
-set(tayweeargs_INCLUDE "${SOURCE_DIR}")
+set(tayweeargs_INCLUDE deps/args)
+
+add_subdirectory(deps/mio)
#set(CMAKE_BUILD_TYPE Release)
@@ -113,16 +105,14 @@ add_executable(mmmulti
)
add_dependencies(mmmulti sdsl-lite)
add_dependencies(mmmulti dynamic)
-add_dependencies(mmmulti ips4o)
add_dependencies(mmmulti atomicqueue)
add_dependencies(mmmulti paryfor)
-add_dependencies(mmmulti tayweeargs)
+
target_include_directories(mmmulti PUBLIC
"${CMAKE_SOURCE_DIR}/src"
"${sdsl-lite_INCLUDE}"
"${sdsl-lite-divsufsort_INCLUDE}"
"${dynamic_INCLUDE}"
- "${ips4o_INCLUDE}"
"${atomicqueue_INCLUDE}"
"${paryfor_INCLUDE}"
"${tayweeargs_INCLUDE}")
@@ -134,6 +124,8 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
"${sdsl-lite-divsufsort_LIB}/libdivsufsort.a"
"${sdsl-lite-divsufsort_LIB}/libdivsufsort64.a"
Threads::Threads
+ ips4o
+ mio
)
elseif (TRUE)
target_link_libraries(mmmulti
@@ -142,6 +134,8 @@ elseif (TRUE)
"${sdsl-lite-divsufsort_LIB}/libdivsufsort64.a"
"-latomic"
Threads::Threads
+ ips4o
+ mio
)
endif()
@@ -150,7 +144,7 @@ if (APPLE)
elseif (TRUE)
# this was hard to track down
# https://stackoverflow.com/questions/35116327/when-g-static-link-pthread-cause-segmentation-fault-why
- set(CMAKE_EXE_LINKER_FLAGS "-static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive")
+ #set(CMAKE_EXE_LINKER_FLAGS "-static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive")
endif()
=====================================
README.md
=====================================
@@ -47,7 +47,7 @@ Calls to `append` are threadsafe once `open_writer` has been called.
To query the `mmmulti::map`, first index it, providing the maximum key to expect (remember, we're working on dense keys!):
```c++
-mmap.index(max_key);
+mmap.index(num_threads, max_key);
```
If we index without providing a maximum key, like this:
@@ -76,6 +76,7 @@ And we can iterate over the values or unique values of a given key with `for_val
## mmmulti::set memory-mapped multiset with iteration
This is similar to `mmmulti::map`, but useful where random access to values is not required, and where contiguity of keys is not possible.
+Unlike the `mmmulti::map`, arbitrary fixed-length data types are allowed, but they must have `operator<` defined to enable sorting.
It drops the index structures and padding, saving space, but preserves the same API.
The `mmmulti::set` only provides iteration across its key space.
As such, it's useful where we need to collect and count a set of entities.
=====================================
src/main.cpp
=====================================
@@ -3,7 +3,6 @@
#include <limits>
#include <cassert>
#include <chrono>
-#include "ips4o.hpp"
#include "args.hxx"
#include <utility>
#include "mmmultimap.hpp"
@@ -46,35 +45,39 @@ int main(int argc, char** argv) {
num_threads = args::get(threads);
}
+ std::random_device rd; //Will be used to obtain a seed for the random number engine
+ // these both stack smash! why??
+ //std::mt19937 gen(100); //Standard mersenne_twister_engine seeded with rd()
+ //std::mt19937_64 gen(100); //Standard mersenne_twister_engine seeded with rd();
+ std::default_random_engine gen(rd());
+
if (!args::get(test_file).empty() && !args::get(test_multiset) && !args::get(test_iitree)) {
if (args::get(test_complex)) {
- std::random_device rd; //Will be used to obtain a seed for the random number engine
- std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
uint64_t max_key = args::get(max_val);
std::uniform_int_distribution<uint64_t> dis(1, max_key);
//std::vector<uint64_t> x; x.reserve(1e8);
std::remove(args::get(test_file).c_str());
- mmmulti::map<uint64_t, std::pair<uint64_t, uint64_t>> mm(args::get(test_file));
+ auto mm = std::make_unique<mmmulti::map<uint64_t, std::pair<uint64_t, uint64_t>>>(args::get(test_file), std::make_pair(0,0));
uint64_t x_len = args::get(test_size);
- mm.open_writer();
+ mm->open_writer();
paryfor::parallel_for<uint64_t>(
0, x_len, num_threads, 1000,
[&](uint64_t n) {
- mm.append(dis(gen), std::make_pair(dis(gen), dis(gen)));
+ mm->append(dis(gen), std::make_pair(dis(gen), dis(gen)));
});
- mm.index(num_threads, max_key);
+ mm->index(num_threads, max_key);
uint64_t i = 0;
uint64_t key_count = 0;
uint64_t value_count = 0;
uint64_t unique_value_count = 0;
bool first = true;
uint64_t last = 0;
- mm.for_each_pair([&](const uint64_t& a, const std::pair<uint64_t, uint64_t>& b) {
+ mm->for_each_pair([&](const uint64_t& a, const std::pair<uint64_t, uint64_t>& b) {
if (first || a > last) {
++key_count;
last = a;
first = false;
- mm.for_unique_values_of(a, [&](const std::pair<uint64_t, uint64_t>& v) {
+ mm->for_unique_values_of(a, [&](const std::pair<uint64_t, uint64_t>& v) {
++unique_value_count;
});
}
@@ -86,7 +89,7 @@ int main(int argc, char** argv) {
uint64_t x = 0;
for (uint64_t i = 0; i < unique_value_test_count; ++i) {
uint64_t q = dis(gen);
- mm.for_unique_values_of(q, [&](const std::pair<uint64_t, uint64_t>& v) {
+ mm->for_unique_values_of(q, [&](const std::pair<uint64_t, uint64_t>& v) {
++x;
});
}
@@ -97,35 +100,35 @@ int main(int argc, char** argv) {
std::cerr << value_count << " values" << std::endl;
std::cerr << unique_value_count << " unique pairs" << std::endl;
if (unique_value_test_count) std::cerr << elapsed_seconds.count()/unique_value_test_count << "s per unique value call" << std::endl;
+ mm.reset();
- } else if (args::get(test_unpadded)) {
- std::random_device rd; //Will be used to obtain a seed for the random number engine
- std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
+ }
+ else if (args::get(test_unpadded)) {
uint64_t max_key = args::get(max_val);
std::uniform_int_distribution<uint64_t> dis(1, max_key);
//std::vector<uint64_t> x; x.reserve(1e8);
std::remove(args::get(test_file).c_str());
- mmmulti::map<uint64_t, uint64_t> mm(args::get(test_file));
+ auto mm = std::make_unique<mmmulti::map<uint64_t, uint64_t>>(args::get(test_file), 0);
uint64_t x_len = args::get(test_size);
- mm.open_writer();
+ mm->open_writer();
paryfor::parallel_for<uint64_t>(
0, x_len, num_threads, 1000,
[&](uint64_t n) {
- mm.append(dis(gen), dis(gen));
+ mm->append(dis(gen), dis(gen));
});
- mm.index(num_threads);
+ mm->index(num_threads);
uint64_t i = 0;
uint64_t key_count = 0;
uint64_t value_count = 0;
uint64_t unique_value_count = 0;
bool first = true;
uint64_t last = 0;
- mm.for_each_pair([&](const uint64_t& a, const uint64_t& b) {
+ mm->for_each_pair([&](const uint64_t& a, const uint64_t& b) {
if (first || a > last) {
++key_count;
last = a;
first = false;
- mm.for_unique_values_of(a, [&](const uint64_t& v) {
+ mm->for_unique_values_of(a, [&](const uint64_t& v) {
++unique_value_count;
});
}
@@ -134,35 +137,33 @@ int main(int argc, char** argv) {
std::cerr << key_count << " keys" << std::endl;
std::cerr << value_count << " values" << std::endl;
std::cerr << unique_value_count << " unique pairs (these can't be found with unpadded mmmultimaps)" << std::endl;
-
- } else {
- std::random_device rd; //Will be used to obtain a seed for the random number engine
- std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
+ }
+ else {
uint64_t max_key = args::get(max_val);
std::uniform_int_distribution<uint64_t> dis(1, max_key);
//std::vector<uint64_t> x; x.reserve(1e8);
std::remove(args::get(test_file).c_str());
- mmmulti::map<uint64_t, uint64_t> mm(args::get(test_file));
+ auto mm = std::make_unique<mmmulti::map<uint64_t, uint64_t>>(args::get(test_file), 0);
uint64_t x_len = args::get(test_size);
- mm.open_writer();
+ mm->open_writer();
paryfor::parallel_for<uint64_t>(
0, x_len, num_threads, 1000,
[&](uint64_t n) {
- mm.append(dis(gen), dis(gen));
+ mm->append(dis(gen), dis(gen));
});
- mm.index(num_threads, max_key);
+ mm->index(num_threads, max_key);
uint64_t i = 0;
uint64_t key_count = 0;
uint64_t value_count = 0;
uint64_t unique_value_count = 0;
bool first = true;
uint64_t last = 0;
- mm.for_each_pair([&](const uint64_t& a, const uint64_t& b) {
+ mm->for_each_pair([&](const uint64_t& a, const uint64_t& b) {
if (first || a > last) {
++key_count;
last = a;
first = false;
- mm.for_unique_values_of(a, [&](const uint64_t& v) {
+ mm->for_unique_values_of(a, [&](const uint64_t& v) {
++unique_value_count;
});
}
@@ -174,7 +175,7 @@ int main(int argc, char** argv) {
uint64_t x = 0;
for (uint64_t i = 0; i < unique_value_test_count; ++i) {
uint64_t q = dis(gen);
- mm.for_unique_values_of(q, [&](const uint64_t& v) {
+ mm->for_unique_values_of(q, [&](const uint64_t& v) {
++x;
});
}
@@ -187,35 +188,33 @@ int main(int argc, char** argv) {
if (unique_value_test_count) std::cerr << elapsed_seconds.count()/unique_value_test_count << "s per unique value call" << std::endl;
}
} else if (!args::get(test_file).empty() && args::get(test_multiset)) {
- std::random_device rd; //Will be used to obtain a seed for the random number engine
- std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
uint64_t max_value = args::get(max_val);
std::uniform_int_distribution<uint64_t> dis(1, max_value);
//std::vector<uint64_t> x; x.reserve(1e8);
std::remove(args::get(test_file).c_str());
- mmmulti::set<uint64_t> ms(args::get(test_file));
+ auto ms = std::make_unique<mmmulti::set<uint64_t>>(args::get(test_file));
uint64_t x_len = args::get(test_size);
- ms.open_writer();
+ ms->open_writer();
paryfor::parallel_for<uint64_t>(
0, x_len, num_threads, 1000,
[&](uint64_t n) {
//Use dis to transform the random unsigned int generated by gen into an int in the range
- ms.append(dis(gen));
+ ms->append(dis(gen));
});
- ms.index(num_threads);
+ ms->index(num_threads);
uint64_t i = 0;
uint64_t value_count = 0;
uint64_t unique_value_count = 0;
uint64_t sum1 = 0;
// exercise unique value search
- ms.for_each_value_count([&](const uint64_t& value, const uint64_t& count) {
+ ms->for_each_value_count([&](const uint64_t& value, const uint64_t& count) {
++unique_value_count;
value_count += count;
sum1 += count * value;
});
uint64_t second_count = 0;
uint64_t sum2 = 0;
- for (auto v = ms.begin(); v != ms.end(); ++v) {
+ for (auto v = ms->begin(); v != ms->end(); ++v) {
++second_count;
sum2 += *v;
}
@@ -225,24 +224,22 @@ int main(int argc, char** argv) {
} else if (!args::get(test_file).empty() && args::get(test_iitree)) {
//} else if (args::get(test_iitree)) {
std::remove(args::get(test_file).c_str());
- mmmulti::iitree<uint64_t, uint64_t> tree(args::get(test_file));
- std::random_device rd; //Will be used to obtain a seed for the random number engine
- std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
+ auto tree = std::make_unique<mmmulti::iitree<uint64_t, uint64_t>>(args::get(test_file));
uint64_t max_value = args::get(max_val);
std::uniform_int_distribution<uint64_t> dis(0, max_value);
uint64_t x_len = args::get(test_size);
- tree.open_writer();
+ tree->open_writer();
paryfor::parallel_for<uint64_t>(
0, x_len, num_threads, 1000,
[&](uint64_t n) {
uint64_t q = dis(gen);
uint64_t r = std::min(q + 100, max_value);
- tree.add(q, r, dis(gen));
+ tree->add(q, r, dis(gen));
});
- tree.index(num_threads);
+ tree->index(num_threads);
for (int n=0; n<max_value; ++n) {
if (n % 1000 == 0) std::cerr << n << "\r";
- tree.overlap(
+ tree->overlap(
n, n+1,
[&](const uint64_t& start,
const uint64_t& end,
@@ -252,12 +249,10 @@ int main(int argc, char** argv) {
}
});
}
- std::cerr << std::endl;
- /*
- tree.for_each_entry([&](const mmmulti::iitree<uint64_t, uint64_t>::Interval& ival) {
- std::cerr << ival.st << ".." << ival.en << " " << ival.data << std::endl;
- });
- */
+ std::cerr << max_value << std::endl;
+ //tree->for_each_entry([&](const mmmulti::iitree<uint64_t, uint64_t>::Interval& ival) {
+ // std::cerr << ival.st << ".." << ival.en << " " << ival.data << std::endl;
+ // });
//std::cerr << std::endl;
}
=====================================
src/mmiitree.hpp
=====================================
@@ -15,6 +15,7 @@
#include <unistd.h>
#include <functional>
#include <thread>
+#include <mio/mmap.hpp>
#include "ips4o.hpp"
#include "atomic_queue.h"
@@ -73,81 +74,27 @@ public:
private:
struct IntervalLess {
- bool operator()(const Interval &a, const Interval &b) const { return a.st < b.st; }
+ bool operator()(const Interval &a, const Interval &b) const { return a.st < b.st || (a.st == b.st && a.en < b.en); }
};
-
- // memory mapped buffer struct
- struct mmap_buffer_t {
- int fd;
- off_t size;
- void *data;
- };
-
- // utilities used by mmmultimap
- int open_mmap_buffer(const char* path, mmap_buffer_t* buffer) {
- buffer->data = nullptr;
- buffer->fd = open(path, O_RDWR);
- if (buffer->fd == -1) {
- goto error;
- }
- struct stat stats;
- if (-1 == fstat(buffer->fd, &stats)) {
- goto error;
- }
- if (!(buffer->data = mmap(nullptr,
- stats.st_size,
- PROT_READ | PROT_WRITE,
- MAP_SHARED,
- buffer->fd,
- 0
- ))) {
- goto error;
- }
- madvise(buffer, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
- buffer->size = stats.st_size;
- return 0;
-
- error:
- perror(path);
- if (buffer->data)
- munmap(buffer->data, stats.st_size);
- if (buffer->fd != -1)
- close(buffer->fd);
- buffer->data = 0;
- buffer->fd = 0;
- return -1;
- }
- void close_mmap_buffer(mmap_buffer_t* buffer) {
- if (buffer->data) {
- munmap(buffer->data, buffer->size);
- buffer->data = 0;
- buffer->size = 0;
- }
-
- if (buffer->fd) {
- close(buffer->fd);
- buffer->fd = 0;
- }
- }
-
- std::ofstream writer;
- char* reader = nullptr;
- int reader_fd = 0;
+ mio::mmap_source reader;
std::string filename;
std::string index_filename;
bool sorted = false;
// key information
uint64_t n_records = 0;
bool indexed = false;
- std::thread writer_thread; // = nullptr;
- atomic_queue::AtomicQueue2<Interval, 2 << 16>* interval_queue = nullptr;
+ std::thread writer_thread;
+ atomic_queue::AtomicQueue2<Interval, 2 << 16> interval_queue;
std::atomic<bool> work_todo;
-
- //std::vector<Interval> a;
uint64_t max_level = 0;
-
- uint64_t index_core(Interval* a, size_t a_size) {
+
+ uint64_t index_core(size_t a_size) {
+ std::error_code error;
+ mio::mmap_sink buf = mio::make_mmap_sink(
+ filename.c_str(), 0, mio::map_entire_file, error);
+ if (error) { assert(false); }
+ Interval* a = (Interval*)buf.begin();
size_t i, last_i; // last_i points to the rightmost node in the tree
S last; // last is the max value at node last_i
int64_t k;
@@ -176,27 +123,36 @@ public:
class iterator;
class const_iterator;
- // constructor
- iitree(void) { }
-
- iitree(const std::string& f) : filename(f) { }
+ iitree(const std::string& f) : filename(f) {
+ work_todo.store(false);
+ }
~iitree(void) {
close_writer();
close_reader();
}
+ // make sure we don't try anything silly
+ iitree(void) = delete;
+ iitree(const iitree& m) = delete;
+ iitree(iitree&& m) = delete;
+ iitree& operator=(iitree&& m) = delete;
+
void set_base_filename(const std::string& f) {
filename = f;
}
void writer_func(void) {
+ std::ofstream writer(filename.c_str(), std::ios::binary | std::ios::trunc);
+ if (writer.fail()) {
+ throw std::ios_base::failure(std::strerror(errno));
+ }
Interval ival;
- while (work_todo.load() || !interval_queue->was_empty()) {
- if (interval_queue->try_pop(ival)) {
+ while (work_todo.load() || !interval_queue.was_empty()) {
+ if (interval_queue.try_pop(ival)) {
do {
writer.write((char*)&ival, sizeof(Interval));
- } while (interval_queue->try_pop(ival));
+ } while (interval_queue.try_pop(ival));
} else {
std::this_thread::sleep_for(std::chrono::nanoseconds(1));
}
@@ -204,69 +160,40 @@ public:
writer.close();
}
- // close/open backing file
+ // open backing file and start writer_thread
void open_writer(void) {
- if (writer.is_open()) {
- writer.seekp(0, std::ios_base::end); // seek to the end for appending
- return;
- }
- assert(!filename.empty());
- // remove the file; we only call this when making an index, and it's done once
- writer.open(filename.c_str(), std::ios::binary | std::ios::trunc);
- if (writer.fail()) {
- throw std::ios_base::failure(std::strerror(errno));
+ if (!work_todo.load()) {
+ work_todo.store(true);
+ writer_thread = std::thread(&iitree::writer_func, this);
}
- interval_queue = new atomic_queue::AtomicQueue2<Interval, 2 << 16>;
- work_todo.store(true);
- writer_thread = std::thread(&iitree::writer_func, this);
}
void close_writer(void) {
if (work_todo.load()) {
work_todo.store(false);
- std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ while (!interval_queue.was_empty()) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
if (writer_thread.joinable()) {
writer_thread.join();
}
- delete interval_queue;
- interval_queue = nullptr;
}
}
void open_reader(void) {
- if (reader_fd) return; //open
- assert(!filename.empty());
- // open in binary mode as we are reading from this interface
- reader_fd = open(filename.c_str(), O_RDWR);
- if (reader_fd == -1) {
- assert(false);
- }
- struct stat stats;
- if (-1 == fstat(reader_fd, &stats)) {
- assert(false);
- }
- if (!(reader =
- (char*) mmap(NULL,
- stats.st_size,
- PROT_READ | PROT_WRITE,
- MAP_SHARED,
- reader_fd,
- 0))) {
- assert(false);
- }
- madvise((void*)reader, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
+ if (reader.is_mapped()) return;
+ std::error_code error;
+ reader = mio::make_mmap_source(
+ filename.c_str(), 0, mio::map_entire_file, error);
+ if (error) { assert(false); }
+ // set up for sequential access
+ madvise((void*)reader.begin(),
+ reader.mapped_length(),
+ POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
}
void close_reader(void) {
- if (reader) {
- size_t c = record_count();
- munmap(reader, c);
- reader = 0;
- }
- if (reader_fd) {
- close(reader_fd);
- reader_fd = 0;
- }
+ if (reader.is_mapped()) reader.unmap();
}
/// write the pair to end of backing file
@@ -278,42 +205,27 @@ public:
/// iterator to first value
iterator begin(void) {
- return iterator((Interval*)reader);
+ return iterator((Interval*)reader.begin());
}
/// iterator to one past end
iterator end(void) {
- return iterator(((Interval*)reader)+n_records);
+ return iterator((Interval*)reader.end());
}
/// const iterator to first value
const_iterator begin(void) const {
- return const_iterator((Interval*)reader);
+ return const_iterator((Interval*)reader.begin());
}
/// const iterator to one past end
const_iterator end(void) const {
- return const_iterator(((Interval*)reader)+n_records);
- }
-
- /// return the backing buffer
- char* get_buffer(void) const {
- return reader;
+ return const_iterator((Interval*)reader.end());
}
/// get the record count
size_t record_count(void) {
- int fd = open(filename.c_str(), O_RDWR);
- if (fd == -1) {
- assert(false);
- }
- struct stat stats;
- if (-1 == fstat(fd, &stats)) {
- assert(false);
- }
- assert(stats.st_size % sizeof(Interval) == 0); // must be even records
- size_t count = stats.st_size / sizeof(Interval);
- return count;
+ return reader.size() / sizeof(Interval);
}
std::ifstream::pos_type filesize(const char* filename) {
@@ -323,19 +235,17 @@ public:
/// sort the record in the backing file by key
void sort(int num_threads) {
- close_writer();
- close_reader();
if (sorted) return;
//std::cerr << "sorting!" << std::endl;
- mmap_buffer_t buffer;
- open_mmap_buffer(filename.c_str(), &buffer);
- uint64_t data_len = buffer.size/sizeof(Interval);
+ std::error_code error;
+ mio::mmap_sink buffer = mio::make_mmap_sink(
+ filename.c_str(), 0, mio::map_entire_file, error);
+ if (error) { assert(false); }
// sort in parallel (uses OpenMP if available, std::thread otherwise)
- ips4o::parallel::sort((Interval*)buffer.data,
- ((Interval*)buffer.data)+data_len,
+ ips4o::parallel::sort((Interval*)buffer.begin(),
+ (Interval*)buffer.end(),
IntervalLess(),
num_threads);
- close_mmap_buffer(&buffer);
sorted = true;
}
@@ -465,24 +375,25 @@ public:
};
Interval* get_array(void) const {
- return (Interval*)reader;
+ return (Interval*)reader.begin();
}
public:
/// write into our write buffer
/// open_writer() must be called first to set up our buffer and writer
void add(const S &s, const S &e, const T &d) {
- interval_queue->push(make_interval(s, e, d));
+ interval_queue.push(make_interval(s, e, d));
}
void index(int num_threads) {
+ close_writer();
sort(num_threads);
open_reader();
n_records = record_count();
indexed = true;
close_reader();
+ max_level = index_core(n_records);
open_reader();
- max_level = index_core(get_array(), n_records);
}
/// get overlaps, callback takes index, start, end, data
=====================================
src/mmmultimap.hpp
=====================================
@@ -16,6 +16,8 @@
#include <sys/types.h>
#include <unistd.h>
+#include <mio/mmap.hpp>
+
#include "sdsl/bit_vectors.hpp"
#include "ips4o.hpp"
#include "atomic_queue.h"
@@ -41,70 +43,16 @@ template <typename Key, typename Value> class map {
private:
- // memory mapped buffer struct
- struct mmap_buffer_t {
- int fd;
- off_t size;
- void *data;
- };
-
- // utilities used by mmmultimap
- int open_mmap_buffer(const char* path, mmap_buffer_t* buffer) {
- buffer->data = nullptr;
- buffer->fd = open(path, O_RDWR);
- if (buffer->fd == -1) {
- goto error;
- }
- struct stat stats;
- if (-1 == fstat(buffer->fd, &stats)) {
- goto error;
- }
- if (!(buffer->data = mmap(nullptr,
- stats.st_size,
- PROT_READ | PROT_WRITE,
- MAP_SHARED,
- buffer->fd,
- 0
- ))) {
- goto error;
- }
- madvise(buffer, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
- buffer->size = stats.st_size;
- return 0;
-
- error:
- perror(path);
- if (buffer->data)
- munmap(buffer->data, stats.st_size);
- if (buffer->fd != -1)
- close(buffer->fd);
- buffer->data = 0;
- buffer->fd = 0;
- return -1;
- }
-
- void close_mmap_buffer(mmap_buffer_t* buffer) {
- if (buffer->data) {
- munmap(buffer->data, buffer->size);
- buffer->data = 0;
- buffer->size = 0;
- }
-
- if (buffer->fd) {
- close(buffer->fd);
- buffer->fd = 0;
- }
- }
-
+ // an entry is a POD struct of Key, Value
typedef struct { Key key; Value value; } Entry;
+
+ // the comparator used to sort the backing array
struct EntryLess {
- bool operator()(const std::pair<Key,Value>& a, const std::pair<Key,Value>& b) const { return a < b; }
+ bool operator()(const std::pair<Key,Value>& a, const std::pair<Key,Value>& b) const { return a < b; }
};
- std::ofstream writer;
- std::vector<std::ofstream> writers;
- char* reader = nullptr;
- int reader_fd = 0;
+ // memory mapped buffer
+ mio::mmap_source reader;
std::string filename;
std::string index_filename;
bool sorted = false;
@@ -122,16 +70,16 @@ private:
sdsl::sd_vector<>::select_1_type key_cbv_select;
bool indexed = false;
uint32_t OUTPUT_VERSION = 1; // update as we change our format
+ // a single writer thread reads from an atomic queue and writes to our (unsorted) backing file
std::thread writer_thread;
- atomic_queue::AtomicQueue2<Entry, 2 << 16>* entry_queue = nullptr;
+ atomic_queue::AtomicQueue2<Entry, 2 << 16> entry_queue;
std::atomic<bool> work_todo;
- void init(void) {
+ void init(Value nullv) {
record_size = sizeof(Key) + sizeof(Value);
nullkey = 0;
- for (size_t i = 0; i < sizeof(Value); ++i) {
- ((uint8_t*)&nullvalue)[i] = 0;
- }
+ nullvalue = nullv;
+ work_todo.store(false);
}
public:
@@ -140,10 +88,16 @@ public:
class iterator;
class const_iterator;
+ // make sure we and our friend the compiler don't try anything silly
+ map(void) = delete;
+ map(const map& m) = delete;
+ map(map&& m) = delete;
+ map& operator=(map&& m) = delete;
+
// constructor
- map(void) { init(); }
+ map(Value nullv) { init(nullv); }
- map(const std::string& f) : filename(f) { init(); }
+ map(const std::string& f, Value nullv) : filename(f) { init(nullv); }
~map(void) {
close_writer();
@@ -196,12 +150,18 @@ public:
}
void writer_func(void) {
+ assert(!filename.empty());
+ // remove the file; we only call this when making an index, and it's done once
+ std::ofstream writer(filename.c_str(), std::ios::binary | std::ios::trunc);
+ if (writer.fail()) {
+ throw std::ios_base::failure(std::strerror(errno));
+ }
Entry entry;
- while (work_todo.load() || !entry_queue->was_empty()) {
- if (entry_queue->try_pop(entry)) {
+ while (work_todo.load() || !entry_queue.was_empty()) {
+ if (entry_queue.try_pop(entry)) {
do {
writer.write((char*)&entry, sizeof(Entry));
- } while (entry_queue->try_pop(entry));
+ } while (entry_queue.try_pop(entry));
} else {
std::this_thread::sleep_for(std::chrono::nanoseconds(1));
}
@@ -211,73 +171,45 @@ public:
// close/open backing file
void open_writer(void) {
- if (writer.is_open()) {
- writer.seekp(0, std::ios_base::end); // seek to the end for appending
- return;
+ if (!work_todo.load()) {
+ work_todo.store(true);
+ writer_thread = std::thread(&map::writer_func, this);
}
- assert(!filename.empty());
- // remove the file; we only call this when making an index, and it's done once
- writer.open(filename.c_str(), std::ios::binary | std::ios::trunc);
- if (writer.fail()) {
- throw std::ios_base::failure(std::strerror(errno));
- }
- entry_queue = new atomic_queue::AtomicQueue2<Entry, 2 << 16>;
- work_todo.store(true);
- writer_thread = std::thread(&map::writer_func, this);
}
void close_writer(void) {
if (work_todo.load()) {
work_todo.store(false);
- std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ while (!entry_queue.was_empty()) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
if (writer_thread.joinable()) {
writer_thread.join();
}
- delete entry_queue;
- entry_queue = nullptr;
}
}
+ // è mio
void open_reader(void) {
- if (reader_fd) return; //open
- assert(!filename.empty());
- // open in binary mode as we are reading from this interface
- reader_fd = open(filename.c_str(), O_RDWR);
- if (reader_fd == -1) {
- assert(false);
- }
- struct stat stats;
- if (-1 == fstat(reader_fd, &stats)) {
- assert(false);
- }
- if (!(reader =
- (char*) mmap(NULL,
- stats.st_size,
- PROT_READ | PROT_WRITE,
- MAP_SHARED,
- reader_fd,
- 0))) {
- assert(false);
- }
- madvise((void*)reader, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
+ if (reader.is_mapped()) return;
+ std::error_code error;
+ reader = mio::make_mmap_source(
+ filename.c_str(), 0, mio::map_entire_file, error);
+ if (error) { assert(false); }
+ // set up for sequential access
+ madvise((void*)reader.begin(),
+ reader.mapped_length(),
+ POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
}
void close_reader(void) {
- if (reader) {
- size_t c = record_count();
- munmap(reader, c);
- reader = 0;
- }
- if (reader_fd) {
- close(reader_fd);
- reader_fd = 0;
- }
+ if (reader.is_mapped()) reader.unmap();
}
/// write the pair to the backing file
/// open_writer() must be called first to set up our buffer and writer
void append(const Key& k, const Value& v) {
- entry_queue->push((Entry){k, v});
+ entry_queue.push((Entry){k, v});
}
/// return the number of records, which will only work after indexing
@@ -290,25 +222,9 @@ public:
return record_size;
}
- /// return the backing buffer
- char* get_buffer(void) const {
- return reader;
- }
-
/// get the record count
size_t record_count(void) {
- int fd = open(filename.c_str(), O_RDWR);
- if (fd == -1) {
- assert(false);
- }
- struct stat stats;
- if (-1 == fstat(fd, &stats)) {
- assert(false);
- }
- assert(stats.st_size % record_size == 0); // must be even records
- size_t count = stats.st_size / record_size;
- close(fd);
- return count;
+ return reader.size() / sizeof(Entry);
}
std::ifstream::pos_type filesize(const char* filename) {
@@ -320,15 +236,15 @@ public:
void sort(int num_threads) {
if (sorted) return;
//std::cerr << "sorting!" << std::endl;
- mmap_buffer_t buffer;
- open_mmap_buffer(filename.c_str(), &buffer);
- uint64_t data_len = buffer.size/record_size;
+ std::error_code error;
+ mio::mmap_sink buffer = mio::make_mmap_sink(
+ filename.c_str(), 0, mio::map_entire_file, error);
+ if (error) { assert(false); }
// sort in parallel (uses OpenMP if available, std::thread otherwise)
- ips4o::parallel::sort((std::pair<Key, Value>*)buffer.data,
- ((std::pair<Key, Value>*)buffer.data)+data_len,
+ ips4o::parallel::sort((std::pair<Key, Value>*)buffer.begin(),
+ (std::pair<Key, Value>*)buffer.end(),
EntryLess(),
num_threads);
- close_mmap_buffer(&buffer);
sorted = true;
}
=====================================
src/mmmultiset.hpp
=====================================
@@ -16,6 +16,8 @@
#include <sys/types.h>
#include <unistd.h>
+#include <mio/mmap.hpp>
+
#include "sdsl/bit_vectors.hpp"
#include "ips4o.hpp"
#include "atomic_queue.h"
@@ -32,70 +34,12 @@ of the multiset, such as to obtain their counts.
template <typename Value> class set {
-private:
-
- // memory mapped buffer struct
- struct mmap_buffer_t {
- int fd;
- off_t size;
- void *data;
- };
-
- // utilities used by mmmultimap
- int open_mmap_buffer(const char* path, mmap_buffer_t* buffer) {
- buffer->data = nullptr;
- buffer->fd = open(path, O_RDWR);
- if (buffer->fd == -1) {
- goto error;
- }
- struct stat stats;
- if (-1 == fstat(buffer->fd, &stats)) {
- goto error;
- }
- if (!(buffer->data = mmap(nullptr,
- stats.st_size,
- PROT_READ | PROT_WRITE,
- MAP_SHARED,
- buffer->fd,
- 0
- ))) {
- goto error;
- }
- madvise(buffer, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
- buffer->size = stats.st_size;
- return 0;
-
- error:
- perror(path);
- if (buffer->data)
- munmap(buffer->data, stats.st_size);
- if (buffer->fd != -1)
- close(buffer->fd);
- buffer->data = 0;
- buffer->fd = 0;
- return -1;
- }
-
- void close_mmap_buffer(mmap_buffer_t* buffer) {
- if (buffer->data) {
- munmap(buffer->data, buffer->size);
- buffer->data = 0;
- buffer->size = 0;
- }
-
- if (buffer->fd) {
- close(buffer->fd);
- buffer->fd = 0;
- }
- }
-
struct ValueLess {
bool operator()(const Value &a, const Value &b) const { return a < b; }
};
-
- std::ofstream writer;
- char* reader = nullptr;
- int reader_fd = 0;
+
+
+ mio::mmap_source reader;
std::string filename;
std::string index_filename;
bool sorted = false;
@@ -103,7 +47,7 @@ private:
uint64_t n_records = 0;
bool indexed = false;
std::thread writer_thread;
- atomic_queue::AtomicQueue2<Value, 2 << 16>* value_queue = nullptr;
+ atomic_queue::AtomicQueue2<Value, 2 << 16> value_queue;
std::atomic<bool> work_todo;
public:
@@ -111,11 +55,16 @@ public:
// forward declaration for iterator types
class iterator;
class const_iterator;
-
- // constructor
- set(void) { }
- set(const std::string& f) : filename(f) { }
+ // make sure we and our friend the compiler don't try anything silly
+ set(void) = delete;
+ set(const set& m) = delete;
+ set(set&& m) = delete;
+ set& operator=(set&& m) = delete;
+
+ set(const std::string& f) : filename(f) {
+ work_todo.store(false);
+ }
~set(void) {
close_writer();
@@ -127,12 +76,16 @@ public:
}
void writer_func(void) {
+ std::ofstream writer(filename.c_str(), std::ios::binary | std::ios::trunc);
+ if (writer.fail()) {
+ throw std::ios_base::failure(std::strerror(errno));
+ }
Value value;
- while (work_todo.load() || !value_queue->was_empty()) {
- if (value_queue->try_pop(value)) {
+ while (work_todo.load() || !value_queue.was_empty()) {
+ if (value_queue.try_pop(value)) {
do {
writer.write((char*)&value, sizeof(Value));
- } while (value_queue->try_pop(value));
+ } while (value_queue.try_pop(value));
} else {
std::this_thread::sleep_for(std::chrono::nanoseconds(1));
}
@@ -140,75 +93,47 @@ public:
writer.close();
}
- // close/open backing file
+ // open backing file and start writer_thread
void open_writer(void) {
- if (writer.is_open()) {
- writer.seekp(0, std::ios_base::end); // seek to the end for appending
- return;
- }
- assert(!filename.empty());
- // remove the file; we only call this when making an index, and it's done once
- writer.open(filename.c_str(), std::ios::binary | std::ios::trunc);
- if (writer.fail()) {
- throw std::ios_base::failure(std::strerror(errno));
+ if (!work_todo.load()) {
+ work_todo.store(true);
+ writer_thread = std::thread(&set::writer_func, this);
}
- value_queue = new atomic_queue::AtomicQueue2<Value, 2 << 16>;
- work_todo.store(true);
- writer_thread = std::thread(&set::writer_func, this);
}
void close_writer(void) {
if (work_todo.load()) {
work_todo.store(false);
- std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ while (!value_queue.was_empty()) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
if (writer_thread.joinable()) {
writer_thread.join();
}
- delete value_queue;
- value_queue = nullptr;
}
}
+ // è mio
void open_reader(void) {
- if (reader_fd) return; //open
- assert(!filename.empty());
- // open in binary mode as we are reading from this interface
- reader_fd = open(filename.c_str(), O_RDWR);
- if (reader_fd == -1) {
- assert(false);
- }
- struct stat stats;
- if (-1 == fstat(reader_fd, &stats)) {
- assert(false);
- }
- if (!(reader =
- (char*) mmap(NULL,
- stats.st_size,
- PROT_READ | PROT_WRITE,
- MAP_SHARED,
- reader_fd,
- 0))) {
- assert(false);
- }
- madvise((void*)reader, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
+ if (reader.is_mapped()) return;
+ std::error_code error;
+ reader = mio::make_mmap_source(
+ filename.c_str(), 0, mio::map_entire_file, error);
+ if (error) { assert(false); }
+ // set up for sequential access
+ madvise((void*)reader.begin(),
+ reader.mapped_length(),
+ POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
}
void close_reader(void) {
- if (reader) {
- size_t c = record_count();
- munmap(reader, c);
- reader = 0;
- }
- if (reader_fd) {
- close(reader_fd);
- reader_fd = 0;
- }
+ if (reader.is_mapped()) reader.unmap();
}
/// write the pair to end of backing file
/// open_writer() must be called first to set up our buffer and writer
void append(const Value& v) {
- value_queue->push(v);
+ value_queue.push(v);
}
/// return the number of records, which will only work after indexing
@@ -218,22 +143,22 @@ public:
/// iterator to first value
iterator begin(void) {
- return iterator((Value*)reader);
+ return iterator((Value*)reader.begin());
}
/// iterator to one past end
iterator end(void) {
- return iterator(((Value*)reader)+n_records);
+ return iterator((Value*)reader.end());
}
/// const iterator to first value
const_iterator begin(void) const {
- return const_iterator((Value*)reader);
+ return const_iterator((Value*)reader.begin());
}
/// const iterator to one past end
const_iterator end(void) const {
- return const_iterator(((Value*)reader)+n_records);
+ return const_iterator((Value*)reader.end());
}
/// return the size of each combined record
@@ -241,50 +166,31 @@ public:
return sizeof(Value);
}
- /// return the backing buffer
- char* get_buffer(void) const {
- return reader;
- }
-
/// get the record count
size_t record_count(void) {
- int fd = open(filename.c_str(), O_RDWR);
- if (fd == -1) {
- assert(false);
- }
- struct stat stats;
- if (-1 == fstat(fd, &stats)) {
- assert(false);
- }
- assert(stats.st_size % get_record_size() == 0); // must be even records
- size_t count = stats.st_size / sizeof(Value);
- close(fd);
- return count;
+ return reader.size() / sizeof(Value);
}
std::ifstream::pos_type filesize(const char* filename) {
std::ifstream in(filename, std::ifstream::ate | std::ifstream::binary);
return in.tellg();
}
-
- /// sort the record in the backing file by key
+
void sort(int num_threads) {
- close_writer();
- close_reader();
if (sorted) return;
//std::cerr << "sorting!" << std::endl;
- mmap_buffer_t buffer;
- open_mmap_buffer(filename.c_str(), &buffer);
- uint64_t data_len = buffer.size/sizeof(Value);
+ std::error_code error;
+ mio::mmap_sink buffer = mio::make_mmap_sink(
+ filename.c_str(), 0, mio::map_entire_file, error);
+ if (error) { assert(false); }
// sort in parallel (uses OpenMP if available, std::thread otherwise)
- ips4o::parallel::sort((Value*)buffer.data,
- ((Value*)buffer.data)+data_len,
+ ips4o::parallel::sort((Value*)buffer.begin(),
+ (Value*)buffer.end(),
ValueLess(),
num_threads);
- close_mmap_buffer(&buffer);
sorted = true;
}
-
+
Value read_value(size_t i) const {
Value v;
memcpy(&v, &reader[i*sizeof(Value)], sizeof(Value));
@@ -293,6 +199,7 @@ public:
// index
void index(int num_threads) {
+ close_writer();
sort(num_threads);
open_reader();
n_records = record_count();
@@ -301,6 +208,12 @@ public:
open_reader();
}
+
+
+ // todo we make a big change, pulling in all the stuff from mmmultimap
+
+
+
void for_each_value(const std::function<void(const Value&)>& lambda) const {
for (size_t i = 0; i < n_records; ++i) {
lambda(read_value(i));
View it on GitLab: https://salsa.debian.org/med-team/libmmmulti/-/commit/6a9378319acdae841f9824d1d13549649cd0da79
--
View it on GitLab: https://salsa.debian.org/med-team/libmmmulti/-/commit/6a9378319acdae841f9824d1d13549649cd0da79
You're receiving this email because of your account on salsa.debian.org. Manage all notifications: https://salsa.debian.org/-/profile/notifications | Help: https://salsa.debian.org/help
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20261028/f9176a0a/attachment-0001.htm>
More information about the debian-med-commit
mailing list