[med-svn] [Git][med-team/libmmmulti][master] 4 commits: For latest vg we need the last upstream commit which is not tagged so adapt...

Andreas Tille (@tille) gitlab at salsa.debian.org
Wed Oct 28 08:30:05 GMT 2026



Andreas Tille pushed to branch master at Debian Med / libmmmulti


Commits:
74c96050 by Andreas Tille at 2026-10-28T09:28:02+01:00
For latest vg we need the last upstream commit which is not tagged so adapt d/watch to fetch latest commit

- - - - -
e317e99f by Andreas Tille at 2026-10-28T09:28:16+01:00
New upstream version

- - - - -
6a937831 by Andreas Tille at 2026-10-28T09:28:17+01:00
New upstream version 0.1+git20210125.d8fb6c9
- - - - -
e797bc55 by Andreas Tille at 2026-10-28T09:28:17+01:00
Update upstream source from tag 'upstream/0.1+git20210125.d8fb6c9'

Update to upstream version '0.1+git20210125.d8fb6c9'
with Debian dir 6c9d1817e9839f391dc952adef1bb460f83b17fc
- - - - -


9 changed files:

- + .gitmodules
- CMakeLists.txt
- README.md
- debian/changelog
- debian/watch
- src/main.cpp
- src/mmiitree.hpp
- src/mmmultimap.hpp
- src/mmmultiset.hpp


Changes:

=====================================
.gitmodules
=====================================
@@ -0,0 +1,25 @@
+[submodule "deps/mio"]
+	path = deps/mio
+	url = https://github.com/mandreyel/mio.git
+	branch = master
+[submodule "deps/args"]
+	path = deps/args
+	url = https://github.com/Taywee/args.git
+[submodule "deps/DYNAMIC"]
+	path = deps/DYNAMIC
+	url = https://github.com/vgteam/DYNAMIC.git
+[submodule "deps/sdsl-lite"]
+	path = deps/sdsl-lite
+	url = https://github.com/vgteam/sdsl-lite.git
+[submodule "deps/ips4o"]
+	path = deps/ips4o
+	url = https://github.com/ips4o/ips4o.git
+[submodule "deps/atomic_queue"]
+	path = deps/atomic_queue
+	url = https://github.com/max0x7ba/atomic_queue.git
+[submodule "deps/paryfor"]
+	path = deps/paryfor
+	url = https://github.com/ekg/paryfor.git
+[submodule "deps/hopscotch-map"]
+	path = deps/hopscotch-map
+	url = https://github.com/Tessil/hopscotch-map.git


=====================================
CMakeLists.txt
=====================================
@@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.1)
 # Project's name
 project(mmmulti)
 # We build using c++14
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
@@ -13,6 +13,9 @@ find_package(Threads REQUIRED)
 # Use all standard-compliant optimizations
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -mcx16 -g")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -mcx16 -g")
+# for debugging, unset and run with ASAN_OPTIONS=detect_leaks=1:symbolize=1 LSAN_OPTIONS=verbosity=1:log_threads=1
+#set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O -mcx16 -g -fsanitize=address")
+#set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O -mcx16 -g -fsanitize=address")
 
 # Set the output folder where your program will be created
 set(CMAKE_BINARY_DIR ${CMAKE_SOURCE_DIR}/bin)
@@ -37,9 +40,8 @@ include(${CMAKE_ROOT}/Modules/ExternalProject.cmake)
 
 # sdsl-lite (full build using its cmake config)
 ExternalProject_Add(sdsl-lite
-  GIT_REPOSITORY "https://github.com/simongog/sdsl-lite.git"
-  GIT_TAG "ddb0fbbc33bb183baa616f17eb48e261ac2a3672"
-  CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_DIR} # TODO ADD static build flag
+  SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/sdsl-lite"
+  CMAKE_ARGS "${CMAKE_ARGS};-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
   UPDATE_COMMAND ""
   INSTALL_COMMAND "")
 ExternalProject_Get_property(sdsl-lite INSTALL_DIR)
@@ -50,45 +52,39 @@ set(sdsl-lite-divsufsort_LIB "${INSTALL_DIR}/src/sdsl-lite-build/external/libdiv
 
 # DYNAMIC (full build using its cmake config)
 ExternalProject_Add(dynamic
-  GIT_REPOSITORY "https://github.com/vgteam/DYNAMIC.git"
-  GIT_TAG "615d8be5276bcd4c5a3d8e31679b4f8e81b2eefc"
-  # we don't actually install dynamic... it's header only
-  #CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${INSTALL_DIR}
+  SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/DYNAMIC"
   UPDATE_COMMAND ""
   INSTALL_COMMAND ""
   BUILD_COMMAND ""
   CONFIGURE_COMMAND "")
-ExternalProject_Get_property(dynamic INSTALL_DIR)
-set(dynamic_INCLUDE "${INSTALL_DIR}/src/dynamic/include")
+ExternalProject_Get_property(dynamic SOURCE_DIR)
+set(dynamic_INCLUDE "${SOURCE_DIR}/include")
 
-# In-place Parallel Super Scalar Samplesort (IPS⁴o), header only
-ExternalProject_Add(ips4o
-  GIT_REPOSITORY "https://github.com/SaschaWitt/ips4o.git"
-  GIT_TAG "a34d7d40c0f1279510e35e0dc2c69637b3c5d0b6"
-  #GIT_REPOSITORY "https://github.com/ekg/ips4o.git"
-  #GIT_TAG "f7eccc5a26928b5c0772922f7b76072414ef1801"
+ExternalProject_Add(hopscotch_map
+  SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/hopscotch-map"
   UPDATE_COMMAND ""
   INSTALL_COMMAND ""
   BUILD_COMMAND ""
   CONFIGURE_COMMAND "")
-ExternalProject_Get_property(ips4o SOURCE_DIR)
-set(ips4o_INCLUDE "${SOURCE_DIR}")
+ExternalProject_Get_property(hopscotch_map SOURCE_DIR)
+set(hopscotch_map_INCLUDE "${SOURCE_DIR}/include")
+
+# In-place Parallel Super Scalar Samplesort (IPS⁴o), header only
+add_subdirectory(${CMAKE_SOURCE_DIR}/deps/ips4o)
 
 # atomic queue
 ExternalProject_Add(atomicqueue
-  GIT_REPOSITORY "https://github.com/max0x7ba/atomic_queue.git"
-  GIT_TAG "430f732da0889b090705ad00ce15d4463fe7b536"
+  SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/atomic_queue/include/atomic_queue"
   UPDATE_COMMAND ""
   INSTALL_COMMAND ""
   BUILD_COMMAND ""
   CONFIGURE_COMMAND "")
 ExternalProject_Get_property(atomicqueue SOURCE_DIR)
-set(atomicqueue_INCLUDE "${SOURCE_DIR}")
+set(atomicqueue_INCLUDE "${SOURCE_DIR}/")
 
 # paryfor parallel_for
 ExternalProject_Add(paryfor
-  GIT_REPOSITORY "https://github.com/ekg/paryfor"
-  GIT_TAG "509b28a092f732a068e2908bb9e359a8562cd32f"
+  SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/paryfor"
   UPDATE_COMMAND ""
   INSTALL_COMMAND ""
   BUILD_COMMAND ""
@@ -97,13 +93,9 @@ ExternalProject_Get_property(paryfor SOURCE_DIR)
 set(paryfor_INCLUDE "${SOURCE_DIR}")
 
 # taywee's C++ args library, header only
-ExternalProject_Add(tayweeargs
-  GIT_REPOSITORY "https://github.com/Taywee/args.git"
-  GIT_TAG "3de44ec671db452cc0c4ef86399b108939768abb"
-  UPDATE_COMMAND ""
-  INSTALL_COMMAND "")
-ExternalProject_Get_property(tayweeargs SOURCE_DIR)
-set(tayweeargs_INCLUDE "${SOURCE_DIR}")
+set(tayweeargs_INCLUDE deps/args)
+
+add_subdirectory(deps/mio)
 
 #set(CMAKE_BUILD_TYPE Release)
 
@@ -113,16 +105,14 @@ add_executable(mmmulti
   )
 add_dependencies(mmmulti sdsl-lite)
 add_dependencies(mmmulti dynamic)
-add_dependencies(mmmulti ips4o)
 add_dependencies(mmmulti atomicqueue)
 add_dependencies(mmmulti paryfor)
-add_dependencies(mmmulti tayweeargs)
+
 target_include_directories(mmmulti PUBLIC
   "${CMAKE_SOURCE_DIR}/src"
   "${sdsl-lite_INCLUDE}"
   "${sdsl-lite-divsufsort_INCLUDE}"
   "${dynamic_INCLUDE}"
-  "${ips4o_INCLUDE}"
   "${atomicqueue_INCLUDE}"
   "${paryfor_INCLUDE}"
   "${tayweeargs_INCLUDE}")
@@ -134,6 +124,8 @@ if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
     "${sdsl-lite-divsufsort_LIB}/libdivsufsort.a"
     "${sdsl-lite-divsufsort_LIB}/libdivsufsort64.a"
     Threads::Threads
+    ips4o
+    mio
     )
 elseif (TRUE)
   target_link_libraries(mmmulti
@@ -142,6 +134,8 @@ elseif (TRUE)
     "${sdsl-lite-divsufsort_LIB}/libdivsufsort64.a"
     "-latomic"
     Threads::Threads
+    ips4o
+    mio
     )
 endif()
 
@@ -150,7 +144,7 @@ if (APPLE)
 elseif (TRUE)
   # this was hard to track down
   # https://stackoverflow.com/questions/35116327/when-g-static-link-pthread-cause-segmentation-fault-why
-  set(CMAKE_EXE_LINKER_FLAGS "-static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive")
+  #set(CMAKE_EXE_LINKER_FLAGS "-static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive")
 endif()
 
 


=====================================
README.md
=====================================
@@ -47,7 +47,7 @@ Calls to `append` are threadsafe once `open_writer` has been called.
 To query the `mmmulti::map`, first index it, providing the maximum key to expect (remember, we're working on dense keys!):
 
 ```c++
-mmap.index(max_key);
+mmap.index(num_threads, max_key);
 ```
 
 If we index without providing a maximum key, like this:
@@ -76,6 +76,7 @@ And we can iterate over the values or unique values of a given key with `for_val
 ## mmmulti::set memory-mapped multiset with iteration
 
 This is similar to `mmmulti::map`, but useful where random access to values is not required, and where contiguity of keys is not possible.
+Unlike the `mmmulti::map`, arbitrary fixed-length data types are allowed, but they must have `operator<` defined to enable sorting.
 It drops the index structures and padding, saving space, but preserves the same API.
 The `mmmulti::set` only provides iteration across its key space.
 As such, it's useful where we need to collect and count a set of entities.


=====================================
debian/changelog
=====================================
@@ -1,3 +1,11 @@
+libmmmulti (0.1+git20210125.d8fb6c9-1) UNRELEASED; urgency=medium
+
+  * For latest vg we need the last upstream commit which is not tagged
+    so adapt d/watch to fetch latest commit
+  * New upstream version
+
+ -- Andreas Tille <tille at debian.org>  Wed, 28 Oct 2026 09:28:16 +0100
+
 libmmmulti (0.1-5) unstable; urgency=medium
 
   * d/watch: Simplify


=====================================
debian/watch
=====================================
@@ -1,4 +1,6 @@
 Version: 5
 
-Template: Github
-Dist: https://github.com/ekg/mmmulti
+Source: https://github.com/ekg/mmmulti
+Mode: git
+Matching-Pattern: HEAD
+Git-Pretty: 0.1+git%cd.%h


=====================================
src/main.cpp
=====================================
@@ -3,7 +3,6 @@
 #include <limits>
 #include <cassert>
 #include <chrono>
-#include "ips4o.hpp"
 #include "args.hxx"
 #include <utility>
 #include "mmmultimap.hpp"
@@ -46,35 +45,39 @@ int main(int argc, char** argv) {
         num_threads = args::get(threads);
     }
 
+    std::random_device rd;  //Will be used to obtain a seed for the random number engine
+    // these both stack smash! why??
+    //std::mt19937 gen(100); //Standard mersenne_twister_engine seeded with rd()
+    //std::mt19937_64 gen(100); //Standard mersenne_twister_engine seeded with rd();
+    std::default_random_engine gen(rd());
+
     if (!args::get(test_file).empty() && !args::get(test_multiset) && !args::get(test_iitree)) {
         if (args::get(test_complex)) {
-            std::random_device rd;  //Will be used to obtain a seed for the random number engine
-            std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
             uint64_t max_key = args::get(max_val);
             std::uniform_int_distribution<uint64_t> dis(1, max_key);
             //std::vector<uint64_t> x; x.reserve(1e8);
             std::remove(args::get(test_file).c_str());
-            mmmulti::map<uint64_t, std::pair<uint64_t, uint64_t>> mm(args::get(test_file));
+            auto mm = std::make_unique<mmmulti::map<uint64_t, std::pair<uint64_t, uint64_t>>>(args::get(test_file), std::make_pair(0,0));
             uint64_t x_len = args::get(test_size);
-            mm.open_writer();
+            mm->open_writer();
             paryfor::parallel_for<uint64_t>(
                 0, x_len, num_threads, 1000,
                 [&](uint64_t n) {
-                    mm.append(dis(gen), std::make_pair(dis(gen), dis(gen)));
+                    mm->append(dis(gen), std::make_pair(dis(gen), dis(gen)));
                 });
-            mm.index(num_threads, max_key);
+            mm->index(num_threads, max_key);
             uint64_t i = 0;
             uint64_t key_count = 0;
             uint64_t value_count = 0;
             uint64_t unique_value_count = 0;
             bool first = true;
             uint64_t last = 0;
-            mm.for_each_pair([&](const uint64_t& a, const std::pair<uint64_t, uint64_t>& b) {
+            mm->for_each_pair([&](const uint64_t& a, const std::pair<uint64_t, uint64_t>& b) {
                     if (first || a > last) {
                         ++key_count;
                         last = a;
                         first = false;
-                        mm.for_unique_values_of(a, [&](const std::pair<uint64_t, uint64_t>& v) {
+                        mm->for_unique_values_of(a, [&](const std::pair<uint64_t, uint64_t>& v) {
                                 ++unique_value_count;
                             });
                     }
@@ -86,7 +89,7 @@ int main(int argc, char** argv) {
             uint64_t x = 0;
             for (uint64_t i = 0; i < unique_value_test_count; ++i) {
                 uint64_t q = dis(gen);
-                mm.for_unique_values_of(q, [&](const std::pair<uint64_t, uint64_t>& v) {
+                mm->for_unique_values_of(q, [&](const std::pair<uint64_t, uint64_t>& v) {
                         ++x;
                     });
             }
@@ -97,35 +100,35 @@ int main(int argc, char** argv) {
             std::cerr << value_count << " values" << std::endl;
             std::cerr << unique_value_count << " unique pairs" << std::endl;
             if (unique_value_test_count) std::cerr << elapsed_seconds.count()/unique_value_test_count << "s per unique value call" << std::endl;
+            mm.reset();
 
-        } else if (args::get(test_unpadded)) {
-            std::random_device rd;  //Will be used to obtain a seed for the random number engine
-            std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
+        }
+        else if (args::get(test_unpadded)) {
             uint64_t max_key = args::get(max_val);
             std::uniform_int_distribution<uint64_t> dis(1, max_key);
             //std::vector<uint64_t> x; x.reserve(1e8);
             std::remove(args::get(test_file).c_str());
-            mmmulti::map<uint64_t, uint64_t> mm(args::get(test_file));
+            auto mm = std::make_unique<mmmulti::map<uint64_t, uint64_t>>(args::get(test_file), 0);
             uint64_t x_len = args::get(test_size);
-            mm.open_writer();
+            mm->open_writer();
             paryfor::parallel_for<uint64_t>(
                 0, x_len, num_threads, 1000,
                 [&](uint64_t n) {
-                    mm.append(dis(gen), dis(gen));
+                    mm->append(dis(gen), dis(gen));
                 });
-            mm.index(num_threads);
+            mm->index(num_threads);
             uint64_t i = 0;
             uint64_t key_count = 0;
             uint64_t value_count = 0;
             uint64_t unique_value_count = 0;
             bool first = true;
             uint64_t last = 0;
-            mm.for_each_pair([&](const uint64_t& a, const uint64_t& b) {
+            mm->for_each_pair([&](const uint64_t& a, const uint64_t& b) {
                     if (first || a > last) {
                         ++key_count;
                         last = a;
                         first = false;
-                        mm.for_unique_values_of(a, [&](const uint64_t& v) {
+                        mm->for_unique_values_of(a, [&](const uint64_t& v) {
                                 ++unique_value_count;
                             });
                     }
@@ -134,35 +137,33 @@ int main(int argc, char** argv) {
             std::cerr << key_count << " keys" << std::endl;
             std::cerr << value_count << " values" << std::endl;
             std::cerr << unique_value_count << " unique pairs (these can't be found with unpadded mmmultimaps)" << std::endl;
-
-        } else {
-            std::random_device rd;  //Will be used to obtain a seed for the random number engine
-            std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
+        }
+        else {
             uint64_t max_key = args::get(max_val);
             std::uniform_int_distribution<uint64_t> dis(1, max_key);
             //std::vector<uint64_t> x; x.reserve(1e8);
             std::remove(args::get(test_file).c_str());
-            mmmulti::map<uint64_t, uint64_t> mm(args::get(test_file));
+            auto mm = std::make_unique<mmmulti::map<uint64_t, uint64_t>>(args::get(test_file), 0);
             uint64_t x_len = args::get(test_size);
-            mm.open_writer();
+            mm->open_writer();
             paryfor::parallel_for<uint64_t>(
                 0, x_len, num_threads, 1000,
                 [&](uint64_t n) {
-                    mm.append(dis(gen), dis(gen));
+                    mm->append(dis(gen), dis(gen));
                 });
-            mm.index(num_threads, max_key);
+            mm->index(num_threads, max_key);
             uint64_t i = 0;
             uint64_t key_count = 0;
             uint64_t value_count = 0;
             uint64_t unique_value_count = 0;
             bool first = true;
             uint64_t last = 0;
-            mm.for_each_pair([&](const uint64_t& a, const uint64_t& b) {
+            mm->for_each_pair([&](const uint64_t& a, const uint64_t& b) {
                     if (first || a > last) {
                         ++key_count;
                         last = a;
                         first = false;
-                        mm.for_unique_values_of(a, [&](const uint64_t& v) {
+                        mm->for_unique_values_of(a, [&](const uint64_t& v) {
                                 ++unique_value_count;
                             });
                     }
@@ -174,7 +175,7 @@ int main(int argc, char** argv) {
             uint64_t x = 0;
             for (uint64_t i = 0; i < unique_value_test_count; ++i) {
                 uint64_t q = dis(gen);
-                mm.for_unique_values_of(q, [&](const uint64_t& v) {
+                mm->for_unique_values_of(q, [&](const uint64_t& v) {
                         ++x;
                     });
             }
@@ -187,35 +188,33 @@ int main(int argc, char** argv) {
             if (unique_value_test_count) std::cerr << elapsed_seconds.count()/unique_value_test_count << "s per unique value call" << std::endl;
         }
     } else if (!args::get(test_file).empty() && args::get(test_multiset)) {
-        std::random_device rd;  //Will be used to obtain a seed for the random number engine
-        std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
         uint64_t max_value = args::get(max_val);
         std::uniform_int_distribution<uint64_t> dis(1, max_value);
         //std::vector<uint64_t> x; x.reserve(1e8);
         std::remove(args::get(test_file).c_str());
-        mmmulti::set<uint64_t> ms(args::get(test_file));
+        auto ms = std::make_unique<mmmulti::set<uint64_t>>(args::get(test_file));
         uint64_t x_len = args::get(test_size);
-        ms.open_writer();
+        ms->open_writer();
         paryfor::parallel_for<uint64_t>(
             0, x_len, num_threads, 1000,
             [&](uint64_t n) {
                 //Use dis to transform the random unsigned int generated by gen into an int in the range
-                ms.append(dis(gen));
+                ms->append(dis(gen));
             });
-        ms.index(num_threads);
+        ms->index(num_threads);
         uint64_t i = 0;
         uint64_t value_count = 0;
         uint64_t unique_value_count = 0;
         uint64_t sum1 = 0;
         // exercise unique value search
-        ms.for_each_value_count([&](const uint64_t& value, const uint64_t& count) {
+        ms->for_each_value_count([&](const uint64_t& value, const uint64_t& count) {
                 ++unique_value_count;
                 value_count += count;
                 sum1 += count * value;
             });
         uint64_t second_count = 0;
         uint64_t sum2 = 0;
-        for (auto v = ms.begin(); v != ms.end(); ++v) {
+        for (auto v = ms->begin(); v != ms->end(); ++v) {
             ++second_count;
             sum2 += *v;
         }
@@ -225,24 +224,22 @@ int main(int argc, char** argv) {
     } else if (!args::get(test_file).empty() && args::get(test_iitree)) {
         //} else if (args::get(test_iitree)) {
         std::remove(args::get(test_file).c_str());
-        mmmulti::iitree<uint64_t, uint64_t> tree(args::get(test_file));
-        std::random_device rd;  //Will be used to obtain a seed for the random number engine
-        std::mt19937 gen(rd()); //Standard mersenne_twister_engine seeded with rd()
+        auto tree = std::make_unique<mmmulti::iitree<uint64_t, uint64_t>>(args::get(test_file));
         uint64_t max_value = args::get(max_val);
         std::uniform_int_distribution<uint64_t> dis(0, max_value);
         uint64_t x_len = args::get(test_size);
-        tree.open_writer();
+        tree->open_writer();
         paryfor::parallel_for<uint64_t>(
             0, x_len, num_threads, 1000,
             [&](uint64_t n) {
                 uint64_t q = dis(gen);
                 uint64_t r = std::min(q + 100, max_value);
-                tree.add(q, r, dis(gen));
+                tree->add(q, r, dis(gen));
             });
-        tree.index(num_threads);
+        tree->index(num_threads);
         for (int n=0; n<max_value; ++n) {
             if (n % 1000 == 0) std::cerr << n << "\r";
-            tree.overlap(
+            tree->overlap(
                 n, n+1,
                 [&](const uint64_t& start,
                     const uint64_t& end,
@@ -252,12 +249,10 @@ int main(int argc, char** argv) {
                     }
                 });
         }
-        std::cerr << std::endl;
-        /*
-        tree.for_each_entry([&](const mmmulti::iitree<uint64_t, uint64_t>::Interval& ival) {
-                std::cerr << ival.st << ".." << ival.en << " " << ival.data << std::endl;
-            });
-        */
+        std::cerr << max_value << std::endl;
+        //tree->for_each_entry([&](const mmmulti::iitree<uint64_t, uint64_t>::Interval& ival) {
+        //        std::cerr << ival.st << ".." << ival.en << " " << ival.data << std::endl;
+        //    });
         //std::cerr << std::endl;
     }
 


=====================================
src/mmiitree.hpp
=====================================
@@ -15,6 +15,7 @@
 #include <unistd.h>
 #include <functional>
 #include <thread>
+#include <mio/mmap.hpp>
 #include "ips4o.hpp"
 #include "atomic_queue.h"
 
@@ -73,81 +74,27 @@ public:
 
 private:
 	struct IntervalLess {
-		bool operator()(const Interval &a, const Interval &b) const { return a.st < b.st; }
+		bool operator()(const Interval &a, const Interval &b) const { return a.st < b.st || (a.st == b.st && a.en < b.en); }
 	};
-    
-    // memory mapped buffer struct
-    struct mmap_buffer_t {
-        int fd;
-        off_t size;
-        void *data;
-    };
-
-    // utilities used by mmmultimap
-    int open_mmap_buffer(const char* path, mmap_buffer_t* buffer) {
-        buffer->data = nullptr;
-        buffer->fd = open(path, O_RDWR);
-        if (buffer->fd == -1) {
-            goto error;
-        }
-        struct stat stats;
-        if (-1 == fstat(buffer->fd, &stats)) {
-            goto error;
-        }
-        if (!(buffer->data = mmap(nullptr,
-                                  stats.st_size,
-                                  PROT_READ | PROT_WRITE,
-                                  MAP_SHARED,
-                                  buffer->fd,
-                                  0
-                  ))) {
-            goto error;
-        }
-        madvise(buffer, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
-        buffer->size = stats.st_size;
-        return 0;
-
-    error:
-        perror(path);
-        if (buffer->data)
-            munmap(buffer->data, stats.st_size);
-        if (buffer->fd != -1)
-            close(buffer->fd);
-        buffer->data = 0;
-        buffer->fd = 0;
-        return -1;
-    }
 
-    void close_mmap_buffer(mmap_buffer_t* buffer) {
-        if (buffer->data) {
-            munmap(buffer->data, buffer->size);
-            buffer->data = 0;
-            buffer->size = 0;
-        }
-
-        if (buffer->fd) {
-            close(buffer->fd);
-            buffer->fd = 0;
-        }
-    }
-
-    std::ofstream writer;
-    char* reader = nullptr;
-    int reader_fd = 0;
+    mio::mmap_source reader;
     std::string filename;
     std::string index_filename;
     bool sorted = false;
     // key information
     uint64_t n_records = 0;
     bool indexed = false;
-    std::thread writer_thread; // = nullptr;
-    atomic_queue::AtomicQueue2<Interval, 2 << 16>* interval_queue = nullptr;
+    std::thread writer_thread;
+    atomic_queue::AtomicQueue2<Interval, 2 << 16> interval_queue;
     std::atomic<bool> work_todo;
-
-    //std::vector<Interval> a;
 	uint64_t max_level = 0;
-    
-	uint64_t index_core(Interval* a, size_t a_size) {
+
+	uint64_t index_core(size_t a_size) {
+        std::error_code error;
+        mio::mmap_sink buf = mio::make_mmap_sink(
+            filename.c_str(), 0, mio::map_entire_file, error);
+        if (error) { assert(false); }
+        Interval* a = (Interval*)buf.begin();
 		size_t i, last_i; // last_i points to the rightmost node in the tree
 		S last; // last is the max value at node last_i
 		int64_t k;
@@ -176,27 +123,36 @@ public:
     class iterator;
     class const_iterator;
     
-    // constructor
-    iitree(void) { }
-
-    iitree(const std::string& f) : filename(f) { }
+    iitree(const std::string& f) : filename(f) {
+        work_todo.store(false);
+    }
 
     ~iitree(void) {
         close_writer();
         close_reader();
     }
 
+    // make sure we don't try anything silly
+    iitree(void) = delete;
+    iitree(const iitree& m) = delete;
+    iitree(iitree&& m) = delete;
+    iitree& operator=(iitree&& m) = delete;
+    
     void set_base_filename(const std::string& f) {
         filename = f;
     }
 
     void writer_func(void) {
+        std::ofstream writer(filename.c_str(), std::ios::binary | std::ios::trunc);
+        if (writer.fail()) {
+            throw std::ios_base::failure(std::strerror(errno));
+        }
         Interval ival;
-        while (work_todo.load() || !interval_queue->was_empty()) {
-            if (interval_queue->try_pop(ival)) {
+        while (work_todo.load() || !interval_queue.was_empty()) {
+            if (interval_queue.try_pop(ival)) {
                 do {
                     writer.write((char*)&ival, sizeof(Interval));
-                } while (interval_queue->try_pop(ival));
+                } while (interval_queue.try_pop(ival));
             } else {
                 std::this_thread::sleep_for(std::chrono::nanoseconds(1));
             }
@@ -204,69 +160,40 @@ public:
         writer.close();
     }
 
-    // close/open backing file
+    // open backing file and start writer_thread
     void open_writer(void) {
-        if (writer.is_open()) {
-            writer.seekp(0, std::ios_base::end); // seek to the end for appending
-            return;
-        }
-        assert(!filename.empty());
-        // remove the file; we only call this when making an index, and it's done once
-        writer.open(filename.c_str(), std::ios::binary | std::ios::trunc);
-        if (writer.fail()) {
-            throw std::ios_base::failure(std::strerror(errno));
+        if (!work_todo.load()) {
+            work_todo.store(true);
+            writer_thread = std::thread(&iitree::writer_func, this);
         }
-        interval_queue = new atomic_queue::AtomicQueue2<Interval, 2 << 16>;
-        work_todo.store(true);
-        writer_thread = std::thread(&iitree::writer_func, this);
     }
 
     void close_writer(void) {
         if (work_todo.load()) {
             work_todo.store(false);
-            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            while (!interval_queue.was_empty()) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            }
             if (writer_thread.joinable()) {
                 writer_thread.join();
             }
-            delete interval_queue;
-            interval_queue = nullptr;
         }
     }
 
     void open_reader(void) {
-        if (reader_fd) return; //open
-        assert(!filename.empty());
-        // open in binary mode as we are reading from this interface
-        reader_fd = open(filename.c_str(), O_RDWR);
-        if (reader_fd == -1) {
-            assert(false);
-        }
-        struct stat stats;
-        if (-1 == fstat(reader_fd, &stats)) {
-            assert(false);
-        }
-        if (!(reader =
-              (char*) mmap(NULL,
-                            stats.st_size,
-                            PROT_READ | PROT_WRITE,
-                            MAP_SHARED,
-                            reader_fd,
-                            0))) {
-            assert(false);
-        }
-        madvise((void*)reader, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
+        if (reader.is_mapped()) return;
+        std::error_code error;
+        reader = mio::make_mmap_source(
+            filename.c_str(), 0, mio::map_entire_file, error);
+        if (error) { assert(false); }
+        // set up for sequential access
+        madvise((void*)reader.begin(),
+                reader.mapped_length(),
+                POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
     }
 
     void close_reader(void) {
-        if (reader) {
-            size_t c = record_count();
-            munmap(reader, c);
-            reader = 0;
-        }
-        if (reader_fd) {
-            close(reader_fd);
-            reader_fd = 0;
-        }
+        if (reader.is_mapped()) reader.unmap();
     }
 
     /// write the pair to end of backing file
@@ -278,42 +205,27 @@ public:
     
     /// iterator to first value
     iterator begin(void) {
-        return iterator((Interval*)reader);
+        return iterator((Interval*)reader.begin());
     }
     
     /// iterator to one past end
     iterator end(void) {
-        return iterator(((Interval*)reader)+n_records);
+        return iterator((Interval*)reader.end());
     }
 
     /// const iterator to first value
     const_iterator begin(void) const {
-        return const_iterator((Interval*)reader);
+        return const_iterator((Interval*)reader.begin());
     }
 
     /// const iterator to one past end
     const_iterator end(void) const {
-        return const_iterator(((Interval*)reader)+n_records);
-    }
-
-    /// return the backing buffer
-    char* get_buffer(void) const {
-        return reader;
+        return const_iterator((Interval*)reader.end());
     }
 
     /// get the record count
     size_t record_count(void) {
-        int fd = open(filename.c_str(), O_RDWR);
-        if (fd == -1) {
-            assert(false);
-        }
-        struct stat stats;
-        if (-1 == fstat(fd, &stats)) {
-            assert(false);
-        }
-        assert(stats.st_size % sizeof(Interval) == 0); // must be even records
-        size_t count = stats.st_size / sizeof(Interval);
-        return count;
+        return reader.size() / sizeof(Interval);
     }
 
     std::ifstream::pos_type filesize(const char* filename) {
@@ -323,19 +235,17 @@ public:
 
     /// sort the record in the backing file by key
     void sort(int num_threads) {
-        close_writer();
-        close_reader();
         if (sorted) return;
         //std::cerr << "sorting!" << std::endl;
-        mmap_buffer_t buffer;
-        open_mmap_buffer(filename.c_str(), &buffer);
-        uint64_t data_len = buffer.size/sizeof(Interval);
+        std::error_code error;
+        mio::mmap_sink buffer = mio::make_mmap_sink(
+            filename.c_str(), 0, mio::map_entire_file, error);
+        if (error) { assert(false); }
         // sort in parallel (uses OpenMP if available, std::thread otherwise)
-        ips4o::parallel::sort((Interval*)buffer.data,
-                              ((Interval*)buffer.data)+data_len,
+        ips4o::parallel::sort((Interval*)buffer.begin(),
+                              (Interval*)buffer.end(),
                               IntervalLess(),
                               num_threads);
-        close_mmap_buffer(&buffer);
         sorted = true;
     }
 
@@ -465,24 +375,25 @@ public:
     };
 
     Interval* get_array(void) const {
-        return (Interval*)reader;
+        return (Interval*)reader.begin();
     }
 
 public:
     /// write into our write buffer
     /// open_writer() must be called first to set up our buffer and writer
 	void add(const S &s, const S &e, const T &d) {
-        interval_queue->push(make_interval(s, e, d));
+        interval_queue.push(make_interval(s, e, d));
     }
 
 	void index(int num_threads) {
+        close_writer();
         sort(num_threads);
         open_reader();
         n_records = record_count();
         indexed = true;
         close_reader();
+        max_level = index_core(n_records);
         open_reader();
-		max_level = index_core(get_array(), n_records);
 	}
 
     /// get overlaps, callback takes index, start, end, data


=====================================
src/mmmultimap.hpp
=====================================
@@ -16,6 +16,8 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <mio/mmap.hpp>
+
 #include "sdsl/bit_vectors.hpp"
 #include "ips4o.hpp"
 #include "atomic_queue.h"
@@ -41,70 +43,16 @@ template <typename Key, typename Value> class map {
 
 private:
 
-    // memory mapped buffer struct
-    struct mmap_buffer_t {
-        int fd;
-        off_t size;
-        void *data;
-    };
-
-    // utilities used by mmmultimap
-    int open_mmap_buffer(const char* path, mmap_buffer_t* buffer) {
-        buffer->data = nullptr;
-        buffer->fd = open(path, O_RDWR);
-        if (buffer->fd == -1) {
-            goto error;
-        }
-        struct stat stats;
-        if (-1 == fstat(buffer->fd, &stats)) {
-            goto error;
-        }
-        if (!(buffer->data = mmap(nullptr,
-                                  stats.st_size,
-                                  PROT_READ | PROT_WRITE,
-                                  MAP_SHARED,
-                                  buffer->fd,
-                                  0
-                  ))) {
-            goto error;
-        }
-        madvise(buffer, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
-        buffer->size = stats.st_size;
-        return 0;
-
-    error:
-        perror(path);
-        if (buffer->data)
-            munmap(buffer->data, stats.st_size);
-        if (buffer->fd != -1)
-            close(buffer->fd);
-        buffer->data = 0;
-        buffer->fd = 0;
-        return -1;
-    }
-
-    void close_mmap_buffer(mmap_buffer_t* buffer) {
-        if (buffer->data) {
-            munmap(buffer->data, buffer->size);
-            buffer->data = 0;
-            buffer->size = 0;
-        }
-
-        if (buffer->fd) {
-            close(buffer->fd);
-            buffer->fd = 0;
-        }
-    }
-
+    // an entry is a POD struct of Key, Value
     typedef struct { Key key; Value value; } Entry;
+
+    // the comparator used to sort the backing array
     struct EntryLess {
-		bool operator()(const std::pair<Key,Value>& a, const std::pair<Key,Value>& b) const { return a < b; }
+        bool operator()(const std::pair<Key,Value>& a, const std::pair<Key,Value>& b) const { return a < b; }
 	};
 
-    std::ofstream writer;
-    std::vector<std::ofstream> writers;
-    char* reader = nullptr;
-    int reader_fd = 0;
+    // memory mapped buffer
+    mio::mmap_source reader;
     std::string filename;
     std::string index_filename;
     bool sorted = false;
@@ -122,16 +70,16 @@ private:
     sdsl::sd_vector<>::select_1_type key_cbv_select;
     bool indexed = false;
     uint32_t OUTPUT_VERSION = 1; // update as we change our format
+    // a single writer thread reads from an atomic queue and writes to our (unsorted) backing file
     std::thread writer_thread;
-    atomic_queue::AtomicQueue2<Entry, 2 << 16>* entry_queue = nullptr;
+    atomic_queue::AtomicQueue2<Entry, 2 << 16> entry_queue;
     std::atomic<bool> work_todo;
 
-    void init(void) {
+    void init(Value nullv) {
         record_size = sizeof(Key) + sizeof(Value);
         nullkey = 0;
-        for (size_t i = 0; i < sizeof(Value); ++i) {
-            ((uint8_t*)&nullvalue)[i] = 0;
-        }
+        nullvalue = nullv;
+        work_todo.store(false);
     }
 
 public:
@@ -140,10 +88,16 @@ public:
     class iterator;
     class const_iterator;
 
+    // make sure we and our friend the compiler don't try anything silly
+    map(void) = delete;
+    map(const map& m) = delete;
+    map(map&& m) = delete;
+    map& operator=(map&& m) = delete;
+
     // constructor
-    map(void) { init(); }
+    map(Value nullv) { init(nullv); }
 
-    map(const std::string& f) : filename(f) { init(); }
+    map(const std::string& f, Value nullv) : filename(f) { init(nullv); }
 
     ~map(void) {
         close_writer();
@@ -196,12 +150,18 @@ public:
     }
 
     void writer_func(void) {
+        assert(!filename.empty());
+        // remove the file; we only call this when making an index, and it's done once
+        std::ofstream writer(filename.c_str(), std::ios::binary | std::ios::trunc);
+        if (writer.fail()) {
+            throw std::ios_base::failure(std::strerror(errno));
+        }
         Entry entry;
-        while (work_todo.load() || !entry_queue->was_empty()) {
-            if (entry_queue->try_pop(entry)) {
+        while (work_todo.load() || !entry_queue.was_empty()) {
+            if (entry_queue.try_pop(entry)) {
                 do {
                     writer.write((char*)&entry, sizeof(Entry));
-                } while (entry_queue->try_pop(entry));
+                } while (entry_queue.try_pop(entry));
             } else {
                 std::this_thread::sleep_for(std::chrono::nanoseconds(1));
             }
@@ -211,73 +171,45 @@ public:
 
     // close/open backing file
     void open_writer(void) {
-        if (writer.is_open()) {
-            writer.seekp(0, std::ios_base::end); // seek to the end for appending
-            return;
+        if (!work_todo.load()) {
+            work_todo.store(true);
+            writer_thread = std::thread(&map::writer_func, this);
         }
-        assert(!filename.empty());
-        // remove the file; we only call this when making an index, and it's done once
-        writer.open(filename.c_str(), std::ios::binary | std::ios::trunc);
-        if (writer.fail()) {
-            throw std::ios_base::failure(std::strerror(errno));
-        }
-        entry_queue = new atomic_queue::AtomicQueue2<Entry, 2 << 16>;
-        work_todo.store(true);
-        writer_thread = std::thread(&map::writer_func, this);
     }
 
     void close_writer(void) {
         if (work_todo.load()) {
             work_todo.store(false);
-            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            while (!entry_queue.was_empty()) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            }
             if (writer_thread.joinable()) {
                 writer_thread.join();
             }
-            delete entry_queue;
-            entry_queue = nullptr;
         }
     }
 
+    // è mio
     void open_reader(void) {
-        if (reader_fd) return; //open
-        assert(!filename.empty());
-        // open in binary mode as we are reading from this interface
-        reader_fd = open(filename.c_str(), O_RDWR);
-        if (reader_fd == -1) {
-            assert(false);
-        }
-        struct stat stats;
-        if (-1 == fstat(reader_fd, &stats)) {
-            assert(false);
-        }
-        if (!(reader =
-              (char*) mmap(NULL,
-                           stats.st_size,
-                           PROT_READ | PROT_WRITE,
-                           MAP_SHARED,
-                           reader_fd,
-                           0))) {
-            assert(false);
-        }
-        madvise((void*)reader, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
+        if (reader.is_mapped()) return;
+        std::error_code error;
+        reader = mio::make_mmap_source(
+            filename.c_str(), 0, mio::map_entire_file, error);
+        if (error) { assert(false); }
+        // set up for sequential access
+        madvise((void*)reader.begin(),
+                reader.mapped_length(),
+                POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
     }
 
     void close_reader(void) {
-        if (reader) {
-            size_t c = record_count();
-            munmap(reader, c);
-            reader = 0;
-        }
-        if (reader_fd) {
-            close(reader_fd);
-            reader_fd = 0;
-        }
+        if (reader.is_mapped()) reader.unmap();
     }
 
     /// write the pair to the backing file
     /// open_writer() must be called first to set up our buffer and writer
     void append(const Key& k, const Value& v) {
-        entry_queue->push((Entry){k, v});
+        entry_queue.push((Entry){k, v});
     }
 
     /// return the number of records, which will only work after indexing
@@ -290,25 +222,9 @@ public:
         return record_size;
     }
 
-    /// return the backing buffer
-    char* get_buffer(void) const {
-        return reader;
-    }
-
     /// get the record count
     size_t record_count(void) {
-        int fd = open(filename.c_str(), O_RDWR);
-        if (fd == -1) {
-            assert(false);
-        }
-        struct stat stats;
-        if (-1 == fstat(fd, &stats)) {
-            assert(false);
-        }
-        assert(stats.st_size % record_size == 0); // must be even records
-        size_t count = stats.st_size / record_size;
-        close(fd);
-        return count;
+        return reader.size() / sizeof(Entry);
     }
 
     std::ifstream::pos_type filesize(const char* filename) {
@@ -320,15 +236,15 @@ public:
     void sort(int num_threads) {
         if (sorted) return;
         //std::cerr << "sorting!" << std::endl;
-        mmap_buffer_t buffer;
-        open_mmap_buffer(filename.c_str(), &buffer);
-        uint64_t data_len = buffer.size/record_size;
+        std::error_code error;
+        mio::mmap_sink buffer = mio::make_mmap_sink(
+            filename.c_str(), 0, mio::map_entire_file, error);
+        if (error) { assert(false); }
         // sort in parallel (uses OpenMP if available, std::thread otherwise)
-        ips4o::parallel::sort((std::pair<Key, Value>*)buffer.data,
-                              ((std::pair<Key, Value>*)buffer.data)+data_len,
+        ips4o::parallel::sort((std::pair<Key, Value>*)buffer.begin(),
+                              (std::pair<Key, Value>*)buffer.end(),
                               EntryLess(),
                               num_threads);
-        close_mmap_buffer(&buffer);
         sorted = true;
     }
 


=====================================
src/mmmultiset.hpp
=====================================
@@ -16,6 +16,8 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#include <mio/mmap.hpp>
+
 #include "sdsl/bit_vectors.hpp"
 #include "ips4o.hpp"
 #include "atomic_queue.h"
@@ -32,70 +34,12 @@ of the multiset, such as to obtain their counts.
 
 template <typename Value> class set {
 
-private:
-    
-    // memory mapped buffer struct
-    struct mmap_buffer_t {
-        int fd;
-        off_t size;
-        void *data;
-    };
-
-    // utilities used by mmmultimap
-    int open_mmap_buffer(const char* path, mmap_buffer_t* buffer) {
-        buffer->data = nullptr;
-        buffer->fd = open(path, O_RDWR);
-        if (buffer->fd == -1) {
-            goto error;
-        }
-        struct stat stats;
-        if (-1 == fstat(buffer->fd, &stats)) {
-            goto error;
-        }
-        if (!(buffer->data = mmap(nullptr,
-                                  stats.st_size,
-                                  PROT_READ | PROT_WRITE,
-                                  MAP_SHARED,
-                                  buffer->fd,
-                                  0
-                  ))) {
-            goto error;
-        }
-        madvise(buffer, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
-        buffer->size = stats.st_size;
-        return 0;
-
-    error:
-        perror(path);
-        if (buffer->data)
-            munmap(buffer->data, stats.st_size);
-        if (buffer->fd != -1)
-            close(buffer->fd);
-        buffer->data = 0;
-        buffer->fd = 0;
-        return -1;
-    }
-
-    void close_mmap_buffer(mmap_buffer_t* buffer) {
-        if (buffer->data) {
-            munmap(buffer->data, buffer->size);
-            buffer->data = 0;
-            buffer->size = 0;
-        }
-
-        if (buffer->fd) {
-            close(buffer->fd);
-            buffer->fd = 0;
-        }
-    }
-
     struct ValueLess {
         bool operator()(const Value &a, const Value &b) const { return a < b; }
 	};
-    
-    std::ofstream writer;
-    char* reader = nullptr;
-    int reader_fd = 0;
+
+
+    mio::mmap_source reader;
     std::string filename;
     std::string index_filename;
     bool sorted = false;
@@ -103,7 +47,7 @@ private:
     uint64_t n_records = 0;
     bool indexed = false;
     std::thread writer_thread;
-    atomic_queue::AtomicQueue2<Value, 2 << 16>* value_queue = nullptr;
+    atomic_queue::AtomicQueue2<Value, 2 << 16> value_queue;
     std::atomic<bool> work_todo;
     
 public:
@@ -111,11 +55,16 @@ public:
     // forward declaration for iterator types
     class iterator;
     class const_iterator;
-    
-    // constructor
-    set(void) { }
 
-    set(const std::string& f) : filename(f) { }
+    // make sure we and our friend the compiler don't try anything silly
+    set(void) = delete;
+    set(const set& m) = delete;
+    set(set&& m) = delete;
+    set& operator=(set&& m) = delete;
+
+    set(const std::string& f) : filename(f) {
+        work_todo.store(false);
+    }
 
     ~set(void) {
         close_writer();
@@ -127,12 +76,16 @@ public:
     }
 
     void writer_func(void) {
+        std::ofstream writer(filename.c_str(), std::ios::binary | std::ios::trunc);
+        if (writer.fail()) {
+            throw std::ios_base::failure(std::strerror(errno));
+        }
         Value value;
-        while (work_todo.load() || !value_queue->was_empty()) {
-            if (value_queue->try_pop(value)) {
+        while (work_todo.load() || !value_queue.was_empty()) {
+            if (value_queue.try_pop(value)) {
                 do {
                     writer.write((char*)&value, sizeof(Value));
-                } while (value_queue->try_pop(value));
+                } while (value_queue.try_pop(value));
             } else {
                 std::this_thread::sleep_for(std::chrono::nanoseconds(1));
             }
@@ -140,75 +93,47 @@ public:
         writer.close();
     }
 
-    // close/open backing file
+    // open backing file and start writer_thread
     void open_writer(void) {
-        if (writer.is_open()) {
-            writer.seekp(0, std::ios_base::end); // seek to the end for appending
-            return;
-        }
-        assert(!filename.empty());
-        // remove the file; we only call this when making an index, and it's done once
-        writer.open(filename.c_str(), std::ios::binary | std::ios::trunc);
-        if (writer.fail()) {
-            throw std::ios_base::failure(std::strerror(errno));
+        if (!work_todo.load()) {
+            work_todo.store(true);
+            writer_thread = std::thread(&set::writer_func, this);
         }
-        value_queue = new atomic_queue::AtomicQueue2<Value, 2 << 16>;
-        work_todo.store(true);
-        writer_thread = std::thread(&set::writer_func, this);
     }
 
     void close_writer(void) {
         if (work_todo.load()) {
             work_todo.store(false);
-            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            while (!value_queue.was_empty()) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            }
             if (writer_thread.joinable()) {
                 writer_thread.join();
             }
-            delete value_queue;
-            value_queue = nullptr;
         }
     }
 
+    // è mio
     void open_reader(void) {
-        if (reader_fd) return; //open
-        assert(!filename.empty());
-        // open in binary mode as we are reading from this interface
-        reader_fd = open(filename.c_str(), O_RDWR);
-        if (reader_fd == -1) {
-            assert(false);
-        }
-        struct stat stats;
-        if (-1 == fstat(reader_fd, &stats)) {
-            assert(false);
-        }
-        if (!(reader =
-              (char*) mmap(NULL,
-                            stats.st_size,
-                            PROT_READ | PROT_WRITE,
-                            MAP_SHARED,
-                            reader_fd,
-                            0))) {
-            assert(false);
-        }
-        madvise((void*)reader, stats.st_size, POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
+        if (reader.is_mapped()) return;
+        std::error_code error;
+        reader = mio::make_mmap_source(
+            filename.c_str(), 0, mio::map_entire_file, error);
+        if (error) { assert(false); }
+        // set up for sequential access
+        madvise((void*)reader.begin(),
+                reader.mapped_length(),
+                POSIX_MADV_WILLNEED | POSIX_MADV_SEQUENTIAL);
     }
 
     void close_reader(void) {
-        if (reader) {
-            size_t c = record_count();
-            munmap(reader, c);
-            reader = 0;
-        }
-        if (reader_fd) {
-            close(reader_fd);
-            reader_fd = 0;
-        }
+        if (reader.is_mapped()) reader.unmap();
     }
 
     /// write the pair to end of backing file
     /// open_writer() must be called first to set up our buffer and writer
     void append(const Value& v) {
-        value_queue->push(v);
+        value_queue.push(v);
     }
 
     /// return the number of records, which will only work after indexing
@@ -218,22 +143,22 @@ public:
     
     /// iterator to first value
     iterator begin(void) {
-        return iterator((Value*)reader);
+        return iterator((Value*)reader.begin());
     }
     
     /// iterator to one past end
     iterator end(void) {
-        return iterator(((Value*)reader)+n_records);
+        return iterator((Value*)reader.end());
     }
 
     /// const iterator to first value
     const_iterator begin(void) const {
-        return const_iterator((Value*)reader);
+        return const_iterator((Value*)reader.begin());
     }
 
     /// const iterator to one past end
     const_iterator end(void) const {
-        return const_iterator(((Value*)reader)+n_records);
+        return const_iterator((Value*)reader.end());
     }
 
     /// return the size of each combined record
@@ -241,50 +166,31 @@ public:
         return sizeof(Value);
     }
 
-    /// return the backing buffer
-    char* get_buffer(void) const {
-        return reader;
-    }
-
     /// get the record count
     size_t record_count(void) {
-        int fd = open(filename.c_str(), O_RDWR);
-        if (fd == -1) {
-            assert(false);
-        }
-        struct stat stats;
-        if (-1 == fstat(fd, &stats)) {
-            assert(false);
-        }
-        assert(stats.st_size % get_record_size() == 0); // must be even records
-        size_t count = stats.st_size / sizeof(Value);
-        close(fd);
-        return count;
+        return reader.size() / sizeof(Value);
     }
 
     std::ifstream::pos_type filesize(const char* filename) {
         std::ifstream in(filename, std::ifstream::ate | std::ifstream::binary);
         return in.tellg();
     }
-    
-    /// sort the record in the backing file by key
+
     void sort(int num_threads) {
-        close_writer();
-        close_reader();
         if (sorted) return;
         //std::cerr << "sorting!" << std::endl;
-        mmap_buffer_t buffer;
-        open_mmap_buffer(filename.c_str(), &buffer);
-        uint64_t data_len = buffer.size/sizeof(Value);
+        std::error_code error;
+        mio::mmap_sink buffer = mio::make_mmap_sink(
+            filename.c_str(), 0, mio::map_entire_file, error);
+        if (error) { assert(false); }
         // sort in parallel (uses OpenMP if available, std::thread otherwise)
-        ips4o::parallel::sort((Value*)buffer.data,
-                              ((Value*)buffer.data)+data_len,
+        ips4o::parallel::sort((Value*)buffer.begin(),
+                              (Value*)buffer.end(),
                               ValueLess(),
                               num_threads);
-        close_mmap_buffer(&buffer);
         sorted = true;
     }
-
+    
     Value read_value(size_t i) const {
         Value v;
         memcpy(&v, &reader[i*sizeof(Value)], sizeof(Value));
@@ -293,6 +199,7 @@ public:
 
     // index
     void index(int num_threads) {
+        close_writer();
         sort(num_threads);
         open_reader();
         n_records = record_count();
@@ -301,6 +208,12 @@ public:
         open_reader();
     }
 
+
+
+    // todo we make a big change, pulling in all the stuff from mmmultimap
+    
+    
+    
     void for_each_value(const std::function<void(const Value&)>& lambda) const {
         for (size_t i = 0; i < n_records; ++i) {
             lambda(read_value(i));



View it on GitLab: https://salsa.debian.org/med-team/libmmmulti/-/compare/eefadf7a8934faf4a592051d77e2961df37f4e2a...e797bc55ae5d3d42bafab7eb74c079baf4fe8571

-- 
View it on GitLab: https://salsa.debian.org/med-team/libmmmulti/-/compare/eefadf7a8934faf4a592051d77e2961df37f4e2a...e797bc55ae5d3d42bafab7eb74c079baf4fe8571
You're receiving this email because of your account on salsa.debian.org. Manage all notifications: https://salsa.debian.org/-/profile/notifications | Help: https://salsa.debian.org/help


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20261028/f1ff69c2/attachment-0001.htm>


More information about the debian-med-commit mailing list