[med-svn] [Git][med-team/libmmap-allocator][upstream] New upstream version 0.4.0+git20200122.adbfbe1
Andreas Tille
gitlab at salsa.debian.org
Sun Jun 7 09:49:28 BST 2020
Andreas Tille pushed to branch upstream at Debian Med / libmmap-allocator
Commits:
2a1e5b81 by Andreas Tille at 2020-06-06T09:36:03+02:00
New upstream version 0.4.0+git20200122.adbfbe1
- - - - -
12 changed files:
- .gitignore
- + CMakeLists.txt
- Makefile
- README.md
- + TODO
- mmap_access_mode.h
- mmap_allocator.h
- + mmap_exception.h
- mmap_file_pool.cpp
- mmap_file_pool.h
- + mmappable_vector.h
- test_allocator.cpp
Changes:
=====================================
.gitignore
=====================================
@@ -1,4 +1,9 @@
testfile
+testfile2
test_allocator
*.o
.*.swp
+*.so
+*.a
+build/
+*~
=====================================
CMakeLists.txt
=====================================
@@ -0,0 +1,51 @@
+# Specify the minimum version for CMake
+
+cmake_minimum_required(VERSION 3.1)
+
+# Project's name
+project(mmap_allocator)
+# We build using c++14
+set(CMAKE_CXX_STANDARD 14)
+
+# Use all standard-compliant optimizations
+set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS} -O3 -mcx16 -g")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -O3 -mcx16 -g")
+
+#set(CMAKE_BUILD_TYPE Release)
+set(CMAKE_BUILD_TYPE Debug)
+
+# set up our target executable and specify its dependencies and includes
+add_library(mmap_allocator_objs OBJECT
+ ${CMAKE_SOURCE_DIR}/mmap_file_pool.cpp)
+
+set(mmap_allocator_INCLUDES
+ ${CMAKE_SOURCE_DIR})
+
+set(mmap_allocator_HEADERS
+ ${CMAKE_SOURCE_DIR}/mmap_access_mode.h
+ ${CMAKE_SOURCE_DIR}/mmap_allocator.h
+ ${CMAKE_SOURCE_DIR}/mmap_exception.h
+ ${CMAKE_SOURCE_DIR}/mmap_file_pool.h
+ ${CMAKE_SOURCE_DIR}/mmappable_vector.h)
+
+target_include_directories(mmap_allocator_objs PUBLIC ${mmap_allocator_INCLUDES})
+set_target_properties(mmap_allocator_objs PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
+
+add_library(libmmap_allocator_static STATIC $<TARGET_OBJECTS:mmap_allocator_objs>)
+set_target_properties(libmmap_allocator_static PROPERTIES OUTPUT_NAME "mmap_allocator")
+set_target_properties(libmmap_allocator_static PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
+add_library(libmmap_allocator_shared SHARED $<TARGET_OBJECTS:mmap_allocator_objs>)
+set_target_properties(libmmap_allocator_shared PROPERTIES OUTPUT_NAME "mmap_allocator")
+set_target_properties(libmmap_allocator_shared PROPERTIES POSITION_INDEPENDENT_CODE TRUE)
+
+set_target_properties(libmmap_allocator_static PROPERTIES PUBLIC_HEADER "${mmap_allocator_HEADERS}")
+set_target_properties(libmmap_allocator_shared PROPERTIES PUBLIC_HEADER "${mmap_allocator_HEADERS}")
+
+add_executable(test_mmap_allocator
+ $<TARGET_OBJECTS:mmap_allocator_objs>
+ ${CMAKE_SOURCE_DIR}/test_allocator.cpp)
+#set_target_properties(mmap_allocator PROPERTIES OUTPUT_NAME "mmap_allocator")
+
+install(TARGETS test_mmap_allocator DESTINATION bin)
+install(TARGETS libmmap_allocator_static ARCHIVE DESTINATION lib LIBRARY DESTINATION lib PUBLIC_HEADER DESTINATION include/mmap_allocator)
+install(TARGETS libmmap_allocator_shared ARCHIVE DESTINATION lib LIBRARY DESTINATION lib PUBLIC_HEADER DESTINATION include/mmap_allocator)
=====================================
Makefile
=====================================
@@ -1,30 +1,54 @@
-CPPFLAGS=-g -Wall
-CFLAGS=-g -Wall
+CPPFLAGS=-g -Wall -fPIC
+CFLAGS=-g -Wall -fPIC
-TARGET_DIR=/home/johannes/re3
+# Enable to test with GCC 3.4
+# CXX=g++34
-all: test mmap_file_pool.o
+PREFIX=/usr
+
+SOURCES=mmap_file_pool.cpp mmap_file_pool.h mmap_allocator.h mmap_access_mode.h mmappable_vector.h mmap_exception.h
+
+HEADERS=mmap_access_mode.h mmap_allocator.h mmap_file_pool.h mmap_exception.h
+LIBRARIES=libmmap_allocator.so libmmap_allocator.a
+
+SRC_INSTALL_TARGET_DIR=/home/johannes/re3
+
+all: test_allocator mmap_file_pool.o $(LIBRARIES)
debug: CPPFLAGS+=-DMMAP_ALLOCATOR_DEBUG=1
debug: CFLAGS+=-DMMAP_ALLOCATOR_DEBUG=1
debug: clean all
-install: mmap_file_pool.cpp mmap_file_pool.h mmap_allocator.h mmap_access_mode.h
- cp mmap_file_pool.cpp mmap_file_pool.h mmap_allocator.h mmap_access_mode.h $(TARGET_DIR)
+libmmap_allocator.so: mmap_file_pool.o
+ g++ -shared -o libmmap_allocator.so mmap_file_pool.o
-test: test_allocator
+libmmap_allocator.a: mmap_file_pool.o
+ ar r libmmap_allocator.a mmap_file_pool.o
+
+install_sources: $(SOURCES)
+ cp $(SOURCES) $(SRC_INSTALL_TARGET_DIR)
+
+install: all
+ install -m 644 $(HEADERS) $(PREFIX)/include
+ install -m 755 $(LIBRARIES) $(PREFIX)/lib
+
+test: all
@echo "Running mmap allocator regression test suite."
- ./test_allocator
+ bash -c 'export LD_LIBRARY_PATH=. ; ./test_allocator'
+
+debugtest: debug
+ @echo "Running mmap allocator regression test suite with verbose enabled."
+ bash -c 'export LD_LIBRARY_PATH=. ; ./test_allocator'
-test_allocator: mmap_allocator.h mmap_file_pool.o test_allocator.o
- g++ mmap_file_pool.o test_allocator.o -o test_allocator
+test_allocator: mmap_allocator.h mmap_file_pool.o test_allocator.o $(LIBRARIES)
+ g++ test_allocator.o -L. -lmmap_allocator -o test_allocator
test_mmap_fixed: test_mmap_fixed.c
gcc $(CFLAGS) test_mmap_fixed.c -o test_mmap_fixed
clean:
- rm -f test_allocator test_mmap_fixed testfile *.o
+ rm -f test_allocator test_mmap_fixed testfile testfile2 *.o $(LIBRARIES)
-mmap_file_pool.o: mmap_file_pool.cpp mmap_file_pool.h mmap_allocator.h mmap_access_mode.h
+mmap_file_pool.o: mmap_file_pool.cpp mmap_file_pool.h mmap_allocator.h mmap_access_mode.h mmappable_vector.h mmap_exception.h
-test_allocator.o: mmap_file_pool.h mmap_allocator.h mmap_access_mode.h
+test_allocator.o: test_allocator.cpp mmap_file_pool.h mmap_allocator.h mmap_access_mode.h mmappable_vector.h mmap_exception.h
=====================================
README.md
=====================================
@@ -4,7 +4,7 @@ mmap_allocator - A STL allocator that mmaps files
Introduction
------------
-When reading large files (>10GB) into memory, read() calls are usually
+When reading large files (>100MB) into memory, read() calls are usually
not very space and time efficient, since the whole data is copiied at
least once. Furthermore, when using STL containers (like a vector for
example), data is copiied another time unless you specify the location
@@ -18,7 +18,7 @@ an interface to do exactly this.
Put short, if you need to handle big files that contain unstructured data
(like doubles or even text files), mmap_allocator is worth a try. It was
-written as part of a consulting project I did for a big Austrian bank.
+written as part of a consulting project I did for a large Austrian bank.
License
-------
@@ -40,41 +40,54 @@ mapped regions directly.
Usage
-----
-To use this, simply copy the mmap_allocator.h file into your project
-and include it. No compilation of the library itself is required (the
-Makefile will compile and run the regression test file).
+To build this, do a
+
+ make
+
+followed by a
+
+ make install
+
+When compiling, include the mmappable_vector.h file and link with
+the -lmmap_allocator library.
Example
-------
Suppose you have a file with 1024 ints. Then:
- mmappable_vector<int, mmap_allocator<int> > my_vector =
- vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile"));
+ mmappable_vector<int> my_vector;
declares a vector that uses mmap_allocator to mmap the file. By calling:
- my_vector.map_into_memory(1024);
+ my_vector.mmap_file("testfile", READ_ONLY, 0, 1024);
+
+the mmappable_vector class will call allocate of the mmap_allocator
+class, which in turn will mmap the file and set the content of the
+vector to the mmapped area. The file's content can then be accessed
+the usual way (my_vector[i]).
+
+Use
+
+ my_vector.munmap_file();
-the STL vector class will call allocate which in turn will mmap the file and
-set the content of the vector to the mmapped area. The file's content can
-then be accessed the usual way (my_vector[i]). Update: Starting with
-version 0.4 this also sets the size of the vector correctly.
+to drop the mapping (it may remain in an internal cache, however). After
+this call all accesses of mmappable_vector elements are invalid, same
+goes for the iterators.
Do not forget to:
- using namespace std;
+
using namespace mmap_allocator_namespace;
and include:
- #include "mmap_allocator.h"
- #include <vector>
+ #include "mmappable_vector.h"
but you probably know that yourself ;)
-Please see the test_allocator.cpp file for more examples (\footnote{
-I used Testdriven development for this project and can highly recommend
-this}).
+Please see the test_allocator.cpp file for more examples
+(I used Testdriven development for this project and can highly recommend
+this development model).
Mode
----
@@ -82,11 +95,11 @@ Mode
As part of the construction of the mmap_allocator object a mode field can be
specified. It does the following:
-DEFAULT_STL_ALLOCATOR: Default STL allocator (malloc based). Reason
+* DEFAULT_STL_ALLOCATOR: Default STL allocator (malloc based). Reason
is to have containers that do both and are compatible
-READ_ONLY: Readonly modus. Segfaults when vector content is written to.
-READ_WRITE_PRIVATE: Read/write access, writes are not propagated to disk.
-READ_WRITE_SHARED: Read/write access, writes are propagated to disk
+* READ_ONLY: Readonly modus. Segfaults when vector content is written to.
+* READ_WRITE_PRIVATE: Read/write access, writes are not propagated to disk.
+* READ_WRITE_SHARED: Read/write access, writes are propagated to disk
(file is modified)
The offset parameter must be page aligned (PAGE_SIZE, usually 4K or 8K),
@@ -102,19 +115,24 @@ We needed this because our program (that uses mmap_allocator) mmaps
a file in many small chunks (10K junks) which eventually lead to a
Out of filedescriptors error (when mapping a 100Meg file in 10K junks).
-Following parameters have been added to the mmap_allocator constructor:
+Following flags can be passed (by |'ing them together) to the mmap_file
+method:
-.) map_whole_file: Set this to true when you know that you need
+* MAP_WHOLE_FILE: Set this flag when you know that you need
the whole (or most parts of the) file later and only want to
request a part of it now.
-.) allow_remap: Set this to true if you are allocating a vector
+* ALLOW_REMAP: Set this flag if you are allocating a vector
from a mmapped file and you know that you do not need previous
mappings any more. Normally this is used when the file size
changes (in particular when it grows). Be aware, however that
this could invalidate all allocations for that file that have
been made before.
+* BYPASS_FILE_POOL: Set this flag if you want a per-vector
+private mapping. This is useful in conjunction with READ_WRITE_PRIVATE
+mappings.
+
Mmappable vector class
----------------------
@@ -125,18 +143,88 @@ the content (which is not wanted since the content is coming from the
mmapped file). From now on, please use the mmapped_vector class for
using mmap_allocator.
+The old method by using the mmap_allocators constructor to set the
+parameters for file mapping is still supported, however deprecated.
+
+READ_WRITE caveats
+------------------
+
+Unlike reading from a file, a mmappable_vector that is mapped via
+the mmap file pool contains all changes already made to the content
+before. Suppose if you have:
+
+ mmappable_vector<int> p;
+ if (method == MMAP) {
+ p.mmap_file("testfile", READ_WRITE_PRIVATE, 0, filesize("testfile"));
+ } else {
+ readFromFileIntoVector(p, "testfile", READ_WRITE_PRIVATE, 0, filesize("testfile")
+ }
+
+and then do something like:
+
+ for (it = p.begin();it != p.end();it++) {
+ *it += 1;
+ }
+
+This will do not what you would expect at first glance when being called
+twice. When the vector maps the file for the second time, it will map it
+from the file pool and hence already have the values increased by one.
+
+To avoid this, use the bypass_file_pool flag which will cause the file
+being mmapped another time with different virtual memory mappings.
+
+Exceptions
+----------
+
+There is a custom exception class mmap_allocator_exception which is
+used at various places, for example when the file to be mapped is
+not found. Use the e.message() method to find out what happened.
+
+Conversion function templates
+-----------------------------
+
+To convert between a STL vector and a mmappable vector, use the
+to_std_vector(mappable_vector) and to_mmappable_vector(std_vector)
+function templates. However, try to avoid this because this will
+copy the whole vector contents.
+
+Verbosity
+---------
+
+To debug the allocator, there are two ways:
+
+* call set_verbosity() with a positive value at runtime.
+* define MMAP_ALLOCATOR_DEBUG as a positive value at compile time.
+
+The latter is done by make test.
+
Version history
---------------
-0.1.0, first release.
-0.2.0, some interface changes.
-0.3.0, mmaped file pool.
-0.3.1, do not remap files when area fits in already mapped file.
-0.3.2, never use MAP_FIXED.
-0.3.3, bugfix in computing pointers.
-0.4.0, mmapped_vector class.
+* 0.1.0, first release.
+* 0.2.0, some interface changes.
+* 0.3.0, mmaped file pool.
+* 0.3.1, do not remap files when area fits in already mapped file.
+* 0.3.2, never use MAP_FIXED.
+* 0.3.3, bugfix in computing pointers.
+* 0.4.0, mmapped_vector class.
+* 0.5.0, cleaner interface, illegal offset bugfix.
+* 0.5.1, bypass_file_pool flag.
+* 0.5.2, changed bool parameters to flags argument.
+* 0.5.3, set_verbosity() to toggle debug output.
+* 0.6.0, to_stl_vector function template.
+* 0.6.1, flags bugfix.
+* 0.7.0, moved mmappable_vector to separate header.
+* 0.8.0, to_mmappable_vector conversion function.
+* 0.8.1, exception now knows what() method.
+* 0.9.0, more standard conformant exception handling.
+* 0.9.1, fixed a permission bug in MMAP_READWRITE_PRIVATE.
+* 0.10.0, KEEP_FOREVER flag: never close any file.
+* 0.10.1, fixed bug with allocating/deallocating 0 bytes.
Author
------
This was written by Johannes Thoma <johannes.thoma at gmx.at>
+Thanks to Piotr Nycz from stackoverflow for contributing
+the constructor function template in mmappable_vector.h
=====================================
TODO
=====================================
@@ -0,0 +1,35 @@
+TODO for mmap_allocator
+-----------------------
+
+Done:
+
+Option to not use mempool (bypass_file_pool).
+
+Convert bool parameters to bit flag.
+
+Have a option to set verbosity at runtime, instead of having to recompile.
+
+Compile as a shared library.
+
+make install should do what one expects (-> install_sources).
+
+Have more converters between mmappable_vectors and plain STL vectors.
+
+For 0.7:
+
+Support for STL strings.
+
+Move mmappable_vector to separate file.
+
+Later:
+
+Support for other platforms
+ At least test different gcc / libstdc++ implementation.
+
+Allow remapping of already mapped vectors.
+
+Conversion function template from std vectors to mmapped vectors (if possible):
+ Only can convert from mmappable_vectors to std::vectors, don't
+ know how to convert the other way.
+
+std:string support
=====================================
mmap_access_mode.h
=====================================
@@ -7,8 +7,6 @@
#define UPPER_ALIGN_TO_PAGE(x) ALIGN_TO_PAGE((x)+(getpagesize()-1))
#define OFFSET_INTO_PAGE(x) ((x) & (getpagesize() - 1))
-// #define MMAP_ALLOCATOR_DEBUG 1
-
namespace mmap_allocator_namespace
{
enum access_mode {
@@ -18,34 +16,15 @@ namespace mmap_allocator_namespace
READ_WRITE_SHARED /* Read/write access, writes are propagated to disk (file is modified) */
};
- class mmap_allocator_exception: public std::exception {
-public:
- mmap_allocator_exception() throw():
- std::exception(),
- msg("Unknown reason")
- {
- }
-
- mmap_allocator_exception(const char *msg_param) throw():
- std::exception(),
- msg(msg_param)
- {
-#ifdef MMAP_ALLOCATOR_DEBUG
- fprintf(stderr, "Throwing exception %s\n", msg_param);
-#endif
- }
-
- virtual ~mmap_allocator_exception(void) throw()
- {
- }
-
- const char *message(void)
- {
- return msg.c_str();
- }
-private:
- std::string msg;
+ enum allocator_flags {
+ MAP_WHOLE_FILE = 1,
+ ALLOW_REMAP = 2,
+ BYPASS_FILE_POOL = 4,
+ KEEP_FOREVER = 8
};
+
+ void set_verbosity(int v);
+ int get_verbosity(void);
}
#endif
=====================================
mmap_allocator.h
=====================================
@@ -6,10 +6,14 @@
#include <stdio.h>
#include <vector>
#include "mmap_access_mode.h"
+#include "mmap_exception.h"
#include "mmap_file_pool.h"
namespace mmap_allocator_namespace
{
+ template <typename T, typename A>
+ class mmappable_vector;
+
template <typename T>
class mmap_allocator: public std::allocator<T>
{
@@ -28,19 +32,26 @@ public:
pointer allocate(size_type n, const void *hint=0)
{
void *the_pointer;
-#ifdef MMAP_ALLOCATOR_DEBUG
- fprintf(stderr, "Alloc %d bytes.\n", n*sizeof(T));
-#endif
+ if (get_verbosity() > 0) {
+ fprintf(stderr, "Alloc %zd bytes.\n", n*sizeof(T));
+ }
if (access_mode == DEFAULT_STL_ALLOCATOR) {
return std::allocator<T>::allocate(n, hint);
} else {
- the_pointer = the_pool.mmap_file(filename, access_mode, offset, n*sizeof(T), map_whole_file, allow_remap);
+ if (n == 0) {
+ return NULL;
+ }
+ if (bypass_file_pool) {
+ the_pointer = private_file.open_and_mmap_file(filename, access_mode, offset, n*sizeof(T), map_whole_file, allow_remap);
+ } else {
+ the_pointer = the_pool.mmap_file(filename, access_mode, offset, n*sizeof(T), map_whole_file, allow_remap);
+ }
if (the_pointer == NULL) {
throw(mmap_allocator_exception("Couldn't mmap file, mmap_file returned NULL"));
}
-#ifdef MMAP_ALLOCATOR_DEBUG
- fprintf(stderr, "pointer = %p\n", the_pointer);
-#endif
+ if (get_verbosity() > 0) {
+ fprintf(stderr, "pointer = %p\n", the_pointer);
+ }
return (pointer)the_pointer;
}
@@ -48,13 +59,22 @@ public:
void deallocate(pointer p, size_type n)
{
-#ifdef MMAP_ALLOCATOR_DEBUG
- fprintf(stderr, "Dealloc %d bytes (%p).\n", n*sizeof(T), p);
-#endif
+ if (get_verbosity() > 0) {
+ fprintf(stderr, "Dealloc %zd bytes (%p).\n", n*sizeof(T), p);
+ }
if (access_mode == DEFAULT_STL_ALLOCATOR) {
std::allocator<T>::deallocate(p, n);
} else {
- the_pool.munmap_file(filename, access_mode, offset, n*sizeof(T));
+ if (n == 0) {
+ return;
+ }
+ if (bypass_file_pool) {
+ private_file.munmap_and_close_file();
+ } else {
+ if (!keep_forever) {
+ the_pool.munmap_file(filename, access_mode, offset, n*sizeof(T));
+ }
+ }
}
}
@@ -64,7 +84,10 @@ public:
offset(0),
access_mode(DEFAULT_STL_ALLOCATOR),
map_whole_file(false),
- allow_remap(false)
+ allow_remap(false),
+ bypass_file_pool(false),
+ private_file(),
+ keep_forever(false)
{ }
mmap_allocator(const std::allocator<T> &a) throw():
@@ -73,7 +96,10 @@ public:
offset(0),
access_mode(DEFAULT_STL_ALLOCATOR),
map_whole_file(false),
- allow_remap(false)
+ allow_remap(false),
+ bypass_file_pool(false),
+ private_file(),
+ keep_forever(false)
{ }
mmap_allocator(const mmap_allocator &a) throw():
@@ -82,89 +108,38 @@ public:
offset(a.offset),
access_mode(a.access_mode),
map_whole_file(a.map_whole_file),
- allow_remap(a.allow_remap)
+ allow_remap(a.allow_remap),
+ bypass_file_pool(a.bypass_file_pool),
+ private_file(a.private_file),
+ keep_forever(a.keep_forever)
{ }
- mmap_allocator(const std::string filename_param, enum access_mode access_mode_param = READ_ONLY, offset_type offset_param = 0, bool map_whole_file_param = false, bool allow_remap_param = false) throw():
+ mmap_allocator(const std::string filename_param, enum access_mode access_mode_param = READ_ONLY, offset_type offset_param = 0, int flags = 0) throw():
std::allocator<T>(),
filename(filename_param),
offset(offset_param),
access_mode(access_mode_param),
- map_whole_file(map_whole_file_param),
- allow_remap(allow_remap_param)
+ map_whole_file((flags & MAP_WHOLE_FILE) != 0),
+ allow_remap((flags & ALLOW_REMAP) != 0),
+ bypass_file_pool((flags & BYPASS_FILE_POOL) != 0),
+ private_file(),
+ keep_forever((flags & KEEP_FOREVER) != 0)
{
}
~mmap_allocator() throw() { }
private:
+ friend class mmappable_vector<T, mmap_allocator<T> >;
+
std::string filename;
offset_type offset;
enum access_mode access_mode;
bool map_whole_file;
bool allow_remap;
+ bool bypass_file_pool;
+ mmapped_file private_file; /* used if bypass is set */
+ bool keep_forever;
};
-
- template <typename T, typename A = mmap_allocator<T> >
- class mmappable_vector: public std::vector<T, A> {
-public:
-/* TODO: are these necessary here? */
- typedef typename std::vector<T, A>::const_iterator const_iterator;
- typedef typename std::vector<T, A>::iterator iterator;
- typedef T value_type;
- typedef A allocator_type;
-
- mmappable_vector():
- std::vector<T,A>()
- {
- }
-
- mmappable_vector(const mmappable_vector<T, A> &other):
- std::vector<T,A>(other)
- {
- }
-
- explicit mmappable_vector(size_t n):
- std::vector<T,A>()
- {
- map_into_memory(n);
- }
-
- explicit mmappable_vector(A alloc):
- std::vector<T,A>(alloc)
- {
- }
-
- mmappable_vector(int n, T val, A alloc):
- std::vector<T,A>(n, val, alloc)
- {
- }
-
- mmappable_vector(int n, T val):
- std::vector<T,A>(n, val)
- {
- }
-
- void map_into_memory(const size_t n)
- {
- std::vector<T,A>::reserve(n);
-#ifdef __GNUC__
- std::vector<T,A>::_M_impl._M_finish = std::vector<T,A>::_M_impl._M_end_of_storage;
-#else
-#error "Not GNU C++, please either implement me or use GCC"
-#endif
- }
- };
-}
-
-template <typename T> std::vector<T> to_std_vector(const std::vector <T, mmap_allocator_namespace::mmap_allocator<T> > &v)
-{
- return std::vector<T, std::allocator<T> >(v.begin(), v.end());
-}
-
-template <typename T>
-std::vector<T, mmap_allocator_namespace::mmap_allocator<T> > to_mmap_vector(std::vector<T, std::allocator<T> > &v)
-{
- return std::vector<T, mmap_allocator_namespace::mmap_allocator<T> >(v.begin(), v.end());
}
#endif /* _MMAP_ALLOCATOR_H */
=====================================
mmap_exception.h
=====================================
@@ -0,0 +1,34 @@
+#ifndef _MMAP_EXCEPTION
+#define _MMAP_EXCEPTION
+
+#include "mmap_access_mode.h"
+#include <stdexcept>
+
+namespace mmap_allocator_namespace
+{
+ class mmap_allocator_exception: public std::runtime_error {
+public:
+ mmap_allocator_exception(const char *msg_param) throw():
+ std::runtime_error(msg_param)
+ {
+ if (get_verbosity() > 0) {
+ fprintf(stderr, "Throwing exception %s\n", msg_param);
+ }
+ }
+
+ mmap_allocator_exception(std::string msg_param) throw():
+ std::runtime_error(msg_param)
+ {
+ if (get_verbosity() > 0) {
+ fprintf(stderr, "Throwing exception %s\n", msg_param.c_str());
+ }
+ }
+
+ virtual ~mmap_allocator_exception(void) throw()
+ {
+ }
+private:
+ };
+}
+
+#endif
=====================================
mmap_file_pool.cpp
=====================================
@@ -1,20 +1,40 @@
#include "mmap_file_pool.h"
+#include "mmap_exception.h"
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
+#include <assert.h>
namespace mmap_allocator_namespace {
- off_t filesize(int fd)
+
+#ifndef MMAP_ALLOCATOR_DEBUG
+#define MMAP_ALLOCATOR_DEBUG 0
+#endif
+
+ int verbosity = MMAP_ALLOCATOR_DEBUG;
+
+ void set_verbosity(int v)
+ {
+ verbosity = v;
+ }
+
+ int get_verbosity(void)
+ {
+ return verbosity;
+ }
+
+ off_t filesize(int fd, std::string fname)
{
struct stat buf;
if (fstat(fd, &buf) < 0) {
-#ifdef MMAP_ALLOCATOR_DEBUG
- perror("stat");
-#endif
- throw mmap_allocator_exception("Cannot stat file");
+ if (get_verbosity() > 0) {
+ perror("stat");
+ }
+
+ throw mmap_allocator_exception("Cannot stat file" + fname);
}
return buf.st_size;
@@ -24,10 +44,10 @@ namespace mmap_allocator_namespace {
{
struct stat buf;
if (stat(fname.c_str(), &buf) < 0) {
-#ifdef MMAP_ALLOCATOR_DEBUG
- perror("stat");
-#endif
- throw mmap_allocator_exception("Cannot stat file");
+ if (get_verbosity() > 0) {
+ perror("stat");
+ }
+ throw mmap_allocator_exception("Cannot stat file "+fname);
}
device = buf.st_dev;
@@ -80,22 +100,23 @@ namespace mmap_allocator_namespace {
switch (access_mode) {
case READ_ONLY: mode = O_RDONLY; prot = PROT_READ; mmap_mode |= MAP_SHARED; break;
case READ_WRITE_SHARED: mode = O_RDWR; prot = PROT_READ | PROT_WRITE; mmap_mode |= MAP_SHARED; break;
- case READ_WRITE_PRIVATE: mode = O_RDWR; prot = PROT_READ | PROT_WRITE; mmap_mode |= MAP_PRIVATE; break;
+ case READ_WRITE_PRIVATE: mode = O_RDONLY; prot = PROT_READ | PROT_WRITE; mmap_mode |= MAP_PRIVATE; break;
default: throw mmap_allocator_exception("Internal error"); break;
}
if (fd == -1) {
fd = open(filename.c_str(), mode);
if (fd < 0) {
-#ifdef MMAP_ALLOCATOR_DEBUG
- perror("open");
-#endif
- throw mmap_allocator_exception("Error opening file");
+ if (get_verbosity() > 0) {
+ perror("open");
+ }
+
+ throw mmap_allocator_exception("Error opening file " + filename);
}
}
if (map_whole_file) {
offset_to_map = 0;
- length_to_map = filesize(fd);
+ length_to_map = filesize(fd, filename);
} else {
offset_to_map = ALIGN_TO_PAGE(offset);
length_to_map = UPPER_ALIGN_TO_PAGE(length);
@@ -103,45 +124,48 @@ namespace mmap_allocator_namespace {
if (offset_to_map == offset_mapped && length_to_map == size_mapped) {
reference_count++;
- return ((char*)memory_area)+offset;
+ return ((char*)memory_area)+offset-offset_mapped;
}
if (offset_to_map >= offset_mapped && length_to_map + offset_to_map - offset_mapped <= size_mapped)
{
reference_count++;
- return ((char*)memory_area)+offset;
+ return ((char*)memory_area)+offset-offset_mapped;
}
if (memory_area != NULL) {
if (munmap(memory_area, size_mapped) < 0) {
-#ifdef MMAP_ALLOCATOR_DEBUG
- perror("munmap");
-#endif
- throw mmap_allocator_exception("Error in munmap");
+ if (get_verbosity() > 0) {
+ perror("munmap");
+ }
+ throw mmap_allocator_exception("Error in munmap file "+filename);
}
}
memory_area = mmap(address_to_map, length_to_map, prot, mmap_mode, fd, offset_to_map);
if (address_to_map != NULL && !allow_remap && memory_area != MAP_FAILED && memory_area != address_to_map) {
if (munmap(memory_area, length_to_map) < 0) {
-#ifdef MMAP_ALLOCATOR_DEBUG
- perror("munmap");
-#endif
- throw mmap_allocator_exception("Error in munmap");
+ if (get_verbosity() > 0) {
+ perror("munmap");
+ }
+ throw mmap_allocator_exception("Error in munmap" + filename);
}
- throw mmap_allocator_exception("Request to remap area but allow_remap is not given");
+ throw mmap_allocator_exception("Request to remap area but allow_remap is not given (remapping "+filename+")");
}
if (memory_area == MAP_FAILED) {
-#ifdef MMAP_ALLOCATOR_DEBUG
- perror("mmap");
-#endif
- throw mmap_allocator_exception("Error in mmap");
+ if (get_verbosity() > 0) {
+ perror("mmap");
+ }
+ throw mmap_allocator_exception("Error in mmap "+filename);
}
offset_mapped = offset_to_map;
size_mapped = length_to_map;
reference_count++;
- return ((char*)memory_area)+offset;
+ void *ret = ((char*)memory_area)+offset-offset_to_map;
+ // assert(ret >= memory_area && ret < (char*)memory_area+size_mapped);
+
+ return ret;
}
bool mmapped_file::munmap_and_close_file(void)
@@ -151,17 +175,18 @@ namespace mmap_allocator_namespace {
return false;
}
if (munmap(memory_area, size_mapped) < 0) {
-#ifdef MMAP_ALLOCATOR_DEBUG
- perror("munmap");
-#endif
+ if (get_verbosity() > 0) {
+ perror("munmap");
+ }
throw mmap_allocator_exception("Error in munmap");
}
if (close(fd)) {
-#ifdef MMAP_ALLOCATOR_DEBUG
- perror("close");
-#endif
+ if (get_verbosity() > 0) {
+ perror("close");
+ }
throw mmap_allocator_exception("Error in close");
}
+ fd = -1;
return true;
}
@@ -194,7 +219,7 @@ namespace mmap_allocator_namespace {
the_map.erase(it);
}
} else {
- throw mmap_allocator_exception("File not found in pool");
+ throw mmap_allocator_exception("File "+fname+" not found in pool");
}
}
=====================================
mmap_file_pool.h
=====================================
@@ -40,6 +40,8 @@ public:
bool munmap_and_close_file(void);
private:
+ friend class mmap_file_pool;
+
int fd;
void *memory_area;
size_t size_mapped;
=====================================
mmappable_vector.h
=====================================
@@ -0,0 +1,137 @@
+#ifndef _MMAPPABLE_VECTOR_H
+#define _MMAPPABLE_VECTOR_H
+
+#include <memory>
+#include <string>
+#include <stdio.h>
+#include <vector>
+#include "mmap_allocator.h"
+
+namespace mmap_allocator_namespace {
+ template <typename T, typename A = mmap_allocator<T> >
+ class mmappable_vector: public std::vector<T, A> {
+public:
+ typedef std::vector<T, A> Base;
+
+ typedef typename Base::const_iterator const_iterator;
+ typedef typename Base::iterator iterator;
+ typedef T value_type;
+ typedef A allocator_type;
+
+ mmappable_vector():
+ Base()
+ {
+ }
+
+ mmappable_vector(const mmappable_vector<T, A> &other):
+ Base(other)
+ {
+ }
+
+ explicit mmappable_vector(size_t n):
+ Base()
+ {
+ mmap_file(n);
+ }
+
+ explicit mmappable_vector(A alloc):
+ Base(alloc)
+ {
+ }
+
+ mmappable_vector(iterator from, iterator to):
+ Base(from, to)
+ {
+ }
+
+ template <typename Iter>
+ mmappable_vector(Iter first, Iter last, A a = A()):
+ Base(first, last, a)
+ {
+ }
+
+ mmappable_vector(int n, T val, A alloc):
+ Base(n, val, alloc)
+ {
+ }
+
+ mmappable_vector(int n, T val):
+ Base(n, val)
+ {
+ }
+
+ mmappable_vector(std::vector<T,std::allocator<T> > v):
+ std::vector<T,std::allocator<T> >(v)
+ {
+ }
+
+/* Use this only when the allocator is already initialized. */
+ void mmap_file(size_t n)
+ {
+ Base::reserve(n);
+ _M_set_finish(n);
+ }
+
+ void mmap_file(std::string filename, enum access_mode access_mode, const off_t offset, const size_t n, int flags = 0)
+ {
+ if (Base::size() > 0) {
+ throw mmap_allocator_exception("Remapping currently not implemented.");
+ }
+#ifdef __GNUC__
+#if __GNUC__ == 3
+ A *the_allocator = static_cast<A*>(&(this->Base::_M_impl));
+#else
+ A *the_allocator = &Base::_M_get_Tp_allocator();
+#endif
+#else
+#error "Not GNU C++, please either implement me or use GCC"
+#endif
+ the_allocator->filename = filename;
+ the_allocator->offset = offset;
+ the_allocator->access_mode = access_mode;
+ the_allocator->map_whole_file = (flags & MAP_WHOLE_FILE) != 0;
+ the_allocator->allow_remap = (flags & ALLOW_REMAP) != 0;
+ the_allocator->bypass_file_pool = (flags & BYPASS_FILE_POOL) != 0;
+ the_allocator->keep_forever = (flags & KEEP_FOREVER) != 0;
+
+ mmap_file(n);
+ }
+
+ void munmap_file(void)
+ {
+ size_t n = Base::size();
+#ifdef __GNUC__
+ Base::_M_deallocate(Base::_M_impl._M_start, n);
+ Base::_M_impl._M_start = 0;
+ Base::_M_impl._M_finish = 0;
+ Base::_M_impl._M_end_of_storage = 0;
+#else
+#error "Not GNU C++, please either implement me or use GCC"
+#endif
+ }
+
+private:
+ void _M_set_finish(size_t n)
+ {
+#ifdef __GNUC__
+ Base::_M_impl._M_finish = Base::_M_impl._M_start + n;
+#else
+#error "Not GNU C++, please either implement me or use GCC"
+#endif
+ }
+ };
+
+ template <typename T>
+ std::vector<T> to_std_vector(const mmappable_vector<T> &v)
+ {
+ return std::vector<T>(v.begin(), v.end());
+ }
+
+ template <typename T>
+ mmappable_vector<T> to_mmappable_vector(const std::vector<T> &v)
+ {
+ return mmappable_vector<T>(v.begin(), v.end());
+ }
+}
+
+#endif /* MMAPPABLE_VECTOR_H */
=====================================
test_allocator.cpp
=====================================
@@ -1,5 +1,8 @@
#include "mmap_allocator.h"
+#include "mmap_exception.h"
+#include "mmappable_vector.h"
#include <stdio.h>
+#include <string.h>
#include <vector>
#include <assert.h>
#include <memory>
@@ -7,16 +10,20 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
+#include <sys/time.h>
using namespace std;
using namespace mmap_allocator_namespace;
-void generate_test_file(int count)
+#define TESTFILE "testfile"
+#define TESTFILE2 "testfile2"
+
+void generate_test_file(int count, const char *fname)
{
FILE *f;
int i;
- f = fopen("testfile", "w+");
+ f = fopen(fname, "w+");
for (i=0;i<count;i++) {
fwrite(&i, 1, sizeof(i), f);
}
@@ -28,9 +35,9 @@ void test_test_file(int count, bool expect_zeros)
FILE *f;
int i, j;
- f = fopen("testfile", "r");
+ f = fopen(TESTFILE, "r");
for (i=0;i<count;i++) {
- fread(&j, 1, sizeof(j), f);
+ size_t r = fread(&j, 1, sizeof(j), f);
assert(j == (expect_zeros ? 0 : i));
}
fclose(f);
@@ -60,13 +67,16 @@ void test_throw_catch(void)
{
try {
throw mmap_allocator_exception("Test");
+ assert(0);
} catch (mmap_allocator_exception e) {
- fprintf(stderr, "Exception message: %s\n", e.message());
+ fprintf(stderr, "Exception message (expected): %s\n", e.what());
+ assert(strcmp(e.what(), "Test") == 0);
}
try {
do_throw();
+ assert(0);
} catch (mmap_allocator_exception e) {
- fprintf(stderr, "Exception message: %s\n", e.message());
+ fprintf(stderr, "Exception message (expected): %s\n", e.what());
}
}
@@ -76,13 +86,14 @@ void test_exceptions(void)
{
bool exception_thrown;
- generate_test_file(1024);
+ generate_test_file(1024, TESTFILE);
exception_thrown = false;
try {
vector<int, mmap_allocator<int> > int_vec(1024, 0, mmap_allocator<int>("", READ_ONLY));
/* Default constructor used, allocate will fail */
+ assert(0);
} catch (mmap_allocator_exception e) {
- fprintf(stderr, "Exception message: %s\n", e.message());
+ fprintf(stderr, "Exception message (expected): %s\n", e.what());
exception_thrown = true;
}
assert(exception_thrown);
@@ -90,17 +101,19 @@ void test_exceptions(void)
exception_thrown = false;
try {
vector<int, mmap_allocator<int> > int_vec_notexsting_file(1024, 0, mmap_allocator<int>("karin", READ_ONLY)); /* no such file or directory */
+ assert(0);
} catch (mmap_allocator_exception e) {
- fprintf(stderr, "Exception message: %s\n", e.message());
+ fprintf(stderr, "Exception message (expected): %s\n", e.what());
exception_thrown = true;
}
assert(exception_thrown);
exception_thrown = false;
try {
- vector<int, mmap_allocator<int> > int_vec_wrong_alignment_file(512, 0, mmap_allocator<int>("testfile", READ_WRITE_PRIVATE, 123)); /* wrong alignment */
+ vector<int, mmap_allocator<int> > int_vec_wrong_alignment_file(512, 0, mmap_allocator<int>(TESTFILE, READ_WRITE_PRIVATE, 123)); /* wrong alignment */
+ /* No exception here expected */
} catch (mmap_allocator_exception &e) {
- fprintf(stderr, "Exception message: %s\n", e.message());
+ fprintf(stderr, "Exception message (not expected): %s\n", e.what());
exception_thrown = true;
}
assert(!exception_thrown);
@@ -111,9 +124,9 @@ void test_mmap(void)
int i;
fprintf(stderr, "Testing int_vec_default\n");
- generate_test_file(1024);
- mmappable_vector<int, mmap_allocator<int> > int_vec_default = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", DEFAULT_STL_ALLOCATOR, 0));
- int_vec_default.map_into_memory(1024);
+ generate_test_file(1024, TESTFILE);
+ mmappable_vector<int, mmap_allocator<int> > int_vec_default = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, DEFAULT_STL_ALLOCATOR, 0));
+ int_vec_default.mmap_file(1024);
assert(int_vec_default.size() == 1024);
for (i=0;i<1024;i++) {
int_vec_default[i] = i; /* no segfault */
@@ -122,33 +135,33 @@ void test_mmap(void)
test_test_file(1024, false);
fprintf(stderr, "Testing int_vec_rw_private\n");
- generate_test_file(1024);
- mmappable_vector<int, mmap_allocator<int> > int_vec_rw_private = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", READ_WRITE_PRIVATE, 0));
- int_vec_rw_private.map_into_memory(1024);
+ generate_test_file(1024, TESTFILE);
+ mmappable_vector<int, mmap_allocator<int> > int_vec_rw_private = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_WRITE_PRIVATE, 0));
+ int_vec_rw_private.mmap_file(1024);
for (i=0;i<1024;i++) {
assert(int_vec_rw_private[i] == i);
}
test_test_file(1024, false);
fprintf(stderr, "Testing int_vec_ro\n");
- mmappable_vector<int, mmap_allocator<int> > int_vec_ro = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", READ_ONLY, 0));
- int_vec_ro.map_into_memory(1024);
+ mmappable_vector<int, mmap_allocator<int> > int_vec_ro = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_ONLY, 0));
+ int_vec_ro.mmap_file(1024);
for (i=0;i<1024;i++) {
assert(int_vec_ro[i] == i);
}
test_test_file(1024, false);
fprintf(stderr, "Testing int_vec_shifted\n");
- mmappable_vector<int, mmap_allocator<int> > int_vec_shifted = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", READ_ONLY, sizeof(int)));
- int_vec_shifted.map_into_memory(1024-1);
+ mmappable_vector<int, mmap_allocator<int> > int_vec_shifted = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_ONLY, sizeof(int)));
+ int_vec_shifted.mmap_file(1024-1);
for (i=0;i<1024-1;i++) {
assert(int_vec_shifted[i] == i+1);
}
test_test_file(1024, false);
fprintf(stderr, "Testing int_vec_pointer\n");
- mmappable_vector<int, mmap_allocator<int> > *int_vec_pointer = new mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", READ_ONLY, 0));
- int_vec_pointer->map_into_memory(1024);
+ mmappable_vector<int, mmap_allocator<int> > *int_vec_pointer = new mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_ONLY, 0));
+ int_vec_pointer->mmap_file(1024);
for (i=0;i<1024;i++) {
assert((*int_vec_pointer)[i] == i);
}
@@ -156,23 +169,23 @@ void test_mmap(void)
delete int_vec_pointer;
fprintf(stderr, "Testing int_vec_initialized_private\n");
- mmappable_vector<int, mmap_allocator<int> > int_vec_initialized_private = mmappable_vector<int, mmap_allocator<int> >(1024, 0, mmap_allocator<int>("testfile", READ_WRITE_PRIVATE, 0));
+ mmappable_vector<int, mmap_allocator<int> > int_vec_initialized_private = mmappable_vector<int, mmap_allocator<int> >(1024, 0, mmap_allocator<int>(TESTFILE, READ_WRITE_PRIVATE, 0));
for (i=0;i<1024;i++) {
assert(int_vec_initialized_private[i] == 0);
}
test_test_file(1024, false);
fprintf(stderr, "Testing int_vec_initialized_shared\n");
- mmappable_vector<int, mmap_allocator<int> > int_vec_initialized_shared = mmappable_vector<int, mmap_allocator<int> >(1024, 0, mmap_allocator<int>("testfile", READ_WRITE_SHARED, 0));
+ mmappable_vector<int, mmap_allocator<int> > int_vec_initialized_shared = mmappable_vector<int, mmap_allocator<int> >(1024, 0, mmap_allocator<int>(TESTFILE, READ_WRITE_SHARED, 0));
for (i=0;i<1024;i++) {
assert(int_vec_initialized_shared[i] == 0);
}
test_test_file(1024, true);
fprintf(stderr, "Testing int_vec_big\n");
- generate_test_file(1024*1024);
- mmappable_vector<int, mmap_allocator<int> > int_vec_big = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", READ_ONLY, 0, true, true));
- int_vec_big.map_into_memory(1024*1024);
+ generate_test_file(1024*1024, TESTFILE);
+ mmappable_vector<int, mmap_allocator<int> > int_vec_big = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_ONLY, 0, MAP_WHOLE_FILE | ALLOW_REMAP));
+ int_vec_big.mmap_file(1024*1024);
for (i=0;i<1024*1024;i++) {
if (int_vec_big[i] != i) { fprintf(stderr, "falsch: i=%d val=%d\n", i, int_vec_big[i]); }
assert(int_vec_big[i] == i);
@@ -180,18 +193,18 @@ if (int_vec_big[i] != i) { fprintf(stderr, "falsch: i=%d val=%d\n", i, int_vec_b
test_test_file(1024*1024, false);
fprintf(stderr, "Testing int_vec_shifted_big\n");
- generate_test_file(1024*1024);
- mmappable_vector<int, mmap_allocator<int> > int_vec_shifted_big = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", READ_ONLY, sizeof(i), true, true));
- int_vec_shifted_big.map_into_memory(1024*1024-1);
+ generate_test_file(1024*1024, TESTFILE);
+ mmappable_vector<int, mmap_allocator<int> > int_vec_shifted_big = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_ONLY, sizeof(i), MAP_WHOLE_FILE | ALLOW_REMAP));
+ int_vec_shifted_big.mmap_file(1024*1024-1);
for (i=0;i<1024*1024-1;i++) {
assert(int_vec_shifted_big[i] == i+1);
}
test_test_file(1024*1024, false);
fprintf(stderr, "Testing int_vec_big_minus_one\n");
- generate_test_file(1024*1024-1);
- mmappable_vector<int, mmap_allocator<int> > int_vec_big_minus_one = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", READ_ONLY, 0, true, true));
- int_vec_big_minus_one.map_into_memory(1024*1024-1);
+ generate_test_file(1024*1024-1, TESTFILE);
+ mmappable_vector<int, mmap_allocator<int> > int_vec_big_minus_one = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_ONLY, 0, MAP_WHOLE_FILE | ALLOW_REMAP));
+ int_vec_big_minus_one.mmap_file(1024*1024-1);
for (i=0;i<1024*1024-1;i++) {
assert(int_vec_big_minus_one[i] == i);
}
@@ -200,20 +213,37 @@ if (int_vec_big[i] != i) { fprintf(stderr, "falsch: i=%d val=%d\n", i, int_vec_b
void test_conversion(void)
{
- vector<int, mmap_allocator<int> > mmap_vector = vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", DEFAULT_STL_ALLOCATOR, 1000));
-
+ mmappable_vector<int> mmap_vector;
vector<int> std_vector;
+ mmappable_vector<int> mmap_vector2;
+ int i;
+
+ fprintf(stderr, "Testing conversion between STL vector and mmap vector.\n");
+ generate_test_file(1024, TESTFILE);
+ mmap_vector.mmap_file(TESTFILE, READ_ONLY, 0, 1024);
+ for (i=0;i<1024;i++) {
+ assert(mmap_vector[i] == i);
+ }
std_vector = to_std_vector(mmap_vector);
- mmap_vector = to_mmap_vector(std_vector);
+ for (i=0;i<1024;i++) {
+ assert(std_vector[i] == i);
+ }
+ for (i=0;i<1024;i++) {
+ std_vector[i] *= 2;
+ }
+ mmap_vector2 = to_mmappable_vector(std_vector);
+ for (i=0;i<1024;i++) {
+ assert(mmap_vector2[i] == i*2);
+ }
}
void test_mmap_file_pool(void)
{
- generate_test_file(1024);
- int *f = (int*)the_pool.mmap_file(string("testfile"), READ_ONLY, 0, 1024, false, false);
- int *f2 = (int*)the_pool.mmap_file(string("testfile"), READ_ONLY, 0, 1024, false, false);
+ generate_test_file(1024, TESTFILE);
+ int *f = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 1024, false, false);
+ int *f2 = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 1024, false, false);
int i;
assert(f == f2);
@@ -221,43 +251,43 @@ void test_mmap_file_pool(void)
for (i=0;i<1024;i++) {
assert(f[i] == i);
}
- the_pool.munmap_file(string("testfile"), READ_ONLY, 0, 1024);
- the_pool.munmap_file(string("testfile"), READ_ONLY, 0, 1024);
+ the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 1024);
+ the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 1024);
}
void test_mapping_smaller_area(void)
{
fprintf(stderr, "Testing mapping of areas that fit in already mapped areas\n");
- generate_test_file(2048);
+ generate_test_file(2048, TESTFILE);
- int *f = (int*)the_pool.mmap_file(string("testfile"), READ_ONLY, 0, 8192, false, false);
- int *first_page = (int*)the_pool.mmap_file(string("testfile"), READ_ONLY, 0, 4096, false, false);
- int *second_page = (int*)the_pool.mmap_file(string("testfile"), READ_ONLY, 4096, 4096, false, false);
+ int *f = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 8192, false, false);
+ int *first_page = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 4096, false, false);
+ int *second_page = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 4096, 4096, false, false);
assert(f == first_page);
assert(f+1024 == second_page);
- the_pool.munmap_file(string("testfile"), READ_ONLY, 0, 8192);
- the_pool.munmap_file(string("testfile"), READ_ONLY, 0, 4096);
- the_pool.munmap_file(string("testfile"), READ_ONLY, 4096, 4096);
+ the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 8192);
+ the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 4096);
+ the_pool.munmap_file(string(TESTFILE), READ_ONLY, 4096, 4096);
}
void test_mapping_smaller_area_whole_file_flag(void)
{
fprintf(stderr, "Testing whole file flag\n");
- generate_test_file(2048);
+ generate_test_file(2048, TESTFILE);
- int *f = (int*)the_pool.mmap_file(string("testfile"), READ_ONLY, 0, 1, true, false);
- int *first_page = (int*)the_pool.mmap_file(string("testfile"), READ_ONLY, 0, 4096, false, false);
- int *second_page = (int*)the_pool.mmap_file(string("testfile"), READ_ONLY, 4096, 4096, false, false);
+ int *f = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 1, true, false);
+ int *first_page = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 0, 4096, false, false);
+ int *second_page = (int*)the_pool.mmap_file(string(TESTFILE), READ_ONLY, 4096, 4096, false, false);
assert(f == first_page);
assert(f+1024 == second_page);
- the_pool.munmap_file(string("testfile"), READ_ONLY, 0, 8192);
- the_pool.munmap_file(string("testfile"), READ_ONLY, 0, 4096);
- the_pool.munmap_file(string("testfile"), READ_ONLY, 4096, 4096);
+ the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 8192);
+ the_pool.munmap_file(string(TESTFILE), READ_ONLY, 0, 4096);
+ the_pool.munmap_file(string(TESTFILE), READ_ONLY, 4096, 4096);
}
@@ -266,32 +296,214 @@ void test_mapping_smaller_area_whole_file_flag_allocator(void)
int i;
fprintf(stderr, "Testing whole file flag via allocator\n");
- generate_test_file(2048);
+ generate_test_file(2048, TESTFILE);
fprintf(stderr, "Testing int_vec_ro\n");
- mmappable_vector<int, mmap_allocator<int> > int_vec_ro = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", READ_ONLY, 0, true, false));
- int_vec_ro.map_into_memory(1024);
+ mmappable_vector<int, mmap_allocator<int> > int_vec_ro = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_ONLY, 0, MAP_WHOLE_FILE));
+ int_vec_ro.mmap_file(1024);
for (i=0;i<1024;i++) {
assert(int_vec_ro[i] == i);
}
- mmappable_vector<int, mmap_allocator<int> > int_vec_ro_second_page = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>("testfile", READ_ONLY, 4096, true, false));
- int_vec_ro_second_page.map_into_memory(1024);
+ mmappable_vector<int, mmap_allocator<int> > int_vec_ro_second_page = mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_ONLY, 4096, MAP_WHOLE_FILE));
+ int_vec_ro_second_page.mmap_file(1024);
for (i=0;i<1024;i++) {
assert(int_vec_ro_second_page[i] == i+1024);
}
}
+void test_mapping_smaller_area_whole_file_flag_allocator_deleting_first_vec(void)
+{
+ int i;
+
+ fprintf(stderr, "Testing whole file flag via allocator, deleting first mapping before allocating second page\n");
+ generate_test_file(2048, TESTFILE);
+
+ fprintf(stderr, "Testing int_vec_ro\n");
+ mmappable_vector<int, mmap_allocator<int> > *int_vec_ro_p = new mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_ONLY, 0, MAP_WHOLE_FILE));
+ int_vec_ro_p->mmap_file(1024);
+ for (i=0;i<1024;i++) {
+ assert((*int_vec_ro_p)[i] == i);
+ }
+ delete int_vec_ro_p;
+
+ mmappable_vector<int, mmap_allocator<int> > *int_vec_ro_second_page_p = new mmappable_vector<int, mmap_allocator<int> >(mmap_allocator<int>(TESTFILE, READ_ONLY, 4096, MAP_WHOLE_FILE));
+ int_vec_ro_second_page_p->mmap_file(1024);
+ for (i=0;i<1024;i++) {
+ assert((*int_vec_ro_second_page_p)[i] == i+1024);
+ }
+ delete int_vec_ro_second_page_p;
+}
+
+
+void test_new_interface(void)
+{
+ int i;
+
+ fprintf(stderr, "Testing new interface\n");
+
+ generate_test_file(1024, TESTFILE);
+
+ mmappable_vector<int> vec;
+ vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024);
+
+ for (i=0;i<1024;i++) {
+ assert(vec[i] == i);
+ }
+ try {
+ vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024);
+ assert(0);
+ } catch (mmap_allocator_exception e) {
+ fprintf(stderr, "Exception message (expected): %s\n", e.what());
+ }
+ vec.munmap_file();
+
+ generate_test_file(2048, TESTFILE);
+ vec.mmap_file(TESTFILE, READ_ONLY, 4096, 1024);
+ for (i=0;i<1024;i++) {
+ assert(vec[i] == i+1024);
+ }
+}
+
+void test_cache_bug(void)
+{
+ mmappable_vector<int> vec;
+ int i;
+
+ fprintf(stderr, "Testing if wrong offset bug in pool is fixed.\n");
+ generate_test_file(2048, TESTFILE);
+ vec.mmap_file(TESTFILE, READ_ONLY, 4096, 1024);
+
+ for (i=0;i<1024;i++) {
+ assert(vec[i] == i+1024);
+ }
+}
+
+void test_private_file_pool(void)
+{
+ mmappable_vector<int> vec;
+ mmappable_vector<int> vec2;
+ int i;
+
+ fprintf(stderr, "Testing if bypass_file_pool works.\n");
+ generate_test_file(1024, TESTFILE);
+ vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024, BYPASS_FILE_POOL);
+ for (i=0;i<1024;i++) {
+ assert(vec[i] == i);
+ }
+
+ vec2.mmap_file(TESTFILE, READ_ONLY, 0, 1024, BYPASS_FILE_POOL);
+ for (i=0;i<1024;i++) {
+ assert(vec[i] == i);
+ }
+ assert(&vec[0] != &vec2[0]);
+}
+
+#define FILESIZE (1024*1024*16)
+
+void read_large_file(enum access_mode mode)
+{
+ struct timeval t, t2;
+ mmappable_vector<int> vec;
+ int i;
+
+ gettimeofday(&t, NULL);
+
+ vec.mmap_file(TESTFILE, mode, 0, FILESIZE);
+ for (i=0;i<FILESIZE;i++) {
+ assert(vec[i] == i);
+ }
+ gettimeofday(&t2, NULL);
+ fprintf(stderr, "Mode: %d Time: %lu.%06lu\n", mode, (t2.tv_sec - t.tv_sec)-(t2.tv_usec < t.tv_usec), (t2.tv_usec < t.tv_usec)*1000000 + (t2.tv_usec - t.tv_usec));
+}
+
+void test_large_file(void)
+{
+ fprintf(stderr, "Testing large file.\n");
+ generate_test_file(FILESIZE, TESTFILE); /* 1G */
+
+ read_large_file(READ_ONLY);
+ read_large_file(READ_WRITE_PRIVATE);
+ read_large_file(READ_WRITE_SHARED);
+}
+
+void test_multiple_open(void)
+{
+ generate_test_file(1024, TESTFILE);
+ generate_test_file(1024, TESTFILE2);
+ mmappable_vector<int> vec1, vec2, vec3, vec4;
+
+ fprintf(stderr, "Testing multiple open (you need to strace this).\n");
+ vec1.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE);
+ vec2.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE);
+ vec3.mmap_file(TESTFILE2, READ_ONLY, 0, 1024, MAP_WHOLE_FILE);
+ vec4.mmap_file(TESTFILE2, READ_ONLY, 0, 1024, MAP_WHOLE_FILE);
+}
+
+void test_keep_forever(void)
+{
+ generate_test_file(1024, TESTFILE);
+ mmappable_vector<int> vec1, vec2, vec3, vec4;
+
+ fprintf(stderr, "Testing multiple open (you need to strace this).\n");
+ {
+ mmappable_vector<int> vec;
+ fprintf(stderr, "Testing mapping without KEEP_FOREVER (you need to strace this: there should be a close).\n");
+ vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE);
+ }
+ {
+ mmappable_vector<int> vec;
+ fprintf(stderr, "Testing mapping without KEEP_FOREVER (you need to strace this: the file should be reopened).\n");
+ vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE);
+ }
+ {
+ mmappable_vector<int> vec;
+ fprintf(stderr, "Testing mapping with KEEP_FOREVER (you need to strace this: there should be NO close).\n");
+ vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE | KEEP_FOREVER);
+ }
+ {
+ mmappable_vector<int> vec;
+ fprintf(stderr, "Testing mapping with KEEP_FOREVER (you need to strace this: the file shouldn't be reopened).\n");
+ vec.mmap_file(TESTFILE, READ_ONLY, 0, 1024, MAP_WHOLE_FILE | KEEP_FOREVER);
+ }
+}
+
+
+void test_allocate_0_bytes(void) /* shouldn't segfault */
+{
+ fprintf(stderr, "Testing vectors of mmappable_vectors.\n");
+
+ vector<mmappable_vector<int> > vecs;
+ vecs.resize(2);
+ for (int i=0; i<2; i++) {
+ vecs[i].mmap_file(TESTFILE, READ_ONLY, 0, 1024, 0);
+ for (int j=0;j<1024;j++) {
+ assert(vecs[i][j] == j);
+ }
+ }
+}
+
+
int main(int argc, char ** argv)
{
+ set_verbosity(1);
+
test_page_align_macros();
test_throw_catch();
test_exceptions();
test_mmap_file_pool();
test_mmap();
test_conversion();
+ test_cache_bug();
test_mapping_smaller_area();
test_mapping_smaller_area_whole_file_flag();
test_mapping_smaller_area_whole_file_flag_allocator();
+ test_mapping_smaller_area_whole_file_flag_allocator_deleting_first_vec();
+ test_new_interface();
+ test_private_file_pool();
+ test_large_file();
+ test_multiple_open();
+ test_keep_forever();
+ test_allocate_0_bytes();
}
View it on GitLab: https://salsa.debian.org/med-team/libmmap-allocator/-/commit/2a1e5b8182e4b9bdd02469d0f32984ccab55093a
--
View it on GitLab: https://salsa.debian.org/med-team/libmmap-allocator/-/commit/2a1e5b8182e4b9bdd02469d0f32984ccab55093a
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/debian-med-commit/attachments/20200607/93920655/attachment-0001.html>
More information about the debian-med-commit
mailing list