[libosmium] 01/04: Imported Upstream version 2.5.0
Sebastiaan Couwenberg
sebastic at moszumanska.debian.org
Thu Nov 5 23:15:22 UTC 2015
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository libosmium.
commit 8d8c25e11844f2958e5bd330ed9be003a78e87c1
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Thu Nov 5 23:40:15 2015 +0100
Imported Upstream version 2.5.0
---
.travis.yml | 181 ++++--
CHANGELOG.md | 22 +
CMakeLists.txt | 6 +-
README.md | 2 +-
cmake/FindOsmium.cmake | 13 +-
cmake/iwyu.sh | 2 +-
examples/CMakeLists.txt | 1 +
examples/osmium_filter_discussions.cpp | 72 +++
include/osmium/area/assembler.hpp | 4 +
include/osmium/area/detail/node_ref_segment.hpp | 6 +-
include/osmium/area/detail/proto_ring.hpp | 9 +-
include/osmium/area/detail/segment_list.hpp | 2 +
include/osmium/area/multipolygon_collector.hpp | 8 +-
include/osmium/area/problem_reporter_ogr.hpp | 2 +-
include/osmium/builder/osm_object_builder.hpp | 62 +-
include/osmium/experimental/flex_reader.hpp | 14 +-
include/osmium/geom/coordinates.hpp | 1 -
include/osmium/geom/factory.hpp | 2 +-
include/osmium/geom/geojson.hpp | 10 +-
include/osmium/geom/tile.hpp | 6 +-
include/osmium/geom/wkb.hpp | 8 +-
include/osmium/geom/wkt.hpp | 10 +-
include/osmium/handler.hpp | 4 +
include/osmium/index/bool_vector.hpp | 4 +-
include/osmium/index/detail/mmap_vector_anon.hpp | 2 +
include/osmium/index/detail/mmap_vector_base.hpp | 6 +-
include/osmium/index/detail/mmap_vector_file.hpp | 8 +-
include/osmium/index/detail/vector_map.hpp | 2 +-
include/osmium/index/detail/vector_multimap.hpp | 2 +-
include/osmium/index/map.hpp | 21 +-
include/osmium/index/map/dummy.hpp | 2 +-
include/osmium/index/map/sparse_mem_map.hpp | 15 +-
include/osmium/index/map/sparse_mem_table.hpp | 3 +-
include/osmium/index/multimap/hybrid.hpp | 12 +-
.../osmium/index/multimap/sparse_mem_multimap.hpp | 2 +-
include/osmium/io/any_input.hpp | 1 +
include/osmium/io/bzip2_compression.hpp | 67 ++-
include/osmium/io/compression.hpp | 91 ++-
include/osmium/io/detail/debug_output_format.hpp | 225 ++++----
include/osmium/io/detail/input_format.hpp | 146 +++--
include/osmium/io/detail/o5m_input_format.hpp | 633 +++++++++++++++++++++
include/osmium/io/detail/opl_output_format.hpp | 135 ++---
include/osmium/io/detail/output_format.hpp | 62 +-
include/osmium/io/detail/pbf.hpp | 1 +
include/osmium/io/detail/pbf_decoder.hpp | 26 +-
include/osmium/io/detail/pbf_input_format.hpp | 183 ++----
include/osmium/io/detail/pbf_output_format.hpp | 298 +++++-----
include/osmium/io/detail/queue_util.hpp | 157 +++++
include/osmium/io/detail/read_thread.hpp | 95 ++--
include/osmium/io/detail/read_write.hpp | 17 +
include/osmium/io/detail/string_table.hpp | 1 +
include/osmium/io/detail/string_util.hpp | 206 +++++++
include/osmium/io/detail/write_thread.hpp | 63 +-
include/osmium/io/detail/xml_input_format.hpp | 540 ++++++++----------
include/osmium/io/detail/xml_output_format.hpp | 275 ++++-----
include/osmium/io/detail/zlib.hpp | 5 +-
include/osmium/io/error.hpp | 12 +
include/osmium/io/file.hpp | 5 +-
include/osmium/io/gzip_compression.hpp | 62 +-
include/osmium/io/input_iterator.hpp | 38 ++
include/osmium/io/{overwrite.hpp => o5m_input.hpp} | 27 +-
include/osmium/io/output_iterator.hpp | 72 ++-
include/osmium/io/overwrite.hpp | 10 +-
include/osmium/io/reader.hpp | 164 ++++--
include/osmium/io/writer.hpp | 252 +++++++-
include/osmium/memory/buffer.hpp | 67 ++-
include/osmium/osm/area.hpp | 1 +
include/osmium/osm/changeset.hpp | 131 ++++-
include/osmium/osm/crc.hpp | 11 +
include/osmium/osm/item_type.hpp | 9 +-
include/osmium/osm/node_ref_list.hpp | 2 +-
include/osmium/osm/object.hpp | 2 +-
include/osmium/osm/timestamp.hpp | 8 +
include/osmium/osm/types.hpp | 1 +
include/osmium/osm/types_from_string.hpp | 16 +-
include/osmium/relations/collector.hpp | 8 +-
include/osmium/thread/function_wrapper.hpp | 28 +-
include/osmium/thread/pool.hpp | 21 +-
include/osmium/thread/util.hpp | 33 +-
include/osmium/util/data_file.hpp | 2 +-
include/osmium/util/delta.hpp | 69 ++-
include/osmium/util/file.hpp | 6 +-
include/osmium/util/memory_mapping.hpp | 4 +-
include/osmium/util/options.hpp | 10 +
include/osmium/visitor.hpp | 3 +
include/protozero/byteswap.hpp | 32 +-
include/protozero/pbf_builder.hpp | 28 +-
include/protozero/pbf_message.hpp | 44 ++
include/protozero/pbf_reader.hpp | 12 +-
include/protozero/pbf_writer.hpp | 4 +-
include/protozero/version.hpp | 6 +-
scripts/travis_install.sh | 20 -
scripts/travis_script.sh | 36 --
test/CMakeLists.txt | 9 +-
test/data-tests/testdata-xml.cpp | 35 +-
test/t/basic/test_changeset.cpp | 62 +-
test/t/buffer/test_buffer_basics.cpp | 34 ++
test/t/geom/test_tile_data.hpp | 4 +-
test/t/io/deleted_nodes.osh | 5 +
test/t/io/deleted_nodes.osh.pbf | Bin 0 -> 189 bytes
test/t/io/test_output_utils.cpp | 153 +++++
test/t/io/test_reader.cpp | 105 +++-
test/t/io/test_reader_with_mock_decompression.cpp | 145 +++++
test/t/io/test_reader_with_mock_parser.cpp | 123 ++++
test/t/io/test_string_table.cpp | 6 +-
test/t/io/test_writer.cpp | 117 ++++
test/t/io/test_writer_with_mock_compression.cpp | 99 ++++
test/t/io/test_writer_with_mock_encoder.cpp | 105 ++++
test/t/thread/test_pool.cpp | 33 +-
test/t/util/test_data_file.cpp | 2 +-
test/t/util/test_delta.cpp | 34 +-
test/t/util/test_file.cpp | 2 +
test/t/util/test_options.cpp | 19 +-
113 files changed, 4575 insertions(+), 1526 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 6ebdd71..ac0d270 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,46 +9,151 @@ language: cpp
sudo: false
matrix:
- include:
- - os: linux
- compiler: clang
- env: BUILD_TYPE=Dev
- - os: linux
- compiler: clang
- env: BUILD_TYPE=Release
- - os: linux
- compiler: gcc
- env: BUILD_TYPE=Dev
- - os: linux
- compiler: gcc
- env: BUILD_TYPE=Release
- - os: osx
- compiler: clang
- env: BUILD_TYPE=Dev
- - os: osx
- compiler: clang
- env: BUILD_TYPE=Release
-
-# http://docs.travis-ci.com/user/apt/
-addons:
- apt:
- sources:
- - boost-latest
- - ubuntu-toolchain-r-test
- packages:
- - g++-4.8
- - gcc-4.8
- - libboost1.55-dev
- - libboost-program-options1.55-dev
- - libgdal-dev
- - libgeos++-dev
- - libproj-dev
- - libsparsehash-dev
- - spatialite-bin
+ include:
+
+ # 1/ Linux Clang Builds
+ - os: linux
+ compiler: clang
+ addons:
+ apt:
+ sources: ['llvm-toolchain-precise-3.5', 'ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['clang-3.5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='clang++-3.5' BUILD_TYPE='Release'
+
+ - os: linux
+ compiler: clang
+ addons:
+ apt:
+ sources: ['llvm-toolchain-precise-3.5', 'ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['clang-3.5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='clang++-3.5' BUILD_TYPE='Dev'
+
+
+ - os: linux
+ compiler: clang
+ addons:
+ apt:
+ sources: ['llvm-toolchain-precise-3.6', 'ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['clang-3.6', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='clang++-3.6' BUILD_TYPE='Release'
+
+ - os: linux
+ compiler: clang
+ addons:
+ apt:
+ sources: ['llvm-toolchain-precise-3.6', 'ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['clang-3.6', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='clang++-3.6' BUILD_TYPE='Dev'
+
+
+ - os: linux
+ compiler: clang
+ addons:
+ apt:
+ sources: ['llvm-toolchain-precise-3.7', 'ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['clang-3.7', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='clang++-3.7' BUILD_TYPE='Release'
+
+ - os: linux
+ compiler: clang
+ addons:
+ apt:
+ sources: ['llvm-toolchain-precise-3.7', 'ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['clang-3.7', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='clang++-3.7' BUILD_TYPE='Dev'
+
+
+ # 2/ Linux GCC Builds
+ - os: linux
+ compiler: gcc
+ addons:
+ apt:
+ sources: ['ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['g++-4.8', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='g++-4.8' COMPILER_FLAGS='-Wno-return-type' BUILD_TYPE='Release'
+
+ - os: linux
+ compiler: gcc
+ addons:
+ apt:
+ sources: ['ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['g++-4.8', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='g++-4.8' COMPILER_FLAGS='-Wno-return-type' BUILD_TYPE='Dev'
+
+
+ - os: linux
+ compiler: gcc
+ addons:
+ apt:
+ sources: ['ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['g++-4.9', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='g++-4.9' BUILD_TYPE='Release'
+
+ - os: linux
+ compiler: gcc
+ addons:
+ apt:
+ sources: ['ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['g++-4.9', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='g++-4.9' BUILD_TYPE='Dev'
+
+
+ - os: linux
+ compiler: gcc
+ addons:
+ apt:
+ sources: ['ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['g++-5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='g++-5' BUILD_TYPE='Release'
+
+ - os: linux
+ compiler: gcc
+ addons:
+ apt:
+ sources: ['ubuntu-toolchain-r-test', 'boost-latest']
+ packages: ['g++-5', 'libboost1.55-all-dev', 'libgdal-dev', 'libgeos++-dev', 'libproj-dev', 'libsparsehash-dev', 'spatialite-bin']
+ env: COMPILER='g++-5' BUILD_TYPE='Dev'
+
+
+ # 3/ OSX Clang Builds
+ - os: osx
+ osx_image: xcode6.4
+ compiler: clang
+ env: COMPILER='clang++' BUILD_TYPE='Dev'
+
+ - os: osx
+ osx_image: xcode6.4
+ compiler: clang
+ env: COMPILER='clang++' BUILD_TYPE='Release'
+
+
+ - os: osx
+ osx_image: xcode7
+ compiler: clang
+ env: COMPILER='clang++' BUILD_TYPE='Dev'
+
+ - os: osx
+ osx_image: xcode7
+ compiler: clang
+ env: COMPILER='clang++' BUILD_TYPE='Release'
+
install:
- - scripts/travis_install.sh
+ - DEPS_DIR="${TRAVIS_BUILD_DIR}/deps"
+ - mkdir -p ${DEPS_DIR} && cd ${DEPS_DIR}
+ - git clone --quiet --depth 1 https://github.com/osmcode/osm-testdata.git
+ - |
+ if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then
+ brew remove gdal
+ brew install cmake boost google-sparsehash gdal
+ fi
+
+before_script:
+ - cd ${TRAVIS_BUILD_DIR}
+ - mkdir build && cd build
+ - CXX=${COMPILER} CXXFLAGS=${COMPILER_FLAGS} cmake -LA .. -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DOSM_TESTDATA="${TRAVIS_BUILD_DIR}/deps/osm-testdata"
script:
- - scripts/travis_script.sh
+ - make VERBOSE=1
+ - ctest --output-on-failure
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2ce0c5c..d5081c6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,10 +8,32 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Added
+- Helper functions to make input iterator ranges and output iterators.
+- Add support for reading o5m and o5c files.
+- Option for osmium::io::Writer to fsync file after writing.
+- Lots of internal asserts() and other robustness checks.
+
### Changed
+- Updated included protozero library to version 1.2.0.
+- Complete overhaul of the I/O system making it much more robust against
+ wrong data and failures during I/O operations.
+- Speed up PBF writing by running parts of it in parallel.
+- OutputIterator doesn't hold an internal buffer any more, but it uses
+ one in Writer. Calling flush() on the OutputIterator isn't needed any
+ more.
+- Reader now throws when trying to read after eof or an error.
+- I/O functions that used to throw std::runtime_error now throw
+ osmium::io_error or derived.
+- Optional parameters on osmium::io::Writer now work in any order.
+
### Fixed
+- PBF reader now decodes locations of invisible nodes properly.
+- Invalid Delta encode iterator dereference.
+- Lots of includes fixed to include (only) what's used.
+- Dangling reference in area assembly code.
+
## [2.4.1] - 2015-08-29
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 129029f..da60fad 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -26,8 +26,8 @@ set(CMAKE_CONFIGURATION_TYPES "Debug;Release;RelWithDebInfo;MinSizeRel;Dev;Cover
project(libosmium)
set(LIBOSMIUM_VERSION_MAJOR 2)
-set(LIBOSMIUM_VERSION_MINOR 4)
-set(LIBOSMIUM_VERSION_PATCH 1)
+set(LIBOSMIUM_VERSION_MINOR 5)
+set(LIBOSMIUM_VERSION_PATCH 0)
set(LIBOSMIUM_VERSION
"${LIBOSMIUM_VERSION_MAJOR}.${LIBOSMIUM_VERSION_MINOR}.${LIBOSMIUM_VERSION_PATCH}")
@@ -264,7 +264,7 @@ find_program(CPPCHECK cppcheck)
if(CPPCHECK)
message(STATUS "Looking for cppcheck - found")
set(CPPCHECK_OPTIONS
- --enable=warning,style,performance,portability,information,missingInclude)
+ --enable=warning,style,performance,portability,information,missingInclude --force -Uassert)
# cpp doesn't find system includes for some reason, suppress that report
set(CPPCHECK_OPTIONS ${CPPCHECK_OPTIONS} --suppress=missingIncludeSystem)
diff --git a/README.md b/README.md
index 9676d80..9ac5a70 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ http://osmcode.org/libosmium
A fast and flexible C++ library for working with OpenStreetMap data.
-[](http://travis-ci.org/osmcode/libosmium)
+[](https://travis-ci.org/osmcode/libosmium)
[](https://ci.appveyor.com/project/Mapbox/libosmium)
Libosmium is developed on Linux, but also works on OSX and Windows (with some
diff --git a/cmake/FindOsmium.cmake b/cmake/FindOsmium.cmake
index b3a4c95..fba8ffb 100644
--- a/cmake/FindOsmium.cmake
+++ b/cmake/FindOsmium.cmake
@@ -62,6 +62,8 @@ find_path(OSMIUM_INCLUDE_DIR osmium/osm.hpp
/opt
)
+set(OSMIUM_INCLUDE_DIRS "${OSMIUM_INCLUDE_DIR}")
+
#----------------------------------------------------------------------
#
# Check for optional components
@@ -251,20 +253,15 @@ endif()
# Check that all required libraries are available
#
#----------------------------------------------------------------------
-list(REMOVE_DUPLICATES OSMIUM_EXTRA_FIND_VARS)
+if (OSMIUM_EXTRA_FIND_VARS)
+ list(REMOVE_DUPLICATES OSMIUM_EXTRA_FIND_VARS)
+endif()
# Handle the QUIETLY and REQUIRED arguments and set OSMIUM_FOUND to TRUE if
# all listed variables are TRUE.
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Osmium REQUIRED_VARS OSMIUM_INCLUDE_DIR ${OSMIUM_EXTRA_FIND_VARS})
unset(OSMIUM_EXTRA_FIND_VARS)
-# Copy the results to the output variables.
-if(OSMIUM_FOUND)
- set(OSMIUM_INCLUDE_DIRS ${OSMIUM_INCLUDE_DIR} ${OSMIUM_INCLUDE_DIRS})
-else()
- set(OSMIUM_INCLUDE_DIRS "")
-endif()
-
#----------------------------------------------------------------------
#
# Add compiler flags
diff --git a/cmake/iwyu.sh b/cmake/iwyu.sh
index f7d8a15..ceea106 100755
--- a/cmake/iwyu.sh
+++ b/cmake/iwyu.sh
@@ -16,7 +16,7 @@ echo "INCLUDE WHAT YOU USE REPORT:" >$log
allok=yes
-for file in `find include/osmium -name \*.hpp`; do
+for file in `find include/osmium -name \*.hpp | sort`; do
mkdir -p `dirname build/check_reports/$file`
ifile="build/check_reports/${file%.hpp}.iwyu"
$cmdline $file >$ifile 2>&1
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 2ee15e1..a04a843 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -14,6 +14,7 @@ set(EXAMPLES
count
create_node_cache
debug
+ filter_discussions
index
read
serdump
diff --git a/examples/osmium_filter_discussions.cpp b/examples/osmium_filter_discussions.cpp
new file mode 100644
index 0000000..bba25b7
--- /dev/null
+++ b/examples/osmium_filter_discussions.cpp
@@ -0,0 +1,72 @@
+/*
+
+ Read OSM changesets with discussions from a changeset dump like the one
+ you get from http://planet.osm.org/planet/discussions-latest.osm.bz2
+ and write out only those changesets which have discussions (ie comments).
+
+ The code in this example file is released into the Public Domain.
+
+*/
+
+#include <algorithm> // for std::copy_if
+#include <iostream> // for std::cout, std::cerr
+
+// we want to read OSM files in XML format
+// (other formats don't support full changesets, so only XML is needed here)
+#include <osmium/io/xml_input.hpp>
+#include <osmium/io/input_iterator.hpp>
+
+// we want to write OSM files in XML format
+#include <osmium/io/xml_output.hpp>
+#include <osmium/io/output_iterator.hpp>
+
+// we want to support any compressioon (.gz2 and .bz2)
+#include <osmium/io/any_compression.hpp>
+
+int main(int argc, char* argv[]) {
+ if (argc != 3) {
+ std::cout << "Usage: " << argv[0] << " INFILE OUTFILE\n";
+ exit(1);
+ }
+
+ // The input file, deduce file format from file suffix
+ osmium::io::File infile(argv[1]);
+
+ // The output file, force class XML OSM file format
+ osmium::io::File outfile(argv[2], "osm");
+
+ // Initialize Reader for the input file.
+ // Read only changesets (will ignore nodes, ways, and
+ // relations if there are any).
+ osmium::io::Reader reader(infile, osmium::osm_entity_bits::changeset);
+
+ // Get the header from the input file
+ osmium::io::Header header = reader.header();
+
+ // Initialize writer for the output file. Use the header from the input
+ // file for the output file. This will copy over some header information.
+ // The last parameter will tell the writer that it is allowed to overwrite
+ // an existing file. Without it, it will refuse to do so.
+ osmium::io::Writer writer(outfile, header, osmium::io::overwrite::allow);
+
+ // Create range of input iterators that will iterator over all changesets
+ // delivered from input file through the "reader".
+ auto input_range = osmium::io::make_input_iterator_range<osmium::Changeset>(reader);
+
+ // Create an output iterator writing through the "writer" object to the
+ // output file.
+ auto output_iterator = osmium::io::make_output_iterator(writer);
+
+ // Copy all changesets from input to output that have at least one comment.
+ std::copy_if(input_range.begin(), input_range.end(), output_iterator, [](const osmium::Changeset& changeset) {
+ return changeset.num_comments() > 0;
+ });
+
+ // Explicitly close the writer and reader. Will throw an exception if
+ // there is a problem. If you wait for the destructor to close the writer
+ // and reader, you will not notice the problem, because destructors must
+ // not throw.
+ writer.close();
+ reader.close();
+}
+
diff --git a/include/osmium/area/assembler.hpp b/include/osmium/area/assembler.hpp
index f172991..87feea2 100644
--- a/include/osmium/area/assembler.hpp
+++ b/include/osmium/area/assembler.hpp
@@ -34,9 +34,13 @@ DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
+#include <cassert>
+#include <cstring>
#include <iostream>
#include <iterator>
#include <list>
+#include <set>
+#include <string>
#include <map>
#include <vector>
diff --git a/include/osmium/area/detail/node_ref_segment.hpp b/include/osmium/area/detail/node_ref_segment.hpp
index 43569a8..ec7b035 100644
--- a/include/osmium/area/detail/node_ref_segment.hpp
+++ b/include/osmium/area/detail/node_ref_segment.hpp
@@ -113,7 +113,7 @@ namespace osmium {
return m_second;
}
- bool to_left_of(const osmium::Location location) const {
+ bool to_left_of(const osmium::Location& location) const {
// std::cerr << "segment " << first() << "--" << second() << " to_left_of(" << location << "\n";
if (first().location() == location || second().location() == location) {
@@ -195,8 +195,8 @@ namespace osmium {
}
inline bool y_range_overlap(const NodeRefSegment& s1, const NodeRefSegment& s2) {
- auto m1 = std::minmax(s1.first().location().y(), s1.second().location().y());
- auto m2 = std::minmax(s2.first().location().y(), s2.second().location().y());
+ const std::pair<int32_t, int32_t> m1 = std::minmax(s1.first().location().y(), s1.second().location().y());
+ const std::pair<int32_t, int32_t> m2 = std::minmax(s2.first().location().y(), s2.second().location().y());
if (m1.first > m2.second || m2.first > m1.second) {
return false;
}
diff --git a/include/osmium/area/detail/proto_ring.hpp b/include/osmium/area/detail/proto_ring.hpp
index c0f545c..162e289 100644
--- a/include/osmium/area/detail/proto_ring.hpp
+++ b/include/osmium/area/detail/proto_ring.hpp
@@ -34,12 +34,14 @@ DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
-#include <cassert>
+#include <cstdint>
+#include <cstdlib>
#include <iostream>
-#include <list>
+#include <iterator>
#include <set>
#include <vector>
+#include <osmium/osm/location.hpp>
#include <osmium/osm/node_ref.hpp>
#include <osmium/area/detail/node_ref_segment.hpp>
@@ -148,7 +150,8 @@ namespace osmium {
}
void swap_segments(ProtoRing& other) {
- std::swap(m_segments, other.m_segments);
+ using std::swap;
+ swap(m_segments, other.m_segments);
}
void add_inner_ring(ProtoRing* ring) {
diff --git a/include/osmium/area/detail/segment_list.hpp b/include/osmium/area/detail/segment_list.hpp
index ca6071e..289ecf0 100644
--- a/include/osmium/area/detail/segment_list.hpp
+++ b/include/osmium/area/detail/segment_list.hpp
@@ -41,6 +41,8 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/area/problem_reporter.hpp>
#include <osmium/area/detail/node_ref_segment.hpp>
#include <osmium/memory/buffer.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/node_ref.hpp>
#include <osmium/osm/relation.hpp>
#include <osmium/osm/way.hpp>
diff --git a/include/osmium/area/multipolygon_collector.hpp b/include/osmium/area/multipolygon_collector.hpp
index bf2a4ce..6761ec6 100644
--- a/include/osmium/area/multipolygon_collector.hpp
+++ b/include/osmium/area/multipolygon_collector.hpp
@@ -87,7 +87,8 @@ namespace osmium {
void flush_output_buffer() {
if (this->callback()) {
osmium::memory::Buffer buffer(initial_output_buffer_size);
- std::swap(buffer, m_output_buffer);
+ using std::swap;
+ swap(buffer, m_output_buffer);
this->callback()(std::move(buffer));
}
}
@@ -206,7 +207,10 @@ namespace osmium {
osmium::memory::Buffer read() {
osmium::memory::Buffer buffer(initial_output_buffer_size, osmium::memory::Buffer::auto_grow::yes);
- std::swap(buffer, m_output_buffer);
+
+ using std::swap;
+ swap(buffer, m_output_buffer);
+
return buffer;
}
diff --git a/include/osmium/area/problem_reporter_ogr.hpp b/include/osmium/area/problem_reporter_ogr.hpp
index d98a5b2..5332997 100644
--- a/include/osmium/area/problem_reporter_ogr.hpp
+++ b/include/osmium/area/problem_reporter_ogr.hpp
@@ -43,11 +43,11 @@ DEALINGS IN THE SOFTWARE.
*/
#include <memory>
-#include <stdexcept>
#include <gdalcpp.hpp>
#include <osmium/area/problem_reporter.hpp>
+#include <osmium/geom/factory.hpp>
#include <osmium/geom/ogr.hpp>
#include <osmium/osm/location.hpp>
#include <osmium/osm/types.hpp>
diff --git a/include/osmium/builder/osm_object_builder.hpp b/include/osmium/builder/osm_object_builder.hpp
index 5827243..343f51f 100644
--- a/include/osmium/builder/osm_object_builder.hpp
+++ b/include/osmium/builder/osm_object_builder.hpp
@@ -33,9 +33,11 @@ DEALINGS IN THE SOFTWARE.
*/
+#include <cassert>
#include <cstring>
#include <initializer_list>
#include <new>
+#include <stdexcept>
#include <string>
#include <utility>
@@ -141,7 +143,7 @@ namespace osmium {
static_cast<Builder*>(this)->add_size(sizeof(osmium::NodeRef));
}
- void add_node_ref(const object_id_type ref, const osmium::Location location = Location()) {
+ void add_node_ref(const object_id_type ref, const osmium::Location& location = Location{}) {
add_node_ref(NodeRef(ref, location));
}
@@ -236,6 +238,64 @@ namespace osmium {
}; // class RelationMemberListBuilder
+ class ChangesetDiscussionBuilder : public ObjectBuilder<ChangesetDiscussion> {
+
+ osmium::ChangesetComment* m_comment = nullptr;
+
+ void add_user(osmium::ChangesetComment& comment, const char* user, const size_t length) {
+ if (length > osmium::max_osm_string_length) {
+ throw std::length_error("OSM user name is too long");
+ }
+ comment.set_user_size(osmium::string_size_type(length) + 1);
+ add_size(append(user, osmium::memory::item_size_type(length)) + append_zero());
+ }
+
+ void add_text(osmium::ChangesetComment& comment, const char* text, const size_t length) {
+ // XXX There is no limit on the length of a comment text. We
+ // limit it here to 2^16-2 characters, because that's all that
+ // will fit into our internal data structure. This is not ideal,
+ // and will have to be discussed and cleared up.
+ if (length > std::numeric_limits<osmium::string_size_type>::max() - 1) {
+ throw std::length_error("OSM changeset comment is too long");
+ }
+ comment.set_text_size(osmium::string_size_type(length) + 1);
+ add_size(append(text, osmium::memory::item_size_type(length)) + append_zero());
+ add_padding(true);
+ }
+
+ public:
+
+ explicit ChangesetDiscussionBuilder(osmium::memory::Buffer& buffer, Builder* parent = nullptr) :
+ ObjectBuilder<ChangesetDiscussion>(buffer, parent) {
+ }
+
+ ~ChangesetDiscussionBuilder() {
+ assert(!m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!");
+ add_padding();
+ }
+
+ void add_comment(osmium::Timestamp date, osmium::user_id_type uid, const char* user) {
+ assert(!m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!");
+ m_comment = reserve_space_for<osmium::ChangesetComment>();
+ new (m_comment) osmium::ChangesetComment(date, uid);
+ add_size(sizeof(ChangesetComment));
+ add_user(*m_comment, user, std::strlen(user));
+ }
+
+ void add_comment_text(const char* text) {
+ assert(m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!");
+ add_text(*m_comment, text, std::strlen(text));
+ m_comment = nullptr;
+ }
+
+ void add_comment_text(const std::string& text) {
+ assert(m_comment && "You have to always call both add_comment() and then add_comment_text() in that order for each comment!");
+ add_text(*m_comment, text.c_str(), text.size());
+ m_comment = nullptr;
+ }
+
+ }; // class ChangesetDiscussionBuilder
+
template <class T>
class OSMObjectBuilder : public ObjectBuilder<T> {
diff --git a/include/osmium/experimental/flex_reader.hpp b/include/osmium/experimental/flex_reader.hpp
index f00a5ec..e99811e 100644
--- a/include/osmium/experimental/flex_reader.hpp
+++ b/include/osmium/experimental/flex_reader.hpp
@@ -33,10 +33,18 @@ DEALINGS IN THE SOFTWARE.
*/
+#include <string>
+#include <vector>
+
+#include <osmium/area/assembler.hpp>
+#include <osmium/area/multipolygon_collector.hpp>
#include <osmium/handler/node_locations_for_ways.hpp>
+#include <osmium/io/file.hpp>
+#include <osmium/io/header.hpp>
+#include <osmium/io/reader.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm/entity_bits.hpp>
#include <osmium/visitor.hpp>
-#include <osmium/area/multipolygon_collector.hpp>
-#include <osmium/area/assembler.hpp>
namespace osmium {
@@ -104,7 +112,7 @@ namespace osmium {
return buffer;
}
- osmium::io::Header header() const {
+ osmium::io::Header header() {
return m_reader.header();
}
diff --git a/include/osmium/geom/coordinates.hpp b/include/osmium/geom/coordinates.hpp
index 2bad57e..6544e68 100644
--- a/include/osmium/geom/coordinates.hpp
+++ b/include/osmium/geom/coordinates.hpp
@@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE.
*/
-#include <cstddef>
#include <iosfwd>
#include <string>
diff --git a/include/osmium/geom/factory.hpp b/include/osmium/geom/factory.hpp
index 13e5955..ef75704 100644
--- a/include/osmium/geom/factory.hpp
+++ b/include/osmium/geom/factory.hpp
@@ -199,7 +199,7 @@ namespace osmium {
/* Point */
- point_type create_point(const osmium::Location location) const {
+ point_type create_point(const osmium::Location& location) const {
return m_impl.make_point(m_projection(location));
}
diff --git a/include/osmium/geom/geojson.hpp b/include/osmium/geom/geojson.hpp
index 7d59535..96ddcd9 100644
--- a/include/osmium/geom/geojson.hpp
+++ b/include/osmium/geom/geojson.hpp
@@ -88,7 +88,10 @@ namespace osmium {
linestring_type linestring_finish(size_t /* num_points */) {
assert(!m_str.empty());
std::string str;
- std::swap(str, m_str);
+
+ using std::swap;
+ swap(str, m_str);
+
str.back() = ']';
str += "}";
return str;
@@ -134,7 +137,10 @@ namespace osmium {
multipolygon_type multipolygon_finish() {
assert(!m_str.empty());
std::string str;
- std::swap(str, m_str);
+
+ using std::swap;
+ swap(str, m_str);
+
str.back() = ']';
str += "}";
return str;
diff --git a/include/osmium/geom/tile.hpp b/include/osmium/geom/tile.hpp
index 1392480..6ca0682 100644
--- a/include/osmium/geom/tile.hpp
+++ b/include/osmium/geom/tile.hpp
@@ -67,10 +67,10 @@ namespace osmium {
explicit Tile(uint32_t zoom, const osmium::Location& location) :
z(zoom) {
osmium::geom::Coordinates c = lonlat_to_mercator(location);
- const int32_t n = 1LL << zoom;
+ const int32_t n = 1 << zoom;
const double scale = detail::max_coordinate_epsg3857 * 2 / n;
- x = detail::restrict_to_range<int32_t>(int32_t((c.x + detail::max_coordinate_epsg3857) / scale), 0, n-1);
- y = detail::restrict_to_range<int32_t>(int32_t((detail::max_coordinate_epsg3857 - c.y) / scale), 0, n-1);
+ x = uint32_t(detail::restrict_to_range<int32_t>(int32_t((c.x + detail::max_coordinate_epsg3857) / scale), 0, n-1));
+ y = uint32_t(detail::restrict_to_range<int32_t>(int32_t((detail::max_coordinate_epsg3857 - c.y) / scale), 0, n-1));
}
}; // struct Tile
diff --git a/include/osmium/geom/wkb.hpp b/include/osmium/geom/wkb.hpp
index a290c02..d2145b1 100644
--- a/include/osmium/geom/wkb.hpp
+++ b/include/osmium/geom/wkb.hpp
@@ -188,7 +188,9 @@ namespace osmium {
linestring_type linestring_finish(size_t num_points) {
set_size(m_linestring_size_offset, num_points);
std::string data;
- std::swap(data, m_data);
+
+ using std::swap;
+ swap(data, m_data);
if (m_out_type == out_type::hex) {
return convert_to_hex(data);
@@ -246,7 +248,9 @@ namespace osmium {
multipolygon_type multipolygon_finish() {
set_size(m_multipolygon_size_offset, m_polygons);
std::string data;
- std::swap(data, m_data);
+
+ using std::swap;
+ swap(data, m_data);
if (m_out_type == out_type::hex) {
return convert_to_hex(data);
diff --git a/include/osmium/geom/wkt.hpp b/include/osmium/geom/wkt.hpp
index 4fea96b..5e1e9a7 100644
--- a/include/osmium/geom/wkt.hpp
+++ b/include/osmium/geom/wkt.hpp
@@ -86,7 +86,10 @@ namespace osmium {
linestring_type linestring_finish(size_t /* num_points */) {
assert(!m_str.empty());
std::string str;
- std::swap(str, m_str);
+
+ using std::swap;
+ swap(str, m_str);
+
str.back() = ')';
return str;
}
@@ -131,7 +134,10 @@ namespace osmium {
multipolygon_type multipolygon_finish() {
assert(!m_str.empty());
std::string str;
- std::swap(str, m_str);
+
+ using std::swap;
+ swap(str, m_str);
+
str.back() = ')';
return str;
}
diff --git a/include/osmium/handler.hpp b/include/osmium/handler.hpp
index 34d8785..a90d779 100644
--- a/include/osmium/handler.hpp
+++ b/include/osmium/handler.hpp
@@ -41,6 +41,7 @@ namespace osmium {
class Relation;
class Area;
class Changeset;
+ class ChangesetDiscussion;
class TagList;
class WayNodeList;
class RelationMemberList;
@@ -89,6 +90,9 @@ namespace osmium {
void inner_ring(const osmium::InnerRing&) const {
}
+ void changeset_discussion(const osmium::ChangesetDiscussion&) const {
+ }
+
void flush() const {
}
diff --git a/include/osmium/index/bool_vector.hpp b/include/osmium/index/bool_vector.hpp
index 94e1f72..04850a5 100644
--- a/include/osmium/index/bool_vector.hpp
+++ b/include/osmium/index/bool_vector.hpp
@@ -56,11 +56,13 @@ namespace osmium {
public:
BoolVector() = default;
+
BoolVector(const BoolVector&) = default;
BoolVector(BoolVector&&) = default;
BoolVector& operator=(const BoolVector&) = default;
BoolVector& operator=(BoolVector&&) = default;
- ~BoolVector() = default;
+
+ ~BoolVector() noexcept = default;
void set(T id, bool value = true) {
if (m_bits.size() <= id) {
diff --git a/include/osmium/index/detail/mmap_vector_anon.hpp b/include/osmium/index/detail/mmap_vector_anon.hpp
index fc01626..12a1803 100644
--- a/include/osmium/index/detail/mmap_vector_anon.hpp
+++ b/include/osmium/index/detail/mmap_vector_anon.hpp
@@ -54,6 +54,8 @@ namespace osmium {
mmap_vector_base<T>() {
}
+ ~mmap_vector_anon() noexcept = default;
+
}; // class mmap_vector_anon
} // namespace detail
diff --git a/include/osmium/index/detail/mmap_vector_base.hpp b/include/osmium/index/detail/mmap_vector_base.hpp
index 9b64768..e5f28e9 100644
--- a/include/osmium/index/detail/mmap_vector_base.hpp
+++ b/include/osmium/index/detail/mmap_vector_base.hpp
@@ -60,7 +60,7 @@ namespace osmium {
public:
- explicit mmap_vector_base(int fd, size_t capacity, size_t size = 0) :
+ mmap_vector_base(int fd, size_t capacity, size_t size = 0) :
m_size(size),
m_mapping(capacity, osmium::util::MemoryMapping::mapping_mode::write_shared, fd) {
}
@@ -70,6 +70,8 @@ namespace osmium {
m_mapping(capacity) {
}
+ ~mmap_vector_base() noexcept = default;
+
typedef T value_type;
typedef T& reference;
typedef const T& const_reference;
@@ -78,8 +80,6 @@ namespace osmium {
typedef T* iterator;
typedef const T* const_iterator;
- ~mmap_vector_base() = default;
-
void close() {
m_mapping.unmap();
}
diff --git a/include/osmium/index/detail/mmap_vector_file.hpp b/include/osmium/index/detail/mmap_vector_file.hpp
index 1dadbcb..54ef513 100644
--- a/include/osmium/index/detail/mmap_vector_file.hpp
+++ b/include/osmium/index/detail/mmap_vector_file.hpp
@@ -50,17 +50,21 @@ namespace osmium {
public:
- explicit mmap_vector_file() : mmap_vector_base<T>(
+ mmap_vector_file() :
+ mmap_vector_base<T>(
osmium::detail::create_tmp_file(),
osmium::detail::mmap_vector_size_increment) {
}
- explicit mmap_vector_file(int fd) : mmap_vector_base<T>(
+ explicit mmap_vector_file(int fd) :
+ mmap_vector_base<T>(
fd,
osmium::util::file_size(fd) / sizeof(T),
osmium::util::file_size(fd) / sizeof(T)) {
}
+ ~mmap_vector_file() noexcept = default;
+
}; // class mmap_vector_file
} // namespace detail
diff --git a/include/osmium/index/detail/vector_map.hpp b/include/osmium/index/detail/vector_map.hpp
index 9c1cf4e..e4f64ac 100644
--- a/include/osmium/index/detail/vector_map.hpp
+++ b/include/osmium/index/detail/vector_map.hpp
@@ -68,7 +68,7 @@ namespace osmium {
m_vector(fd) {
}
- ~VectorBasedDenseMap() = default;
+ ~VectorBasedDenseMap() noexcept = default;
void reserve(const size_t size) override final {
m_vector.reserve(size);
diff --git a/include/osmium/index/detail/vector_multimap.hpp b/include/osmium/index/detail/vector_multimap.hpp
index 1d875fc..dc2e15a 100644
--- a/include/osmium/index/detail/vector_multimap.hpp
+++ b/include/osmium/index/detail/vector_multimap.hpp
@@ -75,7 +75,7 @@ namespace osmium {
m_vector(fd) {
}
- ~VectorBasedSparseMultimap() = default;
+ ~VectorBasedSparseMultimap() noexcept = default;
void set(const TId id, const TValue value) override final {
m_vector.push_back(element_type(id, value));
diff --git a/include/osmium/index/map.hpp b/include/osmium/index/map.hpp
index 61af672..68a9c20 100644
--- a/include/osmium/index/map.hpp
+++ b/include/osmium/index/map.hpp
@@ -84,7 +84,8 @@ namespace osmium {
template <typename TId, typename TValue>
class Map {
- static_assert(std::is_integral<TId>::value && std::is_unsigned<TId>::value, "TId template parameter for class Map must be unsigned integral type");
+ static_assert(std::is_integral<TId>::value && std::is_unsigned<TId>::value,
+ "TId template parameter for class Map must be unsigned integral type");
Map(const Map&) = delete;
Map& operator=(const Map&) = delete;
@@ -104,7 +105,7 @@ namespace osmium {
Map() = default;
- virtual ~Map() = default;
+ virtual ~Map() noexcept = default;
virtual void reserve(const size_t) {
// default implementation is empty
@@ -147,10 +148,16 @@ namespace osmium {
// default implementation is empty
}
+ // This function could usually be const in derived classes,
+ // but not always. It could, for instance, sort internal data.
+ // This is why it is not declared const here.
virtual void dump_as_list(const int /*fd*/) {
throw std::runtime_error("can't dump as list");
}
+ // This function could usually be const in derived classes,
+ // but not always. It could, for instance, sort internal data.
+ // This is why it is not declared const here.
virtual void dump_as_array(const int /*fd*/) {
throw std::runtime_error("can't dump as array");
}
@@ -252,12 +259,14 @@ namespace osmium {
#define OSMIUM_CONCATENATE_DETAIL_(x, y) x##y
#define OSMIUM_CONCATENATE_(x, y) OSMIUM_CONCATENATE_DETAIL_(x, y)
-#define OSMIUM_MAKE_UNIQUE_(x) OSMIUM_CONCATENATE_(x, __COUNTER__)
#define REGISTER_MAP(id, value, klass, name) \
-namespace { \
- const bool OSMIUM_MAKE_UNIQUE_(registered_index_map_##name) = osmium::index::register_map<id, value, klass>(#name); \
-}
+namespace osmium { namespace index { namespace detail { \
+ const bool OSMIUM_CONCATENATE_(registered_, name) = osmium::index::register_map<id, value, klass>(#name); \
+ inline bool OSMIUM_CONCATENATE_(get_registered_, name)() noexcept { \
+ return OSMIUM_CONCATENATE_(registered_, name); \
+ } \
+} } }
} // namespace index
diff --git a/include/osmium/index/map/dummy.hpp b/include/osmium/index/map/dummy.hpp
index de05d1d..5b471df 100644
--- a/include/osmium/index/map/dummy.hpp
+++ b/include/osmium/index/map/dummy.hpp
@@ -56,7 +56,7 @@ namespace osmium {
Dummy() = default;
- ~Dummy() override final = default;
+ ~Dummy() noexcept override final = default;
void set(const TId, const TValue) override final {
// intentionally left blank
diff --git a/include/osmium/index/map/sparse_mem_map.hpp b/include/osmium/index/map/sparse_mem_map.hpp
index 2b9048b..9bad07e 100644
--- a/include/osmium/index/map/sparse_mem_map.hpp
+++ b/include/osmium/index/map/sparse_mem_map.hpp
@@ -71,25 +71,25 @@ namespace osmium {
SparseMemMap() = default;
- ~SparseMemMap() override final = default;
+ ~SparseMemMap() noexcept override final = default;
void set(const TId id, const TValue value) override final {
m_elements[id] = value;
}
const TValue get(const TId id) const override final {
- try {
- return m_elements.at(id);
- } catch (std::out_of_range&) {
+ auto it = m_elements.find(id);
+ if (it == m_elements.end()) {
not_found_error(id);
}
+ return it->second;
}
- size_t size() const override final {
+ size_t size() const noexcept override final {
return m_elements.size();
}
- size_t used_memory() const override final {
+ size_t used_memory() const noexcept override final {
return element_size * m_elements.size();
}
@@ -100,7 +100,8 @@ namespace osmium {
void dump_as_list(const int fd) override final {
typedef typename std::map<TId, TValue>::value_type t;
std::vector<t> v;
- std::copy(m_elements.begin(), m_elements.end(), std::back_inserter(v));
+ v.reserve(m_elements.size());
+ std::copy(m_elements.cbegin(), m_elements.cend(), std::back_inserter(v));
osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(v.data()), sizeof(t) * v.size());
}
diff --git a/include/osmium/index/map/sparse_mem_table.hpp b/include/osmium/index/map/sparse_mem_table.hpp
index 09ee81b..032400e 100644
--- a/include/osmium/index/map/sparse_mem_table.hpp
+++ b/include/osmium/index/map/sparse_mem_table.hpp
@@ -88,7 +88,7 @@ namespace osmium {
m_elements(grow_size) {
}
- ~SparseMemTable() override final = default;
+ ~SparseMemTable() noexcept override final = default;
void set(const TId id, const TValue value) override final {
if (id >= m_elements.size()) {
@@ -123,6 +123,7 @@ namespace osmium {
void dump_as_list(const int fd) override final {
std::vector<std::pair<TId, TValue>> v;
+ v.reserve(m_elements.size());
int n = 0;
for (const TValue value : m_elements) {
if (value != osmium::index::empty_value<TValue>()) {
diff --git a/include/osmium/index/multimap/hybrid.hpp b/include/osmium/index/multimap/hybrid.hpp
index ac2d964..cdf14a3 100644
--- a/include/osmium/index/multimap/hybrid.hpp
+++ b/include/osmium/index/multimap/hybrid.hpp
@@ -62,16 +62,18 @@ namespace osmium {
public:
- explicit HybridIterator(typename main_map_type::iterator begin_main,
- typename main_map_type::iterator end_main,
- typename extra_map_type::iterator begin_extra,
- typename extra_map_type::iterator end_extra) :
+ HybridIterator(typename main_map_type::iterator begin_main,
+ typename main_map_type::iterator end_main,
+ typename extra_map_type::iterator begin_extra,
+ typename extra_map_type::iterator end_extra) :
m_begin_main(begin_main),
m_end_main(end_main),
m_begin_extra(begin_extra),
m_end_extra(end_extra) {
}
+ ~HybridIterator() noexcept = default;
+
HybridIterator& operator++() {
if (m_begin_main == m_end_main) {
++m_begin_extra;
@@ -134,6 +136,8 @@ namespace osmium {
m_extra() {
}
+ ~Hybrid() noexcept = default;
+
size_t size() const override final {
return m_main.size() + m_extra.size();
}
diff --git a/include/osmium/index/multimap/sparse_mem_multimap.hpp b/include/osmium/index/multimap/sparse_mem_multimap.hpp
index 5b47152..353e357 100644
--- a/include/osmium/index/multimap/sparse_mem_multimap.hpp
+++ b/include/osmium/index/multimap/sparse_mem_multimap.hpp
@@ -132,10 +132,10 @@ namespace osmium {
void dump_as_list(const int fd) override final {
std::vector<element_type> v;
+ v.reserve(m_elements.size());
for (const auto& element : m_elements) {
v.emplace_back(element.first, element.second);
}
-// std::copy(m_elements.cbegin(), m_elements.cend(), std::back_inserter(v));
std::sort(v.begin(), v.end());
osmium::io::detail::reliable_write(fd, reinterpret_cast<const char*>(v.data()), sizeof(element_type) * v.size());
}
diff --git a/include/osmium/io/any_input.hpp b/include/osmium/io/any_input.hpp
index d16d069..36f43b7 100644
--- a/include/osmium/io/any_input.hpp
+++ b/include/osmium/io/any_input.hpp
@@ -47,5 +47,6 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/io/pbf_input.hpp> // IWYU pragma: export
#include <osmium/io/xml_input.hpp> // IWYU pragma: export
+#include <osmium/io/o5m_input.hpp> // IWYU pragma: export
#endif // OSMIUM_IO_ANY_INPUT_HPP
diff --git a/include/osmium/io/bzip2_compression.hpp b/include/osmium/io/bzip2_compression.hpp
index e961a87..058d993 100644
--- a/include/osmium/io/bzip2_compression.hpp
+++ b/include/osmium/io/bzip2_compression.hpp
@@ -55,7 +55,9 @@ DEALINGS IN THE SOFTWARE.
#endif
#include <osmium/io/compression.hpp>
+#include <osmium/io/error.hpp>
#include <osmium/io/file_compression.hpp>
+#include <osmium/io/overwrite.hpp>
#include <osmium/util/cast.hpp>
#include <osmium/util/compatibility.hpp>
@@ -65,13 +67,13 @@ namespace osmium {
* Exception thrown when there are problems compressing or
* decompressing bzip2 files.
*/
- struct bzip2_error : public std::runtime_error {
+ struct bzip2_error : public io_error {
int bzip2_error_code;
int system_errno;
bzip2_error(const std::string& what, int error_code) :
- std::runtime_error(what),
+ io_error(what),
bzip2_error_code(error_code),
system_errno(error_code == BZ_IO_ERROR ? errno : 0) {
}
@@ -105,8 +107,8 @@ namespace osmium {
public:
- explicit Bzip2Compressor(int fd) :
- Compressor(),
+ explicit Bzip2Compressor(int fd, fsync sync) :
+ Compressor(sync),
m_file(fdopen(dup(fd), "wb")),
m_bzerror(BZ_OK),
m_bzfile(::BZ2_bzWriteOpen(&m_bzerror, m_file, 6, 0, 0)) {
@@ -115,8 +117,12 @@ namespace osmium {
}
}
- ~Bzip2Compressor() override final {
- close();
+ ~Bzip2Compressor() noexcept override final {
+ try {
+ close();
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
+ }
}
void write(const std::string& data) override final {
@@ -133,7 +139,12 @@ namespace osmium {
::BZ2_bzWriteClose(&error, m_bzfile, 0, nullptr, nullptr);
m_bzfile = nullptr;
if (m_file) {
- fclose(m_file);
+ if (do_fsync()) {
+ osmium::io::detail::reliable_fsync(::fileno(m_file));
+ }
+ if (fclose(m_file) != 0) {
+ throw std::system_error(errno, std::system_category(), "Close failed");
+ }
}
if (error != BZ_OK) {
detail::throw_bzip2_error(m_bzfile, "write close failed", error);
@@ -162,8 +173,12 @@ namespace osmium {
}
}
- ~Bzip2Decompressor() override final {
- close();
+ ~Bzip2Decompressor() noexcept override final {
+ try {
+ close();
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
+ }
}
std::string read() override final {
@@ -209,7 +224,9 @@ namespace osmium {
::BZ2_bzReadClose(&error, m_bzfile);
m_bzfile = nullptr;
if (m_file) {
- fclose(m_file);
+ if (fclose(m_file) != 0) {
+ throw std::system_error(errno, std::system_category(), "Close failed");
+ }
}
if (error != BZ_OK) {
detail::throw_bzip2_error(m_bzfile, "read close failed", error);
@@ -240,8 +257,12 @@ namespace osmium {
}
}
- ~Bzip2BufferDecompressor() override final {
- BZ2_bzDecompressEnd(&m_bzstream);
+ ~Bzip2BufferDecompressor() noexcept override final {
+ try {
+ close();
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
+ }
}
std::string read() override final {
@@ -270,22 +291,28 @@ namespace osmium {
return output;
}
+ void close() override final {
+ BZ2_bzDecompressEnd(&m_bzstream);
+ }
+
}; // class Bzip2BufferDecompressor
- namespace {
+ namespace detail {
-// we want the register_compression() function to run, setting the variable
-// is only a side-effect, it will never be used
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-variable"
+ // we want the register_compression() function to run, setting
+ // the variable is only a side-effect, it will never be used
const bool registered_bzip2_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::bzip2,
- [](int fd) { return new osmium::io::Bzip2Compressor(fd); },
+ [](int fd, fsync sync) { return new osmium::io::Bzip2Compressor(fd, sync); },
[](int fd) { return new osmium::io::Bzip2Decompressor(fd); },
[](const char* buffer, size_t size) { return new osmium::io::Bzip2BufferDecompressor(buffer, size); }
);
-#pragma GCC diagnostic pop
- } // anonymous namespace
+ // dummy function to silence the unused variable warning from above
+ inline bool get_registered_bzip2_compression() noexcept {
+ return registered_bzip2_compression;
+ }
+
+ } // namespace detail
} // namespace io
diff --git a/include/osmium/io/compression.hpp b/include/osmium/io/compression.hpp
index 2529761..64e59cc 100644
--- a/include/osmium/io/compression.hpp
+++ b/include/osmium/io/compression.hpp
@@ -40,6 +40,7 @@ DEALINGS IN THE SOFTWARE.
#include <stdexcept>
#include <string>
#include <system_error>
+#include <tuple>
#include <utility>
#ifndef _MSC_VER
@@ -49,7 +50,9 @@ DEALINGS IN THE SOFTWARE.
#endif
#include <osmium/io/detail/read_write.hpp>
+#include <osmium/io/error.hpp>
#include <osmium/io/file_compression.hpp>
+#include <osmium/io/overwrite.hpp>
#include <osmium/util/compatibility.hpp>
namespace osmium {
@@ -58,11 +61,21 @@ namespace osmium {
class Compressor {
+ fsync m_fsync;
+
+ protected:
+
+ bool do_fsync() const {
+ return m_fsync == fsync::yes;
+ }
+
public:
- Compressor() = default;
+ explicit Compressor(fsync sync) :
+ m_fsync(sync) {
+ }
- virtual ~Compressor() {
+ virtual ~Compressor() noexcept {
}
virtual void write(const std::string& data) = 0;
@@ -85,13 +98,12 @@ namespace osmium {
Decompressor(Decompressor&&) = delete;
Decompressor& operator=(Decompressor&&) = delete;
- virtual ~Decompressor() {
+ virtual ~Decompressor() noexcept {
}
virtual std::string read() = 0;
- virtual void close() {
- }
+ virtual void close() = 0;
}; // class Decompressor
@@ -106,13 +118,16 @@ namespace osmium {
public:
- typedef std::function<osmium::io::Compressor*(int)> create_compressor_type;
+ typedef std::function<osmium::io::Compressor*(int, fsync)> create_compressor_type;
typedef std::function<osmium::io::Decompressor*(int)> create_decompressor_type_fd;
typedef std::function<osmium::io::Decompressor*(const char*, size_t)> create_decompressor_type_buffer;
private:
- typedef std::map<const osmium::io::file_compression, std::tuple<create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer>> compression_map_type;
+ typedef std::map<const osmium::io::file_compression,
+ std::tuple<create_compressor_type,
+ create_decompressor_type_fd,
+ create_decompressor_type_buffer>> compression_map_type;
compression_map_type m_callbacks;
@@ -128,7 +143,7 @@ namespace osmium {
std::string error_message {"Support for compression '"};
error_message += as_string(compression);
error_message += "' not compiled into this binary.";
- throw std::runtime_error(error_message);
+ throw unsupported_file_format_error(error_message);
}
public:
@@ -144,15 +159,20 @@ namespace osmium {
create_decompressor_type_fd create_decompressor_fd,
create_decompressor_type_buffer create_decompressor_buffer) {
- compression_map_type::value_type cc(compression, std::make_tuple(create_compressor, create_decompressor_fd, create_decompressor_buffer));
+ compression_map_type::value_type cc(compression,
+ std::make_tuple(create_compressor,
+ create_decompressor_fd,
+ create_decompressor_buffer));
+
return m_callbacks.insert(cc).second;
}
- std::unique_ptr<osmium::io::Compressor> create_compressor(osmium::io::file_compression compression, int fd) {
+ template <typename... TArgs>
+ std::unique_ptr<osmium::io::Compressor> create_compressor(osmium::io::file_compression compression, TArgs&&... args) {
auto it = m_callbacks.find(compression);
if (it != m_callbacks.end()) {
- return std::unique_ptr<osmium::io::Compressor>(std::get<0>(it->second)(fd));
+ return std::unique_ptr<osmium::io::Compressor>(std::get<0>(it->second)(std::forward<TArgs>(args)...));
}
error(compression);
@@ -186,13 +206,17 @@ namespace osmium {
public:
- NoCompressor(int fd) :
- Compressor(),
+ NoCompressor(int fd, fsync sync) :
+ Compressor(sync),
m_fd(fd) {
}
- ~NoCompressor() override final {
- close();
+ ~NoCompressor() noexcept override final {
+ try {
+ close();
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
+ }
}
void write(const std::string& data) override final {
@@ -201,8 +225,12 @@ namespace osmium {
void close() override final {
if (m_fd >= 0) {
- ::close(m_fd);
+ int fd = m_fd;
m_fd = -1;
+ if (do_fsync()) {
+ osmium::io::detail::reliable_fsync(fd);
+ }
+ osmium::io::detail::reliable_close(fd);
}
}
@@ -230,8 +258,12 @@ namespace osmium {
m_buffer_size(size) {
}
- ~NoDecompressor() override final {
- close();
+ ~NoDecompressor() noexcept override final {
+ try {
+ close();
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
+ }
}
std::string read() override final {
@@ -249,7 +281,7 @@ namespace osmium {
if (nread < 0) {
throw std::system_error(errno, std::system_category(), "Read failed");
}
- buffer.resize(nread);
+ buffer.resize(std::string::size_type(nread));
}
return buffer;
@@ -257,27 +289,30 @@ namespace osmium {
void close() override final {
if (m_fd >= 0) {
- ::close(m_fd);
+ int fd = m_fd;
m_fd = -1;
+ osmium::io::detail::reliable_close(fd);
}
}
}; // class NoDecompressor
- namespace {
+ namespace detail {
-// we want the register_compression() function to run, setting the variable
-// is only a side-effect, it will never be used
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-variable"
+ // we want the register_compression() function to run, setting
+ // the variable is only a side-effect, it will never be used
const bool registered_no_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::none,
- [](int fd) { return new osmium::io::NoCompressor(fd); },
+ [](int fd, fsync sync) { return new osmium::io::NoCompressor(fd, sync); },
[](int fd) { return new osmium::io::NoDecompressor(fd); },
[](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); }
);
-#pragma GCC diagnostic pop
- } // anonymous namespace
+ // dummy function to silence the unused variable warning from above
+ inline bool get_registered_no_compression() noexcept {
+ return registered_no_compression;
+ }
+
+ } // namespace detail
} // namespace io
diff --git a/include/osmium/io/detail/debug_output_format.hpp b/include/osmium/io/detail/debug_output_format.hpp
index 026cdc3..90ec199 100644
--- a/include/osmium/io/detail/debug_output_format.hpp
+++ b/include/osmium/io/detail/debug_output_format.hpp
@@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE.
*/
-#include <chrono>
#include <cinttypes>
#include <cstddef>
#include <cstdint>
@@ -41,14 +40,10 @@ DEALINGS IN THE SOFTWARE.
#include <future>
#include <iterator>
#include <memory>
-#include <ratio>
#include <string>
#include <thread>
#include <utility>
-#include <utf8.h>
-
-#include <osmium/handler.hpp>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/memory/buffer.hpp>
@@ -87,65 +82,32 @@ namespace osmium {
constexpr const char* color_white = "\x1b[37m";
constexpr const char* color_reset = "\x1b[0m";
- /**
- * Writes out one buffer with OSM data in Debug format.
- */
- class DebugOutputBlock : public osmium::handler::Handler {
+ struct debug_output_options {
- static constexpr size_t tmp_buffer_size = 50;
+ /// Should metadata of objects be added?
+ bool add_metadata;
- std::shared_ptr<osmium::memory::Buffer> m_input_buffer;
+ /// Output with ANSI colors?
+ bool use_color;
- std::shared_ptr<std::string> m_out;
+ };
- char m_tmp_buffer[tmp_buffer_size+1];
+ /**
+ * Writes out one buffer with OSM data in Debug format.
+ */
+ class DebugOutputBlock : public OutputBlock {
- bool m_add_metadata;
- bool m_use_color;
+ debug_output_options m_options;
- template <typename... TArgs>
- void output_formatted(const char* format, TArgs&&... args) {
-#ifndef NDEBUG
- int len =
-#endif
-#ifndef _MSC_VER
- snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward<TArgs>(args)...);
-#else
- _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward<TArgs>(args)...);
-#endif
- assert(len > 0 && static_cast<size_t>(len) < tmp_buffer_size);
- *m_out += m_tmp_buffer;
- }
+ const char* m_utf8_prefix = "";
+ const char* m_utf8_suffix = "";
void append_encoded_string(const char* data) {
- const char* end = data + std::strlen(data);
-
- while (data != end) {
- const char* last = data;
- uint32_t c = utf8::next(data, end);
-
- // This is a list of Unicode code points that we let
- // through instead of escaping them. It is incomplete
- // and can be extended later.
- // Generally we don't want to let through any
- // non-printing characters.
- if ((0x0020 <= c && c <= 0x0021) ||
- (0x0023 <= c && c <= 0x003b) ||
- (0x003d == c) ||
- (0x003f <= c && c <= 0x007e) ||
- (0x00a1 <= c && c <= 0x00ac) ||
- (0x00ae <= c && c <= 0x05ff)) {
- m_out->append(last, data);
- } else {
- write_color(color_red);
- output_formatted("<U+%04X>", c);
- write_color(color_blue);
- }
- }
+ append_debug_encoded_string(*m_out, data, m_utf8_prefix, m_utf8_suffix);
}
void write_color(const char* color) {
- if (m_use_color) {
+ if (m_options.use_color) {
*m_out += color;
}
}
@@ -177,15 +139,38 @@ namespace osmium {
*m_out += ": ";
}
+ void write_comment_field(const char* name) {
+ write_color(color_cyan);
+ *m_out += name;
+ write_color(color_reset);
+ *m_out += ": ";
+ }
+
+ void write_counter(int width, int n) {
+ write_color(color_white);
+ output_formatted(" %0*d: ", width, n++);
+ write_color(color_reset);
+ }
+
void write_error(const char* msg) {
write_color(color_red);
*m_out += msg;
write_color(color_reset);
}
+ void write_timestamp(const osmium::Timestamp& timestamp) {
+ if (timestamp.valid()) {
+ *m_out += timestamp.to_iso();
+ output_formatted(" (%d)", timestamp.seconds_since_epoch());
+ } else {
+ write_error("NOT SET");
+ }
+ *m_out += '\n';
+ }
+
void write_meta(const osmium::OSMObject& object) {
output_formatted("%" PRId64 "\n", object.id());
- if (m_add_metadata) {
+ if (m_options.add_metadata) {
write_fieldname("version");
output_formatted(" %d", object.version());
if (object.visible()) {
@@ -196,8 +181,7 @@ namespace osmium {
write_fieldname("changeset");
output_formatted("%d\n", object.changeset());
write_fieldname("timestamp");
- *m_out += object.timestamp().to_iso();
- output_formatted(" (%d)\n", object.timestamp());
+ write_timestamp(object.timestamp());
write_fieldname("user");
output_formatted(" %d ", object.uid());
write_string(object.user());
@@ -255,12 +239,11 @@ namespace osmium {
public:
- explicit DebugOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool use_color) :
- m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
- m_out(std::make_shared<std::string>()),
- m_tmp_buffer(),
- m_add_metadata(add_metadata),
- m_use_color(use_color) {
+ DebugOutputBlock(osmium::memory::Buffer&& buffer, const debug_output_options& options) :
+ OutputBlock(std::move(buffer)),
+ m_options(options),
+ m_utf8_prefix(options.use_color ? color_red : ""),
+ m_utf8_suffix(options.use_color ? color_blue : "") {
}
DebugOutputBlock(const DebugOutputBlock&) = default;
@@ -269,13 +252,15 @@ namespace osmium {
DebugOutputBlock(DebugOutputBlock&&) = default;
DebugOutputBlock& operator=(DebugOutputBlock&&) = default;
- ~DebugOutputBlock() = default;
+ ~DebugOutputBlock() noexcept = default;
std::string operator()() {
osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this);
std::string out;
- std::swap(out, *m_out);
+ using std::swap;
+ swap(out, *m_out);
+
return out;
}
@@ -313,7 +298,8 @@ namespace osmium {
int width = int(log10(way.nodes().size())) + 1;
int n = 0;
for (const auto& node_ref : way.nodes()) {
- output_formatted(" %0*d: %10" PRId64, width, n++, node_ref.ref());
+ write_counter(width, n++);
+ output_formatted("%10" PRId64, node_ref.ref());
if (node_ref.location().valid()) {
output_formatted(" (%.7f,%.7f)", node_ref.location().lon_without_check(), node_ref.location().lat_without_check());
}
@@ -335,7 +321,7 @@ namespace osmium {
int width = int(log10(relation.members().size())) + 1;
int n = 0;
for (const auto& member : relation.members()) {
- output_formatted(" %0*d: ", width, n++);
+ write_counter(width, n++);
*m_out += short_typename[item_type_to_nwr_index(member.type())];
output_formatted(" %10" PRId64 " ", member.ref());
write_string(member.role());
@@ -348,24 +334,26 @@ namespace osmium {
void changeset(const osmium::Changeset& changeset) {
write_object_type("changeset");
output_formatted("%d\n", changeset.id());
+
write_fieldname("num changes");
output_formatted("%d", changeset.num_changes());
if (changeset.num_changes() == 0) {
write_error(" NO CHANGES!");
}
*m_out += '\n';
+
write_fieldname("created at");
*m_out += ' ';
- *m_out += changeset.created_at().to_iso();
- output_formatted(" (%d)\n", changeset.created_at());
+ write_timestamp(changeset.created_at());
+
write_fieldname("closed at");
*m_out += " ";
if (changeset.closed()) {
- *m_out += changeset.closed_at().to_iso();
- output_formatted(" (%d)\n", changeset.closed_at());
+ write_timestamp(changeset.closed_at());
} else {
write_error("OPEN!\n");
}
+
write_fieldname("user");
output_formatted(" %d ", changeset.uid());
write_string(changeset.user());
@@ -374,51 +362,73 @@ namespace osmium {
write_box(changeset.bounds());
write_tags(changeset.tags(), " ");
- *m_out += '\n';
- }
+ if (changeset.num_comments() > 0) {
+ write_fieldname("comments");
+ output_formatted(" %d\n", changeset.num_comments());
- }; // DebugOutputBlock
+ int width = int(log10(changeset.num_comments())) + 1;
+ int n = 0;
+ for (const auto& comment : changeset.discussion()) {
+ write_counter(width, n++);
- class DebugOutputFormat : public osmium::io::detail::OutputFormat {
+ write_comment_field("date");
+ write_timestamp(comment.date());
+ output_formatted(" %*s", width, "");
- bool m_add_metadata;
- bool m_use_color;
+ write_comment_field("user");
+ output_formatted("%d ", comment.uid());
+ write_string(comment.user());
+ output_formatted("\n %*s", width, "");
- public:
+ write_comment_field("text");
+ write_string(comment.text());
+ *m_out += '\n';
+ }
+ }
- DebugOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
- OutputFormat(file, output_queue),
- m_add_metadata(file.get("add_metadata") != "false"),
- m_use_color(file.get("color") == "true") {
+ *m_out += '\n';
}
- DebugOutputFormat(const DebugOutputFormat&) = delete;
- DebugOutputFormat& operator=(const DebugOutputFormat&) = delete;
+ }; // class DebugOutputBlock
- void write_buffer(osmium::memory::Buffer&& buffer) override final {
- m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_add_metadata, m_use_color}));
- }
+ class DebugOutputFormat : public osmium::io::detail::OutputFormat {
+
+ debug_output_options m_options;
void write_fieldname(std::string& out, const char* name) {
out += " ";
- if (m_use_color) {
+ if (m_options.use_color) {
out += color_cyan;
}
out += name;
- if (m_use_color) {
+ if (m_options.use_color) {
out += color_reset;
}
out += ": ";
}
+ public:
+
+ DebugOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) :
+ OutputFormat(output_queue),
+ m_options() {
+ m_options.add_metadata = file.is_not_false("add_metadata");
+ m_options.use_color = file.is_true("color");
+ }
+
+ DebugOutputFormat(const DebugOutputFormat&) = delete;
+ DebugOutputFormat& operator=(const DebugOutputFormat&) = delete;
+
+ ~DebugOutputFormat() noexcept = default;
+
void write_header(const osmium::io::Header& header) override final {
std::string out;
- if (m_use_color) {
+ if (m_options.use_color) {
out += color_bold;
}
out += "header\n";
- if (m_use_color) {
+ if (m_options.use_color) {
out += color_reset;
}
@@ -445,33 +455,26 @@ namespace osmium {
}
out += "\n=============================================\n\n";
- std::promise<std::string> promise;
- m_output_queue.push(promise.get_future());
- promise.set_value(std::move(out));
+ send_to_output_queue(std::move(out));
}
- void close() override final {
- std::string out;
- std::promise<std::string> promise;
- m_output_queue.push(promise.get_future());
- promise.set_value(out);
+ void write_buffer(osmium::memory::Buffer&& buffer) override final {
+ m_output_queue.push(osmium::thread::Pool::instance().submit(DebugOutputBlock{std::move(buffer), m_options}));
}
}; // class DebugOutputFormat
- namespace {
-
-// we want the register_output_format() function to run, setting the variable
-// is only a side-effect, it will never be used
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-variable"
- const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug,
- [](const osmium::io::File& file, data_queue_type& output_queue) {
- return new osmium::io::detail::DebugOutputFormat(file, output_queue);
- });
-#pragma GCC diagnostic pop
-
- } // anonymous namespace
+ // we want the register_output_format() function to run, setting
+ // the variable is only a side-effect, it will never be used
+ const bool registered_debug_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::debug,
+ [](const osmium::io::File& file, future_string_queue_type& output_queue) {
+ return new osmium::io::detail::DebugOutputFormat(file, output_queue);
+ });
+
+ // dummy function to silence the unused variable warning from above
+ inline bool get_registered_debug_output() noexcept {
+ return registered_debug_output;
+ }
} // namespace detail
diff --git a/include/osmium/io/detail/input_format.hpp b/include/osmium/io/detail/input_format.hpp
index 743845c..d26b1ee 100644
--- a/include/osmium/io/detail/input_format.hpp
+++ b/include/osmium/io/detail/input_format.hpp
@@ -33,13 +33,16 @@ DEALINGS IN THE SOFTWARE.
*/
+#include <exception>
#include <functional>
+#include <future>
#include <map>
#include <memory>
#include <stdexcept>
#include <string>
#include <utility>
+#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/header.hpp>
@@ -48,103 +51,156 @@ DEALINGS IN THE SOFTWARE.
namespace osmium {
- namespace thread {
- template <typename T> class Queue;
- } // namespace thread
-
namespace io {
namespace detail {
- /**
- * Virtual base class for all classes reading OSM files in different
- * formats.
- *
- * Do not use this class or derived classes directly. Use the
- * osmium::io::Reader class instead.
- */
- class InputFormat {
+ class Parser {
+
+ future_buffer_queue_type& m_output_queue;
+ std::promise<osmium::io::Header>& m_header_promise;
+ queue_wrapper<std::string> m_input_queue;
+ osmium::osm_entity_bits::type m_read_types;
+ bool m_header_is_done;
protected:
- osmium::io::File m_file;
- osmium::osm_entity_bits::type m_read_which_entities;
- osmium::io::Header m_header;
+ std::string get_input() {
+ return m_input_queue.pop();
+ }
- explicit InputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities) :
- m_file(file),
- m_read_which_entities(read_which_entities) {
- m_header.set_has_multiple_object_versions(m_file.has_multiple_object_versions());
+ bool input_done() const {
+ return m_input_queue.has_reached_end_of_data();
}
- InputFormat(const InputFormat&) = delete;
- InputFormat(InputFormat&&) = delete;
+ osmium::osm_entity_bits::type read_types() const {
+ return m_read_types;
+ }
- InputFormat& operator=(const InputFormat&) = delete;
- InputFormat& operator=(InputFormat&&) = delete;
+ bool header_is_done() const {
+ return m_header_is_done;
+ }
- public:
+ void set_header_value(const osmium::io::Header& header) {
+ if (!m_header_is_done) {
+ m_header_is_done = true;
+ m_header_promise.set_value(header);
+ }
+ }
- virtual ~InputFormat() {
+ void set_header_exception(const std::exception_ptr& exception) {
+ if (!m_header_is_done) {
+ m_header_is_done = true;
+ m_header_promise.set_exception(exception);
+ }
}
- virtual osmium::memory::Buffer read() = 0;
+ /**
+ * Wrap the buffer into a future and add it to the output queue.
+ */
+ void send_to_output_queue(osmium::memory::Buffer&& buffer) {
+ add_to_queue(m_output_queue, std::move(buffer));
+ }
+
+ void send_to_output_queue(std::future<osmium::memory::Buffer>&& future) {
+ m_output_queue.push(std::move(future));
+ }
- virtual void close() {
+ public:
+
+ Parser(future_string_queue_type& input_queue,
+ future_buffer_queue_type& output_queue,
+ std::promise<osmium::io::Header>& header_promise,
+ osmium::osm_entity_bits::type read_types) :
+ m_output_queue(output_queue),
+ m_header_promise(header_promise),
+ m_input_queue(input_queue),
+ m_read_types(read_types),
+ m_header_is_done(false) {
}
- virtual osmium::io::Header header() {
- return m_header;
+ Parser(const Parser&) = delete;
+ Parser& operator=(const Parser&) = delete;
+
+ Parser(Parser&&) = delete;
+ Parser& operator=(Parser&&) = delete;
+
+ virtual ~Parser() noexcept = default;
+
+ virtual void run() = 0;
+
+ void parse() {
+ try {
+ run();
+ } catch (...) {
+ std::exception_ptr exception = std::current_exception();
+ set_header_exception(exception);
+ add_to_queue(m_output_queue, std::move(exception));
+ }
+
+ add_end_of_data_to_queue(m_output_queue);
}
- }; // class InputFormat
+ }; // class Parser
/**
- * This factory class is used to create objects that read OSM data
- * written in a specified format.
+ * This factory class is used to create objects that decode OSM
+ * data written in a specified format.
*
- * Do not use this class directly. Instead use the osmium::io::Reader
- * class.
+ * Do not use this class directly. Use the osmium::io::Reader
+ * class instead.
*/
- class InputFormatFactory {
+ class ParserFactory {
public:
- typedef std::function<osmium::io::detail::InputFormat*(const osmium::io::File&, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>&)> create_input_type;
+ typedef std::function<
+ std::unique_ptr<Parser>(
+ future_string_queue_type&,
+ future_buffer_queue_type&,
+ std::promise<osmium::io::Header>& header_promise,
+ osmium::osm_entity_bits::type read_which_entities
+ )
+ > create_parser_type;
private:
- typedef std::map<osmium::io::file_format, create_input_type> map_type;
+ typedef std::map<osmium::io::file_format, create_parser_type> map_type;
map_type m_callbacks;
- InputFormatFactory() :
+ ParserFactory() :
m_callbacks() {
}
public:
- static InputFormatFactory& instance() {
- static InputFormatFactory factory;
+ static ParserFactory& instance() {
+ static ParserFactory factory;
return factory;
}
- bool register_input_format(osmium::io::file_format format, create_input_type create_function) {
+ bool register_parser(osmium::io::file_format format, create_parser_type create_function) {
if (! m_callbacks.insert(map_type::value_type(format, create_function)).second) {
return false;
}
return true;
}
- create_input_type* get_creator_function(const osmium::io::File& file) {
+ create_parser_type get_creator_function(const osmium::io::File& file) {
auto it = m_callbacks.find(file.format());
if (it == m_callbacks.end()) {
- throw std::runtime_error(std::string("Can not open file '") + file.filename() + "' with type '" + as_string(file.format()) + "'. No support for reading this format in this program.");
+ throw unsupported_file_format_error(
+ std::string("Can not open file '") +
+ file.filename() +
+ "' with type '" +
+ as_string(file.format()) +
+ "'. No support for reading this format in this program.");
}
- return &(it->second);
+ return it->second;
}
- }; // class InputFormatFactory
+ }; // class ParserFactory
} // namespace detail
diff --git a/include/osmium/io/detail/o5m_input_format.hpp b/include/osmium/io/detail/o5m_input_format.hpp
new file mode 100644
index 0000000..b5b5c71
--- /dev/null
+++ b/include/osmium/io/detail/o5m_input_format.hpp
@@ -0,0 +1,633 @@
+#ifndef OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP
+#define OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen at topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <future>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include <protozero/varint.hpp>
+
+#include <osmium/builder/builder.hpp>
+#include <osmium/builder/osm_object_builder.hpp>
+#include <osmium/io/detail/input_format.hpp>
+#include <osmium/io/detail/queue_util.hpp>
+#include <osmium/io/error.hpp>
+#include <osmium/io/file_format.hpp>
+#include <osmium/io/header.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm.hpp>
+#include <osmium/osm/box.hpp>
+#include <osmium/osm/entity_bits.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/location.hpp>
+#include <osmium/osm/object.hpp>
+#include <osmium/osm/types.hpp>
+#include <osmium/util/cast.hpp>
+#include <osmium/util/delta.hpp>
+
+namespace osmium {
+
+ /**
+ * Exception thrown when the o5m deocder failed. The exception contains
+ * (if available) information about the place where the error happened
+ * and the type of error.
+ */
+ struct o5m_error : public io_error {
+
+ o5m_error(const char* what) :
+ io_error(std::string("o5m format error: ") + what) {
+ }
+
+ }; // struct o5m_error
+
+ namespace io {
+
+ namespace detail {
+
+ // Implementation of the o5m/o5c file formats according to the
+ // description at http://wiki.openstreetmap.org/wiki/O5m .
+
+ class ReferenceTable {
+
+ // The following settings are from the o5m description:
+
+ // The maximum number of entries in this table.
+ const uint64_t number_of_entries = 15000;
+
+ // The size of one entry in the table.
+ const unsigned int entry_size = 256;
+
+ // The maximum length of a string in the table including
+ // two \0 bytes.
+ const unsigned int max_length = 250 + 2;
+
+ // The data is stored in this string. It is default constructed
+ // and then resized on demand the first time something is added.
+ // This is done because the ReferenceTable is in a O5mParser
+ // object which will be copied from one thread to another. This
+ // way the string is still small when it is copied.
+ std::string m_table;
+
+ unsigned int current_entry = 0;
+
+ public:
+
+ void clear() {
+ current_entry = 0;
+ }
+
+ void add(const char* string, size_t size) {
+ if (m_table.empty()) {
+ m_table.resize(entry_size * number_of_entries);
+ }
+ if (size <= max_length) {
+ std::copy_n(string, size, &m_table[current_entry * entry_size]);
+ if (++current_entry == number_of_entries) {
+ current_entry = 0;
+ }
+ }
+ }
+
+ const char* get(uint64_t index) const {
+ if (m_table.empty() || index == 0 || index > number_of_entries) {
+ throw o5m_error("reference to non-existing string in table");
+ }
+ auto entry = (current_entry + number_of_entries - index) % number_of_entries;
+ return &m_table[entry * entry_size];
+ }
+
+ }; // class ReferenceTable
+
+ class O5mParser : public Parser {
+
+ static constexpr int buffer_size = 2 * 1000 * 1000;
+
+ osmium::io::Header m_header;
+
+ osmium::memory::Buffer m_buffer;
+
+ std::string m_input;
+
+ const char* m_data;
+ const char* m_end;
+
+ ReferenceTable m_reference_table;
+
+ static int64_t zvarint(const char** data, const char* end) {
+ return protozero::decode_zigzag64(protozero::decode_varint(data, end));
+ }
+
+ bool ensure_bytes_available(size_t need_bytes) {
+ if ((m_end - m_data) >= long(need_bytes)) {
+ return true;
+ }
+
+ if (input_done() && (m_input.size() < need_bytes)) {
+ return false;
+ }
+
+ m_input.erase(0, m_data - m_input.data());
+
+ while (m_input.size() < need_bytes) {
+ std::string data = get_input();
+ if (input_done()) {
+ return false;
+ }
+ m_input.append(data);
+ }
+
+ m_data = m_input.data();
+ m_end = m_input.data() + m_input.size();
+
+ return true;
+ }
+
+ void check_header_magic() {
+ static const unsigned char header_magic[] = { 0xff, 0xe0, 0x04, 'o', '5' };
+
+ if (std::strncmp(reinterpret_cast<const char*>(header_magic), m_data, sizeof(header_magic))) {
+ throw o5m_error("wrong header magic");
+ }
+
+ m_data += sizeof(header_magic);
+ }
+
+ void check_file_type() {
+ if (*m_data == 'm') { // o5m data file
+ m_header.set_has_multiple_object_versions(false);
+ } else if (*m_data == 'c') { // o5c change file
+ m_header.set_has_multiple_object_versions(true);
+ } else {
+ throw o5m_error("wrong header magic");
+ }
+
+ m_data++;
+ }
+
+ void check_file_format_version() {
+ if (*m_data != '2') {
+ throw o5m_error("wrong header magic");
+ }
+
+ m_data++;
+ }
+
+ void decode_header() {
+ if (! ensure_bytes_available(7)) { // overall length of header
+ throw o5m_error("file too short (incomplete header info)");
+ }
+
+ check_header_magic();
+ check_file_type();
+ check_file_format_version();
+ }
+
+ void mark_header_as_done() {
+ set_header_value(m_header);
+ }
+
+ osmium::util::DeltaDecode<osmium::object_id_type> m_delta_id;
+
+ osmium::util::DeltaDecode<int64_t> m_delta_timestamp;
+ osmium::util::DeltaDecode<osmium::changeset_id_type> m_delta_changeset;
+ osmium::util::DeltaDecode<int64_t> m_delta_lon;
+ osmium::util::DeltaDecode<int64_t> m_delta_lat;
+
+ osmium::util::DeltaDecode<osmium::object_id_type> m_delta_way_node_id;
+ osmium::util::DeltaDecode<osmium::object_id_type> m_delta_member_ids[3];
+
+ void reset() {
+ m_reference_table.clear();
+
+ m_delta_id.clear();
+ m_delta_timestamp.clear();
+ m_delta_changeset.clear();
+ m_delta_lon.clear();
+ m_delta_lat.clear();
+
+ m_delta_way_node_id.clear();
+ m_delta_member_ids[0].clear();
+ m_delta_member_ids[1].clear();
+ m_delta_member_ids[2].clear();
+ }
+
+ const char* decode_string(const char** dataptr, const char* const end) {
+ if (**dataptr == 0x00) { // get inline string
+ (*dataptr)++;
+ if (*dataptr == end) {
+ throw o5m_error("string format error");
+ }
+ return *dataptr;
+ } else { // get from reference table
+ auto index = protozero::decode_varint(dataptr, end);
+ return m_reference_table.get(index);
+ }
+ }
+
+ std::pair<osmium::user_id_type, const char*> decode_user(const char** dataptr, const char* const end) {
+ bool update_pointer = (**dataptr == 0x00);
+ const char* data = decode_string(dataptr, end);
+ const char* start = data;
+
+ auto uid = protozero::decode_varint(&data, end);
+
+ if (data == end) {
+ throw o5m_error("missing user name");
+ }
+
+ const char* user = ++data;
+
+ if (uid == 0 && update_pointer) {
+ m_reference_table.add("\0\0", 2);
+ *dataptr = data;
+ return std::make_pair(0, "");
+ }
+
+ while (*data++) {
+ if (data == end) {
+ throw o5m_error("no null byte in user name");
+ }
+ }
+
+ if (update_pointer) {
+ m_reference_table.add(start, data - start);
+ *dataptr = data;
+ }
+
+ return std::make_pair(static_cast_with_assert<osmium::user_id_type>(uid), user);
+ }
+
+ void decode_tags(osmium::builder::Builder* builder, const char** dataptr, const char* const end) {
+ osmium::builder::TagListBuilder tl_builder(m_buffer, builder);
+
+ while(*dataptr != end) {
+ bool update_pointer = (**dataptr == 0x00);
+ const char* data = decode_string(dataptr, end);
+ const char* start = data;
+
+ while (*data++) {
+ if (data == end) {
+ throw o5m_error("no null byte in tag key");
+ }
+ }
+
+ const char* value = data;
+ while (*data++) {
+ if (data == end) {
+ throw o5m_error("no null byte in tag value");
+ }
+ }
+
+ if (update_pointer) {
+ m_reference_table.add(start, data - start);
+ *dataptr = data;
+ }
+
+ tl_builder.add_tag(start, value);
+ }
+ }
+
+ const char* decode_info(osmium::OSMObject& object, const char** dataptr, const char* const end) {
+ const char* user = "";
+
+ if (**dataptr == 0x00) { // no info section
+ ++*dataptr;
+ } else { // has info section
+ object.set_version(static_cast_with_assert<object_version_type>(protozero::decode_varint(dataptr, end)));
+ auto timestamp = m_delta_timestamp.update(zvarint(dataptr, end));
+ if (timestamp != 0) { // has timestamp
+ object.set_timestamp(timestamp);
+ object.set_changeset(m_delta_changeset.update(zvarint(dataptr, end)));
+ if (*dataptr != end) {
+ auto uid_user = decode_user(dataptr, end);
+ object.set_uid(uid_user.first);
+ user = uid_user.second;
+ } else {
+ object.set_uid(user_id_type(0));
+ }
+ }
+ }
+
+ return user;
+ }
+
+ void decode_node(const char* data, const char* const end) {
+ osmium::builder::NodeBuilder builder(m_buffer);
+ osmium::Node& node = builder.object();
+
+ node.set_id(m_delta_id.update(zvarint(&data, end)));
+
+ builder.add_user(decode_info(node, &data, end));
+
+ if (data == end) {
+ // no location, object is deleted
+ builder.object().set_visible(false);
+ builder.object().set_location(osmium::Location{});
+ } else {
+ auto lon = m_delta_lon.update(zvarint(&data, end));
+ auto lat = m_delta_lat.update(zvarint(&data, end));
+ builder.object().set_location(osmium::Location{lon, lat});
+
+ if (data != end) {
+ decode_tags(&builder, &data, end);
+ }
+ }
+
+ m_buffer.commit();
+ }
+
+ void decode_way(const char* data, const char* const end) {
+ osmium::builder::WayBuilder builder(m_buffer);
+ osmium::Way& way = builder.object();
+
+ way.set_id(m_delta_id.update(zvarint(&data, end)));
+
+ builder.add_user(decode_info(way, &data, end));
+
+ if (data == end) {
+ // no reference section, object is deleted
+ builder.object().set_visible(false);
+ } else {
+ auto reference_section_length = protozero::decode_varint(&data, end);
+ if (reference_section_length > 0) {
+ const char* const end_refs = data + reference_section_length;
+ if (end_refs > end) {
+ throw o5m_error("way nodes ref section too long");
+ }
+
+ osmium::builder::WayNodeListBuilder wn_builder(m_buffer, &builder);
+
+ while (data < end_refs) {
+ wn_builder.add_node_ref(m_delta_way_node_id.update(zvarint(&data, end)));
+ }
+ }
+
+ if (data != end) {
+ decode_tags(&builder, &data, end);
+ }
+ }
+
+ m_buffer.commit();
+ }
+
+ osmium::item_type decode_member_type(char c) {
+ if (c < '0' || c > '2') {
+ throw o5m_error("unknown member type");
+ }
+ return osmium::nwr_index_to_item_type(c - '0');
+ }
+
+ std::pair<osmium::item_type, const char*> decode_role(const char** dataptr, const char* const end) {
+ bool update_pointer = (**dataptr == 0x00);
+ const char* data = decode_string(dataptr, end);
+ const char* start = data;
+
+ auto member_type = decode_member_type(*data++);
+ if (data == end) {
+ throw o5m_error("missing role");
+ }
+ const char* role = data;
+
+ while (*data++) {
+ if (data == end) {
+ throw o5m_error("no null byte in role");
+ }
+ }
+
+ if (update_pointer) {
+ m_reference_table.add(start, data - start);
+ *dataptr = data;
+ }
+
+ return std::make_pair(member_type, role);
+ }
+
+ void decode_relation(const char* data, const char* const end) {
+ osmium::builder::RelationBuilder builder(m_buffer);
+ osmium::Relation& relation = builder.object();
+
+ relation.set_id(m_delta_id.update(zvarint(&data, end)));
+
+ builder.add_user(decode_info(relation, &data, end));
+
+ if (data == end) {
+ // no reference section, object is deleted
+ builder.object().set_visible(false);
+ } else {
+ auto reference_section_length = protozero::decode_varint(&data, end);
+ if (reference_section_length > 0) {
+ const char* const end_refs = data + reference_section_length;
+ if (end_refs > end) {
+ throw o5m_error("relation format error");
+ }
+
+ osmium::builder::RelationMemberListBuilder rml_builder(m_buffer, &builder);
+
+ while (data < end_refs) {
+ auto delta_id = zvarint(&data, end);
+ if (data == end) {
+ throw o5m_error("relation member format error");
+ }
+ auto type_role = decode_role(&data, end);
+ auto i = osmium::item_type_to_nwr_index(type_role.first);
+ auto ref = m_delta_member_ids[i].update(delta_id);
+ rml_builder.add_member(type_role.first, ref, type_role.second);
+ }
+ }
+
+ if (data != end) {
+ decode_tags(&builder, &data, end);
+ }
+ }
+
+ m_buffer.commit();
+ }
+
+ void decode_bbox(const char* data, const char* const end) {
+ auto sw_lon = zvarint(&data, end);
+ auto sw_lat = zvarint(&data, end);
+ auto ne_lon = zvarint(&data, end);
+ auto ne_lat = zvarint(&data, end);
+
+ m_header.add_box(osmium::Box{osmium::Location{sw_lon, sw_lat},
+ osmium::Location{ne_lon, ne_lat}});
+ }
+
+ void decode_timestamp(const char* data, const char* const end) {
+ auto timestamp = osmium::Timestamp(zvarint(&data, end)).to_iso();
+ m_header.set("o5m_timestamp", timestamp);
+ m_header.set("timestamp", timestamp);
+ }
+
+ void flush() {
+ osmium::memory::Buffer buffer(buffer_size);
+ using std::swap;
+ swap(m_buffer, buffer);
+ send_to_output_queue(std::move(buffer));
+ }
+
+ enum class dataset_type : unsigned char {
+ node = 0x10,
+ way = 0x11,
+ relation = 0x12,
+ bounding_box = 0xdb,
+ timestamp = 0xdc,
+ header = 0xe0,
+ sync = 0xee,
+ jump = 0xef,
+ reset = 0xff
+ };
+
+ void decode_data() {
+ while (ensure_bytes_available(1)) {
+ dataset_type ds_type = dataset_type(*m_data++);
+ if (ds_type > dataset_type::jump) {
+ if (ds_type == dataset_type::reset) {
+ reset();
+ }
+ } else {
+ ensure_bytes_available(protozero::max_varint_length);
+
+ uint64_t length = 0;
+ try {
+ length = protozero::decode_varint(&m_data, m_end);
+ } catch (protozero::end_of_buffer_exception&) {
+ throw o5m_error("premature end of file");
+ }
+
+ if (! ensure_bytes_available(length)) {
+ throw o5m_error("premature end of file");
+ }
+
+ switch (ds_type) {
+ case dataset_type::node:
+ mark_header_as_done();
+ if (read_types() & osmium::osm_entity_bits::node) {
+ decode_node(m_data, m_data + length);
+ }
+ break;
+ case dataset_type::way:
+ mark_header_as_done();
+ if (read_types() & osmium::osm_entity_bits::way) {
+ decode_way(m_data, m_data + length);
+ }
+ break;
+ case dataset_type::relation:
+ mark_header_as_done();
+ if (read_types() & osmium::osm_entity_bits::relation) {
+ decode_relation(m_data, m_data + length);
+ }
+ break;
+ case dataset_type::bounding_box:
+ decode_bbox(m_data, m_data + length);
+ break;
+ case dataset_type::timestamp:
+ decode_timestamp(m_data, m_data + length);
+ break;
+ default:
+ // ignore unknown datasets
+ break;
+ }
+
+ if (read_types() == osmium::osm_entity_bits::nothing && header_is_done()) {
+ break;
+ }
+
+ m_data += length;
+
+ if (m_buffer.committed() > buffer_size / 10 * 9) {
+ flush();
+ }
+ }
+ }
+
+ if (m_buffer.committed()) {
+ flush();
+ }
+
+ mark_header_as_done();
+ }
+
+ public:
+
+ O5mParser(future_string_queue_type& input_queue,
+ future_buffer_queue_type& output_queue,
+ std::promise<osmium::io::Header>& header_promise,
+ osmium::osm_entity_bits::type read_types) :
+ Parser(input_queue, output_queue, header_promise, read_types),
+ m_header(),
+ m_buffer(buffer_size),
+ m_input(),
+ m_data(m_input.data()),
+ m_end(m_data) {
+ }
+
+ ~O5mParser() noexcept = default;
+
+ void run() override final {
+ decode_header();
+ decode_data();
+ }
+
+ }; // class O5mParser
+
+ // we want the register_parser() function to run, setting
+ // the variable is only a side-effect, it will never be used
+ const bool registered_o5m_parser = ParserFactory::instance().register_parser(
+ file_format::o5m,
+ [](future_string_queue_type& input_queue,
+ future_buffer_queue_type& output_queue,
+ std::promise<osmium::io::Header>& header_promise,
+ osmium::osm_entity_bits::type read_which_entities) {
+ return std::unique_ptr<Parser>(new O5mParser(input_queue, output_queue, header_promise, read_which_entities));
+ });
+
+ // dummy function to silence the unused variable warning from above
+ inline bool get_registered_o5m_parser() noexcept {
+ return registered_o5m_parser;
+ }
+
+ } // namespace detail
+
+ } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_O5M_INPUT_FORMAT_HPP
diff --git a/include/osmium/io/detail/opl_output_format.hpp b/include/osmium/io/detail/opl_output_format.hpp
index a3103d9..5c40bf2 100644
--- a/include/osmium/io/detail/opl_output_format.hpp
+++ b/include/osmium/io/detail/opl_output_format.hpp
@@ -33,7 +33,6 @@ DEALINGS IN THE SOFTWARE.
*/
-#include <chrono>
#include <cinttypes>
#include <cstddef>
#include <cstdint>
@@ -41,14 +40,10 @@ DEALINGS IN THE SOFTWARE.
#include <future>
#include <iterator>
#include <memory>
-#include <ratio>
#include <string>
#include <thread>
#include <utility>
-#include <utf8.h>
-
-#include <osmium/handler.hpp>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/memory/buffer.hpp>
@@ -74,71 +69,27 @@ namespace osmium {
namespace detail {
- /**
- * Writes out one buffer with OSM data in OPL format.
- */
- class OPLOutputBlock : public osmium::handler::Handler {
-
- static constexpr size_t tmp_buffer_size = 100;
+ struct opl_output_options {
- std::shared_ptr<osmium::memory::Buffer> m_input_buffer;
+ /// Should metadata of objects be added?
+ bool add_metadata;
- std::shared_ptr<std::string> m_out;
+ };
- char m_tmp_buffer[tmp_buffer_size+1];
-
- bool m_add_metadata;
+ /**
+ * Writes out one buffer with OSM data in OPL format.
+ */
+ class OPLOutputBlock : public OutputBlock {
- template <typename... TArgs>
- void output_formatted(const char* format, TArgs&&... args) {
-#ifndef NDEBUG
- int len =
-#endif
-#ifndef _MSC_VER
- snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward<TArgs>(args)...);
-#else
- _snprintf(m_tmp_buffer, tmp_buffer_size, format, std::forward<TArgs>(args)...);
-#endif
- assert(len > 0 && static_cast<size_t>(len) < tmp_buffer_size);
- *m_out += m_tmp_buffer;
- }
+ opl_output_options m_options;
void append_encoded_string(const char* data) {
- const char* end = data + std::strlen(data);
-
- while (data != end) {
- const char* last = data;
- uint32_t c = utf8::next(data, end);
-
- // This is a list of Unicode code points that we let
- // through instead of escaping them. It is incomplete
- // and can be extended later.
- // Generally we don't want to let through any character
- // that has special meaning in the OPL format such as
- // space, comma, @, etc. and any non-printing characters.
- if ((0x0021 <= c && c <= 0x0024) ||
- (0x0026 <= c && c <= 0x002b) ||
- (0x002d <= c && c <= 0x003c) ||
- (0x003e <= c && c <= 0x003f) ||
- (0x0041 <= c && c <= 0x007e) ||
- (0x00a1 <= c && c <= 0x00ac) ||
- (0x00ae <= c && c <= 0x05ff)) {
- m_out->append(last, data);
- } else {
- *m_out += '%';
- if (c <= 0xff) {
- output_formatted("%02x", c);
- } else {
- output_formatted("%04x", c);
- }
- *m_out += '%';
- }
- }
+ osmium::io::detail::append_utf8_encoded_string(*m_out, data);
}
void write_meta(const osmium::OSMObject& object) {
output_formatted("%" PRId64, object.id());
- if (m_add_metadata) {
+ if (m_options.add_metadata) {
output_formatted(" v%d d", object.version());
*m_out += (object.visible() ? 'V' : 'D');
output_formatted(" c%d t", object.changeset());
@@ -160,7 +111,7 @@ namespace osmium {
}
}
- void write_location(const osmium::Location location, const char x, const char y) {
+ void write_location(const osmium::Location& location, const char x, const char y) {
if (location) {
output_formatted(" %c%.7f %c%.7f", x, location.lon_without_check(), y, location.lat_without_check());
} else {
@@ -173,11 +124,9 @@ namespace osmium {
public:
- explicit OPLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata) :
- m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
- m_out(std::make_shared<std::string>()),
- m_tmp_buffer(),
- m_add_metadata(add_metadata) {
+ OPLOutputBlock(osmium::memory::Buffer&& buffer, const opl_output_options& options) :
+ OutputBlock(std::move(buffer)),
+ m_options(options) {
}
OPLOutputBlock(const OPLOutputBlock&) = default;
@@ -186,13 +135,15 @@ namespace osmium {
OPLOutputBlock(OPLOutputBlock&&) = default;
OPLOutputBlock& operator=(OPLOutputBlock&&) = default;
- ~OPLOutputBlock() = default;
+ ~OPLOutputBlock() noexcept = default;
std::string operator()() {
osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this);
std::string out;
- std::swap(out, *m_out);
+ using std::swap;
+ swap(out, *m_out);
+
return out;
}
@@ -244,7 +195,7 @@ namespace osmium {
*m_out += changeset.created_at().to_iso();
*m_out += " e";
*m_out += changeset.closed_at().to_iso();
- output_formatted(" i%d u", changeset.uid());
+ output_formatted(" d%d i%d u", changeset.num_comments(), changeset.uid());
append_encoded_string(changeset.user());
write_location(changeset.bounds().bottom_left(), 'x', 'y');
write_location(changeset.bounds().top_right(), 'X', 'Y');
@@ -264,48 +215,42 @@ namespace osmium {
*m_out += '\n';
}
- }; // OPLOutputBlock
+ }; // class OPLOutputBlock
class OPLOutputFormat : public osmium::io::detail::OutputFormat {
- bool m_add_metadata;
+ opl_output_options m_options;
public:
- OPLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
- OutputFormat(file, output_queue),
- m_add_metadata(file.get("add_metadata") != "false") {
+ OPLOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) :
+ OutputFormat(output_queue),
+ m_options() {
+ m_options.add_metadata = file.is_not_false("add_metadata");
}
OPLOutputFormat(const OPLOutputFormat&) = delete;
OPLOutputFormat& operator=(const OPLOutputFormat&) = delete;
- void write_buffer(osmium::memory::Buffer&& buffer) override final {
- m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_add_metadata}));
- }
+ ~OPLOutputFormat() noexcept = default;
- void close() override final {
- std::string out;
- std::promise<std::string> promise;
- m_output_queue.push(promise.get_future());
- promise.set_value(out);
+ void write_buffer(osmium::memory::Buffer&& buffer) override final {
+ m_output_queue.push(osmium::thread::Pool::instance().submit(OPLOutputBlock{std::move(buffer), m_options}));
}
}; // class OPLOutputFormat
- namespace {
-
-// we want the register_output_format() function to run, setting the variable
-// is only a side-effect, it will never be used
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-variable"
- const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl,
- [](const osmium::io::File& file, data_queue_type& output_queue) {
- return new osmium::io::detail::OPLOutputFormat(file, output_queue);
- });
-#pragma GCC diagnostic pop
-
- } // anonymous namespace
+ // we want the register_output_format() function to run, setting
+ // the variable is only a side-effect, it will never be used
+ const bool registered_opl_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::opl,
+ [](const osmium::io::File& file, future_string_queue_type& output_queue) {
+ return new osmium::io::detail::OPLOutputFormat(file, output_queue);
+ });
+
+ // dummy function to silence the unused variable warning from above
+ inline bool get_registered_opl_output() noexcept {
+ return registered_opl_output;
+ }
} // namespace detail
diff --git a/include/osmium/io/detail/output_format.hpp b/include/osmium/io/detail/output_format.hpp
index ed48b38..ba1fb41 100644
--- a/include/osmium/io/detail/output_format.hpp
+++ b/include/osmium/io/detail/output_format.hpp
@@ -34,29 +34,48 @@ DEALINGS IN THE SOFTWARE.
*/
#include <functional>
-#include <future>
#include <map>
#include <memory>
#include <stdexcept>
#include <string>
#include <utility>
+#include <osmium/handler.hpp>
+#include <osmium/io/detail/queue_util.hpp>
+#include <osmium/io/detail/string_util.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
-#include <osmium/io/header.hpp>
-#include <osmium/thread/queue.hpp>
+#include <osmium/memory/buffer.hpp>
namespace osmium {
- namespace memory {
- class Buffer;
+ namespace io {
+ class Header;
}
namespace io {
namespace detail {
- typedef osmium::thread::Queue<std::future<std::string>> data_queue_type;
+ class OutputBlock : public osmium::handler::Handler {
+
+ protected:
+
+ std::shared_ptr<osmium::memory::Buffer> m_input_buffer;
+
+ std::shared_ptr<std::string> m_out;
+
+ explicit OutputBlock(osmium::memory::Buffer&& buffer) :
+ m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
+ m_out(std::make_shared<std::string>()) {
+ }
+
+ template <typename... TArgs>
+ void output_formatted(const char* format, TArgs&&... args) {
+ append_printf_formatted_string(*m_out, format, std::forward<TArgs>(args)...);
+ }
+
+ }; // class OutputBlock;
/**
* Virtual base class for all classes writing OSM files in different
@@ -69,13 +88,19 @@ namespace osmium {
protected:
- osmium::io::File m_file;
- data_queue_type& m_output_queue;
+ future_string_queue_type& m_output_queue;
+
+ /**
+ * Wrap the string into a future and add it to the output
+ * queue.
+ */
+ void send_to_output_queue(std::string&& data) {
+ add_to_queue(m_output_queue, std::move(data));
+ }
public:
- explicit OutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
- m_file(file),
+ explicit OutputFormat(future_string_queue_type& output_queue) :
m_output_queue(output_queue) {
}
@@ -85,15 +110,15 @@ namespace osmium {
OutputFormat& operator=(const OutputFormat&) = delete;
OutputFormat& operator=(OutputFormat&&) = delete;
- virtual ~OutputFormat() {
- }
+ virtual ~OutputFormat() noexcept = default;
virtual void write_header(const osmium::io::Header&) {
}
virtual void write_buffer(osmium::memory::Buffer&&) = 0;
- virtual void close() = 0;
+ virtual void write_end() {
+ }
}; // class OutputFormat
@@ -108,7 +133,7 @@ namespace osmium {
public:
- typedef std::function<osmium::io::detail::OutputFormat*(const osmium::io::File&, data_queue_type&)> create_output_type;
+ typedef std::function<osmium::io::detail::OutputFormat*(const osmium::io::File&, future_string_queue_type&)> create_output_type;
private:
@@ -134,13 +159,18 @@ namespace osmium {
return true;
}
- std::unique_ptr<osmium::io::detail::OutputFormat> create_output(const osmium::io::File& file, data_queue_type& output_queue) {
+ std::unique_ptr<osmium::io::detail::OutputFormat> create_output(const osmium::io::File& file, future_string_queue_type& output_queue) {
auto it = m_callbacks.find(file.format());
if (it != m_callbacks.end()) {
return std::unique_ptr<osmium::io::detail::OutputFormat>((it->second)(file, output_queue));
}
- throw std::runtime_error(std::string("Can not open file '") + file.filename() + "' with type '" + as_string(file.format()) + "'. No support for writing this format in this program.");
+ throw unsupported_file_format_error(
+ std::string("Can not open file '") +
+ file.filename() +
+ "' with type '" +
+ as_string(file.format()) +
+ "'. No support for writing this format in this program.");
}
}; // class OutputFormatFactory
diff --git a/include/osmium/io/detail/pbf.hpp b/include/osmium/io/detail/pbf.hpp
index 15e457a..4a32a63 100644
--- a/include/osmium/io/detail/pbf.hpp
+++ b/include/osmium/io/detail/pbf.hpp
@@ -33,6 +33,7 @@ DEALINGS IN THE SOFTWARE.
*/
+#include <cstdint>
#include <string>
// needed for htonl and ntohl
diff --git a/include/osmium/io/detail/pbf_decoder.hpp b/include/osmium/io/detail/pbf_decoder.hpp
index e9e0e82..09e09bf 100644
--- a/include/osmium/io/detail/pbf_decoder.hpp
+++ b/include/osmium/io/detail/pbf_decoder.hpp
@@ -37,8 +37,12 @@ DEALINGS IN THE SOFTWARE.
#include <cstdint>
#include <cstring>
#include <algorithm>
-#include <iterator>
#include <limits>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
#include <protozero/pbf_message.hpp>
@@ -527,10 +531,14 @@ namespace osmium {
builder.add_user("");
}
+ // even if the node isn't visible, there's still a record
+ // of its lat/lon in the dense arrays.
+ const auto lon = dense_longitude.update(*lons.first++);
+ const auto lat = dense_latitude.update(*lats.first++);
if (visible) {
builder.object().set_location(osmium::Location(
- convert_pbf_coordinate(dense_longitude.update(*lons.first++)),
- convert_pbf_coordinate(dense_latitude.update(*lats.first++))
+ convert_pbf_coordinate(lon),
+ convert_pbf_coordinate(lat)
));
}
@@ -557,7 +565,7 @@ namespace osmium {
public:
- explicit PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) :
+ PBFPrimitiveBlockDecoder(const ptr_len_type& data, osmium::osm_entity_bits::type read_types) :
m_data(data),
m_read_types(read_types) {
}
@@ -568,7 +576,7 @@ namespace osmium {
PBFPrimitiveBlockDecoder(PBFPrimitiveBlockDecoder&&) = delete;
PBFPrimitiveBlockDecoder& operator=(PBFPrimitiveBlockDecoder&&) = delete;
- ~PBFPrimitiveBlockDecoder() = default;
+ ~PBFPrimitiveBlockDecoder() noexcept = default;
osmium::memory::Buffer operator()() {
try {
@@ -699,7 +707,11 @@ namespace osmium {
header.set("generator", pbf_header_block.get_string());
break;
case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_timestamp:
- header.set("osmosis_replication_timestamp", osmium::Timestamp(pbf_header_block.get_int64()).to_iso());
+ {
+ auto timestamp = osmium::Timestamp(pbf_header_block.get_int64()).to_iso();
+ header.set("osmosis_replication_timestamp", timestamp);
+ header.set("timestamp", timestamp);
+ }
break;
case OSMFormat::HeaderBlock::optional_int64_osmosis_replication_sequence_number:
header.set("osmosis_replication_sequence_number", std::to_string(pbf_header_block.get_int64()));
@@ -746,7 +758,7 @@ namespace osmium {
PBFDataBlobDecoder(PBFDataBlobDecoder&&) = default;
PBFDataBlobDecoder& operator=(PBFDataBlobDecoder&&) = default;
- ~PBFDataBlobDecoder() = default;
+ ~PBFDataBlobDecoder() noexcept = default;
osmium::memory::Buffer operator()() {
std::string output;
diff --git a/include/osmium/io/detail/pbf_input_format.hpp b/include/osmium/io/detail/pbf_input_format.hpp
index 7817d27..4464fd7 100644
--- a/include/osmium/io/detail/pbf_input_format.hpp
+++ b/include/osmium/io/detail/pbf_input_format.hpp
@@ -34,17 +34,12 @@ DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
-#include <atomic>
#include <cassert>
-#include <chrono>
#include <cstddef>
#include <cstdint>
#include <cstring>
-#include <future>
#include <memory>
-#include <ratio>
#include <sstream>
-#include <stdexcept>
#include <string>
#include <thread>
#include <type_traits>
@@ -58,40 +53,22 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/io/error.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
-#include <osmium/memory/buffer.hpp>
#include <osmium/osm.hpp>
-#include <osmium/osm/box.hpp>
#include <osmium/osm/entity_bits.hpp>
-#include <osmium/osm/location.hpp>
#include <osmium/osm/object.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/thread/pool.hpp>
-#include <osmium/thread/queue.hpp>
#include <osmium/thread/util.hpp>
-#include <osmium/util/cast.hpp>
#include <osmium/util/config.hpp>
namespace osmium {
namespace io {
- class File;
-
namespace detail {
- /**
- * Class for parsing PBF files.
- */
- class PBFInputFormat : public osmium::io::detail::InputFormat {
-
- typedef osmium::thread::Queue<std::future<osmium::memory::Buffer>> queue_type;
+ class PBFParser : public Parser {
- bool m_use_thread_pool;
- bool m_eof { false };
- queue_type m_queue;
- std::atomic<bool> m_quit_input_thread;
- std::thread m_reader;
- osmium::thread::Queue<std::string>& m_input_queue;
std::string m_input_buffer;
/**
@@ -103,9 +80,8 @@ namespace osmium {
*/
std::string read_from_input_queue(size_t size) {
while (m_input_buffer.size() < size) {
- std::string new_data;
- m_input_queue.wait_and_pop(new_data);
- if (new_data.empty()) {
+ std::string new_data = get_input();
+ if (input_done()) {
throw osmium::pbf_error("truncated data (EOF encountered)");
}
m_input_buffer += new_data;
@@ -113,7 +89,10 @@ namespace osmium {
std::string output { m_input_buffer.substr(size) };
m_input_buffer.resize(size);
- std::swap(output, m_input_buffer);
+
+ using std::swap;
+ swap(output, m_input_buffer);
+
return output;
}
@@ -125,7 +104,7 @@ namespace osmium {
uint32_t size_in_network_byte_order;
try {
- std::string input_data = read_from_input_queue(sizeof(size_in_network_byte_order));
+ const std::string input_data = read_from_input_queue(sizeof(size_in_network_byte_order));
size_in_network_byte_order = *reinterpret_cast<const uint32_t*>(input_data.data());
} catch (osmium::pbf_error&) {
return 0; // EOF
@@ -174,125 +153,85 @@ namespace osmium {
size_t check_type_and_get_blob_size(const char* expected_type) {
assert(expected_type);
- auto size = read_blob_header_size_from_file();
+ const auto size = read_blob_header_size_from_file();
if (size == 0) { // EOF
return 0;
}
- std::string blob_header = read_from_input_queue(size);
+ const std::string blob_header = read_from_input_queue(size);
return decode_blob_header(protozero::pbf_message<FileFormat::BlobHeader>(blob_header), expected_type);
}
- void parse_osm_data(osmium::osm_entity_bits::type read_types) {
- osmium::thread::set_thread_name("_osmium_pbf_in");
+ std::string read_from_input_queue_with_check(size_t size) {
+ if (size > max_uncompressed_blob_size) {
+ throw osmium::pbf_error(std::string("invalid blob size: " +
+ std::to_string(size)));
+ }
+ return read_from_input_queue(size);
+ }
- while (auto size = check_type_and_get_blob_size("OSMData")) {
- std::string input_buffer = read_from_input_queue(size);
- if (input_buffer.size() > max_uncompressed_blob_size) {
- throw osmium::pbf_error(std::string("invalid blob size: " + std::to_string(input_buffer.size())));
- }
+ // Parse the header in the PBF OSMHeader blob.
+ void parse_header_blob() {
+ osmium::io::Header header;
+ const auto size = check_type_and_get_blob_size("OSMHeader");
+ header = decode_header(read_from_input_queue_with_check(size));
+ set_header_value(header);
+ }
- if (m_use_thread_pool) {
- m_queue.push(osmium::thread::Pool::instance().submit(PBFDataBlobDecoder{ std::move(input_buffer), read_types }));
- } else {
- std::promise<osmium::memory::Buffer> promise;
- m_queue.push(promise.get_future());
- PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types };
- promise.set_value(data_blob_parser());
- }
+ void parse_data_blobs() {
+ while (const auto size = check_type_and_get_blob_size("OSMData")) {
+ std::string input_buffer = read_from_input_queue_with_check(size);
- if (m_quit_input_thread) {
- return;
+ PBFDataBlobDecoder data_blob_parser{ std::move(input_buffer), read_types() };
+
+ if (osmium::config::use_pool_threads_for_pbf_parsing()) {
+ send_to_output_queue(osmium::thread::Pool::instance().submit(std::move(data_blob_parser)));
+ } else {
+ send_to_output_queue(data_blob_parser());
}
}
-
- // Send an empty buffer to signal the reader that we are
- // done.
- std::promise<osmium::memory::Buffer> promise;
- m_queue.push(promise.get_future());
- promise.set_value(osmium::memory::Buffer{});
- }
-
- void signal_input_thread_to_quit() {
- m_quit_input_thread = true;
}
public:
- /**
- * Instantiate PBF Parser
- *
- * @param file osmium::io::File instance describing file to be read from.
- * @param read_which_entities Which types of OSM entities (nodes, ways, relations, changesets) should be parsed?
- * @param input_queue String queue where data is read from.
- */
- PBFInputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) :
- osmium::io::detail::InputFormat(file, read_which_entities),
- m_use_thread_pool(osmium::config::use_pool_threads_for_pbf_parsing()),
- m_queue(20, "pbf_parser_results"), // XXX
- m_quit_input_thread(false),
- m_input_queue(input_queue),
+ PBFParser(future_string_queue_type& input_queue,
+ future_buffer_queue_type& output_queue,
+ std::promise<osmium::io::Header>& header_promise,
+ osmium::osm_entity_bits::type read_types) :
+ Parser(input_queue, output_queue, header_promise, read_types),
m_input_buffer() {
-
- // handle OSMHeader
- const auto size = check_type_and_get_blob_size("OSMHeader");
- m_header = decode_header(read_from_input_queue(size));
-
- if (m_read_which_entities != osmium::osm_entity_bits::nothing) {
- m_reader = std::thread(&PBFInputFormat::parse_osm_data, this, m_read_which_entities);
- }
}
- ~PBFInputFormat() {
- signal_input_thread_to_quit();
- if (m_reader.joinable()) {
- m_reader.join();
- }
- }
+ ~PBFParser() noexcept = default;
- /**
- * Returns the next buffer with OSM data read from the PBF
- * file. Blocks if data is not available yet.
- * Returns an empty buffer at end of input.
- */
- osmium::memory::Buffer read() override {
- osmium::memory::Buffer buffer;
- if (m_eof) {
- return buffer;
- }
+ void run() override final {
+ osmium::thread::set_thread_name("_osmium_pbf_in");
- std::future<osmium::memory::Buffer> buffer_future;
- m_queue.wait_and_pop(buffer_future);
+ parse_header_blob();
- try {
- buffer = std::move(buffer_future.get());
- if (!buffer) {
- m_eof = true;
- }
- return buffer;
- } catch (...) {
- m_eof = true;
- signal_input_thread_to_quit();
- throw;
+ if (read_types() != osmium::osm_entity_bits::nothing) {
+ parse_data_blobs();
}
}
- }; // class PBFInputFormat
-
- namespace {
-
-// we want the register_input_format() function to run, setting the variable
-// is only a side-effect, it will never be used
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-variable"
- const bool registered_pbf_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::pbf,
- [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) {
- return new osmium::io::detail::PBFInputFormat(file, read_which_entities, input_queue);
- });
-#pragma GCC diagnostic pop
-
- } // anonymous namespace
+ }; // class PBFParser
+
+ // we want the register_parser() function to run, setting
+ // the variable is only a side-effect, it will never be used
+ const bool registered_pbf_parser = ParserFactory::instance().register_parser(
+ file_format::pbf,
+ [](future_string_queue_type& input_queue,
+ future_buffer_queue_type& output_queue,
+ std::promise<osmium::io::Header>& header_promise,
+ osmium::osm_entity_bits::type read_which_entities) {
+ return std::unique_ptr<Parser>(new PBFParser(input_queue, output_queue, header_promise, read_which_entities));
+ });
+
+ // dummy function to silence the unused variable warning from above
+ inline bool get_registered_pbf_parser() noexcept {
+ return registered_pbf_parser;
+ }
} // namespace detail
diff --git a/include/osmium/io/detail/pbf_output_format.hpp b/include/osmium/io/detail/pbf_output_format.hpp
index 9098b47..97b143b 100644
--- a/include/osmium/io/detail/pbf_output_format.hpp
+++ b/include/osmium/io/detail/pbf_output_format.hpp
@@ -34,16 +34,13 @@ DEALINGS IN THE SOFTWARE.
*/
#include <algorithm>
-#include <chrono>
+#include <cassert>
#include <cmath>
#include <cstdint>
#include <cstdlib>
-#include <future>
#include <iterator>
#include <memory>
-#include <ratio>
#include <string>
-#include <thread>
#include <time.h>
#include <utility>
@@ -73,6 +70,7 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/osm/tag.hpp>
#include <osmium/osm/timestamp.hpp>
#include <osmium/osm/way.hpp>
+#include <osmium/thread/pool.hpp>
#include <osmium/util/cast.hpp>
#include <osmium/util/delta.hpp>
#include <osmium/visitor.hpp>
@@ -83,6 +81,32 @@ namespace osmium {
namespace detail {
+ struct pbf_output_options {
+
+ /// Should nodes be encoded in DenseNodes?
+ bool use_dense_nodes;
+
+ /**
+ * Should the PBF blobs contain zlib compressed data?
+ *
+ * The zlib compression is optional, it's possible to store the
+ * blobs in raw format. Disabling the compression can improve
+ * the writing speed a little but the output will be 2x to 3x
+ * bigger.
+ */
+ bool use_compression;
+
+ /// Should metadata of objects be written?
+ bool add_metadata;
+
+ /// Add the "HistoricalInformation" header flag.
+ bool add_historical_information_flag;
+
+ /// Should the visible flag be added to all OSM objects?
+ bool add_visible_flag;
+
+ };
+
/**
* Maximum number of items in a primitive block.
*
@@ -106,45 +130,81 @@ namespace osmium {
return static_cast<int64_t>(std::round(lonlat * lonlat_resolution / location_granularity));
}
- /**
- * Serialize a protobuf message into a Blob, optionally apply compression
- * and return it together with a BlobHeader ready to be written to a file.
- *
- * @param type Type-string used in the BlobHeader.
- * @param msg Protobuf-message.
- * @param use_compression Should the output be compressed using zlib?
- */
- inline std::string serialize_blob(const std::string& type, const std::string& msg, bool use_compression) {
- assert(msg.size() <= max_uncompressed_blob_size);
+ enum class pbf_blob_type {
+ header = 0,
+ data = 1
+ };
+
+ class SerializeBlob {
+
+ std::string m_msg;
- std::string blob_data;
- protozero::pbf_builder<FileFormat::Blob> pbf_blob(blob_data);
+ pbf_blob_type m_blob_type;
- if (use_compression) {
- pbf_blob.add_int32(FileFormat::Blob::optional_int32_raw_size, int32_t(msg.size()));
- pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_zlib_data, osmium::io::detail::zlib_compress(msg));
- } else {
- pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_raw, msg);
+ bool m_use_compression;
+
+ public:
+
+ /**
+ * Initialize a blob serializer.
+ *
+ * @param msg Protobuf-message containing the blob data
+ * @param type Type of blob.
+ * @param use_compression Should the output be compressed using
+ * zlib?
+ */
+ SerializeBlob(std::string&& msg, pbf_blob_type type, bool use_compression) :
+ m_msg(std::move(msg)),
+ m_blob_type(type),
+ m_use_compression(use_compression) {
}
- std::string blob_header_data;
- protozero::pbf_builder<FileFormat::BlobHeader> pbf_blob_header(blob_header_data);
+ /**
+ * Serialize a protobuf message into a Blob, optionally apply
+ * compression and return it together with a BlobHeader ready
+ * to be written to a file.
+ */
+ std::string operator()() {
+ assert(m_msg.size() <= max_uncompressed_blob_size);
- pbf_blob_header.add_string(FileFormat::BlobHeader::required_string_type, type);
- pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, static_cast_with_assert<int32_t>(blob_data.size()));
+ std::string blob_data;
+ protozero::pbf_builder<FileFormat::Blob> pbf_blob(blob_data);
- uint32_t sz = htonl(static_cast_with_assert<uint32_t>(blob_header_data.size()));
+ if (m_use_compression) {
+ pbf_blob.add_int32(FileFormat::Blob::optional_int32_raw_size, int32_t(m_msg.size()));
+ pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_zlib_data, osmium::io::detail::zlib_compress(m_msg));
+ } else {
+ pbf_blob.add_bytes(FileFormat::Blob::optional_bytes_raw, m_msg);
+ }
- // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob
- std::string output;
- output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size());
- output.append(reinterpret_cast<const char*>(&sz), sizeof(sz));
- output.append(blob_header_data);
- output.append(blob_data);
+ std::string blob_header_data;
+ protozero::pbf_builder<FileFormat::BlobHeader> pbf_blob_header(blob_header_data);
- return output;
- }
+ pbf_blob_header.add_string(FileFormat::BlobHeader::required_string_type, m_blob_type == pbf_blob_type::data ? "OSMData" : "OSMHeader");
+ pbf_blob_header.add_int32(FileFormat::BlobHeader::required_int32_datasize, static_cast_with_assert<int32_t>(blob_data.size()));
+ uint32_t sz = htonl(static_cast_with_assert<uint32_t>(blob_header_data.size()));
+
+ // write to output: the 4-byte BlobHeader-Size followed by the BlobHeader followed by the Blob
+ std::string output;
+ output.reserve(sizeof(sz) + blob_header_data.size() + blob_data.size());
+ output.append(reinterpret_cast<const char*>(&sz), sizeof(sz));
+ output.append(blob_header_data);
+ output.append(blob_data);
+
+ return output;
+ }
+
+ }; // class SerializeBlob
+
+ /**
+ * Contains the code to pack any number of nodes into a DenseNode
+ * structure.
+ *
+ * Because this needs to allocate a lot of memory on the heap,
+ * only one object of this class will be created and then re-used
+ * after calling clear() on it.
+ */
class DenseNodes {
StringTable& m_stringtable;
@@ -162,27 +222,26 @@ namespace osmium {
std::vector<int64_t> m_lons;
std::vector<int32_t> m_tags;
- osmium::util::DeltaEncode<int64_t> m_delta_id;
+ osmium::util::DeltaEncode<object_id_type, int64_t> m_delta_id;
- osmium::util::DeltaEncode<int64_t> m_delta_timestamp;
- osmium::util::DeltaEncode<int64_t> m_delta_changeset;
- osmium::util::DeltaEncode<int32_t> m_delta_uid;
- osmium::util::DeltaEncode<int32_t> m_delta_user_sid;
+ osmium::util::DeltaEncode<time_t, int64_t> m_delta_timestamp;
+ osmium::util::DeltaEncode<changeset_id_type, int64_t> m_delta_changeset;
+ osmium::util::DeltaEncode<user_id_type, int32_t> m_delta_uid;
+ osmium::util::DeltaEncode<uint32_t, int32_t> m_delta_user_sid;
- osmium::util::DeltaEncode<int64_t> m_delta_lat;
- osmium::util::DeltaEncode<int64_t> m_delta_lon;
+ osmium::util::DeltaEncode<int64_t, int64_t> m_delta_lat;
+ osmium::util::DeltaEncode<int64_t, int64_t> m_delta_lon;
- bool m_add_metadata;
- bool m_add_visible;
+ const pbf_output_options& m_options;
public:
- DenseNodes(StringTable& stringtable, bool add_metadata, bool add_visible) :
+ DenseNodes(StringTable& stringtable, const pbf_output_options& options) :
m_stringtable(stringtable),
- m_add_metadata(add_metadata),
- m_add_visible(add_visible) {
+ m_options(options) {
}
+ /// Clear object for re-use. Keep the allocated memory.
void clear() {
m_ids.clear();
@@ -215,13 +274,13 @@ namespace osmium {
void add_node(const osmium::Node& node) {
m_ids.push_back(m_delta_id.update(node.id()));
- if (m_add_metadata) {
- m_versions.push_back(node.version());
+ if (m_options.add_metadata) {
+ m_versions.push_back(static_cast_with_assert<int32_t>(node.version()));
m_timestamps.push_back(m_delta_timestamp.update(node.timestamp()));
m_changesets.push_back(m_delta_changeset.update(node.changeset()));
m_uids.push_back(m_delta_uid.update(node.uid()));
m_user_sids.push_back(m_delta_user_sid.update(m_stringtable.add(node.user())));
- if (m_add_visible) {
+ if (m_options.add_visible_flag) {
m_visibles.push_back(node.visible());
}
}
@@ -230,8 +289,8 @@ namespace osmium {
m_lons.push_back(m_delta_lon.update(lonlat2int(node.location().lon_without_check())));
for (const auto& tag : node.tags()) {
- m_tags.push_back(m_stringtable.add(tag.key()));
- m_tags.push_back(m_stringtable.add(tag.value()));
+ m_tags.push_back(static_cast_with_assert<int32_t>(m_stringtable.add(tag.key())));
+ m_tags.push_back(static_cast_with_assert<int32_t>(m_stringtable.add(tag.value())));
}
m_tags.push_back(0);
}
@@ -242,7 +301,7 @@ namespace osmium {
pbf_dense_nodes.add_packed_sint64(OSMFormat::DenseNodes::packed_sint64_id, m_ids.cbegin(), m_ids.cend());
- if (m_add_metadata) {
+ if (m_options.add_metadata) {
protozero::pbf_builder<OSMFormat::DenseInfo> pbf_dense_info(pbf_dense_nodes, OSMFormat::DenseNodes::optional_DenseInfo_denseinfo);
pbf_dense_info.add_packed_int32(OSMFormat::DenseInfo::packed_int32_version, m_versions.cbegin(), m_versions.cend());
pbf_dense_info.add_packed_sint64(OSMFormat::DenseInfo::packed_sint64_timestamp, m_timestamps.cbegin(), m_timestamps.cend());
@@ -250,7 +309,7 @@ namespace osmium {
pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_uid, m_uids.cbegin(), m_uids.cend());
pbf_dense_info.add_packed_sint32(OSMFormat::DenseInfo::packed_sint32_user_sid, m_user_sids.cbegin(), m_user_sids.cend());
- if (m_add_visible) {
+ if (m_options.add_visible_flag) {
pbf_dense_info.add_packed_bool(OSMFormat::DenseInfo::packed_bool_visible, m_visibles.cbegin(), m_visibles.cend());
}
}
@@ -276,11 +335,11 @@ namespace osmium {
public:
- PrimitiveBlock(bool add_metadata, bool add_visible) :
+ PrimitiveBlock(const pbf_output_options& options) :
m_pbf_primitive_group_data(),
m_pbf_primitive_group(m_pbf_primitive_group_data),
m_stringtable(),
- m_dense_nodes(m_stringtable, add_metadata, add_visible),
+ m_dense_nodes(m_stringtable, options),
m_type(OSMFormat::PrimitiveGroup::unknown),
m_count(0) {
}
@@ -354,24 +413,7 @@ namespace osmium {
class PBFOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler {
- /// Should nodes be encoded in DenseNodes?
- bool m_use_dense_nodes;
-
- /**
- * Should the PBF blobs contain zlib compressed data?
- *
- * The zlib compression is optional, it's possible to store the
- * blobs in raw format. Disabling the compression can improve
- * the writing speed a little but the output will be 2x to 3x
- * bigger.
- */
- bool m_use_compression;
-
- /// Should metadata of objects be written?
- bool m_add_metadata;
-
- /// Should the visible flag be added to objects?
- bool m_add_visible;
+ pbf_output_options m_options;
PrimitiveBlock m_primitive_block;
@@ -390,9 +432,11 @@ namespace osmium {
primitive_block.add_message(OSMFormat::PrimitiveBlock::repeated_PrimitiveGroup_primitivegroup, m_primitive_block.group_data());
- std::promise<std::string> promise;
- m_output_queue.push(promise.get_future());
- promise.set_value(serialize_blob("OSMData", primitive_block_data, m_use_compression));
+ m_output_queue.push(osmium::thread::Pool::instance().submit(
+ SerializeBlob{std::move(primitive_block_data),
+ pbf_blob_type::data,
+ m_options.use_compression}
+ ));
}
template <typename T>
@@ -414,37 +458,44 @@ namespace osmium {
boost::make_transform_iterator(tags.begin(), map_tag_value),
boost::make_transform_iterator(tags.end(), map_tag_value));
- if (m_add_metadata) {
+ if (m_options.add_metadata) {
protozero::pbf_builder<OSMFormat::Info> pbf_info(pbf_object, T::enum_type::optional_Info_info);
- pbf_info.add_int32(OSMFormat::Info::optional_int32_version, object.version());
+ pbf_info.add_int32(OSMFormat::Info::optional_int32_version, static_cast_with_assert<int32_t>(object.version()));
pbf_info.add_int64(OSMFormat::Info::optional_int64_timestamp, object.timestamp());
pbf_info.add_int64(OSMFormat::Info::optional_int64_changeset, object.changeset());
- pbf_info.add_int32(OSMFormat::Info::optional_int32_uid, object.uid());
+ pbf_info.add_int32(OSMFormat::Info::optional_int32_uid, static_cast_with_assert<int32_t>(object.uid()));
pbf_info.add_uint32(OSMFormat::Info::optional_uint32_user_sid, m_primitive_block.store_in_stringtable(object.user()));
- if (m_add_visible) {
+ if (m_options.add_visible_flag) {
pbf_info.add_bool(OSMFormat::Info::optional_bool_visible, object.visible());
}
}
}
- PBFOutputFormat(const PBFOutputFormat&) = delete;
- PBFOutputFormat& operator=(const PBFOutputFormat&) = delete;
+ void switch_primitive_block_type(OSMFormat::PrimitiveGroup type) {
+ if (!m_primitive_block.can_add(type)) {
+ store_primitive_block();
+ m_primitive_block.reset(type);
+ }
+ }
public:
- explicit PBFOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
- OutputFormat(file, output_queue),
- m_use_dense_nodes(file.get("pbf_dense_nodes") != "false"),
- m_use_compression(file.get("pbf_compression") != "none" && file.get("pbf_compression") != "false"),
- m_add_metadata(file.get("pbf_add_metadata") != "false" && file.get("add_metadata") != "false"),
- m_add_visible(file.has_multiple_object_versions()),
- m_primitive_block(m_add_metadata, m_add_visible) {
+ PBFOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) :
+ OutputFormat(output_queue),
+ m_options(),
+ m_primitive_block(m_options) {
+ m_options.use_dense_nodes = file.is_not_false("pbf_dense_nodes");
+ m_options.use_compression = file.get("pbf_compression") != "none" && file.is_not_false("pbf_compression");
+ m_options.add_metadata = file.is_not_false("pbf_add_metadata") && file.is_not_false("add_metadata");
+ m_options.add_historical_information_flag = file.has_multiple_object_versions();
+ m_options.add_visible_flag = file.has_multiple_object_versions();
}
- void write_buffer(osmium::memory::Buffer&& buffer) override final {
- osmium::apply(buffer.cbegin(), buffer.cend(), *this);
- }
+ PBFOutputFormat(const PBFOutputFormat&) = delete;
+ PBFOutputFormat& operator=(const PBFOutputFormat&) = delete;
+
+ ~PBFOutputFormat() noexcept = default;
void write_header(const osmium::io::Header& header) override final {
std::string data;
@@ -462,11 +513,11 @@ namespace osmium {
pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "OsmSchema-V0.6");
- if (m_use_dense_nodes) {
+ if (m_options.use_dense_nodes) {
pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "DenseNodes");
}
- if (m_file.has_multiple_object_versions()) {
+ if (m_options.add_historical_information_flag) {
pbf_header_block.add_string(OSMFormat::HeaderBlock::repeated_string_required_features, "HistoricalInformation");
}
@@ -488,20 +539,23 @@ namespace osmium {
pbf_header_block.add_string(OSMFormat::HeaderBlock::optional_string_osmosis_replication_base_url, osmosis_replication_base_url);
}
- std::promise<std::string> promise;
- m_output_queue.push(promise.get_future());
- promise.set_value(serialize_blob("OSMHeader", data, m_use_compression));
+ m_output_queue.push(osmium::thread::Pool::instance().submit(
+ SerializeBlob{std::move(data),
+ pbf_blob_type::header,
+ m_options.use_compression}
+ ));
}
- void switch_primitive_block_type(OSMFormat::PrimitiveGroup type) {
- if (!m_primitive_block.can_add(type)) {
- store_primitive_block();
- m_primitive_block.reset(type);
- }
+ void write_buffer(osmium::memory::Buffer&& buffer) override final {
+ osmium::apply(buffer.cbegin(), buffer.cend(), *this);
+ }
+
+ void write_end() override final {
+ store_primitive_block();
}
void node(const osmium::Node& node) {
- if (m_use_dense_nodes) {
+ if (m_options.use_dense_nodes) {
switch_primitive_block_type(OSMFormat::PrimitiveGroup::optional_DenseNodes_dense);
m_primitive_block.add_dense_node(node);
return;
@@ -558,41 +612,27 @@ namespace osmium {
it_type last { members.cend(), members.cend(), map_member_ref };
pbf_relation.add_packed_sint64(OSMFormat::Relation::packed_sint64_memids, first, last);
- static auto map_member_type = [](const osmium::RelationMember& member) noexcept -> int {
- return osmium::item_type_to_nwr_index(member.type());
+ static auto map_member_type = [](const osmium::RelationMember& member) noexcept -> int32_t {
+ return int32_t(osmium::item_type_to_nwr_index(member.type()));
};
pbf_relation.add_packed_int32(OSMFormat::Relation::packed_MemberType_types,
boost::make_transform_iterator(relation.members().begin(), map_member_type),
boost::make_transform_iterator(relation.members().end(), map_member_type));
}
- /**
- * Finalize the writing process, flush any open primitive
- * blocks to the file and close the file.
- */
- void close() override final {
- store_primitive_block();
-
- std::promise<std::string> promise;
- m_output_queue.push(promise.get_future());
- promise.set_value(std::string());
- }
-
}; // class PBFOutputFormat
- namespace {
+ // we want the register_output_format() function to run, setting
+ // the variable is only a side-effect, it will never be used
+ const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf,
+ [](const osmium::io::File& file, future_string_queue_type& output_queue) {
+ return new osmium::io::detail::PBFOutputFormat(file, output_queue);
+ });
-// we want the register_output_format() function to run, setting the variable
-// is only a side-effect, it will never be used
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-variable"
- const bool registered_pbf_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::pbf,
- [](const osmium::io::File& file, data_queue_type& output_queue) {
- return new osmium::io::detail::PBFOutputFormat(file, output_queue);
- });
-#pragma GCC diagnostic pop
-
- } // anonymous namespace
+ // dummy function to silence the unused variable warning from above
+ inline bool get_registered_pbf_output() noexcept {
+ return registered_pbf_output;
+ }
} // namespace detail
diff --git a/include/osmium/io/detail/queue_util.hpp b/include/osmium/io/detail/queue_util.hpp
new file mode 100644
index 0000000..47f30da
--- /dev/null
+++ b/include/osmium/io/detail/queue_util.hpp
@@ -0,0 +1,157 @@
+#ifndef OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP
+#define OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen at topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <algorithm>
+#include <exception>
+#include <future>
+#include <string>
+
+#include <osmium/memory/buffer.hpp>
+#include <osmium/thread/queue.hpp>
+
+namespace osmium {
+
+ namespace io {
+
+ namespace detail {
+
+ /**
+ * This type of queue contains buffers with OSM data in them.
+ * The "end of file" is marked by an invalid Buffer.
+ * The buffers are wrapped in a std::future so that they can also
+ * transport exceptions. The future also helps with keeping the
+ * data in order.
+ */
+ using future_buffer_queue_type = osmium::thread::Queue<std::future<osmium::memory::Buffer>>;
+
+ /**
+ * This type of queue contains OSM file data in the form it is
+ * stored on disk, ie encoded as XML, PBF, etc.
+ * The "end of file" is marked by an empty string.
+ */
+ using string_queue_type = osmium::thread::Queue<std::string>;
+
+ /**
+ * This type of queue contains OSM file data in the form it is
+ * stored on disk, ie encoded as XML, PBF, etc.
+ * The "end of file" is marked by an empty string.
+ * The strings are wrapped in a std::future so that they can also
+ * transport exceptions. The future also helps with keeping the
+ * data in order.
+ */
+ using future_string_queue_type = osmium::thread::Queue<std::future<std::string>>;
+
+ template <class T>
+ inline void add_to_queue(osmium::thread::Queue<std::future<T>>& queue, T&& data) {
+ std::promise<T> promise;
+ queue.push(promise.get_future());
+ promise.set_value(std::forward<T>(data));
+ }
+
+ template <class T>
+ inline void add_to_queue(osmium::thread::Queue<std::future<T>>& queue, std::exception_ptr&& exception) {
+ std::promise<T> promise;
+ queue.push(promise.get_future());
+ promise.set_exception(std::move(exception));
+ }
+
+ template <class T>
+ inline void add_end_of_data_to_queue(osmium::thread::Queue<std::future<T>>& queue) {
+ add_to_queue<T>(queue, T{});
+ }
+
+ inline bool at_end_of_data(const std::string& data) {
+ return data.empty();
+ }
+
+ inline bool at_end_of_data(osmium::memory::Buffer& buffer) {
+ return !buffer;
+ }
+
+ template <class T>
+ class queue_wrapper {
+
+ using queue_type = osmium::thread::Queue<std::future<T>>;
+
+ queue_type& m_queue;
+ bool m_has_reached_end_of_data;
+
+ public:
+
+ queue_wrapper(queue_type& queue) :
+ m_queue(queue),
+ m_has_reached_end_of_data(false) {
+ }
+
+ ~queue_wrapper() noexcept {
+ drain();
+ }
+
+ void drain() {
+ while (!m_has_reached_end_of_data) {
+ try {
+ pop();
+ } catch (...) {
+ // Ignore any exceptions.
+ }
+ }
+ }
+
+ bool has_reached_end_of_data() const noexcept {
+ return m_has_reached_end_of_data;
+ }
+
+ T pop() {
+ T data;
+ if (!m_has_reached_end_of_data) {
+ std::future<T> data_future;
+ m_queue.wait_and_pop(data_future);
+ data = std::move(data_future.get());
+ if (at_end_of_data(data)) {
+ m_has_reached_end_of_data = true;
+ }
+ }
+ return data;
+ }
+
+ }; // class queue_wrapper
+
+ } // namespace detail
+
+ } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_QUEUE_UTIL_HPP
diff --git a/include/osmium/io/detail/read_thread.hpp b/include/osmium/io/detail/read_thread.hpp
index bce4f55..6f96c0b 100644
--- a/include/osmium/io/detail/read_thread.hpp
+++ b/include/osmium/io/detail/read_thread.hpp
@@ -34,14 +34,13 @@ DEALINGS IN THE SOFTWARE.
*/
#include <atomic>
-#include <chrono>
-#include <ratio>
+#include <exception>
#include <string>
#include <thread>
#include <utility>
#include <osmium/io/compression.hpp>
-#include <osmium/thread/queue.hpp>
+#include <osmium/io/detail/queue_util.hpp>
#include <osmium/thread/util.hpp>
namespace osmium {
@@ -50,52 +49,80 @@ namespace osmium {
namespace detail {
- class ReadThread {
+ /**
+ * This code uses an internally managed thread to read data from
+ * the input file and (optionally) decompress it. The result is
+ * sent to the given queue. Any exceptions will also be send to
+ * the queue.
+ */
+ class ReadThreadManager {
- osmium::thread::Queue<std::string>& m_queue;
- osmium::io::Decompressor* m_decompressor;
+ // only used in the sub-thread
+ osmium::io::Decompressor& m_decompressor;
+ future_string_queue_type& m_queue;
- // If this is set in the main thread, we have to wrap up at the
- // next possible moment.
- std::atomic<bool>& m_done;
+ // used in both threads
+ std::atomic<bool> m_done;
- public:
+ // only used in the main thread
+ std::thread m_thread;
- explicit ReadThread(osmium::thread::Queue<std::string>& queue, osmium::io::Decompressor* decompressor, std::atomic<bool>& done) :
- m_queue(queue),
- m_decompressor(decompressor),
- m_done(done) {
- }
-
- bool operator()() {
- osmium::thread::set_thread_name("_osmium_input");
+ void run_in_thread() {
+ osmium::thread::set_thread_name("_osmium_read");
try {
while (!m_done) {
- std::string data {m_decompressor->read()};
- if (data.empty()) {
- m_queue.push(std::move(data));
+ std::string data {m_decompressor.read()};
+ if (at_end_of_data(data)) {
break;
}
- m_queue.push(std::move(data));
- while (m_queue.size() > 10 && !m_done) {
- std::this_thread::sleep_for(std::chrono::milliseconds(10));
- }
+ add_to_queue(m_queue, std::move(data));
}
- m_decompressor->close();
+ m_decompressor.close();
} catch (...) {
- // If there is an exception in this thread, we make sure
- // to push an empty string onto the queue to signal the
- // end-of-data to the reading thread so that it will not
- // hang. Then we re-throw the exception.
- m_queue.push(std::string());
- throw;
+ add_to_queue(m_queue, std::current_exception());
+ }
+
+ add_end_of_data_to_queue(m_queue);
+ }
+
+ public:
+
+ ReadThreadManager(osmium::io::Decompressor& decompressor,
+ future_string_queue_type& queue) :
+ m_decompressor(decompressor),
+ m_queue(queue),
+ m_done(false),
+ m_thread(std::thread(&ReadThreadManager::run_in_thread, this)) {
+ }
+
+ ReadThreadManager(const ReadThreadManager&) = delete;
+ ReadThreadManager& operator=(const ReadThreadManager&) = delete;
+
+ ReadThreadManager(ReadThreadManager&&) = delete;
+ ReadThreadManager& operator=(ReadThreadManager&&) = delete;
+
+ ~ReadThreadManager() noexcept {
+ try {
+ close();
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
+ }
+ }
+
+ void stop() noexcept {
+ m_done = true;
+ }
+
+ void close() {
+ stop();
+ if (m_thread.joinable()) {
+ m_thread.join();
}
- return true;
}
- }; // class ReadThread
+ }; // class ReadThreadManager
} // namespace detail
diff --git a/include/osmium/io/detail/read_write.hpp b/include/osmium/io/detail/read_write.hpp
index 773497c..815c6bb 100644
--- a/include/osmium/io/detail/read_write.hpp
+++ b/include/osmium/io/detail/read_write.hpp
@@ -35,6 +35,7 @@ DEALINGS IN THE SOFTWARE.
#include <cerrno>
#include <cstddef>
+#include <errno.h>
#include <fcntl.h>
#include <string>
#include <system_error>
@@ -154,6 +155,22 @@ namespace osmium {
reliable_write(fd, reinterpret_cast<const unsigned char*>(output_buffer), size);
}
+ inline void reliable_fsync(const int fd) {
+#ifdef _WIN32
+ if (_commit(fd) != 0) {
+#else
+ if (::fsync(fd) != 0) {
+#endif
+ throw std::system_error(errno, std::system_category(), "Fsync failed");
+ }
+ }
+
+ inline void reliable_close(const int fd) {
+ if (::close(fd) != 0) {
+ throw std::system_error(errno, std::system_category(), "Close failed");
+ }
+ }
+
} // namespace detail
} // namespace io
diff --git a/include/osmium/io/detail/string_table.hpp b/include/osmium/io/detail/string_table.hpp
index a048de3..a23035d 100644
--- a/include/osmium/io/detail/string_table.hpp
+++ b/include/osmium/io/detail/string_table.hpp
@@ -34,6 +34,7 @@ DEALINGS IN THE SOFTWARE.
*/
#include <cassert>
+#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <iterator>
diff --git a/include/osmium/io/detail/string_util.hpp b/include/osmium/io/detail/string_util.hpp
new file mode 100644
index 0000000..672266a
--- /dev/null
+++ b/include/osmium/io/detail/string_util.hpp
@@ -0,0 +1,206 @@
+#ifndef OSMIUM_IO_DETAIL_STRING_UTIL_HPP
+#define OSMIUM_IO_DETAIL_STRING_UTIL_HPP
+
+/*
+
+This file is part of Osmium (http://osmcode.org/libosmium).
+
+Copyright 2013-2015 Jochen Topf <jochen at topf.org> and others (see README).
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+*/
+
+#include <cassert>
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <utility>
+
+#include <utf8.h>
+
+namespace osmium {
+
+ namespace io {
+
+ namespace detail {
+
+#ifndef _MSC_VER
+# define SNPRINTF std::snprintf
+#else
+# define SNPRINTF _snprintf
+#endif
+
+ template <typename... TArgs>
+ inline int string_snprintf(std::string& out,
+ size_t old_size,
+ size_t max_size,
+ const char* format,
+ TArgs&&... args) {
+ out.resize(old_size + max_size);
+
+ return SNPRINTF(max_size ? const_cast<char*>(out.c_str()) + old_size : nullptr,
+ max_size,
+ format,
+ std::forward<TArgs>(args)...);
+ }
+
+#undef SNPRINTF
+
+ /**
+ * This is a helper function for writing printf-like formatted
+ * data into a std::string.
+ *
+ * @param out The data will be appended to this string.
+ * @param format A string with formatting instructions a la printf.
+ * @param args Any further arguments like in printf.
+ * @throws std::bad_alloc If the string needed to grow and there
+ * wasn't enough memory.
+ */
+ template <typename... TArgs>
+ inline void append_printf_formatted_string(std::string& out,
+ const char* format,
+ TArgs&&... args) {
+
+ // First try to write string with the max_size, if that doesn't
+ // work snprintf will tell us how much space it needs. We
+ // reserve that much space and try again. So this will always
+ // work, even if the output is larger than the given max_size.
+ //
+ // Unfortunately this trick doesn't work on Windows, because
+ // the _snprintf() function there only returns the length it
+ // needs if max_size==0 and the buffer pointer is the null
+ // pointer. So we have to take this into account.
+
+#ifndef _MSC_VER
+ static const size_t max_size = 100;
+#else
+ static const size_t max_size = 0;
+#endif
+
+ size_t old_size = out.size();
+
+ int len = string_snprintf(out,
+ old_size,
+ max_size,
+ format,
+ std::forward<TArgs>(args)...);
+ assert(len > 0);
+
+ if (size_t(len) >= max_size) {
+ int len2 = string_snprintf(out,
+ old_size,
+ size_t(len) + 1,
+ format,
+ std::forward<TArgs>(args)...);
+ assert(len2 == len);
+ }
+
+ out.resize(old_size + size_t(len));
+ }
+
+ inline void append_utf8_encoded_string(std::string& out, const char* data) {
+ const char* end = data + std::strlen(data);
+
+ while (data != end) {
+ const char* last = data;
+ uint32_t c = utf8::next(data, end);
+
+ // This is a list of Unicode code points that we let
+ // through instead of escaping them. It is incomplete
+ // and can be extended later.
+ // Generally we don't want to let through any character
+ // that has special meaning in the OPL format such as
+ // space, comma, @, etc. and any non-printing characters.
+ if ((0x0021 <= c && c <= 0x0024) ||
+ (0x0026 <= c && c <= 0x002b) ||
+ (0x002d <= c && c <= 0x003c) ||
+ (0x003e <= c && c <= 0x003f) ||
+ (0x0041 <= c && c <= 0x007e) ||
+ (0x00a1 <= c && c <= 0x00ac) ||
+ (0x00ae <= c && c <= 0x05ff)) {
+ out.append(last, data);
+ } else {
+ out += '%';
+ if (c <= 0xff) {
+ append_printf_formatted_string(out, "%02x", c);
+ } else {
+ append_printf_formatted_string(out, "%04x", c);
+ }
+ out += '%';
+ }
+ }
+ }
+
+ inline void append_xml_encoded_string(std::string& out, const char* data) {
+ for (; *data != '\0'; ++data) {
+ switch(*data) {
+ case '&': out += "&"; break;
+ case '\"': out += """; break;
+ case '\'': out += "'"; break;
+ case '<': out += "<"; break;
+ case '>': out += ">"; break;
+ case '\n': out += "
"; break;
+ case '\r': out += "
"; break;
+ case '\t': out += "	"; break;
+ default: out += *data; break;
+ }
+ }
+ }
+
+ inline void append_debug_encoded_string(std::string& out, const char* data, const char* prefix, const char* suffix) {
+ const char* end = data + std::strlen(data);
+
+ while (data != end) {
+ const char* last = data;
+ uint32_t c = utf8::next(data, end);
+
+ // This is a list of Unicode code points that we let
+ // through instead of escaping them. It is incomplete
+ // and can be extended later.
+ // Generally we don't want to let through any
+ // non-printing characters.
+ if ((0x0020 <= c && c <= 0x0021) ||
+ (0x0023 <= c && c <= 0x003b) ||
+ (0x003d == c) ||
+ (0x003f <= c && c <= 0x007e) ||
+ (0x00a1 <= c && c <= 0x00ac) ||
+ (0x00ae <= c && c <= 0x05ff)) {
+ out.append(last, data);
+ } else {
+ out.append(prefix);
+ append_printf_formatted_string(out, "<U+%04X>", c);
+ out.append(suffix);
+ }
+ }
+ }
+
+ } // namespace detail
+
+ } // namespace io
+
+} // namespace osmium
+
+#endif // OSMIUM_IO_DETAIL_STRING_UTIL_HPP
diff --git a/include/osmium/io/detail/write_thread.hpp b/include/osmium/io/detail/write_thread.hpp
index fad22ed..81ab194 100644
--- a/include/osmium/io/detail/write_thread.hpp
+++ b/include/osmium/io/detail/write_thread.hpp
@@ -33,11 +33,13 @@ DEALINGS IN THE SOFTWARE.
*/
+#include <algorithm>
+#include <exception>
#include <future>
#include <string>
#include <osmium/io/compression.hpp>
-#include <osmium/io/detail/output_format.hpp>
+#include <osmium/io/detail/queue_util.hpp>
#include <osmium/thread/util.hpp>
namespace osmium {
@@ -46,33 +48,52 @@ namespace osmium {
namespace detail {
+ /**
+ * This codes runs in its own thread, getting data from the given
+ * queue, (optionally) compressing it, and writing it to the output
+ * file.
+ */
class WriteThread {
- typedef osmium::io::detail::data_queue_type data_queue_type;
-
- data_queue_type& m_input_queue;
- osmium::io::Compressor* m_compressor;
+ queue_wrapper<std::string> m_queue;
+ std::unique_ptr<osmium::io::Compressor> m_compressor;
+ std::promise<bool> m_promise;
public:
- explicit WriteThread(data_queue_type& input_queue, osmium::io::Compressor* compressor) :
- m_input_queue(input_queue),
- m_compressor(compressor) {
+ WriteThread(future_string_queue_type& input_queue,
+ std::unique_ptr<osmium::io::Compressor>&& compressor,
+ std::promise<bool>&& promise) :
+ m_queue(input_queue),
+ m_compressor(std::move(compressor)),
+ m_promise(std::move(promise)) {
}
- bool operator()() {
- osmium::thread::set_thread_name("_osmium_output");
-
- std::future<std::string> data_future;
- std::string data;
- do {
- m_input_queue.wait_and_pop(data_future);
- data = data_future.get();
- m_compressor->write(data);
- } while (!data.empty());
-
- m_compressor->close();
- return true;
+ WriteThread(const WriteThread&) = delete;
+ WriteThread& operator=(const WriteThread&) = delete;
+
+ WriteThread(WriteThread&&) = delete;
+ WriteThread& operator=(WriteThread&&) = delete;
+
+ ~WriteThread() noexcept = default;
+
+ void operator()() {
+ osmium::thread::set_thread_name("_osmium_write");
+
+ try {
+ while (true) {
+ std::string data = m_queue.pop();
+ if (at_end_of_data(data)) {
+ break;
+ }
+ m_compressor->write(data);
+ }
+ m_compressor->close();
+ m_promise.set_value(true);
+ } catch (...) {
+ m_promise.set_exception(std::current_exception());
+ m_queue.drain();
+ }
}
}; // class WriteThread
diff --git a/include/osmium/io/detail/xml_input_format.hpp b/include/osmium/io/detail/xml_input_format.hpp
index b0f3da3..23caa8f 100644
--- a/include/osmium/io/detail/xml_input_format.hpp
+++ b/include/osmium/io/detail/xml_input_format.hpp
@@ -33,21 +33,14 @@ DEALINGS IN THE SOFTWARE.
*/
-#include <atomic>
#include <cassert>
-#include <chrono>
#include <cstddef>
#include <cstdlib>
#include <cstring>
#include <exception>
#include <future>
-#include <iostream>
#include <memory>
-#include <ratio>
-#include <sstream>
-#include <stdexcept>
#include <string>
-#include <thread>
#include <utility>
#include <expat.h>
@@ -55,6 +48,7 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/builder/builder.hpp>
#include <osmium/builder/osm_object_builder.hpp>
#include <osmium/io/detail/input_format.hpp>
+#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/error.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/header.hpp>
@@ -130,23 +124,11 @@ namespace osmium {
namespace io {
- class File;
-
namespace detail {
- /**
- * Once the header is fully parsed this exception will be thrown if
- * the caller is not interested in anything else except the header.
- * It will break off the parsing at this point.
- *
- * This exception is never seen by user code, it is caught internally.
- */
- class ParserIsDone : std::exception {
- };
-
- class XMLParser {
+ class XMLParser : public Parser {
- static constexpr int buffer_size = 10 * 1000 * 1000;
+ static constexpr int buffer_size = 2 * 1000 * 1000;
enum class context {
root,
@@ -155,6 +137,9 @@ namespace osmium {
way,
relation,
changeset,
+ discussion,
+ comment,
+ comment_text,
ignored_node,
ignored_way,
ignored_relation,
@@ -175,24 +160,17 @@ namespace osmium {
osmium::memory::Buffer m_buffer;
- std::unique_ptr<osmium::builder::NodeBuilder> m_node_builder;
- std::unique_ptr<osmium::builder::WayBuilder> m_way_builder;
- std::unique_ptr<osmium::builder::RelationBuilder> m_relation_builder;
- std::unique_ptr<osmium::builder::ChangesetBuilder> m_changeset_builder;
-
- std::unique_ptr<osmium::builder::TagListBuilder> m_tl_builder;
- std::unique_ptr<osmium::builder::WayNodeListBuilder> m_wnl_builder;
- std::unique_ptr<osmium::builder::RelationMemberListBuilder> m_rml_builder;
-
- osmium::thread::Queue<std::string>& m_input_queue;
- osmium::thread::Queue<osmium::memory::Buffer>& m_queue;
- std::promise<osmium::io::Header>& m_header_promise;
+ std::unique_ptr<osmium::builder::NodeBuilder> m_node_builder;
+ std::unique_ptr<osmium::builder::WayBuilder> m_way_builder;
+ std::unique_ptr<osmium::builder::RelationBuilder> m_relation_builder;
+ std::unique_ptr<osmium::builder::ChangesetBuilder> m_changeset_builder;
+ std::unique_ptr<osmium::builder::ChangesetDiscussionBuilder> m_changeset_discussion_builder;
- osmium::osm_entity_bits::type m_read_types;
+ std::unique_ptr<osmium::builder::TagListBuilder> m_tl_builder;
+ std::unique_ptr<osmium::builder::WayNodeListBuilder> m_wnl_builder;
+ std::unique_ptr<osmium::builder::RelationMemberListBuilder> m_rml_builder;
- std::atomic<bool>& m_done;
-
- bool m_header_is_done;
+ std::string m_comment_text;
/**
* A C++ wrapper for the Expat parser that makes sure no memory is leaked.
@@ -210,6 +188,10 @@ namespace osmium {
static_cast<XMLParser*>(data)->end_element(element);
}
+ static void XMLCALL character_data_wrapper(void* data, const XML_Char* text, int len) {
+ static_cast<XMLParser*>(data)->characters(text, len);
+ }
+
public:
ExpatXMLParser(T* callback_object) :
@@ -219,6 +201,7 @@ namespace osmium {
}
XML_SetUserData(m_parser, callback_object);
XML_SetElementHandler(m_parser, start_element_wrapper, end_element_wrapper);
+ XML_SetCharacterDataHandler(m_parser, character_data_wrapper);
}
ExpatXMLParser(const ExpatXMLParser&) = delete;
@@ -227,7 +210,7 @@ namespace osmium {
ExpatXMLParser& operator=(const ExpatXMLParser&) = delete;
ExpatXMLParser& operator=(ExpatXMLParser&&) = delete;
- ~ExpatXMLParser() {
+ ~ExpatXMLParser() noexcept {
XML_ParserFree(m_parser);
}
@@ -239,126 +222,14 @@ namespace osmium {
}; // class ExpatXMLParser
- /**
- * A helper class that makes sure a promise is kept. It stores
- * a reference to some piece of data and to a promise and, on
- * destruction, sets the value of the promise from the data.
- */
- template <class T>
- class PromiseKeeper {
-
- T& m_data;
- std::promise<T>& m_promise;
- bool m_done;
-
- public:
-
- PromiseKeeper(T& data, std::promise<T>& promise) :
- m_data(data),
- m_promise(promise),
- m_done(false) {
+ template <typename T>
+ static void check_attributes(const XML_Char** attrs, T check) {
+ while (*attrs) {
+ check(attrs[0], attrs[1]);
+ attrs += 2;
}
-
- void fullfill_promise() {
- if (!m_done) {
- m_promise.set_value(m_data);
- m_done = true;
- }
- }
-
- ~PromiseKeeper() {
- fullfill_promise();
- }
-
- }; // class PromiseKeeper
-
- public:
-
- explicit XMLParser(osmium::thread::Queue<std::string>& input_queue, osmium::thread::Queue<osmium::memory::Buffer>& queue, std::promise<osmium::io::Header>& header_promise, osmium::osm_entity_bits::type read_types, std::atomic<bool>& done) :
- m_context(context::root),
- m_last_context(context::root),
- m_in_delete_section(false),
- m_header(),
- m_buffer(buffer_size),
- m_node_builder(),
- m_way_builder(),
- m_relation_builder(),
- m_changeset_builder(),
- m_tl_builder(),
- m_wnl_builder(),
- m_rml_builder(),
- m_input_queue(input_queue),
- m_queue(queue),
- m_header_promise(header_promise),
- m_read_types(read_types),
- m_done(done),
- m_header_is_done(false) {
- }
-
- /**
- * The copy constructor is needed for storing XMLParser in a std::function.
- * The copy will look the same as if it has been initialized with the
- * same parameters as the original. Any state changes in the original will
- * not be reflected in the copy.
- */
- XMLParser(const XMLParser& other) :
- m_context(context::root),
- m_last_context(context::root),
- m_in_delete_section(false),
- m_header(),
- m_buffer(buffer_size),
- m_node_builder(),
- m_way_builder(),
- m_relation_builder(),
- m_changeset_builder(),
- m_tl_builder(),
- m_wnl_builder(),
- m_rml_builder(),
- m_input_queue(other.m_input_queue),
- m_queue(other.m_queue),
- m_header_promise(other.m_header_promise),
- m_read_types(other.m_read_types),
- m_done(other.m_done),
- m_header_is_done(other.m_header_is_done) {
}
- XMLParser(XMLParser&&) = default;
-
- XMLParser& operator=(const XMLParser&) = delete;
-
- XMLParser& operator=(XMLParser&&) = default;
-
- ~XMLParser() = default;
-
- bool operator()() {
- ExpatXMLParser<XMLParser> parser(this);
- PromiseKeeper<osmium::io::Header> promise_keeper(m_header, m_header_promise);
- bool last;
- do {
- std::string data;
- m_input_queue.wait_and_pop(data);
- last = data.empty();
- try {
- parser(data, last);
- if (m_header_is_done) {
- promise_keeper.fullfill_promise();
- }
- } catch (ParserIsDone&) {
- return true;
- } catch (...) {
- m_queue.push(osmium::memory::Buffer()); // empty buffer to signify eof
- throw;
- }
- } while (!last && !m_done);
- if (m_buffer.committed() > 0) {
- m_queue.push(std::move(m_buffer));
- }
- m_queue.push(osmium::memory::Buffer()); // empty buffer to signify eof
- return true;
- }
-
- private:
-
const char* init_object(osmium::OSMObject& object, const XML_Char** attrs) {
const char* user = "";
@@ -367,17 +238,18 @@ namespace osmium {
}
osmium::Location location;
- for (int count = 0; attrs[count]; count += 2) {
- if (!strcmp(attrs[count], "lon")) {
- location.set_lon(std::atof(attrs[count+1])); // XXX doesn't detect garbage after the number
- } else if (!strcmp(attrs[count], "lat")) {
- location.set_lat(std::atof(attrs[count+1])); // XXX doesn't detect garbage after the number
- } else if (!strcmp(attrs[count], "user")) {
- user = attrs[count+1];
+
+ check_attributes(attrs, [&location, &user, &object](const XML_Char* name, const XML_Char* value) {
+ if (!strcmp(name, "lon")) {
+ location.set_lon(std::atof(value)); // XXX doesn't detect garbage after the number
+ } else if (!strcmp(name, "lat")) {
+ location.set_lat(std::atof(value)); // XXX doesn't detect garbage after the number
+ } else if (!strcmp(name, "user")) {
+ user = value;
} else {
- object.set_attribute(attrs[count], attrs[count+1]);
+ object.set_attribute(name, value);
}
- }
+ });
if (location && object.type() == osmium::item_type::node) {
static_cast<osmium::Node&>(object).set_location(location);
@@ -392,21 +264,21 @@ namespace osmium {
osmium::Location min;
osmium::Location max;
- for (int count = 0; attrs[count]; count += 2) {
- if (!strcmp(attrs[count], "min_lon")) {
- min.set_lon(atof(attrs[count+1]));
- } else if (!strcmp(attrs[count], "min_lat")) {
- min.set_lat(atof(attrs[count+1]));
- } else if (!strcmp(attrs[count], "max_lon")) {
- max.set_lon(atof(attrs[count+1]));
- } else if (!strcmp(attrs[count], "max_lat")) {
- max.set_lat(atof(attrs[count+1]));
- } else if (!strcmp(attrs[count], "user")) {
- user = attrs[count+1];
+ check_attributes(attrs, [&min, &max, &user, &new_changeset](const XML_Char* name, const XML_Char* value) {
+ if (!strcmp(name, "min_lon")) {
+ min.set_lon(atof(value));
+ } else if (!strcmp(name, "min_lat")) {
+ min.set_lat(atof(value));
+ } else if (!strcmp(name, "max_lon")) {
+ max.set_lon(atof(value));
+ } else if (!strcmp(name, "max_lat")) {
+ max.set_lat(atof(value));
+ } else if (!strcmp(name, "user")) {
+ user = value;
} else {
- new_changeset.set_attribute(attrs[count], attrs[count+1]);
+ new_changeset.set_attribute(name, value);
}
- }
+ });
new_changeset.bounds().extend(min);
new_changeset.bounds().extend(max);
@@ -414,32 +286,24 @@ namespace osmium {
builder->add_user(user);
}
- void check_tag(osmium::builder::Builder* builder, const XML_Char* element, const XML_Char** attrs) {
- if (!strcmp(element, "tag")) {
- m_wnl_builder.reset();
- m_rml_builder.reset();
-
- const char* key = "";
- const char* value = "";
- for (int count = 0; attrs[count]; count += 2) {
- if (attrs[count][0] == 'k' && attrs[count][1] == 0) {
- key = attrs[count+1];
- } else if (attrs[count][0] == 'v' && attrs[count][1] == 0) {
- value = attrs[count+1];
- }
+ void get_tag(osmium::builder::Builder* builder, const XML_Char** attrs) {
+ const char* k = "";
+ const char* v = "";
+ check_attributes(attrs, [&k, &v](const XML_Char* name, const XML_Char* value) {
+ if (name[0] == 'k' && name[1] == 0) {
+ k = value;
+ } else if (name[0] == 'v' && name[1] == 0) {
+ v = value;
}
- if (!m_tl_builder) {
- m_tl_builder = std::unique_ptr<osmium::builder::TagListBuilder>(new osmium::builder::TagListBuilder(m_buffer, builder));
- }
- m_tl_builder->add_tag(key, value);
+ });
+ if (!m_tl_builder) {
+ m_tl_builder = std::unique_ptr<osmium::builder::TagListBuilder>(new osmium::builder::TagListBuilder(m_buffer, builder));
}
+ m_tl_builder->add_tag(k, v);
}
- void header_is_done() {
- m_header_is_done = true;
- if (m_read_types == osmium::osm_entity_bits::nothing) {
- throw ParserIsDone();
- }
+ void mark_header_as_done() {
+ set_header_value(m_header);
}
void start_element(const XML_Char* element, const XML_Char** attrs) {
@@ -449,16 +313,16 @@ namespace osmium {
if (!strcmp(element, "osmChange")) {
m_header.set_has_multiple_object_versions(true);
}
- for (int count = 0; attrs[count]; count += 2) {
- if (!strcmp(attrs[count], "version")) {
- m_header.set("version", attrs[count+1]);
- if (strcmp(attrs[count+1], "0.6")) {
- throw osmium::format_version_error(attrs[count+1]);
+ check_attributes(attrs, [this](const XML_Char* name, const XML_Char* value) {
+ if (!strcmp(name, "version")) {
+ m_header.set("version", value);
+ if (strcmp(value, "0.6")) {
+ throw osmium::format_version_error(value);
}
- } else if (!strcmp(attrs[count], "generator")) {
- m_header.set("generator", attrs[count+1]);
+ } else if (!strcmp(name, "generator")) {
+ m_header.set("generator", value);
}
- }
+ });
if (m_header.get("version") == "") {
throw osmium::format_version_error();
}
@@ -470,8 +334,8 @@ namespace osmium {
case context::top:
assert(!m_tl_builder);
if (!strcmp(element, "node")) {
- header_is_done();
- if (m_read_types & osmium::osm_entity_bits::node) {
+ mark_header_as_done();
+ if (read_types() & osmium::osm_entity_bits::node) {
m_node_builder = std::unique_ptr<osmium::builder::NodeBuilder>(new osmium::builder::NodeBuilder(m_buffer));
m_node_builder->add_user(init_object(m_node_builder->object(), attrs));
m_context = context::node;
@@ -479,8 +343,8 @@ namespace osmium {
m_context = context::ignored_node;
}
} else if (!strcmp(element, "way")) {
- header_is_done();
- if (m_read_types & osmium::osm_entity_bits::way) {
+ mark_header_as_done();
+ if (read_types() & osmium::osm_entity_bits::way) {
m_way_builder = std::unique_ptr<osmium::builder::WayBuilder>(new osmium::builder::WayBuilder(m_buffer));
m_way_builder->add_user(init_object(m_way_builder->object(), attrs));
m_context = context::way;
@@ -488,8 +352,8 @@ namespace osmium {
m_context = context::ignored_way;
}
} else if (!strcmp(element, "relation")) {
- header_is_done();
- if (m_read_types & osmium::osm_entity_bits::relation) {
+ mark_header_as_done();
+ if (read_types() & osmium::osm_entity_bits::relation) {
m_relation_builder = std::unique_ptr<osmium::builder::RelationBuilder>(new osmium::builder::RelationBuilder(m_buffer));
m_relation_builder->add_user(init_object(m_relation_builder->object(), attrs));
m_context = context::relation;
@@ -497,8 +361,8 @@ namespace osmium {
m_context = context::ignored_relation;
}
} else if (!strcmp(element, "changeset")) {
- header_is_done();
- if (m_read_types & osmium::osm_entity_bits::changeset) {
+ mark_header_as_done();
+ if (read_types() & osmium::osm_entity_bits::changeset) {
m_changeset_builder = std::unique_ptr<osmium::builder::ChangesetBuilder>(new osmium::builder::ChangesetBuilder(m_buffer));
init_changeset(m_changeset_builder.get(), attrs);
m_context = context::changeset;
@@ -508,17 +372,17 @@ namespace osmium {
} else if (!strcmp(element, "bounds")) {
osmium::Location min;
osmium::Location max;
- for (int count = 0; attrs[count]; count += 2) {
- if (!strcmp(attrs[count], "minlon")) {
- min.set_lon(atof(attrs[count+1]));
- } else if (!strcmp(attrs[count], "minlat")) {
- min.set_lat(atof(attrs[count+1]));
- } else if (!strcmp(attrs[count], "maxlon")) {
- max.set_lon(atof(attrs[count+1]));
- } else if (!strcmp(attrs[count], "maxlat")) {
- max.set_lat(atof(attrs[count+1]));
+ check_attributes(attrs, [&min, &max](const XML_Char* name, const XML_Char* value) {
+ if (!strcmp(name, "minlon")) {
+ min.set_lon(atof(value));
+ } else if (!strcmp(name, "minlat")) {
+ min.set_lat(atof(value));
+ } else if (!strcmp(name, "maxlon")) {
+ max.set_lon(atof(value));
+ } else if (!strcmp(name, "maxlat")) {
+ max.set_lat(atof(value));
}
- }
+ });
osmium::Box box;
box.extend(min).extend(max);
m_header.add_box(box);
@@ -529,7 +393,9 @@ namespace osmium {
case context::node:
m_last_context = context::node;
m_context = context::in_object;
- check_tag(m_node_builder.get(), element, attrs);
+ if (!strcmp(element, "tag")) {
+ get_tag(m_node_builder.get(), attrs);
+ }
break;
case context::way:
m_last_context = context::way;
@@ -541,13 +407,14 @@ namespace osmium {
m_wnl_builder = std::unique_ptr<osmium::builder::WayNodeListBuilder>(new osmium::builder::WayNodeListBuilder(m_buffer, m_way_builder.get()));
}
- for (int count = 0; attrs[count]; count += 2) {
- if (!strcmp(attrs[count], "ref")) {
- m_wnl_builder->add_node_ref(osmium::string_to_object_id(attrs[count+1]));
+ check_attributes(attrs, [this](const XML_Char* name, const XML_Char* value) {
+ if (!strcmp(name, "ref")) {
+ m_wnl_builder->add_node_ref(osmium::string_to_object_id(value));
}
- }
- } else {
- check_tag(m_way_builder.get(), element, attrs);
+ });
+ } else if (!strcmp(element, "tag")) {
+ m_wnl_builder.reset();
+ get_tag(m_way_builder.get(), attrs);
}
break;
case context::relation:
@@ -560,28 +427,68 @@ namespace osmium {
m_rml_builder = std::unique_ptr<osmium::builder::RelationMemberListBuilder>(new osmium::builder::RelationMemberListBuilder(m_buffer, m_relation_builder.get()));
}
- char type = 'x';
- object_id_type ref = 0;
+ item_type type = item_type::undefined;
+ object_id_type ref = 0;
const char* role = "";
- for (int count = 0; attrs[count]; count += 2) {
- if (!strcmp(attrs[count], "type")) {
- type = static_cast<char>(attrs[count+1][0]);
- } else if (!strcmp(attrs[count], "ref")) {
- ref = osmium::string_to_object_id(attrs[count+1]);
- } else if (!strcmp(attrs[count], "role")) {
- role = static_cast<const char*>(attrs[count+1]);
+ check_attributes(attrs, [&type, &ref, &role](const XML_Char* name, const XML_Char* value) {
+ if (!strcmp(name, "type")) {
+ type = char_to_item_type(value[0]);
+ } else if (!strcmp(name, "ref")) {
+ ref = osmium::string_to_object_id(value);
+ } else if (!strcmp(name, "role")) {
+ role = static_cast<const char*>(value);
}
+ });
+ if (type != item_type::node && type != item_type::way && type != item_type::relation) {
+ throw osmium::xml_error("Unknown type on relation member");
}
- // XXX assert type, ref, role are set
- m_rml_builder->add_member(char_to_item_type(type), ref, role);
- } else {
- check_tag(m_relation_builder.get(), element, attrs);
+ if (ref == 0) {
+ throw osmium::xml_error("Missing ref on relation member");
+ }
+ m_rml_builder->add_member(type, ref, role);
+ } else if (!strcmp(element, "tag")) {
+ m_rml_builder.reset();
+ get_tag(m_relation_builder.get(), attrs);
}
break;
case context::changeset:
m_last_context = context::changeset;
- m_context = context::in_object;
- check_tag(m_changeset_builder.get(), element, attrs);
+ if (!strcmp(element, "discussion")) {
+ m_context = context::discussion;
+ m_tl_builder.reset();
+ if (!m_changeset_discussion_builder) {
+ m_changeset_discussion_builder = std::unique_ptr<osmium::builder::ChangesetDiscussionBuilder>(new osmium::builder::ChangesetDiscussionBuilder(m_buffer, m_changeset_builder.get()));
+ }
+ } else if (!strcmp(element, "tag")) {
+ m_context = context::in_object;
+ m_changeset_discussion_builder.reset();
+ get_tag(m_changeset_builder.get(), attrs);
+ }
+ break;
+ case context::discussion:
+ if (!strcmp(element, "comment")) {
+ m_context = context::comment;
+ osmium::Timestamp date;
+ osmium::user_id_type uid = 0;
+ const char* user = "";
+ check_attributes(attrs, [&date, &uid, &user](const XML_Char* name, const XML_Char* value) {
+ if (!strcmp(name, "date")) {
+ date = osmium::Timestamp(value);
+ } else if (!strcmp(name, "uid")) {
+ uid = osmium::string_to_user_id(value);
+ } else if (!strcmp(name, "user")) {
+ user = static_cast<const char*>(value);
+ }
+ });
+ m_changeset_discussion_builder->add_comment(date, uid, user);
+ }
+ break;
+ case context::comment:
+ if (!strcmp(element, "text")) {
+ m_context = context::comment_text;
+ }
+ break;
+ case context::comment_text:
break;
case context::ignored_node:
break;
@@ -604,7 +511,7 @@ namespace osmium {
break;
case context::top:
if (!strcmp(element, "osm") || !strcmp(element, "osmChange")) {
- header_is_done();
+ mark_header_as_done();
m_context = context::root;
} else if (!strcmp(element, "delete")) {
m_in_delete_section = false;
@@ -639,11 +546,25 @@ namespace osmium {
case context::changeset:
assert(!strcmp(element, "changeset"));
m_tl_builder.reset();
+ m_changeset_discussion_builder.reset();
m_changeset_builder.reset();
m_buffer.commit();
m_context = context::top;
flush_buffer();
break;
+ case context::discussion:
+ assert(!strcmp(element, "discussion"));
+ m_context = context::changeset;
+ break;
+ case context::comment:
+ assert(!strcmp(element, "comment"));
+ m_context = context::discussion;
+ break;
+ case context::comment_text:
+ assert(!strcmp(element, "text"));
+ m_context = context::comment;
+ m_changeset_discussion_builder->add_comment_text(m_comment_text);
+ break;
case context::in_object:
m_context = m_last_context;
break;
@@ -670,85 +591,84 @@ namespace osmium {
}
}
+ void characters(const XML_Char* text, int len) {
+ if (m_context == context::comment_text) {
+ m_comment_text.append(text, len);
+ } else {
+ m_comment_text.resize(0);
+ }
+ }
+
void flush_buffer() {
- if (m_buffer.capacity() - m_buffer.committed() < 1000 * 1000) {
- m_queue.push(std::move(m_buffer));
+ if (m_buffer.committed() > buffer_size / 10 * 9) {
+ send_to_output_queue(std::move(m_buffer));
osmium::memory::Buffer buffer(buffer_size);
- std::swap(m_buffer, buffer);
+ using std::swap;
+ swap(m_buffer, buffer);
}
}
- }; // class XMLParser
-
- class XMLInputFormat : public osmium::io::detail::InputFormat {
-
- static constexpr size_t max_queue_size = 100;
-
- osmium::thread::Queue<osmium::memory::Buffer> m_queue;
- std::atomic<bool> m_done;
- std::promise<osmium::io::Header> m_header_promise;
- std::future<bool> m_parser_future;
-
public:
- /**
- * Instantiate XML Parser
- *
- * @param file osmium::io::File instance describing file to be read from.
- * @param read_which_entities Which types of OSM entities (nodes, ways, relations, changesets) should be parsed?
- * @param input_queue String queue where data is read from.
- */
- explicit XMLInputFormat(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) :
- osmium::io::detail::InputFormat(file, read_which_entities),
- m_queue(max_queue_size, "xml_parser_results"),
- m_done(false),
- m_header_promise(),
- m_parser_future(std::async(std::launch::async, XMLParser(input_queue, m_queue, m_header_promise, read_which_entities, m_done))) {
+ XMLParser(future_string_queue_type& input_queue,
+ future_buffer_queue_type& output_queue,
+ std::promise<osmium::io::Header>& header_promise,
+ osmium::osm_entity_bits::type read_types) :
+ Parser(input_queue, output_queue, header_promise, read_types),
+ m_context(context::root),
+ m_last_context(context::root),
+ m_in_delete_section(false),
+ m_header(),
+ m_buffer(buffer_size),
+ m_node_builder(),
+ m_way_builder(),
+ m_relation_builder(),
+ m_changeset_builder(),
+ m_changeset_discussion_builder(),
+ m_tl_builder(),
+ m_wnl_builder(),
+ m_rml_builder() {
}
- ~XMLInputFormat() {
- try {
- close();
- } catch (...) {
- // ignore any exceptions at this point because destructor should not throw
- }
- }
+ ~XMLParser() noexcept = default;
- virtual osmium::io::Header header() override final {
- osmium::thread::check_for_exception(m_parser_future);
- return m_header_promise.get_future().get();
- }
+ void run() override final {
+ osmium::thread::set_thread_name("_osmium_xml_in");
+
+ ExpatXMLParser<XMLParser> parser(this);
- osmium::memory::Buffer read() override {
- osmium::memory::Buffer buffer;
- if (!m_done || !m_queue.empty()) {
- m_queue.wait_and_pop(buffer);
+ while (!input_done()) {
+ std::string data = get_input();
+ parser(data, input_done());
+ if (read_types() == osmium::osm_entity_bits::nothing && header_is_done()) {
+ break;
+ }
}
- osmium::thread::check_for_exception(m_parser_future);
- return buffer;
- }
+ mark_header_as_done();
- void close() override {
- m_done = true;
- osmium::thread::wait_until_done(m_parser_future);
+ if (m_buffer.committed() > 0) {
+ send_to_output_queue(std::move(m_buffer));
+ }
}
- }; // class XMLInputFormat
-
- namespace {
-
-// we want the register_input_format() function to run, setting the variable
-// is only a side-effect, it will never be used
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-variable"
- const bool registered_xml_input = osmium::io::detail::InputFormatFactory::instance().register_input_format(osmium::io::file_format::xml,
- [](const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities, osmium::thread::Queue<std::string>& input_queue) {
- return new osmium::io::detail::XMLInputFormat(file, read_which_entities, input_queue);
- });
-#pragma GCC diagnostic pop
+ }; // class XMLParser
- } // anonymous namespace
+ // we want the register_parser() function to run, setting
+ // the variable is only a side-effect, it will never be used
+ const bool registered_xml_parser = ParserFactory::instance().register_parser(
+ file_format::xml,
+ [](future_string_queue_type& input_queue,
+ future_buffer_queue_type& output_queue,
+ std::promise<osmium::io::Header>& header_promise,
+ osmium::osm_entity_bits::type read_which_entities) {
+ return std::unique_ptr<Parser>(new XMLParser(input_queue, output_queue, header_promise, read_which_entities));
+ });
+
+ // dummy function to silence the unused variable warning from above
+ inline bool get_registered_xml_parser() noexcept {
+ return registered_xml_parser;
+ }
} // namespace detail
diff --git a/include/osmium/io/detail/xml_output_format.hpp b/include/osmium/io/detail/xml_output_format.hpp
index 2a381d5..b91b051 100644
--- a/include/osmium/io/detail/xml_output_format.hpp
+++ b/include/osmium/io/detail/xml_output_format.hpp
@@ -33,20 +33,17 @@ DEALINGS IN THE SOFTWARE.
*/
-#include <cassert>
-#include <chrono>
+#include <algorithm>
#include <cinttypes>
#include <cstddef>
#include <cstdio>
#include <future>
#include <iterator>
#include <memory>
-#include <ratio>
#include <string>
#include <thread>
#include <utility>
-#include <osmium/handler.hpp>
#include <osmium/io/detail/output_format.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/file_format.hpp>
@@ -75,45 +72,23 @@ namespace osmium {
struct XMLWriteError {};
- namespace {
-
- void xml_string(std::string& out, const char* in) {
- for (; *in != '\0'; ++in) {
- switch(*in) {
- case '&': out += "&"; break;
- case '\"': out += """; break;
- case '\'': out += "'"; break;
- case '<': out += "<"; break;
- case '>': out += ">"; break;
- case '\n': out += "
"; break;
- case '\r': out += "
"; break;
- case '\t': out += "	"; break;
- default: out += *in; break;
- }
- }
- }
+ struct xml_output_options {
- const size_t tmp_buffer_size = 100;
-
- template <typename T>
- void oprintf(std::string& out, const char* format, T value) {
- char buffer[tmp_buffer_size+1];
- size_t max_size = sizeof(buffer)/sizeof(char);
-#ifndef NDEBUG
- int len =
-#endif
-#ifndef _MSC_VER
- snprintf(buffer, max_size, format, value);
-#else
- _snprintf(buffer, max_size, format, value);
-#endif
- assert(len > 0 && static_cast<size_t>(len) < max_size);
- out += buffer;
- }
+ /// Should metadata of objects be added?
+ bool add_metadata;
+
+ /// Should the visible flag be added to all OSM objects?
+ bool add_visible_flag;
+
+ /**
+ * Should <create>, <modify>, <delete> "operations" be added?
+ * (This is used for .osc files.)
+ */
+ bool use_change_ops;
- } // anonymous namespace
+ };
- class XMLOutputBlock : public osmium::handler::Handler {
+ class XMLOutputBlock : public OutputBlock {
// operation (create, modify, delete) for osc files
enum class operation {
@@ -123,15 +98,9 @@ namespace osmium {
op_delete = 3
}; // enum class operation
- std::shared_ptr<osmium::memory::Buffer> m_input_buffer;
-
- std::shared_ptr<std::string> m_out;
-
operation m_last_op {operation::op_none};
- const bool m_add_metadata;
- const bool m_write_visible_flag;
- const bool m_write_change_ops;
+ xml_output_options m_options;
void write_spaces(int num) {
for (; num != 0; --num) {
@@ -139,20 +108,20 @@ namespace osmium {
}
}
+ int prefix_spaces() {
+ return m_options.use_change_ops ? 4 : 2;
+ }
+
void write_prefix() {
- if (m_write_change_ops) {
- write_spaces(4);
- } else {
- write_spaces(2);
- }
+ write_spaces(prefix_spaces());
}
void write_meta(const osmium::OSMObject& object) {
- oprintf(*m_out, " id=\"%" PRId64 "\"", object.id());
+ output_formatted(" id=\"%" PRId64 "\"", object.id());
- if (m_add_metadata) {
+ if (m_options.add_metadata) {
if (object.version()) {
- oprintf(*m_out, " version=\"%d\"", object.version());
+ output_formatted(" version=\"%d\"", object.version());
}
if (object.timestamp()) {
@@ -162,16 +131,16 @@ namespace osmium {
}
if (!object.user_is_anonymous()) {
- oprintf(*m_out, " uid=\"%d\" user=\"", object.uid());
- xml_string(*m_out, object.user());
+ output_formatted(" uid=\"%d\" user=\"", object.uid());
+ append_xml_encoded_string(*m_out, object.user());
*m_out += "\"";
}
if (object.changeset()) {
- oprintf(*m_out, " changeset=\"%d\"", object.changeset());
+ output_formatted(" changeset=\"%d\"", object.changeset());
}
- if (m_write_visible_flag) {
+ if (m_options.add_visible_flag) {
if (object.visible()) {
*m_out += " visible=\"true\"";
} else {
@@ -181,17 +150,31 @@ namespace osmium {
}
}
- void write_tags(const osmium::TagList& tags) {
+ void write_tags(const osmium::TagList& tags, int spaces) {
for (const auto& tag : tags) {
- write_prefix();
+ write_spaces(spaces);
*m_out += " <tag k=\"";
- xml_string(*m_out, tag.key());
+ append_xml_encoded_string(*m_out, tag.key());
*m_out += "\" v=\"";
- xml_string(*m_out, tag.value());
+ append_xml_encoded_string(*m_out, tag.value());
*m_out += "\"/>\n";
}
}
+ void write_discussion(const osmium::ChangesetDiscussion& comments) {
+ for (const auto& comment : comments) {
+ output_formatted(" <comment uid=\"%d\" user=\"", comment.uid());
+ append_xml_encoded_string(*m_out, comment.user());
+ *m_out += "\" date=\"";
+ *m_out += comment.date().to_iso();
+ *m_out += "\">\n";
+ *m_out += " <text>";
+ append_xml_encoded_string(*m_out, comment.text());
+ *m_out += "</text>\n </comment>\n";
+ }
+ *m_out += " </discussion>\n";
+ }
+
void open_close_op_tag(const operation op = operation::op_none) {
if (op == m_last_op) {
return;
@@ -230,12 +213,9 @@ namespace osmium {
public:
- explicit XMLOutputBlock(osmium::memory::Buffer&& buffer, bool add_metadata, bool write_visible_flag, bool write_change_ops) :
- m_input_buffer(std::make_shared<osmium::memory::Buffer>(std::move(buffer))),
- m_out(std::make_shared<std::string>()),
- m_add_metadata(add_metadata),
- m_write_visible_flag(write_visible_flag && !write_change_ops),
- m_write_change_ops(write_change_ops) {
+ XMLOutputBlock(osmium::memory::Buffer&& buffer, const xml_output_options& options) :
+ OutputBlock(std::move(buffer)),
+ m_options(options) {
}
XMLOutputBlock(const XMLOutputBlock&) = default;
@@ -244,22 +224,24 @@ namespace osmium {
XMLOutputBlock(XMLOutputBlock&&) = default;
XMLOutputBlock& operator=(XMLOutputBlock&&) = default;
- ~XMLOutputBlock() = default;
+ ~XMLOutputBlock() noexcept = default;
std::string operator()() {
osmium::apply(m_input_buffer->cbegin(), m_input_buffer->cend(), *this);
- if (m_write_change_ops) {
+ if (m_options.use_change_ops) {
open_close_op_tag();
}
std::string out;
- std::swap(out, *m_out);
+ using std::swap;
+ swap(out, *m_out);
+
return out;
}
void node(const osmium::Node& node) {
- if (m_write_change_ops) {
+ if (m_options.use_change_ops) {
open_close_op_tag(node.visible() ? (node.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete);
}
@@ -283,14 +265,14 @@ namespace osmium {
*m_out += ">\n";
- write_tags(node.tags());
+ write_tags(node.tags(), prefix_spaces());
write_prefix();
*m_out += "</node>\n";
}
void way(const osmium::Way& way) {
- if (m_write_change_ops) {
+ if (m_options.use_change_ops) {
open_close_op_tag(way.visible() ? (way.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete);
}
@@ -307,17 +289,17 @@ namespace osmium {
for (const auto& node_ref : way.nodes()) {
write_prefix();
- oprintf(*m_out, " <nd ref=\"%" PRId64 "\"/>\n", node_ref.ref());
+ output_formatted(" <nd ref=\"%" PRId64 "\"/>\n", node_ref.ref());
}
- write_tags(way.tags());
+ write_tags(way.tags(), prefix_spaces());
write_prefix();
*m_out += "</way>\n";
}
void relation(const osmium::Relation& relation) {
- if (m_write_change_ops) {
+ if (m_options.use_change_ops) {
open_close_op_tag(relation.visible() ? (relation.version() == 1 ? operation::op_create : operation::op_modify) : operation::op_delete);
}
@@ -336,22 +318,21 @@ namespace osmium {
write_prefix();
*m_out += " <member type=\"";
*m_out += item_type_to_name(member.type());
- oprintf(*m_out, "\" ref=\"%" PRId64 "\" role=\"", member.ref());
- xml_string(*m_out, member.role());
+ output_formatted("\" ref=\"%" PRId64 "\" role=\"", member.ref());
+ append_xml_encoded_string(*m_out, member.role());
*m_out += "\"/>\n";
}
- write_tags(relation.tags());
+ write_tags(relation.tags(), prefix_spaces());
write_prefix();
*m_out += "</relation>\n";
}
void changeset(const osmium::Changeset& changeset) {
- write_prefix();
- *m_out += "<changeset";
+ *m_out += " <changeset";
- oprintf(*m_out, " id=\"%" PRId32 "\"", changeset.id());
+ output_formatted(" id=\"%" PRId32 "\"", changeset.id());
if (changeset.created_at()) {
*m_out += " created_at=\"";
@@ -359,8 +340,6 @@ namespace osmium {
*m_out += "\"";
}
- oprintf(*m_out, " num_changes=\"%" PRId32 "\"", changeset.num_changes());
-
if (changeset.closed_at()) {
*m_out += " closed_at=\"";
*m_out += changeset.closed_at().to_iso();
@@ -369,64 +348,67 @@ namespace osmium {
*m_out += " open=\"true\"";
}
- if (changeset.bounds()) {
- oprintf(*m_out, " min_lon=\"%.7f\"", changeset.bounds().bottom_left().lon_without_check());
- oprintf(*m_out, " min_lat=\"%.7f\"", changeset.bounds().bottom_left().lat_without_check());
- oprintf(*m_out, " max_lon=\"%.7f\"", changeset.bounds().top_right().lon_without_check());
- oprintf(*m_out, " max_lat=\"%.7f\"", changeset.bounds().top_right().lat_without_check());
- }
-
if (!changeset.user_is_anonymous()) {
*m_out += " user=\"";
- xml_string(*m_out, changeset.user());
- oprintf(*m_out, "\" uid=\"%d\"", changeset.uid());
+ append_xml_encoded_string(*m_out, changeset.user());
+ output_formatted("\" uid=\"%d\"", changeset.uid());
}
- if (changeset.tags().empty()) {
+ if (changeset.bounds()) {
+ output_formatted(" min_lat=\"%.7f\"", changeset.bounds().bottom_left().lat_without_check());
+ output_formatted(" min_lon=\"%.7f\"", changeset.bounds().bottom_left().lon_without_check());
+ output_formatted(" max_lat=\"%.7f\"", changeset.bounds().top_right().lat_without_check());
+ output_formatted(" max_lon=\"%.7f\"", changeset.bounds().top_right().lon_without_check());
+ }
+
+ output_formatted(" num_changes=\"%" PRId32 "\"", changeset.num_changes());
+ output_formatted(" comments_count=\"%" PRId32 "\"", changeset.num_comments());
+
+ // If there are no tags and no comments, we can close the
+ // tag right here and are done.
+ if (changeset.tags().empty() && changeset.num_comments() == 0) {
*m_out += "/>\n";
return;
}
*m_out += ">\n";
- write_tags(changeset.tags());
+ write_tags(changeset.tags(), 0);
- write_prefix();
- *m_out += "</changeset>\n";
+ if (changeset.num_comments() > 0) {
+ *m_out += " <discussion>\n";
+ write_discussion(changeset.discussion());
+ }
+
+ *m_out += " </changeset>\n";
}
}; // class XMLOutputBlock
class XMLOutputFormat : public osmium::io::detail::OutputFormat, public osmium::handler::Handler {
- bool m_add_metadata;
- bool m_write_visible_flag;
+ xml_output_options m_options;
public:
- XMLOutputFormat(const osmium::io::File& file, data_queue_type& output_queue) :
- OutputFormat(file, output_queue),
- m_add_metadata(file.get("add_metadata") != "false"),
- m_write_visible_flag(file.has_multiple_object_versions() || m_file.is_true("force_visible_flag")) {
+ XMLOutputFormat(const osmium::io::File& file, future_string_queue_type& output_queue) :
+ OutputFormat(output_queue),
+ m_options() {
+ m_options.add_metadata = file.is_not_false("add_metadata");
+ m_options.use_change_ops = file.is_true("xml_change_format");
+ m_options.add_visible_flag = (file.has_multiple_object_versions() || file.is_true("force_visible_flag")) && !m_options.use_change_ops;
}
XMLOutputFormat(const XMLOutputFormat&) = delete;
XMLOutputFormat& operator=(const XMLOutputFormat&) = delete;
- ~XMLOutputFormat() override final {
- }
-
- void write_buffer(osmium::memory::Buffer&& buffer) override final {
- m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_add_metadata, m_write_visible_flag, m_file.is_true("xml_change_format")}));
- }
+ ~XMLOutputFormat() noexcept = default;
void write_header(const osmium::io::Header& header) override final {
std::string out = "<?xml version='1.0' encoding='UTF-8'?>\n";
- if (m_file.is_true("xml_change_format")) {
+ if (m_options.use_change_ops) {
out += "<osmChange version=\"0.6\" generator=\"";
- xml_string(out, header.get("generator").c_str());
- out += "\">\n";
} else {
out += "<osm version=\"0.6\"";
@@ -437,57 +419,50 @@ namespace osmium {
out += "\"";
}
out += " generator=\"";
- xml_string(out, header.get("generator").c_str());
- out += "\">\n";
}
+ append_xml_encoded_string(out, header.get("generator").c_str());
+ out += "\">\n";
for (const auto& box : header.boxes()) {
out += " <bounds";
- oprintf(out, " minlon=\"%.7f\"", box.bottom_left().lon());
- oprintf(out, " minlat=\"%.7f\"", box.bottom_left().lat());
- oprintf(out, " maxlon=\"%.7f\"", box.top_right().lon());
- oprintf(out, " maxlat=\"%.7f\"/>\n", box.top_right().lat());
+ append_printf_formatted_string(out, " minlon=\"%.7f\"", box.bottom_left().lon());
+ append_printf_formatted_string(out, " minlat=\"%.7f\"", box.bottom_left().lat());
+ append_printf_formatted_string(out, " maxlon=\"%.7f\"", box.top_right().lon());
+ append_printf_formatted_string(out, " maxlat=\"%.7f\"/>\n", box.top_right().lat());
}
- std::promise<std::string> promise;
- m_output_queue.push(promise.get_future());
- promise.set_value(std::move(out));
+ send_to_output_queue(std::move(out));
}
- void close() override final {
- {
- std::string out;
- if (m_file.is_true("xml_change_format")) {
- out += "</osmChange>\n";
- } else {
- out += "</osm>\n";
- }
+ void write_buffer(osmium::memory::Buffer&& buffer) override final {
+ m_output_queue.push(osmium::thread::Pool::instance().submit(XMLOutputBlock{std::move(buffer), m_options}));
+ }
- std::promise<std::string> promise;
- m_output_queue.push(promise.get_future());
- promise.set_value(std::move(out));
+ void write_end() override final {
+ std::string out;
+
+ if (m_options.use_change_ops) {
+ out += "</osmChange>\n";
+ } else {
+ out += "</osm>\n";
}
- std::promise<std::string> promise;
- m_output_queue.push(promise.get_future());
- promise.set_value(std::string());
+ send_to_output_queue(std::move(out));
}
}; // class XMLOutputFormat
- namespace {
-
-// we want the register_output_format() function to run, setting the variable
-// is only a side-effect, it will never be used
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-variable"
- const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml,
- [](const osmium::io::File& file, data_queue_type& output_queue) {
- return new osmium::io::detail::XMLOutputFormat(file, output_queue);
- });
-#pragma GCC diagnostic pop
-
- } // anonymous namespace
+ // we want the register_output_format() function to run, setting
+ // the variable is only a side-effect, it will never be used
+ const bool registered_xml_output = osmium::io::detail::OutputFormatFactory::instance().register_output_format(osmium::io::file_format::xml,
+ [](const osmium::io::File& file, future_string_queue_type& output_queue) {
+ return new osmium::io::detail::XMLOutputFormat(file, output_queue);
+ });
+
+ // dummy function to silence the unused variable warning from above
+ inline bool get_registered_xml_output() noexcept {
+ return registered_xml_output;
+ }
} // namespace detail
diff --git a/include/osmium/io/detail/zlib.hpp b/include/osmium/io/detail/zlib.hpp
index fc0baf3..4d86168 100644
--- a/include/osmium/io/detail/zlib.hpp
+++ b/include/osmium/io/detail/zlib.hpp
@@ -39,6 +39,7 @@ DEALINGS IN THE SOFTWARE.
#include <zlib.h>
+#include <osmium/io/error.hpp>
#include <osmium/util/cast.hpp>
namespace osmium {
@@ -69,7 +70,7 @@ namespace osmium {
);
if (result != Z_OK) {
- throw std::runtime_error(std::string("failed to compress data: ") + zError(result));
+ throw io_error(std::string("failed to compress data: ") + zError(result));
}
output.resize(output_size);
@@ -99,7 +100,7 @@ namespace osmium {
);
if (result != Z_OK) {
- throw std::runtime_error(std::string("failed to uncompress data: ") + zError(result));
+ throw io_error(std::string("failed to uncompress data: ") + zError(result));
}
return std::make_pair(output.data(), output.size());
diff --git a/include/osmium/io/error.hpp b/include/osmium/io/error.hpp
index 07652bc..0b5393f 100644
--- a/include/osmium/io/error.hpp
+++ b/include/osmium/io/error.hpp
@@ -53,6 +53,18 @@ namespace osmium {
}; // struct io_error
+ struct unsupported_file_format_error : public io_error {
+
+ unsupported_file_format_error(const std::string& what) :
+ io_error(what) {
+ }
+
+ unsupported_file_format_error(const char* what) :
+ io_error(what) {
+ }
+
+ }; // struct unsupported_file_format_error
+
} // namespace osmium
#endif // OSMIUM_IO_ERROR_HPP
diff --git a/include/osmium/io/file.hpp b/include/osmium/io/file.hpp
index f5acef4..87fcc37 100644
--- a/include/osmium/io/file.hpp
+++ b/include/osmium/io/file.hpp
@@ -39,6 +39,7 @@ DEALINGS IN THE SOFTWARE.
#include <string>
#include <vector>
+#include <osmium/io/error.hpp>
#include <osmium/io/file_format.hpp>
#include <osmium/io/file_compression.hpp>
#include <osmium/util/options.hpp>
@@ -255,7 +256,7 @@ namespace osmium {
* Check file format etc. for consistency and throw exception if
* there is a problem.
*
- * @throws std::runtime_error
+ * @throws osmium::io_error
*/
const File& check() const {
if (m_file_format == file_format::unknown) {
@@ -273,7 +274,7 @@ namespace osmium {
msg += "'";
}
msg += ".";
- throw std::runtime_error(msg);
+ throw io_error(msg);
}
return *this;
}
diff --git a/include/osmium/io/gzip_compression.hpp b/include/osmium/io/gzip_compression.hpp
index 204bfd5..c0fdd93 100644
--- a/include/osmium/io/gzip_compression.hpp
+++ b/include/osmium/io/gzip_compression.hpp
@@ -49,7 +49,9 @@ DEALINGS IN THE SOFTWARE.
#include <zlib.h>
#include <osmium/io/compression.hpp>
+#include <osmium/io/error.hpp>
#include <osmium/io/file_compression.hpp>
+#include <osmium/io/overwrite.hpp>
#include <osmium/util/cast.hpp>
#include <osmium/util/compatibility.hpp>
@@ -59,13 +61,13 @@ namespace osmium {
* Exception thrown when there are problems compressing or
* decompressing gzip files.
*/
- struct gzip_error : public std::runtime_error {
+ struct gzip_error : public io_error {
int gzip_error_code;
int system_errno;
gzip_error(const std::string& what, int error_code) :
- std::runtime_error(what),
+ io_error(what),
gzip_error_code(error_code),
system_errno(error_code == Z_ERRNO ? errno : 0) {
}
@@ -93,20 +95,26 @@ namespace osmium {
class GzipCompressor : public Compressor {
+ int m_fd;
gzFile m_gzfile;
public:
- explicit GzipCompressor(int fd) :
- Compressor(),
+ explicit GzipCompressor(int fd, fsync sync) :
+ Compressor(sync),
+ m_fd(dup(fd)),
m_gzfile(::gzdopen(fd, "w")) {
if (!m_gzfile) {
detail::throw_gzip_error(m_gzfile, "write initialization failed");
}
}
- ~GzipCompressor() override final {
- close();
+ ~GzipCompressor() noexcept override final {
+ try {
+ close();
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
+ }
}
void write(const std::string& data) override final {
@@ -125,6 +133,10 @@ namespace osmium {
if (result != Z_OK) {
detail::throw_gzip_error(m_gzfile, "write close failed", result);
}
+ if (do_fsync()) {
+ osmium::io::detail::reliable_fsync(m_fd);
+ }
+ osmium::io::detail::reliable_close(m_fd);
}
}
@@ -144,8 +156,12 @@ namespace osmium {
}
}
- ~GzipDecompressor() override final {
- close();
+ ~GzipDecompressor() noexcept override final {
+ try {
+ close();
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
+ }
}
std::string read() override final {
@@ -194,8 +210,12 @@ namespace osmium {
}
}
- ~GzipBufferDecompressor() override final {
- inflateEnd(&m_zstream);
+ ~GzipBufferDecompressor() noexcept override final {
+ try {
+ close();
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
+ }
}
std::string read() override final {
@@ -227,22 +247,28 @@ namespace osmium {
return output;
}
+ void close() override final {
+ inflateEnd(&m_zstream);
+ }
+
}; // class GzipBufferDecompressor
- namespace {
+ namespace detail {
-// we want the register_compression() function to run, setting the variable
-// is only a side-effect, it will never be used
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-variable"
+ // we want the register_compression() function to run, setting
+ // the variable is only a side-effect, it will never be used
const bool registered_gzip_compression = osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip,
- [](int fd) { return new osmium::io::GzipCompressor(fd); },
+ [](int fd, fsync sync) { return new osmium::io::GzipCompressor(fd, sync); },
[](int fd) { return new osmium::io::GzipDecompressor(fd); },
[](const char* buffer, size_t size) { return new osmium::io::GzipBufferDecompressor(buffer, size); }
);
-#pragma GCC diagnostic pop
- } // anonymous namespace
+ // dummy function to silence the unused variable warning from above
+ inline bool get_registered_gzip_compression() noexcept {
+ return registered_gzip_compression;
+ }
+
+ } // namespace detail
} // namespace io
diff --git a/include/osmium/io/input_iterator.hpp b/include/osmium/io/input_iterator.hpp
index f619729..b0bf56c 100644
--- a/include/osmium/io/input_iterator.hpp
+++ b/include/osmium/io/input_iterator.hpp
@@ -133,6 +133,44 @@ namespace osmium {
}; // class InputIterator
+ template <class TSource, class TItem = osmium::memory::Item>
+ class InputIteratorRange {
+
+ InputIterator<TSource, TItem> m_begin;
+ InputIterator<TSource, TItem> m_end;
+
+ public:
+
+ InputIteratorRange(InputIterator<TSource, TItem>&& begin,
+ InputIterator<TSource, TItem>&& end) :
+ m_begin(std::move(begin)),
+ m_end(std::move(end)) {
+ }
+
+ InputIterator<TSource, TItem> begin() const noexcept {
+ return m_begin;
+ }
+
+ InputIterator<TSource, TItem> end() const noexcept {
+ return m_end;
+ }
+
+ const InputIterator<TSource, TItem> cbegin() const noexcept {
+ return m_begin;
+ }
+
+ const InputIterator<TSource, TItem> cend() const noexcept {
+ return m_end;
+ }
+
+ }; // class InputIteratorRange
+
+ template <class TItem, class TSource>
+ InputIteratorRange<TSource, TItem> make_input_iterator_range(TSource& source) {
+ using it_type = InputIterator<TSource, TItem>;
+ return InputIteratorRange<TSource, TItem>(it_type{source}, it_type{});
+ }
+
} // namespace io
} // namespace osmium
diff --git a/include/osmium/io/overwrite.hpp b/include/osmium/io/o5m_input.hpp
similarity index 80%
copy from include/osmium/io/overwrite.hpp
copy to include/osmium/io/o5m_input.hpp
index e33894b..b63f728 100644
--- a/include/osmium/io/overwrite.hpp
+++ b/include/osmium/io/o5m_input.hpp
@@ -1,5 +1,5 @@
-#ifndef OSMIUM_IO_OVERWRITE_HPP
-#define OSMIUM_IO_OVERWRITE_HPP
+#ifndef OSMIUM_IO_O5M_INPUT_HPP
+#define OSMIUM_IO_O5M_INPUT_HPP
/*
@@ -33,20 +33,13 @@ DEALINGS IN THE SOFTWARE.
*/
-namespace osmium {
+/**
+ * @file
+ *
+ * Include this file if you want to read OSM o5m and o5c files.
+ */
- namespace io {
+#include <osmium/io/reader.hpp> // IWYU pragma: export
+#include <osmium/io/detail/o5m_input_format.hpp> // IWYU pragma: export
- /**
- * Allow overwriting of existing file.
- */
- enum class overwrite : bool {
- no = false,
- allow = true
- };
-
- } // namespace io
-
-} // namespace osmium
-
-#endif // OSMIUM_IO_OVERWRITE_HPP
+#endif // OSMIUM_IO_O5M_INPUT_HPP
diff --git a/include/osmium/io/output_iterator.hpp b/include/osmium/io/output_iterator.hpp
index 608852f..8f2203a 100644
--- a/include/osmium/io/output_iterator.hpp
+++ b/include/osmium/io/output_iterator.hpp
@@ -41,6 +41,14 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/diff_object.hpp>
+#ifdef __GNUC__
+# define DEPRECATED __attribute__((deprecated))
+#elif defined(_MSC_VER)
+# define DEPRECATED __declspec(deprecated)
+#else
+# define DEPRECATED
+#endif
+
namespace osmium {
namespace memory {
@@ -52,27 +60,22 @@ namespace osmium {
template <class TDest>
class OutputIterator : public std::iterator<std::output_iterator_tag, osmium::memory::Item> {
- struct buffer_wrapper {
-
- osmium::memory::Buffer buffer;
-
- buffer_wrapper(size_t buffer_size) :
- buffer(buffer_size, osmium::memory::Buffer::auto_grow::no) {
- }
-
- }; // struct buffer_wrapper
-
- static constexpr size_t default_buffer_size = 10 * 1024 * 1024;
-
TDest* m_destination;
- std::shared_ptr<buffer_wrapper> m_buffer_wrapper;
-
public:
- explicit OutputIterator(TDest& destination, const size_t buffer_size = default_buffer_size) :
- m_destination(&destination),
- m_buffer_wrapper(std::make_shared<buffer_wrapper>(buffer_size)) {
+ explicit OutputIterator(TDest& destination) :
+ m_destination(&destination) {
+ }
+
+ /**
+ * Warning! Use of buffer size argument on OutputIterator
+ * constructor is deprecated. Call Writer::set_buffer_size()
+ * instead if you want to change the default.
+ */
+ DEPRECATED OutputIterator(TDest& destination, const size_t buffer_size) :
+ m_destination(&destination) {
+ destination.set_buffer_size(buffer_size);
}
OutputIterator(const OutputIterator&) = default;
@@ -83,19 +86,16 @@ namespace osmium {
~OutputIterator() = default;
- void flush() {
- osmium::memory::Buffer buffer(m_buffer_wrapper->buffer.capacity(), osmium::memory::Buffer::auto_grow::no);
- std::swap(m_buffer_wrapper->buffer, buffer);
- (*m_destination)(std::move(buffer));
+ /**
+ * Warning! Calling OutputIterator<Writer>::flush() is usually not
+ * needed any more. Call flush() on the Writer instead if needed.
+ */
+ DEPRECATED void flush() {
+ m_destination->flush();
}
OutputIterator& operator=(const osmium::memory::Item& item) {
- try {
- m_buffer_wrapper->buffer.push_back(item);
- } catch (osmium::buffer_is_full&) {
- flush();
- m_buffer_wrapper->buffer.push_back(item);
- }
+ (*m_destination)(item);
return *this;
}
@@ -117,8 +117,26 @@ namespace osmium {
}; // class OutputIterator
+ template <class TDest>
+ OutputIterator<TDest> make_output_iterator(TDest& destination) {
+ return OutputIterator<TDest>{destination};
+ }
+
+ /**
+ * Warning! Use of buffer size argument on make_output_iterator is
+ * deprecated. Call Writer::set_buffer_size() instead if you want to
+ * change the default.
+ */
+ template <class TDest>
+ DEPRECATED OutputIterator<TDest> make_output_iterator(TDest& destination, const size_t buffer_size) {
+ destination.set_buffer_size(buffer_size);
+ return OutputIterator<TDest>{destination};
+ }
+
} // namespace io
} // namespace osmium
+#undef DEPRECATED
+
#endif // OSMIUM_IO_OUTPUT_ITERATOR_HPP
diff --git a/include/osmium/io/overwrite.hpp b/include/osmium/io/overwrite.hpp
index e33894b..0311aed 100644
--- a/include/osmium/io/overwrite.hpp
+++ b/include/osmium/io/overwrite.hpp
@@ -38,13 +38,21 @@ namespace osmium {
namespace io {
/**
- * Allow overwriting of existing file.
+ * Allow overwriting of existing file?
*/
enum class overwrite : bool {
no = false,
allow = true
};
+ /**
+ * Should writer do an fsync before closing the file?
+ */
+ enum class fsync : bool {
+ no = false,
+ yes = true
+ };
+
} // namespace io
} // namespace osmium
diff --git a/include/osmium/io/reader.hpp b/include/osmium/io/reader.hpp
index e093533..54c518e 100644
--- a/include/osmium/io/reader.hpp
+++ b/include/osmium/io/reader.hpp
@@ -33,10 +33,10 @@ DEALINGS IN THE SOFTWARE.
*/
-#include <atomic>
#include <cerrno>
#include <cstdlib>
#include <fcntl.h>
+#include <future>
#include <memory>
#include <string>
#include <system_error>
@@ -55,12 +55,13 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/io/detail/input_format.hpp>
#include <osmium/io/detail/read_thread.hpp>
#include <osmium/io/detail/read_write.hpp>
+#include <osmium/io/detail/queue_util.hpp>
+#include <osmium/io/error.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/header.hpp>
#include <osmium/memory/buffer.hpp>
#include <osmium/osm/entity_bits.hpp>
#include <osmium/thread/util.hpp>
-#include <osmium/thread/queue.hpp>
namespace osmium {
@@ -74,18 +75,46 @@ namespace osmium {
*/
class Reader {
+ static constexpr size_t max_input_queue_size = 20; // XXX
+ static constexpr size_t max_osmdata_queue_size = 20; // XXX
+
osmium::io::File m_file;
- osmium::io::detail::InputFormatFactory::create_input_type* m_input_format_creator;
osmium::osm_entity_bits::type m_read_which_entities;
- std::atomic<bool> m_input_done;
+
+ enum class status {
+ okay = 0, // normal reading
+ error = 1, // some error occurred while reading
+ closed = 2, // close() called successfully after eof
+ eof = 3 // eof of file was reached without error
+ } m_status;
+
int m_childpid;
- osmium::thread::Queue<std::string> m_input_queue;
+ detail::future_string_queue_type m_input_queue;
std::unique_ptr<osmium::io::Decompressor> m_decompressor;
- std::future<bool> m_read_future;
- std::unique_ptr<osmium::io::detail::InputFormat> m_input;
+ osmium::io::detail::ReadThreadManager m_read_thread_manager;
+
+ detail::future_buffer_queue_type m_osmdata_queue;
+ detail::queue_wrapper<osmium::memory::Buffer> m_osmdata_queue_wrapper;
+
+ std::future<osmium::io::Header> m_header_future;
+ osmium::io::Header m_header;
+
+ osmium::thread::thread_handler m_thread;
+
+ // This function will run in a separate thread.
+ static void parser_thread(const osmium::io::File& file,
+ detail::future_string_queue_type& input_queue,
+ detail::future_buffer_queue_type& osmdata_queue,
+ std::promise<osmium::io::Header>&& header_promise,
+ osmium::osm_entity_bits::type read_which_entities) {
+ std::promise<osmium::io::Header> promise = std::move(header_promise);
+ auto creator = detail::ParserFactory::instance().get_creator_function(file);
+ auto parser = creator(input_queue, osmdata_queue, promise, read_which_entities);
+ parser->parse();
+ }
#ifndef _WIN32
/**
@@ -150,7 +179,7 @@ namespace osmium {
#ifndef _WIN32
return execute("curl", filename, childpid);
#else
- throw std::runtime_error("Reading OSM files from the network currently not supported on Windows.");
+ throw io_error("Reading OSM files from the network currently not supported on Windows.");
#endif
} else {
return osmium::io::detail::open_for_reading(filename);
@@ -170,16 +199,22 @@ namespace osmium {
*/
explicit Reader(const osmium::io::File& file, osmium::osm_entity_bits::type read_which_entities = osmium::osm_entity_bits::all) :
m_file(file.check()),
- m_input_format_creator(osmium::io::detail::InputFormatFactory::instance().get_creator_function(m_file)),
m_read_which_entities(read_which_entities),
- m_input_done(false),
+ m_status(status::okay),
m_childpid(0),
- m_input_queue(20, "raw_input"), // XXX
+ m_input_queue(max_input_queue_size, "raw_input"),
m_decompressor(m_file.buffer() ?
osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), m_file.buffer(), m_file.buffer_size()) :
osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), open_input_file_or_url(m_file.filename(), &m_childpid))),
- m_read_future(std::async(std::launch::async, detail::ReadThread(m_input_queue, m_decompressor.get(), m_input_done))),
- m_input((*m_input_format_creator)(m_file, m_read_which_entities, m_input_queue)) {
+ m_read_thread_manager(*m_decompressor.get(), m_input_queue),
+ m_osmdata_queue(max_osmdata_queue_size, "parser_results"),
+ m_osmdata_queue_wrapper(m_osmdata_queue),
+ m_header_future(),
+ m_header(),
+ m_thread() {
+ std::promise<osmium::io::Header> header_promise;
+ m_header_future = header_promise.get_future();
+ m_thread = osmium::thread::thread_handler{parser_thread, std::ref(m_file), std::ref(m_input_queue), std::ref(m_osmdata_queue), std::move(header_promise), read_which_entities};
}
explicit Reader(const std::string& filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) :
@@ -193,27 +228,37 @@ namespace osmium {
Reader(const Reader&) = delete;
Reader& operator=(const Reader&) = delete;
- ~Reader() {
+ Reader(Reader&&) = default;
+ Reader& operator=(Reader&&) = default;
+
+ ~Reader() noexcept {
try {
close();
- }
- catch (...) {
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
}
}
/**
* Close down the Reader. A call to this is optional, because the
* destructor of Reader will also call this. But if you don't call
- * this function first, the destructor might throw an exception
- * which is not good.
+ * this function first, you might miss an exception, because the
+ * destructor is not allowed to throw.
*
- * @throws Some form of std::runtime_error when there is a problem.
+ * @throws Some form of osmium::io_error when there is a problem.
*/
void close() {
- // Signal to input child process that it should wrap up.
- m_input_done = true;
+ m_status = status::closed;
- m_input->close();
+ m_read_thread_manager.stop();
+
+ m_osmdata_queue_wrapper.drain();
+
+ try {
+ m_read_thread_manager.close();
+ } catch (...) {
+ // Ignore any exceptions.
+ }
#ifndef _WIN32
if (m_childpid) {
@@ -228,15 +273,32 @@ namespace osmium {
m_childpid = 0;
}
#endif
-
- osmium::thread::wait_until_done(m_read_future);
}
/**
* Get the header data from the file.
+ *
+ * @returns Header.
+ * @throws Some form of osmium::io_error if there is an error.
*/
- osmium::io::Header header() const {
- return m_input->header();
+ osmium::io::Header header() {
+ if (m_status == status::error) {
+ throw io_error("Can not get header from reader when in status 'error'");
+ }
+
+ try {
+ if (m_header_future.valid()) {
+ m_header = m_header_future.get();
+ if (m_read_which_entities == osmium::osm_entity_bits::nothing) {
+ m_status = status::eof;
+ }
+ }
+ } catch (...) {
+ close();
+ m_status = status::error;
+ throw;
+ }
+ return m_header;
}
/**
@@ -247,32 +309,36 @@ namespace osmium {
* constructed.
*
* @returns Buffer.
- * @throws Some form of std::runtime_error if there is an error.
+ * @throws Some form of osmium::io_error if there is an error.
*/
osmium::memory::Buffer read() {
- // If an exception happened in the input thread, re-throw
- // it in this (the main) thread.
- osmium::thread::check_for_exception(m_read_future);
-
- if (m_read_which_entities == osmium::osm_entity_bits::nothing || m_input_done) {
- // If the caller didn't want anything but the header, it will
- // always get an empty buffer here.
- return osmium::memory::Buffer();
+ osmium::memory::Buffer buffer;
+
+ if (m_status != status::okay ||
+ m_read_which_entities == osmium::osm_entity_bits::nothing) {
+ throw io_error("Can not read from reader when in status 'closed', 'eof', or 'error'");
}
- // m_input->read() can return an invalid buffer to signal EOF,
- // or a valid buffer with or without data. A valid buffer
- // without data is not an error, it just means we have to
- // keep getting the next buffer until there is one with data.
- while (true) {
- osmium::memory::Buffer buffer = m_input->read();
- if (!buffer) {
- m_input_done = true;
- return buffer;
- }
- if (buffer.committed() > 0) {
- return buffer;
+ try {
+ // m_input_format.read() can return an invalid buffer to signal EOF,
+ // or a valid buffer with or without data. A valid buffer
+ // without data is not an error, it just means we have to
+ // keep getting the next buffer until there is one with data.
+ while (true) {
+ buffer = m_osmdata_queue_wrapper.pop();
+ if (detail::at_end_of_data(buffer)) {
+ m_status = status::eof;
+ m_read_thread_manager.close();
+ return buffer;
+ }
+ if (buffer.committed() > 0) {
+ return buffer;
+ }
}
+ } catch (...) {
+ close();
+ m_status = status::error;
+ throw;
}
}
@@ -281,7 +347,7 @@ namespace osmium {
* data has been read. It is also set by calling close().
*/
bool eof() const {
- return m_input_done;
+ return m_status == status::eof || m_status == status::closed;
}
}; // class Reader
@@ -294,7 +360,7 @@ namespace osmium {
* unless you are working with small OSM files and/or have lots of
* RAM.
*/
- template <class... TArgs>
+ template <typename... TArgs>
osmium::memory::Buffer read_file(TArgs&&... args) {
osmium::memory::Buffer buffer(1024*1024, osmium::memory::Buffer::auto_grow::yes);
diff --git a/include/osmium/io/writer.hpp b/include/osmium/io/writer.hpp
index 018dd11..4b8823e 100644
--- a/include/osmium/io/writer.hpp
+++ b/include/osmium/io/writer.hpp
@@ -33,14 +33,20 @@ DEALINGS IN THE SOFTWARE.
*/
+#include <cassert>
+#include <future>
#include <memory>
+#include <stdexcept>
#include <string>
+#include <thread>
#include <utility>
#include <osmium/io/compression.hpp>
#include <osmium/io/detail/output_format.hpp>
+#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/detail/read_write.hpp>
#include <osmium/io/detail/write_thread.hpp>
+#include <osmium/io/error.hpp>
#include <osmium/io/file.hpp>
#include <osmium/io/header.hpp>
#include <osmium/io/overwrite.hpp>
@@ -54,27 +60,120 @@ namespace osmium {
/**
* This is the user-facing interface for writing OSM files. Instantiate
* an object of this class with a file name or osmium::io::File object
- * and optionally the data for the header and then call operator() on it
- * to write Buffers. Call close() to finish up.
+ * and optionally the data for the header and then call operator() on
+ * it to write Buffers or Items.
+ *
+ * The writer uses multithreading internally to do the actual encoding
+ * of the data into the intended format, possible compress the data and
+ * then write it out. But this is intentionally hidden from the user
+ * of this class who can use it without knowing those details.
+ *
+ * If you are done call the close() method to finish up. Only if you
+ * don't get an exception from the close() method, you can be sure
+ * the data is written correctly (modulo operating system buffering).
+ * The destructor of this class will also do the right thing if you
+ * forget to call close(), but because the destructor can't throw you
+ * will not get informed about any problems.
+ *
+ * The writer is usually used to write complete blocks of data stored
+ * in osmium::memory::Buffers. But you can also write single
+ * osmium::memory::Items. In this case the Writer uses an internal
+ * Buffer.
*/
class Writer {
+ static constexpr size_t default_buffer_size = 10 * 1024 * 1024;
+
osmium::io::File m_file;
- osmium::io::detail::data_queue_type m_output_queue;
+ detail::future_string_queue_type m_output_queue;
std::unique_ptr<osmium::io::detail::OutputFormat> m_output;
- std::unique_ptr<osmium::io::Compressor> m_compressor;
+ osmium::memory::Buffer m_buffer;
+
+ size_t m_buffer_size;
std::future<bool> m_write_future;
+ osmium::thread::thread_handler m_thread;
+
+ enum class status {
+ okay = 0, // normal writing
+ error = 1, // some error occurred while writing
+ closed = 2 // close() called successfully
+ } m_status;
+
+ // This function will run in a separate thread.
+ static void write_thread(detail::future_string_queue_type& output_queue,
+ std::unique_ptr<osmium::io::Compressor>&& compressor,
+ std::promise<bool>&& write_promise) {
+ detail::WriteThread write_thread{output_queue,
+ std::move(compressor),
+ std::move(write_promise)};
+ write_thread();
+ }
+
+ void do_write(osmium::memory::Buffer&& buffer) {
+ if (buffer && buffer.committed() > 0) {
+ m_output->write_buffer(std::move(buffer));
+ }
+ }
+
+ void do_flush() {
+ if (m_buffer && m_buffer.committed() > 0) {
+ osmium::memory::Buffer buffer{m_buffer_size,
+ osmium::memory::Buffer::auto_grow::no};
+ using std::swap;
+ swap(m_buffer, buffer);
+
+ m_output->write_buffer(std::move(buffer));
+ }
+ }
+
+ template <typename TFunction, typename ...TArgs>
+ void ensure_cleanup(TFunction func, TArgs&&... args) {
+ if (m_status != status::okay) {
+ throw io_error("Can not write to writer when in status 'closed' or 'error'");
+ }
+
+ try {
+ osmium::thread::check_for_exception(m_write_future);
+ func(std::forward<TArgs>(args)...);
+ } catch (...) {
+ m_status = status::error;
+ detail::add_to_queue(m_output_queue, std::current_exception());
+ detail::add_end_of_data_to_queue(m_output_queue);
+ throw;
+ }
+ }
+
+ struct options_type {
+ osmium::io::Header header;
+ overwrite allow_overwrite = overwrite::no;
+ fsync sync = fsync::no;
+ };
+
+ static void set_option(options_type& options, const osmium::io::Header& header) {
+ options.header = header;
+ }
+
+ static void set_option(options_type& options, overwrite value) {
+ options.allow_overwrite = value;
+ }
+
+ static void set_option(options_type& options, fsync value) {
+ options.sync = value;
+ }
+
public:
/**
* The constructor of the Writer object opens a file and writes the
* header to it.
*
+ * All parameters except the first one can be in any order.
+ *
* @param file File (contains name and format info) to open.
* @param header Optional header data. If this is not given sensible
* defaults will be used. See the default constructor
@@ -82,58 +181,155 @@ namespace osmium {
* @param allow_overwrite Allow overwriting of existing file? Can be
* osmium::io::overwrite::allow or osmium::io::overwrite::no
* (default).
+ * @param fsync Should fsync be called on the file before closing it?
+ * Can be osmium::io::fsync::yes or
+ * osmium::io::fsync::no (default).
*
- * @throws std::runtime_error If the file could not be opened.
+ * @throws osmium::io_error If there was an error.
* @throws std::system_error If the file could not be opened.
*/
- explicit Writer(const osmium::io::File& file, const osmium::io::Header& header = osmium::io::Header(), overwrite allow_overwrite = overwrite::no) :
+ template <typename... TArgs>
+ explicit Writer(const osmium::io::File& file, TArgs&&... args) :
m_file(file.check()),
m_output_queue(20, "raw_output"), // XXX
m_output(osmium::io::detail::OutputFormatFactory::instance().create_output(m_file, m_output_queue)),
- m_compressor(osmium::io::CompressionFactory::instance().create_compressor(file.compression(), osmium::io::detail::open_for_writing(m_file.filename(), allow_overwrite))),
- m_write_future(std::async(std::launch::async, detail::WriteThread(m_output_queue, m_compressor.get()))) {
- assert(!m_file.buffer());
- m_output->write_header(header);
+ m_buffer(),
+ m_buffer_size(default_buffer_size),
+ m_write_future(),
+ m_thread(),
+ m_status(status::okay) {
+ assert(!m_file.buffer()); // XXX can't handle pseudo-files
+
+ options_type options;
+ (void)std::initializer_list<int>{
+ (set_option(options, args), 0)...
+ };
+
+ std::unique_ptr<osmium::io::Compressor> compressor =
+ CompressionFactory::instance().create_compressor(file.compression(),
+ osmium::io::detail::open_for_writing(m_file.filename(), options.allow_overwrite),
+ options.sync);
+
+ std::promise<bool> write_promise;
+ m_write_future = write_promise.get_future();
+ m_thread = osmium::thread::thread_handler{write_thread, std::ref(m_output_queue), std::move(compressor), std::move(write_promise)};
+
+ ensure_cleanup([&](){
+ m_output->write_header(options.header);
+ });
}
- explicit Writer(const std::string& filename, const osmium::io::Header& header = osmium::io::Header(), overwrite allow_overwrite = overwrite::no) :
- Writer(osmium::io::File(filename), header, allow_overwrite) {
+ template <typename... TArgs>
+ explicit Writer(const std::string& filename, TArgs&&... args) :
+ Writer(osmium::io::File(filename), std::forward<TArgs>(args)...) {
}
- explicit Writer(const char* filename, const osmium::io::Header& header = osmium::io::Header(), overwrite allow_overwrite = overwrite::no) :
- Writer(osmium::io::File(filename), header, allow_overwrite) {
+ template <typename... TArgs>
+ explicit Writer(const char* filename, TArgs&&... args) :
+ Writer(osmium::io::File(filename), std::forward<TArgs>(args)...) {
}
Writer(const Writer&) = delete;
Writer& operator=(const Writer&) = delete;
- ~Writer() {
- close();
+ Writer(Writer&&) = default;
+ Writer& operator=(Writer&&) = default;
+
+ ~Writer() noexcept {
+ try {
+ close();
+ } catch (...) {
+ // Ignore any exceptions because destructor must not throw.
+ }
+ }
+
+ /**
+ * Get the currently configured size of the internal buffer.
+ */
+ size_t buffer_size() const noexcept {
+ return m_buffer_size;
+ }
+
+ /**
+ * Set the size of the internal buffer. This will only take effect
+ * if you have not yet written anything or after the next flush().
+ */
+ void set_buffer_size(size_t size) noexcept {
+ m_buffer_size = size;
+ }
+
+ /**
+ * Flush the internal buffer if it contains any data. This is
+ * usually not needed as the buffer gets flushed on close()
+ * automatically.
+ *
+ * @throws Some form of osmium::io_error when there is a problem.
+ */
+ void flush() {
+ ensure_cleanup([&](){
+ do_flush();
+ });
}
/**
- * Write contents of a buffer to the output file.
+ * Write contents of a buffer to the output file. The buffer is
+ * moved into this function and will be in an undefined moved-from
+ * state afterwards.
*
- * @throws Some form of std::runtime_error when there is a problem.
+ * @param buffer Buffer that is being written out.
+ * @throws Some form of osmium::io_error when there is a problem.
*/
void operator()(osmium::memory::Buffer&& buffer) {
- osmium::thread::check_for_exception(m_write_future);
- if (buffer.committed() > 0) {
- m_output->write_buffer(std::move(buffer));
- }
+ ensure_cleanup([&](){
+ do_flush();
+ do_write(std::move(buffer));
+ });
+ }
+
+ /**
+ * Add item to the internal buffer for eventual writing to the
+ * output file.
+ *
+ * @param item Item to write (usually an OSM object).
+ * @throws Some form of osmium::io_error when there is a problem.
+ */
+ void operator()(const osmium::memory::Item& item) {
+ ensure_cleanup([&](){
+ if (!m_buffer) {
+ m_buffer = osmium::memory::Buffer{m_buffer_size,
+ osmium::memory::Buffer::auto_grow::no};
+ }
+ try {
+ m_buffer.push_back(item);
+ } catch (osmium::buffer_is_full&) {
+ do_flush();
+ m_buffer.push_back(item);
+ }
+ });
}
/**
- * Flush writes to output file and closes it. If you do not
+ * Flushes internal buffer and closes output file. If you do not
* call this, the destructor of Writer will also do the same
- * thing. But because this call might thrown an exception,
- * it is better to call close() explicitly.
+ * thing. But because this call might throw an exception, which
+ * the destructor will ignore, it is better to call close()
+ * explicitly.
*
- * @throws Some form of std::runtime_error when there is a problem.
+ * @throws Some form of osmium::io_error when there is a problem.
*/
void close() {
- m_output->close();
- osmium::thread::wait_until_done(m_write_future);
+ if (m_status == status::okay) {
+ ensure_cleanup([&](){
+ do_write(std::move(m_buffer));
+ m_output->write_end();
+ m_status = status::closed;
+ detail::add_end_of_data_to_queue(m_output_queue);
+ });
+ }
+
+ if (m_write_future.valid()) {
+ m_write_future.get();
+ }
}
}; // class Writer
diff --git a/include/osmium/memory/buffer.hpp b/include/osmium/memory/buffer.hpp
index d800c68..949f005 100644
--- a/include/osmium/memory/buffer.hpp
+++ b/include/osmium/memory/buffer.hpp
@@ -117,7 +117,7 @@ namespace osmium {
/**
* The constructor without any parameters creates a non-initialized
* buffer, ie an empty hull of a buffer that has no actual memory
- * associated with it. It can be used to signify end-of-input.
+ * associated with it. It can be used to signify end-of-data.
*/
Buffer() noexcept :
m_memory(),
@@ -201,11 +201,13 @@ namespace osmium {
* Return a pointer to data inside the buffer.
*/
unsigned char* data() const noexcept {
+ assert(m_data);
return m_data;
}
/**
* Returns the capacity of the buffer, ie how many bytes it can contain.
+ * Always returns 0 on invalid buffers.
*/
size_t capacity() const noexcept {
return m_capacity;
@@ -213,6 +215,7 @@ namespace osmium {
/**
* Returns the number of bytes already filled in this buffer.
+ * Always returns 0 on invalid buffers.
*/
size_t committed() const noexcept {
return m_committed;
@@ -221,6 +224,7 @@ namespace osmium {
/**
* Returns the number of bytes currently filled in this buffer that
* are not yet committed.
+ * Always returns 0 on invalid buffers.
*/
size_t written() const noexcept {
return m_written;
@@ -229,16 +233,24 @@ namespace osmium {
/**
* This tests if the current state of the buffer is aligned
* properly. Can be used for asserts.
+ *
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
*/
bool is_aligned() const noexcept {
+ assert(m_data);
return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
}
/**
* Set functor to be called whenever the buffer is full
* instead of throwing buffer_is_full.
+ *
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
*/
void set_full_callback(std::function<void(Buffer&)> full) {
+ assert(m_data);
m_full = full;
}
@@ -248,9 +260,13 @@ namespace osmium {
* If the given size is not larger than the current capacity, nothing is done.
* Already written but not committed data is discarded.
*
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
+ *
* @param size New capacity.
*/
void grow(size_t size) {
+ assert(m_data);
if (m_memory.empty()) {
throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
}
@@ -267,9 +283,13 @@ namespace osmium {
/**
* Mark currently written bytes in the buffer as committed.
*
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
+ *
* @returns Last number of committed bytes before this commit.
*/
size_t commit() {
+ assert(m_data);
assert(is_aligned());
const size_t offset = m_committed;
@@ -279,14 +299,20 @@ namespace osmium {
/**
* Roll back changes in buffer to last committed state.
+ *
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
*/
void rollback() {
+ assert(m_data);
m_written = m_committed;
}
/**
* Clear the buffer.
*
+ * No-op on an invalid buffer.
+ *
* @returns Number of bytes in the buffer before it was cleared.
*/
size_t clear() {
@@ -299,11 +325,15 @@ namespace osmium {
/**
* Get the data in the buffer at the given offset.
*
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
+ *
* @tparam T Type we want to the data to be interpreted as.
* @returns Reference of given type pointing to the data in the buffer.
*/
template <class T>
T& get(const size_t offset) const {
+ assert(m_data);
return *reinterpret_cast<T*>(&m_data[offset]);
}
@@ -326,6 +356,9 @@ namespace osmium {
* the new data will fit.
* * Else the buffer_is_full exception is thrown.
*
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
+ *
* @param size Number of bytes to reserve.
* @returns Pointer to reserved space. Note that this pointer is
* only guaranteed to be valid until the next call to
@@ -333,6 +366,7 @@ namespace osmium {
* @throws osmium::buffer_is_full Might be thrown if the buffer is full.
*/
unsigned char* reserve_space(const size_t size) {
+ assert(m_data);
if (m_written + size > m_capacity) {
if (m_full) {
m_full(*this);
@@ -359,12 +393,16 @@ namespace osmium {
* Note that you have to eventually call commit() to actually
* commit this data.
*
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
+ *
* @tparam T Class of the item to be copied.
* @param item Reference to the item to be copied.
* @returns Reference to newly copied data in the buffer.
*/
template <class T>
T& add_item(const T& item) {
+ assert(m_data);
unsigned char* target = reserve_space(item.padded_size());
std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
return *reinterpret_cast<T*>(target);
@@ -373,10 +411,14 @@ namespace osmium {
/**
* Add committed contents of the given buffer to this buffer.
*
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
+ *
* Note that you have to eventually call commit() to actually
* commit this data.
*/
void add_buffer(const Buffer& buffer) {
+ assert(m_data);
unsigned char* target = reserve_space(buffer.committed());
std::copy_n(reinterpret_cast<const unsigned char*>(buffer.data()), buffer.committed(), target);
}
@@ -384,8 +426,12 @@ namespace osmium {
/**
* Add an item to the buffer. This function is provided so that
* you can use std::back_inserter.
+ *
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
*/
void push_back(const osmium::memory::Item& item) {
+ assert(m_data);
add_item(item);
commit();
}
@@ -405,55 +451,67 @@ namespace osmium {
template <class T>
t_iterator<T> begin() {
+ assert(m_data);
return t_iterator<T>(m_data, m_data + m_committed);
}
iterator begin() {
+ assert(m_data);
return iterator(m_data, m_data + m_committed);
}
template <class T>
t_iterator<T> get_iterator(size_t offset) {
+ assert(m_data);
return t_iterator<T>(m_data + offset, m_data + m_committed);
}
iterator get_iterator(size_t offset) {
+ assert(m_data);
return iterator(m_data + offset, m_data + m_committed);
}
template <class T>
t_iterator<T> end() {
+ assert(m_data);
return t_iterator<T>(m_data + m_committed, m_data + m_committed);
}
iterator end() {
+ assert(m_data);
return iterator(m_data + m_committed, m_data + m_committed);
}
template <class T>
t_const_iterator<T> cbegin() const {
+ assert(m_data);
return t_const_iterator<T>(m_data, m_data + m_committed);
}
const_iterator cbegin() const {
+ assert(m_data);
return const_iterator(m_data, m_data + m_committed);
}
template <class T>
t_const_iterator<T> get_iterator(size_t offset) const {
+ assert(m_data);
return t_const_iterator<T>(m_data + offset, m_data + m_committed);
}
const_iterator get_iterator(size_t offset) const {
+ assert(m_data);
return const_iterator(m_data + offset, m_data + m_committed);
}
template <class T>
t_const_iterator<T> cend() const {
+ assert(m_data);
return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
}
const_iterator cend() const {
+ assert(m_data);
return const_iterator(m_data + m_committed, m_data + m_committed);
}
@@ -505,9 +563,13 @@ namespace osmium {
* the old and new offsets in the buffer where the object used to
* be and is now, respectively. This call can be used to update any
* indexes.
+ *
+ * The behaviour is undefined if you call this on an invalid
+ * buffer.
*/
template <class TCallbackClass>
void purge_removed(TCallbackClass* callback) {
+ assert(m_data);
if (begin() == end()) {
return;
}
@@ -538,6 +600,9 @@ namespace osmium {
}; // class Buffer
inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
+ if (!lhs || !rhs) {
+ return !lhs && !rhs;
+ }
return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
}
diff --git a/include/osmium/osm/area.hpp b/include/osmium/osm/area.hpp
index 3e129d0..4e11c8b 100644
--- a/include/osmium/osm/area.hpp
+++ b/include/osmium/osm/area.hpp
@@ -167,6 +167,7 @@ namespace osmium {
case osmium::item_type::way_node_list:
case osmium::item_type::relation_member_list:
case osmium::item_type::relation_member_list_with_full_members:
+ case osmium::item_type::changeset_discussion:
assert(false && "Children of Area can only be outer/inner_ring and tag_list.");
break;
}
diff --git a/include/osmium/osm/changeset.hpp b/include/osmium/osm/changeset.hpp
index 07bc0dd..c5e8358 100644
--- a/include/osmium/osm/changeset.hpp
+++ b/include/osmium/osm/changeset.hpp
@@ -48,9 +48,103 @@ DEALINGS IN THE SOFTWARE.
namespace osmium {
namespace builder {
+ class ChangesetDiscussionBuilder;
template <class T> class ObjectBuilder;
}
+ class Changeset;
+
+ class ChangesetComment : public osmium::memory::detail::ItemHelper {
+
+ friend class osmium::builder::ChangesetDiscussionBuilder;
+
+ osmium::Timestamp m_date;
+ osmium::user_id_type m_uid {0};
+ string_size_type m_user_size;
+ string_size_type m_text_size;
+
+ ChangesetComment(const ChangesetComment&) = delete;
+ ChangesetComment(ChangesetComment&&) = delete;
+
+ ChangesetComment& operator=(const ChangesetComment&) = delete;
+ ChangesetComment& operator=(ChangesetComment&&) = delete;
+
+ unsigned char* endpos() {
+ return data() + osmium::memory::padded_length(sizeof(ChangesetComment) + m_user_size + m_text_size);
+ }
+
+ const unsigned char* endpos() const {
+ return data() + osmium::memory::padded_length(sizeof(ChangesetComment) + m_user_size + m_text_size);
+ }
+
+ template <class TMember>
+ friend class osmium::memory::CollectionIterator;
+
+ unsigned char* next() {
+ return endpos();
+ }
+
+ unsigned const char* next() const {
+ return endpos();
+ }
+
+ void set_user_size(string_size_type size) noexcept {
+ m_user_size = size;
+ }
+
+ void set_text_size(string_size_type size) noexcept {
+ m_text_size = size;
+ }
+
+ public:
+
+ static constexpr item_type collection_type = item_type::changeset_discussion;
+
+ ChangesetComment(osmium::Timestamp date, osmium::user_id_type uid) noexcept :
+ m_date(date),
+ m_uid(uid),
+ m_user_size(0),
+ m_text_size(0) {
+ }
+
+ osmium::Timestamp date() const noexcept {
+ return m_date;
+ }
+
+ osmium::user_id_type uid() const noexcept {
+ return m_uid;
+ }
+
+ const char* user() const noexcept {
+ return reinterpret_cast<const char*>(data() + sizeof(ChangesetComment));
+ }
+
+ const char* text() const noexcept {
+ return reinterpret_cast<const char*>(data() + sizeof(ChangesetComment) + m_user_size);
+ }
+
+ }; // class ChangesetComment
+
+ class ChangesetDiscussion : public osmium::memory::Collection<ChangesetComment, osmium::item_type::changeset_discussion> {
+
+ friend class osmium::builder::ObjectBuilder<osmium::Changeset>;
+
+ public:
+
+ typedef size_t size_type;
+
+ ChangesetDiscussion() :
+ osmium::memory::Collection<ChangesetComment, osmium::item_type::changeset_discussion>() {
+ }
+
+ size_type size() const noexcept {
+ return static_cast<size_type>(std::distance(begin(), end()));
+ }
+
+ }; // class ChangesetDiscussion
+
+ static_assert(sizeof(ChangesetDiscussion) % osmium::memory::align_bytes == 0, "Class osmium::ChangesetDiscussion has wrong size to be aligned properly!");
+
/**
* \brief An OSM Changeset, a group of changes made by a single user over
* a short period of time.
@@ -62,13 +156,16 @@ namespace osmium {
friend class osmium::builder::ObjectBuilder<osmium::Changeset>;
+ osmium::Box m_bounds;
osmium::Timestamp m_created_at;
osmium::Timestamp m_closed_at;
- osmium::Box m_bounds;
changeset_id_type m_id {0};
num_changes_type m_num_changes {0};
+ num_comments_type m_num_comments {0};
user_id_type m_uid {0};
string_size_type m_user_size;
+ int16_t m_padding1 {0};
+ int32_t m_padding2 {0};
Changeset() :
OSMEntity(sizeof(Changeset), osmium::item_type::changeset) {
@@ -188,7 +285,7 @@ namespace osmium {
* @param timestamp Timestamp
* @returns Reference to changeset to make calls chainable.
*/
- Changeset& set_created_at(const osmium::Timestamp timestamp) {
+ Changeset& set_created_at(const osmium::Timestamp& timestamp) {
m_created_at = timestamp;
return *this;
}
@@ -199,7 +296,7 @@ namespace osmium {
* @param timestamp Timestamp
* @returns Reference to changeset to make calls chainable.
*/
- Changeset& set_closed_at(const osmium::Timestamp timestamp) {
+ Changeset& set_closed_at(const osmium::Timestamp& timestamp) {
m_closed_at = timestamp;
return *this;
}
@@ -216,10 +313,26 @@ namespace osmium {
}
/// Set the number of changes in this changeset
- Changeset& set_num_changes(const char* num_changes) noexcept {
+ Changeset& set_num_changes(const char* num_changes) {
return set_num_changes(osmium::string_to_num_changes(num_changes));
}
+ /// Get the number of comments in this changeset
+ num_comments_type num_comments() const noexcept {
+ return m_num_comments;
+ }
+
+ /// Set the number of comments in this changeset
+ Changeset& set_num_comments(num_comments_type num_comments) noexcept {
+ m_num_comments = num_comments;
+ return *this;
+ }
+
+ /// Set the number of comments in this changeset
+ Changeset& set_num_comments(const char* num_comments) {
+ return set_num_comments(osmium::string_to_num_comments(num_comments));
+ }
+
/**
* Get the bounding box of this changeset.
*
@@ -260,6 +373,8 @@ namespace osmium {
set_id(value);
} else if (!strcmp(attr, "num_changes")) {
set_num_changes(value);
+ } else if (!strcmp(attr, "comments_count")) {
+ set_num_comments(value);
} else if (!strcmp(attr, "created_at")) {
set_created_at(osmium::Timestamp(value));
} else if (!strcmp(attr, "closed_at")) {
@@ -296,6 +411,14 @@ namespace osmium {
return cend();
}
+ ChangesetDiscussion& discussion() {
+ return osmium::detail::subitem_of_type<ChangesetDiscussion>(begin(), end());
+ }
+
+ const ChangesetDiscussion& discussion() const {
+ return osmium::detail::subitem_of_type<const ChangesetDiscussion>(cbegin(), cend());
+ }
+
}; // class Changeset
static_assert(sizeof(Changeset) % osmium::memory::align_bytes == 0, "Class osmium::Changeset has wrong size to be aligned properly!");
diff --git a/include/osmium/osm/crc.hpp b/include/osmium/osm/crc.hpp
index fc87994..cf81476 100644
--- a/include/osmium/osm/crc.hpp
+++ b/include/osmium/osm/crc.hpp
@@ -213,15 +213,26 @@ namespace osmium {
}
}
+ void update(const osmium::ChangesetDiscussion& discussion) {
+ for (const auto& comment : discussion) {
+ update(comment.date());
+ update_int32(comment.uid());
+ update_string(comment.user());
+ update_string(comment.text());
+ }
+ }
+
void update(const osmium::Changeset& changeset) {
update_int64(changeset.id());
update(changeset.created_at());
update(changeset.closed_at());
update(changeset.bounds());
update_int32(changeset.num_changes());
+ update_int32(changeset.num_comments());
update_int32(changeset.uid());
update_string(changeset.user());
update(changeset.tags());
+ update(changeset.discussion());
}
}; // class CRC
diff --git a/include/osmium/osm/item_type.hpp b/include/osmium/osm/item_type.hpp
index 54975e3..d8f5296 100644
--- a/include/osmium/osm/item_type.hpp
+++ b/include/osmium/osm/item_type.hpp
@@ -53,7 +53,8 @@ namespace osmium {
relation_member_list = 0x13,
relation_member_list_with_full_members = 0x23,
outer_ring = 0x40,
- inner_ring = 0x41
+ inner_ring = 0x41,
+ changeset_discussion = 0x80
}; // enum class item_type
@@ -102,6 +103,8 @@ namespace osmium {
return item_type::outer_ring;
case 'I':
return item_type::inner_ring;
+ case 'D':
+ return item_type::changeset_discussion;
default:
return item_type::undefined;
}
@@ -136,6 +139,8 @@ namespace osmium {
return 'O';
case item_type::inner_ring:
return 'I';
+ case item_type::changeset_discussion:
+ return 'D';
}
}
@@ -165,6 +170,8 @@ namespace osmium {
return "outer_ring";
case item_type::inner_ring:
return "inner_ring";
+ case item_type::changeset_discussion:
+ return "changeset_discussion";
}
}
#pragma GCC diagnostic pop
diff --git a/include/osmium/osm/node_ref_list.hpp b/include/osmium/osm/node_ref_list.hpp
index f0dfedb..c6c4213 100644
--- a/include/osmium/osm/node_ref_list.hpp
+++ b/include/osmium/osm/node_ref_list.hpp
@@ -66,7 +66,7 @@ namespace osmium {
* Returns the number of nodes in the list.
*/
size_t size() const noexcept {
- auto size_node_refs = osmium::memory::Item::byte_size() - sizeof(NodeRefList);
+ auto size_node_refs = byte_size() - sizeof(NodeRefList);
assert(size_node_refs % sizeof(NodeRef) == 0);
return size_node_refs / sizeof(NodeRef);
}
diff --git a/include/osmium/osm/object.hpp b/include/osmium/osm/object.hpp
index 8c745ce..039e741 100644
--- a/include/osmium/osm/object.hpp
+++ b/include/osmium/osm/object.hpp
@@ -281,7 +281,7 @@ namespace osmium {
* @param timestamp Timestamp
* @returns Reference to object to make calls chainable.
*/
- OSMObject& set_timestamp(const osmium::Timestamp timestamp) noexcept {
+ OSMObject& set_timestamp(const osmium::Timestamp& timestamp) noexcept {
m_timestamp = timestamp;
return *this;
}
diff --git a/include/osmium/osm/timestamp.hpp b/include/osmium/osm/timestamp.hpp
index 390f0e7..2145fcb 100644
--- a/include/osmium/osm/timestamp.hpp
+++ b/include/osmium/osm/timestamp.hpp
@@ -78,6 +78,14 @@ namespace osmium {
}
/**
+ * Returns true if this timestamp is valid (ie set to something other
+ * than 0).
+ */
+ bool valid() const noexcept {
+ return m_timestamp != 0;
+ }
+
+ /**
* Construct timestamp from ISO date/time string.
* Throws std::invalid_argument, if the timestamp can not be parsed.
*/
diff --git a/include/osmium/osm/types.hpp b/include/osmium/osm/types.hpp
index c9ab423..e4250d9 100644
--- a/include/osmium/osm/types.hpp
+++ b/include/osmium/osm/types.hpp
@@ -49,6 +49,7 @@ namespace osmium {
typedef uint32_t user_id_type; ///< Type for OSM user IDs.
typedef int32_t signed_user_id_type; ///< Type for signed OSM user IDs.
typedef uint32_t num_changes_type; ///< Type for changeset num_changes.
+ typedef uint32_t num_comments_type; ///< Type for changeset num_comments.
/**
* Size for strings in OSM data such as user names, tag keys, roles, etc.
diff --git a/include/osmium/osm/types_from_string.hpp b/include/osmium/osm/types_from_string.hpp
index b8de14c..b0e22a7 100644
--- a/include/osmium/osm/types_from_string.hpp
+++ b/include/osmium/osm/types_from_string.hpp
@@ -43,6 +43,7 @@ DEALINGS IN THE SOFTWARE.
#include <osmium/osm/entity_bits.hpp>
#include <osmium/osm/types.hpp>
+#include <osmium/util/cast.hpp>
namespace osmium {
@@ -75,7 +76,7 @@ namespace osmium {
namespace detail {
- inline long string_to_ulong(const char* input, const char *name) {
+ inline unsigned long string_to_ulong(const char* input, const char *name) {
if (*input != '\0' && *input != '-' && !std::isspace(*input)) {
char* end;
auto value = std::strtoul(input, &end, 10);
@@ -90,12 +91,12 @@ namespace osmium {
inline object_version_type string_to_object_version(const char* input) {
assert(input);
- return static_cast<object_version_type>(detail::string_to_ulong(input, "version"));
+ return static_cast_with_assert<object_version_type>(detail::string_to_ulong(input, "version"));
}
inline changeset_id_type string_to_changeset_id(const char* input) {
assert(input);
- return static_cast<changeset_id_type>(detail::string_to_ulong(input, "changeset"));
+ return static_cast_with_assert<changeset_id_type>(detail::string_to_ulong(input, "changeset"));
}
inline signed_user_id_type string_to_user_id(const char* input) {
@@ -103,12 +104,17 @@ namespace osmium {
if (input[0] == '-' && input[1] == '1' && input[2] == '\0') {
return -1;
}
- return static_cast<signed_user_id_type>(detail::string_to_ulong(input, "user id"));
+ return static_cast_with_assert<signed_user_id_type>(detail::string_to_ulong(input, "user id"));
}
inline num_changes_type string_to_num_changes(const char* input) {
assert(input);
- return static_cast<num_changes_type>(detail::string_to_ulong(input, "value for num changes"));
+ return static_cast_with_assert<num_changes_type>(detail::string_to_ulong(input, "value for num changes"));
+ }
+
+ inline num_comments_type string_to_num_comments(const char* input) {
+ assert(input);
+ return static_cast_with_assert<num_comments_type>(detail::string_to_ulong(input, "value for num comments"));
}
} // namespace osmium
diff --git a/include/osmium/relations/collector.hpp b/include/osmium/relations/collector.hpp
index 40e3773..040e392 100644
--- a/include/osmium/relations/collector.hpp
+++ b/include/osmium/relations/collector.hpp
@@ -39,7 +39,7 @@ DEALINGS IN THE SOFTWARE.
#include <cstdint>
#include <functional>
#include <iomanip>
-#include <iostream>
+//#include <iostream>
#include <vector>
#include <osmium/osm/item_type.hpp>
@@ -506,13 +506,15 @@ namespace osmium {
void possibly_purge_removed_members() {
++m_count_complete;
if (m_count_complete > 10000) { // XXX
- const size_t size_before = m_members_buffer.committed();
+// const size_t size_before = m_members_buffer.committed();
m_members_buffer.purge_removed(this);
+/*
const size_t size_after = m_members_buffer.committed();
double percent = static_cast<double>(size_before - size_after);
percent /= size_before;
percent *= 100;
-// std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
+ std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
+*/
m_count_complete = 0;
}
}
diff --git a/include/osmium/thread/function_wrapper.hpp b/include/osmium/thread/function_wrapper.hpp
index fe0a492..95b85d6 100644
--- a/include/osmium/thread/function_wrapper.hpp
+++ b/include/osmium/thread/function_wrapper.hpp
@@ -50,7 +50,9 @@ namespace osmium {
struct impl_base {
virtual ~impl_base() = default;
- virtual void call() = 0;
+ virtual bool call() {
+ return true;
+ }
}; // struct impl_base
@@ -58,28 +60,38 @@ namespace osmium {
template <typename F>
struct impl_type : impl_base {
+
F m_functor;
impl_type(F&& functor) :
- m_functor(std::move(functor)) {
+ m_functor(std::forward<F>(functor)) {
}
- void call() override {
+ bool call() override {
m_functor();
+ return false;
}
+
}; // struct impl_type
public:
// Constructor must not be "explicit" for wrapper
// to work seemlessly.
- template <typename F>
- function_wrapper(F&& f) :
- impl(new impl_type<F>(std::move(f))) {
+ template <typename TFunction>
+ function_wrapper(TFunction&& f) :
+ impl(new impl_type<TFunction>(std::forward<TFunction>(f))) {
+ }
+
+ // The integer parameter is only used to signal that we want
+ // the special function wrapper that makes the worker thread
+ // shut down.
+ function_wrapper(int) :
+ impl(new impl_base()) {
}
- void operator()() {
- impl->call();
+ bool operator()() {
+ return impl->call();
}
function_wrapper() = default;
diff --git a/include/osmium/thread/pool.hpp b/include/osmium/thread/pool.hpp
index 3916031..dd1023b 100644
--- a/include/osmium/thread/pool.hpp
+++ b/include/osmium/thread/pool.hpp
@@ -83,7 +83,6 @@ namespace osmium {
}; // class thread_joiner
- std::atomic<bool> m_done;
osmium::thread::Queue<function_wrapper> m_work_queue;
std::vector<std::thread> m_threads;
thread_joiner m_joiner;
@@ -91,11 +90,15 @@ namespace osmium {
void worker_thread() {
osmium::thread::set_thread_name("_osmium_worker");
- while (!m_done) {
+ while (true) {
function_wrapper task;
m_work_queue.wait_and_pop_with_timeout(task);
if (task) {
- task();
+ if (task()) {
+ // The called tasks returns true only when the
+ // worker thread should shut down.
+ return;
+ }
}
}
}
@@ -113,7 +116,6 @@ namespace osmium {
* In all cases the minimum number of threads in the pool is 1.
*/
explicit Pool(int num_threads, size_t max_queue_size) :
- m_done(false),
m_work_queue(max_queue_size, "work"),
m_threads(),
m_joiner(m_threads),
@@ -132,7 +134,7 @@ namespace osmium {
m_threads.push_back(std::thread(&Pool::worker_thread, this));
}
} catch (...) {
- m_done = true;
+ shutdown_all_workers();
throw;
}
}
@@ -147,8 +149,15 @@ namespace osmium {
return pool;
}
+ void shutdown_all_workers() {
+ for (int i = 0; i < m_num_threads; ++i) {
+ // The special function wrapper makes a worker shut down.
+ m_work_queue.push(function_wrapper{0});
+ }
+ }
+
~Pool() {
- m_done = true;
+ shutdown_all_workers();
m_work_queue.shutdown();
}
diff --git a/include/osmium/thread/util.hpp b/include/osmium/thread/util.hpp
index ca4f6dd..5a1ab16 100644
--- a/include/osmium/thread/util.hpp
+++ b/include/osmium/thread/util.hpp
@@ -71,15 +71,44 @@ namespace osmium {
* Set name of current thread for debugging. This only works on Linux.
*/
#ifdef __linux__
- inline void set_thread_name(const char* name) {
+ inline void set_thread_name(const char* name) noexcept {
prctl(PR_SET_NAME, name, 0, 0, 0);
}
#else
- inline void set_thread_name(const char*) {
+ inline void set_thread_name(const char*) noexcept {
// intentionally left blank
}
#endif
+ class thread_handler {
+
+ std::thread m_thread;
+
+ public:
+
+ thread_handler() :
+ m_thread() {
+ }
+
+ template <typename TFunction, typename... TArgs>
+ thread_handler(TFunction&& f, TArgs&&... args) :
+ m_thread(std::forward<TFunction>(f), std::forward<TArgs>(args)...) {
+ }
+
+ thread_handler(const thread_handler&) = delete;
+ thread_handler& operator=(const thread_handler&) = delete;
+
+ thread_handler(thread_handler&&) = default;
+ thread_handler& operator=(thread_handler&&) = default;
+
+ ~thread_handler() {
+ if (m_thread.joinable()) {
+ m_thread.join();
+ }
+ }
+
+ }; // class thread_handler
+
} // namespace thread
} // namespace osmium
diff --git a/include/osmium/util/data_file.hpp b/include/osmium/util/data_file.hpp
index 22bf191..53bb81c 100644
--- a/include/osmium/util/data_file.hpp
+++ b/include/osmium/util/data_file.hpp
@@ -137,7 +137,7 @@ namespace osmium {
try {
close();
} catch (std::system_error&) {
- // ignore
+ // Ignore any exceptions because destructor must not throw.
}
}
diff --git a/include/osmium/util/delta.hpp b/include/osmium/util/delta.hpp
index f5d6719..cdf3674 100644
--- a/include/osmium/util/delta.hpp
+++ b/include/osmium/util/delta.hpp
@@ -37,6 +37,8 @@ DEALINGS IN THE SOFTWARE.
#include <type_traits>
#include <utility>
+#include <osmium/util/cast.hpp>
+
namespace osmium {
namespace util {
@@ -44,25 +46,39 @@ namespace osmium {
/**
* Helper class for delta encoding.
*/
- template <typename T>
+ template <typename TValue, typename TDelta = int64_t>
class DeltaEncode {
- T m_value;
+ static_assert(std::is_integral<TValue>::value,
+ "DeltaEncode value type must be some integer");
+
+ static_assert(std::is_integral<TDelta>::value && std::is_signed<TDelta>::value,
+ "DeltaEncode delta type must be some signed integer");
+
+ TValue m_value;
public:
- DeltaEncode(T value = 0) :
+ using value_type = TValue;
+ using delta_type = TDelta;
+
+ DeltaEncode(TValue value = 0) :
m_value(value) {
}
- void clear() {
+ void clear() noexcept {
m_value = 0;
}
- T update(T new_value) {
+ TValue value() const noexcept {
+ return m_value;
+ }
+
+ TDelta update(TValue new_value) noexcept {
using std::swap;
swap(m_value, new_value);
- return m_value - new_value;
+ return static_cast_with_assert<TDelta>(m_value) -
+ static_cast_with_assert<TDelta>(new_value);
}
}; // class DeltaEncode
@@ -70,52 +86,63 @@ namespace osmium {
/**
* Helper class for delta decoding.
*/
- template <typename T>
+ template <typename TValue, typename TDelta = int64_t>
class DeltaDecode {
- T m_value;
+ static_assert(std::is_integral<TValue>::value,
+ "DeltaDecode value type must be some integer");
+
+ static_assert(std::is_integral<TDelta>::value && std::is_signed<TDelta>::value,
+ "DeltaDecode delta type must be some signed integer");
+
+ TValue m_value;
public:
+ using value_type = TValue;
+ using delta_type = TDelta;
+
DeltaDecode() :
m_value(0) {
}
- void clear() {
+ void clear() noexcept {
m_value = 0;
}
- T update(T delta) {
- m_value += delta;
+ TValue update(TDelta delta) noexcept {
+ m_value = static_cast_with_assert<TValue>(
+ static_cast_with_assert<TDelta>(m_value) + delta);
return m_value;
}
}; // class DeltaDecode
- template <typename TBaseIterator, typename TTransform, typename TValue>
+ template <typename TBaseIterator, typename TTransform, typename TValue, typename TDelta = int64_t>
class DeltaEncodeIterator : public std::iterator<std::input_iterator_tag, TValue> {
- typedef TValue value_type;
-
TBaseIterator m_it;
TBaseIterator m_end;
TTransform m_trans;
- value_type m_delta;
- DeltaEncode<value_type> m_value;
+ DeltaEncode<TValue, TDelta> m_value;
+ TDelta m_delta;
public:
+ using value_type = TValue;
+ using delta_type = TDelta;
+
DeltaEncodeIterator(TBaseIterator first, TBaseIterator last, TTransform& trans) :
m_it(first),
m_end(last),
m_trans(trans),
- m_delta(m_it != m_end ? m_trans(m_it) : 0),
- m_value(m_delta) {
+ m_value(m_it != m_end ? m_trans(m_it) : 0),
+ m_delta(static_cast_with_assert<TDelta>(m_value.value())) {
}
DeltaEncodeIterator& operator++() {
- if (m_it != m_end) {
- m_delta = m_value.update(m_trans(++m_it));
+ if (++m_it != m_end) {
+ m_delta = m_value.update(m_trans(m_it));
}
return *this;
}
@@ -126,7 +153,7 @@ namespace osmium {
return tmp;
}
- value_type operator*() {
+ TDelta operator*() {
return m_delta;
}
diff --git a/include/osmium/util/file.hpp b/include/osmium/util/file.hpp
index 461f4e6..39e01af 100644
--- a/include/osmium/util/file.hpp
+++ b/include/osmium/util/file.hpp
@@ -52,6 +52,8 @@ DEALINGS IN THE SOFTWARE.
# define ftruncate _chsize_s
#endif
+#include <osmium/util/cast.hpp>
+
namespace osmium {
namespace util {
@@ -92,7 +94,7 @@ namespace osmium {
* @throws std::system_error If ftruncate(2) call failed
*/
inline void resize_file(int fd, size_t new_size) {
- if (::ftruncate(fd, new_size) != 0) {
+ if (::ftruncate(fd, static_cast_with_assert<off_t>(new_size)) != 0) {
throw std::system_error(errno, std::system_category(), "ftruncate failed");
}
}
@@ -108,7 +110,7 @@ namespace osmium {
return si.dwPageSize;
#else
// Unix implementation
- return ::sysconf(_SC_PAGESIZE);
+ return size_t(::sysconf(_SC_PAGESIZE));
#endif
}
diff --git a/include/osmium/util/memory_mapping.hpp b/include/osmium/util/memory_mapping.hpp
index 4bb3641..d5a057d 100644
--- a/include/osmium/util/memory_mapping.hpp
+++ b/include/osmium/util/memory_mapping.hpp
@@ -213,7 +213,7 @@ private:
try {
unmap();
} catch (std::system_error&) {
- // ignore
+ // Ignore any exceptions because destructor must not throw.
}
}
@@ -379,7 +379,7 @@ private:
* Releases the mapping by calling unmap(). Will never throw.
* Call unmap() instead if you want to be notified of any error.
*/
- ~TypedMemoryMapping() = default;
+ ~TypedMemoryMapping() noexcept = default;
/**
* Unmap a mapping. If the mapping is not valid, it will do
diff --git a/include/osmium/util/options.hpp b/include/osmium/util/options.hpp
index fea0752..24c0918 100644
--- a/include/osmium/util/options.hpp
+++ b/include/osmium/util/options.hpp
@@ -112,12 +112,22 @@ namespace osmium {
/**
* Is this option set to a true value ("true" or "yes")?
+ * Will return false if the value is unset.
*/
bool is_true(const std::string& key) const noexcept {
std::string value = get(key);
return (value == "true" || value == "yes");
}
+ /**
+ * Is this option not set to a false value ("false" or "no")?
+ * Will return true if the value is unset.
+ */
+ bool is_not_false(const std::string& key) const noexcept {
+ std::string value = get(key);
+ return !(value == "false" || value == "no");
+ }
+
size_t size() const noexcept {
return m_options.size();
}
diff --git a/include/osmium/visitor.hpp b/include/osmium/visitor.hpp
index 0250f11..a52e8c3 100644
--- a/include/osmium/visitor.hpp
+++ b/include/osmium/visitor.hpp
@@ -98,6 +98,9 @@ namespace osmium {
case osmium::item_type::inner_ring:
handler.inner_ring(static_cast<ConstIfConst<TItem, osmium::InnerRing>&>(item));
break;
+ case osmium::item_type::changeset_discussion:
+ handler.changeset_discussion(static_cast<ConstIfConst<TItem, osmium::ChangesetDiscussion>&>(item));
+ break;
}
}
diff --git a/include/protozero/byteswap.hpp b/include/protozero/byteswap.hpp
index d019c28..29c312a 100644
--- a/include/protozero/byteswap.hpp
+++ b/include/protozero/byteswap.hpp
@@ -10,30 +10,49 @@ documentation.
*****************************************************************************/
+/**
+ * @file byteswap.hpp
+ *
+ * @brief Contains functions to swap bytes in values (for different endianness).
+ */
+
+#include <cstdint>
#include <cassert>
namespace protozero {
+/**
+ * Swap N byte value between endianness formats. This template function must
+ * be specialized to actually work.
+ */
template <int N>
inline void byteswap(const char* /*data*/, char* /*result*/) {
- assert(false);
-}
-
-template <>
-inline void byteswap<1>(const char* data, char* result) {
- result[0] = data[0];
+ static_assert(N == 1, "Can only swap 4 or 8 byte values");
}
+/**
+ * Swap 4 byte value (int32_t, uint32_t, float) between endianness formats.
+ */
template <>
inline void byteswap<4>(const char* data, char* result) {
+# if defined(__GNUC__) || defined(__clang__)
+ *reinterpret_cast<uint32_t*>(result) = __builtin_bswap32(*reinterpret_cast<const uint32_t*>(data));
+# else
result[3] = data[0];
result[2] = data[1];
result[1] = data[2];
result[0] = data[3];
+#endif
}
+/**
+ * Swap 8 byte value (int64_t, uint64_t, double) between endianness formats.
+ */
template <>
inline void byteswap<8>(const char* data, char* result) {
+# if defined(__GNUC__) || defined(__clang__)
+ *reinterpret_cast<uint64_t*>(result) = __builtin_bswap64(*reinterpret_cast<const uint64_t*>(data));
+# else
result[7] = data[0];
result[6] = data[1];
result[5] = data[2];
@@ -42,6 +61,7 @@ inline void byteswap<8>(const char* data, char* result) {
result[2] = data[5];
result[1] = data[6];
result[0] = data[7];
+#endif
}
} // end namespace protozero
diff --git a/include/protozero/pbf_builder.hpp b/include/protozero/pbf_builder.hpp
index d49a7ba..063fa9c 100644
--- a/include/protozero/pbf_builder.hpp
+++ b/include/protozero/pbf_builder.hpp
@@ -10,6 +10,12 @@ documentation.
*****************************************************************************/
+/**
+ * @file pbf_builder.hpp
+ *
+ * @brief Contains the pbf_builder template class.
+ */
+
#include <type_traits>
#include <protozero/pbf_types.hpp>
@@ -17,10 +23,22 @@ documentation.
namespace protozero {
+/**
+ * The pbf_builder is used to write PBF formatted messages into a buffer. It
+ * is based on the pbf_writer class and has all the same methods. The
+ * difference is that whereever the pbf_writer class takes an integer tag,
+ * this template class takes a tag of the template type T.
+ *
+ * Almost all methods in this class can throw an std::bad_alloc exception if
+ * the std::string used as a buffer wants to resize.
+ *
+ * Read the tutorial to understand how this class is used.
+ */
template <typename T>
class pbf_builder : public pbf_writer {
- static_assert(std::is_same<pbf_tag_type, typename std::underlying_type<T>::type>::value, "T must be enum with underlying type protozero::pbf_tag_type");
+ static_assert(std::is_same<pbf_tag_type, typename std::underlying_type<T>::type>::value,
+ "T must be enum with underlying type protozero::pbf_tag_type");
public:
@@ -35,6 +53,7 @@ public:
pbf_writer(parent_writer, pbf_tag_type(tag)) {
}
+/// @cond INTERNAL
#define PROTOZERO_WRITER_WRAP_ADD_SCALAR(name, type) \
inline void add_##name(T tag, type value) { \
pbf_writer::add_##name(pbf_tag_type(tag), value); \
@@ -55,6 +74,9 @@ public:
PROTOZERO_WRITER_WRAP_ADD_SCALAR(float, float)
PROTOZERO_WRITER_WRAP_ADD_SCALAR(double, double)
+#undef PROTOZERO_WRITER_WRAP_ADD_SCALAR
+/// @endcond
+
inline void add_bytes(T tag, const char* value, size_t size) {
pbf_writer::add_bytes(pbf_tag_type(tag), value, size);
}
@@ -83,6 +105,7 @@ public:
pbf_writer::add_message(pbf_tag_type(tag), value);
}
+/// @cond INTERNAL
#define PROTOZERO_WRITER_WRAP_ADD_PACKED(name) \
template <typename InputIterator> \
inline void add_packed_##name(T tag, InputIterator first, InputIterator last) { \
@@ -104,6 +127,9 @@ public:
PROTOZERO_WRITER_WRAP_ADD_PACKED(float)
PROTOZERO_WRITER_WRAP_ADD_PACKED(double)
+#undef PROTOZERO_WRITER_WRAP_ADD_PACKED
+/// @endcond
+
};
} // end namespace protozero
diff --git a/include/protozero/pbf_message.hpp b/include/protozero/pbf_message.hpp
index af29a00..7fef06f 100644
--- a/include/protozero/pbf_message.hpp
+++ b/include/protozero/pbf_message.hpp
@@ -10,6 +10,12 @@ documentation.
*****************************************************************************/
+/**
+ * @file pbf_message.hpp
+ *
+ * @brief Contains the pbf_message class.
+ */
+
#include <type_traits>
#include <protozero/pbf_reader.hpp>
@@ -17,6 +23,44 @@ documentation.
namespace protozero {
+/**
+ * This class represents a protobuf message. Either a top-level message or
+ * a nested sub-message. Top-level messages can be created from any buffer
+ * with a pointer and length:
+ *
+ * @code
+ * enum class Message : protozero::pbf_tag_type {
+ * ...
+ * };
+ *
+ * std::string buffer;
+ * // fill buffer...
+ * pbf_message<Message> message(buffer.data(), buffer.size());
+ * @endcode
+ *
+ * Sub-messages are created using get_message():
+ *
+ * @code
+ * enum class SubMessage : protozero::pbf_tag_type {
+ * ...
+ * };
+ *
+ * pbf_message<Message> message(...);
+ * message.next();
+ * pbf_message<SubMessage> submessage = message.get_message();
+ * @endcode
+ *
+ * All methods of the pbf_message class except get_bytes() and get_string()
+ * provide the strong exception guarantee, ie they either succeed or do not
+ * change the pbf_message object they are called on. Use the get_data() method
+ * instead of get_bytes() or get_string(), if you need this guarantee.
+ *
+ * This template class is based on the pbf_reader class and has all the same
+ * methods. The difference is that whereever the pbf_reader class takes an
+ * integer tag, this template class takes a tag of the template type T.
+ *
+ * Read the tutorial to understand how this class is used.
+ */
template <typename T>
class pbf_message : public pbf_reader {
diff --git a/include/protozero/pbf_reader.hpp b/include/protozero/pbf_reader.hpp
index 1c5ed0d..ac3220c 100644
--- a/include/protozero/pbf_reader.hpp
+++ b/include/protozero/pbf_reader.hpp
@@ -866,8 +866,16 @@ bool pbf_reader::next() {
protozero_assert(((m_tag > 0 && m_tag < 19000) || (m_tag > 19999 && m_tag <= ((1 << 29) - 1))) && "tag out of range");
m_wire_type = pbf_wire_type(value & 0x07);
-// XXX do we want this check? or should it throw an exception?
-// protozero_assert((m_wire_type <=2 || m_wire_type == 5) && "illegal wire type");
+ switch (m_wire_type) {
+ case pbf_wire_type::varint:
+ case pbf_wire_type::fixed64:
+ case pbf_wire_type::length_delimited:
+ case pbf_wire_type::fixed32:
+ break;
+ default:
+ throw unknown_pbf_wire_type_exception();
+ }
+
return true;
}
diff --git a/include/protozero/pbf_writer.hpp b/include/protozero/pbf_writer.hpp
index 53cbfdf..e4e02de 100644
--- a/include/protozero/pbf_writer.hpp
+++ b/include/protozero/pbf_writer.hpp
@@ -229,7 +229,9 @@ public:
*/
inline void add_bool(pbf_tag_type tag, bool value) {
add_field(tag, pbf_wire_type::varint);
- add_fixed<char>(value);
+ protozero_assert(m_pos == 0 && "you can't add fields to a parent pbf_writer if there is an existing pbf_writer for a submessage");
+ protozero_assert(m_data);
+ m_data->append(1, value);
}
/**
diff --git a/include/protozero/version.hpp b/include/protozero/version.hpp
index 098492e..f11d303 100644
--- a/include/protozero/version.hpp
+++ b/include/protozero/version.hpp
@@ -11,12 +11,12 @@ documentation.
*****************************************************************************/
#define PROTOZERO_VERSION_MAJOR 1
-#define PROTOZERO_VERSION_MINOR 1
-#define PROTOZERO_VERSION_PATCH 0
+#define PROTOZERO_VERSION_MINOR 2
+#define PROTOZERO_VERSION_PATCH 2
#define PROTOZERO_VERSION_CODE (PROTOZERO_VERSION_MAJOR * 10000 + PROTOZERO_VERSION_MINOR * 100 + PROTOZERO_VERSION_PATCH)
-#define PROTOZERO_VERSION_STRING "1.1.0"
+#define PROTOZERO_VERSION_STRING "1.2.2"
#endif // PROTOZERO_VERSION_HPP
diff --git a/scripts/travis_install.sh b/scripts/travis_install.sh
deleted file mode 100755
index 119e9fd..0000000
--- a/scripts/travis_install.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/sh
-#
-# travis_install.sh
-#
-
-if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-
- brew install google-sparsehash || true
-
- brew install --without-python boost || true
-
- # workaround for gdal homebrew problem
- brew remove gdal
- brew install gdal
-
-fi
-
-cd ..
-git clone --quiet --depth 1 https://github.com/osmcode/osm-testdata.git
-
diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh
deleted file mode 100755
index d11ac79..0000000
--- a/scripts/travis_script.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh
-#
-# travis_script.sh
-#
-
-mkdir build
-cd build
-
-# GCC ignores the pragmas in the code that disable the "return-type" warning
-# selectively, so use this workaround.
-if [ "${CXX}" = "g++" ]; then
- WORKAROUND="-DCMAKE_CXX_FLAGS=-Wno-return-type"
-else
- WORKAROUND=""
-fi
-
-if [ "${CXX}" = "g++" ]; then
- CXX=g++-4.8
- CC=gcc-4.8
-fi
-
-echo "travis_fold:start:cmake\nRunning cmake..."
-cmake -LA \
- -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
- ${WORKAROUND} \
- ..
-echo "travis_fold:end:cmake"
-
-echo "travis_fold:start:make\nRunning make..."
-make VERBOSE=1
-echo "travis_fold:end:make"
-
-echo "travis_fold:start:ctest\nRunning ctest..."
-ctest --output-on-failure
-echo "travis_fold:end:ctest"
-
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 0047457..b9f02ee 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -106,6 +106,7 @@ add_unit_test(basic test_timestamp)
add_unit_test(basic test_types_from_string)
add_unit_test(basic test_way)
+add_unit_test(buffer test_buffer_basics)
add_unit_test(buffer test_buffer_node)
add_unit_test(buffer test_buffer_purge)
@@ -133,9 +134,15 @@ add_unit_test(index test_id_to_location ENABLE_IF ${SPARSEHASH_FOUND})
add_unit_test(io test_bzip2 ENABLE_IF ${BZIP2_FOUND} LIBS ${BZIP2_LIBRARIES})
add_unit_test(io test_file_formats)
-add_unit_test(io test_reader LIBS "${OSMIUM_XML_LIBRARIES}")
+add_unit_test(io test_reader LIBS "${OSMIUM_XML_LIBRARIES};${OSMIUM_PBF_LIBRARIES}")
+add_unit_test(io test_reader_with_mock_decompression ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES})
+add_unit_test(io test_reader_with_mock_parser ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT})
+add_unit_test(io test_output_utils)
add_unit_test(io test_output_iterator ENABLE_IF ${Threads_FOUND} LIBS ${CMAKE_THREAD_LIBS_INIT})
add_unit_test(io test_string_table)
+add_unit_test(io test_writer ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES})
+add_unit_test(io test_writer_with_mock_compression ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES})
+add_unit_test(io test_writer_with_mock_encoder ENABLE_IF ${Threads_FOUND} LIBS ${OSMIUM_XML_LIBRARIES})
add_unit_test(tags test_filter)
add_unit_test(tags test_operators)
diff --git a/test/data-tests/testdata-xml.cpp b/test/data-tests/testdata-xml.cpp
index 8102759..b5a0e90 100644
--- a/test/data-tests/testdata-xml.cpp
+++ b/test/data-tests/testdata-xml.cpp
@@ -8,6 +8,7 @@
#include <iostream>
#include <string>
+#include <osmium/io/detail/queue_util.hpp>
#include <osmium/io/xml_input.hpp>
#include <osmium/io/gzip_compression.hpp>
#include <osmium/visitor.hpp>
@@ -72,24 +73,27 @@ std::string read_gz_file(const char* test_id, const char* suffix) {
header_buffer_type parse_xml(std::string input) {
- osmium::thread::Queue<std::string> input_queue;
- osmium::thread::Queue<osmium::memory::Buffer> output_queue;
+ osmium::io::detail::future_string_queue_type input_queue;
+ osmium::io::detail::future_buffer_queue_type output_queue;
std::promise<osmium::io::Header> header_promise;
- std::atomic<bool> done {false};
- input_queue.push(input);
- input_queue.push(std::string()); // EOF marker
+ std::future<osmium::io::Header> header_future = header_promise.get_future();
- osmium::io::detail::XMLParser parser(input_queue, output_queue, header_promise, osmium::osm_entity_bits::all, done);
- parser();
+ osmium::io::detail::add_to_queue(input_queue, std::move(input));
+ osmium::io::detail::add_to_queue(input_queue, std::string{});
+
+ osmium::io::detail::XMLParser parser(input_queue, output_queue, header_promise, osmium::osm_entity_bits::all);
+ parser.parse();
header_buffer_type result;
- result.header = header_promise.get_future().get();
- output_queue.wait_and_pop(result.buffer);
+ result.header = header_future.get();
+ std::future<osmium::memory::Buffer> future_buffer;
+ output_queue.wait_and_pop(future_buffer);
+ result.buffer = future_buffer.get();
if (result.buffer) {
- osmium::memory::Buffer buffer;
- output_queue.wait_and_pop(buffer);
- assert(!buffer);
+ std::future<osmium::memory::Buffer> future_buffer2;
+ output_queue.wait_and_pop(future_buffer2);
+ assert(!future_buffer2.get());
}
return result;
@@ -534,9 +538,10 @@ TEST_CASE("Reading OSM XML 200") {
osmium::io::Header header = reader.header();
REQUIRE(header.get("generator") == "testdata");
- osmium::memory::Buffer buffer = reader.read();
- REQUIRE(0 == buffer.committed());
- REQUIRE(! buffer);
+ REQUIRE_THROWS({
+ reader.read();
+ });
+
reader.close();
}
diff --git a/test/t/basic/test_changeset.cpp b/test/t/basic/test_changeset.cpp
index d214698..d1f3fde 100644
--- a/test/t/basic/test_changeset.cpp
+++ b/test/t/basic/test_changeset.cpp
@@ -21,11 +21,13 @@ TEST_CASE("Basic Changeset") {
.set_created_at(100)
.set_closed_at(200)
.set_num_changes(7)
+ .set_num_comments(3)
.set_uid(9);
REQUIRE(42 == cs1.id());
REQUIRE(9 == cs1.uid());
REQUIRE(7 == cs1.num_changes());
+ REQUIRE(3 == cs1.num_comments());
REQUIRE(true == cs1.closed());
REQUIRE(osmium::Timestamp(100) == cs1.created_at());
REQUIRE(osmium::Timestamp(200) == cs1.closed_at());
@@ -33,7 +35,7 @@ TEST_CASE("Basic Changeset") {
REQUIRE(std::string("user") == cs1.user());
crc32.update(cs1);
- REQUIRE(crc32().checksum() == 0xda0cd932);
+ REQUIRE(crc32().checksum() == 0x502e8c0e);
osmium::Changeset& cs2 = buffer_add_changeset(buffer,
"user",
@@ -42,11 +44,13 @@ TEST_CASE("Basic Changeset") {
cs2.set_id(43)
.set_created_at(120)
.set_num_changes(21)
+ .set_num_comments(osmium::num_comments_type(0))
.set_uid(9);
REQUIRE(43 == cs2.id());
REQUIRE(9 == cs2.uid());
REQUIRE(21 == cs2.num_changes());
+ REQUIRE(0 == cs2.num_comments());
REQUIRE(false == cs2.closed());
REQUIRE(osmium::Timestamp(120) == cs2.created_at());
REQUIRE(osmium::Timestamp() == cs2.closed_at());
@@ -61,3 +65,59 @@ TEST_CASE("Basic Changeset") {
REQUIRE(false == (cs1 >= cs2));
}
+
+TEST_CASE("Create changeset without helper") {
+ osmium::memory::Buffer buffer(10 * 1000);
+ osmium::builder::ChangesetBuilder builder(buffer);
+
+ osmium::Changeset& cs1 = builder.object();
+ cs1.set_id(42)
+ .set_created_at(100)
+ .set_closed_at(200)
+ .set_num_changes(7)
+ .set_num_comments(2)
+ .set_uid(9);
+
+ builder.add_user("user");
+ add_tags(buffer, builder, {
+ {"key1", "val1"},
+ {"key2", "val2"}
+ });
+
+ {
+ osmium::builder::ChangesetDiscussionBuilder disc_builder(buffer, &builder);
+ disc_builder.add_comment(osmium::Timestamp(300), 10, "user2");
+ disc_builder.add_comment_text("foo");
+ disc_builder.add_comment(osmium::Timestamp(400), 9, "user");
+ disc_builder.add_comment_text("bar");
+ }
+
+ buffer.commit();
+
+ REQUIRE(42 == cs1.id());
+ REQUIRE(9 == cs1.uid());
+ REQUIRE(7 == cs1.num_changes());
+ REQUIRE(2 == cs1.num_comments());
+ REQUIRE(true == cs1.closed());
+ REQUIRE(osmium::Timestamp(100) == cs1.created_at());
+ REQUIRE(osmium::Timestamp(200) == cs1.closed_at());
+ REQUIRE(2 == cs1.tags().size());
+ REQUIRE(std::string("user") == cs1.user());
+
+ auto cit = cs1.discussion().begin();
+
+ REQUIRE(cit != cs1.discussion().end());
+ REQUIRE(cit->date() == osmium::Timestamp(300));
+ REQUIRE(cit->uid() == 10);
+ REQUIRE(std::string("user2") == cit->user());
+ REQUIRE(std::string("foo") == cit->text());
+
+ REQUIRE(++cit != cs1.discussion().end());
+ REQUIRE(cit->date() == osmium::Timestamp(400));
+ REQUIRE(cit->uid() == 9);
+ REQUIRE(std::string("user") == cit->user());
+ REQUIRE(std::string("bar") == cit->text());
+
+ REQUIRE(++cit == cs1.discussion().end());
+}
+
diff --git a/test/t/buffer/test_buffer_basics.cpp b/test/t/buffer/test_buffer_basics.cpp
new file mode 100644
index 0000000..ffe7251
--- /dev/null
+++ b/test/t/buffer/test_buffer_basics.cpp
@@ -0,0 +1,34 @@
+#include "catch.hpp"
+
+#include <osmium/memory/buffer.hpp>
+
+TEST_CASE("Buffer basics") {
+
+ osmium::memory::Buffer invalid_buffer1;
+ osmium::memory::Buffer invalid_buffer2;
+ osmium::memory::Buffer empty_buffer1(1024);
+ osmium::memory::Buffer empty_buffer2(2048);
+
+ REQUIRE(!invalid_buffer1);
+ REQUIRE(!invalid_buffer2);
+ REQUIRE(empty_buffer1);
+ REQUIRE(empty_buffer2);
+
+ REQUIRE(invalid_buffer1 == invalid_buffer2);
+ REQUIRE(invalid_buffer1 != empty_buffer1);
+ REQUIRE(empty_buffer1 != empty_buffer2);
+
+ REQUIRE(invalid_buffer1.capacity() == 0);
+ REQUIRE(invalid_buffer1.written() == 0);
+ REQUIRE(invalid_buffer1.committed() == 0);
+
+ REQUIRE(empty_buffer1.capacity() == 1024);
+ REQUIRE(empty_buffer1.written() == 0);
+ REQUIRE(empty_buffer1.committed() == 0);
+
+ REQUIRE(empty_buffer2.capacity() == 2048);
+ REQUIRE(empty_buffer2.written() == 0);
+ REQUIRE(empty_buffer2.committed() == 0);
+
+}
+
diff --git a/test/t/geom/test_tile_data.hpp b/test/t/geom/test_tile_data.hpp
index e5c0953..8a22cfa 100644
--- a/test/t/geom/test_tile_data.hpp
+++ b/test/t/geom/test_tile_data.hpp
@@ -1,5 +1,7 @@
-std::string s = R"(127.4864358 16.8380041 223904 118630 18
+#include <string>
+
+static std::string s = R"(127.4864358 16.8380041 223904 118630 18
163.1103174 39.4760232 121 48 7
-4.1372725 -22.5105386 31 36 6
98.7193066 -36.2312406 1 1 1
diff --git a/test/t/io/deleted_nodes.osh b/test/t/io/deleted_nodes.osh
new file mode 100644
index 0000000..639e051
--- /dev/null
+++ b/test/t/io/deleted_nodes.osh
@@ -0,0 +1,5 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<osm version="0.6" generator="osmium/1.2.1">
+ <node id="1" version="1" timestamp="2015-09-17T11:52:00Z" uid="1" user="user_1" changeset="1" visible="false"/>
+ <node id="2" version="1" timestamp="2015-09-17T11:52:00Z" uid="1" user="user_1" changeset="1" visible="true" lat="1" lon="1"/>
+</osm>
diff --git a/test/t/io/deleted_nodes.osh.pbf b/test/t/io/deleted_nodes.osh.pbf
new file mode 100644
index 0000000..8a94870
Binary files /dev/null and b/test/t/io/deleted_nodes.osh.pbf differ
diff --git a/test/t/io/test_output_utils.cpp b/test/t/io/test_output_utils.cpp
new file mode 100644
index 0000000..a760683
--- /dev/null
+++ b/test/t/io/test_output_utils.cpp
@@ -0,0 +1,153 @@
+
+#include "catch.hpp"
+
+#include <locale>
+
+#include <osmium/io/detail/string_util.hpp>
+
+TEST_CASE("output formatted") {
+
+ std::string out;
+
+ SECTION("small results") {
+ osmium::io::detail::append_printf_formatted_string(out, "%d", 17);
+ REQUIRE(out == "17");
+ }
+
+ SECTION("several parameters") {
+ osmium::io::detail::append_printf_formatted_string(out, "%d %s", 17, "foo");
+ REQUIRE(out == "17 foo");
+
+ }
+
+ SECTION("string already containing something") {
+ out += "foo";
+ osmium::io::detail::append_printf_formatted_string(out, " %d", 23);
+ REQUIRE(out == "foo 23");
+ }
+
+ SECTION("large results") {
+ const char* str =
+ "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+ "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+ "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+ "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+ "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
+
+ osmium::io::detail::append_printf_formatted_string(out, "%s", str);
+
+ REQUIRE(out == str);
+ }
+
+}
+
+TEST_CASE("UTF8 encoding") {
+
+ std::string out;
+
+ SECTION("append to string") {
+ out += "1234";
+ osmium::io::detail::append_utf8_encoded_string(out, "abc");
+ REQUIRE(out == "1234abc");
+ }
+
+ SECTION("don't encode alphabetic characters") {
+ const char* s = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+ osmium::io::detail::append_utf8_encoded_string(out, s);
+ REQUIRE(out == s);
+ }
+
+ SECTION("don't encode numeric characters") {
+ const char* s = "0123456789";
+ osmium::io::detail::append_utf8_encoded_string(out, s);
+ REQUIRE(out == s);
+ }
+
+ SECTION("don't encode lots of often used characters characters") {
+ const char* s = ".-;:_#+";
+ osmium::io::detail::append_utf8_encoded_string(out, s);
+ REQUIRE(out == s);
+ }
+
+ SECTION("encode characters that are special in OPL") {
+ osmium::io::detail::append_utf8_encoded_string(out, " \n,=@");
+ REQUIRE(out == "%20%%0a%%2c%%3d%%40%");
+ }
+
+// workaround for missing support for u8 string literals on Windows
+#if !defined(_MSC_VER)
+
+ SECTION("encode multibyte character") {
+ osmium::io::detail::append_utf8_encoded_string(out, u8"\u30dc_\U0001d11e_\U0001f6eb");
+ REQUIRE(out == "%30dc%_%1d11e%_%1f6eb%");
+ }
+
+#endif
+
+}
+
+TEST_CASE("html encoding") {
+
+ std::string out;
+
+ SECTION("do not encode normal characters") {
+ const char* s = "abc123,.-";
+ osmium::io::detail::append_xml_encoded_string(out, s);
+ REQUIRE(out == s);
+ }
+
+ SECTION("encode special XML characters") {
+ const char* s = "& \" \' < > \n \r \t";
+ osmium::io::detail::append_xml_encoded_string(out, s);
+ REQUIRE(out == "& " ' < > 
 
 	");
+ }
+
+}
+
+TEST_CASE("debug encoding") {
+
+ std::string out;
+
+ SECTION("do not encode normal characters") {
+ const char* s = "abc123,.-";
+ osmium::io::detail::append_debug_encoded_string(out, s, "[", "]");
+ REQUIRE(out == s);
+ }
+
+ SECTION("encode some unicode characters") {
+ const char* s = u8"\n_\u30dc_\U0001d11e_\U0001f6eb";
+ osmium::io::detail::append_debug_encoded_string(out, s, "[", "]");
+ REQUIRE(out == "[<U+000A>]_[<U+30DC>]_[<U+1D11E>]_[<U+1F6EB>]");
+ }
+
+}
+
+TEST_CASE("encoding of non-printable characters in the first 127 characters") {
+
+ std::locale cloc("C");
+ char s[] = "a\0";
+
+ for (char c = 1; c < 0x7f; ++c) {
+ std::string out;
+ s[0] = c;
+
+ SECTION("utf8 encode") {
+ osmium::io::detail::append_utf8_encoded_string(out, s);
+
+ if (!std::isprint(c, cloc)) {
+ REQUIRE(out[0] == '%');
+ }
+ }
+
+ SECTION("debug encode") {
+ osmium::io::detail::append_debug_encoded_string(out, s, "", "");
+
+ if (!std::isprint(c, cloc)) {
+ REQUIRE(out[0] == '<');
+ }
+ }
+
+ }
+
+}
+
diff --git a/test/t/io/test_reader.cpp b/test/t/io/test_reader.cpp
index 9a06d84..a83af52 100644
--- a/test/t/io/test_reader.cpp
+++ b/test/t/io/test_reader.cpp
@@ -4,6 +4,7 @@
#include <osmium/handler.hpp>
#include <osmium/io/any_compression.hpp>
#include <osmium/io/xml_input.hpp>
+#include <osmium/io/pbf_input.hpp>
#include <osmium/visitor.hpp>
#include <osmium/memory/buffer.hpp>
@@ -17,6 +18,27 @@ struct CountHandler : public osmium::handler::Handler {
}; // class CountHandler
+struct ZeroPositionNodeCountHandler : public osmium::handler::Handler {
+
+ // number of nodes seen at zero position, or visible with undefined
+ // location.
+ int count = 0;
+ int total_count = 0; // total number of nodes seen
+ const osmium::Location zero = osmium::Location(int32_t(0), int32_t(0));
+
+ void node(osmium::Node &n) {
+ // no nodes in the history file have a zero location, and
+ // no visible nodes should have an undefined location.
+ if ((n.location() == zero) ||
+ (n.visible() && !n.location())) {
+ ++count;
+ }
+ ++total_count;
+ }
+
+}; // class ZeroPositionNodeCountHandler
+
+
TEST_CASE("Reader") {
SECTION("reader can be initialized with file") {
@@ -34,7 +56,7 @@ TEST_CASE("Reader") {
osmium::apply(reader, handler);
}
- SECTION("should return invalid buffer after eof") {
+ SECTION("should throw after eof") {
osmium::io::File file(with_data_dir("t/io/data.osm"));
osmium::io::Reader reader(file);
@@ -45,9 +67,9 @@ TEST_CASE("Reader") {
REQUIRE(reader.eof());
- // extra read always returns invalid buffer
- osmium::memory::Buffer buffer = reader.read();
- REQUIRE(!buffer);
+ REQUIRE_THROWS_AS({
+ reader.read();
+ }, osmium::io_error);
}
SECTION("should not hang when apply() is called twice on reader") {
@@ -56,7 +78,9 @@ TEST_CASE("Reader") {
osmium::handler::Handler handler;
osmium::apply(reader, handler);
- osmium::apply(reader, handler);
+ REQUIRE_THROWS_AS({
+ osmium::apply(reader, handler);
+ }, osmium::io_error);
}
SECTION("should work with a buffer with uncompressed data") {
@@ -113,5 +137,76 @@ TEST_CASE("Reader") {
REQUIRE(handler.count == 1);
}
+ SECTION("should decode zero node positions in history (XML)") {
+ osmium::io::Reader reader(with_data_dir("t/io/deleted_nodes.osh"),
+ osmium::osm_entity_bits::node);
+ ZeroPositionNodeCountHandler handler;
+
+ REQUIRE(handler.count == 0);
+ REQUIRE(handler.total_count == 0);
+
+ osmium::apply(reader, handler);
+
+ REQUIRE(handler.count == 0);
+ REQUIRE(handler.total_count == 2);
+ }
+
+ SECTION("should decode zero node positions in history (PBF)") {
+ osmium::io::Reader reader(with_data_dir("t/io/deleted_nodes.osh.pbf"),
+ osmium::osm_entity_bits::node);
+ ZeroPositionNodeCountHandler handler;
+
+ REQUIRE(handler.count == 0);
+ REQUIRE(handler.total_count == 0);
+
+ osmium::apply(reader, handler);
+
+ REQUIRE(handler.count == 0);
+ REQUIRE(handler.total_count == 2);
+ }
+
+}
+
+TEST_CASE("Reader failure modes") {
+
+ SECTION("should fail with nonexistent file") {
+ REQUIRE_THROWS({
+ osmium::io::Reader reader(with_data_dir("t/io/nonexistent-file.osm"));
+ });
+ }
+
+ SECTION("should fail with nonexistent file (gz)") {
+ REQUIRE_THROWS({
+ osmium::io::Reader reader(with_data_dir("t/io/nonexistent-file.osm.gz"));
+ });
+ }
+
+ SECTION("should fail with nonexistent file (pbf)") {
+ REQUIRE_THROWS({
+ osmium::io::Reader reader(with_data_dir("t/io/nonexistent-file.osm.pbf"));
+ });
+ }
+
+ SECTION("should work when there is an exception in main thread before getting header") {
+ try {
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm"));
+ REQUIRE(!reader.eof());
+ throw std::runtime_error("foo");
+ } catch (...) {
+ }
+
+ }
+
+ SECTION("should work when there is an exception in main thread while reading") {
+ try {
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm"));
+ REQUIRE(!reader.eof());
+ auto header = reader.header();
+ throw std::runtime_error("foo");
+ } catch (...) {
+ }
+
+ }
+
}
diff --git a/test/t/io/test_reader_with_mock_decompression.cpp b/test/t/io/test_reader_with_mock_decompression.cpp
new file mode 100644
index 0000000..7a88671
--- /dev/null
+++ b/test/t/io/test_reader_with_mock_decompression.cpp
@@ -0,0 +1,145 @@
+
+#include "catch.hpp"
+#include "utils.hpp"
+
+#include <string>
+
+#include <osmium/io/compression.hpp>
+#include <osmium/io/xml_input.hpp>
+
+// The MockDecompressor behaves like other Decompressor classes, but "invents"
+// OSM data in XML format that can be read. Through a parameter to the
+// constructor it can be instructed to throw an exception in specific parts
+// of its code. This is then used to test the internals of the Reader.
+
+class MockDecompressor : public osmium::io::Decompressor {
+
+ std::string m_fail_in;
+ int m_read_count = 0;
+
+public:
+
+ MockDecompressor(const std::string& fail_in) :
+ Decompressor(),
+ m_fail_in(fail_in) {
+ if (m_fail_in == "constructor") {
+ throw std::runtime_error("error constructor");
+ }
+ }
+
+ ~MockDecompressor() noexcept override final = default;
+
+ void add_node(std::string& s, int i) {
+ s += "<node id='";
+ s += std::to_string(i);
+ s += "' version='1' timestamp='2014-01-01T00:00:00Z' uid='1' user='test' changeset='1' lon='1.02' lat='1.02'/>\n";
+ }
+
+ std::string read() override final {
+ std::string buffer;
+ ++m_read_count;
+
+ if (m_read_count == 1) {
+ if (m_fail_in == "first read") {
+ throw std::runtime_error("error first read");
+ } else {
+ buffer += "<?xml version='1.0' encoding='UTF-8'?>\n<osm version='0.6' generator='testdata'>\n";
+ for (int i = 0; i < 1000; ++i) {
+ add_node(buffer, i);
+ }
+ }
+ } else if (m_read_count == 2) {
+ if (m_fail_in == "second read") {
+ throw std::runtime_error("error second read");
+ } else {
+ for (int i = 1000; i < 2000; ++i) {
+ add_node(buffer, i);
+ }
+ }
+ } else if (m_read_count == 3) {
+ buffer += "</osm>";
+ }
+
+ return buffer;
+ }
+
+ void close() override final {
+ if (m_fail_in == "close") {
+ throw std::runtime_error("error close");
+ }
+ }
+
+}; // class MockDecompressor
+
+TEST_CASE("Test Reader using MockDecompressor") {
+
+ std::string fail_in;
+
+ osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip,
+ [](int, osmium::io::fsync) { return nullptr; },
+ [&](int) { return new MockDecompressor(fail_in); },
+ [](const char*, size_t) { return nullptr; }
+ );
+
+ SECTION("fail in constructor") {
+ fail_in = "constructor";
+
+ try {
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz"));
+ REQUIRE(false);
+ } catch (std::runtime_error& e) {
+ REQUIRE(std::string{e.what()} == "error constructor");
+ }
+ }
+
+ SECTION("fail in first read") {
+ fail_in = "first read";
+
+ try {
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz"));
+ reader.read();
+ REQUIRE(false);
+ } catch (std::runtime_error& e) {
+ REQUIRE(std::string{e.what()} == "error first read");
+ }
+ }
+
+ SECTION("fail in second read") {
+ fail_in = "second read";
+
+ try {
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz"));
+ reader.read();
+ reader.read();
+ REQUIRE(false);
+ } catch (std::runtime_error& e) {
+ REQUIRE(std::string{e.what()} == "error second read");
+ }
+ }
+
+ SECTION("fail in close") {
+ fail_in = "close";
+
+ try {
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz"));
+ reader.read();
+ reader.read();
+ reader.read();
+ reader.close();
+ REQUIRE(false);
+ } catch (std::runtime_error& e) {
+ REQUIRE(std::string{e.what()} == "error close");
+ }
+ }
+
+ SECTION("not failing") {
+ fail_in = "not";
+
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm.gz"));
+ reader.read();
+ reader.close();
+ REQUIRE(true);
+ }
+
+}
+
diff --git a/test/t/io/test_reader_with_mock_parser.cpp b/test/t/io/test_reader_with_mock_parser.cpp
new file mode 100644
index 0000000..02a944b
--- /dev/null
+++ b/test/t/io/test_reader_with_mock_parser.cpp
@@ -0,0 +1,123 @@
+
+#include "catch.hpp"
+#include "utils.hpp"
+
+#include <string>
+
+#include <osmium/builder/osm_object_builder.hpp>
+#include <osmium/io/compression.hpp>
+#include <osmium/io/detail/input_format.hpp>
+#include <osmium/io/detail/queue_util.hpp>
+#include <osmium/io/file_format.hpp>
+#include <osmium/io/header.hpp>
+#include <osmium/io/reader.hpp>
+#include <osmium/thread/queue.hpp>
+#include <osmium/thread/util.hpp>
+
+class MockParser : public osmium::io::detail::Parser {
+
+ std::string m_fail_in;
+
+public:
+
+ MockParser(osmium::io::detail::future_string_queue_type& input_queue,
+ osmium::io::detail::future_buffer_queue_type& output_queue,
+ std::promise<osmium::io::Header>& header_promise,
+ osmium::osm_entity_bits::type read_types,
+ const std::string& fail_in) :
+ Parser(input_queue, output_queue, header_promise, read_types),
+ m_fail_in(fail_in) {
+ }
+
+ osmium::memory::Buffer create_testdata() {
+ osmium::memory::Buffer buffer(1000);
+
+ {
+ osmium::builder::NodeBuilder nb(buffer);
+ nb.add_user("foo");
+ }
+ buffer.commit();
+
+ return buffer;
+ }
+
+ void run() override final {
+ osmium::thread::set_thread_name("_osmium_mock_in");
+
+ if (m_fail_in == "header") {
+ throw std::runtime_error("error in header");
+ }
+
+ set_header_value(osmium::io::Header{});
+
+ send_to_output_queue(create_testdata());
+
+ if (m_fail_in == "read") {
+ throw std::runtime_error("error in read");
+ }
+ }
+
+}; // class MockParser
+
+TEST_CASE("Test Reader using MockParser") {
+
+ std::string fail_in;
+
+ osmium::io::detail::ParserFactory::instance().register_parser(
+ osmium::io::file_format::xml,
+ [&](osmium::io::detail::future_string_queue_type& input_queue,
+ osmium::io::detail::future_buffer_queue_type& output_queue,
+ std::promise<osmium::io::Header>& header_promise,
+ osmium::osm_entity_bits::type read_which_entities) {
+ return std::unique_ptr<osmium::io::detail::Parser>(new MockParser(input_queue, output_queue, header_promise, read_which_entities, fail_in));
+ });
+
+ SECTION("no failure") {
+ fail_in = "";
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm"));
+ auto header = reader.header();
+ REQUIRE(reader.read());
+ REQUIRE(!reader.read());
+ REQUIRE(reader.eof());
+ reader.close();
+ }
+
+ SECTION("throw in header") {
+ fail_in = "header";
+ try {
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm"));
+ reader.header();
+ } catch (std::runtime_error& e) {
+ REQUIRE(std::string{e.what()} == "error in header");
+ }
+ }
+
+ SECTION("throw in read") {
+ fail_in = "read";
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm"));
+ reader.header();
+ try {
+ reader.read();
+ } catch (std::runtime_error& e) {
+ REQUIRE(std::string{e.what()} == "error in read");
+ }
+ reader.close();
+ }
+
+ SECTION("throw in user code") {
+ fail_in = "";
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm"));
+ reader.header();
+ try {
+ throw std::runtime_error("error in user code");
+ } catch (std::runtime_error& e) {
+ REQUIRE(std::string{e.what()} == "error in user code");
+ }
+ REQUIRE(reader.read());
+ REQUIRE(!reader.read());
+ REQUIRE(reader.eof());
+ reader.close();
+ }
+
+}
+
diff --git a/test/t/io/test_string_table.cpp b/test/t/io/test_string_table.cpp
index 7fedfcf..ab977e8 100644
--- a/test/t/io/test_string_table.cpp
+++ b/test/t/io/test_string_table.cpp
@@ -33,9 +33,9 @@ TEST_CASE("String store") {
}
SECTION("add zero-length string and longer strings") {
- const char* s1 = ss.add("");
- const char* s2 = ss.add("xxx");
- const char* s3 = ss.add("yyyyy");
+ ss.add("");
+ ss.add("xxx");
+ ss.add("yyyyy");
auto it = ss.begin();
REQUIRE(std::string(*it++) == "");
diff --git a/test/t/io/test_writer.cpp b/test/t/io/test_writer.cpp
new file mode 100644
index 0000000..45593cf
--- /dev/null
+++ b/test/t/io/test_writer.cpp
@@ -0,0 +1,117 @@
+#include "catch.hpp"
+#include "utils.hpp"
+
+#include <algorithm>
+
+#include <osmium/io/any_compression.hpp>
+#include <osmium/io/xml_input.hpp>
+#include <osmium/io/xml_output.hpp>
+#include <osmium/io/output_iterator.hpp>
+#include <osmium/memory/buffer.hpp>
+
+TEST_CASE("Writer") {
+
+ osmium::io::Header header;
+ header.set("generator", "test_writer.cpp");
+
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm"));
+ osmium::memory::Buffer buffer = reader.read();
+ REQUIRE(buffer);
+ REQUIRE(buffer.committed() > 0);
+ auto num = std::distance(buffer.cbegin<osmium::OSMObject>(), buffer.cend<osmium::OSMObject>());
+ REQUIRE(num > 0);
+ REQUIRE(buffer.cbegin<osmium::OSMObject>()->id() == 1);
+
+ std::string filename;
+
+ SECTION("Empty writes") {
+
+ SECTION("Empty buffer") {
+ filename = "test-writer-out-empty-buffer.osm";
+ osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow);
+ osmium::memory::Buffer empty_buffer(1024);
+ writer(std::move(empty_buffer));
+ writer.close();
+ }
+
+ SECTION("Invalid buffer") {
+ filename = "test-writer-out-invalid-buffer.osm";
+ osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow);
+ osmium::memory::Buffer invalid_buffer;
+ writer(std::move(invalid_buffer));
+ writer.close();
+ }
+
+ osmium::io::Reader reader_check(filename);
+ osmium::memory::Buffer buffer_check = reader_check.read();
+ REQUIRE(!buffer_check);
+ }
+
+ SECTION("Successfull writes") {
+
+ SECTION("Writer buffer") {
+ filename = "test-writer-out-buffer.osm";
+ osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow);
+ writer(std::move(buffer));
+ writer.close();
+
+ REQUIRE_THROWS_AS({
+ writer(osmium::memory::Buffer{});
+ }, osmium::io_error);
+ }
+
+ SECTION("Writer item") {
+ filename = "test-writer-out-item.osm";
+ osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow);
+ for (const auto& item : buffer) {
+ writer(item);
+ }
+ writer.close();
+ }
+
+ SECTION("Writer output iterator") {
+ filename = "test-writer-out-iterator.osm";
+ osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow);
+ auto it = osmium::io::make_output_iterator(writer);
+ std::copy(buffer.cbegin(), buffer.cend(), it);
+ writer.close();
+ }
+
+ osmium::io::Reader reader_check(filename);
+ osmium::memory::Buffer buffer_check = reader_check.read();
+ REQUIRE(buffer_check);
+ REQUIRE(buffer_check.committed() > 0);
+ REQUIRE(std::distance(buffer_check.cbegin<osmium::OSMObject>(), buffer_check.cend<osmium::OSMObject>()) == num);
+ REQUIRE(buffer_check.cbegin<osmium::OSMObject>()->id() == 1);
+
+ }
+
+ SECTION("Interrupted write") {
+
+ int error = 0;
+ try {
+
+ SECTION("fail after open") {
+ filename = "test-writer-out-fail1.osm";
+ osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow);
+ throw 1;
+ }
+
+ SECTION("fail after write") {
+ filename = "test-writer-out-fail2.osm";
+ osmium::io::Writer writer(filename, header, osmium::io::overwrite::allow);
+ writer(std::move(buffer));
+ throw 2;
+ }
+
+ } catch (int e) {
+ error = e;
+ }
+
+ REQUIRE(error > 0);
+
+ }
+
+}
+
+
diff --git a/test/t/io/test_writer_with_mock_compression.cpp b/test/t/io/test_writer_with_mock_compression.cpp
new file mode 100644
index 0000000..fb7e3e4
--- /dev/null
+++ b/test/t/io/test_writer_with_mock_compression.cpp
@@ -0,0 +1,99 @@
+
+#include "catch.hpp"
+#include "utils.hpp"
+
+#include <stdexcept>
+#include <string>
+
+#include <osmium/io/compression.hpp>
+#include <osmium/io/xml_input.hpp>
+#include <osmium/io/xml_output.hpp>
+
+class MockCompressor : public osmium::io::Compressor {
+
+ std::string m_fail_in;
+
+public:
+
+ MockCompressor(const std::string& fail_in) :
+ Compressor(osmium::io::fsync::no),
+ m_fail_in(fail_in) {
+ if (m_fail_in == "constructor") {
+ throw std::logic_error("constructor");
+ }
+ }
+
+ ~MockCompressor() noexcept override final = default;
+
+ void write(const std::string&) override final {
+ if (m_fail_in == "write") {
+ throw std::logic_error("write");
+ }
+ }
+
+ void close() override final {
+ if (m_fail_in == "close") {
+ throw std::logic_error("close");
+ }
+ }
+
+}; // class MockCompressor
+
+TEST_CASE("Write with mock compressor") {
+
+ std::string fail_in;
+
+ osmium::io::CompressionFactory::instance().register_compression(osmium::io::file_compression::gzip,
+ [&](int, osmium::io::fsync) { return new MockCompressor(fail_in); },
+ [](int) { return nullptr; },
+ [](const char*, size_t) { return nullptr; }
+ );
+
+ osmium::io::Header header;
+ header.set("generator", "test_writer_with_mock_compression.cpp");
+
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm"));
+ osmium::memory::Buffer buffer = reader.read();
+ REQUIRE(buffer);
+ REQUIRE(buffer.committed() > 0);
+ auto num = std::distance(buffer.cbegin<osmium::OSMObject>(), buffer.cend<osmium::OSMObject>());
+ REQUIRE(num > 0);
+
+ SECTION("fail on construction") {
+
+ fail_in = "constructor";
+
+ REQUIRE_THROWS_AS({
+ osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm.gz", header, osmium::io::overwrite::allow);
+ writer(std::move(buffer));
+ writer.close();
+ }, std::logic_error);
+
+ }
+
+ SECTION("fail on write") {
+
+ fail_in = "write";
+
+ REQUIRE_THROWS_AS({
+ osmium::io::Writer writer("test-writer-mock-fail-on-write.osm.gz", header, osmium::io::overwrite::allow);
+ writer(std::move(buffer));
+ writer.close();
+ }, std::logic_error);
+
+ }
+
+ SECTION("fail on close") {
+
+ fail_in = "close";
+
+ REQUIRE_THROWS_AS({
+ osmium::io::Writer writer("test-writer-mock-fail-on-close.osm.gz", header, osmium::io::overwrite::allow);
+ writer(std::move(buffer));
+ writer.close();
+ }, std::logic_error);
+
+ }
+
+}
+
diff --git a/test/t/io/test_writer_with_mock_encoder.cpp b/test/t/io/test_writer_with_mock_encoder.cpp
new file mode 100644
index 0000000..23cebd9
--- /dev/null
+++ b/test/t/io/test_writer_with_mock_encoder.cpp
@@ -0,0 +1,105 @@
+
+#include "catch.hpp"
+#include "utils.hpp"
+
+#include <stdexcept>
+#include <string>
+
+#include <osmium/io/compression.hpp>
+#include <osmium/io/detail/output_format.hpp>
+#include <osmium/io/detail/queue_util.hpp>
+#include <osmium/io/xml_input.hpp>
+#include <osmium/io/writer.hpp>
+
+class MockOutputFormat : public osmium::io::detail::OutputFormat {
+
+ std::string m_fail_in;
+
+public:
+
+ MockOutputFormat(const osmium::io::File&, osmium::io::detail::future_string_queue_type& output_queue, const std::string& fail_in) :
+ OutputFormat(output_queue),
+ m_fail_in(fail_in) {
+ }
+
+ void write_header(const osmium::io::Header&) override final {
+ if (m_fail_in == "header") {
+ throw std::logic_error("header");
+ }
+ send_to_output_queue(std::string{"header"});
+ }
+
+ void write_buffer(osmium::memory::Buffer&&) override final {
+ if (m_fail_in == "write") {
+ throw std::logic_error("write");
+ }
+ send_to_output_queue(std::string{"write"});
+ }
+
+ void write_end() override final {
+ if (m_fail_in == "write_end") {
+ throw std::logic_error("write_end");
+ }
+ send_to_output_queue(std::string{"end"});
+ }
+
+}; // class MockOutputFormat
+
+TEST_CASE("Test Writer with MockOutputFormat") {
+
+ std::string fail_in;
+
+ osmium::io::detail::OutputFormatFactory::instance().register_output_format(
+ osmium::io::file_format::xml,
+ [&](const osmium::io::File& file, osmium::io::detail::future_string_queue_type& output_queue) {
+ return new MockOutputFormat(file, output_queue, fail_in);
+ });
+
+ osmium::io::Header header;
+ header.set("generator", "test_writer_with_mock_encoder.cpp");
+
+ osmium::io::Reader reader(with_data_dir("t/io/data.osm"));
+ osmium::memory::Buffer buffer = reader.read();
+ REQUIRE(buffer);
+ REQUIRE(buffer.committed() > 0);
+ auto num = std::distance(buffer.cbegin<osmium::OSMObject>(), buffer.cend<osmium::OSMObject>());
+ REQUIRE(num > 0);
+
+ SECTION("error in header") {
+
+ fail_in = "header";
+
+ REQUIRE_THROWS_AS({
+ osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm", header, osmium::io::overwrite::allow);
+ writer(std::move(buffer));
+ writer.close();
+ }, std::logic_error);
+
+ }
+
+ SECTION("error in write") {
+
+ fail_in = "write";
+
+ REQUIRE_THROWS_AS({
+ osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm", header, osmium::io::overwrite::allow);
+ writer(std::move(buffer));
+ writer.close();
+ }, std::logic_error);
+
+ }
+
+ SECTION("error in write_end") {
+
+ fail_in = "write_end";
+
+ REQUIRE_THROWS_AS({
+ osmium::io::Writer writer("test-writer-mock-fail-on-construction.osm", header, osmium::io::overwrite::allow);
+ writer(std::move(buffer));
+ writer.close();
+ }, std::logic_error);
+
+ }
+
+}
+
diff --git a/test/t/thread/test_pool.cpp b/test/t/thread/test_pool.cpp
index 5fa6bba..7b6d20d 100644
--- a/test/t/thread/test_pool.cpp
+++ b/test/t/thread/test_pool.cpp
@@ -5,14 +5,7 @@
#include <thread>
#include <osmium/thread/pool.hpp>
-
-static std::atomic<int> result;
-
-struct test_job_ok {
- void operator()() const {
- result = 1;
- }
-};
+#include <osmium/util/compatibility.hpp>
struct test_job_with_result {
int operator()() const {
@@ -21,44 +14,26 @@ struct test_job_with_result {
};
struct test_job_throw {
- void operator()() const {
+ OSMIUM_NORETURN void operator()() const {
throw std::runtime_error("exception in pool thread");
}
};
TEST_CASE("thread") {
+ auto& pool = osmium::thread::Pool::instance();
+
SECTION("can get access to thread pool") {
- auto& pool = osmium::thread::Pool::instance();
REQUIRE(pool.queue_empty());
}
SECTION("can send job to thread pool") {
- auto& pool = osmium::thread::Pool::instance();
- result = 0;
- auto future = pool.submit(test_job_ok {});
-
- // wait a bit for the other thread to get a chance to run
- std::this_thread::sleep_for(std::chrono::milliseconds(1));
-
- REQUIRE(result == 1);
-
- future.get();
-
- REQUIRE(true);
- }
-
- SECTION("can send job to thread pool") {
- auto& pool = osmium::thread::Pool::instance();
auto future = pool.submit(test_job_with_result {});
REQUIRE(future.get() == 42);
}
SECTION("can throw from job in thread pool") {
- auto& pool = osmium::thread::Pool::instance();
- result = 0;
-
auto future = pool.submit(test_job_throw {});
REQUIRE_THROWS_AS(future.get(), std::runtime_error);
diff --git a/test/t/util/test_data_file.cpp b/test/t/util/test_data_file.cpp
index 3f432f9..792cedf 100644
--- a/test/t/util/test_data_file.cpp
+++ b/test/t/util/test_data_file.cpp
@@ -55,7 +55,7 @@ TEST_CASE("named file") {
REQUIRE(file.size() == 6);
char buf[10];
- int len = ::read(fd, buf, sizeof(buf));
+ auto len = ::read(fd, buf, sizeof(buf));
REQUIRE(len == 6);
REQUIRE(!strncmp(buf, "foobar", 6));
diff --git a/test/t/util/test_delta.cpp b/test/t/util/test_delta.cpp
index cebcca8..667c9b4 100644
--- a/test/t/util/test_delta.cpp
+++ b/test/t/util/test_delta.cpp
@@ -4,24 +4,50 @@
#include <osmium/util/delta.hpp>
-TEST_CASE("delta encode") {
+TEST_CASE("delta encode int") {
osmium::util::DeltaEncode<int> x;
SECTION("int") {
REQUIRE(x.update(17) == 17);
REQUIRE(x.update(10) == -7);
+ REQUIRE(x.update(-10) == -20);
}
}
-TEST_CASE("delta decode") {
+TEST_CASE("delta decode int") {
osmium::util::DeltaDecode<int> x;
SECTION("int") {
REQUIRE(x.update(17) == 17);
REQUIRE(x.update(10) == 27);
+ REQUIRE(x.update(-40) == -13);
+ }
+
+}
+
+TEST_CASE("delta encode unsigned int") {
+
+ osmium::util::DeltaEncode<unsigned int> x;
+
+ SECTION("int") {
+ REQUIRE(x.update(17) == 17);
+ REQUIRE(x.update(10) == -7);
+ REQUIRE(x.update(0) == -10);
+ }
+
+}
+
+TEST_CASE("delta decode unsigned int") {
+
+ osmium::util::DeltaDecode<unsigned int> x;
+
+ SECTION("int") {
+ REQUIRE(x.update(17) == 17);
+ REQUIRE(x.update(10) == 27);
+ REQUIRE(x.update(-15) == 12);
}
}
@@ -30,13 +56,13 @@ TEST_CASE("delta encode and decode") {
std::vector<int> a = { 5, -9, 22, 13, 0, 23 };
- osmium::util::DeltaEncode<int> de;
+ osmium::util::DeltaEncode<int, int> de;
std::vector<int> b;
for (int x : a) {
b.push_back(de.update(x));
}
- osmium::util::DeltaDecode<int> dd;
+ osmium::util::DeltaDecode<int, int> dd;
std::vector<int> c;
for (int x : b) {
c.push_back(dd.update(x));
diff --git a/test/t/util/test_file.cpp b/test/t/util/test_file.cpp
index 2787261..475f285 100644
--- a/test/t/util/test_file.cpp
+++ b/test/t/util/test_file.cpp
@@ -3,6 +3,7 @@
#include <osmium/util/file.hpp>
#ifdef _WIN32
+#include <crtdbg.h>
// https://msdn.microsoft.com/en-us/library/ksazx244.aspx
// https://msdn.microsoft.com/en-us/library/a9yf33zb.aspx
class DoNothingInvalidParameterHandler {
@@ -23,6 +24,7 @@ public:
DoNothingInvalidParameterHandler() :
old_handler(_set_invalid_parameter_handler(invalid_parameter_handler)) {
+ _CrtSetReportMode(_CRT_ASSERT, 0);
}
~DoNothingInvalidParameterHandler() {
diff --git a/test/t/util/test_options.cpp b/test/t/util/test_options.cpp
index 969f201..c1e13bd 100644
--- a/test/t/util/test_options.cpp
+++ b/test/t/util/test_options.cpp
@@ -6,41 +6,52 @@
TEST_CASE("Options") {
+ osmium::util::Options o;
+
SECTION("set_simple") {
- osmium::util::Options o;
o.set("foo", "bar");
REQUIRE("bar" == o.get("foo"));
REQUIRE("" == o.get("empty"));
REQUIRE("default" == o.get("empty", "default"));
+
REQUIRE(!o.is_true("foo"));
REQUIRE(!o.is_true("empty"));
+
+ REQUIRE(o.is_not_false("foo"));
+ REQUIRE(o.is_not_false("empty"));
+
REQUIRE(1 == o.size());
}
SECTION("set_from_bool") {
- osmium::util::Options o;
o.set("t", true);
o.set("f", false);
REQUIRE("true" == o.get("t"));
REQUIRE("false" == o.get("f"));
REQUIRE("" == o.get("empty"));
+
REQUIRE(o.is_true("t"));
REQUIRE(!o.is_true("f"));
+
+ REQUIRE(o.is_not_false("t"));
+ REQUIRE(!o.is_not_false("f"));
+
REQUIRE(2 == o.size());
}
SECTION("set_from_single_string_with_equals") {
- osmium::util::Options o;
o.set("foo=bar");
REQUIRE("bar" == o.get("foo"));
REQUIRE(1 == o.size());
}
SECTION("set_from_single_string_without_equals") {
- osmium::util::Options o;
o.set("foo");
REQUIRE("true" == o.get("foo"));
+
REQUIRE(o.is_true("foo"));
+ REQUIRE(o.is_not_false("foo"));
+
REQUIRE(1 == o.size());
}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/libosmium.git
More information about the Pkg-grass-devel
mailing list