[osmcoastline] 01/07: Imported Upstream version 2.1.3
Sebastiaan Couwenberg
sebastic at moszumanska.debian.org
Thu Mar 31 10:02:29 UTC 2016
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository osmcoastline.
commit 6ce404e77652872dca08580ae0c7918578d7bbe1
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Thu Mar 31 07:42:46 2016 +0200
Imported Upstream version 2.1.3
---
CHANGELOG.md | 21 ++++++++++++++++++-
CMakeLists.txt | 2 +-
include/gdalcpp.hpp | 4 ++++
src/osmcoastline.cpp | 40 +++++++----------------------------
src/osmcoastline_filter.cpp | 49 +++++++++++++++++++++++++++++++++++++------
src/osmcoastline_ways.cpp | 4 +++-
src/output_database.cpp | 24 ++++++++++-----------
src/verbose_output.hpp | 51 ---------------------------------------------
8 files changed, 90 insertions(+), 105 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d1068a7..aa6d6ba 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,24 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Fixed
+## [2.1.3] - 2016-03-30
+
+### Added
+
+- Add verbose option to osmcoastline_filter.
+- osmcoastline_filter now shows memory used in verbose mode.
+
+### Changed
+
+- Optimized osmcoastline_filter program.
+- Use more features from newest libosmium.
+
+### Fixed
+
+- Setting the sqlite output to unsynchronized speeds up writing to database.
+- Now also works on GDAL 2. Fixes an error in the transaction handling.
+
+
## [2.1.2] - 2016-01-05
### Added
@@ -59,7 +77,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
- Added man pages
-[unreleased]: https://github.com/osmcode/osmium-tool/compare/v2.1.2...HEAD
+[unreleased]: https://github.com/osmcode/osmium-tool/compare/v2.1.3...HEAD
+[2.1.3]: https://github.com/osmcode/osmium-tool/compare/v2.1.2...v2.1.3
[2.1.2]: https://github.com/osmcode/osmium-tool/compare/v2.1.1...v2.1.2
[2.1.1]: https://github.com/osmcode/osmium-tool/compare/v2.1.0...v2.1.1
[2.1.0]: https://github.com/osmcode/osmium-tool/compare/v2.0.1...v2.1.0
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 894b51a..e03ecd3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,7 +22,7 @@ project(osmcoastline)
set(OSMCOASTLINE_VERSION_MAJOR 2)
set(OSMCOASTLINE_VERSION_MINOR 1)
-set(OSMCOASTLINE_VERSION_PATCH 2)
+set(OSMCOASTLINE_VERSION_PATCH 3)
set(OSMCOASTLINE_VERSION
${OSMCOASTLINE_VERSION_MAJOR}.${OSMCOASTLINE_VERSION_MINOR}.${OSMCOASTLINE_VERSION_PATCH})
diff --git a/include/gdalcpp.hpp b/include/gdalcpp.hpp
index 1502f2f..ac3b104 100644
--- a/include/gdalcpp.hpp
+++ b/include/gdalcpp.hpp
@@ -347,18 +347,22 @@ namespace gdalcpp {
}
Layer& start_transaction() {
+#if GDAL_VERSION_MAJOR < 2
OGRErr result = m_layer->StartTransaction();
if (result != OGRERR_NONE) {
throw gdal_error(std::string("starting transaction on layer '") + name() + "' failed", result, m_dataset.driver_name(), m_dataset.dataset_name(), name());
}
+#endif
return *this;
}
Layer& commit_transaction() {
+#if GDAL_VERSION_MAJOR < 2
OGRErr result = m_layer->CommitTransaction();
if (result != OGRERR_NONE) {
throw gdal_error(std::string("committing transaction on layer '") + name() + "' failed", result, m_dataset.driver_name(), m_dataset.dataset_name(), name());
}
+#endif
return *this;
}
diff --git a/src/osmcoastline.cpp b/src/osmcoastline.cpp
index 262b81b..1ede948 100644
--- a/src/osmcoastline.cpp
+++ b/src/osmcoastline.cpp
@@ -25,6 +25,8 @@
#include <unistd.h>
#include <osmium/io/any_input.hpp>
+#include <osmium/util/memory.hpp>
+#include <osmium/util/verbose_output.hpp>
#include <osmium/visitor.hpp>
#include "return_codes.hpp"
@@ -38,7 +40,6 @@
#include "coastline_handlers.hpp"
#include "srs.hpp"
#include "util.hpp"
-#include "verbose_output.hpp"
// The global SRS object is used in many places to transform
// from WGS84 to the output SRS etc.
@@ -104,38 +105,10 @@ polygon_vector_type create_polygons(CoastlineRingCollection& coastline_rings, Ou
/* ================================================== */
-std::pair<int, int> get_memory_usage() {
- char filename[100];
- sprintf(filename, "/proc/%d/status", getpid());
- std::ifstream status_file(filename);
- std::string line;
-
- int vmpeak = 0;
- int vmsize = 0;
- if (status_file.is_open()) {
- while (! status_file.eof() ) {
- std::getline(status_file, line);
- if (line.substr(0, 6) == "VmPeak") {
- int f = line.find_first_of("0123456789");
- int l = line.find_last_of("0123456789");
- vmpeak = atoi(line.substr(f, l-f+1).c_str());
- }
- if (line.substr(0, 6) == "VmSize") {
- int f = line.find_first_of("0123456789");
- int l = line.find_last_of("0123456789");
- vmsize = atoi(line.substr(f, l-f+1).c_str());
- }
- }
- status_file.close();
- }
-
- return std::make_pair(vmsize / 1024, vmpeak / 1024);
-}
-
std::string memory_usage() {
- std::pair<int, int> mem = get_memory_usage();
+ osmium::MemoryUsage mem;
std::ostringstream s;
- s << "Memory used currently: " << mem.first << " MB (Peak was: " << mem.second << " MB).\n";
+ s << "Memory used: current: " << mem.current() << " MBytes, peak: " << mem.peak() << " MBytes\n";
return s.str();
}
@@ -152,11 +125,12 @@ int main(int argc, char *argv[]) {
// The vout object is an output stream we can write to instead of
// std::cerr. Nothing is written if we are not in verbose mode.
// The running time will be prepended to output lines.
- VerboseOutput vout(options.verbose);
+ osmium::util::VerboseOutput vout(options.verbose);
debug = options.debug;
CPLSetConfigOption("OGR_ENABLE_PARTIAL_REPROJECTION", "TRUE");
+ CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "OFF");
vout << "Using SRS " << options.epsg << " for output. (Change with the --srs/s option.)\n";
if (!srs.set_output(options.epsg)) {
std::cerr << "Setting up output transformation failed\n";
@@ -320,7 +294,7 @@ int main(int argc, char *argv[]) {
vout << memory_usage();
vout << "Committing database transactions...\n";
- output_database.set_meta(vout.runtime(), get_memory_usage().second, stats);
+ output_database.set_meta(vout.runtime(), osmium::MemoryUsage{}.peak(), stats);
output_database.commit();
vout << "All done.\n";
vout << memory_usage();
diff --git a/src/osmcoastline_filter.cpp b/src/osmcoastline_filter.cpp
index 9a3f951..1700769 100644
--- a/src/osmcoastline_filter.cpp
+++ b/src/osmcoastline_filter.cpp
@@ -20,9 +20,10 @@
*/
#include <algorithm>
-#include <iostream>
-#include <set>
#include <getopt.h>
+#include <iostream>
+#include <string>
+#include <vector>
#include <osmium/io/any_input.hpp>
#include <osmium/io/input_iterator.hpp>
@@ -30,6 +31,8 @@
#include <osmium/io/pbf_output.hpp>
#include <osmium/handler.hpp>
#include <osmium/osm/entity_bits.hpp>
+#include <osmium/util/memory.hpp>
+#include <osmium/util/verbose_output.hpp>
#include "return_codes.hpp"
@@ -38,22 +41,25 @@ void print_help() {
<< "\nOptions:\n"
<< " -h, --help - This help message\n"
<< " -o, --output=OSMFILE - Where to write output (default: none)\n"
+ << " -v, --verbose - Verbose output\n"
<< " -V, --version - Show version and exit\n"
<< "\n";
}
int main(int argc, char* argv[]) {
std::string output_filename;
+ bool verbose = false;
static struct option long_options[] = {
{"help", no_argument, 0, 'h'},
{"output", required_argument, 0, 'o'},
+ {"verbose", no_argument, 0, 'v'},
{"version", no_argument, 0, 'V'},
{0, 0, 0, 0}
};
while (1) {
- int c = getopt_long(argc, argv, "ho:V", long_options, 0);
+ int c = getopt_long(argc, argv, "ho:vV", long_options, 0);
if (c == -1)
break;
@@ -64,6 +70,9 @@ int main(int argc, char* argv[]) {
case 'o':
output_filename = optarg;
break;
+ case 'v':
+ verbose = true;
+ break;
case 'V':
std::cout << "osmcoastline_filter version " OSMCOASTLINE_VERSION "\n"
<< "Copyright (C) 2012-2016 Jochen Topf <jochen at topf.org>\n"
@@ -76,6 +85,11 @@ int main(int argc, char* argv[]) {
}
}
+ // The vout object is an output stream we can write to instead of
+ // std::cerr. Nothing is written if we are not in verbose mode.
+ // The running time will be prepended to output lines.
+ osmium::util::VerboseOutput vout(verbose);
+
if (output_filename.empty()) {
std::cerr << "Missing -o/--output=OSMFILE option\n";
exit(return_code_cmdline);
@@ -98,6 +112,7 @@ int main(int argc, char* argv[]) {
std::vector<osmium::object_id_type> ids;
+ vout << "Reading ways (1st pass through input file)...\n";
{
osmium::io::Reader reader(infile, osmium::osm_entity_bits::way);
auto ways = osmium::io::make_input_iterator_range<const osmium::Way>(reader);
@@ -113,17 +128,32 @@ int main(int argc, char* argv[]) {
reader.close();
}
+ vout << "Preparing node ID list...\n";
std::sort(ids.begin(), ids.end());
auto last = std::unique(ids.begin(), ids.end());
+ vout << "Reading nodes (2nd pass through input file)...\n";
{
osmium::io::Reader reader(infile, osmium::osm_entity_bits::node);
auto nodes = osmium::io::make_input_iterator_range<const osmium::Node>(reader);
- std::copy_if(nodes.cbegin(), nodes.cend(), output_it, [&ids, &last](const osmium::Node& node){
+
+ auto first = ids.begin();
+ std::copy_if(nodes.cbegin(), nodes.cend(), output_it, [&first, &last](const osmium::Node& node){
+ while (*first < node.id() && first != last) {
+ ++first;
+ }
+
+ if (node.id() == *first) {
+ if (first != last) {
+ ++first;
+ }
+ return true;
+ }
+
const char* natural = node.get_value_by_key("natural");
- return std::binary_search(ids.begin(), last, node.id()) ||
- (natural && !strcmp(natural, "coastline"));
+ return natural && !strcmp(natural, "coastline");
});
+
reader.close();
}
@@ -132,5 +162,12 @@ int main(int argc, char* argv[]) {
std::cerr << "io error: " << e.what() << "'\n";
exit(return_code_fatal);
}
+
+ vout << "All done.\n";
+ osmium::MemoryUsage mem;
+ if (mem.current() > 0) {
+ vout << "Memory used: current: " << mem.current() << " MBytes\n"
+ << " peak: " << mem.peak() << " MBytes\n";
+ }
}
diff --git a/src/osmcoastline_ways.cpp b/src/osmcoastline_ways.cpp
index cd38493..09c8af4 100644
--- a/src/osmcoastline_ways.cpp
+++ b/src/osmcoastline_ways.cpp
@@ -51,7 +51,7 @@ public:
CoastlineWaysHandler(const std::string& db_filename) :
m_length(0.0),
- m_dataset("SQLite", db_filename, gdalcpp::SRS{}, {"SPATIALITE=TRUE", "OGR_SQLITE_SYNCHRONOUS=OFF", "INIT_WITH_EPSG=no" }),
+ m_dataset("SQLite", db_filename, gdalcpp::SRS{}, {"SPATIALITE=TRUE", "INIT_WITH_EPSG=no" }),
m_layer_ways(m_dataset, "ways", wkbLineString) {
m_layer_ways.add_field("way_id", OFTString, 10);
@@ -110,6 +110,8 @@ int main(int argc, char* argv[]) {
exit(return_code_cmdline);
}
+ CPLSetConfigOption("OGR_SQLITE_SYNCHRONOUS", "OFF");
+
std::string input_osm_filename { argv[1] };
std::string output_db_filename { "coastline-ways.db" };
diff --git a/src/output_database.cpp b/src/output_database.cpp
index b1b385a..293da90 100644
--- a/src/output_database.cpp
+++ b/src/output_database.cpp
@@ -33,7 +33,7 @@
OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_index) :
m_with_index(with_index),
m_srs(srs),
- m_dataset("SQLite", outdb, gdalcpp::SRS(*srs.out()), { "SPATIALITE=TRUE", "OGR_SQLITE_SYNCHRONOUS=OFF", "INIT_WITH_EPSG=no" }),
+ m_dataset("SQLite", outdb, gdalcpp::SRS(*srs.out()), { "SPATIALITE=TRUE", "INIT_WITH_EPSG=no" }),
m_layer_error_points(m_dataset, "error_points", wkbPoint, layer_options()),
m_layer_error_lines(m_dataset, "error_lines", wkbLineString, layer_options()),
m_layer_rings(m_dataset, "rings", wkbPolygon, layer_options()),
@@ -43,11 +43,9 @@ OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_ind
m_layer_error_points.add_field("osm_id", OFTString, 10);
m_layer_error_points.add_field("error", OFTString, 16);
- m_layer_error_points.start_transaction();
m_layer_error_lines.add_field("osm_id", OFTString, 10);
m_layer_error_lines.add_field("error", OFTString, 16);
- m_layer_error_lines.start_transaction();
m_layer_rings.add_field("osm_id", OFTString, 10);
m_layer_rings.add_field("nways", OFTInteger, 6);
@@ -55,13 +53,6 @@ OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_ind
m_layer_rings.add_field("fixed", OFTInteger, 1);
m_layer_rings.add_field("land", OFTInteger, 1);
m_layer_rings.add_field("valid", OFTInteger, 1);
- m_layer_rings.start_transaction();
-
- m_layer_land_polygons.start_transaction();
-
- m_layer_water_polygons.start_transaction();
-
- m_layer_lines.start_transaction();
m_dataset.exec("CREATE TABLE options (overlap REAL, close_distance REAL, max_points_in_polygons INTEGER, split_large_polygons INTEGER)");
m_dataset.exec("CREATE TABLE meta ("
@@ -76,6 +67,14 @@ OutputDatabase::OutputDatabase(const std::string& outdb, SRS& srs, bool with_ind
"num_rings_turned_around INTEGER, "
"num_land_polygons_before_split INTEGER, "
"num_land_polygons_after_split INTEGER)");
+
+ m_dataset.start_transaction();
+ m_layer_rings.start_transaction();
+ m_layer_land_polygons.start_transaction();
+ m_layer_water_polygons.start_transaction();
+ m_layer_lines.start_transaction();
+ m_layer_error_points.start_transaction();
+ m_layer_error_lines.start_transaction();
}
void OutputDatabase::set_options(const Options& options) {
@@ -119,12 +118,13 @@ void OutputDatabase::set_meta(int runtime, int memory_usage, const Stats& stats)
}
void OutputDatabase::commit() {
+ m_layer_error_lines.commit_transaction();
+ m_layer_error_points.commit_transaction();
m_layer_lines.commit_transaction();
m_layer_water_polygons.commit_transaction();
m_layer_land_polygons.commit_transaction();
m_layer_rings.commit_transaction();
- m_layer_error_lines.commit_transaction();
- m_layer_error_points.commit_transaction();
+ m_dataset.commit_transaction();
}
void OutputDatabase::add_error_point(std::unique_ptr<OGRPoint>&& point, const char* error, osmium::object_id_type id) {
diff --git a/src/verbose_output.hpp b/src/verbose_output.hpp
deleted file mode 100644
index 5284b0b..0000000
--- a/src/verbose_output.hpp
+++ /dev/null
@@ -1,51 +0,0 @@
-#ifndef VERBOSE_OUTPUT_HPP
-#define VERBOSE_OUTPUT_HPP
-
-#include <iostream>
-#include <sstream>
-
-class VerboseOutput {
-
- time_t m_start;
- bool m_verbose;
- bool m_newline;
-
-public:
-
- VerboseOutput(bool verbose) :
- m_start(time(nullptr)),
- m_verbose(verbose),
- m_newline(true) {
- }
-
- int runtime() const {
- return time(nullptr) - m_start;
- }
-
- void start_line() {
- if (m_newline) {
- int elapsed = time(nullptr) - m_start;
- char timestr[20];
- snprintf(timestr, sizeof(timestr)-1, "[%2d:%02d] ", elapsed / 60, elapsed % 60);
- std::cerr << timestr;
- m_newline = false;
- }
- }
-
- template<typename T>
- friend VerboseOutput& operator<<(VerboseOutput& out, T t) {
- if (out.m_verbose) {
- std::ostringstream o;
- o << t;
- out.start_line();
- std::cerr << o.str();
- if (o.str()[o.str().size()-1] == '\n') {
- out.m_newline = true;
- }
- }
- return out;
- }
-
-}; // class VerboseOutput
-
-#endif // VERBOSE_OUTPUT_HPP
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/osmcoastline.git
More information about the Pkg-grass-devel
mailing list