[osm2pgsql] 01/10: New upstream version 0.94.0~rc1+ds
Bas Couwenberg
sebastic at debian.org
Mon Sep 18 23:12:16 UTC 2017
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository osm2pgsql.
commit 273f46fe86eaca2044a62bb3df46c48b0f737e1d
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Tue Sep 19 00:40:34 2017 +0200
New upstream version 0.94.0~rc1+ds
---
.clang-format | 9 +
.travis.yml | 7 +-
CMakeLists.txt | 32 +-
CONTRIBUTING.md | 36 +-
README.md | 28 +-
appveyor.yml | 45 +-
docs/lua.md | 2 +-
docs/migrations.md | 5 +
docs/osm2pgsql.1 | 5 +-
docs/pgsql.md | 4 +-
docs/usage.md | 7 +-
expire-tiles.cpp | 391 ++++++-------
expire-tiles.hpp | 207 +++++--
geometry-builder.cpp | 739 -----------------------
geometry-builder.hpp | 114 ----
geometry-processor.cpp | 101 ++--
geometry-processor.hpp | 48 +-
middle-pgsql.cpp | 627 +++++++++-----------
middle-pgsql.hpp | 55 +-
middle-ram.cpp | 99 ++--
middle-ram.hpp | 52 +-
middle.hpp | 54 +-
multi.lua | 16 +-
node-persistent-cache.cpp | 658 ++-------------------
node-persistent-cache.hpp | 100 +---
node-ram-cache.cpp | 406 +++++++------
node-ram-cache.hpp | 90 +--
options.cpp | 111 ++--
options.hpp | 12 +-
osm2pgsql.cpp | 6 +-
osmdata.cpp | 93 +--
osmdata.hpp | 22 +-
osmium-builder.cpp | 410 +++++++++++++
osmium-builder.hpp | 51 ++
osmtypes.hpp | 111 +++-
output-gazetteer.cpp | 804 ++++++++++++--------------
output-gazetteer.hpp | 139 ++---
output-multi.cpp | 300 +++++-----
output-multi.hpp | 62 +-
output-null.cpp | 30 +-
output-null.hpp | 34 +-
output-pgsql.cpp | 463 +++++++--------
output-pgsql.hpp | 67 +--
output.cpp | 4 +-
output.hpp | 12 +-
parse-osmium.cpp | 81 +--
parse-osmium.hpp | 27 +-
pgsql.cpp | 45 +-
pgsql.hpp | 19 +-
processor-line.cpp | 21 +-
processor-line.hpp | 17 +-
processor-point.cpp | 14 +-
processor-point.hpp | 10 +-
processor-polygon.cpp | 21 +-
processor-polygon.hpp | 18 +-
style.lua | 16 +-
table.cpp | 59 +-
table.hpp | 21 +-
taginfo.cpp | 54 +-
taginfo_impl.hpp | 14 +-
tagtransform-c.cpp | 421 ++++++++++++++
tagtransform-c.hpp | 31 +
tagtransform-lua.cpp | 202 +++++++
tagtransform-lua.hpp | 38 ++
tagtransform.cpp | 675 +--------------------
tagtransform.hpp | 70 +--
tests/000466354.osc.gz | Bin 161335 -> 125326 bytes
tests/CMakeLists.txt | 1 +
tests/common-pg.cpp | 8 +-
tests/common.hpp | 3 +-
tests/liechtenstein-2013-08-03.osm.pbf | Bin 666646 -> 592570 bytes
tests/middle-tests.cpp | 425 ++++++++------
tests/mockups.hpp | 137 ++---
tests/regression-test.py | 90 +--
tests/test-expire-tiles.cpp | 430 +++++++++-----
tests/test-hstore-match-only.cpp | 2 +-
tests/test-middle-flat.cpp | 1 -
tests/test-middle-pgsql.cpp | 1 -
tests/test-middle-ram.cpp | 1 -
tests/test-options-database.cpp | 14 +-
tests/test-options-parse.cpp | 41 +-
tests/test-options-projection.cpp | 2 +-
tests/test-output-multi-line-storage.cpp | 2 +-
tests/test-output-multi-line.cpp | 8 +-
tests/test-output-multi-point-multi-table.cpp | 4 +-
tests/test-output-multi-point.cpp | 4 +-
tests/test-output-multi-poly-trivial.cpp | 2 +-
tests/test-output-multi-polygon.cpp | 4 +-
tests/test-output-multi-tags.cpp | 2 +-
tests/test-output-pgsql-area.cpp | 3 +-
tests/test-output-pgsql-schema.cpp | 2 +-
tests/test-output-pgsql-tablespace.cpp | 6 +-
tests/test-output-pgsql-validgeom.cpp | 4 +-
tests/test-output-pgsql-z_order.cpp | 2 +-
tests/test-output-pgsql.cpp | 43 +-
tests/test-parse-diff.cpp | 32 +-
tests/test-parse-xml2.cpp | 63 +-
tests/test-persistent-node-cache.cpp | 119 ++++
tests/test_output_multi_poly_trivial.lua | 6 +-
win_fsync.h | 71 ---
wkb.hpp | 363 ++++++++++++
101 files changed, 4968 insertions(+), 5470 deletions(-)
diff --git a/.clang-format b/.clang-format
new file mode 100644
index 0000000..5f5881d
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,9 @@
+---
+Language: Cpp
+BasedOnStyle: LLVM
+AccessModifierOffset: -4
+BreakBeforeBraces: Mozilla
+IndentWidth: 4
+ConstructorInitializerIndentWidth: 0
+ReflowComments: false
+AlwaysBreakTemplateDeclarations: true
diff --git a/.travis.yml b/.travis.yml
index c5070e2..993f279 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,8 +8,6 @@ addons:
packages:
- g++-4.8
- libexpat1-dev
- - libgeos-dev
- - libgeos++-dev
- libpq-dev
- libbz2-dev
- libproj-dev
@@ -25,10 +23,10 @@ matrix:
env: CXXFLAGS="-pedantic -Werror"
- os: linux
compiler: gcc
- env: RUNTEST="-L NoDB" CXXFLAGS="-pedantic -Werror"
+ env: RUNTEST="-L NoDB" CXXFLAGS="-pedantic -Werror -fsanitize=address"
- os: osx
compiler: clang
- env: RUNTEST="-L NoDB" CXXFLAGS="-pedantic -Werror"
+ env: RUNTEST="-L NoDB" CXXFLAGS="-pedantic -Werror -fsanitize=address"
before_install:
- if [[ $TRAVIS_OS_NAME == 'osx' ]]; then
brew install lua;
@@ -44,7 +42,6 @@ install:
before_script:
- $CXX --version
- xml2-config --version
- - geos-config --version
- proj | head -n1
- lua -v
script:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 05696b4..f2434c5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,6 +1,6 @@
set(PACKAGE osm2pgsql)
set(PACKAGE_NAME osm2pgsql)
-set(PACKAGE_VERSION 0.92.1)
+set(PACKAGE_VERSION 0.93.0-dev)
cmake_minimum_required(VERSION 2.8.7)
@@ -23,7 +23,7 @@ if (NOT WIN32 AND NOT APPLE)
endif()
# Just in case user installed RPMs from http://yum.postgresql.org/
-list(APPEND PostgreSQL_ADDITIONAL_SEARCH_PATHS /usr/pgsql-9.3 /usr/pgsql-9.4 /usr/pgsql-9.5)
+list(APPEND CMAKE_PREFIX_PATH /usr/pgsql-9.3 /usr/pgsql-9.4 /usr/pgsql-9.5 /usr/pgsql-9.6)
if (PROJECT_SOURCE_DIR STREQUAL PROJECT_BINARY_DIR)
message(FATAL_ERROR "In-source builds are not allowed, please use a separate build directory like `mkdir build && cd build && cmake ..`")
@@ -71,17 +71,6 @@ if (WIN32)
set(HAVE_LIBGEN_H FALSE)
endif()
-CHECK_FUNCTION_EXISTS(lseek64 HAVE_LSEEK64)
-CHECK_FUNCTION_EXISTS(posix_fallocate HAVE_POSIX_FALLOCATE)
-CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE)
-CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE)
-
-CHECK_TYPE_SIZE("off_t" SIZEOF_OFF_T)
-
-if (NOT WIN32 AND NOT APPLE AND NOT HAVE_LSEEK64 AND NOT SIZEOF_OFF_T EQUAL 8)
- message(FATAL_ERROR "Flat nodes cache requires a 64 bit capable seek")
-endif()
-
#############################################################
# Find necessary libraries
#############################################################
@@ -92,7 +81,7 @@ endif()
include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
-find_package(Osmium REQUIRED COMPONENTS io geos proj)
+find_package(Osmium REQUIRED COMPONENTS io proj)
include_directories(SYSTEM ${OSMIUM_INCLUDE_DIRS})
if (WITH_LUA)
@@ -101,11 +90,6 @@ if (WITH_LUA)
set(HAVE_LUA 1)
endif()
-if (MSVC)
- # Boost thread needs extra libraries
- set(BOOST_EXTRA date_time chrono)
-endif()
-
if (NOT Boost_ADDITIONAL_VERSIONS)
set(Boost_ADDITIONAL_VERSIONS "1.55;1.56;1.57;1.58;1.59;1.60;1.61")
endif()
@@ -167,7 +151,6 @@ endif()
set(osm2pgsql_lib_SOURCES
expire-tiles.cpp
- geometry-builder.cpp
geometry-processor.cpp
id-tracker.cpp
middle-pgsql.cpp
@@ -177,6 +160,7 @@ set(osm2pgsql_lib_SOURCES
node-ram-cache.cpp
options.cpp
osmdata.cpp
+ osmium-builder.cpp
output-gazetteer.cpp
output-multi.cpp
output-null.cpp
@@ -192,10 +176,10 @@ set(osm2pgsql_lib_SOURCES
table.cpp
taginfo.cpp
tagtransform.cpp
+ tagtransform-c.cpp
util.cpp
wildcmp.cpp
expire-tiles.hpp
- geometry-builder.hpp
geometry-processor.hpp
id-tracker.hpp
middle-pgsql.hpp
@@ -205,6 +189,7 @@ set(osm2pgsql_lib_SOURCES
node-ram-cache.hpp
options.hpp
osmdata.hpp
+ osmium-builder.hpp
osmtypes.hpp
output-gazetteer.hpp
output-multi.hpp
@@ -224,8 +209,13 @@ set(osm2pgsql_lib_SOURCES
tagtransform.hpp
util.hpp
wildcmp.hpp
+ wkb.hpp
)
+if (LUA_FOUND)
+ list(APPEND osm2pgsql_lib_SOURCES tagtransform-lua.cpp)
+endif()
+
add_library(osm2pgsql_lib STATIC ${osm2pgsql_lib_SOURCES})
set_target_properties(osm2pgsql_lib PROPERTIES OUTPUT_NAME osm2pgsql)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 47f305a..5ffa71d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -32,37 +32,25 @@ for easy bug fixes, or if a patch backporting a fix is provided.
## Code style
-The current codebase is a mix of styles, but new code should be written in the
+Code must be written in the
[K&R 1TBS style](https://en.wikipedia.org/wiki/Indent_style#Variant:_1TBS) with
-4 spaces indentation. Tabs should never be used in the C++ code.
-
-e.g.
-
-```
-int main(int argc, char *argv[])
-{
- ...
- while (x == y) {
- something();
- somethingelse();
-
- if (some_error) {
- do_correct();
- } else {
- continue_as_usual();
- }
- }
-
- finalthing();
- ...
-}
-```
+4 spaces indentation. Tabs should never be used in the C++ code. Braces must
+always be used for code blocks, even one-liners.
Names should use underscores, not camel case, with class/struct names ending in `_t`.
+Template parameters must use all upper case.
Headers should be included in the order `config.h`, C++ standard library headers,
C library headers, Boost headers, and last osm2pgsql files.
+There is a .clang-format configuration avialable and all code must be run through
+clang-format before submitting. You can use git-clang-format after staging all
+your changes:
+
+ git-clang-format --style=file *pp tests/*pp
+
+clang-format 3.8 or later is required.
+
## Documentation
User documentation is stored in `docs/`. Pages on the OpenStreetMap wiki are
diff --git a/README.md b/README.md
index 6f15533..59c2d30 100644
--- a/README.md
+++ b/README.md
@@ -18,9 +18,12 @@ Nominatim, or general analysis.
## Installing ##
-Most Linux distributions include osm2pgsql. It is also available on macOS with [Homebrew](http://brew.sh/). Unoffical builds for Windows are built by [AppVeyor](https://ci.appveyor.com/project/openstreetmap/osm2pgsql/history) but you need to find the right build artifacts.
+Most Linux distributions include osm2pgsql. It is also available on macOS with [Homebrew](http://brew.sh/).
-The latest source code is available in the OSM git repository on github
+Unoffical builds for Windows are available from [AppVeyor](https://ci.appveyor.com/project/openstreetmap/osm2pgsql/history) but you need to find the right build artifacts. The latest
+release is [0.92.0](https://ci.appveyor.com/api/projects/openstreetmap/osm2pgsql/artifacts/osm2pgsql_Release.zip?tag=0.92.0).
+
+The latest source code is available in the osm2pgsql git repository on GitHub
and can be downloaded as follows:
```sh
@@ -35,7 +38,6 @@ to configure and build itself and requires
Required libraries are
* [expat](http://www.libexpat.org/)
-* [geos](http://trac.osgeo.org/geos)
* [proj](http://proj.osgeo.org/)
* [bzip2](http://www.bzip.org/)
* [zlib](http://www.zlib.net/)
@@ -57,25 +59,29 @@ On a Debian or Ubuntu system, this can be done with:
```sh
sudo apt-get install make cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
- libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev lua5.2 \
- liblua5.2-dev
+ libbz2-dev libpq-dev libproj-dev lua5.2 liblua5.2-dev
```
On a Fedora system, use
```sh
-sudo yum install cmake gcc-c++ boost-devel expat-devel zlib-devel bzip2-devel \
- postgresql-devel geos-devel proj-devel proj-epsg lua-devel
+sudo dnf install cmake make gcc-c++ boost-devel expat-devel zlib-devel \
+ bzip2-devel postgresql-devel proj-devel proj-epsg lua-devel
```
On RedHat / CentOS first run `sudo yum install epel-release` then install
-dependencies like on Fedora.
+dependencies with:
+
+```sh
+sudo yum install cmake make gcc-c++ boost-devel expat-devel zlib-devel \
+ bzip2-devel postgresql-devel proj-devel proj-epsg lua-devel
+```
On a FreeBSD system, use
```sh
pkg install devel/cmake devel/boost-libs textproc/expat2 \
- databases/postgresql94-client graphics/geos graphics/proj lang/lua52
+ databases/postgresql94-client graphics/proj lang/lua52
```
Once dependencies are installed, use CMake to build the Makefiles in a separate folder
@@ -110,7 +116,7 @@ cmake .. -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON
## Usage ##
-Osm2pgsql has one program, the executable itself, which has **44** command line
+Osm2pgsql has one program, the executable itself, which has **43** command line
options.
Before loading into a database, the database must be created and the PostGIS
@@ -141,7 +147,7 @@ osm2pgsql -c -d gis --slim -C <cache size> \
```
where
* ``<cache size>`` is about 75% of memory in MiB, to a maximum of about 30000. Additional RAM will not be used.
-* ``<flat nodes>`` is a location where a 24GiB file can be saved.
+* ``<flat nodes>`` is a location where a 36GiB+ file can be saved.
Many different data files (e.g., .pbf) can be found at [planet.osm.org](http://planet.osm.org/).
diff --git a/appveyor.yml b/appveyor.yml
index 60f64e7..42cffeb 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -2,11 +2,11 @@ environment:
global:
PGUSER: postgres
PGPASSWORD: Password12!
- BOOST_ROOT: c:\libs\boost
+ BOOST_ROOT: c:/libs/boost
PREFIX: c:\libs
- PSQL_ROOT: C:\Program Files\PostgreSQL\9.4
+ PSQL_ROOT: C:/Program Files/PostgreSQL/9.4
CMAKE_PREFIX_PATH: c:\libs;C:\Program Files\PostgreSQL\9.4
- PYTHONPATH: c:\libs
+ POSTGIS_FILE: postgis-pg94-binaries-2.3.2w64gcc48
matrix:
- configuration: Release
@@ -14,8 +14,12 @@ environment:
os: Visual Studio 2015
+cache:
+ - C:\libs\%POSTGIS_FILE%.zip -> appveyor.yml
+ - C:\osm2pgsql_win_deps_release.7z -> appveyor.yml
+
services:
-# - postgresql # enable when Postgis will be available
+ - postgresql94 # enable when Postgis will be available
# scripts that are called at very beginning, before repo cloning
init:
@@ -31,23 +35,36 @@ platform: x64
install:
# by default, all script lines are interpreted as batch
- cd c:\
+ - if exist libs ( rmdir c:\libs /s /q )
- mkdir libs
- - curl -O https://dl.dropboxusercontent.com/u/63393258/libs_osm2pgsql_%Configuration%.7z
- - 7z x libs_osm2pgsql_%Configuration%.7z | find ":"
+ - if not exist osm2pgsql_win_deps_release.7z ( curl -O http://lonvia.dev.openstreetmap.org/osm2pgsql-winbuild/osm2pgsql_win_deps_release.7z )
+ - 7z x osm2pgsql_win_deps_release.7z | find ":"
+ - cd c:\libs
+ - echo Downloading and installing PostGIS:
+ - if not exist %POSTGIS_FILE%.zip ( curl -L -O -S -s http://lonvia.dev.openstreetmap.org/osm2pgsql-winbuild/%POSTGIS_FILE%.zip )
+ - 7z x %POSTGIS_FILE%.zip
+ - echo xcopy /e /y /q %POSTGIS_FILE% %PSQL_ROOT%
+ - xcopy /e /y /q %POSTGIS_FILE% "%PSQL_ROOT%"
+ - echo Creating tablespace for tablespace test...
+ - mkdir temp
+ - cacls temp /T /E /G Users:F
+ - cacls temp /T /E /G "Network Service":F
+ - echo Installing psycopg2 Python module...
+ - python -V
+ - pip install psycopg2
build_script:
- mkdir c:\osm2pgsql\build
- cd c:\osm2pgsql\build
- echo Running cmake...
- call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64
- - cmake .. -LA -G "NMake Makefiles" -DBOOST_ROOT=%BOOST_ROOT% -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_INSTALL_PREFIX=%PREFIX% -DBoost_USE_STATIC_LIBS=ON -DBUILD_TESTS=ON -DBoost_ADDITIONAL_VERSIONS=1.57;1.58;1.59
+ - cmake .. -LA -G "NMake Makefiles" -DBOOST_ROOT=%BOOST_ROOT% -DCMAKE_BUILD_TYPE=%Configuration% -DCMAKE_INSTALL_PREFIX=%PREFIX% -DBoost_USE_STATIC_LIBS=ON -DBUILD_TESTS=ON
- nmake
- mkdir osm2pgsql-bin
- copy /y *.exe osm2pgsql-bin
- copy /y ..\*.style osm2pgsql-bin
- copy /y ..\*.lua osm2pgsql-bin
- - copy /y %PREFIX%\bin\lua.dll osm2pgsql-bin
- - copy /y %PREFIX%\bin\geos.dll osm2pgsql-bin
+ - copy /y %PREFIX%\bin\*.dll osm2pgsql-bin
- copy /y "%PSQL_ROOT%\bin\libpq.dll" osm2pgsql-bin
- copy /y "%PSQL_ROOT%\bin\libintl-8.dll" osm2pgsql-bin
- copy /y "%PSQL_ROOT%\bin\libeay32.dll" osm2pgsql-bin
@@ -55,14 +72,14 @@ build_script:
- 7z a c:\osm2pgsql\osm2pgsql_%Configuration%.zip osm2pgsql-bin -tzip
test_script:
+ - |
+ "%PSQL_ROOT%/bin/psql" -c "CREATE TABLESPACE tablespacetest LOCATION 'c:/libs/temp'"
- set PATH=c:\osm2pgsql\build\osm2pgsql-bin;%PATH%
+ - set PROJ_LIB=c:\libs\share
- cd c:\osm2pgsql\build
- - ctest -VV -L NoDB
-# - ctest -VV -LE FlatNodes # enable when Postgis will be available
+# - ctest -VV -L NoDB
+ - ctest -VV -LE FlatNodes # enable when Postgis will be available
-#deploy_script:
-# - cd c:\build
-# - curl -T osm2pgsql_%Configuration%.zip --user %ACCOUNT% https://webdav.yandex.ru/libs/osm2pgsql_%Configuration%.zip
artifacts:
- path: osm2pgsql_Release.zip
diff --git a/docs/lua.md b/docs/lua.md
index 6ebf9da..bb666f7 100644
--- a/docs/lua.md
+++ b/docs/lua.md
@@ -8,7 +8,7 @@ This allows you to unify disparate tagging (for example, `highway=path; foot=yes
Pass a Lua script to osm2pgsql using the command line switch `--tag-transform-script`:
- osm2pgsql -S your.style --tag-transform-script your.lua --hstore-all extract.osm.pbf
+ osm2pgsql -d gis -S your.style --tag-transform-script your.lua --hstore-all extract.osm.pbf
This Lua script needs to implement the following functions:
diff --git a/docs/migrations.md b/docs/migrations.md
index 0aa0fb6..001a335 100644
--- a/docs/migrations.md
+++ b/docs/migrations.md
@@ -7,6 +7,11 @@ the default `planet_osm` prefix.
It is frequently better to reimport as this will also recluster the tables and
remove table or index bloat.
+## 0.93 unprojected slim coordinates ##
+
+The method of storing coordinates in the slim tables has changed. There is no
+migration and a reload is required.
+
## 0.91 default projection ##
The default projection was moved from 900913 to 3857. This does not effect
diff --git a/docs/osm2pgsql.1 b/docs/osm2pgsql.1
index b0fac79..2bcea4a 100644
--- a/docs/osm2pgsql.1
+++ b/docs/osm2pgsql.1
@@ -1,4 +1,4 @@
-.TH OSM2PGSQL 1 "October 31, 2016"
+.TH OSM2PGSQL 1 "February 5, 2017"
.\" Please adjust this date whenever revising the manpage.
.SH NAME
osm2pgsql \- Openstreetmap data to PostgreSQL converter.
@@ -53,8 +53,7 @@ Remove existing data from the database. This is the
default if \fB\-\-append\fR is not specified.
.TP
\fB\-d\fR|\-\-database name
-The name of the PostgreSQL database to connect
-to (default: gis).
+The name of the PostgreSQL database to connect to.
.TP
\fB\-i\fR|\-\-tablespace\-index tablespacename
Store all indices in a separate PostgreSQL tablespace named by this parameter.
diff --git a/docs/pgsql.md b/docs/pgsql.md
index afc1a4f..620fa1d 100644
--- a/docs/pgsql.md
+++ b/docs/pgsql.md
@@ -1,8 +1,8 @@
# Pgsql Backend #
The pgsql backend is designed for rendering OpenStreetMap data, principally
-with Mapnik, but is also useful for [analysis](docs/analysis.md) and
-[exporting](docs/exporting.md) to other formats.
+with Mapnik, but is also useful for [analysis](analysis.md) and
+[exporting](export.md) to other formats.
## Database Layout ##
It connects to a PostgreSQL database and stores the data in four tables
diff --git a/docs/usage.md b/docs/usage.md
index 50fd1af..3a85dfa 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -45,8 +45,8 @@ that only impact performance.
## Database options ##
osm2pgsql supports standard options for how to connect to PostgreSQL. If left
-unset, it will attempt to connect to the ``gis`` database using a unix socket.
-Most usage only requires setting ``--database``.
+unset, it will attempt to connect to the default database (usually the username)
+using a unix socket. Most usage only requires setting ``--database``.
``--tablespace`` options allow the location of main and slim tables and indexes
to be set to different tablespaces independently, typically on machines with
@@ -144,6 +144,3 @@ reimporting the database, at the cost of a
* ``--keep-coastlines`` disables a hard-coded rule that would otherwise
discard ``natural=coastline`` ways.
-
-* ``--exclude-invalid-polygon`` prevents osm2pgsql from attempting to form
- valid polygons from invalid ones and just rejects the invalid ones.
diff --git a/expire-tiles.cpp b/expire-tiles.cpp
index 40cbaa5..86a285d 100644
--- a/expire-tiles.cpp
+++ b/expire-tiles.cpp
@@ -16,178 +16,95 @@
#include "expire-tiles.hpp"
#include "options.hpp"
-#include "geometry-builder.hpp"
#include "reprojection.hpp"
#include "table.hpp"
+#include "wkb.hpp"
#define EARTH_CIRCUMFERENCE 40075016.68
#define HALF_EARTH_CIRCUMFERENCE (EARTH_CIRCUMFERENCE / 2)
#define TILE_EXPIRY_LEEWAY 0.1 /* How many tiles worth of space to leave either side of a changed feature */
-/*
- * We store the dirty tiles in an in-memory tree during runtime
- * and dump them out to a file at the end. This allows us to easilly drop
- * duplicate tiles from the output.
- *
- * This data structure consists of a node, representing a tile at zoom level 0,
- * which contains 4 pointers to nodes representing each of the child tiles at
- * zoom level 1, and so on down the the zoom level specified in
- * Options->expire_tiles_zoom.
- *
- * The memory allowed to this structure is not capped, but daily deltas
- * generally produce a few hundred thousand expired tiles at zoom level 17,
- * which are easilly accommodated.
- */
-
-
-struct tile_output_file : public expire_tiles::tile_output
+tile_output_t::tile_output_t(const char *filename)
+: outfile(fopen(filename, "a"))
{
- tile_output_file(const char *expire_tiles_filename, int zmin)
- : outcount(0), min_zoom(zmin), outfile(fopen(expire_tiles_filename, "a"))
- {
- if (outfile == nullptr) {
- fprintf(stderr, "Failed to open expired tiles file (%s). Tile expiry list will not be written!\n", strerror(errno));
- }
- }
-
- ~tile_output_file() {
- if (outfile) {
- fclose(outfile);
- }
- }
-
- void output_dirty_tile(int x, int y, int zoom) override
- {
if (outfile == nullptr) {
- return;
+ fprintf(stderr, "Failed to open expired tiles file (%s). Tile expiry "
+ "list will not be written!\n",
+ strerror(errno));
}
+}
- int out_zoom = std::max(zoom, min_zoom);
- int zoom_diff = out_zoom - zoom;
- int x_max = (x + 1) << zoom_diff;
- int y_max = (y + 1) << zoom_diff;
-
- for (int x_iter = x << zoom_diff; x_iter < x_max; ++x_iter) {
- for (int y_iter = y << zoom_diff; y_iter < y_max; ++y_iter) {
- ++outcount;
- if ((outcount % 1000) == 0) {
- fprintf(stderr, "\rWriting dirty tile list (%iK)", outcount / 1000);
- }
- fprintf(outfile, "%i/%i/%i\n", out_zoom, x_iter, y_iter);
- }
- }
- }
-
-private:
- int outcount;
- int min_zoom;
- FILE *outfile;
-};
-
-
-int tile::mark_tile(int x, int y, int zoom, int this_zoom)
+tile_output_t::~tile_output_t()
{
- int zoom_diff = zoom - this_zoom - 1;
- int sub = ((x >> zoom_diff) & 1) << 1 | ((y >> zoom_diff) & 1);
-
- if (!complete[sub]) {
- if (zoom_diff <= 0) {
- complete[sub] = 1;
- subtiles[sub].reset();
- } else {
- if (!subtiles[sub])
- subtiles[sub].reset(new tile);
- int done = subtiles[sub]->mark_tile(x, y, zoom, this_zoom + 1);
- if (done >= 4) {
- complete[sub] = 1;
- subtiles[sub].reset();
- }
- }
+ if (outfile) {
+ fclose(outfile);
}
-
- return num_complete();
}
-void tile::output_and_destroy(expire_tiles::tile_output *output,
- int x, int y, int this_zoom)
+void tile_output_t::output_dirty_tile(uint32_t x, uint32_t y, uint32_t zoom)
{
- int sub_x = x << 1;
- int sub_y = y << 1;
-
- for (int i = 0; i < 4; ++i) {
- if (complete[i]) {
- output->output_dirty_tile(sub_x + sub2x(i), sub_y + sub2y(i),
- this_zoom + 1);
- }
- if (subtiles[i]) {
- subtiles[i]->output_and_destroy(output,
- sub_x + sub2x(i), sub_y + sub2y(i),
- this_zoom + 1);
- subtiles[i].reset();
+ if (outfile) {
+ fprintf(outfile, "%i/%i/%i\n", zoom, x, y);
+ ++outcount;
+ if (outcount % 1000 == 0) {
+ fprintf(stderr, "\rWriting dirty tile list (%iK)", outcount / 1000);
}
}
}
-int tile::merge(tile *other)
+void expire_tiles::output_and_destroy(const char *filename, uint32_t minzoom)
{
- for (int i = 0; i < 4; ++i) {
- // if other is complete, then the merge tree must be complete too
- if (other->complete[i]) {
- complete[i] = 1;
- subtiles[i].reset();
- // if our subtree is complete don't bother moving anything
- } else if (!complete[i]) {
- if (subtiles[i]) {
- if (other->subtiles[i]) {
- int done = subtiles[i]->merge(other->subtiles[i].get());
- if (done >= 4) {
- complete[i] = 1;
- subtiles[i].reset();
- }
- }
- } else {
- subtiles[i] = std::move(other->subtiles[i]);
- }
- }
- other->subtiles[i].reset();
- }
-
- return num_complete();
+ tile_output_t output_writer(filename);
+ output_and_destroy<tile_output_t>(output_writer, minzoom);
}
-void expire_tiles::output_and_destroy(tile_output *output)
+expire_tiles::expire_tiles(uint32_t max, double bbox,
+ const std::shared_ptr<reprojection> &proj)
+: max_bbox(bbox), maxzoom(max), projection(proj)
{
- if (!dirty)
- return;
-
- dirty->output_and_destroy(output, 0, 0, 0);
- dirty.reset();
+ map_width = 1 << maxzoom;
+ tile_width = EARTH_CIRCUMFERENCE / map_width;
+ last_tile_x = static_cast<uint32_t>(map_width) + 1;
+ last_tile_y = static_cast<uint32_t>(map_width) + 1;
}
-void expire_tiles::output_and_destroy(const char *filename, int minzoom)
+uint64_t expire_tiles::xy_to_quadkey(uint32_t x, uint32_t y, uint32_t zoom)
{
- if (maxzoom >= 0) {
- tile_output_file output(filename, minzoom);
-
- output_and_destroy(&output);
- }
+ uint64_t quadkey = 0;
+ // the two highest bits are the bits of zoom level 1, the third and fourth bit are level 2, …
+ for (uint32_t z = 0; z < zoom; z++) {
+ quadkey |= ((x & (1ULL << z)) << z);
+ quadkey |= ((y & (1ULL << z)) << (z + 1));
+ }
+ return quadkey;
}
-expire_tiles::expire_tiles(int max, double bbox, const std::shared_ptr<reprojection> &proj)
-: max_bbox(bbox), maxzoom(max), projection(proj)
+xy_coord_t expire_tiles::quadkey_to_xy(uint64_t quadkey_coord, uint32_t zoom)
{
- if (maxzoom >= 0) {
- map_width = 1 << maxzoom;
- tile_width = EARTH_CIRCUMFERENCE / map_width;
+ xy_coord_t result;
+ for (uint32_t z = zoom; z > 0; --z) {
+ /* The quadkey contains Y and X bits interleaved in following order: YXYX...
+ * We have to pick out the bit representing the y/x bit of the current zoom
+ * level and then shift it back to the right on its position in a y-/x-only
+ * coordinate.*/
+ result.y = result.y + static_cast<uint32_t>(
+ (quadkey_coord & (1ULL << (2 * z - 1))) >> z);
+ result.x = result.x +
+ static_cast<uint32_t>(
+ (quadkey_coord & (1ULL << (2 * (z - 1)))) >> (z - 1));
}
+ return result;
}
-void expire_tiles::expire_tile(int x, int y)
+void expire_tiles::expire_tile(uint32_t x, uint32_t y)
{
- if (!dirty)
- dirty.reset(new tile);
-
- dirty->mark_tile(x, y, maxzoom, 0);
+ // Only try to insert to tile into the set if the last inserted tile
+ // is different from this tile.
+ if (last_tile_x != x || last_tile_y != y) {
+ m_dirty_tiles.insert(xy_to_quadkey(x, y, maxzoom));
+ last_tile_x = x;
+ last_tile_y = y;
+ }
}
int expire_tiles::normalise_tile_x_coord(int x) {
@@ -296,7 +213,7 @@ int expire_tiles::from_bbox(double min_lon, double min_lat, double max_lon, doub
double tmp_x;
double tmp_y;
- if (maxzoom < 0) return 0;
+ if (maxzoom == 0) return 0;
width = max_lon - min_lon;
height = max_lat - min_lat;
@@ -333,70 +250,114 @@ int expire_tiles::from_bbox(double min_lon, double min_lat, double max_lon, doub
return 0;
}
-void expire_tiles::from_nodes_line(const nodelist_t &nodes)
-{
- if (maxzoom < 0 || nodes.empty())
- return;
- if (nodes.size() == 1) {
- from_bbox(nodes[0].lon, nodes[0].lat, nodes[0].lon, nodes[0].lat);
- } else {
- for (size_t i = 1; i < nodes.size(); ++i)
- from_line(nodes[i-1].lon, nodes[i-1].lat, nodes[i].lon, nodes[i].lat);
- }
-}
-
-/*
- * Calculate a bounding box from a list of nodes and expire all tiles within it
- */
-void expire_tiles::from_nodes_poly(const nodelist_t &nodes, osmid_t osm_id)
+void expire_tiles::from_wkb(const char *wkb, osmid_t osm_id)
{
- if (maxzoom < 0 || nodes.empty())
+ if (maxzoom == 0) {
return;
-
- double min_lon = nodes[0].lon;
- double min_lat = nodes[0].lat;
- double max_lon = nodes[0].lon;
- double max_lat = nodes[0].lat;
-
- for (size_t i = 1; i < nodes.size(); ++i) {
- if (nodes[i].lon < min_lon) min_lon = nodes[i].lon;
- if (nodes[i].lat < min_lat) min_lat = nodes[i].lat;
- if (nodes[i].lon > max_lon) max_lon = nodes[i].lon;
- if (nodes[i].lat > max_lat) max_lat = nodes[i].lat;
}
- if (from_bbox(min_lon, min_lat, max_lon, max_lat)) {
- /* Bounding box too big - just expire tiles on the line */
- fprintf(stderr, "\rLarge polygon (%.0f x %.0f metres, OSM ID %" PRIdOSMID ") - only expiring perimeter\n", max_lon - min_lon, max_lat - min_lat, osm_id);
- from_nodes_line(nodes);
+ auto parse = ewkb::parser_t(wkb);
+
+ switch (parse.read_header()) {
+ case ewkb::wkb_point:
+ from_wkb_point(&parse);
+ break;
+ case ewkb::wkb_line:
+ from_wkb_line(&parse);
+ break;
+ case ewkb::wkb_polygon:
+ from_wkb_polygon(&parse, osm_id);
+ break;
+ case ewkb::wkb_multi_line: {
+ auto num = parse.read_length();
+ for (unsigned i = 0; i < num; ++i) {
+ parse.read_header();
+ from_wkb_line(&parse);
+ }
+ break;
+ }
+ case ewkb::wkb_multi_polygon: {
+ auto num = parse.read_length();
+ for (unsigned i = 0; i < num; ++i) {
+ parse.read_header();
+ from_wkb_polygon(&parse, osm_id);
+ }
+ break;
+ }
+ default:
+ fprintf(stderr, "OSM id %" PRIdOSMID
+ ": Unknown geometry type, cannot expire.\n",
+ osm_id);
}
}
-void expire_tiles::from_xnodes_poly(const multinodelist_t &xnodes, osmid_t osm_id)
+void expire_tiles::from_wkb_point(ewkb::parser_t *wkb)
{
- for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it)
- from_nodes_poly(*it, osm_id);
+ auto c = wkb->read_point();
+ from_bbox(c.x, c.y, c.x, c.y);
}
-void expire_tiles::from_xnodes_line(const multinodelist_t &xnodes)
+void expire_tiles::from_wkb_line(ewkb::parser_t *wkb)
{
- for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it)
- from_nodes_line(*it);
+ auto sz = wkb->read_length();
+
+ if (sz == 0) {
+ return;
+ }
+
+ if (sz == 1) {
+ from_wkb_point(wkb);
+ } else {
+ auto prev = wkb->read_point();
+ for (size_t i = 1; i < sz; ++i) {
+ auto cur = wkb->read_point();
+ from_line(prev.x, prev.y, cur.x, cur.y);
+ prev = cur;
+ }
+ }
}
-void expire_tiles::from_wkb(const char* wkb, osmid_t osm_id)
+void expire_tiles::from_wkb_polygon(ewkb::parser_t *wkb, osmid_t osm_id)
{
- if (maxzoom < 0) return;
-
- multinodelist_t xnodes;
- bool polygon;
+ auto num_rings = wkb->read_length();
+ assert(num_rings > 0);
+
+ auto start = wkb->save_pos();
+
+ auto num_pt = wkb->read_length();
+ auto initpt = wkb->read_point();
+
+ osmium::geom::Coordinates min{initpt}, max{initpt};
+
+ for (size_t i = 1; i < num_pt; ++i) {
+ auto c = wkb->read_point();
+ if (c.x < min.x)
+ min.x = c.x;
+ if (c.y < min.y)
+ min.y = c.y;
+ if (c.x > max.x)
+ max.x = c.x;
+ if (c.y > max.y)
+ max.y = c.y;
+ }
- if (geometry_builder::parse_wkb(wkb, xnodes, &polygon) == 0) {
- if (polygon)
- from_xnodes_poly(xnodes, osm_id);
- else
- from_xnodes_line(xnodes);
+ if (from_bbox(min.x, min.y, max.x, max.y)) {
+ /* Bounding box too big - just expire tiles on the line */
+ fprintf(stderr,
+ "\rLarge polygon (%.0f x %.0f metres, OSM ID %" PRIdOSMID
+ ") - only expiring perimeter\n",
+ max.x - min.x, max.y - min.y, osm_id);
+ wkb->rewind(start);
+ for (unsigned ring = 0; ring < num_rings; ++ring) {
+ from_wkb_line(wkb);
+ }
+ } else {
+ // ignore inner rings
+ for (unsigned ring = 1; ring < num_rings; ++ring) {
+ auto inum_pt = wkb->read_length();
+ wkb->skip_points(inum_pt);
+ }
}
}
@@ -412,7 +373,7 @@ void expire_tiles::from_wkb(const char* wkb, osmid_t osm_id)
*/
int expire_tiles::from_db(table_t* table, osmid_t osm_id) {
//bail if we dont care about expiry
- if (maxzoom < 0)
+ if (maxzoom == 0)
return -1;
//grab the geom for this id
@@ -420,8 +381,10 @@ int expire_tiles::from_db(table_t* table, osmid_t osm_id) {
//dirty the stuff
const char* wkb = nullptr;
- while((wkb = wkbs.get_next()))
- from_wkb(wkb, osm_id);
+ while ((wkb = wkbs.get_next())) {
+ auto binwkb = ewkb::parser_t::wkb_from_hex(wkb);
+ from_wkb(binwkb.c_str(), osm_id);
+ }
//return how many rows were affected
return wkbs.get_count();
@@ -429,28 +392,28 @@ int expire_tiles::from_db(table_t* table, osmid_t osm_id) {
void expire_tiles::merge_and_destroy(expire_tiles &other)
{
- if (!other.dirty) {
- return;
- }
-
- if (map_width != other.map_width) {
- throw std::runtime_error((boost::format("Unable to merge tile expiry sets when "
- "map_width does not match: %1% != %2%.")
- % map_width % other.map_width).str());
- }
-
- if (tile_width != other.tile_width) {
- throw std::runtime_error((boost::format("Unable to merge tile expiry sets when "
- "tile_width does not match: %1% != %2%.")
- % tile_width % other.tile_width).str());
- }
-
-
- if (!dirty) {
- dirty = std::move(other.dirty);
- } else {
- dirty->merge(other.dirty.get());
- }
-
- other.dirty.reset();
+ if (map_width != other.map_width) {
+ throw std::runtime_error(
+ (boost::format("Unable to merge tile expiry sets when "
+ "map_width does not match: %1% != %2%.") %
+ map_width % other.map_width)
+ .str());
+ }
+
+ if (tile_width != other.tile_width) {
+ throw std::runtime_error(
+ (boost::format("Unable to merge tile expiry sets when "
+ "tile_width does not match: %1% != %2%.") %
+ tile_width % other.tile_width)
+ .str());
+ }
+
+ if (m_dirty_tiles.size() == 0) {
+ m_dirty_tiles = std::move(other.m_dirty_tiles);
+ } else {
+ m_dirty_tiles.insert(other.m_dirty_tiles.begin(),
+ other.m_dirty_tiles.end());
+ }
+
+ other.m_dirty_tiles.clear();
}
diff --git a/expire-tiles.hpp b/expire-tiles.hpp
index 840e4fd..ceb8b3f 100644
--- a/expire-tiles.hpp
+++ b/expire-tiles.hpp
@@ -2,82 +2,193 @@
#define EXPIRE_TILES_H
#include <memory>
+#include <unordered_set>
#include "osmtypes.hpp"
class reprojection;
class table_t;
class tile;
+namespace ewkb {
+class parser_t;
+}
+
+/**
+ * \brief Simple struct for the x and y index of a tile ID.
+ */
+struct xy_coord_t
+{
+ uint32_t x;
+ uint32_t y;
+ xy_coord_t() : x(0), y(0) {}
+};
+
+/**
+ * Implementation of the output of the tile expiry list to a file.
+ */
+class tile_output_t
+{
+ FILE *outfile;
+ uint32_t outcount = 0;
+
+public:
+ tile_output_t(const char *filename);
+
+ ~tile_output_t();
+
+ /**
+ * Output dirty tile.
+ *
+ * \param x x index
+ * \param y y index
+ * \param zoom zoom level of the tile
+ */
+ void output_dirty_tile(uint32_t x, uint32_t y, uint32_t zoom);
+};
struct expire_tiles
{
- expire_tiles(int maxzoom, double maxbbox,
+ expire_tiles(uint32_t maxzoom, double maxbbox,
const std::shared_ptr<reprojection> &projection);
int from_bbox(double min_lon, double min_lat, double max_lon, double max_lat);
- void from_nodes_line(const nodelist_t &nodes);
- void from_nodes_poly(const nodelist_t &nodes, osmid_t osm_id);
void from_wkb(const char* wkb, osmid_t osm_id);
int from_db(table_t* table, osmid_t osm_id);
- /* customisable tile output. this can be passed into the
- * `output_and_destroy` function to override output to a file.
- * this is primarily useful for testing.
+ /**
+ * Write the list of expired tiles to a file.
+ *
+ * You will probably use tile_output_t as template argument for production code
+ * and another class which does not write to a file for unit tests.
+ *
+ * \param filename name of the file
+ * \param minzoom minimum zoom level
*/
- struct tile_output {
- virtual ~tile_output() = default;
- // dirty a tile at x, y & zoom, and all descendants of that
- // tile at the given zoom if zoom < min_zoom.
- virtual void output_dirty_tile(int x, int y, int zoom) = 0;
- };
-
- // output the list of expired tiles to a file. note that this
- // consumes the list of expired tiles destructively.
- void output_and_destroy(const char *filename, int minzoom);
-
- // output the list of expired tiles using a `tile_output`
- // functor. this consumes the list of expired tiles destructively.
- void output_and_destroy(tile_output *output);
+ void output_and_destroy(const char *filename, uint32_t minzoom);
+
+ /**
+ * Output expired tiles on all requested zoom levels.
+ *
+ * \tparam TILE_WRITER class which implements the method
+ * output_dirty_tile(uint32_t x, uint32_t y, uint32_t zoom) which usually writes the tile ID to a file
+ * (production code) or does something else (usually unit tests)
+ *
+ * \param minzoom minimum zoom level
+ */
+ template <class TILE_WRITER>
+ void output_and_destroy(TILE_WRITER &output_writer, uint32_t minzoom)
+ {
+ assert(minzoom <= maxzoom);
+ // build a sorted vector of all expired tiles
+ std::vector<uint64_t> tiles_maxzoom(m_dirty_tiles.begin(),
+ m_dirty_tiles.end());
+ std::sort(tiles_maxzoom.begin(), tiles_maxzoom.end());
+ /* Loop over all requested zoom levels (from maximum down to the minimum zoom level).
+ * Tile IDs of the tiles enclosing this tile at lower zoom levels are calculated using
+ * bit shifts.
+ *
+ * last_quadkey is initialized with a value which is not expected to exist
+ * (larger than largest possible quadkey). */
+ uint64_t last_quadkey = 1ULL << (2 * maxzoom);
+ for (std::vector<uint64_t>::const_iterator it = tiles_maxzoom.cbegin();
+ it != tiles_maxzoom.cend(); ++it) {
+ for (uint32_t dz = 0; dz <= maxzoom - minzoom; dz++) {
+ // scale down to the current zoom level
+ uint64_t qt_current = *it >> (dz * 2);
+ /* If dz > 0, there are propably multiple elements whose quadkey
+ * is equal because they are all sub-tiles of the same tile at the current
+ * zoom level. We skip all of them after we have written the first sibling.
+ */
+ if (qt_current == last_quadkey >> (dz * 2)) {
+ continue;
+ }
+ xy_coord_t xy = quadkey_to_xy(qt_current, maxzoom - dz);
+ output_writer.output_dirty_tile(xy.x, xy.y, maxzoom - dz);
+ }
+ last_quadkey = *it;
+ }
+ }
- // merge the list of expired tiles in the other object into this
- // object, destroying the list in the other object.
- void merge_and_destroy(expire_tiles &);
+ /**
+ * merge the list of expired tiles in the other object into this
+ * object, destroying the list in the other object.
+ */
+ void merge_and_destroy(expire_tiles &other);
+
+ /**
+ * Helper method to convert a tile ID (x and y) into a quadkey
+ * using bitshifts.
+ *
+ * Quadkeys are interleaved this way: YXYX…
+ *
+ * \param x x index
+ * \param y y index
+ * \param zoom zoom level
+ * \returns quadtree ID as integer
+ */
+ static uint64_t xy_to_quadkey(uint32_t x, uint32_t y, uint32_t zoom);
+
+ /**
+ * Convert a quadkey into a tile ID (x and y) using bitshifts.
+ *
+ * Quadkeys coordinates are interleaved this way: YXYX…
+ *
+ * \param quadkey the quadkey to be converted
+ * \param zoom zoom level
+ */
+ static xy_coord_t quadkey_to_xy(uint64_t quadkey, uint32_t zoom);
private:
- void expire_tile(int x, int y);
+ /**
+ * Expire a single tile.
+ *
+ * \param x x index of the tile to be expired.
+ * \param y y index of the tile to be expired.
+ */
+ void expire_tile(uint32_t x, uint32_t y);
int normalise_tile_x_coord(int x);
void from_line(double lon_a, double lat_a, double lon_b, double lat_b);
- void from_xnodes_poly(const multinodelist_t &xnodes, osmid_t osm_id);
- void from_xnodes_line(const multinodelist_t &xnodes);
+
+ void from_wkb_point(ewkb::parser_t *wkb);
+ void from_wkb_line(ewkb::parser_t *wkb);
+ void from_wkb_polygon(ewkb::parser_t *wkb, osmid_t osm_id);
double tile_width;
double max_bbox;
int map_width;
- int maxzoom;
+ uint32_t maxzoom;
std::shared_ptr<reprojection> projection;
- std::unique_ptr<tile> dirty;
-};
-
-
-class tile
-{
-public:
- int mark_tile(int x, int y, int zoom, int this_zoom);
- void output_and_destroy(expire_tiles::tile_output *output,
- int x, int y, int this_zoom);
- int merge(tile *other);
-private:
- int sub2x(int sub) const { return sub >> 1; }
- int sub2y(int sub) const { return sub & 1; }
-
- int num_complete() const
- {
- return complete[0] + complete[1] + complete[2] + complete[3];
- }
+ /**
+ * x coordinate of the tile which has been added as last tile to the unordered set
+ */
+ uint32_t last_tile_x;
- std::unique_ptr<tile> subtiles[4];
- char complete[4] = {0, 0, 0, 0};
+ /**
+ * y coordinate of the tile which has been added as last tile to the unordered set
+ */
+ uint32_t last_tile_y;
+
+ /**
+ * manages which tiles have been marked as empty
+ *
+ * This set stores the IDs of the tiles at the maximum zoom level. We don't
+ * store the IDs of the expired tiles of lower zoom levels. They are calculated
+ * on the fly at the end.
+ *
+ * Tile IDs are converted into so-called quadkeys as used by Bing Maps.
+ * https://msdn.microsoft.com/en-us/library/bb259689.aspx
+ * A quadkey is generated by interleaving the x and y index in following order:
+ * YXYX...
+ *
+ * Example:
+ * x = 3 = 0b011, y = 5 = 0b101
+ * results in the quadkey 0b100111.
+ *
+ * Bing Maps itself uses the quadkeys as a base-4 number converted to a string.
+ * We interpret this IDs as simple 64-bit integers due to performance reasons.
+ */
+ std::unordered_set<uint64_t> m_dirty_tiles;
};
#endif
diff --git a/geometry-builder.cpp b/geometry-builder.cpp
deleted file mode 100644
index e978341..0000000
--- a/geometry-builder.cpp
+++ /dev/null
@@ -1,739 +0,0 @@
-/*
-#-----------------------------------------------------------------------------
-# Part of osm2pgsql utility
-#-----------------------------------------------------------------------------
-# By Artem Pavlenko, Copyright 2007
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#-----------------------------------------------------------------------------
-*/
-
-#include <iostream>
-#include <algorithm>
-#include <cmath>
-#include <cstddef>
-#include <sstream>
-#include <stdexcept>
-#include <memory>
-#include <new>
-#include <numeric>
-
-#if defined(__CYGWIN__)
-#define GEOS_INLINE
-#endif
-
-#include <geos/geom/prep/PreparedGeometry.h>
-#include <geos/geom/prep/PreparedGeometryFactory.h>
-#include <geos/geom/GeometryFactory.h>
-#include <geos/geom/Coordinate.h>
-#include <geos/geom/CoordinateSequence.h>
-#include <geos/geom/CoordinateSequenceFactory.h>
-#include <geos/geom/Geometry.h>
-#include <geos/geom/GeometryCollection.h>
-#include <geos/geom/LineString.h>
-#include <geos/geom/LinearRing.h>
-#include <geos/geom/MultiLineString.h>
-#include <geos/geom/Polygon.h>
-#include <geos/geom/MultiPolygon.h>
-#include <geos/io/WKBReader.h>
-#include <geos/io/WKBWriter.h>
-#include <geos/util/GEOSException.h>
-#include <geos/opLinemerge.h>
-using namespace geos::geom;
-using namespace geos::util;
-using namespace geos::operation::linemerge;
-
-#include "geometry-builder.hpp"
-#include "reprojection.hpp"
-
-typedef std::unique_ptr<Geometry> geom_ptr;
-typedef std::unique_ptr<CoordinateSequence> coord_ptr;
-
-namespace {
-
-void coords2nodes(CoordinateSequence * coords, nodelist_t &nodes)
-{
- size_t num_coords = coords->getSize();
- nodes.reserve(num_coords);
-
- for (size_t i = 0; i < num_coords; i++) {
- Coordinate coord = coords->getAt(i);
- nodes.push_back(osmNode(coord.x, coord.y));
- }
-}
-
-coord_ptr nodes2coords(GeometryFactory &gf, const nodelist_t &nodes)
-{
- coord_ptr coords(gf.getCoordinateSequenceFactory()->create(size_t(0), size_t(2)));
-
- for (const auto& nd: nodes) {
- coords->add(Coordinate(nd.lon, nd.lat), 0);
- }
-
- return coords;
-}
-
-geom_ptr create_multi_line(GeometryFactory &gf, const multinodelist_t &xnodes)
-{
- // XXX leaks memory if an exception is thrown
- std::unique_ptr<std::vector<Geometry*> > lines(new std::vector<Geometry*>);
- lines->reserve(xnodes.size());
-
- for (const auto& nodes: xnodes) {
- auto coords = nodes2coords(gf, nodes);
- if (coords->getSize() > 1) {
- lines->push_back(gf.createLineString(coords.release()));
- }
- }
-
- return geom_ptr(gf.createMultiLineString(lines.release()));
-}
-
-bool is_polygon_line(CoordinateSequence * coords)
-{
- return (coords->getSize() >= 4)
- && (coords->getAt(coords->getSize() - 1).equals2D(coords->getAt(0)));
-}
-
-/**
- * Reprojects given Linear Ring from target projection to spherical mercator.
- * Caller takes ownership of return value.
- */
-LinearRing* reproject_linearring(const LineString *ls, const reprojection *proj)
-{
- auto *gf = ls->getFactory();
- coord_ptr coords(gf->getCoordinateSequenceFactory()->create(size_t(0), size_t(2)));
- for (auto i : *(ls->getCoordinatesRO()->toVector())) {
- Coordinate c(i.x, i.y);
- proj->target_to_tile(&c.y, &c.x);
- coords->add(c);
- }
- return gf->createLinearRing(coords.release());
-}
-
-
-/**
- * Computes area of given polygonal geometry.
- * \return the area in projected units, or in EPSG 3857 if area reprojection is enabled
- */
-double get_area(const geos::geom::Geometry *geom, reprojection *proj)
-{
- // reprojection is not necessary, or has not been asked for.
- if (!proj) {
- return geom->getArea();
- }
-
- // MultiPolygon - return sum of individual areas
- if (const auto *multi = dynamic_cast<const MultiPolygon *>(geom)) {
- return std::accumulate(multi->begin(), multi->end(), 0.0,
- [=](double a, const Geometry *g) {
- return a + get_area(g, proj);
- });
- }
-
- const auto *poly = dynamic_cast<const geos::geom::Polygon *>(geom);
- if (!poly) {
- return 0.0;
- }
-
- // standard polygon - reproject rings individually, then assemble polygon and
- // compute area.
-
- const auto *ext = poly->getExteriorRing();
- std::unique_ptr<LinearRing> projectedExt(reproject_linearring(ext, proj));
- auto nholes = poly->getNumInteriorRing();
- std::unique_ptr<std::vector<Geometry *> > projectedHoles(new std::vector<Geometry *>);
- for (std::size_t i=0; i < nholes; i++) {
- auto* hole = poly->getInteriorRingN(i);
- projectedHoles->push_back(reproject_linearring(hole, proj));
- }
- const geom_ptr projectedPoly(poly->getFactory()->createPolygon(projectedExt.release(), projectedHoles.release()));
-
- return projectedPoly->getArea();
-}
-
-
-struct polygondata
-{
- std::unique_ptr<Polygon> polygon;
- std::unique_ptr<LinearRing> ring;
- double area;
- bool iscontained;
- unsigned containedbyid;
-
- polygondata(std::unique_ptr<Polygon> p, LinearRing* r, double a)
- : polygon(std::move(p)), ring(r), area(a),
- iscontained(false), containedbyid(0)
- {}
-};
-
-struct polygondata_comparearea {
- bool operator()(const polygondata& lhs, const polygondata& rhs) {
- return lhs.area > rhs.area;
- }
-};
-
-} // anonymous namespace
-
-
-void geometry_builder::pg_geom_t::set(const geos::geom::Geometry *g, bool poly,
- reprojection *p)
-{
- geos::io::WKBWriter writer(2, getMachineByteOrder(), true);
- std::stringstream stream(std::ios_base::out);
- writer.writeHEX(*g, stream);
- geom = stream.str();
-
- if (valid()) {
- area = poly ? get_area(g, p) : 0;
- polygon = poly;
- }
-}
-
-geom_ptr geometry_builder::create_simple_poly(GeometryFactory &gf,
- std::unique_ptr<CoordinateSequence> coords) const
-{
- std::unique_ptr<LinearRing> shell(gf.createLinearRing(coords.release()));
- std::unique_ptr<std::vector<Geometry *> > empty(new std::vector<Geometry *>);
- geom_ptr geom(gf.createPolygon(shell.release(), empty.release()));
-
- if (geom->isEmpty()) {
- throw std::runtime_error("Excluding empty polygon.");
- }
- if (!geom->isValid()) {
- if (excludepoly) {
- throw std::runtime_error("Excluding broken polygon.");
- } else {
- geom = geom_ptr(geom->buffer(0));
- if (geom->isEmpty() || !geom->isValid()) {
- throw std::runtime_error("Excluding unrecoverable broken polygon.");
- }
- }
- }
- geom->normalize(); // Fix direction of ring
-
- return geom;
-}
-
-geometry_builder::pg_geom_t geometry_builder::get_wkb_simple(const nodelist_t &nodes, int polygon) const
-{
- pg_geom_t wkb;
-
- try
- {
- GeometryFactory gf;
- auto coords = nodes2coords(gf, nodes);
- if (polygon && is_polygon_line(coords.get())) {
- auto geom = create_simple_poly(gf, std::move(coords));
- wkb.set(geom.get(), true, projection);
- } else {
- if (coords->getSize() < 2)
- throw std::runtime_error("Excluding degenerate line.");
- geom_ptr geom(gf.createLineString(coords.release()));
- wkb.set(geom.get(), false);
- }
- }
- catch (const std::bad_alloc&)
- {
- std::cerr << std::endl << "Exception caught processing way. You are likelly running out of memory." << std::endl;
- std::cerr << "Try in slim mode, using -s parameter." << std::endl;
- }
- catch (const std::runtime_error& e)
- {
- //std::cerr << std::endl << "Exception caught processing way: " << e.what() << std::endl;
- }
- catch (...)
- {
- std::cerr << std::endl << "Exception caught processing way" << std::endl;
- }
-
- return wkb;
-}
-
-geometry_builder::pg_geoms_t geometry_builder::get_wkb_split(const nodelist_t &nodes, int polygon, double split_at) const
-{
- //TODO: use count to get some kind of hint of how much we should reserve?
- pg_geoms_t wkbs;
-
- try
- {
- GeometryFactory gf;
- auto coords = nodes2coords(gf, nodes);
-
- if (polygon && is_polygon_line(coords.get())) {
- auto geom = create_simple_poly(gf, std::move(coords));
- wkbs.emplace_back(geom.get(), true, projection);
- } else {
- if (coords->getSize() < 2)
- throw std::runtime_error("Excluding degenerate line.");
-
- double distance = 0;
- std::unique_ptr<CoordinateSequence> segment(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
- segment->add(coords->getAt(0));
- for(size_t i=1; i<coords->getSize(); i++) {
- const Coordinate this_pt = coords->getAt(i);
- const Coordinate prev_pt = coords->getAt(i-1);
- const double delta = this_pt.distance(prev_pt);
- assert(!std::isnan(delta));
- // figure out if the addition of this point would take the total
- // length of the line in `segment` over the `split_at` distance.
-
- if (distance + delta > split_at) {
- const size_t splits = (size_t) std::floor((distance + delta) / split_at);
- // use the splitting distance to split the current segment up
- // into as many parts as necessary to keep each part below
- // the `split_at` distance.
- for (size_t j = 0; j < splits; ++j) {
- double frac = (double(j + 1) * split_at - distance) / delta;
- const Coordinate interpolated(frac * (this_pt.x - prev_pt.x) + prev_pt.x,
- frac * (this_pt.y - prev_pt.y) + prev_pt.y);
- segment->add(interpolated);
- geom_ptr geom(gf.createLineString(segment.release()));
-
- wkbs.emplace_back(geom.get(), false);
-
- segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
- segment->add(interpolated);
- }
- // reset the distance based on the final splitting point for
- // the next iteration.
- distance = segment->getAt(0).distance(this_pt);
-
- } else {
- // if not split then just push this point onto the sequence
- // being saved up.
- distance += delta;
- }
-
- // always add this point
- segment->add(this_pt);
-
- // on the last iteration, close out the line.
- if (i == coords->getSize()-1) {
- geom_ptr geom(gf.createLineString(segment.release()));
-
- wkbs.emplace_back(geom.get(), false);
- }
- }
- }
- }
- catch (const std::bad_alloc&)
- {
- std::cerr << std::endl << "Exception caught processing way. You are likely running out of memory." << std::endl;
- std::cerr << "Try in slim mode, using -s parameter." << std::endl;
- }
- catch (const std::runtime_error& e)
- {
- //std::cerr << std::endl << "Exception caught processing way: " << e.what() << std::endl;
- }
- catch (...)
- {
- std::cerr << std::endl << "Exception caught processing way" << std::endl;
- }
- return wkbs;
-}
-
-int geometry_builder::parse_wkb(const char* wkb, multinodelist_t &nodes, bool *polygon) {
- GeometryFactory gf;
- geos::io::WKBReader reader(gf);
-
- *polygon = false;
- std::stringstream stream(wkb, std::ios_base::in);
- geom_ptr geometry(reader.readHEX(stream));
- switch (geometry->getGeometryTypeId()) {
- // Single geometries
- case GEOS_POLYGON:
- // Drop through
- case GEOS_LINEARRING:
- *polygon = true;
- // Drop through
- case GEOS_POINT:
- // Drop through
- case GEOS_LINESTRING:
- {
- nodes.push_back(nodelist_t());
- coord_ptr coords(geometry->getCoordinates());
- coords2nodes(coords.get(), nodes.back());
- break;
- }
- // Geometry collections
- case GEOS_MULTIPOLYGON:
- *polygon = true;
- // Drop through
- case GEOS_MULTIPOINT:
- // Drop through
- case GEOS_MULTILINESTRING:
- {
- auto gc = dynamic_cast<GeometryCollection *>(geometry.get());
- size_t num_geometries = gc->getNumGeometries();
- nodes.assign(num_geometries, nodelist_t());
- for (size_t i = 0; i < num_geometries; i++) {
- const Geometry *subgeometry = gc->getGeometryN(i);
- coord_ptr coords(subgeometry->getCoordinates());
- coords2nodes(coords.get(), nodes[i]);
- }
- break;
- }
- default:
- std::cerr << std::endl << "unexpected object type while processing PostGIS data" << std::endl;
- return -1;
- }
-
- return 0;
-}
-
-geometry_builder::pg_geoms_t geometry_builder::build_polygons(const multinodelist_t &xnodes,
- bool enable_multi, osmid_t osm_id) const
-{
- pg_geoms_t wkbs;
-
- try
- {
- GeometryFactory gf;
- geom_ptr mline = create_multi_line(gf, xnodes);
-
- //geom_ptr noded (segment->Union(mline.get()));
- LineMerger merger;
- //merger.add(noded.get());
- merger.add(mline.get());
- std::unique_ptr<std::vector<LineString *>> merged(merger.getMergedLineStrings());
-
- // Procces ways into lines or simple polygon list
- std::vector<polygondata> polys;
- polys.reserve(merged->size());
-
- for (auto *line: *merged) {
- // stuff into unique pointer for auto-destruct
- std::unique_ptr<LineString> pline(line);
- if (pline->getNumPoints() > 3 && pline->isClosed()) {
- std::unique_ptr<Polygon> poly(gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0));
- double area = get_area(poly.get(), projection);
- if (area > 0.0) {
- polys.emplace_back(std::move(poly),
- gf.createLinearRing(pline->getCoordinates()),
- area);
- }
- }
- }
-
- if (!polys.empty())
- {
- std::sort(polys.begin(), polys.end(), polygondata_comparearea());
-
- unsigned toplevelpolygons = 0;
- int istoplevelafterall;
- size_t totalpolys = polys.size();
-
- geos::geom::prep::PreparedGeometryFactory pgf;
- for (unsigned i=0 ;i < totalpolys; ++i)
- {
- if (polys[i].iscontained) continue;
- toplevelpolygons++;
- const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon.get());
-
- for (unsigned j=i+1; j < totalpolys; ++j)
- {
- // Does preparedtoplevelpolygon contain the smaller polygon[j]?
- if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon.get()))
- {
- // are we in a [i] contains [k] contains [j] situation
- // which would actually make j top level
- istoplevelafterall = 0;
- for (unsigned k=i+1; k < j; ++k)
- {
- if (polys[k].iscontained && polys[k].containedbyid == i && polys[k].polygon->contains(polys[j].polygon.get()))
- {
- istoplevelafterall = 1;
- break;
- }
- }
- if (istoplevelafterall == 0)
- {
- polys[j].iscontained = true;
- polys[j].containedbyid = i;
- }
- }
- }
- pgf.destroy(preparedtoplevelpolygon);
- }
- // polys now is a list of polygons tagged with which ones are inside each other
-
- // List of polygons for multipolygon
- std::unique_ptr<std::vector<Geometry*>> polygons(new std::vector<Geometry*>);
-
- // For each top level polygon create a new polygon including any holes
- for (unsigned i=0 ;i < totalpolys; ++i)
- {
- if (polys[i].iscontained) continue;
-
- // List of holes for this top level polygon
- std::unique_ptr<std::vector<Geometry*> > interior(new std::vector<Geometry*>);
- for (unsigned j=i+1; j < totalpolys; ++j)
- {
- if (polys[j].iscontained && polys[j].containedbyid == i)
- {
- interior->push_back(polys[j].ring.release());
- }
- }
-
- Polygon* poly(gf.createPolygon(polys[i].ring.release(), interior.release()));
- poly->normalize();
- polygons->push_back(poly);
- }
-
- // Make a multipolygon if required
- if ((toplevelpolygons > 1) && enable_multi)
- {
- geom_ptr multipoly(gf.createMultiPolygon(polygons.release()));
-
- if (!multipoly->isEmpty()) {
- if (!multipoly->isValid() && !excludepoly) {
- multipoly = geom_ptr(multipoly->buffer(0));
- multipoly->normalize();
- if (!multipoly->isEmpty() && multipoly->isValid()) {
- wkbs.emplace_back(multipoly.get(), true, projection);
- }
- } else {
- multipoly->normalize();
- wkbs.emplace_back(multipoly.get(), true, projection);
- }
- }
- } else {
- for(unsigned i=0; i<toplevelpolygons; i++) {
- geom_ptr poly(polygons->at(i));
- if (!poly->isEmpty()) {
- if (!poly->isValid() && !excludepoly) {
- poly = geom_ptr(poly->buffer(0));
- poly->normalize();
- if (!poly->isEmpty() && poly->isValid()) {
- wkbs.emplace_back(poly.get(), true, projection);
- }
- } else {
- poly->normalize();
- wkbs.emplace_back(poly.get(), true, projection);
- }
- }
- }
- }
- }
- }//TODO: don't show in message id when osm_id == -1
- catch (const std::exception& e)
- {
- std::cerr << std::endl << "Standard exception processing relation_id="<< osm_id << ": " << e.what() << std::endl;
- }
- catch (...)
- {
- std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl;
- }
-
- return wkbs;
-}
-
-geometry_builder::pg_geom_t geometry_builder::build_multilines(const multinodelist_t &xnodes, osmid_t osm_id) const
-{
- pg_geom_t wkb;
-
- try
- {
- GeometryFactory gf;
- geom_ptr mline = create_multi_line(gf, xnodes);
-
- wkb.set(mline.get(), false);
- }//TODO: don't show in message id when osm_id == -1
- catch (const std::exception& e)
- {
- std::cerr << std::endl << "Standard exception processing relation_id="<< osm_id << ": " << e.what() << std::endl;
- }
- catch (...)
- {
- std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl;
- }
- return wkb;
-}
-
-geometry_builder::pg_geoms_t geometry_builder::build_both(const multinodelist_t &xnodes,
- int make_polygon, int enable_multi,
- double split_at, osmid_t osm_id) const
-{
- pg_geoms_t wkbs;
-
- try
- {
- GeometryFactory gf;
- geom_ptr mline = create_multi_line(gf, xnodes);
- //geom_ptr noded (segment->Union(mline.get()));
- LineMerger merger;
- //merger.add(noded.get());
- merger.add(mline.get());
- std::unique_ptr<std::vector<LineString *> > merged(merger.getMergedLineStrings());
-
- // Procces ways into lines or simple polygon list
- std::vector<polygondata> polys;
- polys.reserve(merged->size());
-
- for (auto *line: *merged) {
- // stuff into unique pointer to ensure auto-destruct
- std::unique_ptr<LineString> pline(line);
- if (make_polygon && pline->getNumPoints() > 3 && pline->isClosed()) {
- std::unique_ptr<Polygon> poly(gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0));
- double area = get_area(poly.get(), projection);
- if (area > 0.0) {
- polys.emplace_back(std::move(poly),
- gf.createLinearRing(pline->getCoordinates()),
- area);
- }
- } else {
- double distance = 0;
- std::unique_ptr<CoordinateSequence> segment;
- segment = std::unique_ptr<CoordinateSequence>(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
- segment->add(pline->getCoordinateN(0));
- for(int j=1; j<(int)pline->getNumPoints(); ++j) {
- segment->add(pline->getCoordinateN(j));
- distance += pline->getCoordinateN(j).distance(pline->getCoordinateN(j-1));
- if ((distance >= split_at) || (j == (int)pline->getNumPoints()-1)) {
- geom_ptr geom = geom_ptr(gf.createLineString(segment.release()));
-
- wkbs.emplace_back(geom.get(), false);
-
- segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
- distance=0;
- segment->add(pline->getCoordinateN(j));
- }
- }
- }
- }
-
- if (!polys.empty())
- {
- std::sort(polys.begin(), polys.end(), polygondata_comparearea());
-
- unsigned toplevelpolygons = 0;
- int istoplevelafterall;
- size_t totalpolys = polys.size();
-
- geos::geom::prep::PreparedGeometryFactory pgf;
- for (unsigned i=0 ;i < totalpolys; ++i)
- {
- if (polys[i].iscontained) continue;
- toplevelpolygons++;
- const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon.get());
-
- for (unsigned j=i+1; j < totalpolys; ++j)
- {
- // Does preparedtoplevelpolygon contain the smaller polygon[j]?
- if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon.get()))
- {
- // are we in a [i] contains [k] contains [j] situation
- // which would actually make j top level
- istoplevelafterall = 0;
- for (unsigned k=i+1; k < j; ++k)
- {
- if (polys[k].iscontained && polys[k].containedbyid == i && polys[k].polygon->contains(polys[j].polygon.get()))
- {
- istoplevelafterall = 1;
- break;
- }
-#if 0
- else if (polys[k].polygon->intersects(polys[j].polygon) || polys[k].polygon->touches(polys[j].polygon))
- {
- // FIXME: This code does not work as intended
- // It should be setting the polys[k].ring in order to update this object
- // but the value of polys[k].polygon calculated is normally NULL
-
- // Add polygon this polygon (j) to k since they intersect
- // Mark ourselfs to be dropped (2), delete the original k
- Geometry* polyunion = polys[k].polygon->Union(polys[j].polygon);
- delete(polys[k].polygon);
- polys[k].polygon = dynamic_cast<Polygon*>(polyunion);
- polys[j].iscontained = 2; // Drop
- istoplevelafterall = 2;
- break;
- }
-#endif
- }
- if (istoplevelafterall == 0)
- {
- polys[j].iscontained = true;
- polys[j].containedbyid = i;
- }
- }
- }
- pgf.destroy(preparedtoplevelpolygon);
- }
- // polys now is a list of polygons tagged with which ones are inside each other
-
- // List of polygons for multipolygon
- std::unique_ptr<std::vector<Geometry*> > polygons(new std::vector<Geometry*>);
-
- // For each top level polygon create a new polygon including any holes
- for (unsigned i=0 ;i < totalpolys; ++i)
- {
- if (polys[i].iscontained) continue;
-
- // List of holes for this top level polygon
- std::unique_ptr<std::vector<Geometry*> > interior(new std::vector<Geometry*>);
- for (unsigned j=i+1; j < totalpolys; ++j)
- {
- if (polys[j].iscontained && polys[j].containedbyid == i)
- {
- interior->push_back(polys[j].ring.release());
- }
- }
-
- Polygon* poly(gf.createPolygon(polys[i].ring.release(), interior.release()));
- poly->normalize();
- polygons->push_back(poly);
- }
-
- // Make a multipolygon if required
- if ((toplevelpolygons > 1) && enable_multi)
- {
- geom_ptr multipoly(gf.createMultiPolygon(polygons.release()));
- if (!multipoly->isValid() && !excludepoly) {
- multipoly = geom_ptr(multipoly->buffer(0));
- }
- multipoly->normalize();
-
- if (!multipoly->isEmpty() && multipoly->isValid()) {
- wkbs.emplace_back(multipoly.get(), true, projection);
- }
- }
- else
- {
- for(unsigned i=0; i<toplevelpolygons; i++)
- {
- geom_ptr poly(polygons->at(i));
- if (!poly->isValid() && !excludepoly) {
- poly = geom_ptr(poly->buffer(0));
- poly->normalize();
- }
- if (!poly->isEmpty() && poly->isValid()) {
- wkbs.emplace_back(poly.get(), true, projection);
- }
- }
- }
- }
- }//TODO: don't show in message id when osm_id == -1
- catch (const std::exception& e)
- {
- std::cerr << std::endl << "Standard exception processing relation id="<< osm_id << ": " << e.what() << std::endl;
- }
- catch (...)
- {
- std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl;
- }
-
- return wkbs;
-}
diff --git a/geometry-builder.hpp b/geometry-builder.hpp
deleted file mode 100644
index 56457c1..0000000
--- a/geometry-builder.hpp
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
-#-----------------------------------------------------------------------------
-# Part of osm2pgsql utility
-#-----------------------------------------------------------------------------
-# By Artem Pavlenko, Copyright 2007
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#-----------------------------------------------------------------------------
-*/
-
-#ifndef GEOMETRY_BUILDER_H
-#define GEOMETRY_BUILDER_H
-
-#include "osmtypes.hpp"
-
-#include <vector>
-#include <string>
-#include <memory>
-
-namespace geos { namespace geom {
-class Geometry;
-class GeometryFactory;
-class CoordinateSequence;
-}}
-
-class reprojection;
-
-class geometry_builder
-{
-public:
- struct pg_geom_t
- {
- pg_geom_t(const std::string &geom_str, bool poly, double geom_area = 0)
- : geom(geom_str), area(geom_area), polygon(poly)
- {}
-
- /// Create an invalid geometry.
- pg_geom_t()
- : area(0), polygon(false)
- {}
-
- pg_geom_t(const geos::geom::Geometry *g, bool poly, reprojection *p = nullptr)
- {
- set(g, poly, p);
- }
-
- /**
- * Set geometry from a Geos geometry.
- */
- void set(const geos::geom::Geometry *geom, bool poly, reprojection *p = nullptr);
-
-
- bool is_polygon() const
- {
- return polygon;
- }
-
- bool valid() const
- {
- return !geom.empty();
- }
-
- std::string geom;
- double area;
- bool polygon;
- };
-
- typedef std::vector<geometry_builder::pg_geom_t> pg_geoms_t;
-
- static int parse_wkb(const char *wkb, multinodelist_t &nodes, bool *polygon);
- pg_geom_t get_wkb_simple(const nodelist_t &nodes, int polygon) const;
- pg_geoms_t get_wkb_split(const nodelist_t &nodes, int polygon, double split_at) const;
- pg_geoms_t build_both(const multinodelist_t &xnodes, int make_polygon,
- int enable_multi, double split_at, osmid_t osm_id = -1) const;
- pg_geoms_t build_polygons(const multinodelist_t &xnodes, bool enable_multi, osmid_t osm_id = -1) const;
- /** Output relation as a multiline.
- *
- * Used by gazetteer only.
- */
- pg_geom_t build_multilines(const multinodelist_t &xnodes, osmid_t osm_id) const;
-
- void set_exclude_broken_polygon(bool exclude)
- {
- excludepoly = exclude;
- }
-
- void set_reprojection(reprojection *r)
- {
- projection = r;
- }
-
-
-private:
- std::unique_ptr<geos::geom::Geometry>
- create_simple_poly(geos::geom::GeometryFactory &gf,
- std::unique_ptr<geos::geom::CoordinateSequence> coords) const;
-
- bool excludepoly = false;
- reprojection *projection = nullptr;
-};
-
-#endif
diff --git a/geometry-processor.cpp b/geometry-processor.cpp
index 5b23eed..c4d89b7 100644
--- a/geometry-processor.cpp
+++ b/geometry-processor.cpp
@@ -14,16 +14,15 @@
std::shared_ptr<geometry_processor> geometry_processor::create(const std::string &type,
const options_t *options) {
std::shared_ptr<geometry_processor> ptr;
- int srid = options->projection->target_srs();
if (type == "point") {
- ptr = std::make_shared<processor_point>(srid);
+ ptr = std::make_shared<processor_point>(options->projection);
}
else if (type == "line") {
- ptr = std::make_shared<processor_line>(srid);
+ ptr = std::make_shared<processor_line>(options->projection);
}
else if (type == "polygon") {
- ptr = std::make_shared<processor_polygon>(srid, options->enable_multi);
+ ptr = std::make_shared<processor_polygon>(options->projection);
}
else {
throw std::runtime_error((boost::format("Unable to construct geometry processor "
@@ -38,8 +37,7 @@ geometry_processor::geometry_processor(int srid, const std::string &type, unsign
: m_srid(srid), m_type(type), m_interests(interests) {
}
-geometry_processor::~geometry_processor() {
-}
+geometry_processor::~geometry_processor() = default;
int geometry_processor::srid() const {
return m_srid;
@@ -57,83 +55,50 @@ bool geometry_processor::interests(unsigned int interested) const {
return (interested & m_interests) == interested;
}
-geometry_builder::pg_geom_t geometry_processor::process_node(double, double) {
- return geometry_builder::pg_geom_t();
-}
-
-geometry_builder::pg_geom_t geometry_processor::process_way(const nodelist_t &) {
- return geometry_builder::pg_geom_t();
-}
-
-geometry_builder::pg_geoms_t geometry_processor::process_relation(const multinodelist_t &) {
- return geometry_builder::pg_geoms_t();
+geometry_processor::wkb_t
+geometry_processor::process_node(osmium::Location const &,
+ geom::osmium_builder_t *)
+{
+ return wkb_t();
}
-way_helper::way_helper()
+geometry_processor::wkb_t
+geometry_processor::process_way(osmium::Way const &, geom::osmium_builder_t *)
{
+ return wkb_t();
}
-way_helper::~way_helper()
+
+geometry_processor::wkbs_t
+geometry_processor::process_relation(osmium::Relation const &,
+ osmium::memory::Buffer const &,
+ geom::osmium_builder_t *)
{
+ return wkbs_t();
}
-size_t way_helper::set(const idlist_t &node_ids, const middle_query_t *mid)
-{
- node_cache.clear();
- mid->nodes_get_list(node_cache, node_ids);
- // equivalent to returning node_count for complete ways, different for partial extracts
- return node_cache.size();
-}
relation_helper::relation_helper()
-{
-}
+: data(1024, osmium::memory::Buffer::auto_grow::yes)
+{}
-relation_helper::~relation_helper()
-{
-}
-
-size_t relation_helper::set(const memberlist_t *member_list, const middle_t* mid)
+size_t relation_helper::set(osmium::Relation const &rel, middle_t const *mid)
{
// cleanup
- input_way_ids.clear();
- ways.clear();
- tags.clear();
- nodes.clear();
+ data.clear();
roles.clear();
- //keep a few things
- members = member_list;
-
- //grab the way members' ids
- input_way_ids.reserve(member_list->size());
- for (memberlist_t::const_iterator it = members->begin(); it != members->end(); ++it) {
- if(it->type == OSMTYPE_WAY)
- input_way_ids.push_back(it->id);
- }
+ // get the nodes and roles of the ways
+ auto num_ways = mid->rel_way_members_get(rel, &roles, data);
- //if we didn't end up using any we'll bail
- if (input_way_ids.empty())
- return 0;
-
- //get the nodes of the ways
- mid->ways_get_list(input_way_ids, ways, tags, nodes);
-
- //grab the roles of each way
- roles.reserve(ways.size());
- size_t memberpos = 0;
- for (idlist_t::const_iterator it = ways.begin(); it != ways.end(); ++it) {
- while (memberpos < members->size()) {
- if (members->at(memberpos).id == *it) {
- roles.push_back(&(members->at(memberpos).role));
- memberpos++;
- break;
- }
- memberpos++;
- }
- }
+ // mark the ends of each so whoever uses them will know where they end..
+ superseded.resize(num_ways);
- //mark the ends of each so whoever uses them will know where they end..
- superseeded.resize(ways.size());
+ return num_ways;
+}
- return ways.size();
+void relation_helper::add_way_locations(middle_t const *mid)
+{
+ for (auto &w : data.select<osmium::Way>()) {
+ mid->nodes_get_list(&(w.nodes()));
+ }
}
diff --git a/geometry-processor.hpp b/geometry-processor.hpp
index 83849fa..67c5846 100644
--- a/geometry-processor.hpp
+++ b/geometry-processor.hpp
@@ -5,14 +5,21 @@
#include <string>
#include <vector>
#include <memory>
-#include "geometry-builder.hpp"
+
+#include <osmium/memory/buffer.hpp>
+
+#include "osmium-builder.hpp"
#include "osmtypes.hpp"
+#include "tagtransform.hpp"
struct middle_query_t;
struct middle_t;
struct options_t;
+class reprojection;
struct geometry_processor {
+ using wkb_t = geom::osmium_builder_t::wkb_t;
+ using wkbs_t = geom::osmium_builder_t::wkbs_t;
// factory method for creating various types of geometry processors either by name or by geometry column type
static std::shared_ptr<geometry_processor> create(const std::string &type,
const options_t *options);
@@ -40,17 +47,21 @@ struct geometry_processor {
// process a node, optionally returning a WKB string describing
// geometry to be inserted into the table.
- virtual geometry_builder::pg_geom_t process_node(double lat, double lon);
+ virtual wkb_t process_node(osmium::Location const &loc,
+ geom::osmium_builder_t *builder);
// process a way
// position data and optionally returning WKB-encoded geometry
// for insertion into the table.
- virtual geometry_builder::pg_geom_t process_way(const nodelist_t &nodes);
+ virtual wkb_t process_way(osmium::Way const &way,
+ geom::osmium_builder_t *builder);
// process a way, taking a middle query object to get way and
// node position data. optionally returns an array of WKB-encoded geometry
// for insertion into the table.
- virtual geometry_builder::pg_geoms_t process_relation(const multinodelist_t &nodes);
+ virtual wkbs_t process_relation(osmium::Relation const &rel,
+ osmium::memory::Buffer const &ways,
+ geom::osmium_builder_t *builder);
// returns the SRID of the output geometry.
int srid() const;
@@ -69,33 +80,18 @@ protected:
geometry_processor(int srid, const std::string &type, unsigned int interests);
};
-
-//various bits for continuous processing of ways
-struct way_helper
-{
- way_helper();
- ~way_helper();
- size_t set(const idlist_t &node_ids, const middle_query_t *mid);
-
- nodelist_t node_cache;
-};
-
//various bits for continuous processing of members of relations
-struct relation_helper
+class relation_helper
{
+public:
relation_helper();
- ~relation_helper();
- size_t set(const memberlist_t *member_list, const middle_t *mid);
- const memberlist_t *members;
- multitaglist_t tags;
- multinodelist_t nodes;
- idlist_t ways;
- rolelist_t roles;
- std::vector<int> superseeded;
+ size_t set(osmium::Relation const &rel, middle_t const *mid);
+ void add_way_locations(middle_t const *mid);
-private:
- idlist_t input_way_ids;
+ rolelist_t roles;
+ std::vector<int> superseded;
+ osmium::memory::Buffer data;
};
#endif /* GEOMETRY_PROCESSOR_HPP */
diff --git a/middle-pgsql.cpp b/middle-pgsql.cpp
index f065f13..ebd914c 100644
--- a/middle-pgsql.cpp
+++ b/middle-pgsql.cpp
@@ -27,6 +27,8 @@ using namespace std;
#include <future>
#include <boost/format.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/builder/osm_object_builder.hpp>
#include <libpq-fe.h>
@@ -96,43 +98,66 @@ inline const char *decode_upto( const char *src, char *dst )
return src;
}
-void pgsql_parse_tags(const char *string, taglist_t &tags)
+template <typename T>
+void pgsql_parse_tags(const char *string, osmium::memory::Buffer &buffer, T &obuilder)
{
- char key[1024];
- char val[1024];
-
- if( *string == '\0' )
- return;
+ if( *string++ != '{' )
+ return;
- if( *string++ != '{' )
- return;
- while( *string != '}' )
- {
- string = decode_upto( string, key );
- // String points to the comma */
- string++;
- string = decode_upto( string, val );
- // String points to the comma or closing '}' */
- tags.push_back(tag_t(key, val));
- if( *string == ',' )
- string++;
- }
+ char key[1024];
+ char val[1024];
+ osmium::builder::TagListBuilder builder(buffer, &obuilder);
+
+ while( *string != '}' ) {
+ string = decode_upto(string, key);
+ // String points to the comma */
+ string++;
+ string = decode_upto(string, val);
+ builder.add_tag(key, val);
+ // String points to the comma or closing '}' */
+ if( *string == ',' ) {
+ string++;
+ }
+ }
}
-// Parses an array of integers */
-void pgsql_parse_nodes(const char *string, idlist_t &nds)
+void pgsql_parse_members(const char *string, osmium::memory::Buffer &buffer,
+ osmium::builder::RelationBuilder &obuilder)
{
- if( *string++ != '{' )
- return;
+ if( *string++ != '{' )
+ return;
- while( *string != '}' )
- {
- char *ptr;
- nds.push_back(strtoosmid( string, &ptr, 10 ));
- string = ptr;
- if( *string == ',' )
- string++;
- }
+ char role[1024];
+ osmium::builder::RelationMemberListBuilder builder(buffer, &obuilder);
+
+ while( *string != '}' ) {
+ char type = string[0];
+ char *endp;
+ osmid_t id = strtoosmid(string + 1, &endp, 10);
+ // String points to the comma */
+ string = decode_upto(endp + 1, role);
+ builder.add_member(osmium::char_to_item_type(type), id, role);
+ // String points to the comma or closing '}' */
+ if( *string == ',' ) {
+ string++;
+ }
+ }
+}
+
+void pgsql_parse_nodes(const char *string, osmium::memory::Buffer &buffer,
+ osmium::builder::WayBuilder &builder)
+{
+ if (*string++ == '{') {
+ osmium::builder::WayNodeListBuilder wnl_builder(buffer, &builder);
+ while (*string != '}') {
+ char *ptr;
+ wnl_builder.add_node_ref(strtoosmid(string, &ptr, 10));
+ string = ptr;
+ if (*string == ',') {
+ string++;
+ }
+ }
+ }
}
int pgsql_endCopy(middle_pgsql_t::table_desc *table)
@@ -146,13 +171,11 @@ int pgsql_endCopy(middle_pgsql_t::table_desc *table)
util::exit_nicely();
}
- PGresult *res = PQgetResult(sql_conn);
- if (PQresultStatus(res) != PGRES_COMMAND_OK) {
+ pg_result_t res(PQgetResult(sql_conn));
+ if (PQresultStatus(res.get()) != PGRES_COMMAND_OK) {
fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
- PQclear(res);
util::exit_nicely();
}
- PQclear(res);
table->copyMode = 0;
}
return 0;
@@ -160,23 +183,6 @@ int pgsql_endCopy(middle_pgsql_t::table_desc *table)
} // anonymous namespace
-void middle_pgsql_t::buffer_store_nodes(idlist_t const &nds)
-{
- if (nds.size() == 0) {
- copy_buffer += "{}";
- return;
- }
-
- copy_buffer += "{";
-
- for (auto const &it : nds) {
- copy_buffer += std::to_string(it);
- copy_buffer += ',';
- }
-
- copy_buffer[copy_buffer.size() - 1] = '}';
-}
-
void middle_pgsql_t::buffer_store_string(std::string const &in, bool escape)
{
for (char const c: in) {
@@ -209,25 +215,31 @@ void middle_pgsql_t::buffer_store_string(std::string const &in, bool escape)
}
// escape means we return '\N' for copy mode, otherwise we return just nullptr
-void middle_pgsql_t::buffer_store_tags(taglist_t const &tags, bool escape)
+void middle_pgsql_t::buffer_store_tags(osmium::OSMObject const &obj, bool attrs,
+ bool escape)
{
copy_buffer += "{";
- bool first = true;
- for (auto const &it : tags) {
- if (!first) {
- copy_buffer += ',';
- }
+ for (auto const &it : obj.tags()) {
copy_buffer += "\"";
- buffer_store_string(it.key, escape);
+ buffer_store_string(it.key(), escape);
copy_buffer += "\",\"";
- buffer_store_string(it.value, escape);
- copy_buffer += '"';
-
- first = false;
+ buffer_store_string(it.value(), escape);
+ copy_buffer += "\",";
+ }
+ if (attrs) {
+ taglist_t extra;
+ extra.add_attributes(obj);
+ for (auto const &it : extra) {
+ copy_buffer += "\"";
+ copy_buffer += it.key;
+ copy_buffer += "\",\"";
+ buffer_store_string(it.value.c_str(), escape);
+ copy_buffer += "\",";
+ }
}
- copy_buffer += "}";
+ copy_buffer[copy_buffer.size() - 1] = '}';
}
void middle_pgsql_t::buffer_correct_params(char const **param, size_t size)
@@ -242,159 +254,110 @@ void middle_pgsql_t::buffer_correct_params(char const **param, size_t size)
}
}
-void middle_pgsql_t::local_nodes_set(osmid_t id, double lat, double lon,
- const taglist_t &tags)
+void middle_pgsql_t::local_nodes_set(osmium::Node const &node)
{
- copy_buffer.reserve(tags.size() * 24 + 64);
+ copy_buffer.reserve(node.tags().byte_size() + 100);
bool copy = node_table->copyMode;
char delim = copy ? '\t' : '\0';
- const char *paramValues[4] = { copy_buffer.c_str(), };
+ const char *paramValues[4] = {
+ copy_buffer.c_str(),
+ };
- copy_buffer = std::to_string(id);
+ copy_buffer = std::to_string(node.id());
copy_buffer += delim;
-#ifdef FIXED_POINT
- ramNode n(lon, lat);
paramValues[1] = paramValues[0] + copy_buffer.size();
- copy_buffer += std::to_string(n.int_lat());
+ copy_buffer += std::to_string(node.location().y());
copy_buffer += delim;
paramValues[2] = paramValues[0] + copy_buffer.size();
- copy_buffer += std::to_string(n.int_lon());
- copy_buffer += delim;
-#else
- paramValues[1] = paramValues[0] + copy_buffer.size();
- copy_buffer += std::to_string(lat);
- copy_buffer += delim;
-
- paramValues[2] = paramValues[0] + copy_buffer.size();
- copy_buffer += std::to_string(lon);
- copy_buffer += delim;
-#endif
-
- if (tags.size() == 0) {
- paramValues[3] = nullptr;
- copy_buffer += "\\N";
- } else {
- paramValues[3] = paramValues[0] + copy_buffer.size();
- buffer_store_tags(tags, copy);
- }
+ copy_buffer += std::to_string(node.location().x());
if (copy) {
copy_buffer += '\n';
pgsql_CopyData(__FUNCTION__, node_table->sql_conn, copy_buffer);
} else {
buffer_correct_params(paramValues, 4);
- pgsql_execPrepared(node_table->sql_conn, "insert_node", 4,
- (const char * const *)paramValues, PGRES_COMMAND_OK);
+ pgsql_execPrepared(node_table->sql_conn, "insert_node", 3,
+ (const char *const *)paramValues, PGRES_COMMAND_OK);
}
}
-// This should be made more efficient by using an IN(ARRAY[]) construct */
-size_t middle_pgsql_t::local_nodes_get_list(nodelist_t &out, const idlist_t nds) const
+size_t middle_pgsql_t::local_nodes_get_list(osmium::WayNodeList *nodes) const
{
- assert(out.empty());
-
- char tmp[16];
-
- char *tmp2 = static_cast<char *>(malloc(sizeof(char) * nds.size() * 16));
- if (tmp2 == nullptr) return 0; //failed to allocate memory, return */
-
-
- // create a list of ids in tmp2 to query the database */
- sprintf(tmp2, "{");
- int countDB = 0;
- for(idlist_t::const_iterator it = nds.begin(); it != nds.end(); ++it) {
- // Check cache first */
- osmNode loc;
- if (cache->get(&loc, *it) == 0) {
- out.push_back(loc);
- continue;
+ size_t count = 0;
+ std::string buffer("{");
+
+ // get nodes where possible from cache,
+ // at the same time build a list for querying missing nodes from DB
+ size_t pos = 0;
+ for (auto &n : *nodes) {
+ auto loc = cache->get(n.ref());
+ if (loc.valid()) {
+ n.set_location(loc);
+ ++count;
+ } else {
+ buffer += std::to_string(n.ref());
+ buffer += ',';
}
-
- countDB++;
- // Mark nodes as needing to be fetched from the DB */
- out.push_back(osmNode());
-
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", *it);
- strncat(tmp2, tmp, sizeof(char)*(nds.size()*16 - 2));
+ ++pos;
}
- tmp2[strlen(tmp2) - 1] = '}'; // replace last , with } to complete list of ids*/
- if (countDB == 0) {
- free(tmp2);
- return nds.size(); // All ids where in cache, so nothing more to do */
+ if (count == pos) {
+ return count; // all ids found in cache, nothing more to do
}
+ // get any remaining nodes from the DB
+ buffer[buffer.size() - 1] = '}';
+
pgsql_endCopy(node_table);
PGconn *sql_conn = node_table->sql_conn;
char const *paramValues[1];
- paramValues[0] = tmp2;
- PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues, PGRES_TUPLES_OK);
- int countPG = PQntuples(res);
-
- //store the pg results in a hashmap and telling it how many we expect
- std::unordered_map<osmid_t, osmNode> pg_nodes(countPG);
-
- for (int i = 0; i < countPG; i++) {
- osmid_t id = strtoosmid(PQgetvalue(res, i, 0), nullptr, 10);
- osmNode node;
-#ifdef FIXED_POINT
- ramNode n((int) strtol(PQgetvalue(res, i, 2), nullptr, 10),
- (int) strtol(PQgetvalue(res, i, 1), nullptr, 10));
-
- node.lat = n.lat();
- node.lon = n.lon();
-#else
- node.lat = strtod(PQgetvalue(res, i, 1), nullptr);
- node.lon = strtod(PQgetvalue(res, i, 2), nullptr);
-#endif
- pg_nodes.emplace(id, node);
+ paramValues[0] = buffer.c_str();
+ auto res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues,
+ PGRES_TUPLES_OK);
+ auto countPG = PQntuples(res.get());
+
+ std::unordered_map<osmid_t, osmium::Location> locs;
+ for (int i = 0; i < countPG; ++i) {
+ locs.emplace(
+ strtoosmid(PQgetvalue(res.get(), i, 0), nullptr, 10),
+ osmium::Location(
+ (int)strtol(PQgetvalue(res.get(), i, 2), nullptr, 10),
+ (int)strtol(PQgetvalue(res.get(), i, 1), nullptr, 10)));
}
- PQclear(res);
- free(tmp2);
-
- // If some of the nodes in the way don't exist, the returning list has holes.
- // Merge the two lists removing any holes.
- size_t wrtidx = 0;
- for (size_t i = 0; i < nds.size(); ++i) {
- if (std::isnan(out[i].lat)) {
- std::unordered_map<osmid_t, osmNode>::iterator found = pg_nodes.find(nds[i]);
- if(found != pg_nodes.end()) {
- out[wrtidx] = found->second;
- ++wrtidx;
- }
- } else {
- if (wrtidx < i)
- out[wrtidx] = out[i];
- ++wrtidx;
+ for (auto &n : *nodes) {
+ auto el = locs.find(n.ref());
+ if (el != locs.end()) {
+ n.set_location(el->second);
+ ++count;
}
+
}
- out.resize(wrtidx);
- return wrtidx;
+ return count;
}
-
-void middle_pgsql_t::nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags) {
- cache->set( id, lat, lon, tags );
+void middle_pgsql_t::nodes_set(osmium::Node const &node)
+{
+ cache->set(node.id(), node.location());
if (out_options->flat_node_cache_enabled) {
- persistent_cache->set(id, lat, lon);
+ persistent_cache->set(node.id(), node.location());
} else {
- local_nodes_set(id, lat, lon, tags);
+ local_nodes_set(node);
}
}
-size_t middle_pgsql_t::nodes_get_list(nodelist_t &out, const idlist_t nds) const
+size_t middle_pgsql_t::nodes_get_list(osmium::WayNodeList *nodes) const
{
return (out_options->flat_node_cache_enabled)
- ? persistent_cache->get_list(out, nds)
- : local_nodes_get_list(out, nds);
+ ? persistent_cache->get_list(nodes)
+ : local_nodes_get_list(nodes);
}
void middle_pgsql_t::local_nodes_delete(osmid_t osm_id)
@@ -412,7 +375,7 @@ void middle_pgsql_t::local_nodes_delete(osmid_t osm_id)
void middle_pgsql_t::nodes_delete(osmid_t osm_id)
{
if (out_options->flat_node_cache_enabled) {
- persistent_cache->set(osm_id, NAN, NAN);
+ persistent_cache->set(osm_id, osmium::Location());
} else {
local_nodes_delete(osm_id);
}
@@ -435,47 +398,53 @@ void middle_pgsql_t::node_changed(osmid_t osm_id)
//keep track of whatever ways and rels these nodes intersect
//TODO: dont need to stop the copy above since we are only reading?
- PGresult* res = pgsql_execPrepared(way_table->sql_conn, "mark_ways_by_node", 1, paramValues, PGRES_TUPLES_OK );
- for(int i = 0; i < PQntuples(res); ++i)
- {
+ auto res = pgsql_execPrepared(way_table->sql_conn, "mark_ways_by_node", 1,
+ paramValues, PGRES_TUPLES_OK);
+ for (int i = 0; i < PQntuples(res.get()); ++i) {
char *end;
- osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10);
+ osmid_t marked = strtoosmid(PQgetvalue(res.get(), i, 0), &end, 10);
ways_pending_tracker->mark(marked);
}
- PQclear(res);
//do the rels too
res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels_by_node", 1, paramValues, PGRES_TUPLES_OK );
- for(int i = 0; i < PQntuples(res); ++i)
- {
+ for (int i = 0; i < PQntuples(res.get()); ++i) {
char *end;
- osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10);
+ osmid_t marked = strtoosmid(PQgetvalue(res.get(), i, 0), &end, 10);
rels_pending_tracker->mark(marked);
}
- PQclear(res);
}
-void middle_pgsql_t::ways_set(osmid_t way_id, const idlist_t &nds, const taglist_t &tags)
+void middle_pgsql_t::ways_set(osmium::Way const &way)
{
- copy_buffer.reserve(nds.size() * 10 + tags.size() * 24 + 64);
+ copy_buffer.reserve(way.nodes().size() * 10 + way.tags().byte_size() + 100);
bool copy = way_table->copyMode;
char delim = copy ? '\t' : '\0';
// Three params: id, nodes, tags */
const char *paramValues[4] = { copy_buffer.c_str(), };
- copy_buffer = std::to_string(way_id);
+ copy_buffer = std::to_string(way.id());
copy_buffer += delim;
paramValues[1] = paramValues[0] + copy_buffer.size();
- buffer_store_nodes(nds);
+ if (way.nodes().size() == 0) {
+ copy_buffer += "{}";
+ } else {
+ copy_buffer += "{";
+ for (auto const &n : way.nodes()) {
+ copy_buffer += std::to_string(n.ref());
+ copy_buffer += ',';
+ }
+ copy_buffer[copy_buffer.size() - 1] = '}';
+ }
copy_buffer += delim;
- if (tags.size() == 0) {
+ if (way.tags().empty() && !out_options->extra_attributes) {
paramValues[2] = nullptr;
copy_buffer += "\\N";
} else {
paramValues[2] = paramValues[0] + copy_buffer.size();
- buffer_store_tags(tags, copy);
+ buffer_store_tags(way, out_options->extra_attributes, copy);
}
if (copy) {
@@ -488,106 +457,106 @@ void middle_pgsql_t::ways_set(osmid_t way_id, const idlist_t &nds, const taglist
}
}
-bool middle_pgsql_t::ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const
+bool middle_pgsql_t::ways_get(osmid_t id, osmium::memory::Buffer &buffer) const
{
char const *paramValues[1];
PGconn *sql_conn = way_table->sql_conn;
// Make sure we're out of copy mode */
- pgsql_endCopy( way_table );
+ pgsql_endCopy(way_table);
char tmp[16];
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
paramValues[0] = tmp;
- PGresult *res = pgsql_execPrepared(sql_conn, "get_way", 1, paramValues, PGRES_TUPLES_OK);
+ auto res = pgsql_execPrepared(sql_conn, "get_way", 1, paramValues,
+ PGRES_TUPLES_OK);
- if (PQntuples(res) != 1) {
- PQclear(res);
+ if (PQntuples(res.get()) != 1) {
return false;
}
- pgsql_parse_tags( PQgetvalue(res, 0, 1), tags );
+ {
+ osmium::builder::WayBuilder builder(buffer);
+ builder.set_id(id);
- size_t num_nodes = strtoul(PQgetvalue(res, 0, 2), nullptr, 10);
- idlist_t list;
- pgsql_parse_nodes( PQgetvalue(res, 0, 0), list);
- if (num_nodes != list.size()) {
- fprintf(stderr, "parse_nodes problem for way %s: expected nodes %zu got %zu\n",
- tmp, num_nodes, list.size());
- util::exit_nicely();
+ pgsql_parse_nodes(PQgetvalue(res.get(), 0, 0), buffer, builder);
+ pgsql_parse_tags(PQgetvalue(res.get(), 0, 1), buffer, builder);
}
- PQclear(res);
- nodes_get_list(nodes, list);
+ buffer.commit();
+
return true;
}
-size_t middle_pgsql_t::ways_get_list(const idlist_t &ids, idlist_t &way_ids,
- multitaglist_t &tags, multinodelist_t &nodes) const {
- if (ids.empty())
- return 0;
-
+size_t middle_pgsql_t::rel_way_members_get(osmium::Relation const &rel,
+ rolelist_t *roles,
+ osmium::memory::Buffer &buffer) const
+{
char tmp[16];
- std::unique_ptr<char[]> tmp2(new (std::nothrow) char[ids.size() * 16]);
char const *paramValues[1];
- if (tmp2 == nullptr) return 0; //failed to allocate memory, return */
+ // create a list of ids in tmp2 to query the database
+ std::string tmp2("{");
+ for (auto const &m : rel.members()) {
+ if (m.type() == osmium::item_type::way) {
+ snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", m.ref());
+ tmp2.append(tmp);
+ }
+ }
- // create a list of ids in tmp2 to query the database */
- sprintf(tmp2.get(), "{");
- for(idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) {
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", *it);
- strncat(tmp2.get(), tmp, sizeof(char)*(ids.size()*16 - 2));
+ if (tmp2.length() == 1) {
+ return 0; // no ways found
}
- tmp2[strlen(tmp2.get()) - 1] = '}'; // replace last , with } to complete list of ids*/
+ // replace last , with } to complete list of ids
+ tmp2[tmp2.length() - 1] = '}';
pgsql_endCopy(way_table);
PGconn *sql_conn = way_table->sql_conn;
- paramValues[0] = tmp2.get();
- PGresult *res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues, PGRES_TUPLES_OK);
- int countPG = PQntuples(res);
+ paramValues[0] = tmp2.c_str();
+ auto res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues,
+ PGRES_TUPLES_OK);
+ int countPG = PQntuples(res.get());
idlist_t wayidspg;
for (int i = 0; i < countPG; i++) {
- wayidspg.push_back(strtoosmid(PQgetvalue(res, i, 0), nullptr, 10));
+ wayidspg.push_back(
+ strtoosmid(PQgetvalue(res.get(), i, 0), nullptr, 10));
}
-
// Match the list of ways coming from postgres in a different order
// back to the list of ways given by the caller */
- for(idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) {
+ size_t outres = 0;
+ for (auto const &m : rel.members()) {
+ if (m.type() != osmium::item_type::way) {
+ continue;
+ }
for (int j = 0; j < countPG; j++) {
- if (*it == wayidspg[j]) {
- way_ids.push_back(*it);
- tags.push_back(taglist_t());
- pgsql_parse_tags(PQgetvalue(res, j, 2), tags.back());
-
- size_t num_nodes = strtoul(PQgetvalue(res, j, 3), nullptr, 10);
- idlist_t list;
- pgsql_parse_nodes( PQgetvalue(res, j, 1), list);
- if (num_nodes != list.size()) {
- fprintf(stderr, "parse_nodes problem for way %s: expected nodes %zu got %zu\n",
- tmp, num_nodes, list.size());
- util::exit_nicely();
+ if (m.ref() == wayidspg[j]) {
+ {
+ osmium::builder::WayBuilder builder(buffer);
+ builder.set_id(m.ref());
+
+ pgsql_parse_nodes(PQgetvalue(res.get(), j, 1), buffer,
+ builder);
+ pgsql_parse_tags(PQgetvalue(res.get(), j, 2), buffer,
+ builder);
}
- nodes.push_back(nodelist_t());
- nodes_get_list(nodes.back(), list);
-
+ buffer.commit();
+ if (roles) {
+ roles->emplace_back(m.role());
+ }
+ outres++;
break;
}
}
}
- assert(way_ids.size() <= ids.size());
-
- PQclear(res);
-
- return way_ids.size();
+ return outres;
}
@@ -634,84 +603,80 @@ void middle_pgsql_t::way_changed(osmid_t osm_id)
//keep track of whatever rels this way intersects
//TODO: dont need to stop the copy above since we are only reading?
- PGresult* res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels_by_way", 1, paramValues, PGRES_TUPLES_OK );
- for(int i = 0; i < PQntuples(res); ++i)
- {
+ auto res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels_by_way", 1,
+ paramValues, PGRES_TUPLES_OK);
+ for (int i = 0; i < PQntuples(res.get()); ++i) {
char *end;
- osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10);
+ osmid_t marked = strtoosmid(PQgetvalue(res.get(), i, 0), &end, 10);
rels_pending_tracker->mark(marked);
}
- PQclear(res);
}
-void middle_pgsql_t::relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags)
+void middle_pgsql_t::relations_set(osmium::Relation const &rel)
{
- taglist_t member_list;
- char buf[64];
-
- idlist_t all_parts, node_parts, way_parts, rel_parts;
- all_parts.reserve(members.size());
- node_parts.reserve(members.size());
- way_parts.reserve(members.size());
- rel_parts.reserve(members.size());
-
- for (memberlist_t::const_iterator it = members.begin(); it != members.end(); ++it) {
- char type = 0;
- switch (it->type)
- {
- case OSMTYPE_NODE: node_parts.push_back(it->id); type = 'n'; break;
- case OSMTYPE_WAY: way_parts.push_back(it->id); type = 'w'; break;
- case OSMTYPE_RELATION: rel_parts.push_back(it->id); type = 'r'; break;
- default:
- fprintf(stderr, "Internal error: Unknown member type %d\n", it->type);
- util::exit_nicely();
- }
- sprintf( buf, "%c%" PRIdOSMID, type, it->id );
- member_list.push_back(tag_t(buf, it->role));
- }
+ idlist_t parts[3];
- all_parts.insert(all_parts.end(), node_parts.begin(), node_parts.end());
- all_parts.insert(all_parts.end(), way_parts.begin(), way_parts.end());
- all_parts.insert(all_parts.end(), rel_parts.begin(), rel_parts.end());
+ for (auto const &m : rel.members()) {
+ parts[osmium::item_type_to_nwr_index(m.type())].push_back(m.ref());
+ }
- copy_buffer.reserve(all_parts.size() * 10 + member_list.size() * 24
- + tags.size() * 24 + 64);
+ copy_buffer.reserve(rel.members().byte_size() * 2 + rel.tags().byte_size() + 128);
// Params: id, way_off, rel_off, parts, members, tags */
const char *paramValues[6] = { copy_buffer.c_str(), };
bool copy = rel_table->copyMode;
char delim = copy ? '\t' : '\0';
- copy_buffer = std::to_string(id);
+ copy_buffer = std::to_string(rel.id());
copy_buffer+= delim;
paramValues[1] = paramValues[0] + copy_buffer.size();
- copy_buffer += std::to_string(node_parts.size());
+ copy_buffer += std::to_string(parts[0].size());
copy_buffer+= delim;
paramValues[2] = paramValues[0] + copy_buffer.size();
- copy_buffer += std::to_string(node_parts.size() + way_parts.size());
+ copy_buffer += std::to_string(parts[0].size() + parts[1].size());
copy_buffer+= delim;
paramValues[3] = paramValues[0] + copy_buffer.size();
- buffer_store_nodes(all_parts);
+ if (rel.members().empty()) {
+ copy_buffer += "{}";
+ } else {
+ copy_buffer += "{";
+ for (int i = 0; i < 3; ++i) {
+ for (auto it : parts[i]) {
+ copy_buffer += std::to_string(it);
+ copy_buffer += ',';
+ }
+ }
+ copy_buffer[copy_buffer.size() - 1] = '}';
+ }
copy_buffer+= delim;
- if (member_list.size() == 0) {
+ if (rel.members().empty()) {
paramValues[4] = nullptr;
copy_buffer += "\\N";
} else {
paramValues[4] = paramValues[0] + copy_buffer.size();
- buffer_store_tags(member_list, copy);
+ copy_buffer += "{";
+ for (auto const &m : rel.members()) {
+ copy_buffer += '"';
+ copy_buffer += osmium::item_type_to_char(m.type());
+ copy_buffer += std::to_string(m.ref());
+ copy_buffer += "\",\"";
+ buffer_store_string(m.role(), copy);
+ copy_buffer += "\",";
+ }
+ copy_buffer[copy_buffer.size() - 1] = '}';
}
copy_buffer+= delim;
- if (tags.size() == 0) {
+ if (rel.tags().empty() && !out_options->extra_attributes) {
paramValues[5] = nullptr;
copy_buffer += "\\N";
} else {
paramValues[5] = paramValues[0] + copy_buffer.size();
- buffer_store_tags(tags, copy);
+ buffer_store_tags(rel, out_options->extra_attributes, copy);
}
if (copy) {
@@ -724,7 +689,7 @@ void middle_pgsql_t::relations_set(osmid_t id, const memberlist_t &members, cons
}
}
-bool middle_pgsql_t::relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const
+bool middle_pgsql_t::relations_get(osmid_t id, osmium::memory::Buffer &buffer) const
{
char tmp[16];
char const *paramValues[1];
@@ -732,36 +697,29 @@ bool middle_pgsql_t::relations_get(osmid_t id, memberlist_t &members, taglist_t
taglist_t member_temp;
// Make sure we're out of copy mode */
- pgsql_endCopy( rel_table );
+ pgsql_endCopy(rel_table);
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
paramValues[0] = tmp;
- PGresult *res = pgsql_execPrepared(sql_conn, "get_rel", 1, paramValues, PGRES_TUPLES_OK);
+ auto res = pgsql_execPrepared(sql_conn, "get_rel", 1, paramValues,
+ PGRES_TUPLES_OK);
// Fields are: members, tags, member_count */
- if (PQntuples(res) != 1) {
- PQclear(res);
+ if (PQntuples(res.get()) != 1) {
return false;
}
- pgsql_parse_tags(PQgetvalue(res, 0, 1), tags);
- pgsql_parse_tags(PQgetvalue(res, 0, 0), member_temp);
+ {
+ osmium::builder::RelationBuilder builder(buffer);
+ builder.set_id(id);
- if (member_temp.size() != strtoul(PQgetvalue(res, 0, 2), nullptr, 10)) {
- fprintf(stderr, "Unexpected member_count reading relation %" PRIdOSMID "\n", id);
- util::exit_nicely();
+ pgsql_parse_members(PQgetvalue(res.get(), 0, 0), buffer, builder);
+ pgsql_parse_tags(PQgetvalue(res.get(), 0, 1), buffer, builder);
}
- PQclear(res);
+ buffer.commit();
- for (taglist_t::const_iterator it = member_temp.begin(); it != member_temp.end(); ++it) {
- char tag = it->key[0];
- OsmType type = (tag == 'n')?OSMTYPE_NODE:(tag == 'w')?OSMTYPE_WAY:(tag == 'r')?OSMTYPE_RELATION:((OsmType)-1);
- members.push_back(member(type,
- strtoosmid(it->key.c_str()+1, nullptr, 10 ),
- it->value));
- }
return true;
}
@@ -779,14 +737,13 @@ void middle_pgsql_t::relations_delete(osmid_t osm_id)
//keep track of whatever ways this relation interesects
//TODO: dont need to stop the copy above since we are only reading?
- PGresult* res = pgsql_execPrepared(way_table->sql_conn, "mark_ways_by_rel", 1, paramValues, PGRES_TUPLES_OK );
- for(int i = 0; i < PQntuples(res); ++i)
- {
+ auto res = pgsql_execPrepared(way_table->sql_conn, "mark_ways_by_rel", 1,
+ paramValues, PGRES_TUPLES_OK);
+ for (int i = 0; i < PQntuples(res.get()); ++i) {
char *end;
- osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10);
+ osmid_t marked = strtoosmid(PQgetvalue(res.get(), i, 0), &end, 10);
ways_pending_tracker->mark(marked);
}
- PQclear(res);
}
void middle_pgsql_t::iterate_relations(pending_processor& pf)
@@ -820,14 +777,13 @@ void middle_pgsql_t::relation_changed(osmid_t osm_id)
//keep track of whatever ways and rels these nodes intersect
//TODO: dont need to stop the copy above since we are only reading?
//TODO: can we just mark the id without querying? the where clause seems intersect reltable.parts with the id
- PGresult* res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels", 1, paramValues, PGRES_TUPLES_OK );
- for(int i = 0; i < PQntuples(res); ++i)
- {
+ auto res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels", 1,
+ paramValues, PGRES_TUPLES_OK);
+ for (int i = 0; i < PQntuples(res.get()); ++i) {
char *end;
- osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10);
+ osmid_t marked = strtoosmid(PQgetvalue(res.get(), i, 0), &end, 10);
rels_pending_tracker->mark(marked);
}
- PQclear(res);
}
idlist_t middle_pgsql_t::relations_using_way(osmid_t way_id) const
@@ -840,14 +796,14 @@ idlist_t middle_pgsql_t::relations_using_way(osmid_t way_id) const
sprintf(buffer, "%" PRIdOSMID, way_id);
paramValues[0] = buffer;
- PGresult *result = pgsql_execPrepared(rel_table->sql_conn, "rels_using_way",
- 1, paramValues, PGRES_TUPLES_OK );
- const int ntuples = PQntuples(result);
- idlist_t rel_ids(ntuples);
+ auto result = pgsql_execPrepared(rel_table->sql_conn, "rels_using_way", 1,
+ paramValues, PGRES_TUPLES_OK);
+ const int ntuples = PQntuples(result.get());
+ idlist_t rel_ids;
+ rel_ids.resize((size_t) ntuples);
for (int i = 0; i < ntuples; ++i) {
- rel_ids[i] = strtoosmid(PQgetvalue(result, i, 0), nullptr, 10);
+ rel_ids[i] = strtoosmid(PQgetvalue(result.get(), i, 0), nullptr, 10);
}
- PQclear(result);
return rel_ids;
}
@@ -1013,10 +969,14 @@ void middle_pgsql_t::start(const options_t *out_options_)
// staying set for the second.
build_indexes = !append && !out_options->droptemp;
- cache.reset(new node_ram_cache( out_options->alloc_chunkwise | ALLOC_LOSSY, out_options->cache, out_options->scale));
- if (out_options->flat_node_cache_enabled) persistent_cache.reset(new node_persistent_cache(out_options, out_options->append, false, cache));
+ cache.reset(new node_ram_cache(out_options->alloc_chunkwise | ALLOC_LOSSY,
+ out_options->cache));
+
+ if (out_options->flat_node_cache_enabled) {
+ persistent_cache.reset(new node_persistent_cache(out_options, cache));
+ }
- fprintf(stderr, "Mid: pgsql, scale=%d cache=%d\n", out_options->scale, out_options->cache);
+ fprintf(stderr, "Mid: pgsql, cache=%d\n", out_options->cache);
// We use a connection per table to enable the use of COPY */
for (auto& table: tables) {
@@ -1041,7 +1001,7 @@ void middle_pgsql_t::start(const options_t *out_options_)
pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET client_min_messages = WARNING");
if (dropcreate) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS %s", table.name);
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS %s CASCADE", table.name);
}
if (table.start) {
@@ -1081,9 +1041,6 @@ void middle_pgsql_t::commit(void) {
table.transactionMode = 0;
}
}
- // Make sure the flat nodes are committed to disk or there will be
- // surprises later.
- if (out_options->flat_node_cache_enabled) persistent_cache.reset();
}
void middle_pgsql_t::pgsql_stop_one(table_desc *table)
@@ -1130,23 +1087,17 @@ void middle_pgsql_t::stop(void)
}
middle_pgsql_t::middle_pgsql_t()
- : tables(), num_tables(0), node_table(nullptr), way_table(nullptr), rel_table(nullptr),
- append(false), mark_pending(true), cache(), persistent_cache(), build_indexes(true)
+: num_tables(0), node_table(nullptr), way_table(nullptr), rel_table(nullptr),
+ append(false), mark_pending(true), build_indexes(true)
{
+ // clang-format off
/*table = t_node,*/
tables.push_back(table_desc(
/*name*/ "%p_nodes",
/*start*/ "BEGIN;\n",
-#ifdef FIXED_POINT
- /*create*/ "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat int4 not null, lon int4 not null, tags text[]) {TABLESPACE %t};\n",
- /*create_index*/ nullptr,
- /*prepare*/ "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", int4, int4, text[]) AS INSERT INTO %p_nodes VALUES ($1,$2,$3,$4);\n"
-#else
- /*create*/ "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat double precision not null, lon double precision not null, tags text[]) {TABLESPACE %t};\n",
+ /*create*/ "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat int4 not null, lon int4 not null) {TABLESPACE %t};\n",
/*create_index*/ nullptr,
- /*prepare*/ "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", double precision, double precision, text[]) AS INSERT INTO %p_nodes VALUES ($1,$2,$3,$4);\n"
-#endif
- "PREPARE get_node (" POSTGRES_OSMID_TYPE ") AS SELECT lat,lon,tags FROM %p_nodes WHERE id = $1 LIMIT 1;\n"
+ /*prepare*/ "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", int4, int4) AS INSERT INTO %p_nodes VALUES ($1,$2,$3);\n"
"PREPARE get_node_list(" POSTGRES_OSMID_TYPE "[]) AS SELECT id, lat, lon FROM %p_nodes WHERE id = ANY($1::" POSTGRES_OSMID_TYPE "[]);\n"
"PREPARE delete_node (" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_nodes WHERE id = $1;\n",
/*prepare_intarray*/ nullptr,
@@ -1193,6 +1144,7 @@ middle_pgsql_t::middle_pgsql_t()
/*stop*/ "COMMIT;\n",
/*array_indexes*/ "CREATE INDEX %p_rels_parts ON %p_rels USING gin (parts) WITH (FASTUPDATE=OFF) {TABLESPACE %i};\n"
));
+ // clang-format on
// set up the rest of the variables from the tables.
num_tables = tables.size();
@@ -1221,10 +1173,7 @@ std::shared_ptr<const middle_query_t> middle_pgsql_t::get_instance() const {
//NOTE: this is thread safe for use in pending async processing only because
//during that process they are only read from
mid->cache = cache;
- // The persistent cache on the other hand is not thread-safe for reading,
- // so we create one per instance.
- if (out_options->flat_node_cache_enabled)
- mid->persistent_cache.reset(new node_persistent_cache(out_options, 1, true, cache));
+ mid->persistent_cache = persistent_cache;
// We use a connection per table to enable the use of COPY */
for(int i=0; i<num_tables; i++) {
diff --git a/middle-pgsql.hpp b/middle-pgsql.hpp
index 1584570..40186df 100644
--- a/middle-pgsql.hpp
+++ b/middle-pgsql.hpp
@@ -20,36 +20,36 @@ struct middle_pgsql_t : public slim_middle_t {
middle_pgsql_t();
virtual ~middle_pgsql_t();
- void start(const options_t *out_options_);
- void stop(void);
- void analyze(void);
- void end(void);
- void commit(void);
+ void start(const options_t *out_options_) override;
+ void stop(void) override;
+ void analyze(void) override;
+ void end(void) override;
+ void commit(void) override;
- void nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags);
- size_t nodes_get_list(nodelist_t &out, const idlist_t nds) const;
- void nodes_delete(osmid_t id);
- void node_changed(osmid_t id);
+ void nodes_set(osmium::Node const &node) override;
+ size_t nodes_get_list(osmium::WayNodeList *nodes) const override;
+ void nodes_delete(osmid_t id) override;
+ void node_changed(osmid_t id) override;
- void ways_set(osmid_t id, const idlist_t &nds, const taglist_t &tags);
- bool ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const;
- size_t ways_get_list(const idlist_t &ids, idlist_t &way_ids,
- multitaglist_t &tags, multinodelist_t &nodes) const;
+ void ways_set(osmium::Way const &way) override;
+ bool ways_get(osmid_t id, osmium::memory::Buffer &buffer) const override;
+ size_t rel_way_members_get(osmium::Relation const &rel, rolelist_t *roles,
+ osmium::memory::Buffer &buffer) const override;
- void ways_delete(osmid_t id);
- void way_changed(osmid_t id);
+ void ways_delete(osmid_t id) override;
+ void way_changed(osmid_t id) override;
- bool relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const;
- void relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags);
- void relations_delete(osmid_t id);
- void relation_changed(osmid_t id);
+ bool relations_get(osmid_t id, osmium::memory::Buffer &buffer) const override;
+ void relations_set(osmium::Relation const &rel) override;
+ void relations_delete(osmid_t id) override;
+ void relation_changed(osmid_t id) override;
- void iterate_ways(middle_t::pending_processor& pf);
- void iterate_relations(pending_processor& pf);
+ void iterate_ways(middle_t::pending_processor& pf) override;
+ void iterate_relations(pending_processor& pf) override;
- size_t pending_count() const;
+ size_t pending_count() const override;
- std::vector<osmid_t> relations_using_way(osmid_t way_id) const;
+ idlist_t relations_using_way(osmid_t way_id) const override;
struct table_desc {
table_desc(const char *name_ = NULL,
@@ -79,7 +79,7 @@ struct middle_pgsql_t : public slim_middle_t {
struct pg_conn *sql_conn;
};
- virtual std::shared_ptr<const middle_query_t> get_instance() const;
+ std::shared_ptr<const middle_query_t> get_instance() const override;
private:
void pgsql_stop_one(table_desc *table);
@@ -87,8 +87,8 @@ private:
* Sets up sql_conn for the table
*/
void connect(table_desc& table);
- void local_nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags);
- size_t local_nodes_get_list(nodelist_t &out, const idlist_t nds) const;
+ void local_nodes_set(osmium::Node const &node);
+ size_t local_nodes_get_list(osmium::WayNodeList *nodes) const;
void local_nodes_delete(osmid_t osm_id);
std::vector<table_desc> tables;
@@ -103,9 +103,8 @@ private:
std::shared_ptr<id_tracker> ways_pending_tracker, rels_pending_tracker;
- void buffer_store_nodes(idlist_t const &nodes);
void buffer_store_string(std::string const &in, bool escape);
- void buffer_store_tags(taglist_t const &tags, bool escape);
+ void buffer_store_tags(osmium::OSMObject const &obj, bool attrs, bool escape);
void buffer_correct_params(char const **param, size_t size);
diff --git a/middle-ram.cpp b/middle-ram.cpp
index 9c449d9..97ea1a8 100644
--- a/middle-ram.cpp
+++ b/middle-ram.cpp
@@ -12,6 +12,8 @@
#include <cassert>
#include <cstdio>
+#include <osmium/builder/attr.hpp>
+
#include "id-tracker.hpp"
#include "middle-ram.hpp"
#include "node-ram-cache.hpp"
@@ -31,30 +33,34 @@
*
*/
-
-void middle_ram_t::nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags) {
- cache->set(id, lat, lon, tags);
+void middle_ram_t::nodes_set(osmium::Node const &node)
+{
+ cache->set(node.id(), node.location());
}
-void middle_ram_t::ways_set(osmid_t id, const idlist_t &nds, const taglist_t &tags)
+void middle_ram_t::ways_set(osmium::Way const &way)
{
- ways.set(id, new ramWay(tags, nds));
+ ways.set(way.id(), new ramWay(way, out_options->extra_attributes));
}
-void middle_ram_t::relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags)
+void middle_ram_t::relations_set(osmium::Relation const &rel)
{
- rels.set(id, new ramRel(tags, members));
+ rels.set(rel.id(), new ramRel(rel, out_options->extra_attributes));
}
-size_t middle_ram_t::nodes_get_list(nodelist_t &out, const idlist_t nds) const
+size_t middle_ram_t::nodes_get_list(osmium::WayNodeList *nodes) const
{
- for (idlist_t::const_iterator it = nds.begin(); it != nds.end(); ++it) {
- osmNode n;
- if (!cache->get(&n, *it))
- out.push_back(n);
+ size_t count = 0;
+
+ for (auto &n : *nodes) {
+ auto loc = cache->get(n.ref());
+ n.set_location(loc);
+ if (loc.valid()) {
+ ++count;
+ }
}
- return int(out.size());
+ return count;
}
void middle_ram_t::iterate_relations(pending_processor& pf)
@@ -94,7 +100,7 @@ void middle_ram_t::release_ways()
ways.clear();
}
-bool middle_ram_t::ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const
+bool middle_ram_t::ways_get(osmid_t id, osmium::memory::Buffer &buffer) const
{
if (simulate_ways_deleted) {
return false;
@@ -106,44 +112,30 @@ bool middle_ram_t::ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) cons
return false;
}
- tags = ele->tags;
- nodes_get_list(nodes, ele->ndids);
+ using namespace osmium::builder::attr;
+ osmium::builder::add_way(buffer, _id(id), _tags(ele->tags), _nodes(ele->ndids));
return true;
}
-size_t middle_ram_t::ways_get_list(const idlist_t &ids, idlist_t &way_ids,
- multitaglist_t &tags, multinodelist_t &nodes) const
+size_t middle_ram_t::rel_way_members_get(osmium::Relation const &rel,
+ rolelist_t *roles,
+ osmium::memory::Buffer &buffer) const
{
- if (ids.empty())
- {
- return 0;
- }
-
- assert(way_ids.empty());
- tags.assign(ids.size(), taglist_t());
- nodes.assign(ids.size(), nodelist_t());
-
size_t count = 0;
- for (idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) {
- if (ways_get(*it, tags[count], nodes[count])) {
- way_ids.push_back(*it);
- count++;
- } else {
- tags[count].clear();
- nodes[count].clear();
+ for (auto const &m : rel.members()) {
+ if (m.type() == osmium::item_type::way && ways_get(m.ref(), buffer)) {
+ if (roles) {
+ roles->emplace_back(m.role());
+ }
+ ++count;
}
}
- if (count < ids.size()) {
- tags.resize(count);
- nodes.resize(count);
- }
-
- return int(count);
+ return count;
}
-bool middle_ram_t::relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const
+bool middle_ram_t::relations_get(osmid_t id, osmium::memory::Buffer &buffer) const
{
auto const *ele = rels.get(id);
@@ -151,8 +143,9 @@ bool middle_ram_t::relations_get(osmid_t id, memberlist_t &members, taglist_t &t
return false;
}
- tags = ele->tags;
- members = ele->members;
+ using namespace osmium::builder::attr;
+ osmium::builder::add_relation(buffer, _id(id), _members(ele->members.for_builder()),
+ _tags(ele->tags));
return true;
}
@@ -170,12 +163,8 @@ void middle_ram_t::end(void)
void middle_ram_t::start(const options_t *out_options_)
{
out_options = out_options_;
- /* latlong has a range of +-180, mercator +-20000
- The fixed poing scaling needs adjusting accordingly to
- be stored accurately in an int */
- cache.reset(new node_ram_cache(out_options->alloc_chunkwise, out_options->cache, out_options->scale));
-
- fprintf( stderr, "Mid: Ram, scale=%d\n", out_options->scale );
+ cache.reset(
+ new node_ram_cache(out_options->alloc_chunkwise, out_options->cache));
}
void middle_ram_t::stop(void)
@@ -198,19 +187,21 @@ middle_ram_t::~middle_ram_t() {
//instance.reset();
}
-std::vector<osmid_t> middle_ram_t::relations_using_way(osmid_t way_id) const
+idlist_t middle_ram_t::relations_using_way(osmid_t) const
{
// this function shouldn't be called - relations_using_way is only used in
// slim mode, and a middle_ram_t shouldn't be constructed if the slim mode
// option is set.
- throw std::runtime_error("middle_ram_t::relations_using_way is unimlpemented, and "
- "should not have been called. This is probably a bug, please "
- "report it at https://github.com/openstreetmap/osm2pgsql/issues");
+ throw std::runtime_error(
+ "middle_ram_t::relations_using_way is unimlpemented, and "
+ "should not have been called. This is probably a bug, please "
+ "report it at https://github.com/openstreetmap/osm2pgsql/issues");
}
namespace {
-void no_delete(const middle_ram_t * middle) {
+void no_delete(const middle_ram_t *)
+{
// boost::shared_ptr thinks we are going to delete
// the middle object, but we are not. Heh heh heh.
// So yeah, this is a hack...
diff --git a/middle-ram.hpp b/middle-ram.hpp
index 32f3bff..afcecd9 100644
--- a/middle-ram.hpp
+++ b/middle-ram.hpp
@@ -75,7 +75,7 @@ public:
void clear()
{
for (auto &ele : arr) {
- ele.release();
+ ele.reset();
}
}
};
@@ -84,38 +84,38 @@ struct middle_ram_t : public middle_t {
middle_ram_t();
virtual ~middle_ram_t();
- void start(const options_t *out_options_);
- void stop(void);
- void analyze(void);
- void end(void);
- void commit(void);
+ void start(const options_t *out_options_) override;
+ void stop(void) override;
+ void analyze(void) override;
+ void end(void) override;
+ void commit(void) override;
- void nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags);
- size_t nodes_get_list(nodelist_t &out, const idlist_t nds) const;
+ void nodes_set(osmium::Node const &node) override;
+ size_t nodes_get_list(osmium::WayNodeList *nodes) const override;
int nodes_delete(osmid_t id);
int node_changed(osmid_t id);
- void ways_set(osmid_t id, const idlist_t &nds, const taglist_t &tags);
- bool ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const;
- size_t ways_get_list(const idlist_t &ids, idlist_t &way_ids,
- multitaglist_t &tags, multinodelist_t &nodes) const;
+ void ways_set(osmium::Way const &way) override;
+ bool ways_get(osmid_t id, osmium::memory::Buffer &buffer) const override;
+ size_t rel_way_members_get(osmium::Relation const &rel, rolelist_t *roles,
+ osmium::memory::Buffer &buffer) const override;
int ways_delete(osmid_t id);
int way_changed(osmid_t id);
- bool relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const;
- void relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags);
+ bool relations_get(osmid_t id, osmium::memory::Buffer &buffer) const override;
+ void relations_set(osmium::Relation const &rel) override;
int relations_delete(osmid_t id);
int relation_changed(osmid_t id);
- std::vector<osmid_t> relations_using_way(osmid_t way_id) const;
+ idlist_t relations_using_way(osmid_t way_id) const override;
- void iterate_ways(middle_t::pending_processor& pf);
- void iterate_relations(pending_processor& pf);
+ void iterate_ways(middle_t::pending_processor& pf) override;
+ void iterate_relations(pending_processor& pf) override;
- size_t pending_count() const;
+ size_t pending_count() const override;
- virtual std::shared_ptr<const middle_query_t> get_instance() const;
+ std::shared_ptr<const middle_query_t> get_instance() const override;
private:
void release_ways();
@@ -125,14 +125,24 @@ private:
taglist_t tags;
idlist_t ndids;
- ramWay(const taglist_t &t, const idlist_t &n) : tags(t), ndids(n) {}
+ ramWay(osmium::Way const &way, bool add_attributes)
+ : tags(way.tags()), ndids(way.nodes())
+ {
+ if (add_attributes)
+ tags.add_attributes(way);
+ }
};
struct ramRel {
taglist_t tags;
memberlist_t members;
- ramRel(const taglist_t &t, const memberlist_t &m) : tags(t), members(m) {}
+ ramRel(osmium::Relation const &rel, bool add_attributes)
+ : tags(rel.tags()), members(rel.members())
+ {
+ if (add_attributes)
+ tags.add_attributes(rel);
+ }
};
elem_cache_t<ramWay, 10> ways;
diff --git a/middle.hpp b/middle.hpp
index b85da7b..0e6a6ab 100644
--- a/middle.hpp
+++ b/middle.hpp
@@ -7,11 +7,14 @@
#ifndef MIDDLE_H
#define MIDDLE_H
-#include "osmtypes.hpp"
+#include <osmium/memory/buffer.hpp>
#include <cstddef>
#include <memory>
+#include "osmtypes.hpp"
+#include "reprojection.hpp"
+
struct options_t;
/**
@@ -20,25 +23,48 @@ struct options_t;
struct middle_query_t {
virtual ~middle_query_t() {}
- virtual size_t nodes_get_list(nodelist_t &out, const idlist_t nds) const = 0;
+ /**
+ * Retrives node locations for the given node list.
+ *
+ * The locations are saved directly in the input list.
+ */
+ virtual size_t nodes_get_list(osmium::WayNodeList *nodes) const = 0;
/**
- * Retrives a single way from the ways storage.
+ * Retrives a single way from the ways storage
+ * and stores it in the given osmium buffer.
+ *
+ * \param id id of the way to retrive
+ * \param buffer osmium buffer where to put the way
+ *
+ * The function does not retrieve the node locations.
+ *
* \return true if the way was retrieved
- * \param id id of the way to retrive
*/
- virtual bool ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const = 0;
+ virtual bool ways_get(osmid_t id, osmium::memory::Buffer &buffer) const = 0;
- virtual size_t ways_get_list(const idlist_t &ids, idlist_t &way_ids,
- multitaglist_t &tags,
- multinodelist_t &nodes) const = 0;
+ /**
+ * Retrives the way members of a relation and stores them in
+ * the given osmium buffer.
+ *
+ * \param rel Relation to get the members for.
+ * \param[out] roles Roles for the ways that where retrived.
+ * \param[out] buffer Buffer where to store the members in.
+ */
+ virtual size_t
+ rel_way_members_get(osmium::Relation const &rel, rolelist_t *roles,
+ osmium::memory::Buffer &buffer) const = 0;
/**
- * Retrives a single relation from the relation storage.
+ * Retrives a single relation from the relation storage
+ * and stores it in the given osmium buffer.
+ *
+ * \param id id of the relation to retrive
+ * \param buffer osmium buffer where to put the relation
+ *
* \return true if the relation was retrieved
- * \param id id of the relation to retrive
*/
- virtual bool relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const = 0;
+ virtual bool relations_get(osmid_t id, osmium::memory::Buffer &buffer) const = 0;
/*
* Retrieve a list of relations with a particular way as a member
@@ -63,9 +89,9 @@ struct middle_t : public middle_query_t {
virtual void end(void) = 0;
virtual void commit(void) = 0;
- virtual void nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags) = 0;
- virtual void ways_set(osmid_t id, const idlist_t &nds, const taglist_t &tags) = 0;
- virtual void relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags) = 0;
+ virtual void nodes_set(osmium::Node const &node) = 0;
+ virtual void ways_set(osmium::Way const &way) = 0;
+ virtual void relations_set(osmium::Relation const &rel) = 0;
struct pending_processor {
virtual ~pending_processor() {}
diff --git a/multi.lua b/multi.lua
index f33e1b3..03bec17 100644
--- a/multi.lua
+++ b/multi.lua
@@ -218,9 +218,9 @@ function generic_rel_members (f, keyvals, keyvaluemembers, roles, membercount, t
--mark each way of the relation to tell the caller if its going
--to be used in the relation or by itself as its own standalone way
--we start by assuming each way will not be used as part of the relation
- membersuperseeded = {}
+ membersuperseded = {}
for i = 1, membercount do
- membersuperseeded[i] = 0
+ membersuperseded[i] = 0
end
--remember the type on the relation and erase it from the tags
@@ -249,23 +249,23 @@ function generic_rel_members (f, keyvals, keyvaluemembers, roles, membercount, t
end
if filter == 1 then
tags = t(keyvals)
- return filter, tags, membersuperseeded, boundary, polygon, roads
+ return filter, tags, membersuperseded, boundary, polygon, roads
end
--for each tag of each member if the relation have the tag or has a non matching value for it
- --then we say the member will not be used in the relation and is there for not superseeded
+ --then we say the member will not be used in the relation and is there for not superseded
--ie it is kept as a standalone way
for i = 1,membercount do
- superseeded = 1
+ superseded = 1
for k,v in pairs(keyvaluemembers[i]) do
if ((keyvals[k] == nil) or (keyvals[k] ~= v)) then
- superseeded = 0;
+ superseded = 0;
break
end
end
- membersuperseeded[i] = superseeded
+ membersuperseded[i] = superseded
end
end
tags = t(keyvals)
- return filter, tags, membersuperseeded, boundary, polygon, roads
+ return filter, tags, membersuperseded, boundary, polygon, roads
end
diff --git a/node-persistent-cache.cpp b/node-persistent-cache.cpp
index 317459d..45f96de 100644
--- a/node-persistent-cache.cpp
+++ b/node-persistent-cache.cpp
@@ -1,656 +1,78 @@
#define _LARGEFILE64_SOURCE /* See feature_test_macrors(7) */
-#include "config.h"
-
-#include <algorithm>
-#include <stdexcept>
-
-#include <cerrno>
-#include <climits>
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
-
-#include <fcntl.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
#include "node-persistent-cache.hpp"
#include "options.hpp"
-#include "osmtypes.hpp"
-#include "output.hpp"
-#include "util.hpp"
-
-#ifdef _WIN32
- #include "win_fsync.h"
- #define lseek64 _lseeki64
- #ifndef S_IRUSR
- #define S_IRUSR S_IREAD
- #endif
- #ifndef S_IWUSR
- #define S_IWUSR S_IWRITE
- #endif
-#else
- #ifdef __APPLE__
- #define lseek64 lseek
- #else
- #ifndef HAVE_LSEEK64
- #if SIZEOF_OFF_T == 8
- #define lseek64 lseek
- #else
- #error Flat nodes cache requires a 64 bit capable seek
- #endif
- #endif
- #endif
-#endif
-
-void node_persistent_cache::writeout_dirty_nodes()
-{
- for (int i = 0; i < READ_NODE_CACHE_SIZE; i++)
- {
- if (readNodeBlockCache[i].dirty())
- {
- if (lseek64(node_cache_fd,
- ((osmid_t) readNodeBlockCache[i].block_offset
- << READ_NODE_BLOCK_SHIFT)
- * sizeof(ramNode)
- + sizeof(persistentCacheHeader),
- SEEK_SET) < 0) {
- fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- };
- if (write(node_cache_fd, readNodeBlockCache[i].nodes,
- READ_NODE_BLOCK_SIZE * sizeof(ramNode))
- < ssize_t(READ_NODE_BLOCK_SIZE * sizeof(ramNode)))
- {
- fprintf(stderr, "Failed to write out node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- }
- }
- readNodeBlockCache[i].reset_used();
- }
-}
-
-
-/**
- * Find the cache block with the lowest usage count for replacement
- */
-size_t node_persistent_cache::replace_block()
-{
- int min_used = INT_MAX;
- int block_id = -1;
-
- for (int i = 0; i < READ_NODE_CACHE_SIZE; i++)
- {
- if (readNodeBlockCache[i].used() < min_used)
- {
- min_used = readNodeBlockCache[i].used();
- block_id = i;
- }
- }
- if (min_used > 0)
- {
- for (int i = 0; i < READ_NODE_CACHE_SIZE; i++)
- {
- if (readNodeBlockCache[i].used() > 1)
- {
- readNodeBlockCache[i].dec_used();
- }
- }
- }
- return block_id;
-}
-
-/**
- * Find cache block number by block_offset
- */
-int node_persistent_cache::find_block(osmid_t block_offset)
-{
- cache_index::iterator it = std::lower_bound(readNodeBlockCacheIdx.begin(),
- readNodeBlockCacheIdx.end(),
- block_offset);
- if (it != readNodeBlockCacheIdx.end() && it->key == block_offset)
- return it->value;
-
- return -1;
-}
-
-void node_persistent_cache::remove_from_cache_idx(osmid_t block_offset)
-{
- cache_index::iterator it = std::lower_bound(readNodeBlockCacheIdx.begin(),
- readNodeBlockCacheIdx.end(),
- block_offset);
-
- if (it == readNodeBlockCacheIdx.end() || it->key != block_offset)
- return;
-
- readNodeBlockCacheIdx.erase(it);
-}
-
-void node_persistent_cache::add_to_cache_idx(cache_index_entry const &entry)
-{
- cache_index::iterator it = std::lower_bound(readNodeBlockCacheIdx.begin(),
- readNodeBlockCacheIdx.end(),
- entry);
- readNodeBlockCacheIdx.insert(it, entry);
-}
-
-// A cache block with invalid nodes, just for writing out empty cache blocks
-static const ramNode nullNodes[READ_NODE_BLOCK_SIZE];
-/**
- * Initialise the persistent cache with NaN values to identify which IDs are valid or not
- */
-void node_persistent_cache::expand_cache(osmid_t block_offset)
-{
- /* Need to expand the persistent node cache */
- if (lseek64(node_cache_fd,
- cacheHeader.max_initialised_id * sizeof(ramNode)
- + sizeof(persistentCacheHeader), SEEK_SET) < 0) {
- fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- };
- for (osmid_t i = cacheHeader.max_initialised_id >> READ_NODE_BLOCK_SHIFT;
- i <= block_offset; i++)
- {
- if (write(node_cache_fd, nullNodes, sizeof(nullNodes))
- < ssize_t(sizeof(nullNodes)))
- {
- fprintf(stderr, "Failed to expand persistent node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- }
- }
- cacheHeader.max_initialised_id = ((block_offset + 1)
- << READ_NODE_BLOCK_SHIFT) - 1;
- if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
- fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- };
- if (write(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader))
- != ssize_t(sizeof(struct persistentCacheHeader)))
- {
- fprintf(stderr, "Failed to update persistent cache header: %s\n",
- strerror(errno));
- util::exit_nicely();
- }
- fsync(node_cache_fd);
-}
-
-void node_persistent_cache::nodes_prefetch_async(osmid_t id)
+void node_persistent_cache::set(osmid_t id, const osmium::Location &coord)
{
-#ifdef HAVE_POSIX_FADVISE
- osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT;
-
- const int block_id = find_block(block_offset);
-
- if (block_id < 0) {
- // The needed block isn't in cache already, so initiate loading
- if (cacheHeader.max_initialised_id < id) {
- fprintf(stderr, "Warning: reading node outside node cache. (%lu vs. %lu)\n",
- cacheHeader.max_initialised_id, id);
- return;
- }
-
- if (posix_fadvise(node_cache_fd, (block_offset << READ_NODE_BLOCK_SHIFT) * sizeof(ramNode)
- + sizeof(persistentCacheHeader), READ_NODE_BLOCK_SIZE * sizeof(ramNode),
- POSIX_FADV_WILLNEED | POSIX_FADV_RANDOM) != 0) {
- fprintf(stderr, "Info: async prefetch of node cache failed. This might reduce performance\n");
- };
+ if (id < 0) {
+ throw std::runtime_error("Flatnode store cannot save negative IDs.");
}
-#endif
+ m_index->set(static_cast<osmium::unsigned_object_id_type>(id), coord);
}
-
-/**
- * Load block offset in a synchronous way.
- */
-int node_persistent_cache::load_block(osmid_t block_offset)
+osmium::Location node_persistent_cache::get(osmid_t id)
{
- const size_t block_id = replace_block();
-
- if (readNodeBlockCache[block_id].dirty())
- {
- if (lseek64(node_cache_fd,
- ((osmid_t) readNodeBlockCache[block_id].block_offset
- << READ_NODE_BLOCK_SHIFT) * sizeof(ramNode)
- + sizeof(struct persistentCacheHeader), SEEK_SET) < 0) {
- fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- };
- if (write(node_cache_fd, readNodeBlockCache[block_id].nodes,
- READ_NODE_BLOCK_SIZE * sizeof(ramNode))
- < ssize_t(READ_NODE_BLOCK_SIZE * sizeof(ramNode)))
- {
- fprintf(stderr, "Failed to write out node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- }
- readNodeBlockCache[block_id].reset_used();
- }
-
- if (readNodeBlockCache[block_id].nodes) {
- remove_from_cache_idx((osmid_t) readNodeBlockCache[block_id].block_offset);
- new(readNodeBlockCache[block_id].nodes) ramNode[READ_NODE_BLOCK_SIZE];
- } else {
- readNodeBlockCache[block_id].nodes = new ramNode[READ_NODE_BLOCK_SIZE];
- if (!readNodeBlockCache[block_id].nodes) {
- fprintf(stderr, "Out of memory: Failed to allocate node read cache\n");
- util::exit_nicely();
- }
- }
- readNodeBlockCache[block_id].block_offset = block_offset;
- readNodeBlockCache[block_id].set_used(READ_NODE_CACHE_SIZE);
-
- /* Make sure the node cache is correctly initialised for the block that will be read */
- if (cacheHeader.max_initialised_id
- < ((block_offset + 1) << READ_NODE_BLOCK_SHIFT))
- {
- expand_cache(block_offset);
- }
-
- /* Read the block into cache */
- if (lseek64(node_cache_fd,
- (block_offset << READ_NODE_BLOCK_SHIFT) * sizeof(ramNode)
- + sizeof(struct persistentCacheHeader), SEEK_SET) < 0) {
- fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- };
- if (read(node_cache_fd, readNodeBlockCache[block_id].nodes,
- READ_NODE_BLOCK_SIZE * sizeof(ramNode))
- != READ_NODE_BLOCK_SIZE * sizeof(ramNode))
- {
- fprintf(stderr, "Failed to read from node cache: %s\n",
- strerror(errno));
- exit(1);
- }
- add_to_cache_idx(cache_index_entry(block_offset, block_id));
-
- return block_id;
-}
-
-void node_persistent_cache::nodes_set_create_writeout_block()
-{
- if (write(node_cache_fd, writeNodeBlock.nodes,
- WRITE_NODE_BLOCK_SIZE * sizeof(ramNode))
- < ssize_t(WRITE_NODE_BLOCK_SIZE * sizeof(ramNode)))
- {
- fprintf(stderr, "Failed to write out node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- }
-#ifdef HAVE_SYNC_FILE_RANGE
- /* writing out large files can cause trouble on some operating systems.
- * For one, if to much dirty data is in RAM, the whole OS can stall until
- * enough dirty data is written out which can take a while. It can also interfere
- * with other disk caching operations and might push things out to swap. By forcing the OS to
- * immediately write out the data and blocking after a while, we ensure that no more
- * than a couple of 10s of MB are dirty in RAM at a time.
- * Secondly, the nodes are stored in an additional ram cache during import. Keeping the
- * node cache file in buffer cache therefore duplicates the data wasting 16GB of ram.
- * Therefore tell the OS not to cache the node-persistent-cache during initial import.
- * */
- if (sync_file_range(node_cache_fd, (osmid_t) writeNodeBlock.block_offset*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) +
- sizeof(persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(ramNode),
- SYNC_FILE_RANGE_WRITE) < 0) {
- fprintf(stderr, "Info: Sync_file_range writeout has an issue. This shouldn't be anything to worry about.: %s\n",
- strerror(errno));
- };
-
- if (writeNodeBlock.block_offset > 16) {
- if(sync_file_range(node_cache_fd, ((osmid_t) writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) +
- sizeof(persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(ramNode),
- SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER) < 0) {
- fprintf(stderr, "Info: Sync_file_range block has an issue. This shouldn't be anything to worry about.: %s\n",
- strerror(errno));
-
- }
-#ifdef HAVE_POSIX_FADVISE
- if (posix_fadvise(node_cache_fd, ((osmid_t) writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) +
- sizeof(persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(ramNode), POSIX_FADV_DONTNEED) !=0 ) {
- fprintf(stderr, "Info: Posix_fadvise failed. This shouldn't be anything to worry about.: %s\n",
- strerror(errno));
- };
-#endif
- }
-#endif
-}
-
-void node_persistent_cache::set_create(osmid_t id, double lat, double lon)
-{
- assert(!append_mode);
- assert(!read_mode);
-
- int32_t block_offset = id >> WRITE_NODE_BLOCK_SHIFT;
-
- if (writeNodeBlock.block_offset != block_offset)
- {
- if (writeNodeBlock.dirty())
- {
- nodes_set_create_writeout_block();
- /* After writing out the node block, the file pointer is at the next block level */
- writeNodeBlock.block_offset++;
- cacheHeader.max_initialised_id = ((osmid_t) writeNodeBlock.block_offset
- << WRITE_NODE_BLOCK_SHIFT) - 1;
- }
- if (writeNodeBlock.block_offset > block_offset)
- {
- fprintf(stderr,
- "ERROR: Block_offset not in sequential order: %d %d\n",
- writeNodeBlock.block_offset, block_offset);
- util::exit_nicely();
+ if (id >= 0) {
+ try {
+ return m_index->get(
+ static_cast<osmium::unsigned_object_id_type>(id));
+ } catch (osmium::not_found const &) {
}
-
- new(writeNodeBlock.nodes) ramNode[WRITE_NODE_BLOCK_SIZE];
-
- /* We need to fill the intermediate node cache with node nodes to identify which nodes are valid */
- while (writeNodeBlock.block_offset < block_offset)
- {
- nodes_set_create_writeout_block();
- writeNodeBlock.block_offset++;
- }
-
- }
-
- writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK] = ramNode(lon, lat);
- writeNodeBlock.set_dirty();
-}
-
-void node_persistent_cache::set_append(osmid_t id, double lat, double lon)
-{
- assert(!read_mode);
-
- osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT;
-
- int block_id = find_block(block_offset);
-
- if (block_id < 0)
- block_id = load_block(block_offset);
-
- if (std::isnan(lat) && std::isnan(lon)) {
- readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK] = ramNode();
- } else {
- readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK] = ramNode(lon, lat);
}
- readNodeBlockCache[block_id].inc_used();
- readNodeBlockCache[block_id].set_dirty();
-}
-void node_persistent_cache::set(osmid_t id, double lat, double lon)
-{
- if (append_mode) {
- set_append(id, lat, lon);
- } else {
- set_create(id, lat, lon);
- }
+ return osmium::Location();
}
-int node_persistent_cache::get(osmNode *out, osmid_t id)
+size_t node_persistent_cache::get_list(osmium::WayNodeList *nodes)
{
- set_read_mode();
-
- osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT;
-
- int block_id = find_block(block_offset);
+ size_t count = 0;
- if (block_id < 0)
- {
- block_id = load_block(block_offset);
- }
-
- readNodeBlockCache[block_id].inc_used();
-
- if (!readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].is_valid())
- return 1;
-
- out->lat = readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat();
- out->lon = readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon();
-
- return 0;
-}
-
-size_t node_persistent_cache::get_list(nodelist_t &out, const idlist_t nds)
-{
- set_read_mode();
-
- out.assign(nds.size(), osmNode());
-
- bool need_fetch = false;
- for (size_t i = 0; i < nds.size(); ++i) {
+ for (auto &n : *nodes) {
+ auto loc = m_ram_cache->get(n.ref());
/* Check cache first */
- if (ram_cache->get(&out[i], nds[i]) != 0) {
- /* In order to have a higher OS level I/O queue depth
- issue posix_fadvise(WILLNEED) requests for all I/O */
- nodes_prefetch_async(nds[i]);
- need_fetch = true;
- }
- }
- if (!need_fetch)
- return out.size();
-
- size_t wrtidx = 0;
- for (size_t i = 0; i < nds.size(); i++) {
- if (std::isnan(out[i].lat) && std::isnan(out[i].lon)) {
- if (get(&(out[wrtidx]), nds[i]) == 0)
- wrtidx++;
- } else {
- if (wrtidx < i)
- out[wrtidx] = out[i];
- wrtidx++;
+ if (!loc.valid() && n.ref() >= 0) {
+ try {
+ loc = m_index->get(
+ static_cast<osmium::unsigned_object_id_type>(n.ref()));
+ } catch (osmium::not_found const &) {
+ }
}
- }
-
- out.resize(wrtidx);
-
- return wrtidx;
-}
-
-void node_persistent_cache::set_read_mode()
-{
- if (read_mode)
- return;
-
- if (writeNodeBlock.dirty()) {
- assert(!append_mode);
- nodes_set_create_writeout_block();
- writeNodeBlock.reset_used();
- writeNodeBlock.block_offset++;
- cacheHeader.max_initialised_id = ((osmid_t) writeNodeBlock.block_offset
- << WRITE_NODE_BLOCK_SHIFT) - 1;
-
- /* write out the header */
- if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
- fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- };
- if (write(node_cache_fd, &cacheHeader, sizeof(persistentCacheHeader))
- != sizeof(persistentCacheHeader)) {
- fprintf(stderr, "Failed to update persistent cache header: %s\n",
- strerror(errno));
- util::exit_nicely();
+ n.set_location(loc);
+ if (loc.valid()) {
+ ++count;
}
}
- read_mode = true;
+ return count;
}
-node_persistent_cache::node_persistent_cache(const options_t *options, bool append,
- bool ro, std::shared_ptr<node_ram_cache> ptr)
- : node_cache_fd(0), node_cache_fname(nullptr), append_mode(append), cacheHeader(),
- writeNodeBlock(), readNodeBlockCache(nullptr), read_mode(ro), ram_cache(ptr)
+node_persistent_cache::node_persistent_cache(
+ const options_t *options, std::shared_ptr<node_ram_cache> ptr)
+: m_ram_cache(ptr), m_fd(-1)
{
- if (options->flat_node_file) {
- node_cache_fname = options->flat_node_file->c_str();
- } else {
+ if (!options->flat_node_file) {
throw std::runtime_error("Unable to set up persistent cache: the name "
"of the flat node file was not set.");
}
- fprintf(stderr, "Mid: loading persistent node cache from %s\n",
- node_cache_fname);
- readNodeBlockCacheIdx.reserve(READ_NODE_CACHE_SIZE);
+ auto fname = options->flat_node_file->c_str();
+ fprintf(stderr, "Mid: loading persistent node cache from %s\n", fname);
- /* Setup the file for the node position cache */
- if (append_mode)
- {
- node_cache_fd = open(node_cache_fname, O_RDWR, S_IRUSR | S_IWUSR);
- if (node_cache_fd < 0)
- {
- fprintf(stderr, "Failed to open node cache file: %s\n",
- strerror(errno));
- util::exit_nicely();
- }
+ m_fd = open(fname, O_RDWR | O_CREAT, 0644);
+ if (m_fd < 0) {
+ fprintf(stderr, "Cannot open location cache file '%s': %s\n", fname,
+ std::strerror(errno));
+ throw std::runtime_error("Unable to open flatnode file\n");
}
- else
- {
- if (read_mode)
- {
- node_cache_fd = open(node_cache_fname, O_RDWR, S_IRUSR | S_IWUSR);
- }
- else
- {
- node_cache_fd = open(node_cache_fname, O_RDWR | O_CREAT | O_TRUNC,
- S_IRUSR | S_IWUSR);
- }
-
- if (node_cache_fd < 0)
- {
- fprintf(stderr, "Failed to create node cache file: %s\n",
- strerror(errno));
- util::exit_nicely();
- }
- if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
- fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- };
-
- writeNodeBlock.block_offset = 0;
-
- if (!read_mode)
- {
- #ifdef HAVE_POSIX_FALLOCATE
- int err;
- if ((err = posix_fallocate(node_cache_fd, 0,
- sizeof(ramNode) * MAXIMUM_INITIAL_ID)) != 0) {
- if (err == ENOSPC) {
- fprintf(stderr, "Failed to allocate space for node cache file: No space on disk\n");
- } else if (err == EFBIG) {
- fprintf(stderr, "Failed to allocate space for node cache file: File is too big\n");
- } else {
- fprintf(stderr, "Failed to allocate space for node cache file: Internal error %i\n", err);
- }
-
- close(node_cache_fd);
- util::exit_nicely();
- }
- fprintf(stderr, "Allocated space for persistent node cache file\n");
- #endif
-
- writeNodeBlock.nodes = new ramNode[WRITE_NODE_BLOCK_SIZE];
- if (!writeNodeBlock.nodes) {
- fprintf(stderr, "Out of memory: Failed to allocate node writeout buffer\n");
- util::exit_nicely();
- }
- cacheHeader.format_version = PERSISTENT_CACHE_FORMAT_VERSION;
- cacheHeader.id_size = sizeof(osmid_t);
- cacheHeader.max_initialised_id = 0;
- if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
- fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- };
- if (write(node_cache_fd, &cacheHeader,
- sizeof(struct persistentCacheHeader))
- != sizeof(struct persistentCacheHeader))
- {
- fprintf(stderr, "Failed to write persistent cache header: %s\n",
- strerror(errno));
- util::exit_nicely();
- }
- }
- }
- if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
- fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- };
- if (read(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader))
- != sizeof(struct persistentCacheHeader))
- {
- fprintf(stderr, "Failed to read persistent cache header: %s\n",
- strerror(errno));
- util::exit_nicely();
- }
- if (cacheHeader.format_version != PERSISTENT_CACHE_FORMAT_VERSION)
- {
- fprintf(stderr, "Persistent cache header is wrong version\n");
- util::exit_nicely();
- }
-
- if (cacheHeader.id_size != sizeof(osmid_t))
- {
- fprintf(stderr, "Persistent cache header is wrong id type\n");
- util::exit_nicely();
- }
-
- fprintf(stderr,"Maximum node in persistent node cache: %" PRIdOSMID "\n", cacheHeader.max_initialised_id);
-
- readNodeBlockCache = new ramNodeBlock[READ_NODE_CACHE_SIZE];
- if (!readNodeBlockCache) {
- fprintf(stderr, "Out of memory: Failed to allocate node read cache\n");
- util::exit_nicely();
- }
+ m_index.reset(new index_t{m_fd});
}
node_persistent_cache::~node_persistent_cache()
{
- if (writeNodeBlock.dirty())
- nodes_set_create_writeout_block();
-
- writeout_dirty_nodes();
-
- if (writeNodeBlock.nodes)
- delete[] writeNodeBlock.nodes;
-
- if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
- fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
- strerror(errno));
- util::exit_nicely();
- };
- if (write(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader))
- != sizeof(struct persistentCacheHeader))
- {
- fprintf(stderr, "Failed to update persistent cache header: %s\n",
- strerror(errno));
- util::exit_nicely();
- }
- fprintf(stderr,"Maximum node in persistent node cache: %" PRIdOSMID "\n", cacheHeader.max_initialised_id);
-
- fsync(node_cache_fd);
-
- if (close(node_cache_fd) != 0)
- {
- fprintf(stderr, "Failed to close node cache file: %s\n",
- strerror(errno));
- }
-
- if (readNodeBlockCache) {
- for (int i = 0; i < READ_NODE_CACHE_SIZE; i++)
- {
- if (readNodeBlockCache[i].nodes)
- delete[] readNodeBlockCache[i].nodes;
- }
- delete[] readNodeBlockCache;
+ m_index.reset();
+ if (m_fd >= 0) {
+ close(m_fd);
}
}
diff --git a/node-persistent-cache.hpp b/node-persistent-cache.hpp
index 9428c82..1902d96 100644
--- a/node-persistent-cache.hpp
+++ b/node-persistent-cache.hpp
@@ -1,95 +1,37 @@
#ifndef NODE_PERSISTENT_CACHE_H
#define NODE_PERSISTENT_CACHE_H
-#include "osmtypes.hpp"
-#include "node-ram-cache.hpp"
#include <memory>
-#include <vector>
-
-#define MAXIMUM_INITIAL_ID 2600000000
-
-#define READ_NODE_CACHE_SIZE 10000
-#define READ_NODE_BLOCK_SHIFT 10l
-#define READ_NODE_BLOCK_SIZE (1l << READ_NODE_BLOCK_SHIFT)
-#define READ_NODE_BLOCK_MASK 0x03FFl
-
-#define WRITE_NODE_BLOCK_SHIFT 20l
-#define WRITE_NODE_BLOCK_SIZE (1l << WRITE_NODE_BLOCK_SHIFT)
-#define WRITE_NODE_BLOCK_MASK 0x0FFFFFl
-
-#define PERSISTENT_CACHE_FORMAT_VERSION 1
-
-struct persistentCacheHeader {
- int format_version;
- int id_size;
- osmid_t max_initialised_id;
-};
-
-struct cache_index_entry {
- osmid_t key;
- int value;
-
- cache_index_entry(osmid_t k, int v) : key(k), value(v) {}
- cache_index_entry() {}
-};
-
-inline bool operator<(cache_index_entry const &a, cache_index_entry const &b)
-{
- return a.key < b.key;
-}
+#include <osmium/index/map/dense_file_array.hpp>
+#include <osmium/osm/location.hpp>
-inline bool operator<(cache_index_entry const &a, osmid_t b)
-{
- return a.key < b;
-}
+#include "node-ram-cache.hpp"
+#include "osmtypes.hpp"
-inline bool operator<(osmid_t a, cache_index_entry const &b)
-{
- return a < b.key;
-}
+struct options_t;
+class reprojection;
-struct node_persistent_cache : public boost::noncopyable
+class node_persistent_cache
{
- node_persistent_cache(const struct options_t *options, bool append,
- bool ro, std::shared_ptr<node_ram_cache> ptr);
+public:
+ node_persistent_cache(options_t const *options,
+ std::shared_ptr<node_ram_cache> ptr);
~node_persistent_cache();
- void set(osmid_t id, double lat, double lon);
- int get(osmNode *out, osmid_t id);
- size_t get_list(nodelist_t &out, const idlist_t nds);
+ void set(osmid_t id, osmium::Location const &coord);
+ osmium::Location get(osmid_t id);
+ size_t get_list(osmium::WayNodeList *nodes);
private:
-
- void set_append(osmid_t id, double lat, double lon);
- void set_create(osmid_t id, double lat, double lon);
-
- void writeout_dirty_nodes();
- size_t replace_block();
- int find_block(osmid_t block_offset);
- void expand_cache(osmid_t block_offset);
- void nodes_prefetch_async(osmid_t id);
- int load_block(osmid_t block_offset);
- void nodes_set_create_writeout_block();
-
- void remove_from_cache_idx(osmid_t block_offset);
- void add_to_cache_idx(cache_index_entry const &entry);
- void set_read_mode();
-
- int node_cache_fd;
- const char * node_cache_fname;
- bool append_mode;
-
- persistentCacheHeader cacheHeader;
- ramNodeBlock writeNodeBlock; /* larger node block for more efficient initial sequential writing of node cache */
- ramNodeBlock * readNodeBlockCache;
-
- typedef std::vector<cache_index_entry> cache_index;
- cache_index readNodeBlockCacheIdx;
-
- bool read_mode;
-
- std::shared_ptr<node_ram_cache> ram_cache;
+ // Dense node cache for unsigned IDs only
+ using index_t =
+ osmium::index::map::DenseFileArray<osmium::unsigned_object_id_type,
+ osmium::Location>;
+
+ std::shared_ptr<node_ram_cache> m_ram_cache;
+ int m_fd;
+ std::unique_ptr<index_t> m_index;
};
#endif
diff --git a/node-ram-cache.cpp b/node-ram-cache.cpp
index ef74191..f42b39e 100644
--- a/node-ram-cache.cpp
+++ b/node-ram-cache.cpp
@@ -54,75 +54,69 @@
* Reuse old block: O(log maxBlocks)
*/
-
-
#define BLOCK_SHIFT 13
-#define PER_BLOCK (((osmid_t)1) << BLOCK_SHIFT)
+#define PER_BLOCK (((osmid_t)1) << BLOCK_SHIFT)
#define NUM_BLOCKS (((osmid_t)1) << (36 - BLOCK_SHIFT))
-#define SAFETY_MARGIN 1024*PER_BLOCK*sizeof(ramNode)
-
-#ifdef FIXED_POINT
-int ramNode::scale;
-#endif
+#define SAFETY_MARGIN (1024 * PER_BLOCK * sizeof(osmium::Location))
static int32_t id2block(osmid_t id)
{
/* + NUM_BLOCKS/2 allows for negative IDs */
- return (id >> BLOCK_SHIFT) + NUM_BLOCKS/2;
+ return (id >> BLOCK_SHIFT) + NUM_BLOCKS / 2;
}
-static int id2offset(osmid_t id)
-{
- return id & (PER_BLOCK-1);
-}
+static int id2offset(osmid_t id) { return id & (PER_BLOCK - 1); }
static osmid_t block2id(int32_t block, int offset)
{
- return (((osmid_t) block - NUM_BLOCKS/2) << BLOCK_SHIFT) + (osmid_t) offset;
+ return (((osmid_t)block - NUM_BLOCKS / 2) << BLOCK_SHIFT) + (osmid_t)offset;
}
-#define Swap(a,b) { ramNodeBlock * __tmp = a; a = b; b = __tmp; }
+#define Swap(a, b) \
+ { \
+ ramNodeBlock *__tmp = a; \
+ a = b; \
+ b = __tmp; \
+ }
-void node_ram_cache::percolate_up( int pos )
+void node_ram_cache::percolate_up(int pos)
{
int i = pos;
- while( i > 0 )
- {
- int parent = (i-1)>>1;
- if( queue[i]->used() < queue[parent]->used() )
- {
- Swap( queue[i], queue[parent] )
- i = parent;
- }
- else
- break;
+ while (i > 0) {
+ int parent = (i - 1) >> 1;
+ if (queue[i]->used() < queue[parent]->used()) {
+ Swap(queue[i], queue[parent]) i = parent;
+ } else
+ break;
}
}
-ramNode *node_ram_cache::next_chunk() {
- if ( (allocStrategy & ALLOC_DENSE_CHUNK) == 0 ) {
+osmium::Location *node_ram_cache::next_chunk()
+{
+ if ((allocStrategy & ALLOC_DENSE_CHUNK) == 0) {
// allocate starting from the upper end of the block cache
- blockCachePos += PER_BLOCK * sizeof(ramNode);
+ blockCachePos += PER_BLOCK * sizeof(osmium::Location);
char *result = blockCache + cacheSize - blockCachePos + SAFETY_MARGIN;
- return new(result) ramNode[PER_BLOCK];
+ return new (result) osmium::Location[PER_BLOCK];
} else {
- return new ramNode[PER_BLOCK];
+ return new osmium::Location[PER_BLOCK];
}
}
-
-void node_ram_cache::set_sparse(osmid_t id, const ramNode &coord) {
+void node_ram_cache::set_sparse(osmid_t id, const osmium::Location &coord)
+{
// Sparse cache depends on ordered nodes, reject out-of-order ids.
// Also check that there is still space.
- if ((maxSparseId && id < maxSparseId)
- || (sizeSparseTuples > maxSparseTuples)
- || ( cacheUsed > cacheSize)) {
+ if ((maxSparseId && id < maxSparseId) ||
+ (sizeSparseTuples > maxSparseTuples) || (cacheUsed > cacheSize)) {
if (allocStrategy & ALLOC_LOSSY) {
return;
} else {
- fprintf(stderr, "\nNode cache size is too small to fit all nodes. Please increase cache size\n");
+ fprintf(stderr,
+ "\nNode cache size is too small to fit all nodes. Please "
+ "increase cache size\n");
util::exit_nicely();
}
}
@@ -135,47 +129,62 @@ void node_ram_cache::set_sparse(osmid_t id, const ramNode &coord) {
storedNodes++;
}
-void node_ram_cache::set_dense(osmid_t id, const ramNode &coord) {
- int32_t const block = id2block(id);
+void node_ram_cache::set_dense(osmid_t id, const osmium::Location &coord)
+{
+ int32_t const block = id2block(id);
int const offset = id2offset(id);
if (maxBlocks == 0) {
- return;
+ return;
}
if (!blocks[block].nodes) {
- if (((allocStrategy & ALLOC_SPARSE) > 0) && ( usedBlocks < maxBlocks) && ( cacheUsed > cacheSize)) {
- /* TODO: It is more memory efficient to drop nodes from the sparse node cache than from the dense node cache */
+ if (((allocStrategy & ALLOC_SPARSE) > 0) && (usedBlocks < maxBlocks) &&
+ (cacheUsed > cacheSize)) {
+ /* TODO: It is more memory efficient to drop nodes from the sparse node
+ * cache than from the dense node cache */
}
- if ((usedBlocks < maxBlocks ) && (cacheUsed < cacheSize)) {
- /* if usedBlocks > 0 then the previous block is used up. Need to correctly handle it. */
- if ( usedBlocks > 0 ) {
- /* If sparse allocation is also set, then check if the previous block has sufficient density
- * to store it in dense representation. If not, push all elements of the block
- * to the sparse node cache and reuse memory of the previous block for the current block */
- if ( ((allocStrategy & ALLOC_SPARSE) == 0) ||
- ((queue[usedBlocks - 1]->used() / (double)(1<< BLOCK_SHIFT)) >
- (sizeof(ramNode) / (double)sizeof(ramNodeID)))) {
+ if ((usedBlocks < maxBlocks) && (cacheUsed < cacheSize)) {
+ /* if usedBlocks > 0 then the previous block is used up. Need to correctly
+ * handle it. */
+ if (usedBlocks > 0) {
+ /* If sparse allocation is also set, then check if the previous block
+ * has sufficient density
+ * to store it in dense representation. If not, push all elements of the
+ * block
+ * to the sparse node cache and reuse memory of the previous block for
+ * the current block */
+ if (((allocStrategy & ALLOC_SPARSE) == 0) ||
+ ((queue[usedBlocks - 1]->used() /
+ (double)(1 << BLOCK_SHIFT)) >
+ (sizeof(osmium::Location) / (double)sizeof(ramNodeID)))) {
/* Block has reached the level to keep it in dense representation */
- /* We've just finished with the previous block, so we need to percolate it up the queue to its correct position */
+ /* We've just finished with the previous block, so we need to
+ * percolate it up the queue to its correct position */
/* Upto log(usedBlocks) iterations */
- percolate_up( usedBlocks-1 );
+ percolate_up(usedBlocks - 1);
blocks[block].nodes = next_chunk();
} else {
- /* previous block was not dense enough, so push it into the sparse node cache instead */
+ /* previous block was not dense enough, so push it into the sparse
+ * node cache instead */
for (int i = 0; i < (1 << BLOCK_SHIFT); i++) {
- if (queue[usedBlocks -1]->nodes[i].is_valid()) {
- set_sparse(block2id(queue[usedBlocks - 1]->block_offset, i),
- queue[usedBlocks -1]->nodes[i]);
- queue[usedBlocks -1]->nodes[i] = ramNode(); // invalidate
+ if (queue[usedBlocks - 1]->nodes[i].valid()) {
+ set_sparse(
+ block2id(queue[usedBlocks - 1]->block_offset,
+ i),
+ queue[usedBlocks - 1]->nodes[i]);
+ // invalidate location
+ queue[usedBlocks - 1]->nodes[i] =
+ osmium::Location();
}
}
- /* reuse previous block, as its content is now in the sparse representation */
+ /* reuse previous block, as its content is now in the sparse
+ * representation */
storedNodes -= queue[usedBlocks - 1]->used();
blocks[block].nodes = queue[usedBlocks - 1]->nodes;
blocks[queue[usedBlocks - 1]->block_offset].nodes = nullptr;
usedBlocks--;
- cacheUsed -= PER_BLOCK * sizeof(ramNode);
+ cacheUsed -= PER_BLOCK * sizeof(osmium::Location);
}
} else {
blocks[block].nodes = next_chunk();
@@ -189,43 +198,45 @@ void node_ram_cache::set_dense(osmid_t id, const ramNode &coord) {
}
queue[usedBlocks] = &blocks[block];
usedBlocks++;
- cacheUsed += PER_BLOCK * sizeof(ramNode);
+ cacheUsed += PER_BLOCK * sizeof(osmium::Location);
/* If we've just used up the last possible block we enter the
- * transition and we change the invariant. To do this we percolate
- * the newly allocated block straight to the head */
- if (( usedBlocks == maxBlocks ) || ( cacheUsed > cacheSize ))
- percolate_up( usedBlocks-1 );
+ * transition and we change the invariant. To do this we percolate
+ * the newly allocated block straight to the head */
+ if ((usedBlocks == maxBlocks) || (cacheUsed > cacheSize))
+ percolate_up(usedBlocks - 1);
} else {
if ((allocStrategy & ALLOC_LOSSY) == 0) {
- fprintf(stderr, "\nNode cache size is too small to fit all nodes. Please increase cache size\n");
+ fprintf(stderr,
+ "\nNode cache size is too small to fit all nodes. "
+ "Please increase cache size\n");
util::exit_nicely();
}
/* We've reached the maximum number of blocks, so now we push the
- * current head of the tree down to the right level to restore the
- * priority queue invariant. Upto log(maxBlocks) iterations */
+ * current head of the tree down to the right level to restore the
+ * priority queue invariant. Upto log(maxBlocks) iterations */
int i = 0;
- while( 2*i+1 < usedBlocks - 1 ) {
- if( queue[2*i+1]->used() <= queue[2*i+2]->used() ) {
- if( queue[i]->used() > queue[2*i+1]->used() ) {
- Swap( queue[i], queue[2*i+1] );
- i = 2*i+1;
- }
- else
+ while (2 * i + 1 < usedBlocks - 1) {
+ if (queue[2 * i + 1]->used() <= queue[2 * i + 2]->used()) {
+ if (queue[i]->used() > queue[2 * i + 1]->used()) {
+ Swap(queue[i], queue[2 * i + 1]);
+ i = 2 * i + 1;
+ } else
break;
} else {
- if( queue[i]->used() > queue[2*i+2]->used() ) {
- Swap( queue[i], queue[2*i+2] );
- i = 2*i+2;
+ if (queue[i]->used() > queue[2 * i + 2]->used()) {
+ Swap(queue[i], queue[2 * i + 2]);
+ i = 2 * i + 2;
} else
break;
}
}
- /* Now the head of the queue is the smallest, so it becomes our replacement candidate */
+ /* Now the head of the queue is the smallest, so it becomes our
+ * replacement candidate */
blocks[block].nodes = queue[0]->nodes;
blocks[block].reset_used();
- new(blocks[block].nodes) ramNode[PER_BLOCK];
+ new (blocks[block].nodes) osmium::Location[PER_BLOCK];
/* Clear old head block and point to new block */
storedNodes -= queue[0]->used();
@@ -235,18 +246,21 @@ void node_ram_cache::set_dense(osmid_t id, const ramNode &coord) {
}
} else {
/* Insert into an existing block. We can't allow this in general or it
- * will break the invariant. However, it will work fine if all the
- * nodes come in numerical order, which is the common case */
+ * will break the invariant. However, it will work fine if all the
+ * nodes come in numerical order, which is the common case */
int expectedpos;
- if (( usedBlocks < maxBlocks ) && (cacheUsed < cacheSize))
- expectedpos = usedBlocks-1;
+ if ((usedBlocks < maxBlocks) && (cacheUsed < cacheSize))
+ expectedpos = usedBlocks - 1;
else
expectedpos = 0;
- if( queue[expectedpos] != &blocks[block] ) {
+ if (queue[expectedpos] != &blocks[block]) {
if (!warn_node_order) {
- fprintf( stderr, "WARNING: Found Out of order node %" PRIdOSMID " (%d,%d) - this will impact the cache efficiency\n", id, block, offset );
+ fprintf(stderr,
+ "WARNING: Found Out of order node %" PRIdOSMID
+ " (%d,%d) - this will impact the cache efficiency\n",
+ id, block, offset);
warn_node_order++;
}
return;
@@ -258,21 +272,21 @@ void node_ram_cache::set_dense(osmid_t id, const ramNode &coord) {
storedNodes++;
}
-
-int node_ram_cache::get_sparse(osmNode *out, osmid_t id) {
+osmium::Location node_ram_cache::get_sparse(osmid_t id)
+{
int64_t pivotPos = sizeSparseTuples >> 1;
int64_t minPos = 0;
int64_t maxPos = sizeSparseTuples;
while (minPos <= maxPos) {
- if ( sparseBlock[pivotPos].id == id ) {
- out->lat = sparseBlock[pivotPos].coord.lat();
- out->lon = sparseBlock[pivotPos].coord.lon();
- return 0;
+ if (sparseBlock[pivotPos].id == id) {
+ return sparseBlock[pivotPos].coord;
+ }
+ if ((pivotPos == minPos) || (pivotPos == maxPos)) {
+ return osmium::Location();
}
- if ( (pivotPos == minPos) || (pivotPos == maxPos)) return 1;
- if ( sparseBlock[pivotPos].id > id ) {
+ if (sparseBlock[pivotPos].id > id) {
maxPos = pivotPos;
pivotPos = minPos + ((maxPos - minPos) >> 1);
} else {
@@ -281,157 +295,171 @@ int node_ram_cache::get_sparse(osmNode *out, osmid_t id) {
}
}
- return 1;
+ return osmium::Location();
}
-int node_ram_cache::get_dense(osmNode *out, osmid_t id) {
- int32_t const block = id2block(id);
- int const offset = id2offset(id);
+osmium::Location node_ram_cache::get_dense(osmid_t id)
+{
+ const int32_t block = id2block(id);
+ const int offset = id2offset(id);
if (!blocks[block].nodes)
- return 1;
-
- if (!blocks[block].nodes[offset].is_valid())
- return 1;
+ return osmium::Location();
- out->lat = blocks[block].nodes[offset].lat();
- out->lon = blocks[block].nodes[offset].lon();
-
- return 0;
+ return blocks[block].nodes[offset];
}
-
-node_ram_cache::node_ram_cache( int strategy, int cacheSizeMB, int fixpointscale )
- : allocStrategy(ALLOC_DENSE), blocks(nullptr), usedBlocks(0),
- maxBlocks(0), blockCache(nullptr), queue(nullptr), sparseBlock(nullptr),
- maxSparseTuples(0), sizeSparseTuples(0), maxSparseId(0), cacheUsed(0),
- cacheSize(0), storedNodes(0), totalNodes(0), nodesCacheHits(0),
- nodesCacheLookups(0), warn_node_order(0) {
-#ifdef FIXED_POINT
- ramNode::scale = fixpointscale;
-#endif
+node_ram_cache::node_ram_cache(int strategy, int cacheSizeMB)
+: allocStrategy(strategy), blocks(nullptr), usedBlocks(0), maxBlocks(0),
+ blockCache(nullptr), queue(nullptr), sparseBlock(nullptr), maxSparseTuples(0),
+ sizeSparseTuples(0), maxSparseId(0), cacheUsed(0),
+ cacheSize((int64_t)cacheSizeMB * 1024 * 1024), storedNodes(0), totalNodes(0),
+ nodesCacheHits(0), nodesCacheLookups(0), warn_node_order(0)
+{
blockCache = 0;
blockCachePos = 0;
- cacheUsed = 0;
- cacheSize = (int64_t)cacheSizeMB*(1024*1024);
/* How much we can fit, and make sure it's odd */
- maxBlocks = (cacheSize/(PER_BLOCK*sizeof(ramNode)));
- maxSparseTuples = (cacheSize/sizeof(ramNodeID))+1;
+ maxBlocks = (cacheSize / (PER_BLOCK * sizeof(osmium::Location)));
+ maxSparseTuples = (cacheSize / sizeof(ramNodeID)) + 1;
- allocStrategy = strategy;
-
- if ((allocStrategy & ALLOC_DENSE) > 0 ) {
+ if ((allocStrategy & ALLOC_DENSE) > 0) {
fprintf(stderr, "Allocating memory for dense node cache\n");
- blocks = (ramNodeBlock *)calloc(NUM_BLOCKS,sizeof(ramNodeBlock));
+ blocks = (ramNodeBlock *)calloc(NUM_BLOCKS, sizeof(ramNodeBlock));
if (!blocks) {
- fprintf(stderr, "Out of memory for node cache dense index, try using \"--cache-strategy sparse\" instead \n");
+ fprintf(stderr,
+ "Out of memory for node cache dense index, try using "
+ "\"--cache-strategy sparse\" instead \n");
util::exit_nicely();
}
- queue = (ramNodeBlock **)calloc( maxBlocks,sizeof(ramNodeBlock *) );
+ queue = (ramNodeBlock **)calloc(maxBlocks, sizeof(ramNodeBlock *));
/* Use this method of allocation if virtual memory is limited,
- * or if OS allocs physical memory right away, rather than page by page
- * once it is needed.
- */
- if( (allocStrategy & ALLOC_DENSE_CHUNK) > 0 ) {
- fprintf(stderr, "Allocating dense node cache in block sized chunks\n");
+ * or if OS allocs physical memory right away, rather than page by page
+ * once it is needed.
+ */
+ if ((allocStrategy & ALLOC_DENSE_CHUNK) > 0) {
+ fprintf(stderr,
+ "Allocating dense node cache in block sized chunks\n");
if (!queue) {
fprintf(stderr, "Out of memory, reduce --cache size\n");
util::exit_nicely();
}
} else {
fprintf(stderr, "Allocating dense node cache in one big chunk\n");
- blockCache = (char *)malloc((maxBlocks + 1024) * PER_BLOCK * sizeof(ramNode));
+ blockCache = (char *)malloc((maxBlocks + 1024) * PER_BLOCK *
+ sizeof(osmium::Location));
if (!queue || !blockCache) {
- fprintf(stderr, "Out of memory for dense node cache, reduce --cache size\n");
+ fprintf(stderr, "Out of memory for dense node cache, reduce "
+ "--cache size\n");
util::exit_nicely();
}
}
}
/* Allocate the full amount of memory given by --cache parameter in one go.
- * If both dense and sparse cache alloc is set, this will allocate up to twice
- * as much virtual memory as specified by --cache. This relies on the OS doing
- * lazy allocation of physical RAM. Extra accounting during setting of nodes is done
- * to ensure physical RAM usage should roughly be no more than --cache
- */
+ * If both dense and sparse cache alloc is set, this will allocate up to twice
+ * as much virtual memory as specified by --cache. This relies on the OS doing
+ * lazy allocation of physical RAM. Extra accounting during setting of nodes
+ * is done
+ * to ensure physical RAM usage should roughly be no more than --cache
+ */
- if ((allocStrategy & ALLOC_SPARSE) > 0 ) {
+ if ((allocStrategy & ALLOC_SPARSE) > 0) {
fprintf(stderr, "Allocating memory for sparse node cache\n");
if (!blockCache) {
- sparseBlock = (ramNodeID *)malloc(maxSparseTuples * sizeof(ramNodeID));
+ sparseBlock =
+ (ramNodeID *)malloc(maxSparseTuples * sizeof(ramNodeID));
} else {
fprintf(stderr, "Sharing dense sparse\n");
sparseBlock = (ramNodeID *)blockCache;
}
if (!sparseBlock) {
- fprintf(stderr, "Out of memory for sparse node cache, reduce --cache size\n");
+ fprintf(
+ stderr,
+ "Out of memory for sparse node cache, reduce --cache size\n");
util::exit_nicely();
}
}
- fprintf( stderr, "Node-cache: cache=%" PRId64 "MB, maxblocks=%d*%" PRId64 ", allocation method=%i\n", (cacheSize >> 20), maxBlocks, (int64_t) PER_BLOCK*sizeof(ramNode), allocStrategy );
+ fprintf(stderr, "Node-cache: cache=%" PRId64 "MB, maxblocks=%d*%" PRId64
+ ", allocation method=%i\n",
+ (cacheSize >> 20), maxBlocks,
+ (int64_t)PER_BLOCK * sizeof(osmium::Location), allocStrategy);
}
-node_ram_cache::~node_ram_cache() {
- fprintf( stderr, "node cache: stored: %" PRIdOSMID "(%.2f%%), storage efficiency: %.2f%% (dense blocks: %i, sparse nodes: %" PRId64 "), hit rate: %.2f%%\n",
- storedNodes, 100.0f*storedNodes/totalNodes, 100.0f*storedNodes*sizeof(ramNode)/cacheUsed,
- usedBlocks, sizeSparseTuples,
- 100.0f*nodesCacheHits/nodesCacheLookups );
-
- if ( (allocStrategy & ALLOC_DENSE) > 0 ) {
- if ( (allocStrategy & ALLOC_DENSE_CHUNK) > 0 ) {
- for(int i = 0; i < usedBlocks; ++i) {
- delete[] queue[i]->nodes;
- queue[i]->nodes = nullptr;
- }
- } else {
- free(blockCache);
- blockCache = 0;
- }
- free(blocks);
- free(queue);
- }
- if ( ((allocStrategy & ALLOC_SPARSE) > 0) && ((allocStrategy & ALLOC_DENSE) == 0)) {
- free(sparseBlock);
- }
+node_ram_cache::~node_ram_cache()
+{
+ fprintf(stderr, "node cache: stored: %" PRIdOSMID
+ "(%.2f%%), storage efficiency: %.2f%% (dense blocks: %i, "
+ "sparse nodes: %" PRId64 "), hit rate: %.2f%%\n",
+ storedNodes, 100.0f * storedNodes / totalNodes,
+ 100.0f * storedNodes * sizeof(osmium::Location) / cacheUsed,
+ usedBlocks, sizeSparseTuples,
+ 100.0f * nodesCacheHits / nodesCacheLookups);
+
+ if ((allocStrategy & ALLOC_DENSE) > 0) {
+ if ((allocStrategy & ALLOC_DENSE_CHUNK) > 0) {
+ for (int i = 0; i < usedBlocks; ++i) {
+ delete[] queue[i]->nodes;
+ queue[i]->nodes = nullptr;
+ }
+ } else {
+ free(blockCache);
+ blockCache = 0;
+ }
+ free(blocks);
+ free(queue);
+ }
+ if (((allocStrategy & ALLOC_SPARSE) > 0) &&
+ ((allocStrategy & ALLOC_DENSE) == 0)) {
+ free(sparseBlock);
+ }
}
-void node_ram_cache::set(osmid_t id, double lat, double lon, const taglist_t &) {
- if ((id > 0 && id >> BLOCK_SHIFT >> 32) || (id < 0 && ~id >> BLOCK_SHIFT >> 32 )) {
- fprintf(stderr, "\nAbsolute node IDs must not be larger than %" PRId64 " (got%" PRId64 " )\n",
- (int64_t) 1 << 42, (int64_t) id);
+void node_ram_cache::set(osmid_t id, const osmium::Location &coord)
+{
+ if ((id > 0 && id >> BLOCK_SHIFT >> 32) ||
+ (id < 0 && ~id >> BLOCK_SHIFT >> 32)) {
+ fprintf(stderr, "\nAbsolute node IDs must not be larger than %" PRId64
+ " (got%" PRId64 " )\n",
+ (int64_t)1 << 42, (int64_t)id);
util::exit_nicely();
}
totalNodes++;
/* if ALLOC_DENSE and ALLOC_SPARSE are set, send it through
- * ram_nodes_set_dense. If a block is non dense, it will automatically
- * get pushed to the sparse cache if a block is sparse and ALLOC_SPARSE is set
- */
- if ( (allocStrategy & ALLOC_DENSE) > 0 ) {
- set_dense(id, ramNode(lon, lat));
- } else if ( (allocStrategy & ALLOC_SPARSE) > 0 ) {
- set_sparse(id, ramNode(lon, lat));
+ * ram_nodes_set_dense. If a block is non dense, it will automatically
+ * get pushed to the sparse cache if a block is sparse and ALLOC_SPARSE is set
+ */
+ if ((allocStrategy & ALLOC_DENSE) > 0) {
+ set_dense(id, coord);
+ } else if ((allocStrategy & ALLOC_SPARSE) > 0) {
+ set_sparse(id, coord);
} else {
// Command line options always have ALLOC_DENSE | ALLOC_SPARSE
- throw std::logic_error((boost::format("Unexpected cache strategy in node_ram_cache::set with allocStrategy %1%") % allocStrategy).str());
+ throw std::logic_error(
+ (boost::format(
+ "Unexpected cache strategy in node_ram_cache::set with "
+ "allocStrategy %1%") %
+ allocStrategy)
+ .str());
}
}
-int node_ram_cache::get(osmNode *out, osmid_t id) {
- nodesCacheLookups++;
+osmium::Location node_ram_cache::get(osmid_t id)
+{
+ osmium::Location coord;
- if ((allocStrategy & ALLOC_DENSE) > 0) {
- if (get_dense(out, id) == 0) {
- nodesCacheHits++;
- return 0;
- }
+ if (allocStrategy & ALLOC_DENSE) {
+ coord = get_dense(id);
}
- if ((allocStrategy & ALLOC_SPARSE) > 0) {
- if (get_sparse(out, id) == 0) {
- nodesCacheHits++;
- return 0;
- }
+
+ if (allocStrategy & ALLOC_SPARSE && !coord.valid()) {
+ coord = get_sparse(id);
}
- return 1;
+ if (coord.valid()) {
+ nodesCacheHits++;
+ }
+ nodesCacheLookups++;
+
+ return coord;
}
diff --git a/node-ram-cache.hpp b/node-ram-cache.hpp
index 7c47f45..5b3d0b4 100644
--- a/node-ram-cache.hpp
+++ b/node-ram-cache.hpp
@@ -8,14 +8,14 @@
#ifndef NODE_RAM_CACHE_H
#define NODE_RAM_CACHE_H
-#include "config.h"
-
#include <climits>
#include <cstddef>
#include <cstdint>
#include <boost/noncopyable.hpp>
+#include <osmium/osm/location.hpp>
+
#include "osmtypes.hpp"
#define ALLOC_SPARSE 1
@@ -23,69 +23,14 @@
#define ALLOC_DENSE_CHUNK 4
#define ALLOC_LOSSY 8
-/**
- * A set of coordinates, for caching in RAM or on disk.
- *
- * If FIXED_POINT is enabled, it uses internally a more efficient
- * representation as integer.
- */
-class ramNode {
-public:
-#ifdef FIXED_POINT
- static int scale;
-
- /// Default constructor creates an invalid node
- ramNode() : _lon(INT_MIN), _lat(INT_MIN) {}
- /**
- * Standard constructor takes geographic coordinates and saves them
- * in the internal node representation.
- */
- ramNode(double lon, double lat) : _lon(dbl2fix(lon)), _lat(dbl2fix(lat)) {}
- /**
- * Internal constructor which takes already encoded nodes.
- *
- * Used by middle-pgsql which stores encoded nodes in the DB.
- */
- ramNode(int lon, int lat) : _lon(lon), _lat(lat) {}
-
- /// Return true if the node currently stores valid coordinates.
- bool is_valid() const { return _lon != INT_MIN; }
- /// Return longitude (converting from internal representation)
- double lon() const { return fix2dbl(_lon); }
- /// Return latitude (converting from internal representation)
- double lat() const { return fix2dbl(_lat); }
- /// Return internal representation of longitude (for external storage).
- int int_lon() const { return _lon; }
- /// Return internal representation of latitude (for external storage).
- int int_lat() const { return _lat; }
-
-private:
- int _lon;
- int _lat;
-
- int dbl2fix(const double x) const { return (int) (x * scale + 0.4); }
- double fix2dbl(const int x) const { return (double)x / scale; }
-#else
-public:
- ramNode() : _lat(NAN), _lon(NAN) {}
- ramNode(double lon, double lat) : _lon(lon), _lat(lat) {}
-
- bool is_valid() const { return !std::isnan(_lon); }
- double lon() const { return _lon; }
- double lat() const { return _lat; }
-private:
- double _lon;
- double _lat;
-
-#endif
-};
-
-struct ramNodeID {
+struct ramNodeID
+{
osmid_t id;
- ramNode coord;
+ osmium::Location coord;
};
-class ramNodeBlock {
+class ramNodeBlock
+{
public:
ramNodeBlock() : nodes(nullptr), block_offset(-1), _used(0) {}
@@ -98,27 +43,28 @@ public:
void set_used(int used) { _used = (used << 1) || (_used & 1); }
int used() const { return _used >> 1; }
- ramNode *nodes;
+ osmium::Location *nodes;
int32_t block_offset;
+
private:
int32_t _used; // 0-bit indicates dirty
};
struct node_ram_cache : public boost::noncopyable
{
- node_ram_cache(int strategy, int cacheSizeMB, int fixpointscale);
+ node_ram_cache(int strategy, int cacheSizeMB);
~node_ram_cache();
- void set(osmid_t id, double lat, double lon, const taglist_t &tags);
- int get(osmNode *out, osmid_t id);
+ void set(osmid_t id, const osmium::Location &coord);
+ osmium::Location get(osmid_t id);
private:
- void percolate_up( int pos );
- ramNode *next_chunk();
- void set_sparse(osmid_t id, const ramNode &coord);
- void set_dense(osmid_t id, const ramNode& coord);
- int get_sparse(osmNode *out, osmid_t id);
- int get_dense(osmNode *out, osmid_t id);
+ void percolate_up(int pos);
+ osmium::Location *next_chunk();
+ void set_sparse(osmid_t id, const osmium::Location &coord);
+ void set_dense(osmid_t id, const osmium::Location &coord);
+ osmium::Location get_sparse(osmid_t id);
+ osmium::Location get_dense(osmid_t id);
int allocStrategy;
diff --git a/options.cpp b/options.cpp
index c2b9be5..025dc9b 100644
--- a/options.cpp
+++ b/options.cpp
@@ -61,7 +61,6 @@ namespace
{"drop", 0, 0, 206},
{"unlogged", 0, 0, 207},
{"flat-nodes",1,0,209},
- {"exclude-invalid-polygon",0,0,210},
{"tag-transform-script",1,0,212},
{"reproject-area",0,0,213},
{0, 0, 0, 0}
@@ -102,8 +101,7 @@ namespace
-C|--cache Use up to this many MB for caching nodes (default: 800)\n\
\n\
Database options:\n\
- -d|--database The name of the PostgreSQL database to connect\n\
- to (default: gis).\n\
+ -d|--database The name of the PostgreSQL database to connect to.\n\
-U|--username PostgreSQL user name (specify passsword in PGPASS\n\
environment variable or use -W).\n\
-W|--password Force password prompt.\n\
@@ -204,7 +202,6 @@ namespace
-K|--keep-coastlines Keep coastline data rather than filtering it out.\n\
By default natural=coastline tagged data will be discarded\n\
because renderers usually have shape files for them.\n\
- --exclude-invalid-polygon do not attempt to recover invalid geometries.\n\
--reproject-area compute area column using spherical mercator coordinates.\n\
-h|--help Help information.\n\
-v|--verbose Verbose output.\n");
@@ -234,7 +231,7 @@ namespace
} // anonymous namespace
database_options_t::database_options_t():
- db("gis"), username(boost::none), host(boost::none),
+ db(boost::none), username(boost::none), host(boost::none),
password(boost::none), port(boost::none)
{
@@ -244,8 +241,10 @@ std::string database_options_t::conninfo() const
{
std::ostringstream out;
- out << "dbname='" << db << "'";
-
+ out << "fallback_application_name='osm2pgsql'";
+ if (db) {
+ out << " dbname='" << *db << "'";
+ }
if (username) {
out << " user='" << *username << "'";
}
@@ -262,22 +261,29 @@ std::string database_options_t::conninfo() const
return out.str();
}
-options_t::options_t():
- prefix("planet_osm"), scale(DEFAULT_SCALE), projection(reprojection::create_projection(PROJ_SPHERE_MERC)), append(false), slim(false),
- cache(800), tblsmain_index(boost::none), tblsslim_index(boost::none), tblsmain_data(boost::none), tblsslim_data(boost::none), style(OSM2PGSQL_DATADIR "/default.style"),
- expire_tiles_zoom(-1), expire_tiles_zoom_min(-1), expire_tiles_max_bbox(20000.0), expire_tiles_filename("dirty_tiles"),
- hstore_mode(HSTORE_NONE), enable_hstore_index(false),
- enable_multi(false), hstore_columns(), keep_coastlines(false), parallel_indexing(true),
- #ifdef __amd64__
- alloc_chunkwise(ALLOC_SPARSE | ALLOC_DENSE),
- #else
- alloc_chunkwise(ALLOC_SPARSE),
- #endif
- droptemp(false), unlogged(false), hstore_match_only(false), flat_node_cache_enabled(false), excludepoly(false), reproject_area(false), flat_node_file(boost::none),
- tag_transform_script(boost::none), tag_transform_node_func(boost::none), tag_transform_way_func(boost::none),
- tag_transform_rel_func(boost::none), tag_transform_rel_mem_func(boost::none),
- create(false), long_usage_bool(false), pass_prompt(false), output_backend("pgsql"), input_reader("auto"), bbox(boost::none),
- extra_attributes(false), verbose(false)
+options_t::options_t()
+: prefix("planet_osm"),
+ projection(reprojection::create_projection(PROJ_SPHERE_MERC)), append(false),
+ slim(false), cache(800), tblsmain_index(boost::none),
+ tblsslim_index(boost::none), tblsmain_data(boost::none),
+ tblsslim_data(boost::none), style(OSM2PGSQL_DATADIR "/default.style"),
+ expire_tiles_zoom(0), expire_tiles_zoom_min(0),
+ expire_tiles_max_bbox(20000.0), expire_tiles_filename("dirty_tiles"),
+ hstore_mode(HSTORE_NONE), enable_hstore_index(false), enable_multi(false),
+ hstore_columns(), keep_coastlines(false), parallel_indexing(true),
+#ifdef __amd64__
+ alloc_chunkwise(ALLOC_SPARSE | ALLOC_DENSE),
+#else
+ alloc_chunkwise(ALLOC_SPARSE),
+#endif
+ droptemp(false), unlogged(false), hstore_match_only(false),
+ flat_node_cache_enabled(false), reproject_area(false),
+ flat_node_file(boost::none), tag_transform_script(boost::none),
+ tag_transform_node_func(boost::none), tag_transform_way_func(boost::none),
+ tag_transform_rel_func(boost::none), tag_transform_rel_mem_func(boost::none),
+ create(false), long_usage_bool(false), pass_prompt(false),
+ output_backend("pgsql"), input_reader("auto"), bbox(boost::none),
+ extra_attributes(false), verbose(false)
{
num_procs = std::thread::hardware_concurrency();
if (num_procs < 1) {
@@ -292,7 +298,6 @@ options_t::~options_t()
options_t::options_t(int argc, char *argv[]): options_t()
{
- const char *temparg;
int c;
//keep going while there are args left to handle
@@ -371,12 +376,37 @@ options_t::options_t(int argc, char *argv[]): options_t()
tblsmain_index = optarg;
break;
case 'e':
- expire_tiles_zoom_min = atoi(optarg);
- temparg = strchr(optarg, '-');
- if (temparg)
- expire_tiles_zoom = atoi(temparg + 1);
- if (expire_tiles_zoom < expire_tiles_zoom_min)
+ if (!optarg || optarg[0] == '-') {
+ throw std::runtime_error("Missing argument for option -e. Zoom "
+ "levels must be positive.\n");
+ }
+ char *next_char;
+ expire_tiles_zoom_min =
+ static_cast<uint32_t>(std::strtoul(optarg, &next_char, 10));
+ if (expire_tiles_zoom_min == 0) {
+ throw std::runtime_error(
+ "Missing zoom level for tile expiry.\n");
+ }
+ // The first character after the number is ignored because that is the separating hyphen.
+ if (*next_char == '-') {
+ ++next_char;
+ // Second number must not be negative because zoom levels must be positive.
+ if (next_char && *next_char != '-' && isdigit(*next_char)) {
+ char *after_maxzoom;
+ expire_tiles_zoom = static_cast<uint32_t>(
+ std::strtoul(next_char, &after_maxzoom, 10));
+ } else {
+ throw std::runtime_error(
+ "Invalid maximum zoom level given for tile expiry.\n");
+ }
+ } else {
+ throw std::runtime_error("Minimum and maximum zoom level for "
+ "tile expiry must be separated by "
+ "'-'.\n");
+ }
+ if (expire_tiles_zoom < expire_tiles_zoom_min) {
expire_tiles_zoom = expire_tiles_zoom_min;
+ }
break;
case 'o':
expire_tiles_filename = optarg;
@@ -446,9 +476,6 @@ options_t::options_t(int argc, char *argv[]): options_t()
flat_node_cache_enabled = true;
flat_node_file = optarg;
break;
- case 210:
- excludepoly = true;
- break;
case 211:
enable_hstore_index = true;
break;
@@ -495,11 +522,6 @@ options_t::options_t(int argc, char *argv[]): options_t()
database_options.password = std::string(prompt);
}
}
-
-
- //NOTE: this is hugely important if you set it inappropriately and are are caching nodes
- //you could get overflow when working with larger coordinates (mercator) and larger scales
- scale = (projection->target_latlon()) ? 10000000 : 100;
}
void options_t::check_options()
@@ -516,7 +538,7 @@ void options_t::check_options()
throw std::runtime_error("--drop only makes sense with --slim.\n");
}
- if (unlogged && !create) {
+ if (unlogged && append) {
fprintf(stderr, "Warning: --unlogged only makes sense with --create; ignored.\n");
unlogged = false;
}
@@ -551,4 +573,17 @@ void options_t::check_options()
fprintf(stderr, "!! exceptions during import, you should try running in slim\n");
fprintf(stderr, "!! mode using parameter -s.\n");
}
+
+ // zoom level 31 is the technical limit because we use 32-bit integers for the x and y index of a tile ID
+ if (expire_tiles_zoom_min >= 32) {
+ expire_tiles_zoom_min = 31;
+ fprintf(stderr, "WARNING: mimimum zoom level for tile expiry is too "
+ "large and has been set to 31.\n\n");
+ }
+
+ if (expire_tiles_zoom >= 32) {
+ expire_tiles_zoom = 31;
+ fprintf(stderr, "WARNING: maximum zoom level for tile expiry is too "
+ "large and has been set to 31.\n\n");
+ }
}
diff --git a/options.hpp b/options.hpp
index c3bb54e..0df2cf1 100644
--- a/options.hpp
+++ b/options.hpp
@@ -17,16 +17,13 @@
/* create a hstore column for all tags */
#define HSTORE_ALL 2
-/* Scale is chosen such that 40,000 * SCALE < 2^32 */
-enum { DEFAULT_SCALE = 100 };
-
/**
* Database options, not specific to a table
*/
class database_options_t {
public:
database_options_t();
- std::string db;
+ boost::optional<std::string> db;
boost::optional<std::string> username;
boost::optional<std::string> host;
boost::optional<std::string> password;
@@ -49,7 +46,6 @@ public:
virtual ~options_t();
std::string prefix; ///< prefix for table names
- int scale; ///< scale for converting coordinates to fixed point
std::shared_ptr<reprojection> projection; ///< SRS of projection
bool append; ///< Append to existing data
bool slim; ///< In slim mode
@@ -59,8 +55,9 @@ public:
boost::optional<std::string> tblsmain_data; ///< Pg Tablespace to store main tables (no default TABLESPACE)
boost::optional<std::string> tblsslim_data; ///< Pg Tablespace to store slim tables (no default TABLESPACE)
std::string style; ///< style file to use
- int expire_tiles_zoom; ///< Zoom level for tile expiry list
- int expire_tiles_zoom_min; ///< Minimum zoom level for tile expiry list
+ uint32_t expire_tiles_zoom = 0; ///< Zoom level for tile expiry list
+ uint32_t expire_tiles_zoom_min =
+ 0; ///< Minimum zoom level for tile expiry list
double expire_tiles_max_bbox; ///< Max bbox size in either dimension to expire full bbox for a polygon
std::string expire_tiles_filename; ///< File name to output expired tiles list to
int hstore_mode; ///< add an additional hstore column with objects key/value pairs, and what type of hstore column
@@ -75,7 +72,6 @@ public:
bool unlogged; ///< use unlogged tables where possible
bool hstore_match_only; ///< only copy rows that match an explicitly listed key
bool flat_node_cache_enabled;
- bool excludepoly;
bool reproject_area;
boost::optional<std::string> flat_node_file;
/**
diff --git a/osm2pgsql.cpp b/osm2pgsql.cpp
index 8b51a07..af6158b 100644
--- a/osm2pgsql.cpp
+++ b/osm2pgsql.cpp
@@ -60,7 +60,7 @@ int main(int argc, char *argv[])
std::vector<std::shared_ptr<output_t> > outputs = output_t::create_outputs(middle.get(), options);
//let osmdata orchestrate between the middle and the outs
- osmdata_t osmdata(middle, outputs);
+ osmdata_t osmdata(middle, outputs, options.projection);
fprintf(stderr, "Using projection SRS %d (%s)\n",
options.projection->target_srs(),
@@ -82,9 +82,7 @@ int main(int argc, char *argv[])
fprintf(stderr, "\nReading in file: %s\n", filename.c_str());
time_t start = time(nullptr);
- parse_osmium_t parser(options.extra_attributes,
- options.bbox, options.projection.get(),
- options.append, &osmdata);
+ parse_osmium_t parser(options.bbox, options.append, &osmdata);
parser.stream_file(filename, options.input_reader);
stats.update(parser.stats());
diff --git a/osmdata.cpp b/osmdata.cpp
index ed7c9d2..0363107 100644
--- a/osmdata.cpp
+++ b/osmdata.cpp
@@ -11,13 +11,18 @@
#include "osmdata.hpp"
#include "output.hpp"
-osmdata_t::osmdata_t(std::shared_ptr<middle_t> mid_, const std::shared_ptr<output_t>& out_): mid(mid_)
+osmdata_t::osmdata_t(std::shared_ptr<middle_t> mid_,
+ std::shared_ptr<output_t> const &out_,
+ std::shared_ptr<reprojection> proj)
+: mid(mid_), projection(proj)
{
outs.push_back(out_);
}
-osmdata_t::osmdata_t(std::shared_ptr<middle_t> mid_, const std::vector<std::shared_ptr<output_t> > &outs_)
- : mid(mid_), outs(outs_)
+osmdata_t::osmdata_t(std::shared_ptr<middle_t> mid_,
+ std::vector<std::shared_ptr<output_t> > const &outs_,
+ std::shared_ptr<reprojection> proj)
+: mid(mid_), outs(outs_), projection(proj)
{
if (outs.empty()) {
throw std::runtime_error("Must have at least one output, but none have "
@@ -29,86 +34,86 @@ osmdata_t::~osmdata_t()
{
}
-int osmdata_t::node_add(osmid_t id, double lat, double lon, const taglist_t &tags) {
- mid->nodes_set(id, lat, lon, tags);
-
- // guarantee that we use the same values as in the node cache
- ramNode n(lon, lat);
+int osmdata_t::node_add(osmium::Node const &node)
+{
+ mid->nodes_set(node);
int status = 0;
- for (auto& out: outs) {
- status |= out->node_add(id, n.lat(), n.lon(), tags);
+ for (auto &out : outs) {
+ status |= out->node_add(node);
}
return status;
}
-int osmdata_t::way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags) {
- mid->ways_set(id, nodes, tags);
+int osmdata_t::way_add(osmium::Way *way)
+{
+ mid->ways_set(*way);
int status = 0;
for (auto& out: outs) {
- status |= out->way_add(id, nodes, tags);
+ status |= out->way_add(way);
}
return status;
}
-int osmdata_t::relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) {
- mid->relations_set(id, members, tags);
+int osmdata_t::relation_add(osmium::Relation const &rel)
+{
+ mid->relations_set(rel);
int status = 0;
for (auto& out: outs) {
- status |= out->relation_add(id, members, tags);
+ status |= out->relation_add(rel);
}
return status;
}
-int osmdata_t::node_modify(osmid_t id, double lat, double lon, const taglist_t &tags) {
+int osmdata_t::node_modify(osmium::Node const &node)
+{
slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid.get());
- slim->nodes_delete(id);
- slim->nodes_set(id, lat, lon, tags);
-
- // guarantee that we use the same values as in the node cache
- ramNode n(lon, lat);
+ slim->nodes_delete(node.id());
+ slim->nodes_set(node);
int status = 0;
for (auto& out: outs) {
- status |= out->node_modify(id, n.lat(), n.lon(), tags);
+ status |= out->node_modify(node);
}
- slim->node_changed(id);
+ slim->node_changed(node.id());
return status;
}
-int osmdata_t::way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags) {
+int osmdata_t::way_modify(osmium::Way *way)
+{
slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid.get());
- slim->ways_delete(id);
- slim->ways_set(id, nodes, tags);
+ slim->ways_delete(way->id());
+ slim->ways_set(*way);
int status = 0;
for (auto& out: outs) {
- status |= out->way_modify(id, nodes, tags);
+ status |= out->way_modify(way);
}
- slim->way_changed(id);
+ slim->way_changed(way->id());
return status;
}
-int osmdata_t::relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags) {
+int osmdata_t::relation_modify(osmium::Relation const &rel)
+{
slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid.get());
- slim->relations_delete(id);
- slim->relations_set(id, members, tags);
+ slim->relations_delete(rel.id());
+ slim->relations_set(rel);
int status = 0;
for (auto& out: outs) {
- status |= out->relation_modify(id, members, tags);
+ status |= out->relation_modify(rel);
}
- slim->relation_changed(id);
+ slim->relation_changed(rel.id());
return status;
}
@@ -170,10 +175,6 @@ struct pending_threaded_processor : public middle_t::pending_processor {
typedef std::pair<std::shared_ptr<const middle_query_t>, output_vec_t> clone_t;
static void do_jobs(output_vec_t const& outputs, pending_queue_t& queue, size_t& ids_done, std::mutex& mutex, int append, bool ways) {
-#ifdef _MSC_VER
- // Avoid problems when GEOS WKT-related methods switch the locale
- _configthreadlocale(_ENABLE_PER_THREAD_LOCALE);
-#endif
while (true) {
//get the job off the queue synchronously
pending_job_t job;
@@ -201,11 +202,18 @@ struct pending_threaded_processor : public middle_t::pending_processor {
}
//starts up count threads and works on the queue
- pending_threaded_processor(std::shared_ptr<middle_query_t> mid, const output_vec_t& outs, size_t thread_count, size_t job_count, int append)
+ pending_threaded_processor(std::shared_ptr<middle_query_t> mid,
+ const output_vec_t &outs, size_t thread_count,
+ int append)
//note that we cant hint to the stack how large it should be ahead of time
//we could use a different datastructure like a deque or vector but then
//the outputs the enqueue jobs would need the version check for the push(_back) method
- : outs(outs), ids_queued(0), append(append), queue(), ids_done(0) {
+ : outs(outs),
+ ids_queued(0),
+ append(append),
+ queue(),
+ ids_done(0)
+ {
//clone all the things we need
clones.reserve(thread_count);
@@ -373,19 +381,18 @@ void osmdata_t::stop() {
* access the data simultanious to process the rest in parallel
* as well as see the newly created tables.
*/
- size_t pending_count = mid->pending_count();
mid->commit();
for (auto& out: outs) {
//TODO: each of the outs can be in parallel
out->commit();
- pending_count += out->pending_count();
}
// should be the same for all outputs
const bool append = outs[0]->get_options()->append;
//threaded pending processing
- pending_threaded_processor ptp(mid, outs, outs[0]->get_options()->num_procs, pending_count, append);
+ pending_threaded_processor ptp(mid, outs, outs[0]->get_options()->num_procs,
+ append);
if (!outs.empty()) {
//This stage takes ways which were processed earlier, but might be
diff --git a/osmdata.hpp b/osmdata.hpp
index dff7621..8759262 100644
--- a/osmdata.hpp
+++ b/osmdata.hpp
@@ -12,23 +12,28 @@
class output_t;
struct middle_t;
+class reprojection;
class osmdata_t {
public:
- osmdata_t(std::shared_ptr<middle_t> mid_, const std::shared_ptr<output_t>& out_);
- osmdata_t(std::shared_ptr<middle_t> mid_, const std::vector<std::shared_ptr<output_t> > &outs_);
+ osmdata_t(std::shared_ptr<middle_t> mid_,
+ std::shared_ptr<output_t> const &out_,
+ std::shared_ptr<reprojection> proj);
+ osmdata_t(std::shared_ptr<middle_t> mid_,
+ std::vector<std::shared_ptr<output_t> > const &outs_,
+ std::shared_ptr<reprojection> proj);
~osmdata_t();
void start();
void stop();
- int node_add(osmid_t id, double lat, double lon, const taglist_t &tags);
- int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags);
- int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags);
+ int node_add(osmium::Node const &node);
+ int way_add(osmium::Way *way);
+ int relation_add(osmium::Relation const &rel);
- int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags);
- int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags);
- int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags);
+ int node_modify(osmium::Node const &node);
+ int way_modify(osmium::Way *way);
+ int relation_modify(osmium::Relation const &rel);
int node_delete(osmid_t id);
int way_delete(osmid_t id);
@@ -37,6 +42,7 @@ public:
private:
std::shared_ptr<middle_t> mid;
std::vector<std::shared_ptr<output_t> > outs;
+ std::shared_ptr<reprojection> projection;
};
#endif
diff --git a/osmium-builder.cpp b/osmium-builder.cpp
new file mode 100644
index 0000000..d0cb62d
--- /dev/null
+++ b/osmium-builder.cpp
@@ -0,0 +1,410 @@
+#include <algorithm>
+#include <cassert>
+#include <tuple>
+#include <vector>
+
+#include <osmium/area/geom_assembler.hpp>
+
+#include "osmium-builder.hpp"
+
+namespace {
+
+inline double distance(osmium::geom::Coordinates p1,
+ osmium::geom::Coordinates p2)
+{
+ double dx = p1.x - p2.x;
+ double dy = p1.y - p2.y;
+ return std::sqrt(dx * dx + dy * dy);
+}
+
+inline osmium::geom::Coordinates interpolate(osmium::geom::Coordinates p1,
+ osmium::geom::Coordinates p2,
+ double frac)
+{
+ return osmium::geom::Coordinates(frac * (p1.x - p2.x) + p2.x,
+ frac * (p1.y - p2.y) + p2.y);
+}
+
+template <typename ITERATOR>
+inline void add_nodes_to_builder(osmium::builder::WayNodeListBuilder &builder,
+ ITERATOR const &begin, ITERATOR const &end,
+ bool skip_first)
+{
+ auto it = begin;
+ if (skip_first) {
+ ++it;
+ }
+
+ while (it != end) {
+ if (it->location().valid()) {
+ builder.add_node_ref(*it);
+ }
+ ++it;
+ }
+}
+
+} // name space
+
+namespace geom {
+
+osmium_builder_t::wkb_t
+osmium_builder_t::get_wkb_node(osmium::Location const &loc) const
+{
+ return m_writer.make_point(m_proj->reproject(loc));
+}
+
+osmium_builder_t::wkbs_t
+osmium_builder_t::get_wkb_line(osmium::WayNodeList const &nodes, double split_at)
+{
+ wkbs_t ret;
+
+ bool do_split = split_at > 0.0;
+
+ double dist = 0;
+ osmium::geom::Coordinates prev_pt;
+ m_writer.linestring_start();
+ size_t curlen = 0;
+
+ for (auto const &node : nodes) {
+ if (!node.location().valid())
+ continue;
+
+ auto const this_pt = m_proj->reproject(node.location());
+ if (prev_pt.valid()) {
+ if (prev_pt == this_pt) {
+ continue;
+ }
+
+ if (do_split) {
+ double const delta = distance(prev_pt, this_pt);
+
+ // figure out if the addition of this point would take the total
+ // length of the line in `segment` over the `split_at` distance.
+
+ if (dist + delta > split_at) {
+ size_t const splits =
+ (size_t)std::floor((dist + delta) / split_at);
+ // use the splitting distance to split the current segment up
+ // into as many parts as necessary to keep each part below
+ // the `split_at` distance.
+ osmium::geom::Coordinates ipoint;
+ for (size_t j = 0; j < splits; ++j) {
+ double const frac =
+ ((double)(j + 1) * split_at - dist) / delta;
+ ipoint = interpolate(this_pt, prev_pt, frac);
+ m_writer.add_location(ipoint);
+ ret.push_back(m_writer.linestring_finish(curlen + 1));
+ // start a new segment
+ m_writer.linestring_start();
+ m_writer.add_location(ipoint);
+ curlen = 1;
+ }
+ // reset the distance based on the final splitting point for
+ // the next iteration.
+ if (this_pt == ipoint) {
+ dist = 0;
+ m_writer.linestring_finish(0);
+ m_writer.linestring_start();
+ curlen = 0;
+ } else {
+ dist = distance(this_pt, ipoint);
+ }
+ } else {
+ dist += delta;
+ }
+ }
+ }
+
+ m_writer.add_location(this_pt);
+ ++curlen;
+
+ prev_pt = this_pt;
+ }
+
+ auto wkb = m_writer.linestring_finish(curlen);
+ if (curlen > 1) {
+ ret.push_back(wkb);
+ }
+
+ return ret;
+}
+
+osmium_builder_t::wkb_t
+osmium_builder_t::get_wkb_polygon(osmium::Way const &way)
+{
+ osmium::area::AssemblerConfig area_config;
+ area_config.ignore_invalid_locations = true;
+ osmium::area::GeomAssembler assembler{area_config};
+
+ m_buffer.clear();
+ if (!assembler(way, m_buffer)) {
+ return wkb_t();
+ }
+
+ auto wkbs = create_polygons(m_buffer.get<osmium::Area>(0));
+
+ return wkbs.empty() ? wkb_t() : wkbs[0];
+}
+
+osmium_builder_t::wkbs_t
+osmium_builder_t::get_wkb_multipolygon(osmium::Relation const &rel,
+ osmium::memory::Buffer const &ways)
+{
+ wkbs_t ret;
+ osmium::area::AssemblerConfig area_config;
+ area_config.ignore_invalid_locations = true;
+ osmium::area::GeomAssembler assembler{area_config};
+
+ m_buffer.clear();
+ if (assembler(rel, ways, m_buffer)) {
+ if (m_build_multigeoms) {
+ ret.push_back(create_multipolygon(m_buffer.get<osmium::Area>(0)));
+ } else {
+ ret = create_polygons(m_buffer.get<osmium::Area>(0));
+ }
+ }
+
+ return ret;
+}
+
+osmium_builder_t::wkbs_t
+osmium_builder_t::get_wkb_multiline(osmium::memory::Buffer const &ways,
+ double split_at)
+{
+ // make a list of all endpoints
+ using endpoint_t = std::tuple<osmium::object_id_type, size_t, bool>;
+ std::vector<endpoint_t> endpoints;
+ // and a list of way connections
+ enum lmt : size_t
+ {
+ NOCONN = -1UL
+ };
+ std::vector<std::tuple<size_t, osmium::Way const *, size_t>> conns;
+
+ // initialise the two lists
+ for (auto const &w : ways.select<osmium::Way>()) {
+ if (w.nodes().size() > 1) {
+ endpoints.emplace_back(w.nodes().front().ref(), conns.size(), true);
+ endpoints.emplace_back(w.nodes().back().ref(), conns.size(), false);
+ conns.emplace_back(NOCONN, &w, NOCONN);
+ }
+ }
+ // sort by node id
+ std::sort(endpoints.begin(), endpoints.end());
+ // now fill the connection list based on the sorted list
+ {
+ endpoint_t const *prev = nullptr;
+ for (auto const &pt : endpoints) {
+ if (prev) {
+ if (std::get<0>(*prev) == std::get<0>(pt)) {
+ auto previd = std::get<1>(*prev);
+ auto ptid = std::get<1>(pt);
+ if (std::get<2>(*prev)) {
+ std::get<0>(conns[previd]) = ptid;
+ } else {
+ std::get<2>(conns[previd]) = ptid;
+ }
+ if (std::get<2>(pt)) {
+ std::get<0>(conns[ptid]) = previd;
+ } else {
+ std::get<2>(conns[ptid]) = previd;
+ }
+ prev = nullptr;
+ continue;
+ }
+ }
+
+ prev = &pt;
+ }
+ }
+
+ wkbs_t ret;
+
+ size_t done_ways = 0;
+ size_t todo_ways = conns.size();
+ for (size_t i = 0; i < todo_ways; ++i) {
+ if (!std::get<1>(conns[i]) || (std::get<0>(conns[i]) != NOCONN &&
+ std::get<2>(conns[i]) != NOCONN)) {
+ continue; // way already done or not the beginning of a segment
+ }
+
+ m_buffer.clear();
+ {
+ osmium::builder::WayNodeListBuilder wnl_builder(m_buffer);
+ size_t prev = NOCONN;
+ size_t cur = i;
+
+ do {
+ auto &conn = conns[cur];
+ assert(std::get<1>(conn));
+ auto &nl = std::get<1>(conn)->nodes();
+ bool skip_first = prev != NOCONN;
+ bool forward = std::get<0>(conn) == prev;
+ prev = cur;
+ // add way nodes
+ if (forward) {
+ add_nodes_to_builder(wnl_builder, nl.cbegin(), nl.cend(),
+ skip_first);
+ cur = std::get<2>(conn);
+ } else {
+ add_nodes_to_builder(wnl_builder, nl.crbegin(), nl.crend(),
+ skip_first);
+ cur = std::get<0>(conn);
+ }
+ // mark way as done
+ std::get<1>(conns[prev]) = nullptr;
+ ++done_ways;
+ } while (cur != NOCONN);
+ }
+
+ // found a line end, create the wkbs
+ m_buffer.commit();
+ auto linewkbs =
+ get_wkb_line(m_buffer.get<osmium::WayNodeList>(0), split_at);
+ std::move(linewkbs.begin(), linewkbs.end(),
+ std::inserter(ret, ret.end()));
+ }
+
+ if (done_ways < todo_ways) {
+ // oh dear, there must be circular ways without an end
+ // need to do the same shebang again
+ for (size_t i = 0; i < todo_ways; ++i) {
+ if (!std::get<1>(conns[i])) {
+ continue; // way already done
+ }
+
+ m_buffer.clear();
+
+ {
+ osmium::builder::WayNodeListBuilder wnl_builder(m_buffer);
+ size_t prev = std::get<0>(conns[i]);
+ size_t cur = i;
+ bool skip_first = false;
+
+ do {
+ auto &conn = conns[cur];
+ assert(std::get<1>(conn));
+ auto &nl = std::get<1>(conn)->nodes();
+ bool forward = std::get<0>(conn) == prev;
+ prev = cur;
+ if (forward) {
+ // add way forwards
+ add_nodes_to_builder(wnl_builder, nl.cbegin(), nl.cend(),
+ skip_first);
+ cur = std::get<2>(conn);
+ } else {
+ // add way backwards
+ add_nodes_to_builder(wnl_builder, nl.crbegin(), nl.crend(),
+ skip_first);
+ cur = std::get<0>(conn);
+ }
+ // mark way as done
+ std::get<1>(conns[prev]) = nullptr;
+ skip_first = true;
+ } while (cur != i);
+ }
+
+ // found a line end, create the wkbs
+ m_buffer.commit();
+ auto linewkbs =
+ get_wkb_line(m_buffer.get<osmium::WayNodeList>(0), split_at);
+ std::move(linewkbs.begin(), linewkbs.end(),
+ std::inserter(ret, ret.end()));
+ }
+ }
+
+ if (split_at <= 0.0 && !ret.empty()) {
+ auto num_lines = ret.size();
+ m_writer.multilinestring_start();
+ for (auto const &line : ret) {
+ m_writer.add_sub_geometry(line);
+ }
+ ret.clear();
+ ret.push_back(m_writer.multilinestring_finish(num_lines));
+ }
+
+ return ret;
+}
+
+size_t osmium_builder_t::add_mp_points(const osmium::NodeRefList &nodes)
+{
+ size_t num_points = 0;
+ osmium::Location last_location;
+ for (const osmium::NodeRef &node_ref : nodes) {
+ if (node_ref.location().valid() &&
+ last_location != node_ref.location()) {
+ last_location = node_ref.location();
+ m_writer.add_location(m_proj->reproject(last_location));
+ ++num_points;
+ }
+ }
+
+ return num_points;
+}
+
+osmium_builder_t::wkb_t
+osmium_builder_t::create_multipolygon(osmium::Area const &area)
+{
+ wkb_t ret;
+
+ auto polys = create_polygons(area);
+
+ switch (polys.size()) {
+ case 0:
+ break; //nothing
+ case 1:
+ ret = polys[0];
+ break;
+ default:
+ m_writer.multipolygon_start();
+ for (auto const &p : polys) {
+ m_writer.add_sub_geometry(p);
+ }
+ ret = m_writer.multipolygon_finish(polys.size());
+ break;
+ }
+
+ return ret;
+}
+
+osmium_builder_t::wkbs_t
+osmium_builder_t::create_polygons(osmium::Area const &area)
+{
+ wkbs_t ret;
+
+ try {
+ size_t num_rings = 0;
+
+ for (auto it = area.cbegin(); it != area.cend(); ++it) {
+ if (it->type() == osmium::item_type::outer_ring) {
+ auto &ring = static_cast<const osmium::OuterRing &>(*it);
+ if (num_rings > 0) {
+ ret.push_back(m_writer.polygon_finish(num_rings));
+ num_rings = 0;
+ }
+ m_writer.polygon_start();
+ m_writer.polygon_ring_start();
+ auto num_points = add_mp_points(ring);
+ m_writer.polygon_ring_finish(num_points);
+ ++num_rings;
+ } else if (it->type() == osmium::item_type::inner_ring) {
+ auto &ring = static_cast<const osmium::InnerRing &>(*it);
+ m_writer.polygon_ring_start();
+ auto num_points = add_mp_points(ring);
+ m_writer.polygon_ring_finish(num_points);
+ ++num_rings;
+ }
+ }
+
+ auto wkb = m_writer.polygon_finish(num_rings);
+ if (num_rings > 0) {
+ ret.push_back(wkb);
+ }
+
+ } catch (osmium::geometry_error) { /* ignored */
+ }
+
+ return ret;
+}
+
+} // name space
diff --git a/osmium-builder.hpp b/osmium-builder.hpp
new file mode 100644
index 0000000..10ccaa5
--- /dev/null
+++ b/osmium-builder.hpp
@@ -0,0 +1,51 @@
+#ifndef OSMIUM_BUILDER_H
+#define OSMIUM_BUILDER_H
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <osmium/memory/buffer.hpp>
+#include <osmium/osm.hpp>
+
+#include "reprojection.hpp"
+#include "wkb.hpp"
+
+namespace geom {
+
+class osmium_builder_t
+{
+public:
+ typedef std::string wkb_t;
+ typedef std::vector<std::string> wkbs_t;
+
+ explicit osmium_builder_t(std::shared_ptr<reprojection> const &proj,
+ bool build_multigeoms)
+ : m_proj(proj), m_buffer(1024, osmium::memory::Buffer::auto_grow::yes),
+ m_writer(m_proj->target_srs()), m_build_multigeoms(build_multigeoms)
+ {
+ }
+
+ wkb_t get_wkb_node(osmium::Location const &loc) const;
+ wkbs_t get_wkb_line(osmium::WayNodeList const &way, double split_at);
+ wkb_t get_wkb_polygon(osmium::Way const &way);
+
+ wkbs_t get_wkb_multipolygon(osmium::Relation const &rel,
+ osmium::memory::Buffer const &ways);
+ wkbs_t get_wkb_multiline(osmium::memory::Buffer const &ways, double split_at);
+
+private:
+ wkb_t create_multipolygon(osmium::Area const &area);
+ wkbs_t create_polygons(osmium::Area const &area);
+ size_t add_mp_points(const osmium::NodeRefList &nodes);
+
+ std::shared_ptr<reprojection> m_proj;
+ // internal buffer for creating areas
+ osmium::memory::Buffer m_buffer;
+ ewkb::writer_t m_writer;
+ bool m_build_multigeoms;
+};
+
+} // namespace
+
+#endif // OSMIUM_BUILDER_H
diff --git a/osmtypes.hpp b/osmtypes.hpp
index 1fd6151..9951c64 100644
--- a/osmtypes.hpp
+++ b/osmtypes.hpp
@@ -13,39 +13,58 @@
#include <vector>
#include <cmath>
+#include <osmium/builder/attr.hpp>
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/osm.hpp>
+
typedef int64_t osmid_t;
#define strtoosmid strtoll
#define PRIdOSMID PRId64
#define POSTGRES_OSMID_TYPE "int8"
-enum OsmType { OSMTYPE_WAY, OSMTYPE_NODE, OSMTYPE_RELATION };
-
-struct osmNode {
- double lon;
- double lat;
+struct member {
+ osmium::item_type type;
+ osmid_t id;
+ std::string role;
- osmNode() : lon(NAN), lat(NAN) {}
+ operator osmium::builder::attr::member_type const() const
+ {
+ return osmium::builder::attr::member_type(type, id, role.c_str());
+ }
- osmNode(double x, double y) : lon(x), lat(y) {}
+ member(osmium::item_type t, osmid_t i, const std::string &r)
+ : type(t), id(i), role(r) {}
};
-typedef std::vector<osmNode> nodelist_t;
-typedef std::vector<nodelist_t> multinodelist_t;
+struct memberlist_t : public std::vector<member> {
+ memberlist_t() {}
-struct member {
- OsmType type;
- osmid_t id;
- std::string role;
+ explicit memberlist_t(osmium::RelationMemberList const &list) {
+ for (auto const &m: list) {
+ emplace_back(m.type(), m.ref(), m.role());
+ }
+ }
- member(OsmType t, osmid_t i, const std::string &r) : type(t), id(i), role(r) {}
-};
+ std::vector<osmium::builder::attr::member_type> for_builder() const
+ {
+ std::vector<osmium::builder::attr::member_type> ret;
+ for (auto const &m : *this) {
+ ret.emplace_back(m.type, m.id, m.role.c_str());
+ }
-typedef std::vector<member> memberlist_t;
+ return ret;
+ }
+};
struct tag_t {
std::string key;
std::string value;
+ operator std::pair<char const *, char const *> const() const
+ {
+ return std::pair<char const *, char const *>(key.c_str(), value.c_str());
+ }
+
tag_t(const std::string &k, const std::string &v) : key(k), value(v) {}
};
@@ -55,6 +74,24 @@ class taglist_t : public std::vector<tag_t> {
typedef std::vector<tag_t> base_t;
public:
+ taglist_t() {}
+
+ explicit taglist_t(osmium::TagList const &list)
+ {
+ for (auto const &t : list) {
+ emplace_back(t.key(), t.value());
+ }
+ }
+
+ void add_attributes(const osmium::OSMObject &obj)
+ {
+ emplace_back("osm_user", obj.user());
+ emplace_back("osm_uid", std::to_string(obj.uid()));
+ emplace_back("osm_version", std::to_string(obj.version()));
+ emplace_back("osm_timestamp", obj.timestamp().to_iso());
+ emplace_back("osm_changeset", std::to_string(obj.changeset()));
+ }
+
const tag_t *find(const std::string &key) const { return _find(key); }
tag_t *find(const std::string &key) { return const_cast<tag_t *>(_find(key)); }
@@ -77,20 +114,28 @@ public:
return 0;
}
+ static bool value_to_bool(char const *value, bool defval)
+ {
+ if (!defval &&
+ (strcmp(value, "yes") == 0
+ || strcmp(value, "true") == 0
+ || strcmp(value, "1") == 0))
+ return true;
+ if (defval &&
+ (strcmp(value, "no") == 0 || strcmp(value, "false") == 0 || strcmp(value, "0") == 0))
+ return false;
+
+ return defval;
+ }
+
bool get_bool(const std::string &key, bool defval) const
{
- for (base_t::const_iterator it = begin() ; it != end(); ++it)
- if (it->key == key) {
- if (!defval &&
- (it->value == "yes" || it->value == "true" || it->value == "1"))
- return true;
- if (defval &&
- (it->value == "no" || it->value == "false" || it->value == "0"))
- return false;
- return defval;
- }
+ for (auto const &t : *this)
+ if (t.key == key) {
+ return value_to_bool(t.value.c_str(), defval);
+ }
- return defval;
+ return defval;
}
void push_dedupe(const tag_t& t)
@@ -125,10 +170,16 @@ private:
};
-typedef std::vector<taglist_t> multitaglist_t;
+struct idlist_t : public std::vector<osmid_t> {
+ idlist_t() {}
-typedef std::vector<osmid_t> idlist_t;
+ explicit idlist_t(osmium::NodeRefList const &list) {
+ for (auto const &n : list) {
+ push_back(n.ref());
+ }
+ }
+};
-typedef std::vector<const std::string *> rolelist_t;
+typedef std::vector<char const *> rolelist_t;
#endif
diff --git a/output-gazetteer.cpp b/output-gazetteer.cpp
index 1de4fa4..272df41 100644
--- a/output-gazetteer.cpp
+++ b/output-gazetteer.cpp
@@ -2,53 +2,25 @@
#include <boost/algorithm/string/predicate.hpp>
#include <boost/format.hpp>
-#include "osmtypes.hpp"
#include "middle.hpp"
-#include "pgsql.hpp"
-#include "reprojection.hpp"
-#include "output-gazetteer.hpp"
#include "options.hpp"
+#include "osmtypes.hpp"
+#include "output-gazetteer.hpp"
+#include "pgsql.hpp"
#include "util.hpp"
+#include "wkb.hpp"
#include <algorithm>
+#include <cstring>
#include <iostream>
#include <memory>
-#define CREATE_PLACE_TABLE \
- "CREATE TABLE place (" \
- " osm_type CHAR(1) NOT NULL," \
- " osm_id " POSTGRES_OSMID_TYPE " NOT NULL," \
- " class TEXT NOT NULL," \
- " type TEXT NOT NULL," \
- " name HSTORE," \
- " admin_level INTEGER," \
- " housenumber TEXT," \
- " street TEXT," \
- " addr_place TEXT," \
- " isin TEXT," \
- " postcode TEXT," \
- " country_code VARCHAR(2)," \
- " extratags HSTORE" \
- ") %s %s"
-
-#define ADMINLEVEL_NONE 100
-
-#define CREATE_PLACE_ID_INDEX \
- "CREATE INDEX place_id_idx ON place USING BTREE (osm_type, osm_id) %s %s"
-
-
-enum { BUFFER_SIZE = 4092 };
+enum : int { MAX_ADMINLEVEL = 15 };
void place_tag_processor::clear()
{
// set members to sane defaults
- src = nullptr;
- admin_level = ADMINLEVEL_NONE;
- countrycode = 0;
- housenumber.assign("\\N");
- street = 0;
- addr_place = 0;
- postcode = 0;
+ admin_level = MAX_ADMINLEVEL;
places.clear();
extratags.clear();
@@ -77,249 +49,246 @@ struct UnnamedPredicate
}
};
-void place_tag_processor::process_tags(const taglist_t &tags)
+void place_tag_processor::process_tags(osmium::OSMObject const &o)
{
bool placeadmin = false;
bool placehouse = false;
bool placebuilding = false;
- const tag_t *place = 0;
- const tag_t *junction = 0;
- const tag_t *landuse = 0;
+ osmium::Tag const *place = 0;
+ osmium::Tag const *junction = 0;
+ osmium::Tag const *landuse = 0;
bool isnamed = false;
bool isinterpolation = false;
- const std::string *house_nr = 0;
- const std::string *conscr_nr = 0;
- const std::string *street_nr = 0;
clear();
- src = &tags;
- for (const auto& item: tags) {
- if (boost::ends_with(item.key, "source")) {
+ for (const auto &item: o.tags()) {
+ char const *k = item.key();
+ char const *v = item.value();
+ if (boost::ends_with(k, "source")) {
// ignore
- } else if (item.key == "name:prefix" ||
- item.key == "name:botanical" ||
- boost::ends_with(item.key, "wikidata")) {
+ } else if (strcmp(k, "name:prefix") == 0 ||
+ strcmp(k, "name:botanical") == 0 ||
+ boost::ends_with(k, "wikidata")) {
extratags.push_back(&item);
- } else if (item.key == "ref" ||
- item.key == "int_ref" ||
- item.key == "nat_ref" ||
- item.key == "reg_ref" ||
- item.key == "loc_ref" ||
- item.key == "old_ref" ||
- item.key == "iata" ||
- item.key == "icao" ||
- item.key == "operator" ||
- item.key == "pcode" ||
- boost::starts_with(item.key, "pcode:")) {
+ } else if (strcmp(k, "ref") == 0 ||
+ strcmp(k, "int_ref") == 0 ||
+ strcmp(k, "nat_ref") == 0 ||
+ strcmp(k, "reg_ref") == 0 ||
+ strcmp(k, "loc_ref") == 0 ||
+ strcmp(k, "old_ref") == 0 ||
+ strcmp(k, "iata") == 0 ||
+ strcmp(k, "icao") == 0 ||
+ strcmp(k, "operator") == 0 ||
+ strcmp(k, "pcode") == 0 ||
+ boost::starts_with(k, "pcode:")) {
names.push_back(&item);
- } else if (item.key == "name" ||
- boost::starts_with(item.key, "name:") ||
- item.key == "int_name" ||
- boost::starts_with(item.key, "int_name:") ||
- item.key == "nat_name" ||
- boost::starts_with(item.key, "nat_name:") ||
- item.key == "reg_name" ||
- boost::starts_with(item.key, "reg_name:") ||
- item.key == "loc_name" ||
- boost::starts_with(item.key, "loc_name:") ||
- item.key == "old_name" ||
- boost::starts_with(item.key, "old_name:") ||
- item.key == "alt_name" ||
- boost::starts_with(item.key, "alt_name:") ||
- boost::starts_with(item.key, "alt_name_") ||
- item.key == "official_name" ||
- boost::starts_with(item.key, "official_name:") ||
- item.key == "place_name" ||
- boost::starts_with(item.key, "place_name:") ||
- item.key == "short_name" ||
- boost::starts_with(item.key, "short_name:") ||
- item.key == "brand") {
+ } else if (strcmp(k, "name") == 0 ||
+ boost::starts_with(k, "name:") ||
+ strcmp(k, "int_name") == 0 ||
+ boost::starts_with(k, "int_name:") ||
+ strcmp(k, "nat_name") == 0 ||
+ boost::starts_with(k, "nat_name:") ||
+ strcmp(k, "reg_name") == 0 ||
+ boost::starts_with(k, "reg_name:") ||
+ strcmp(k, "loc_name") == 0 ||
+ boost::starts_with(k, "loc_name:") ||
+ strcmp(k, "old_name") == 0 ||
+ boost::starts_with(k, "old_name:") ||
+ strcmp(k, "alt_name") == 0 ||
+ boost::starts_with(k, "alt_name:") ||
+ boost::starts_with(k, "alt_name_") ||
+ strcmp(k, "official_name") == 0 ||
+ boost::starts_with(k, "official_name:") ||
+ strcmp(k, "place_name") == 0 ||
+ boost::starts_with(k, "place_name:") ||
+ strcmp(k, "short_name") == 0 ||
+ boost::starts_with(k, "short_name:") ||
+ strcmp(k, "brand") == 0) {
names.push_back(&item);
isnamed = true;
- } else if (item.key == "addr:housename") {
+ } else if (strcmp(k, "addr:housename") == 0) {
names.push_back(&item);
placehouse = true;
- } else if (item.key == "emergency") {
- if (item.value != "fire_hydrant" &&
- item.value != "yes" &&
- item.value != "no")
- places.push_back(item);
- } else if (item.key == "tourism" ||
- item.key == "historic" ||
- item.key == "military") {
- if (item.value != "no" && item.value != "yes")
- places.push_back(item);
- } else if (item.key == "natural") {
- if (item.value != "no" &&
- item.value != "yes" &&
- item.value != "coastline")
- places.push_back(item);
- } else if (item.key == "landuse") {
- if (item.value == "cemetry")
- places.push_back(item);
+ } else if (strcmp(k, "emergency") == 0) {
+ if (strcmp(v, "fire_hydrant") != 0 &&
+ strcmp(v, "yes") != 0 &&
+ strcmp(v, "no") != 0)
+ places.emplace_back(k, v);
+ } else if (strcmp(k, "tourism") == 0 ||
+ strcmp(k, "historic") == 0 ||
+ strcmp(k, "military") == 0) {
+ if (strcmp(v, "no") != 0 && strcmp(v, "yes") != 0)
+ places.emplace_back(k, v);
+ } else if (strcmp(k, "natural") == 0) {
+ if (strcmp(v, "no") != 0 &&
+ strcmp(v, "yes") != 0 &&
+ strcmp(v, "coastline") != 0)
+ places.emplace_back(k, v);
+ } else if (strcmp(k, "landuse") == 0) {
+ if (strcmp(v, "cemetry") == 0)
+ places.emplace_back(k, v);
else
landuse = &item;
- } else if (item.key == "highway") {
- if (item.value == "footway") {
- auto *footway = tags.get("footway");
- if (footway == nullptr || *footway != "sidewalk")
- places.push_back(item);
- } else if (item.value != "no" &&
- item.value != "turning_circle" &&
- item.value != "mini_roundabout" &&
- item.value != "noexit" &&
- item.value != "crossing")
- places.push_back(item);
- } else if (item.key == "railway") {
- if (item.value != "level_crossing" &&
- item.value != "no")
- places.push_back(item);
- } else if (item.key == "man_made") {
- if (item.value != "survey_point" &&
- item.value != "cutline")
- places.push_back(item);
- } else if (item.key == "aerialway") {
- if (item.value != "pylon" &&
- item.value != "no")
- places.push_back(item);
- } else if (item.key == "boundary") {
- if (item.value == "administrative")
+ } else if (strcmp(k, "highway") == 0) {
+ if (strcmp(v, "footway") == 0) {
+ auto *footway = o.tags()["footway"];
+ if (footway == nullptr || strcmp(footway, "sidewalk") != 0)
+ places.emplace_back(k, v);
+ } else if (strcmp(v, "no") != 0 &&
+ strcmp(v, "turning_circle") != 0 &&
+ strcmp(v, "mini_roundabout") != 0 &&
+ strcmp(v, "noexit") != 0 &&
+ strcmp(v, "crossing") != 0)
+ places.emplace_back(k, v);
+ } else if (strcmp(k, "railway") == 0) {
+ if (strcmp(v, "level_crossing") != 0 &&
+ strcmp(v, "no") != 0)
+ places.emplace_back(k, v);
+ } else if (strcmp(k, "man_made") == 0) {
+ if (strcmp(v, "survey_point") != 0 &&
+ strcmp(v, "cutline") != 0)
+ places.emplace_back(k, v);
+ } else if (strcmp(k, "aerialway") == 0) {
+ if (strcmp(v, "pylon") != 0 &&
+ strcmp(v, "no") != 0)
+ places.emplace_back(k, v);
+ } else if (strcmp(k, "boundary") == 0) {
+ if (strcmp(v, "administrative") == 0)
placeadmin = true;
- places.push_back(item);
- } else if (item.key == "aeroway" ||
- item.key == "amenity" ||
- item.key == "boundary" ||
- item.key == "bridge" ||
- item.key == "craft" ||
- item.key == "leisure" ||
- item.key == "office" ||
- item.key == "shop" ||
- item.key == "tunnel" ||
- item.key == "mountain_pass") {
- if (item.value != "no")
+ places.emplace_back(k, v);
+ } else if (strcmp(k, "aeroway") == 0 ||
+ strcmp(k, "amenity") == 0 ||
+ strcmp(k, "club") == 0 ||
+ strcmp(k, "boundary") == 0 ||
+ strcmp(k, "bridge") == 0 ||
+ strcmp(k, "craft") == 0 ||
+ strcmp(k, "leisure") == 0 ||
+ strcmp(k, "office") == 0 ||
+ strcmp(k, "shop") == 0 ||
+ strcmp(k, "tunnel") == 0 ||
+ strcmp(k, "mountain_pass") == 0) {
+ if (strcmp(v, "no") != 0)
{
- places.push_back(item);
+ places.emplace_back(k, v);
}
- } else if (item.key == "waterway") {
- if (item.value != "riverbank")
- places.push_back(item);
- } else if (item.key == "place") {
+ } else if (strcmp(k, "waterway") == 0) {
+ if (strcmp(v, "riverbank") != 0)
+ places.emplace_back(k, v);
+ } else if (strcmp(k, "place") == 0) {
place = &item;
- } else if (item.key == "junction") {
+ } else if (strcmp(k, "junction") == 0) {
junction = &item;
- } else if (item.key == "addr:interpolation") {
- housenumber.clear();
- escape(item.value, housenumber);
- isinterpolation = true;
- } else if (item.key == "addr:housenumber") {
- house_nr = &item.value;
- placehouse = true;
- } else if (item.key == "addr:conscriptionnumber") {
- conscr_nr = &item.value;
- placehouse = true;
- } else if (item.key == "addr:streetnumber") {
- street_nr = &item.value;
- placehouse = true;
- } else if (item.key == "addr:street") {
- street = &item.value;
- } else if (item.key == "addr:place") {
- addr_place = &item.value;
- } else if (item.key == "postal_code" ||
- item.key == "postcode" ||
- item.key == "addr:postcode" ||
- item.key == "tiger:zip_left" ||
- item.key == "tiger:zip_right") {
- if (!postcode)
- postcode = &item.value;
- } else if (item.key == "country_code" ||
- item.key == "ISO3166-1" ||
- item.key == "is_in:country_code" ||
- item.key == "addr:country" ||
- item.key == "addr:country_code") {
- if (item.value.length() == 2)
- countrycode = &item.value;
- } else if (boost::starts_with(item.key, "addr:") ||
- item.key == "is_in" ||
- boost::starts_with(item.key, "is_in:") ||
- item.key == "tiger:county") {
- address.push_back(&item);
- } else if (item.key == "admin_level") {
- admin_level = atoi(item.value.c_str());
- if (admin_level <= 0 || admin_level > 100)
- admin_level = 100;
- } else if (item.key == "tracktype" ||
- item.key == "traffic_calming" ||
- item.key == "service" ||
- item.key == "cuisine" ||
- item.key == "capital" ||
- item.key == "dispensing" ||
- item.key == "religion" ||
- item.key == "denomination" ||
- item.key == "sport" ||
- item.key == "internet_access" ||
- item.key == "lanes" ||
- item.key == "surface" ||
- item.key == "smoothness" ||
- item.key == "width" ||
- item.key == "est_width" ||
- item.key == "incline" ||
- item.key == "opening_hours" ||
- item.key == "collection_times" ||
- item.key == "service_times" ||
- item.key == "disused" ||
- item.key == "wheelchair" ||
- item.key == "sac_scale" ||
- item.key == "trail_visibility" ||
- item.key == "mtb:scale" ||
- item.key == "mtb:description" ||
- item.key == "wood" ||
- item.key == "drive_through" ||
- item.key == "drive_in" ||
- item.key == "access" ||
- item.key == "vehicle" ||
- item.key == "bicyle" ||
- item.key == "foot" ||
- item.key == "goods" ||
- item.key == "hgv" ||
- item.key == "motor_vehicle" ||
- item.key == "motor_car" ||
- boost::starts_with(item.key, "access:") ||
- boost::starts_with(item.key, "contact:") ||
- boost::starts_with(item.key, "drink:") ||
- item.key == "oneway" ||
- item.key == "date_on" ||
- item.key == "date_off" ||
- item.key == "day_on" ||
- item.key == "day_off" ||
- item.key == "hour_on" ||
- item.key == "hour_off" ||
- item.key == "maxweight" ||
- item.key == "maxheight" ||
- item.key == "maxspeed" ||
- item.key == "fee" ||
- item.key == "toll" ||
- boost::starts_with(item.key, "toll:") ||
- item.key == "charge" ||
- item.key == "population" ||
- item.key == "description" ||
- item.key == "image" ||
- item.key == "attribution" ||
- item.key == "fax" ||
- item.key == "email" ||
- item.key == "url" ||
- item.key == "website" ||
- item.key == "phone" ||
- item.key == "real_ale" ||
- item.key == "smoking" ||
- item.key == "food" ||
- item.key == "camera" ||
- item.key == "brewery" ||
- item.key == "locality" ||
- item.key == "wikipedia" ||
- boost::starts_with(item.key, "wikipedia:")) {
+ } else if (strcmp(k, "postal_code") == 0 ||
+ strcmp(k, "postcode") == 0 ||
+ strcmp(k, "addr:postcode") == 0 ||
+ strcmp(k, "tiger:zip_left") == 0 ||
+ strcmp(k, "tiger:zip_right") == 0) {
+ if (address.find("postcode") == address.end()) {
+ address.emplace("postcode", v);
+ }
+ } else if (strcmp(k, "country_code") == 0 ||
+ strcmp(k, "ISO3166-1") == 0 ||
+ strcmp(k, "is_in:country_code") == 0 ||
+ strcmp(k, "is_in:country") == 0 ||
+ strcmp(k, "addr:country") == 0 ||
+ strcmp(k, "addr:country_code") == 0) {
+ if (strlen(v) == 2 && address.find("country") == address.end()) {
+ address.emplace("country", v);
+ }
+ } else if (boost::starts_with(k, "addr:")) {
+ if (strcmp(k, "addr:interpolation") == 0) {
+ isinterpolation = true;
+ }
+ if (strcmp(k, "addr:housenumber") == 0 ||
+ strcmp(k, "addr:conscriptionnumber") == 0 ||
+ strcmp(k, "addr:streetnumber") == 0) {
+ placehouse = true;
+ }
+ address.emplace(k + 5, v);
+ } else if (boost::starts_with(k, "is_in:")) {
+ if (address.find(k + 6) == address.end()) {
+ address.emplace(k + 6, v);
+ }
+ } else if (strcmp(k, "is_in") == 0 ||
+ strcmp(k, "tiger:county") == 0) {
+ address.emplace(k, v);
+ } else if (strcmp(k, "admin_level") == 0) {
+ admin_level = atoi(v);
+ if (admin_level <= 0 || admin_level > MAX_ADMINLEVEL)
+ admin_level = MAX_ADMINLEVEL;
+ } else if (strcmp(k, "tracktype") == 0 ||
+ strcmp(k, "traffic_calming") == 0 ||
+ strcmp(k, "service") == 0 ||
+ strcmp(k, "cuisine") == 0 ||
+ strcmp(k, "capital") == 0 ||
+ strcmp(k, "dispensing") == 0 ||
+ strcmp(k, "religion") == 0 ||
+ strcmp(k, "denomination") == 0 ||
+ strcmp(k, "sport") == 0 ||
+ strcmp(k, "internet_access") == 0 ||
+ strcmp(k, "lanes") == 0 ||
+ strcmp(k, "surface") == 0 ||
+ strcmp(k, "smoothness") == 0 ||
+ strcmp(k, "width") == 0 ||
+ strcmp(k, "est_width") == 0 ||
+ strcmp(k, "incline") == 0 ||
+ strcmp(k, "opening_hours") == 0 ||
+ strcmp(k, "collection_times") == 0 ||
+ strcmp(k, "service_times") == 0 ||
+ strcmp(k, "disused") == 0 ||
+ strcmp(k, "wheelchair") == 0 ||
+ strcmp(k, "sac_scale") == 0 ||
+ strcmp(k, "trail_visibility") == 0 ||
+ strcmp(k, "mtb:scale") == 0 ||
+ strcmp(k, "mtb:description") == 0 ||
+ strcmp(k, "wood") == 0 ||
+ strcmp(k, "drive_through") == 0 ||
+ strcmp(k, "drive_in") == 0 ||
+ strcmp(k, "access") == 0 ||
+ strcmp(k, "vehicle") == 0 ||
+ strcmp(k, "bicyle") == 0 ||
+ strcmp(k, "foot") == 0 ||
+ strcmp(k, "goods") == 0 ||
+ strcmp(k, "hgv") == 0 ||
+ strcmp(k, "motor_vehicle") == 0 ||
+ strcmp(k, "motor_car") == 0 ||
+ boost::starts_with(k, "access:") ||
+ boost::starts_with(k, "contact:") ||
+ boost::starts_with(k, "drink:") ||
+ strcmp(k, "oneway") == 0 ||
+ strcmp(k, "date_on") == 0 ||
+ strcmp(k, "date_off") == 0 ||
+ strcmp(k, "day_on") == 0 ||
+ strcmp(k, "day_off") == 0 ||
+ strcmp(k, "hour_on") == 0 ||
+ strcmp(k, "hour_off") == 0 ||
+ strcmp(k, "maxweight") == 0 ||
+ strcmp(k, "maxheight") == 0 ||
+ strcmp(k, "maxspeed") == 0 ||
+ strcmp(k, "fee") == 0 ||
+ strcmp(k, "toll") == 0 ||
+ boost::starts_with(k, "toll:") ||
+ strcmp(k, "charge") == 0 ||
+ strcmp(k, "population") == 0 ||
+ strcmp(k, "description") == 0 ||
+ strcmp(k, "image") == 0 ||
+ strcmp(k, "attribution") == 0 ||
+ strcmp(k, "fax") == 0 ||
+ strcmp(k, "email") == 0 ||
+ strcmp(k, "url") == 0 ||
+ strcmp(k, "website") == 0 ||
+ strcmp(k, "phone") == 0 ||
+ strcmp(k, "real_ale") == 0 ||
+ strcmp(k, "smoking") == 0 ||
+ strcmp(k, "food") == 0 ||
+ strcmp(k, "camera") == 0 ||
+ strcmp(k, "brewery") == 0 ||
+ strcmp(k, "locality") == 0 ||
+ strcmp(k, "wikipedia") == 0 ||
+ boost::starts_with(k, "wikipedia:")) {
extratags.push_back(&item);
- } else if (item.key == "building") {
+ } else if (strcmp(k, "building") == 0) {
placebuilding = true;
}
}
@@ -333,73 +302,54 @@ void place_tag_processor::process_tags(const taglist_t &tags)
}
if (isinterpolation)
- places.push_back(tag_t("place", "houses"));
+ places.emplace_back("place", "houses");
if (place) {
if (isinterpolation ||
(placeadmin &&
- place ->value != "island" &&
- place ->value != "islet"))
- extratags.push_back(place);
+ strcmp(place->value(), "island") != 0 &&
+ strcmp(place->value(), "islet") != 0))
+ extratags.emplace_back(place);
else
- places.push_back(*place);
+ places.emplace_back(place->key(), place->value());
}
if (isnamed && places.empty()) {
if (junction)
- places.push_back(*junction);
+ places.emplace_back(junction->key(), junction->value());
else if (landuse)
- places.push_back(*landuse);
+ places.emplace_back(landuse->key(), landuse->value());
}
if (places.empty()) {
+ bool postcode = address.find("postcode") != address.end();
if (placebuilding && (!names.empty() || placehouse || postcode)) {
- places.push_back(tag_t("building", "yes"));
+ places.emplace_back("building", "yes");
} else if (placehouse) {
- places.push_back(tag_t("place", "house"));
+ places.emplace_back("place", "house");
} else if (postcode) {
- places.push_back(tag_t("place", "postcode"));
+ places.emplace_back("place", "postcode");
}
}
-
- // housenumbers
- if (!isinterpolation) {
- if (street_nr && conscr_nr) {
- housenumber.clear();
- escape(*conscr_nr, housenumber);
- housenumber.append("/");
- escape(*street_nr, housenumber);
- } else if (conscr_nr) {
- housenumber.clear();
- escape(*conscr_nr, housenumber);
- } else if (street_nr) {
- housenumber.clear();
- escape(*street_nr, housenumber);
- } else if (house_nr) {
- housenumber.clear();
- escape(*house_nr, housenumber);
- }
- }
-
}
-void place_tag_processor::copy_out(char osm_type, osmid_t osm_id,
+void place_tag_processor::copy_out(osmium::OSMObject const &o,
const std::string &geom,
std::string &buffer)
{
for (const auto& place: places) {
std::string name;
if (place.key == "bridge" || place.key == "tunnel") {
- name = domain_name(place.key);
+ name = domain_name(place.key, o.tags());
if (name.empty())
continue; // don't include unnamed bridges and tunnels
}
// osm_type
- buffer += osm_type;
+ buffer += (char) toupper(osmium::item_type_to_char(o.type()));
buffer += '\t';
// osm_id
- buffer += (single_fmt % osm_id).str();
+ buffer += (single_fmt % o.id()).str();
// class
escape(place.key, buffer);
buffer += '\t';
@@ -418,73 +368,66 @@ void place_tag_processor::copy_out(char osm_type, osmid_t osm_id,
(place.key == "amenity") ||
(place.key == "tourism");
for (const auto entry: names) {
- if (!shop && (entry->key == "operator"))
+ if (!shop && strcmp(entry->key(), "operator") == 0)
continue;
- if (first)
+ if (first) {
first = false;
- else
+ } else {
buffer += ',';
+ }
buffer += "\"";
- escape_array_record(entry->key, buffer);
+ escape_array_record(entry->key(), buffer);
buffer += "\"=>\"";
- escape_array_record(entry->value, buffer);
+ escape_array_record(entry->value(), buffer);
buffer += "\"";
}
- buffer += '\t';
+
+ buffer += first ? "\\N\t" : "\t";
} else
buffer += "\\N\t";
// admin_level
buffer += (single_fmt % admin_level).str();
- // house number
- buffer += housenumber;
- buffer += '\t';
- // street
- copy_opt_string(street, buffer);
- // addr_place
- copy_opt_string(addr_place, buffer);
- // isin
- if (!address.empty()) {
- for (const auto entry: address) {
- if (entry->key == "tiger:county") {
- escape(std::string(entry->value, 0, entry->value.find(",")),
- buffer);
+ // address
+ if (address.empty()) {
+ buffer += "\\N\t";
+ } else {
+ for (auto const &a : address) {
+ buffer += "\"";
+ escape_array_record(a.first, buffer);
+ buffer += "\"=>\"";
+ if (a.first == "tiger:county") {
+ auto *end = strchr(a.second, ',');
+ if (end) {
+ size_t len = (size_t) (end - a.second);
+ escape_array_record(std::string(a.second, len), buffer);
+ } else {
+ escape_array_record(a.second, buffer);
+ }
buffer += " county";
} else {
- escape(entry->value, buffer);
+ escape_array_record(a.second, buffer);
}
- buffer += ',';
+ buffer += "\",";
}
buffer[buffer.length() - 1] = '\t';
- } else
- buffer += "\\N\t";
- // postcode
- copy_opt_string(postcode, buffer);
- // country code
- copy_opt_string(countrycode, buffer);
+ }
// extra tags
if (extratags.empty()) {
buffer += "\\N\t";
} else {
- bool first = true;
for (const auto entry: extratags) {
- if (first)
- first = false;
- else
- buffer += ',';
-
buffer += "\"";
- escape_array_record(entry->key, buffer);
+ escape_array_record(entry->key(), buffer);
buffer += "\"=>\"";
- escape_array_record(entry->value, buffer);
- buffer += "\"";
+ escape_array_record(entry->value(), buffer);
+ buffer += "\",";
}
- buffer += "\t";
+ buffer[buffer.length() - 1] = '\t';
}
- // geometry
- buffer += srid_str;
- buffer += geom;
+ // add the geometry - encoding it to hex along the way
+ ewkb::writer_t::write_as_hex(buffer, geom);
buffer += '\n';
}
}
@@ -509,17 +452,12 @@ void output_gazetteer_t::stop_copy(void)
}
/* Check the result */
- PGresult *res = PQgetResult(Connection);
- if (PQresultStatus(res) != PGRES_COMMAND_OK)
- {
+ pg_result_t res(PQgetResult(Connection));
+ if (PQresultStatus(res.get()) != PGRES_COMMAND_OK) {
std::cerr << "COPY_END for place failed: " << PQerrorMessage(Connection) << "\n";
- PQclear(res);
util::exit_nicely();
}
- /* Discard the result */
- PQclear(res);
-
/* We no longer have an active copy */
copy_active = false;
}
@@ -532,18 +470,17 @@ void output_gazetteer_t::delete_unused_classes(char osm_type, osmid_t osm_id) {
char const *paramValues[2];
paramValues[0] = tmp2;
paramValues[1] = (single_fmt % osm_id).str().c_str();
- PGresult *res = pgsql_execPrepared(ConnectionDelete, "get_classes", 2,
- paramValues, PGRES_TUPLES_OK);
+ auto res = pgsql_execPrepared(ConnectionDelete, "get_classes", 2,
+ paramValues, PGRES_TUPLES_OK);
- int sz = PQntuples(res);
+ int sz = PQntuples(res.get());
if (sz > 0 && !places.has_data()) {
- PQclear(res);
/* unconditional delete of all places */
delete_place(osm_type, osm_id);
} else {
std::string clslist;
for (int i = 0; i < sz; i++) {
- std::string cls(PQgetvalue(res, i, 0));
+ std::string cls(PQgetvalue(res.get(), i, 0));
if (!places.has_place(cls)) {
clslist.reserve(clslist.length() + cls.length() + 3);
if (!clslist.empty())
@@ -554,7 +491,6 @@ void output_gazetteer_t::delete_unused_classes(char osm_type, osmid_t osm_id) {
}
}
- PQclear(res);
if (!clslist.empty()) {
/* Stop any active copy */
@@ -606,43 +542,51 @@ int output_gazetteer_t::connect() {
int output_gazetteer_t::start()
{
- int srid = m_options.projection->target_srs();
- builder.set_exclude_broken_polygon(m_options.excludepoly);
-
- places.srid_str = (boost::format("SRID=%1%;") % srid).str();
-
- if(connect())
- util::exit_nicely();
-
- /* Start a transaction */
- pgsql_exec(Connection, PGRES_COMMAND_OK, "BEGIN");
-
- /* (Re)create the table unless we are appending */
- if (!m_options.append) {
- /* Drop any existing table */
- pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS place");
-
- /* Create the new table */
- if (m_options.tblsmain_data) {
- pgsql_exec(Connection, PGRES_COMMAND_OK,
- CREATE_PLACE_TABLE, "TABLESPACE", m_options.tblsmain_data->c_str());
- } else {
- pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_TABLE, "", "");
- }
- if (m_options.tblsmain_index) {
- pgsql_exec(Connection, PGRES_COMMAND_OK,
- CREATE_PLACE_ID_INDEX, "TABLESPACE", m_options.tblsmain_index->c_str());
- } else {
- pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_ID_INDEX, "", "");
- }
-
- pgsql_exec(Connection, PGRES_TUPLES_OK, "SELECT AddGeometryColumn('place', 'geometry', %d, 'GEOMETRY', 2)", srid);
- pgsql_exec(Connection, PGRES_COMMAND_OK, "ALTER TABLE place ALTER COLUMN geometry SET NOT NULL");
- }
-
- return 0;
-}
+ int srid = m_options.projection->target_srs();
+ if (connect()) {
+ util::exit_nicely();
+ }
+
+ /* Start a transaction */
+ pgsql_exec(Connection, PGRES_COMMAND_OK, "BEGIN");
+
+ /* (Re)create the table unless we are appending */
+ if (!m_options.append) {
+ /* Drop any existing table */
+ pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS place CASCADE");
+
+ /* Create the new table */
+
+ std::string sql =
+ "CREATE TABLE place ("
+ " osm_id " POSTGRES_OSMID_TYPE " NOT NULL,"
+ " osm_type char(1) NOT NULL,"
+ " class TEXT NOT NULL,"
+ " type TEXT NOT NULL,"
+ " name HSTORE,"
+ " admin_level SMALLINT,"
+ " address HSTORE,"
+ " extratags HSTORE," +
+ (boost::format(" geometry Geometry(Geometry,%1%) NOT NULL") % srid)
+ .str() +
+ ")";
+ if (m_options.tblsmain_data) {
+ sql += " TABLESPACE " + m_options.tblsmain_data.get();
+ }
+
+ pgsql_exec_simple(Connection, PGRES_COMMAND_OK, sql);
+
+ std::string index_sql =
+ "CREATE INDEX place_id_idx ON place USING BTREE (osm_type, osm_id)";
+ if (m_options.tblsmain_index) {
+ index_sql += " TABLESPACE " + m_options.tblsmain_index.get();
+ }
+ pgsql_exec_simple(Connection, PGRES_COMMAND_OK, index_sql);
+ }
+
+ return 0;
+}
void output_gazetteer_t::stop()
{
@@ -662,114 +606,106 @@ void output_gazetteer_t::stop()
return;
}
-int output_gazetteer_t::process_node(osmid_t id, double lat, double lon,
- const taglist_t &tags)
+int output_gazetteer_t::process_node(osmium::Node const &node)
{
- places.process_tags(tags);
+ places.process_tags(node);
if (m_options.append)
- delete_unused_classes('N', id);
+ delete_unused_classes('N', node.id());
/* Are we interested in this item? */
if (places.has_data()) {
- std::string wkt = (point_fmt % lon % lat).str();
- places.copy_out('N', id, wkt, buffer);
+ auto wkb = m_builder.get_wkb_node(node.location());
+ places.copy_out(node, wkb, buffer);
flush_place_buffer();
}
return 0;
}
-int output_gazetteer_t::process_way(osmid_t id, const idlist_t &nds, const taglist_t &tags)
+int output_gazetteer_t::process_way(osmium::Way *way)
{
- places.process_tags(tags);
+ places.process_tags(*way);
if (m_options.append)
- delete_unused_classes('W', id);
+ delete_unused_classes('W', way->id());
/* Are we interested in this item? */
if (places.has_data()) {
/* Fetch the node details */
- nodelist_t nodes;
- m_mid->nodes_get_list(nodes, nds);
+ m_mid->nodes_get_list(&(way->nodes()));
/* Get the geometry of the object */
- auto geom = builder.get_wkb_simple(nodes, 1);
- if (geom.valid()) {
- places.copy_out('W', id, geom.geom, buffer);
- flush_place_buffer();
+ geom::osmium_builder_t::wkb_t geom;
+ if (way->is_closed()) {
+ geom = m_builder.get_wkb_polygon(*way);
}
+ if (geom.empty()) {
+ auto wkbs = m_builder.get_wkb_line(way->nodes(), 0.0);
+ if (wkbs.empty()) {
+ return 0;
+ }
+
+ geom = wkbs[0];
+ }
+
+ places.copy_out(*way, geom, buffer);
+ flush_place_buffer();
}
return 0;
}
-int output_gazetteer_t::process_relation(osmid_t id, const memberlist_t &members,
- const taglist_t &tags)
+int output_gazetteer_t::process_relation(osmium::Relation const &rel)
{
- const std::string *type = tags.get("type");
+ auto const &tags = rel.tags();
+ char const *type = tags["type"];
if (!type) {
- delete_unused_full('R', id);
+ delete_unused_full('R', rel.id());
return 0;
}
- int cmp_waterway = type->compare("waterway");
+ bool is_waterway = strcmp(type, "waterway") == 0;
- if (*type == "associatedStreet"
- || !(*type == "boundary" || *type == "multipolygon" || !cmp_waterway)) {
- delete_unused_full('R', id);
+ if (strcmp(type, "associatedStreet") == 0
+ || !(strcmp(type, "boundary") == 0
+ || strcmp(type, "multipolygon") == 0 || is_waterway)) {
+ delete_unused_full('R', rel.id());
return 0;
}
- places.process_tags(tags);
+ places.process_tags(rel);
if (m_options.append)
- delete_unused_classes('R', id);
+ delete_unused_classes('R', rel.id());
/* Are we interested in this item? */
if (!places.has_data())
return 0;
/* get the boundary path (ways) */
- idlist_t xid2;
- for (const auto& member: members) {
- /* only interested in ways */
- if (member.type == OSMTYPE_WAY)
- xid2.push_back(member.id);
- }
+ osmium_buffer.clear();
+ auto num_ways = m_mid->rel_way_members_get(rel, nullptr, osmium_buffer);
- if (xid2.empty()) {
+ if (num_ways == 0) {
if (m_options.append)
- delete_unused_full('R', id);
+ delete_unused_full('R', rel.id());
return 0;
}
- multitaglist_t xtags;
- multinodelist_t xnodes;
- idlist_t xid;
- m_mid->ways_get_list(xid2, xid, xtags, xnodes);
-
- if (cmp_waterway) {
- auto geoms = builder.build_both(xnodes, 1, 1, 1000000, id);
- for (const auto& geom: geoms) {
- if (geom.is_polygon()) {
- places.copy_out('R', id, geom.geom, buffer);
- flush_place_buffer();
- } else {
- /* add_polygon_error('R', id, "boundary", "adminitrative", &names, countrycode, wkt); */
- }
- }
- } else {
- /* waterways result in multilinestrings */
- auto geom = builder.build_multilines(xnodes, id);
- if (geom.valid()) {
- places.copy_out('R', id, geom.geom, buffer);
- flush_place_buffer();
- }
+ for (auto &w : osmium_buffer.select<osmium::Way>()) {
+ m_mid->nodes_get_list(&(w.nodes()));
}
- return 0;
-}
+ auto geoms = is_waterway
+ ? m_builder.get_wkb_multiline(osmium_buffer, 0.0)
+ : m_builder.get_wkb_multipolygon(rel, osmium_buffer);
+ if (!geoms.empty()) {
+ places.copy_out(rel, geoms[0], buffer);
+ flush_place_buffer();
+ }
+ return 0;
+}
diff --git a/output-gazetteer.hpp b/output-gazetteer.hpp
index 511a629..0a2c26a 100644
--- a/output-gazetteer.hpp
+++ b/output-gazetteer.hpp
@@ -3,11 +3,14 @@
#include <memory>
#include <string>
+#include <unordered_map>
+#include <vector>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/format.hpp>
+#include <osmium/memory/buffer.hpp>
-#include "geometry-builder.hpp"
+#include "osmium-builder.hpp"
#include "osmtypes.hpp"
#include "output.hpp"
#include "pgsql.hpp"
@@ -29,7 +32,7 @@ public:
~place_tag_processor() {}
- void process_tags(const taglist_t &tags);
+ void process_tags(osmium::OSMObject const &o);
bool has_data() const { return !places.empty(); }
@@ -43,42 +46,42 @@ public:
return false;
}
- void copy_out(char osm_type, osmid_t osm_id, const std::string &geom,
+ void copy_out(osmium::OSMObject const &o, const std::string &geom,
std::string &buffer);
void clear();
private:
- void copy_opt_string(const std::string *val, std::string &buffer)
+ void copy_opt_string(char const *val, std::string &buffer)
{
if (val) {
- escape(*val, buffer);
+ escape(val, buffer);
buffer += "\t";
} else {
buffer += "\\N\t";
}
}
- std::string domain_name(const std::string &cls)
+ std::string domain_name(const std::string &cls, osmium::TagList const &tags)
{
std::string ret;
bool hasname = false;
std::string prefix(cls + ":name");
- for (const auto& item: *src) {
- if (boost::starts_with(item.key, prefix) &&
- (item.key.length() == prefix.length()
- || item.key[prefix.length()] == ':')) {
+ for (const auto& item: tags) {
+ char const *k = item.key();
+ if (boost::starts_with(k, prefix)
+ && (k[prefix.length()] == '\0' || k[prefix.length()] == ':')) {
if (!hasname) {
- ret.reserve(item.key.length() + item.value.length() + 10);
hasname = true;
- } else
+ } else {
ret += ",";
+ }
ret += "\"";
- escape_array_record(std::string(item.key, cls.length() + 1), ret);
+ escape_array_record(k + cls.length() + 1, ret);
ret += "\"=>\"";
- escape_array_record(item.value, ret);
+ escape_array_record(item.value(), ret);
ret += "\"";
}
}
@@ -111,135 +114,121 @@ private:
std::vector<tag_t> places;
- std::vector<const tag_t *> names;
- std::vector<const tag_t *> extratags;
- std::vector<const tag_t *> address;
- const taglist_t *src;
+ std::vector<osmium::Tag const *> names;
+ std::vector<osmium::Tag const *> extratags;
+ std::unordered_map<std::string, char const *> address;
int admin_level;
- const std::string *countrycode;
- std::string housenumber;
- const std::string *street;
- const std::string *addr_place;
- const std::string *postcode;
boost::format single_fmt;
-public:
- std::string srid_str;
};
class output_gazetteer_t : public output_t {
public:
- output_gazetteer_t(const middle_query_t* mid_, const options_t &options_)
- : output_t(mid_, options_),
- Connection(NULL),
- ConnectionDelete(NULL),
- ConnectionError(NULL),
- copy_active(false),
- single_fmt("%1%"),
- point_fmt("POINT(%.15g %.15g)")
+ output_gazetteer_t(const middle_query_t *mid_, const options_t &options_)
+ : output_t(mid_, options_), Connection(NULL), ConnectionDelete(NULL),
+ ConnectionError(NULL), copy_active(false),
+ m_builder(options_.projection, true), single_fmt("%1%"),
+ osmium_buffer(PLACE_BUFFER_SIZE, osmium::memory::Buffer::auto_grow::yes)
{
buffer.reserve(PLACE_BUFFER_SIZE);
}
- output_gazetteer_t(const output_gazetteer_t& other)
- : output_t(other.m_mid, other.m_options),
- Connection(NULL),
- ConnectionDelete(NULL),
- ConnectionError(NULL),
- copy_active(false),
- reproj(other.reproj),
- single_fmt(other.single_fmt),
- point_fmt(other.point_fmt)
+ output_gazetteer_t(const output_gazetteer_t &other)
+ : output_t(other.m_mid, other.m_options), Connection(NULL),
+ ConnectionDelete(NULL), ConnectionError(NULL), copy_active(false),
+ m_builder(other.m_options.projection, true), single_fmt(other.single_fmt),
+ osmium_buffer(PLACE_BUFFER_SIZE, osmium::memory::Buffer::auto_grow::yes)
{
buffer.reserve(PLACE_BUFFER_SIZE);
- builder.set_exclude_broken_polygon(m_options.excludepoly);
connect();
}
virtual ~output_gazetteer_t() {}
- virtual std::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const
+ std::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const override
{
output_gazetteer_t *clone = new output_gazetteer_t(*this);
clone->m_mid = cloned_middle;
return std::shared_ptr<output_t>(clone);
}
- int start();
- void stop();
- void commit() {}
+ int start() override;
+ void stop() override;
+ void commit() override {}
- void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {}
- int pending_way(osmid_t id, int exists) { return 0; }
+ void enqueue_ways(pending_queue_t &, osmid_t, size_t, size_t&) override {}
+ int pending_way(osmid_t, int) override { return 0; }
- void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {}
- int pending_relation(osmid_t id, int exists) { return 0; }
+ void enqueue_relations(pending_queue_t &, osmid_t, size_t, size_t&) override {}
+ int pending_relation(osmid_t, int) override { return 0; }
- int node_add(osmid_t id, double lat, double lon, const taglist_t &tags)
+ int node_add(osmium::Node const &node) override
{
- return process_node(id, lat, lon, tags);
+ return process_node(node);
}
- int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags)
+ int way_add(osmium::Way *way) override
{
- return process_way(id, nodes, tags);
+ return process_way(way);
}
- int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags)
+ int relation_add(osmium::Relation const &rel) override
{
- return process_relation(id, members, tags);
+ return process_relation(rel);
}
- int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags)
+ int node_modify(osmium::Node const &node) override
{
- return process_node(id, lat, lon, tags);
+ return process_node(node);
}
- int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags)
+ int way_modify(osmium::Way *way) override
{
- return process_way(id, nodes, tags);
+ return process_way(way);
}
- int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags)
+ int relation_modify(osmium::Relation const &rel) override
{
- return process_relation(id, members, tags);
+ return process_relation(rel);
}
- int node_delete(osmid_t id)
+ int node_delete(osmid_t id) override
{
delete_place('N', id);
return 0;
}
- int way_delete(osmid_t id)
+ int way_delete(osmid_t id) override
{
delete_place('W', id);
return 0;
}
- int relation_delete(osmid_t id)
+ int relation_delete(osmid_t id) override
{
delete_place('R', id);
return 0;
}
private:
- enum { PLACE_BUFFER_SIZE = 4092 };
+ enum { PLACE_BUFFER_SIZE = 4096 };
void stop_copy(void);
void delete_unused_classes(char osm_type, osmid_t osm_id);
void delete_place(char osm_type, osmid_t osm_id);
- int process_node(osmid_t id, double lat, double lon, const taglist_t &tags);
- int process_way(osmid_t id, const idlist_t &nodes, const taglist_t &tags);
- int process_relation(osmid_t id, const memberlist_t &members, const taglist_t &tags);
+ int process_node(osmium::Node const &node);
+ int process_way(osmium::Way *way);
+ int process_relation(osmium::Relation const &rel);
int connect();
void flush_place_buffer()
{
if (!copy_active)
{
- pgsql_exec(Connection, PGRES_COPY_IN, "COPY place (osm_type, osm_id, class, type, name, admin_level, housenumber, street, addr_place, isin, postcode, country_code, extratags, geometry) FROM STDIN");
+ pgsql_exec(Connection, PGRES_COPY_IN,
+ "COPY place (osm_type, osm_id, class, type, name, "
+ "admin_level, address, extratags, geometry) FROM STDIN");
copy_active = true;
}
@@ -264,16 +253,12 @@ private:
std::string buffer;
place_tag_processor places;
- geometry_builder builder;
-
- std::shared_ptr<reprojection> reproj;
+ geom::osmium_builder_t m_builder;
// string formatters
// Need to be part of the class, so we have one per thread.
boost::format single_fmt;
- boost::format point_fmt;
+ osmium::memory::Buffer osmium_buffer;
};
-extern output_gazetteer_t out_gazetteer;
-
#endif
diff --git a/output-multi.cpp b/output-multi.cpp
index e8de3bc..b45b47d 100644
--- a/output-multi.cpp
+++ b/output-multi.cpp
@@ -1,12 +1,12 @@
#include "output-multi.hpp"
-#include "taginfo_impl.hpp"
+#include "expire-tiles.hpp"
+#include "id-tracker.hpp"
+#include "middle.hpp"
+#include "options.hpp"
#include "table.hpp"
+#include "taginfo_impl.hpp"
#include "tagtransform.hpp"
-#include "options.hpp"
-#include "middle.hpp"
-#include "id-tracker.hpp"
-#include "geometry-builder.hpp"
-#include "expire-tiles.hpp"
+#include "wkb.hpp"
#include <boost/algorithm/string/predicate.hpp>
#include <vector>
@@ -14,34 +14,49 @@
output_multi_t::output_multi_t(const std::string &name,
std::shared_ptr<geometry_processor> processor_,
const struct export_list &export_list_,
- const middle_query_t* mid_, const options_t &options_)
- : output_t(mid_, options_),
- m_tagtransform(new tagtransform(&m_options)),
- m_export_list(new export_list(export_list_)),
- m_processor(processor_),
- //TODO: we could in fact have something that is interested in nodes and ways..
- m_osm_type(m_processor->interests(geometry_processor::interest_node) ? OSMTYPE_NODE : OSMTYPE_WAY),
- m_table(new table_t(m_options.database_options.conninfo(), name, m_processor->column_type(),
- m_export_list->normal_columns(m_osm_type),
- m_options.hstore_columns, m_processor->srid(),
- m_options.append, m_options.slim, m_options.droptemp,
- m_options.hstore_mode, m_options.enable_hstore_index,
- m_options.tblsmain_data, m_options.tblsmain_index)),
- ways_done_tracker(new id_tracker()),
- m_expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox,
- m_options.projection)
-{}
-
-output_multi_t::output_multi_t(const output_multi_t& other):
- output_t(other.m_mid, other.m_options), m_tagtransform(new tagtransform(&m_options)), m_export_list(new export_list(*other.m_export_list)),
- m_processor(other.m_processor), m_osm_type(other.m_osm_type), m_table(new table_t(*other.m_table)),
- //NOTE: we need to know which ways were used by relations so each thread
- //must have a copy of the original marked done ways, its read only so its ok
- ways_done_tracker(other.ways_done_tracker),
- m_expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox,
- m_options.projection)
-{}
+ const middle_query_t *mid_,
+ const options_t &options_)
+: output_t(mid_, options_),
+ m_tagtransform(tagtransform_t::make_tagtransform(&m_options)),
+ m_export_list(new export_list(export_list_)), m_processor(processor_),
+ m_proj(m_options.projection),
+ // TODO: we could in fact have something that is interested in nodes and
+ // ways..
+ m_osm_type(m_processor->interests(geometry_processor::interest_node)
+ ? osmium::item_type::node
+ : osmium::item_type::way),
+ m_table(new table_t(
+ m_options.database_options.conninfo(), name, m_processor->column_type(),
+ m_export_list->normal_columns(m_osm_type), m_options.hstore_columns,
+ m_processor->srid(), m_options.append, m_options.slim, m_options.droptemp,
+ m_options.hstore_mode, m_options.enable_hstore_index,
+ m_options.tblsmain_data, m_options.tblsmain_index)),
+ ways_done_tracker(new id_tracker()),
+ m_expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox,
+ m_options.projection),
+ buffer(1024, osmium::memory::Buffer::auto_grow::yes),
+ m_builder(m_options.projection, m_options.enable_multi),
+ m_way_area(m_export_list->has_column(m_osm_type, "way_area"))
+{
+}
+output_multi_t::output_multi_t(const output_multi_t &other)
+: output_t(other.m_mid, other.m_options),
+ m_tagtransform(tagtransform_t::make_tagtransform(&m_options)),
+ m_export_list(new export_list(*other.m_export_list)),
+ m_processor(other.m_processor), m_proj(other.m_proj),
+ m_osm_type(other.m_osm_type), m_table(new table_t(*other.m_table)),
+ // NOTE: we need to know which ways were used by relations so each thread
+ // must have a copy of the original marked done ways, its read only so its
+ // ok
+ ways_done_tracker(other.ways_done_tracker),
+ m_expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox,
+ m_options.projection),
+ buffer(1024, osmium::memory::Buffer::auto_grow::yes),
+ m_builder(m_options.projection, m_options.enable_multi),
+ m_way_area(other.m_way_area)
+{
+}
output_multi_t::~output_multi_t() = default;
@@ -101,14 +116,13 @@ void output_multi_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t
}
int output_multi_t::pending_way(osmid_t id, int exists) {
- taglist_t tags_int;
- nodelist_t nodes_int;
int ret = 0;
// Try to fetch the way from the DB
- if (m_mid->ways_get(id, tags_int, nodes_int)) {
+ buffer.clear();
+ if (m_mid->ways_get(id, buffer)) {
// Output the way
- ret = reprocess_way(id, nodes_int, tags_int, exists);
+ ret = reprocess_way(&buffer.get<osmium::Way>(0), exists);
}
return ret;
@@ -152,13 +166,13 @@ void output_multi_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, s
}
int output_multi_t::pending_relation(osmid_t id, int exists) {
- taglist_t tags_int;
- memberlist_t members_int;
int ret = 0;
// Try to fetch the relation from the DB
- if (m_mid->relations_get(id, members_int, tags_int)) {
- ret = process_relation(id, members_int, tags_int, exists);
+ buffer.clear();
+ if (m_mid->relations_get(id, buffer)) {
+ auto const &rel = buffer.get<osmium::Relation>(0);
+ ret = process_relation(rel, exists, true);
}
return ret;
@@ -167,7 +181,7 @@ int output_multi_t::pending_relation(osmid_t id, int exists) {
void output_multi_t::stop()
{
m_table->stop();
- if (m_options.expire_tiles_zoom_min >= 0) {
+ if (m_options.expire_tiles_zoom_min > 0) {
m_expire.output_and_destroy(m_options.expire_tiles_filename.c_str(),
m_options.expire_tiles_zoom_min);
}
@@ -177,68 +191,69 @@ void output_multi_t::commit() {
m_table->commit();
}
-int output_multi_t::node_add(osmid_t id, double lat, double lon, const taglist_t &tags) {
+int output_multi_t::node_add(osmium::Node const &node)
+{
if (m_processor->interests(geometry_processor::interest_node)) {
- return process_node(id, lat, lon, tags);
+ return process_node(node);
}
return 0;
}
-int output_multi_t::way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags) {
- if (m_processor->interests(geometry_processor::interest_way) && nodes.size() > 1) {
- return process_way(id, nodes, tags);
+int output_multi_t::way_add(osmium::Way *way) {
+ if (m_processor->interests(geometry_processor::interest_way) && way->nodes().size() > 1) {
+ return process_way(way);
}
return 0;
}
-int output_multi_t::relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) {
- if (m_processor->interests(geometry_processor::interest_relation) && !members.empty()) {
- return process_relation(id, members, tags, 0);
+int output_multi_t::relation_add(osmium::Relation const &rel) {
+ if (m_processor->interests(geometry_processor::interest_relation)
+ && !rel.members().empty()) {
+ return process_relation(rel, 0);
}
return 0;
}
-int output_multi_t::node_modify(osmid_t id, double lat, double lon, const taglist_t &tags) {
+int output_multi_t::node_modify(osmium::Node const &node)
+{
if (m_processor->interests(geometry_processor::interest_node)) {
// TODO - need to know it's a node?
- delete_from_output(id);
+ delete_from_output(node.id());
// TODO: need to mark any ways or relations using it - depends on what
// type of output this is... delegate to the geometry processor??
- return process_node(id, lat, lon, tags);
-
- } else {
- return 0;
+ return process_node(node);
}
+
+ return 0;
}
-int output_multi_t::way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags) {
+int output_multi_t::way_modify(osmium::Way *way) {
if (m_processor->interests(geometry_processor::interest_way)) {
// TODO - need to know it's a way?
- delete_from_output(id);
+ delete_from_output(way->id());
// TODO: need to mark any relations using it - depends on what
// type of output this is... delegate to the geometry processor??
- return process_way(id, nodes, tags);
-
- } else {
- return 0;
+ return process_way(way);
}
+
+ return 0;
}
-int output_multi_t::relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags) {
+int output_multi_t::relation_modify(osmium::Relation const &rel) {
if (m_processor->interests(geometry_processor::interest_relation)) {
// TODO - need to know it's a relation?
- delete_from_output(-id);
+ delete_from_output(-rel.id());
// TODO: need to mark any other relations using it - depends on what
// type of output this is... delegate to the geometry processor??
- return process_relation(id, members, tags, false);
+ return process_relation(rel, false);
- } else {
- return 0;
}
+
+ return 0;
}
int output_multi_t::node_delete(osmid_t id) {
@@ -265,151 +280,140 @@ int output_multi_t::relation_delete(osmid_t id) {
return 0;
}
-int output_multi_t::process_node(osmid_t id, double lat, double lon, const taglist_t &tags) {
- //check if we are keeping this node
+int output_multi_t::process_node(osmium::Node const &node)
+{
+ // check if we are keeping this node
taglist_t outtags;
- unsigned int filter = m_tagtransform->filter_node_tags(tags, *m_export_list.get(), outtags, true);
+ auto filter = m_tagtransform->filter_tags(node, 0, 0, *m_export_list.get(),
+ outtags, true);
if (!filter) {
- //grab its geom
- auto geom = m_processor->process_node(lat, lon);
- if (geom.valid()) {
- m_expire.from_bbox(lon, lat, lon, lat);
- copy_node_to_table(id, geom.geom, outtags);
+ // grab its geom
+ auto geom = m_processor->process_node(node.location(), &m_builder);
+ if (!geom.empty()) {
+ m_expire.from_wkb(geom.c_str(), node.id());
+ copy_node_to_table(node.id(), geom, outtags);
}
}
return 0;
}
-int output_multi_t::reprocess_way(osmid_t id, const nodelist_t &nodes, const taglist_t &tags, bool exists)
+int output_multi_t::reprocess_way(osmium::Way *way, bool exists)
{
//if the way could exist already we have to make the relation pending and reprocess it later
//but only if we actually care about relations
if(m_processor->interests(geometry_processor::interest_relation) && exists) {
- way_delete(id);
- const std::vector<osmid_t> rel_ids = m_mid->relations_using_way(id);
+ way_delete(way->id());
+ const std::vector<osmid_t> rel_ids =
+ m_mid->relations_using_way(way->id());
for (std::vector<osmid_t>::const_iterator itr = rel_ids.begin(); itr != rel_ids.end(); ++itr) {
rels_pending_tracker.mark(*itr);
}
}
//check if we are keeping this way
- int polygon = 0, roads = 0;
taglist_t outtags;
- unsigned int filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads,
- *m_export_list.get(), outtags, true);
+ unsigned int filter = m_tagtransform->filter_tags(
+ *way, 0, 0, *m_export_list.get(), outtags, true);
if (!filter) {
- //grab its geom
- auto geom = m_processor->process_way(nodes);
- if (geom.valid()) {
- copy_to_table(id, geom, outtags, polygon);
+ m_mid->nodes_get_list(&(way->nodes()));
+ auto geom = m_processor->process_way(*way, &m_builder);
+ if (!geom.empty()) {
+ copy_to_table(way->id(), geom, outtags);
}
}
return 0;
}
-int output_multi_t::process_way(osmid_t id, const idlist_t &nodes, const taglist_t &tags) {
+int output_multi_t::process_way(osmium::Way *way) {
//check if we are keeping this way
- int polygon = 0, roads = 0;
taglist_t outtags;
- unsigned filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads,
- *m_export_list.get(), outtags, true);
+ auto filter = m_tagtransform->filter_tags(*way, 0, 0, *m_export_list.get(), outtags, true);
if (!filter) {
//get the geom from the middle
- if(m_way_helper.set(nodes, m_mid) < 1)
+ if (m_mid->nodes_get_list(&(way->nodes())) < 1)
return 0;
//grab its geom
- auto geom = m_processor->process_way(m_way_helper.node_cache);
+ auto geom = m_processor->process_way(*way, &m_builder);
- if (geom.valid()) {
+ if (!geom.empty()) {
//if we are also interested in relations we need to mark
//this way pending just in case it shows up in one
if (m_processor->interests(geometry_processor::interest_relation)) {
- ways_pending_tracker.mark(id);
+ ways_pending_tracker.mark(way->id());
} else {
// We wouldn't be interested in this as a relation, so no need to mark it pending.
// TODO: Does this imply anything for non-multipolygon relations?
- copy_to_table(id, geom, outtags, polygon);
+ copy_to_table(way->id(), geom, outtags);
}
}
}
return 0;
}
-int output_multi_t::process_relation(osmid_t id, const memberlist_t &members,
- const taglist_t &tags, bool exists, bool pending) {
+
+int output_multi_t::process_relation(osmium::Relation const &rel,
+ bool exists, bool pending)
+{
//if it may exist already, delete it first
if(exists)
- relation_delete(id);
+ relation_delete(rel.id());
//does this relation have anything interesting to us
taglist_t rel_outtags;
- unsigned filter = m_tagtransform->filter_rel_tags(tags, *m_export_list.get(),
- rel_outtags, true);
+ auto filter = m_tagtransform->filter_tags(rel, 0, 0, *m_export_list.get(),
+ rel_outtags, true);
if (!filter) {
//TODO: move this into geometry processor, figure a way to come back for tag transform
//grab ways/nodes of the members in the relation, bail if none were used
- if(m_relation_helper.set(&members, (middle_t*)m_mid) < 1)
+ if (m_relation_helper.set(rel, (middle_t *)m_mid) < 1)
return 0;
- //filter the tags on each member because we got them from the middle
- //and since the middle is no longer tied to the output it no longer
- //shares any kind of tag transform and therefore has all original tags
- //so we filter here because each individual outputs cares about different tags
- int polygon, roads;
- multitaglist_t filtered(m_relation_helper.tags.size(), taglist_t());
- for(size_t i = 0; i < m_relation_helper.tags.size(); ++i)
- {
- m_tagtransform->filter_way_tags(m_relation_helper.tags[i], &polygon,
- &roads, *m_export_list.get(), filtered[i]);
- //TODO: if the filter says that this member is now not interesting we
- //should decrement the count and remove his nodes and tags etc. for
- //now we'll just keep him with no tags so he will get filtered later
- }
-
- //do the members of this relation have anything interesting to us
//NOTE: make_polygon is preset here this is to force the tag matching/superseded stuff
//normally this wouldnt work but we tell the tag transform to allow typeless relations
//this is needed because the type can get stripped off by the rel_tag filter above
//if the export list did not include the type tag.
//TODO: find a less hacky way to do the matching/superseded and tag copying stuff without
//all this trickery
- int make_boundary, make_polygon = 1;
+ int roads;
+ int make_boundary, make_polygon;
taglist_t outtags;
- filter = m_tagtransform->filter_rel_member_tags(rel_outtags, filtered, m_relation_helper.roles,
- &m_relation_helper.superseeded.front(),
- &make_boundary, &make_polygon, &roads,
- *m_export_list.get(), outtags, true);
+ filter = m_tagtransform->filter_rel_member_tags(
+ rel_outtags, m_relation_helper.data, m_relation_helper.roles,
+ &m_relation_helper.superseded.front(), &make_boundary,
+ &make_polygon, &roads, *m_export_list.get(), outtags, true);
if (!filter)
{
- auto geoms = m_processor->process_relation(m_relation_helper.nodes);
- for (const auto geom: geoms) {
- //TODO: we actually have the nodes in the m_relation_helper and could use them
- //instead of having to reparse the wkb in the expiry code
- m_expire.from_wkb(geom.geom.c_str(), -id);
- //what part of the code relies on relation members getting negative ids?
- copy_to_table(-id, geom, outtags, make_polygon);
+ m_relation_helper.add_way_locations((middle_t *)m_mid);
+ auto geoms = m_processor->process_relation(
+ rel, m_relation_helper.data, &m_builder);
+ for (const auto geom : geoms) {
+ copy_to_table(-rel.id(), geom, outtags);
}
//TODO: should this loop be inside the if above just in case?
//take a look at each member to see if its superseded (tags on it matched the tags on the relation)
- for(size_t i = 0; i < m_relation_helper.ways.size(); ++i) {
+ size_t i = 0;
+ for (auto const &w : m_relation_helper.data.select<osmium::Way>()) {
//tags matched so we are keeping this one with this relation
- if (m_relation_helper.superseeded[i]) {
+ if (m_relation_helper.superseded[i]) {
//just in case it wasnt previously with this relation we get rid of them
- way_delete(m_relation_helper.ways[i]);
+ way_delete(w.id());
//the other option is that we marked them pending in the way processing so here we mark them
//done so when we go back over the pendings we can just skip it because its in the done list
//TODO: dont do this when working with pending relations to avoid thread races
if(!pending)
- ways_done_tracker->mark(m_relation_helper.ways[i]);
+ ways_done_tracker->mark(w.id());
}
+ ++i;
}
}
}
return 0;
}
-void output_multi_t::copy_node_to_table(osmid_t id, const std::string &geom, taglist_t &tags) {
+void output_multi_t::copy_node_to_table(osmid_t id, std::string const &geom,
+ taglist_t &tags)
+{
m_table->write_row(id, tags, geom);
}
@@ -422,25 +426,23 @@ void output_multi_t::copy_node_to_table(osmid_t id, const std::string &geom, tag
*
* \pre geom must be valid.
*/
-void output_multi_t::copy_to_table(const osmid_t id, const geometry_builder::pg_geom_t &geom, taglist_t &tags, int polygon) {
- if (geom.is_polygon()) {
+void output_multi_t::copy_to_table(const osmid_t id,
+ geometry_processor::wkb_t const &geom,
+ taglist_t &tags)
+{
+ // XXX really should depend on expected output type
+ if (m_way_area) {
// It's a polygon table (implied by it turning into a poly),
// and it got formed into a polygon, so expire as a polygon and write the geom
- m_expire.from_nodes_poly(m_way_helper.node_cache, id);
- if (geom.area > 0.0) {
- char tmp[32];
- snprintf(tmp, sizeof(tmp), "%g", geom.area);
- tags.push_override(tag_t("way_area", tmp));
- }
- m_table->write_row(id, tags, geom.geom);
- } else {
- // Linestring
- if (!polygon) {
- // non-polygons are okay
- m_expire.from_nodes_line(m_way_helper.node_cache);
- m_table->write_row(id, tags, geom.geom);
- }
+ auto area =
+ ewkb::parser_t(geom).get_area<osmium::geom::IdentityProjection>();
+ char tmp[32];
+ snprintf(tmp, sizeof(tmp), "%g", area);
+ tags.push_override(tag_t("way_area", tmp));
}
+
+ m_expire.from_wkb(geom.c_str(), id);
+ m_table->write_row(id, tags, geom);
}
void output_multi_t::delete_from_output(osmid_t id) {
diff --git a/output-multi.hpp b/output-multi.hpp
index 0054695..2091ee0 100644
--- a/output-multi.hpp
+++ b/output-multi.hpp
@@ -18,7 +18,7 @@
#include <memory>
class table_t;
-class tagtransform;
+class tagtransform_t;
struct export_list;
struct middle_query_t;
struct options_t;
@@ -32,55 +32,59 @@ public:
output_multi_t(const output_multi_t& other);
virtual ~output_multi_t();
- virtual std::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const;
+ std::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const override;
- int start();
- void stop();
- void commit();
+ int start() override;
+ void stop() override;
+ void commit() override;
- void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
- int pending_way(osmid_t id, int exists);
+ void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) override;
+ int pending_way(osmid_t id, int exists) override;
- void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
- int pending_relation(osmid_t id, int exists);
+ void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) override;
+ int pending_relation(osmid_t id, int exists) override;
- int node_add(osmid_t id, double lat, double lon, const taglist_t &tags);
- int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags);
- int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags);
+ int node_add(osmium::Node const &node) override;
+ int way_add(osmium::Way *way) override;
+ int relation_add(osmium::Relation const &rel) override;
- int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags);
- int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags);
- int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags);
+ int node_modify(osmium::Node const &node) override;
+ int way_modify(osmium::Way *way) override;
+ int relation_modify(osmium::Relation const &rel) override;
- int node_delete(osmid_t id);
- int way_delete(osmid_t id);
- int relation_delete(osmid_t id);
+ int node_delete(osmid_t id) override;
+ int way_delete(osmid_t id) override;
+ int relation_delete(osmid_t id) override;
- size_t pending_count() const;
+ size_t pending_count() const override;
- void merge_pending_relations(output_t *other);
- void merge_expire_trees(output_t *other);
+ void merge_pending_relations(output_t *other) override;
+ void merge_expire_trees(output_t *other) override;
protected:
void delete_from_output(osmid_t id);
- int process_node(osmid_t id, double lat, double lon, const taglist_t &tags);
- int process_way(osmid_t id, const idlist_t &nodes, const taglist_t &tags);
- int reprocess_way(osmid_t id, const nodelist_t &nodes, const taglist_t &tags, bool exists);
- int process_relation(osmid_t id, const memberlist_t &members, const taglist_t &tags, bool exists, bool pending=false);
+ int process_node(osmium::Node const &node);
+ int process_way(osmium::Way *way);
+ int reprocess_way(osmium::Way *way, bool exists);
+ int process_relation(osmium::Relation const &rel, bool exists, bool pending=false);
void copy_node_to_table(osmid_t id, const std::string &geom, taglist_t &tags);
- void copy_to_table(const osmid_t id, const geometry_builder::pg_geom_t &geom, taglist_t &tags, int polygon);
+ void copy_to_table(const osmid_t id, geometry_processor::wkb_t const &geom,
+ taglist_t &tags);
- std::unique_ptr<tagtransform> m_tagtransform;
+ std::unique_ptr<tagtransform_t> m_tagtransform;
std::unique_ptr<export_list> m_export_list;
std::shared_ptr<geometry_processor> m_processor;
- const OsmType m_osm_type;
+ std::shared_ptr<reprojection> m_proj;
+ osmium::item_type const m_osm_type;
std::unique_ptr<table_t> m_table;
id_tracker ways_pending_tracker, rels_pending_tracker;
std::shared_ptr<id_tracker> ways_done_tracker;
expire_tiles m_expire;
- way_helper m_way_helper;
relation_helper m_relation_helper;
+ osmium::memory::Buffer buffer;
+ geom::osmium_builder_t m_builder;
+ bool m_way_area;
};
#endif
diff --git a/output-null.cpp b/output-null.cpp
index f2383a5..81d5715 100644
--- a/output-null.cpp
+++ b/output-null.cpp
@@ -17,53 +17,49 @@ void output_null_t::stop() {
void output_null_t::commit() {
}
-void output_null_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+void output_null_t::enqueue_ways(pending_queue_t &, osmid_t, size_t, size_t&) {
}
-int output_null_t::pending_way(osmid_t id, int exists) {
+int output_null_t::pending_way(osmid_t, int) {
return 0;
}
-void output_null_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+void output_null_t::enqueue_relations(pending_queue_t &, osmid_t, size_t, size_t&) {
}
-int output_null_t::pending_relation(osmid_t id, int exists) {
+int output_null_t::pending_relation(osmid_t, int) {
return 0;
}
-int output_null_t::node_add(osmid_t, double, double, const taglist_t &) {
- return 0;
-}
+int output_null_t::node_add(osmium::Node const &) { return 0; }
-int output_null_t::way_add(osmid_t a, const idlist_t &, const taglist_t &) {
+int output_null_t::way_add(osmium::Way *) {
return 0;
}
-int output_null_t::relation_add(osmid_t a, const memberlist_t &, const taglist_t &) {
+int output_null_t::relation_add(osmium::Relation const &) {
return 0;
}
-int output_null_t::node_delete(osmid_t i) {
+int output_null_t::node_delete(osmid_t) {
return 0;
}
-int output_null_t::way_delete(osmid_t i) {
+int output_null_t::way_delete(osmid_t) {
return 0;
}
-int output_null_t::relation_delete(osmid_t i) {
+int output_null_t::relation_delete(osmid_t) {
return 0;
}
-int output_null_t::node_modify(osmid_t, double, double, const taglist_t &) {
- return 0;
-}
+int output_null_t::node_modify(osmium::Node const &) { return 0; }
-int output_null_t::way_modify(osmid_t, const idlist_t &, const taglist_t &) {
+int output_null_t::way_modify(osmium::Way *) {
return 0;
}
-int output_null_t::relation_modify(osmid_t, const memberlist_t &, const taglist_t &) {
+int output_null_t::relation_modify(osmium::Relation const &) {
return 0;
}
diff --git a/output-null.hpp b/output-null.hpp
index 079fbb8..14f5eb6 100644
--- a/output-null.hpp
+++ b/output-null.hpp
@@ -12,30 +12,30 @@ public:
output_null_t(const output_null_t& other);
virtual ~output_null_t();
- virtual std::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const;
+ std::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const override;
- int start();
- void stop();
- void commit();
+ int start() override;
+ void stop() override;
+ void commit() override;
void cleanup(void);
- void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
- int pending_way(osmid_t id, int exists);
+ void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) override;
+ int pending_way(osmid_t id, int exists) override;
- void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
- int pending_relation(osmid_t id, int exists);
+ void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) override;
+ int pending_relation(osmid_t id, int exists) override;
- int node_add(osmid_t id, double lat, double lon, const taglist_t &tags);
- int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags);
- int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags);
+ int node_add(osmium::Node const &node) override;
+ int way_add(osmium::Way *way) override;
+ int relation_add(osmium::Relation const &rel) override;
- int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags);
- int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags);
- int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags);
+ int node_modify(osmium::Node const &node) override;
+ int way_modify(osmium::Way *way) override;
+ int relation_modify(osmium::Relation const &rel) override;
- int node_delete(osmid_t id);
- int way_delete(osmid_t id);
- int relation_delete(osmid_t id);
+ int node_delete(osmid_t id) override;
+ int way_delete(osmid_t id) override;
+ int relation_delete(osmid_t id) override;
};
#endif
diff --git a/output-pgsql.cpp b/output-pgsql.cpp
index be1ac08..c2059d0 100644
--- a/output-pgsql.cpp
+++ b/output-pgsql.cpp
@@ -38,6 +38,7 @@
#include "tagtransform.hpp"
#include "util.hpp"
#include "wildcmp.hpp"
+#include "wkb.hpp"
/* make the diagnostic information work with older versions of
* boost - the function signature changed at version 1.54.
@@ -61,18 +62,6 @@ psql - 01 01000020 E6100000 30CCA462B6C3D4BF92998C9B38E04940
Workaround - output SRID=4326;<WKB>
*/
-int output_pgsql_t::pgsql_out_node(osmid_t id, const taglist_t &tags, double node_lat, double node_lon)
-{
- taglist_t outtags;
- if (m_tagtransform->filter_node_tags(tags, *m_export_list.get(), outtags))
- return 1;
-
- expire.from_bbox(node_lon, node_lat, node_lon, node_lat);
- m_tables[t_point]->write_node(id, outtags, node_lat, node_lon);
-
- return 0;
-}
-
/*
COPY planet_osm (osm_id, name, place, landuse, leisure, "natural", man_made, waterway, highway, railway, amenity, tourism, learning, bu
@@ -82,127 +71,39 @@ E4C1421D5BF24D06053E7DF4940
212696 Oswald Road \N \N \N \N \N \N minor \N \N \N \N \N \N \N 0102000020E610000004000000467D923B6C22D5BFA359D93EE4DF4940B3976DA7AD11D5BF84BBB376DBDF4940997FF44D9A06D5BF4223D8B8FEDF49404D158C4AEA04D
5BF5BB39597FCDF4940
*/
-int output_pgsql_t::pgsql_out_way(osmid_t id, taglist_t &outtags,
- const nodelist_t &nodes,
- int polygon, int roads)
-{
- /* Split long ways after around 1 degree or 100km */
- double split_at;
- if (m_options.projection->target_latlon())
- split_at = 1;
- else
- split_at = 100 * 1000;
-
- char tmp[32];
- auto wkbs = builder.get_wkb_split(nodes, polygon, split_at);
- for (const auto& wkb: wkbs) {
- /* FIXME: there should be a better way to detect polygons */
- if (wkb.is_polygon()) {
- expire.from_nodes_poly(nodes, id);
- if ((wkb.area > 0.0) && m_enable_way_area) {
- snprintf(tmp, sizeof(tmp), "%g", wkb.area);
- outtags.push_override(tag_t("way_area", tmp));
- }
- m_tables[t_poly]->write_row(id, outtags, wkb.geom);
- } else {
- expire.from_nodes_line(nodes);
- m_tables[t_line]->write_row(id, outtags, wkb.geom);
- if (roads)
- m_tables[t_roads]->write_row(id, outtags, wkb.geom);
- }
- }
-
- return 0;
-}
-
-int output_pgsql_t::pgsql_out_relation(osmid_t id, const taglist_t &rel_tags,
- const multinodelist_t &xnodes, const multitaglist_t & xtags,
- const idlist_t &xid, const rolelist_t &xrole,
- bool pending)
+void output_pgsql_t::pgsql_out_way(osmium::Way const &way, taglist_t *tags,
+ bool polygon, bool roads)
{
- if (xnodes.empty())
- return 0;
-
- int roads = 0;
- int make_polygon = 0;
- int make_boundary = 0;
- double split_at;
-
- std::vector<int> members_superseeded(xnodes.size(), 0);
- taglist_t outtags;
-
- //if its a route relation make_boundary and make_polygon will be false otherwise one or the other will be true
- if (m_tagtransform->filter_rel_member_tags(rel_tags, xtags, xrole,
- &(members_superseeded[0]), &make_boundary, &make_polygon, &roads,
- *m_export_list.get(), outtags)) {
- return 0;
- }
-
- /* Split long linear ways after around 1 degree or 100km (polygons not effected) */
- if (m_options.projection->target_latlon())
- split_at = 1;
- else
- split_at = 100 * 1000;
-
- //this will either make lines or polygons (unless the lines arent a ring or are less than 3 pts) depending on the tag transform above
- //TODO: pick one or the other based on which we expect to care about
- auto wkbs = builder.build_both(xnodes, make_polygon, m_options.enable_multi, split_at, id);
-
- if (wkbs.empty()) {
- return 0;
- }
-
- char tmp[32];
- for (const auto& wkb: wkbs) {
- expire.from_wkb(wkb.geom.c_str(), -id);
- /* FIXME: there should be a better way to detect polygons */
- if (wkb.is_polygon()) {
- if ((wkb.area > 0.0) && m_enable_way_area) {
- snprintf(tmp, sizeof(tmp), "%g", wkb.area);
- outtags.push_override(tag_t("way_area", tmp));
+ if (polygon && way.is_closed()) {
+ auto wkb = m_builder.get_wkb_polygon(way);
+ if (!wkb.empty()) {
+ expire.from_wkb(wkb.c_str(), way.id());
+ if (m_enable_way_area) {
+ char tmp[32];
+ auto const area =
+ m_options.reproject_area
+ ? ewkb::parser_t(wkb).get_area<reprojection>(
+ m_options.projection.get())
+ : ewkb::parser_t(wkb)
+ .get_area<osmium::geom::IdentityProjection>();
+ snprintf(tmp, sizeof(tmp), "%g", area);
+ tags->push_override(tag_t("way_area", tmp));
}
- m_tables[t_poly]->write_row(-id, outtags, wkb.geom);
- } else {
- m_tables[t_line]->write_row(-id, outtags, wkb.geom);
- if (roads)
- m_tables[t_roads]->write_row(-id, outtags, wkb.geom);
+ m_tables[t_poly]->write_row(way.id(), *tags, wkb);
}
- }
-
- /* Tagtransform will have marked those member ways of the relation that
- * have fully been dealt with as part of the multi-polygon entry.
- * Set them in the database as done and delete their entry to not
- * have duplicates */
- //dont do this when working with pending relations as its not needed
- if (make_polygon) {
- for (size_t i=0; i < xid.size(); i++) {
- if (members_superseeded[i]) {
- pgsql_delete_way_from_output(xid[i]);
- if(!pending)
- ways_done_tracker->mark(xid[i]);
+ } else {
+ double const split_at = m_options.projection->target_latlon() ? 1 : 100 * 1000;
+ for (auto const &wkb : m_builder.get_wkb_line(way.nodes(), split_at)) {
+ expire.from_wkb(wkb.c_str(), way.id());
+ m_tables[t_line]->write_row(way.id(), *tags, wkb);
+ if (roads) {
+ m_tables[t_roads]->write_row(way.id(), *tags, wkb);
}
}
- }
- // If the tag transform said the polygon looked like a boundary we want to make that as well
- // If we are making a boundary then also try adding any relations which form complete rings
- // The linear variants will have already been processed above
- if (make_boundary) {
- wkbs = builder.build_polygons(xnodes, m_options.enable_multi, id);
- for (const auto& wkb: wkbs) {
- expire.from_wkb(wkb.geom.c_str(), -id);
- if ((wkb.area > 0.0) && m_enable_way_area) {
- snprintf(tmp, sizeof(tmp), "%g", wkb.area);
- outtags.push_override(tag_t("way_area", tmp));
- }
- m_tables[t_poly]->write_row(-id, outtags, wkb.geom);
- }
}
-
- return 0;
}
-
void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
osmid_t const prev = ways_pending_tracker.last_returned();
if (id_tracker::is_valid(prev) && prev >= id) {
@@ -243,11 +144,9 @@ void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t
}
int output_pgsql_t::pending_way(osmid_t id, int exists) {
- taglist_t tags_int;
- nodelist_t nodes_int;
-
// Try to fetch the way from the DB
- if (m_mid->ways_get(id, tags_int, nodes_int)) {
+ buffer.clear();
+ if (m_mid->ways_get(id, buffer)) {
/* If the flag says this object may exist already, delete it first */
if (exists) {
pgsql_delete_way_from_output(id);
@@ -263,9 +162,14 @@ int output_pgsql_t::pending_way(osmid_t id, int exists) {
taglist_t outtags;
int polygon;
int roads;
- if (!m_tagtransform->filter_way_tags(tags_int, &polygon, &roads,
- *m_export_list.get(), outtags)) {
- return pgsql_out_way(id, outtags, nodes_int, polygon, roads);
+ auto &way = buffer.get<osmium::Way>(0);
+ if (!m_tagtransform->filter_tags(way, &polygon, &roads,
+ *m_export_list.get(), outtags)) {
+ auto nnodes = m_mid->nodes_get_list(&(way.nodes()));
+ if (nnodes > 1) {
+ pgsql_out_way(way, &outtags, polygon, roads);
+ return 1;
+ }
}
}
@@ -310,16 +214,22 @@ void output_pgsql_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, s
}
int output_pgsql_t::pending_relation(osmid_t id, int exists) {
- taglist_t tags_int;
- memberlist_t members_int;
- int ret = 0;
-
// Try to fetch the relation from the DB
- if (m_mid->relations_get(id, members_int, tags_int)) {
- ret = pgsql_process_relation(id, members_int, tags_int, exists, true);
+ // Note that we cannot use the global buffer here because
+ // we cannot keep a reference to the relation and an autogrow buffer
+ // might be relocated when more data is added.
+ rels_buffer.clear();
+ if (m_mid->relations_get(id, rels_buffer)) {
+ // If the flag says this object may exist already, delete it first.
+ if (exists) {
+ pgsql_delete_relation_from_output(id);
+ }
+
+ auto const &rel = rels_buffer.get<osmium::Relation>(0);
+ return pgsql_process_relation(rel, true);
}
- return ret;
+ return 0;
}
void output_pgsql_t::commit()
@@ -352,113 +262,167 @@ void output_pgsql_t::stop()
}
}
- if (m_options.expire_tiles_zoom_min >= 0) {
+ if (m_options.expire_tiles_zoom_min > 0) {
expire.output_and_destroy(m_options.expire_tiles_filename.c_str(),
m_options.expire_tiles_zoom_min);
}
}
-int output_pgsql_t::node_add(osmid_t id, double lat, double lon, const taglist_t &tags)
+int output_pgsql_t::node_add(osmium::Node const &node)
{
- pgsql_out_node(id, tags, lat, lon);
+ taglist_t outtags;
+ if (m_tagtransform->filter_tags(node, nullptr, nullptr,
+ *m_export_list.get(), outtags))
+ return 1;
- return 0;
+ auto wkb = m_builder.get_wkb_node(node.location());
+ expire.from_wkb(wkb.c_str(), node.id());
+ m_tables[t_point]->write_row(node.id(), outtags, wkb);
+
+ return 0;
}
-int output_pgsql_t::way_add(osmid_t id, const idlist_t &nds, const taglist_t &tags)
+int output_pgsql_t::way_add(osmium::Way *way)
{
- int polygon = 0;
- int roads = 0;
- taglist_t outtags;
+ int polygon = 0;
+ int roads = 0;
+ taglist_t outtags;
- /* Check whether the way is: (1) Exportable, (2) Maybe a polygon */
- auto filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads,
- *m_export_list.get(), outtags);
+ /* Check whether the way is: (1) Exportable, (2) Maybe a polygon */
+ auto filter = m_tagtransform->filter_tags(*way, &polygon, &roads,
+ *m_export_list.get(), outtags);
- /* If this isn't a polygon then it can not be part of a multipolygon
- Hence only polygons are "pending" */
- if (!filter && polygon) { ways_pending_tracker.mark(id); }
+ /* If this isn't a polygon then it can not be part of a multipolygon
+ Hence only polygons are "pending" */
+ if (!filter && polygon) { ways_pending_tracker.mark(way->id()); }
- if( !polygon && !filter )
- {
- /* Get actual node data and generate output */
- nodelist_t nodes;
- m_mid->nodes_get_list(nodes, nds);
- pgsql_out_way(id, outtags, nodes, polygon, roads);
- }
- return 0;
+ if( !polygon && !filter )
+ {
+ /* Get actual node data and generate output */
+ auto nnodes = m_mid->nodes_get_list(&(way->nodes()));
+ if (nnodes > 1) {
+ pgsql_out_way(*way, &outtags, polygon, roads);
+ }
+ }
+ return 0;
}
/* This is the workhorse of pgsql_add_relation, split out because it is used as the callback for iterate relations */
-int output_pgsql_t::pgsql_process_relation(osmid_t id, const memberlist_t &members,
- const taglist_t &tags, int exists, bool pending)
+int output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel,
+ bool pending)
{
- /* If the flag says this object may exist already, delete it first */
- if(exists)
- pgsql_delete_relation_from_output(id);
+ taglist_t prefiltered_tags;
+ if (m_tagtransform->filter_tags(rel, nullptr, nullptr, *m_export_list.get(),
+ prefiltered_tags)) {
+ return 1;
+ }
+
+ idlist_t xid2;
+ for (auto const &m : rel.members()) {
+ /* Need to handle more than just ways... */
+ if (m.type() == osmium::item_type::way) {
+ xid2.push_back(m.ref());
+ }
+ }
+ buffer.clear();
+ rolelist_t xrole;
+ auto num_ways = m_mid->rel_way_members_get(rel, &xrole, buffer);
+
+ if (num_ways == 0)
+ return 0;
+
+ int roads = 0;
+ int make_polygon = 0;
+ int make_boundary = 0;
+ std::vector<int> members_superseded(num_ways, 0);
taglist_t outtags;
- if (m_tagtransform->filter_rel_tags(tags, *m_export_list.get(), outtags))
- return 1;
+ // If it's a route relation make_boundary and make_polygon will be false
+ // otherwise one or the other will be true.
+ if (m_tagtransform->filter_rel_member_tags(
+ prefiltered_tags, buffer, xrole, &(members_superseded[0]),
+ &make_boundary, &make_polygon, &roads, *m_export_list.get(),
+ outtags)) {
+ return 0;
+ }
- idlist_t xid2;
- multitaglist_t xtags2;
- multinodelist_t xnodes;
+ for (auto &w : buffer.select<osmium::Way>()) {
+ m_mid->nodes_get_list(&(w.nodes()));
+ }
- for (memberlist_t::const_iterator it = members.begin(); it != members.end(); ++it)
- {
- /* Need to handle more than just ways... */
- if (it->type == OSMTYPE_WAY)
- xid2.push_back(it->id);
+ // linear features and boundaries
+ // Needs to be done before the polygon treatment below because
+ // for boundaries the way_area tag may be added.
+ if (!make_polygon) {
+ double const split_at = m_options.projection->target_latlon() ? 1 : 100 * 1000;
+ auto wkbs = m_builder.get_wkb_multiline(buffer, split_at);
+ for (auto const &wkb : wkbs) {
+ expire.from_wkb(wkb.c_str(), -rel.id());
+ m_tables[t_line]->write_row(-rel.id(), outtags, wkb);
+ if (roads)
+ m_tables[t_roads]->write_row(-rel.id(), outtags, wkb);
+ }
}
- idlist_t xid;
- m_mid->ways_get_list(xid2, xid, xtags2, xnodes);
- int polygon = 0, roads = 0;
- multitaglist_t xtags(xid.size(), taglist_t());
- rolelist_t xrole(xid.size(), 0);
-
- for (size_t i = 0; i < xid.size(); i++) {
- for (size_t j = i; j < members.size(); j++) {
- if (members[j].id == xid[i]) {
- //filter the tags on this member because we got it from the middle
- //and since the middle is no longer tied to the output it no longer
- //shares any kind of tag transform and therefore all original tags
- //will come back and need to be filtered by individual outputs before
- //using these ways
- m_tagtransform->filter_way_tags(xtags2[i], &polygon, &roads,
- *m_export_list.get(), xtags[i]);
- //TODO: if the filter says that this member is now not interesting we
- //should decrement the count and remove his nodes and tags etc. for
- //now we'll just keep him with no tags so he will get filtered later
- xrole[i] = &members[j].role;
- break;
+ // multipolygons and boundaries
+ if (make_boundary || make_polygon) {
+ auto wkbs = m_builder.get_wkb_multipolygon(rel, buffer);
+
+ char tmp[32];
+ for (auto const &wkb : wkbs) {
+ expire.from_wkb(wkb.c_str(), -rel.id());
+ if (m_enable_way_area) {
+ auto const area =
+ m_options.reproject_area
+ ? ewkb::parser_t(wkb).get_area<reprojection>(
+ m_options.projection.get())
+ : ewkb::parser_t(wkb)
+ .get_area<osmium::geom::IdentityProjection>();
+ snprintf(tmp, sizeof(tmp), "%g", area);
+ outtags.push_override(tag_t("way_area", tmp));
}
+ m_tables[t_poly]->write_row(-rel.id(), outtags, wkb);
}
- }
- /* At some point we might want to consider storing the retrieved data in the members, rather than as separate arrays */
- pgsql_out_relation(id, outtags, xnodes, xtags, xid, xrole, pending);
+ /* Tagtransform will have marked those member ways of the relation that
+ * have fully been dealt with as part of the multi-polygon entry.
+ * Set them in the database as done and delete their entry to not
+ * have duplicates */
+ if (make_polygon) {
+ size_t j = 0;
+ for (auto &w : buffer.select<osmium::Way>()) {
+ if (members_superseded[j]) {
+ pgsql_delete_way_from_output(w.id());
+ // When working with pending relations this is not needed.
+ if (!pending) {
+ ways_done_tracker->mark(w.id());
+ }
+ }
+ ++j;
+ }
+ }
+ }
return 0;
}
-int output_pgsql_t::relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags)
+int output_pgsql_t::relation_add(osmium::Relation const &rel)
{
- const std::string *type = tags.get("type");
-
- /* Must have a type field or we ignore it */
- if (!type)
- return 0;
+ char const *type = rel.tags()["type"];
- /* Only a limited subset of type= is supported, ignore other */
- if ( (*type != "route") && (*type != "multipolygon") && (*type != "boundary"))
- return 0;
+ /* Must have a type field or we ignore it */
+ if (!type)
+ return 0;
+ /* Only a limited subset of type= is supported, ignore other */
+ if (strcmp(type, "route") != 0 && strcmp(type, "multipolygon") != 0
+ && strcmp(type, "boundary") != 0) {
+ return 0;
+ }
- return pgsql_process_relation(id, members, tags, 0);
+ return pgsql_process_relation(rel, false);
}
/* Delete is easy, just remove all traces of this object. We don't need to
@@ -532,40 +496,39 @@ int output_pgsql_t::relation_delete(osmid_t osm_id)
/* Modify is slightly trickier. The basic idea is we simply delete the
* object and create it with the new parameters. Then we need to mark the
* objects that depend on this one */
-int output_pgsql_t::node_modify(osmid_t osm_id, double lat, double lon, const taglist_t &tags)
+int output_pgsql_t::node_modify(osmium::Node const &node)
{
- if( !m_options.slim )
- {
- fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
+ if (!m_options.slim) {
+ fprintf(stderr, "Cannot apply diffs unless in slim mode\n");
util::exit_nicely();
}
- node_delete(osm_id);
- node_add(osm_id, lat, lon, tags);
+ node_delete(node.id());
+ node_add(node);
return 0;
}
-int output_pgsql_t::way_modify(osmid_t osm_id, const idlist_t &nodes, const taglist_t &tags)
+int output_pgsql_t::way_modify(osmium::Way *way)
{
if( !m_options.slim )
{
fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
util::exit_nicely();
}
- way_delete(osm_id);
- way_add(osm_id, nodes, tags);
+ way_delete(way->id());
+ way_add(way);
return 0;
}
-int output_pgsql_t::relation_modify(osmid_t osm_id, const memberlist_t &members, const taglist_t &tags)
+int output_pgsql_t::relation_modify(osmium::Relation const &rel)
{
if( !m_options.slim )
{
fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
util::exit_nicely();
}
- relation_delete(osm_id);
- relation_add(osm_id, members, tags);
+ relation_delete(rel.id());
+ relation_add(rel);
return 0;
}
@@ -587,21 +550,19 @@ std::shared_ptr<output_t> output_pgsql_t::clone(const middle_query_t* cloned_mid
return std::shared_ptr<output_t>(clone);
}
-output_pgsql_t::output_pgsql_t(const middle_query_t* mid, const options_t &o)
- : output_t(mid, o),
- expire(o.expire_tiles_zoom, o.expire_tiles_max_bbox, o.projection),
- ways_done_tracker(new id_tracker())
+output_pgsql_t::output_pgsql_t(const middle_query_t *mid, const options_t &o)
+: output_t(mid, o), m_builder(o.projection, o.enable_multi),
+ expire(o.expire_tiles_zoom, o.expire_tiles_max_bbox, o.projection),
+ ways_done_tracker(new id_tracker()),
+ buffer(32768, osmium::memory::Buffer::auto_grow::yes),
+ rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes)
{
- reproj = m_options.projection;
- builder.set_exclude_broken_polygon(m_options.excludepoly);
- if (m_options.reproject_area) builder.set_reprojection(reproj.get());
-
m_export_list.reset(new export_list());
m_enable_way_area = read_style_file( m_options.style, m_export_list.get() );
try {
- m_tagtransform.reset(new tagtransform(&m_options));
+ m_tagtransform = tagtransform_t::make_tagtransform(&m_options);
}
catch(const std::runtime_error& e) {
fprintf(stderr, "%s\n", e.what());
@@ -614,7 +575,9 @@ output_pgsql_t::output_pgsql_t(const middle_query_t* mid, const options_t &o)
for (int i = 0; i < t_MAX; i++) {
//figure out the columns this table needs
- columns_t columns = m_export_list->normal_columns((i == t_point)?OSMTYPE_NODE:OSMTYPE_WAY);
+ columns_t columns = m_export_list->normal_columns((i == t_point)
+ ? osmium::item_type::node
+ : osmium::item_type::way);
//figure out what name we are using for this and what type
std::string name = m_options.prefix;
@@ -645,29 +608,29 @@ output_pgsql_t::output_pgsql_t(const middle_query_t* mid, const options_t &o)
//tremble in awe of this massive constructor! seriously we are trying to avoid passing an
//options object because we want to make use of the table_t in output_mutli_t which could
//have a different tablespace/hstores/etc per table
- m_tables.push_back(std::shared_ptr<table_t>(
- new table_t(
- m_options.database_options.conninfo(), name, type, columns, m_options.hstore_columns,
- reproj->target_srs(),
- m_options.append, m_options.slim, m_options.droptemp, m_options.hstore_mode,
- m_options.enable_hstore_index, m_options.tblsmain_data, m_options.tblsmain_index
- )
- ));
+ m_tables.push_back(std::shared_ptr<table_t>(new table_t(
+ m_options.database_options.conninfo(), name, type, columns,
+ m_options.hstore_columns, m_options.projection->target_srs(),
+ m_options.append, m_options.slim, m_options.droptemp,
+ m_options.hstore_mode, m_options.enable_hstore_index,
+ m_options.tblsmain_data, m_options.tblsmain_index)));
}
}
-output_pgsql_t::output_pgsql_t(const output_pgsql_t& other):
- output_t(other.m_mid, other.m_options), m_tagtransform(new tagtransform(&m_options)), m_enable_way_area(other.m_enable_way_area),
- m_export_list(new export_list(*other.m_export_list)),
- expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox,
- m_options.projection),
- reproj(other.reproj),
- //NOTE: we need to know which ways were used by relations so each thread
- //must have a copy of the original marked done ways, its read only so its ok
- ways_done_tracker(other.ways_done_tracker)
+output_pgsql_t::output_pgsql_t(const output_pgsql_t &other)
+: output_t(other.m_mid, other.m_options),
+ m_tagtransform(tagtransform_t::make_tagtransform(&m_options)),
+ m_enable_way_area(other.m_enable_way_area),
+ m_export_list(new export_list(*other.m_export_list)),
+ m_builder(m_options.projection, other.m_options.enable_multi),
+ expire(m_options.expire_tiles_zoom, m_options.expire_tiles_max_bbox,
+ m_options.projection),
+ //NOTE: we need to know which ways were used by relations so each thread
+ //must have a copy of the original marked done ways, its read only so its ok
+ ways_done_tracker(other.ways_done_tracker),
+ buffer(1024, osmium::memory::Buffer::auto_grow::yes),
+ rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes)
{
- builder.set_exclude_broken_polygon(m_options.excludepoly);
- if (m_options.reproject_area) builder.set_reprojection(reproj.get());
for(std::vector<std::shared_ptr<table_t> >::const_iterator t = other.m_tables.begin(); t != other.m_tables.end(); ++t) {
//copy constructor will just connect to the already there table
m_tables.push_back(std::shared_ptr<table_t>(new table_t(**t)));
diff --git a/output-pgsql.hpp b/output-pgsql.hpp
index 3bffce3..a6d53d0 100644
--- a/output-pgsql.hpp
+++ b/output-pgsql.hpp
@@ -6,13 +6,12 @@
#ifndef OUTPUT_PGSQL_H
#define OUTPUT_PGSQL_H
-#include "output.hpp"
-#include "tagtransform.hpp"
-#include "geometry-builder.hpp"
-#include "reprojection.hpp"
#include "expire-tiles.hpp"
#include "id-tracker.hpp"
+#include "osmium-builder.hpp"
+#include "output.hpp"
#include "table.hpp"
+#include "tagtransform.hpp"
#include <vector>
#include <memory>
@@ -27,49 +26,43 @@ public:
virtual ~output_pgsql_t();
output_pgsql_t(const output_pgsql_t& other);
- virtual std::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const;
+ std::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const override;
- int start();
- void stop();
- void commit();
+ int start() override;
+ void stop() override;
+ void commit() override;
- void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
- int pending_way(osmid_t id, int exists);
+ void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) override;
+ int pending_way(osmid_t id, int exists) override;
- void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
- int pending_relation(osmid_t id, int exists);
+ void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) override;
+ int pending_relation(osmid_t id, int exists) override;
- int node_add(osmid_t id, double lat, double lon, const taglist_t &tags);
- int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags);
- int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags);
+ int node_add(osmium::Node const &node) override;
+ int way_add(osmium::Way *way) override;
+ int relation_add(osmium::Relation const &rel) override;
- int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags);
- int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags);
- int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags);
+ int node_modify(osmium::Node const &node) override;
+ int way_modify(osmium::Way *way) override;
+ int relation_modify(osmium::Relation const &rel) override;
- int node_delete(osmid_t id);
- int way_delete(osmid_t id);
- int relation_delete(osmid_t id);
+ int node_delete(osmid_t id) override;
+ int way_delete(osmid_t id) override;
+ int relation_delete(osmid_t id) override;
- size_t pending_count() const;
+ size_t pending_count() const override;
- void merge_pending_relations(output_t *other);
- void merge_expire_trees(output_t *other);
+ void merge_pending_relations(output_t *other) override;
+ void merge_expire_trees(output_t *other) override;
protected:
-
- int pgsql_out_node(osmid_t id, const taglist_t &tags, double node_lat, double node_lon);
- int pgsql_out_way(osmid_t id, taglist_t &tags, const nodelist_t &nodes,
- int polygons, int roads);
- int pgsql_out_relation(osmid_t id, const taglist_t &rel_tags,
- const multinodelist_t &xnodes, const multitaglist_t & xtags,
- const idlist_t &xid, const rolelist_t &xrole,
- bool pending);
- int pgsql_process_relation(osmid_t id, const memberlist_t &members, const taglist_t &tags, int exists, bool pending=false);
+ void pgsql_out_way(osmium::Way const &way, taglist_t *tags, bool polygon,
+ bool roads);
+ int pgsql_process_relation(osmium::Relation const &rel, bool pending);
int pgsql_delete_way_from_output(osmid_t osm_id);
int pgsql_delete_relation_from_output(osmid_t osm_id);
- std::unique_ptr<tagtransform> m_tagtransform;
+ std::unique_ptr<tagtransform_t> m_tagtransform;
//enable output of a generated way_area tag to either hstore or its own column
int m_enable_way_area;
@@ -78,13 +71,13 @@ protected:
std::unique_ptr<export_list> m_export_list;
- geometry_builder builder;
+ geom::osmium_builder_t m_builder;
expire_tiles expire;
- std::shared_ptr<reprojection> reproj;
-
id_tracker ways_pending_tracker, rels_pending_tracker;
std::shared_ptr<id_tracker> ways_done_tracker;
+ osmium::memory::Buffer buffer;
+ osmium::memory::Buffer rels_buffer;
};
#endif
diff --git a/output.cpp b/output.cpp
index 745c294..8977cbe 100644
--- a/output.cpp
+++ b/output.cpp
@@ -60,8 +60,8 @@ std::shared_ptr<output_t> parse_multi_single(const pt::ptree &conf,
geometry_processor::create(proc_type, &new_opts);
// TODO: we're faking this up, but there has to be a better way?
- OsmType osm_type = ((processor->interests() & geometry_processor::interest_node) > 0)
- ? OSMTYPE_NODE : OSMTYPE_WAY;
+ osmium::item_type osm_type = ((processor->interests() & geometry_processor::interest_node) > 0)
+ ? osmium::item_type::node : osmium::item_type::way;
export_list columns;
const pt::ptree &tags = conf.get_child("tags");
diff --git a/output.hpp b/output.hpp
index e7c7ec4..19c02f7 100644
--- a/output.hpp
+++ b/output.hpp
@@ -49,13 +49,13 @@ public:
virtual void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) = 0;
virtual int pending_relation(osmid_t id, int exists) = 0;
- virtual int node_add(osmid_t id, double lat, double lon, const taglist_t &tags) = 0;
- virtual int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags) = 0;
- virtual int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) = 0;
+ virtual int node_add(osmium::Node const &node) = 0;
+ virtual int way_add(osmium::Way *way) = 0;
+ virtual int relation_add(osmium::Relation const &rel) = 0;
- virtual int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags) = 0;
- virtual int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags) = 0;
- virtual int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags) = 0;
+ virtual int node_modify(osmium::Node const &node) = 0;
+ virtual int way_modify(osmium::Way *way) = 0;
+ virtual int relation_modify(osmium::Relation const &rel) = 0;
virtual int node_delete(osmid_t id) = 0;
virtual int way_delete(osmid_t id) = 0;
diff --git a/parse-osmium.cpp b/parse-osmium.cpp
index 681032e..346fd9c 100644
--- a/parse-osmium.cpp
+++ b/parse-osmium.cpp
@@ -60,9 +60,14 @@ void parse_stats_t::print_summary() const
rel.count > 0 ? (int) (end_rel - rel.start) : 0);
}
-void parse_stats_t::print_status() const
+void parse_stats_t::print_status()
{
time_t now = time(nullptr);
+
+ if (print_time >= now) {
+ return;
+ }
+
time_t end_nodes = way.start > 0 ? way.start : now;
time_t end_way = rel.start > 0 ? rel.start : now;
time_t end_rel = now;
@@ -73,14 +78,14 @@ void parse_stats_t::print_status() const
way.count / 1000,
way.count > 0 ? (double) way.count / 1000.0 / ((double) (end_way - way.start) > 0.0 ? (double) (end_way - way.start) : 1.0) : 0.0, rel.count,
rel.count > 0 ? (double) rel.count / ((double) (end_rel - rel.start) > 0.0 ? (double) (end_rel - rel.start) : 1.0) : 0.0);
+
+ print_time = now;
}
-parse_osmium_t::parse_osmium_t(bool extra_attrs,
- const boost::optional<std::string> &bbox,
- const reprojection *proj, bool do_append,
- osmdata_t *osmdata)
-: m_data(osmdata), m_append(do_append), m_attributes(extra_attrs), m_proj(proj)
+parse_osmium_t::parse_osmium_t(const boost::optional<std::string> &bbox,
+ bool do_append, osmdata_t *osmdata)
+: m_data(osmdata), m_append(do_append)
{
if (bbox) {
m_bbox = parse_bbox(bbox);
@@ -124,7 +129,7 @@ void parse_osmium_t::stream_file(const std::string &filename, const std::string
reader.close();
}
-void parse_osmium_t::node(osmium::Node& node)
+void parse_osmium_t::node(osmium::Node const &node)
{
if (node.deleted()) {
m_data->node_delete(node.id());
@@ -142,13 +147,10 @@ void parse_osmium_t::node(osmium::Node& node)
}
if (!m_bbox || m_bbox->contains(node.location())) {
- auto c = m_proj->reproject(node.location());
-
- convert_tags(node);
if (m_append) {
- m_data->node_modify(node.id(), c.y, c.x, tags);
+ m_data->node_modify(node);
} else {
- m_data->node_add(node.id(), c.y, c.x, tags);
+ m_data->node_add(node);
}
m_stats.add_node(node.id());
}
@@ -160,18 +162,16 @@ void parse_osmium_t::way(osmium::Way& way)
if (way.deleted()) {
m_data->way_delete(way.id());
} else {
- convert_tags(way);
- convert_nodes(way.nodes());
if (m_append) {
- m_data->way_modify(way.id(), nds, tags);
+ m_data->way_modify(&way);
} else {
- m_data->way_add(way.id(), nds, tags);
+ m_data->way_add(&way);
}
}
m_stats.add_way(way.id());
}
-void parse_osmium_t::relation(osmium::Relation& rel)
+void parse_osmium_t::relation(osmium::Relation const &rel)
{
if (rel.deleted()) {
m_data->relation_delete(rel.id());
@@ -179,54 +179,11 @@ void parse_osmium_t::relation(osmium::Relation& rel)
if (rel.members().size() > 32767) {
return;
}
- convert_tags(rel);
- convert_members(rel.members());
if (m_append) {
- m_data->relation_modify(rel.id(), members, tags);
+ m_data->relation_modify(rel);
} else {
- m_data->relation_add(rel.id(), members, tags);
+ m_data->relation_add(rel);
}
}
m_stats.add_rel(rel.id());
}
-
-void parse_osmium_t::convert_tags(const osmium::OSMObject &obj)
-{
- tags.clear();
- for (auto const &t : obj.tags()) {
- tags.emplace_back(t.key(), t.value());
- }
- if (m_attributes) {
- tags.emplace_back("osm_user", obj.user());
- tags.emplace_back("osm_uid", std::to_string(obj.uid()));
- tags.emplace_back("osm_version", std::to_string(obj.version()));
- tags.emplace_back("osm_timestamp", obj.timestamp().to_iso());
- tags.emplace_back("osm_changeset", std::to_string(obj.changeset()));
- }
-}
-
-void parse_osmium_t::convert_nodes(const osmium::NodeRefList &in_nodes)
-{
- nds.clear();
-
- for (auto const &n : in_nodes) {
- nds.push_back(n.ref());
- }
-}
-
-void parse_osmium_t::convert_members(const osmium::RelationMemberList &in_rels)
-{
- members.clear();
-
- for (auto const &m: in_rels) {
- OsmType type;
- switch (m.type()) {
- case osmium::item_type::node: type = OSMTYPE_NODE; break;
- case osmium::item_type::way: type = OSMTYPE_WAY; break;
- case osmium::item_type::relation: type = OSMTYPE_RELATION; break;
- default:
- fprintf(stderr, "Unsupported type: %u""\n", unsigned(m.type()));
- }
- members.emplace_back(type, m.ref(), m.role());
- }
-}
diff --git a/parse-osmium.hpp b/parse-osmium.hpp
index 1264ef4..4b1e23a 100644
--- a/parse-osmium.hpp
+++ b/parse-osmium.hpp
@@ -35,7 +35,6 @@
#include <osmium/handler.hpp>
-class reprojection;
class osmdata_t;
class parse_stats_t
@@ -73,9 +72,11 @@ class parse_stats_t
};
public:
+ parse_stats_t() : print_time(time(nullptr)) {}
+
void update(const parse_stats_t &other);
void print_summary() const;
- void print_status() const;
+ void print_status();
inline void add_node(osmid_t id)
{
@@ -100,20 +101,21 @@ public:
private:
Counter node, way, rel;
+ time_t print_time;
};
class parse_osmium_t: public osmium::handler::Handler
{
public:
- parse_osmium_t(bool extra_attrs, const boost::optional<std::string> &bbox,
- const reprojection *proj, bool do_append, osmdata_t *osmdata);
+ parse_osmium_t(const boost::optional<std::string> &bbox,
+ bool do_append, osmdata_t *osmdata);
void stream_file(const std::string &filename, const std::string &fmt);
- void node(osmium::Node& node);
+ void node(osmium::Node const &node);
void way(osmium::Way& way);
- void relation(osmium::Relation& rel);
+ void relation(osmium::Relation const &rel);
parse_stats_t const &stats() const
{
@@ -121,25 +123,12 @@ public:
}
private:
- void convert_tags(const osmium::OSMObject &obj);
- void convert_nodes(const osmium::NodeRefList &in_nodes);
- void convert_members(const osmium::RelationMemberList &in_rels);
-
osmium::Box parse_bbox(const boost::optional<std::string> &bbox);
osmdata_t *m_data;
bool m_append;
boost::optional<osmium::Box> m_bbox;
- bool m_attributes;
- const reprojection *m_proj;
parse_stats_t m_stats;
-
- /* Since {node,way} elements are not nested we can guarantee that
- elements are parsed sequentially and can therefore be cached.
- */
- taglist_t tags;
- idlist_t nds;
- memberlist_t members;
};
#endif
diff --git a/pgsql.cpp b/pgsql.cpp
index 667792e..abd1d38 100644
--- a/pgsql.cpp
+++ b/pgsql.cpp
@@ -23,24 +23,23 @@ void escape(const std::string &src, std::string &dst)
}
}
-
-std::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const std::string& sql)
+pg_result_t pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect,
+ const std::string &sql)
{
return pgsql_exec_simple(sql_conn, expect, sql.c_str());
}
-std::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const char *sql)
+pg_result_t pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect,
+ const char *sql)
{
- PGresult* res;
#ifdef DEBUG_PGSQL
fprintf( stderr, "Executing: %s\n", sql );
#endif
- res = PQexec(sql_conn, sql);
- if (PQresultStatus(res) != expect) {
- PQclear(res);
+ pg_result_t res(PQexec(sql_conn, sql));
+ if (PQresultStatus(res.get()) != expect) {
throw std::runtime_error((boost::format("%1% failed: %2%\n") % sql % PQerrorMessage(sql_conn)).str());
}
- return std::shared_ptr<PGresult>(res, &PQclear);
+ return res;
}
int pgsql_exec(PGconn *sql_conn, const ExecStatusType expect, const char *fmt, ...)
@@ -80,15 +79,13 @@ int pgsql_exec(PGconn *sql_conn, const ExecStatusType expect, const char *fmt, .
#ifdef DEBUG_PGSQL
fprintf( stderr, "Executing: %s\n", sql );
#endif
- PGresult* res = PQexec(sql_conn, sql);
- if (PQresultStatus(res) != expect) {
+ pg_result_t res(PQexec(sql_conn, sql));
+ if (PQresultStatus(res.get()) != expect) {
std::string err_msg = (boost::format("%1% failed: %2%") % sql % PQerrorMessage(sql_conn)).str();
free(sql);
- PQclear(res);
throw std::runtime_error(err_msg);
}
free(sql);
- PQclear(res);
return 0;
}
@@ -112,16 +109,22 @@ void pgsql_CopyData(const char *context, PGconn *sql_conn, std::string const &sq
}
}
-PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, const int nParams, const char *const * paramValues, const ExecStatusType expect)
+pg_result_t pgsql_execPrepared(PGconn *sql_conn, const char *stmtName,
+ const int nParams,
+ const char *const *paramValues,
+ const ExecStatusType expect)
{
#ifdef DEBUG_PGSQL
fprintf( stderr, "ExecPrepared: %s\n", stmtName );
#endif
//run the prepared statement
- PGresult *res = PQexecPrepared(sql_conn, stmtName, nParams, paramValues, nullptr, nullptr, 0);
- if(PQresultStatus(res) != expect)
- {
- std::string message = (boost::format("%1% failed: %2%(%3%)\n") % stmtName % PQerrorMessage(sql_conn) % PQresultStatus(res)).str();
+ pg_result_t res(PQexecPrepared(sql_conn, stmtName, nParams, paramValues,
+ nullptr, nullptr, 0));
+ if (PQresultStatus(res.get()) != expect) {
+ std::string message =
+ (boost::format("%1% failed: %2%(%3%)\n") % stmtName %
+ PQerrorMessage(sql_conn) % PQresultStatus(res.get()))
+ .str();
if(nParams)
{
message += "Arguments were: ";
@@ -131,16 +134,8 @@ PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, const int
message += ", ";
}
}
- PQclear(res);
throw std::runtime_error(message);
}
- //TODO: this seems a bit strange
- //if you decided you wanted to expect something other than this you didnt want to use the result?
- if( expect != PGRES_TUPLES_OK )
- {
- PQclear(res);
- res = nullptr;
- }
return res;
}
diff --git a/pgsql.hpp b/pgsql.hpp
index 6bd0ed0..9f4f217 100644
--- a/pgsql.hpp
+++ b/pgsql.hpp
@@ -11,10 +11,23 @@
#include <libpq-fe.h>
#include <memory>
-PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, const int nParams, const char *const * paramValues, const ExecStatusType expect);
+struct pg_result_deleter_t
+{
+ void operator()(PGresult *p) const { PQclear(p); }
+};
+
+typedef std::unique_ptr<PGresult, pg_result_deleter_t> pg_result_t;
+
+pg_result_t pgsql_execPrepared(PGconn *sql_conn, const char *stmtName,
+ int nParams, const char *const *paramValues,
+ ExecStatusType expect);
void pgsql_CopyData(const char *context, PGconn *sql_conn, std::string const &sql);
-std::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const std::string& sql);
-std::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const char *sql);
+
+pg_result_t pgsql_exec_simple(PGconn *sql_conn, ExecStatusType expect,
+ std::string const &sql);
+pg_result_t pgsql_exec_simple(PGconn *sql_conn, ExecStatusType expect,
+ const char *sql);
+
int pgsql_exec(PGconn *sql_conn, const ExecStatusType expect, const char *fmt, ...)
#ifndef _MSC_VER
__attribute__ ((format (printf, 3, 4)))
diff --git a/processor-line.cpp b/processor-line.cpp
index bc1b23a..807bec2 100644
--- a/processor-line.cpp
+++ b/processor-line.cpp
@@ -1,19 +1,24 @@
#include "processor-line.hpp"
-processor_line::processor_line(int srid) : geometry_processor(srid, "LINESTRING", interest_way | interest_relation )
+processor_line::processor_line(std::shared_ptr<reprojection> const &proj)
+: geometry_processor(proj->target_srs(), "LINESTRING",
+ interest_way | interest_relation)
{
}
-processor_line::~processor_line()
+geometry_processor::wkb_t
+processor_line::process_way(osmium::Way const &way,
+ geom::osmium_builder_t *builder)
{
-}
+ auto wkbs = builder->get_wkb_line(way.nodes(), 1000000);
-geometry_builder::pg_geom_t processor_line::process_way(const nodelist_t &nodes)
-{
- return builder.get_wkb_simple(nodes, false);
+ return wkbs.empty() ? wkb_t() : wkbs[0];
}
-geometry_builder::pg_geoms_t processor_line::process_relation(const multinodelist_t &nodes)
+geometry_processor::wkbs_t
+processor_line::process_relation(osmium::Relation const &,
+ osmium::memory::Buffer const &ways,
+ geom::osmium_builder_t *builder)
{
- return builder.build_both(nodes, false, false, 1000000);
+ return builder->get_wkb_multiline(ways, 1000000);
}
diff --git a/processor-line.hpp b/processor-line.hpp
index 1a011b6..aa46940 100644
--- a/processor-line.hpp
+++ b/processor-line.hpp
@@ -3,15 +3,16 @@
#include "geometry-processor.hpp"
-struct processor_line : public geometry_processor {
- processor_line(int srid);
- virtual ~processor_line();
+class processor_line : public geometry_processor
+{
+public:
+ processor_line(std::shared_ptr<reprojection> const &proj);
- geometry_builder::pg_geom_t process_way(const nodelist_t &nodes);
- geometry_builder::pg_geoms_t process_relation(const multinodelist_t &nodes);
-
-private:
- geometry_builder builder;
+ wkb_t process_way(osmium::Way const &way,
+ geom::osmium_builder_t *builder) override;
+ wkbs_t process_relation(osmium::Relation const &rel,
+ osmium::memory::Buffer const &ways,
+ geom::osmium_builder_t *builder) override;
};
#endif /* PROCESSOR_LINE_HPP */
diff --git a/processor-point.cpp b/processor-point.cpp
index d404725..4f75589 100644
--- a/processor-point.cpp
+++ b/processor-point.cpp
@@ -5,14 +5,14 @@
#include "processor-point.hpp"
#include "util.hpp"
-processor_point::processor_point(int srid)
- : geometry_processor(srid, "POINT", interest_node) {
-}
-
-processor_point::~processor_point() {
+processor_point::processor_point(std::shared_ptr<reprojection> const &proj)
+: geometry_processor(proj->target_srs(), "POINT", interest_node)
+{
}
-geometry_builder::pg_geom_t processor_point::process_node(double lat, double lon)
+geometry_processor::wkb_t
+processor_point::process_node(osmium::Location const &loc,
+ geom::osmium_builder_t *builder)
{
- return geometry_builder::pg_geom_t((boost::format("POINT(%.15g %.15g)") % lon % lat).str(), false);
+ return builder->get_wkb_node(loc);
}
diff --git a/processor-point.hpp b/processor-point.hpp
index 6cdf58e..0b48d1f 100644
--- a/processor-point.hpp
+++ b/processor-point.hpp
@@ -3,11 +3,13 @@
#include "geometry-processor.hpp"
-struct processor_point : public geometry_processor {
- processor_point(int srid);
- virtual ~processor_point();
+class processor_point : public geometry_processor
+{
+public:
+ processor_point(std::shared_ptr<reprojection> const &proj);
- geometry_builder::pg_geom_t process_node(double lat, double lon);
+ wkb_t process_node(osmium::Location const &loc,
+ geom::osmium_builder_t *builder) override;
};
#endif /* PROCESSOR_POINT_HPP */
diff --git a/processor-polygon.cpp b/processor-polygon.cpp
index 73c3392..7546037 100644
--- a/processor-polygon.cpp
+++ b/processor-polygon.cpp
@@ -1,19 +1,22 @@
#include "processor-polygon.hpp"
-processor_polygon::processor_polygon(int srid, bool enable_multi) : geometry_processor(srid, "GEOMETRY", interest_way | interest_relation), enable_multi(enable_multi)
+processor_polygon::processor_polygon(std::shared_ptr<reprojection> const &proj)
+: geometry_processor(proj->target_srs(), "GEOMETRY",
+ interest_way | interest_relation)
{
}
-processor_polygon::~processor_polygon()
+geometry_processor::wkb_t
+processor_polygon::process_way(osmium::Way const &way,
+ geom::osmium_builder_t *builder)
{
+ return builder->get_wkb_polygon(way);
}
-geometry_builder::pg_geom_t processor_polygon::process_way(const nodelist_t &nodes)
+geometry_processor::wkbs_t
+processor_polygon::process_relation(osmium::Relation const &rel,
+ osmium::memory::Buffer const &ways,
+ geom::osmium_builder_t *builder)
{
- return builder.get_wkb_simple(nodes, true);
-}
-
-geometry_builder::pg_geoms_t processor_polygon::process_relation(const multinodelist_t &nodes)
-{
- return builder.build_polygons(nodes, enable_multi, -1);
+ return builder->get_wkb_multipolygon(rel, ways);
}
diff --git a/processor-polygon.hpp b/processor-polygon.hpp
index 6d562eb..e40d11e 100644
--- a/processor-polygon.hpp
+++ b/processor-polygon.hpp
@@ -3,16 +3,16 @@
#include "geometry-processor.hpp"
-struct processor_polygon : public geometry_processor {
- processor_polygon(int srid, bool enable_multi);
- virtual ~processor_polygon();
+class processor_polygon : public geometry_processor
+{
+public:
+ processor_polygon(std::shared_ptr<reprojection> const &proj);
- geometry_builder::pg_geom_t process_way(const nodelist_t &nodes);
- geometry_builder::pg_geoms_t process_relation(const multinodelist_t &nodes);
-
-private:
- bool enable_multi;
- geometry_builder builder;
+ wkb_t process_way(osmium::Way const &nodes,
+ geom::osmium_builder_t *builder) override;
+ wkbs_t process_relation(osmium::Relation const &rel,
+ osmium::memory::Buffer const &ways,
+ geom::osmium_builder_t *builder) override;
};
#endif /* PROCESSOR_POLYGON_HPP */
diff --git a/style.lua b/style.lua
index 87669bd..7ce5abb 100644
--- a/style.lua
+++ b/style.lua
@@ -261,7 +261,8 @@ function filter_tags_relation_member (keyvalues, keyvaluemembers, roles, memberc
-- Count the number of polygon tags of the object
for i,k in ipairs(polygon_keys) do
if keyvalues[k] then
- polytagcount = polytagcount + 1
+ polytagcount = 1
+ break
end
end
-- If there are no polygon tags, add tags from all outer elements to the multipolygon itself
@@ -273,6 +274,19 @@ function filter_tags_relation_member (keyvalues, keyvaluemembers, roles, memberc
end
end
end
+
+ f, keyvalues = filter_tags_generic(keyvalues, 1)
+ -- check again if there are still polygon tags left
+ polytagcount = 0
+ for i,k in ipairs(polygon_keys) do
+ if keyvalues[k] then
+ polytagcount = 1
+ break
+ end
+ end
+ if polytagcount == 0 then
+ filter = 1
+ end
end
-- For any member of the multipolygon, set membersuperseded to 1 (i.e. don't deal with it as area as well),
-- except when the member has a key/value combination such that
diff --git a/table.cpp b/table.cpp
index 271a317..87ccf7d 100644
--- a/table.cpp
+++ b/table.cpp
@@ -1,8 +1,3 @@
-#include "table.hpp"
-#include "options.hpp"
-#include "util.hpp"
-#include "taginfo.hpp"
-
#include <exception>
#include <algorithm>
#include <cstring>
@@ -10,6 +5,12 @@
#include <utility>
#include <time.h>
+#include "options.hpp"
+#include "table.hpp"
+#include "taginfo.hpp"
+#include "util.hpp"
+#include "wkb.hpp"
+
using std::string;
typedef boost::format fmt;
@@ -32,7 +33,6 @@ table_t::table_t(const string& conninfo, const string& name, const string& type,
//we use these a lot, so instead of constantly allocating them we predefine these
single_fmt = fmt("%1%");
- point_fmt = fmt("POINT(%.15g %.15g)");
del_fmt = fmt("DELETE FROM %1% WHERE osm_id = %2%");
}
@@ -40,7 +40,7 @@ table_t::table_t(const table_t& other):
conninfo(other.conninfo), name(other.name), type(other.type), sql_conn(nullptr), copyMode(false), buffer(), srid(other.srid),
append(other.append), slim(other.slim), drop_temp(other.drop_temp), hstore_mode(other.hstore_mode), enable_hstore_index(other.enable_hstore_index),
columns(other.columns), hstore_columns(other.hstore_columns), copystr(other.copystr), table_space(other.table_space),
- table_space_index(other.table_space_index), single_fmt(other.single_fmt), point_fmt(other.point_fmt), del_fmt(other.del_fmt)
+ table_space_index(other.table_space_index), single_fmt(other.single_fmt), del_fmt(other.del_fmt)
{
// if the other table has already started, then we want to execute
// the same stuff to get into the same state. but if it hasn't, then
@@ -108,7 +108,7 @@ void table_t::start()
//we are making a new table
if (!append)
{
- pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("DROP TABLE IF EXISTS %1%") % name).str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("DROP TABLE IF EXISTS %1% CASCADE") % name).str());
}
/* These _tmp tables can be left behind if we run out of disk space */
@@ -161,7 +161,9 @@ void table_t::start()
}//appending
else {
//check the columns against those in the existing table
- std::shared_ptr<PGresult> res = pgsql_exec_simple(sql_conn, PGRES_TUPLES_OK, (fmt("SELECT * FROM %1% LIMIT 0") % name).str());
+ auto res =
+ pgsql_exec_simple(sql_conn, PGRES_TUPLES_OK,
+ (fmt("SELECT * FROM %1% LIMIT 0") % name).str());
for (auto const &column : columns) {
if (PQfnumber(res.get(), ('"' + column.name + '"').c_str()) < 0) {
#if 0
@@ -247,7 +249,6 @@ void table_t::stop()
}
}
fprintf(stderr, "Creating indexes on %s finished\n", name.c_str());
- pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("GRANT SELECT ON %1% TO PUBLIC") % name).str());
pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ANALYZE %1%") % name).str());
time(&end);
fprintf(stderr, "All indexes on %s created in %ds\n", name.c_str(), (int)(end - start));
@@ -259,7 +260,6 @@ void table_t::stop()
void table_t::stop_copy()
{
- PGresult* res;
int stop;
//we werent copying anyway
@@ -278,28 +278,20 @@ void table_t::stop_copy()
throw std::runtime_error((fmt("stop COPY_END for %1% failed: %2%\n") % name % PQerrorMessage(sql_conn)).str());
//get the result
- res = PQgetResult(sql_conn);
- if (PQresultStatus(res) != PGRES_COMMAND_OK)
- {
- PQclear(res);
+ pg_result_t res(PQgetResult(sql_conn));
+ if (PQresultStatus(res.get()) != PGRES_COMMAND_OK) {
throw std::runtime_error((fmt("result COPY_END for %1% failed: %2%\n") % name % PQerrorMessage(sql_conn)).str());
}
- PQclear(res);
copyMode = false;
}
-void table_t::write_node(const osmid_t id, const taglist_t &tags, double lat, double lon)
-{
- write_row(id, tags, (point_fmt % lon % lat).str());
-}
-
void table_t::delete_row(const osmid_t id)
{
stop_copy();
pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (del_fmt % name % id).str());
}
-void table_t::write_row(const osmid_t id, const taglist_t &tags, const std::string &geom)
+void table_t::write_row(osmid_t id, taglist_t const &tags, std::string const &geom)
{
//add the osm id
buffer.append((single_fmt % id).str());
@@ -312,7 +304,7 @@ void table_t::write_row(const osmid_t id, const taglist_t &tags, const std::stri
used.assign(tags.size(), false);
//get the regular columns' values
- write_columns(tags, buffer, hstore_mode == HSTORE_NORM?&used:nullptr);
+ write_columns(tags, buffer, hstore_mode == HSTORE_NORM ? &used : nullptr);
//get the hstore columns' values
write_hstore_columns(tags, buffer);
@@ -321,25 +313,20 @@ void table_t::write_row(const osmid_t id, const taglist_t &tags, const std::stri
if (hstore_mode != HSTORE_NONE)
write_tags_column(tags, buffer, used);
- //give the geometry an srid
- buffer.append("SRID=");
- buffer.append(srid);
- buffer.push_back(';');
- //add the geometry
- buffer.append(geom);
+ //add the geometry - encoding it to hex along the way
+ ewkb::writer_t::write_as_hex(buffer, geom);
+
//we need \n because we are copying from stdin
buffer.push_back('\n');
//tell the db we are copying if for some reason we arent already
- if (!copyMode)
- {
+ if (!copyMode) {
pgsql_exec_simple(sql_conn, PGRES_COPY_IN, copystr);
copyMode = true;
}
//send all the data to postgres
- if(buffer.length() > BUFFER_SEND_SIZE)
- {
+ if (buffer.length() > BUFFER_SEND_SIZE) {
pgsql_CopyData(name.c_str(), sql_conn, buffer);
buffer.clear();
}
@@ -526,6 +513,8 @@ table_t::wkb_reader table_t::get_wkb_reader(const osmid_t id)
//the prepared statement get_wkb will behave differently depending on the sql_conn
//each table has its own sql_connection with the get_way referring to the appropriate table
- PGresult* res = pgsql_execPrepared(sql_conn, "get_wkb", 1, (const char * const *)paramValues, PGRES_TUPLES_OK);
- return wkb_reader(res);
+ auto res =
+ pgsql_execPrepared(sql_conn, "get_wkb", 1,
+ (const char *const *)paramValues, PGRES_TUPLES_OK);
+ return wkb_reader(std::move(res));
}
diff --git a/table.hpp b/table.hpp
index f4b2423..1aa404b 100644
--- a/table.hpp
+++ b/table.hpp
@@ -31,21 +31,11 @@ class table_t
void begin();
void commit();
- void write_row(const osmid_t id, const taglist_t &tags, const std::string &geom);
- void write_node(const osmid_t id, const taglist_t &tags, double lat, double lon);
+ void write_row(osmid_t id, taglist_t const &tags, std::string const &geom);
void delete_row(const osmid_t id);
std::string const& get_name();
- struct pg_result_closer
- {
- void operator() (PGresult* result)
- {
- PQclear(result);
- }
-
- };
-
//interface from retrieving well known binary geometry from the table
class wkb_reader
{
@@ -67,11 +57,12 @@ class table_t
m_current = 0;
}
private:
- wkb_reader(PGresult* result)
- : m_result(result), m_count(PQntuples(result)), m_current(0)
+ wkb_reader(pg_result_t &&result)
+ : m_result(std::move(result)), m_count(PQntuples(m_result.get())),
+ m_current(0)
{}
- std::unique_ptr<PGresult, pg_result_closer> m_result;
+ pg_result_t m_result;
int m_count;
int m_current;
};
@@ -108,7 +99,7 @@ class table_t
boost::optional<std::string> table_space;
boost::optional<std::string> table_space_index;
- boost::format single_fmt, point_fmt, del_fmt;
+ boost::format single_fmt, del_fmt;
};
#endif
diff --git a/taginfo.cpp b/taginfo.cpp
index 52e1260..dec87fe 100644
--- a/taginfo.cpp
+++ b/taginfo.cpp
@@ -43,37 +43,50 @@ taginfo::taginfo(const taginfo &other)
flags(other.flags) {
}
-export_list::export_list()
- : num_tables(0), exportList() {
-}
-
-void export_list::add(enum OsmType id, const taginfo &info) {
+void export_list::add(osmium::item_type id, const taginfo &info) {
std::vector<taginfo> &infos = get(id);
infos.push_back(info);
}
-std::vector<taginfo> &export_list::get(enum OsmType id) {
- if (id >= num_tables) {
- exportList.resize(id+1);
- num_tables = id + 1;
+std::vector<taginfo> &export_list::get(osmium::item_type id) {
+ auto idx = item_type_to_nwr_index(id);
+ if (idx >= exportList.size()) {
+ exportList.resize(idx+1);
}
- return exportList[id];
+ return exportList[idx];
}
-const std::vector<taginfo> &export_list::get(enum OsmType id) const {
+const std::vector<taginfo> &export_list::get(osmium::item_type id) const {
// this fakes as if we have infinite taginfo vectors, but
// means we don't actually have anything allocated unless
// the info object has been assigned.
static const std::vector<taginfo> empty;
- if (id < num_tables) {
- return exportList[id];
+ auto idx = item_type_to_nwr_index(id);
+ if (idx < exportList.size()) {
+ return exportList[idx];
} else {
return empty;
}
}
-columns_t export_list::normal_columns(OsmType id) const {
+bool export_list::has_column(osmium::item_type id, char const *name) const
+{
+ auto idx = item_type_to_nwr_index(id);
+ if (idx >= exportList.size()) {
+ return false;
+ }
+
+ for (auto const &info : exportList[idx]) {
+ if (info.name == name) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+columns_t export_list::normal_columns(osmium::item_type id) const {
columns_t columns;
for (auto const &info : get(id)) {
@@ -145,6 +158,7 @@ int read_style_file( const std::string &filename, export_list *exlist )
if( fields < 3 )
{
fprintf( stderr, "Error reading style file line %d (fields=%d)\n", lineno, fields );
+ fclose(in);
util::exit_nicely();
}
@@ -163,6 +177,7 @@ int read_style_file( const std::string &filename, export_list *exlist )
((temp.name.find('?') != std::string::npos) ||
(temp.name.find('*') != std::string::npos))) {
fprintf( stderr, "wildcard '%s' in non-delete style entry\n",temp.name.c_str());
+ fclose(in);
util::exit_nicely();
}
@@ -176,20 +191,21 @@ int read_style_file( const std::string &filename, export_list *exlist )
//keep this tag info if it applies to nodes
if( strstr( osmtype, "node" ) )
{
- exlist->add(OSMTYPE_NODE, temp);
+ exlist->add(osmium::item_type::node, temp);
kept = true;
}
//keep this tag info if it applies to ways
if( strstr( osmtype, "way" ) )
{
- exlist->add(OSMTYPE_WAY, temp);
+ exlist->add(osmium::item_type::way, temp);
kept = true;
}
//do we really want to completely quit on an unusable line?
if( !kept )
{
+ fclose(in);
throw std::runtime_error((boost::format("Weird style line %1%:%2%")
% filename % lineno).str());
}
@@ -198,13 +214,15 @@ int read_style_file( const std::string &filename, export_list *exlist )
if (ferror(in)) {
+ int err = errno;
+ fclose(in);
throw std::runtime_error((boost::format("%1%: %2%")
- % filename % strerror(errno)).str());
+ % filename % strerror(err)).str());
}
+ fclose(in);
if (num_read == 0) {
throw std::runtime_error("Unable to parse any valid columns from "
"the style file. Aborting.");
}
- fclose(in);
return enable_way_area;
}
diff --git a/taginfo_impl.hpp b/taginfo_impl.hpp
index 19a4120..c6313be 100644
--- a/taginfo_impl.hpp
+++ b/taginfo_impl.hpp
@@ -37,16 +37,14 @@ struct taginfo {
};
struct export_list {
- export_list();
+ void add(osmium::item_type id, const taginfo &info);
+ std::vector<taginfo> &get(osmium::item_type id);
+ const std::vector<taginfo> &get(osmium::item_type id) const;
- void add(enum OsmType id, const taginfo &info);
- std::vector<taginfo> &get(enum OsmType id);
- const std::vector<taginfo> &get(enum OsmType id) const;
+ columns_t normal_columns(osmium::item_type id) const;
+ bool has_column(osmium::item_type id, char const *name) const;
- columns_t normal_columns(OsmType id) const;
-
- int num_tables;
- std::vector<std::vector<taginfo> > exportList; /* Indexed by enum OsmType */
+ std::vector<std::vector<taginfo> > exportList; /* Indexed osmium nwr index */
};
/* Parse a comma or whitespace delimited list of tags to apply to
diff --git a/tagtransform-c.cpp b/tagtransform-c.cpp
new file mode 100644
index 0000000..f617b37
--- /dev/null
+++ b/tagtransform-c.cpp
@@ -0,0 +1,421 @@
+#include <boost/algorithm/string/predicate.hpp>
+#include <cstdlib>
+#include <cstring>
+
+#include "options.hpp"
+#include "taginfo_impl.hpp"
+#include "tagtransform-c.hpp"
+#include "wildcmp.hpp"
+
+namespace {
+
+static const struct
+{
+ int offset;
+ const char *highway;
+ int roads;
+} layers[] = {{1, "proposed", 0}, {2, "construction", 0},
+ {10, "steps", 0}, {10, "cycleway", 0},
+ {10, "bridleway", 0}, {10, "footway", 0},
+ {10, "path", 0}, {11, "track", 0},
+ {15, "service", 0},
+
+ {24, "tertiary_link", 0}, {25, "secondary_link", 1},
+ {27, "primary_link", 1}, {28, "trunk_link", 1},
+ {29, "motorway_link", 1},
+
+ {30, "raceway", 0}, {31, "pedestrian", 0},
+ {32, "living_street", 0}, {33, "road", 0},
+ {33, "unclassified", 0}, {33, "residential", 0},
+ {34, "tertiary", 0}, {36, "secondary", 1},
+ {37, "primary", 1}, {38, "trunk", 1},
+ {39, "motorway", 1}};
+
+static const unsigned int nLayers = (sizeof(layers) / sizeof(*layers));
+
+void add_z_order(taglist_t &tags, int *roads)
+{
+ const std::string *layer = tags.get("layer");
+ const std::string *highway = tags.get("highway");
+ bool bridge = tags.get_bool("bridge", false);
+ bool tunnel = tags.get_bool("tunnel", false);
+ const std::string *railway = tags.get("railway");
+ const std::string *boundary = tags.get("boundary");
+
+ int z_order = 0;
+
+ int l = layer ? (int)strtol(layer->c_str(), NULL, 10) : 0;
+ z_order = 100 * l;
+ *roads = 0;
+
+ if (highway) {
+ for (unsigned i = 0; i < nLayers; i++) {
+ if (!strcmp(layers[i].highway, highway->c_str())) {
+ z_order += layers[i].offset;
+ *roads = layers[i].roads;
+ break;
+ }
+ }
+ }
+
+ if (railway && !railway->empty()) {
+ z_order += 35;
+ *roads = 1;
+ }
+ /* Administrative boundaries are rendered at low zooms so we prefer to use the roads table */
+ if (boundary && *boundary == "administrative")
+ *roads = 1;
+
+ if (bridge)
+ z_order += 100;
+
+ if (tunnel)
+ z_order -= 100;
+
+ char z[13];
+ snprintf(z, sizeof(z), "%d", z_order);
+ tags.push_back(tag_t("z_order", z));
+}
+
+} // anonymous namespace
+
+c_tagtransform_t::c_tagtransform_t(options_t const *options)
+: m_options(options)
+{
+}
+
+bool c_tagtransform_t::check_key(std::vector<taginfo> const &infos,
+ char const *k, bool *filter, int *flags,
+ bool strict)
+{
+ //go through the actual tags found on the item and keep the ones in the export list
+ size_t i = 0;
+ for (; i < infos.size(); i++) {
+ const taginfo &info = infos[i];
+ if (info.flags & FLAG_DELETE) {
+ if (wildMatch(info.name.c_str(), k)) {
+ return false;
+ }
+ } else if (strcmp(info.name.c_str(), k) == 0) {
+ *filter = false;
+ *flags |= info.flags;
+
+ return true;
+ }
+ }
+
+ // if we didn't find any tags that we wanted to export
+ // and we aren't strictly adhering to the list
+ if (!strict) {
+ if (m_options->hstore_mode != HSTORE_NONE) {
+ /* ... but if hstore_match_only is set then don't take this
+ as a reason for keeping the object */
+ if (!m_options->hstore_match_only)
+ *filter = false;
+ /* with hstore, copy all tags... */
+ return true;
+ } else if (m_options->hstore_columns.size() > 0) {
+ /* does this column match any of the hstore column prefixes? */
+ size_t j = 0;
+ for (; j < m_options->hstore_columns.size(); ++j) {
+ if (boost::starts_with(k, m_options->hstore_columns[j])) {
+ /* ... but if hstore_match_only is set then don't take this
+ as a reason for keeping the object */
+ if (!m_options->hstore_match_only) {
+ *filter = false;
+ }
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+}
+
+bool c_tagtransform_t::filter_tags(osmium::OSMObject const &o, int *polygon,
+ int *roads, export_list const &exlist,
+ taglist_t &out_tags, bool strict)
+{
+ //assume we dont like this set of tags
+ bool filter = true;
+
+ int flags = 0;
+ int add_area_tag = 0;
+
+ auto export_type = o.type();
+ if (o.type() == osmium::item_type::relation) {
+ export_type = osmium::item_type::way;
+ }
+ const std::vector<taginfo> &infos = exlist.get(export_type);
+
+ /* We used to only go far enough to determine if it's a polygon or not,
+ but now we go through and filter stuff we don't need
+ pop each tag off and keep it in the temp list if we like it */
+ for (auto const &item : o.tags()) {
+ char const *k = item.key();
+ char const *v = item.value();
+ //if we want to do more than the export list says
+ if (!strict) {
+ if (o.type() == osmium::item_type::relation &&
+ strcmp("type", k) == 0) {
+ out_tags.emplace_back(k, v);
+ filter = false;
+ continue;
+ }
+ /* Allow named islands to appear as polygons */
+ if (strcmp("natural", k) == 0 && strcmp("coastline", v) == 0) {
+ add_area_tag = 1;
+
+ /* Discard natural=coastline tags (we render these from a shapefile instead) */
+ if (!m_options->keep_coastlines) {
+ continue;
+ }
+ }
+ }
+
+ //go through the actual tags found on the item and keep the ones in the export list
+ if (check_key(infos, k, &filter, &flags, strict)) {
+ out_tags.emplace_back(k, v);
+ }
+ }
+ if (m_options->extra_attributes && o.version() > 0) {
+ out_tags.add_attributes(o);
+ }
+
+ if (polygon) {
+ if (add_area_tag) {
+ /* If we need to force this as a polygon, append an area tag */
+ out_tags.push_dedupe(tag_t("area", "yes"));
+ *polygon = 1;
+ } else {
+ auto const *area = o.tags()["area"];
+ if (area)
+ *polygon = taglist_t::value_to_bool(area, flags & FLAG_POLYGON);
+ else
+ *polygon = flags & FLAG_POLYGON;
+ }
+ }
+
+ if (roads && !filter && (o.type() == osmium::item_type::way)) {
+ add_z_order(out_tags, roads);
+ }
+
+ return filter;
+}
+
+bool c_tagtransform_t::filter_rel_member_tags(
+ taglist_t const &rel_tags, osmium::memory::Buffer const &members,
+ rolelist_t const &member_roles, int *member_superseded, int *make_boundary,
+ int *make_polygon, int *roads, export_list const &exlist,
+ taglist_t &out_tags, bool allow_typeless)
+{
+ auto const &infos = exlist.get(osmium::item_type::way);
+ //if it has a relation figure out what kind it is
+ const std::string *type = rel_tags.get("type");
+ bool is_route = false, is_boundary = false, is_multipolygon = false;
+ if (type) {
+ //what kind of relation is it
+ if (*type == "route")
+ is_route = true;
+ else if (*type == "boundary")
+ is_boundary = true;
+ else if (*type == "multipolygon")
+ is_multipolygon = true;
+ else if (!allow_typeless)
+ return true;
+ } //you didnt have a type and it was required
+ else if (!allow_typeless) {
+ return true;
+ }
+
+ /* Clone tags from relation */
+ for (const auto &rel_tag : rel_tags) {
+ //copy the name tag as "route_name"
+ if (is_route && (rel_tag.key == "name"))
+ out_tags.push_dedupe(tag_t("route_name", rel_tag.value));
+ //copy all other tags except for "type"
+ if (rel_tag.key != "type")
+ out_tags.push_dedupe(rel_tag);
+ }
+
+ if (is_route) {
+ const std::string *netw = rel_tags.get("network");
+ int networknr = -1;
+
+ if (netw != nullptr) {
+ const std::string *state = rel_tags.get("state");
+ std::string statetype("yes");
+ if (state) {
+ if (*state == "alternate")
+ statetype = "alternate";
+ else if (*state == "connection")
+ statetype = "connection";
+ }
+ if (*netw == "lcn") {
+ networknr = 10;
+ out_tags.push_dedupe(tag_t("lcn", statetype));
+ } else if (*netw == "rcn") {
+ networknr = 11;
+ out_tags.push_dedupe(tag_t("rcn", statetype));
+ } else if (*netw == "ncn") {
+ networknr = 12;
+ out_tags.push_dedupe(tag_t("ncn", statetype));
+ } else if (*netw == "lwn") {
+ networknr = 20;
+ out_tags.push_dedupe(tag_t("lwn", statetype));
+ } else if (*netw == "rwn") {
+ networknr = 21;
+ out_tags.push_dedupe(tag_t("rwn", statetype));
+ } else if (*netw == "nwn") {
+ networknr = 22;
+ out_tags.push_dedupe(tag_t("nwn", statetype));
+ }
+ }
+
+ const std::string *prefcol = rel_tags.get("preferred_color");
+ if (prefcol != NULL && prefcol->size() == 1) {
+ if ((*prefcol)[0] == '0' || (*prefcol)[0] == '1' ||
+ (*prefcol)[0] == '2' || (*prefcol)[0] == '3' ||
+ (*prefcol)[0] == '4') {
+ out_tags.push_dedupe(tag_t("route_pref_color", *prefcol));
+ } else {
+ out_tags.push_dedupe(tag_t("route_pref_color", "0"));
+ }
+ } else {
+ out_tags.push_dedupe(tag_t("route_pref_color", "0"));
+ }
+
+ const std::string *relref = rel_tags.get("ref");
+ if (relref != NULL) {
+ if (networknr == 10) {
+ out_tags.push_dedupe(tag_t("lcn_ref", *relref));
+ } else if (networknr == 11) {
+ out_tags.push_dedupe(tag_t("rcn_ref", *relref));
+ } else if (networknr == 12) {
+ out_tags.push_dedupe(tag_t("ncn_ref", *relref));
+ } else if (networknr == 20) {
+ out_tags.push_dedupe(tag_t("lwn_ref", *relref));
+ } else if (networknr == 21) {
+ out_tags.push_dedupe(tag_t("rwn_ref", *relref));
+ } else if (networknr == 22) {
+ out_tags.push_dedupe(tag_t("nwn_ref", *relref));
+ }
+ }
+ } else if (is_boundary) {
+ /* Boundaries will get converted into multiple geometries:
+ - Linear features will end up in the line and roads tables (useful for admin boundaries)
+ - Polygon features also go into the polygon table (useful for national_forests)
+ The edges of the polygon also get treated as linear fetaures allowing these to be rendered seperately. */
+ *make_boundary = 1;
+ } else if (is_multipolygon && out_tags.contains("boundary")) {
+ /* Treat type=multipolygon exactly like type=boundary if it has a boundary tag. */
+ *make_boundary = 1;
+ } else if (is_multipolygon) {
+ *make_polygon = 1;
+
+ // Check if the relation has any polygon-like tags. In that case
+ // we have a new-style polygon.
+ bool newstyle_mp = false;
+ for (const auto &tag : out_tags) {
+ if (tag.key == "area") {
+ newstyle_mp = true;
+ } else {
+ for (const auto &info : infos) {
+ if (info.name == tag.key) {
+ newstyle_mp = info.flags & FLAG_POLYGON;
+ break;
+ }
+ }
+ }
+ if (newstyle_mp) {
+ break;
+ }
+ }
+
+ // Old-style MP: copy the tags from the outer way(s). Only use tags
+ // that appear in all outer rings.
+ if (!newstyle_mp) {
+ taglist_t poly_tags;
+ bool first_outerway = true;
+ size_t i = 0;
+ for (auto const &w : members.select<osmium::Way>()) {
+ if (member_roles[i] && strcmp(member_roles[i], "inner") == 0)
+ continue;
+
+ /* insert all tags of the first outerway to the potential list of copied tags. */
+ if (first_outerway) {
+ for (auto const &tag : w.tags()) {
+ poly_tags.emplace_back(tag.key(), tag.value());
+ }
+ first_outerway = false;
+ } else {
+ /* Check if all of the tags in the list of potential tags are present on this way,
+ otherwise remove from the list of potential tags. Tags need to be present on
+ all outer ways to be copied over to the relation */
+ auto it = poly_tags.begin();
+ while (it != poly_tags.end()) {
+ if (!w.tags().has_key(it->key.c_str()))
+ /* This tag is not present on all member outer ways, so don't copy it over to relation */
+ it = poly_tags.erase(it);
+ else
+ ++it;
+ }
+ }
+ ++i;
+ }
+ // Copy the list identified outer way tags over to the relation
+ // filtering for wanted tags on the way.
+ bool filter;
+ int flags;
+ for (const auto &poly_tag : poly_tags) {
+ if (check_key(infos, poly_tag.key.c_str(), &filter, &flags,
+ false)) {
+ out_tags.push_dedupe(poly_tag);
+ }
+ }
+
+ if (!(flags & FLAG_POLYGON)) {
+ out_tags.clear();
+ return true;
+ }
+ }
+ }
+
+ if (out_tags.empty()) {
+ return true;
+ }
+
+ /* If we are creating a multipolygon then we
+ mark each member so that we can skip them during iterate_ways
+ but only if the polygon-tags look the same as the outer ring */
+ if (make_polygon) {
+ size_t i = 0;
+ for (auto const &w : members.select<osmium::Way>()) {
+ member_superseded[i] = 1;
+ for (const auto &member_tag : w.tags()) {
+ auto const *v = out_tags.get(member_tag.key());
+ bool filt;
+ int flag;
+ if ((!v && check_key(infos, member_tag.key(), &filt, &flag, false))
+ || (v && *v != member_tag.value())) {
+ /* z_order and osm_ are automatically generated tags, so ignore them */
+ if (strcmp(member_tag.key(), "z_order") &&
+ strcmp(member_tag.key(), "osm_user") &&
+ strcmp(member_tag.key(), "osm_version") &&
+ strcmp(member_tag.key(), "osm_uid") &&
+ strcmp(member_tag.key(), "osm_changeset") &&
+ strcmp(member_tag.key(), "osm_timestamp")) {
+ member_superseded[i] = 0;
+ break;
+ }
+ }
+ }
+ ++i;
+ }
+ }
+
+ add_z_order(out_tags, roads);
+
+ return 0;
+}
diff --git a/tagtransform-c.hpp b/tagtransform-c.hpp
new file mode 100644
index 0000000..68fd7b8
--- /dev/null
+++ b/tagtransform-c.hpp
@@ -0,0 +1,31 @@
+#ifndef TAGTRANSFORM_C_H
+#define TAGTRANSFORM_C_H
+
+#include "taginfo_impl.hpp"
+#include "tagtransform.hpp"
+
+class c_tagtransform_t : public tagtransform_t
+{
+public:
+ c_tagtransform_t(options_t const *options);
+
+ bool filter_tags(osmium::OSMObject const &o, int *polygon, int *roads,
+ export_list const &exlist, taglist_t &out_tags,
+ bool strict = false) override;
+
+ bool filter_rel_member_tags(taglist_t const &rel_tags,
+ osmium::memory::Buffer const &members,
+ rolelist_t const &member_roles,
+ int *member_superseded, int *make_boundary,
+ int *make_polygon, int *roads,
+ export_list const &exlist, taglist_t &out_tags,
+ bool allow_typeless = false) override;
+
+private:
+ bool check_key(std::vector<taginfo> const &infos, char const *k,
+ bool *filter, int *flags, bool strict);
+
+ options_t const *m_options;
+};
+
+#endif // TAGTRANSFORM_C_H
diff --git a/tagtransform-lua.cpp b/tagtransform-lua.cpp
new file mode 100644
index 0000000..cf9021b
--- /dev/null
+++ b/tagtransform-lua.cpp
@@ -0,0 +1,202 @@
+extern "C" {
+#include <lauxlib.h>
+#include <lualib.h>
+}
+
+#include <boost/format.hpp>
+
+#include "options.hpp"
+#include "tagtransform-lua.hpp"
+
+lua_tagtransform_t::lua_tagtransform_t(options_t const *options)
+: L(luaL_newstate()), m_node_func(options->tag_transform_node_func.get_value_or(
+ "filter_tags_node")),
+ m_way_func(options->tag_transform_way_func.get_value_or("filter_tags_way")),
+ m_rel_func(
+ options->tag_transform_rel_func.get_value_or("filter_basic_tags_rel")),
+ m_rel_mem_func(options->tag_transform_rel_mem_func.get_value_or(
+ "filter_tags_relation_member")),
+ m_extra_attributes(options->extra_attributes)
+{
+ luaL_openlibs(L);
+ luaL_dofile(L, options->tag_transform_script->c_str());
+
+ check_lua_function_exists(m_node_func);
+ check_lua_function_exists(m_way_func);
+ check_lua_function_exists(m_rel_func);
+ check_lua_function_exists(m_rel_mem_func);
+}
+
+lua_tagtransform_t::~lua_tagtransform_t() { lua_close(L); }
+
+void lua_tagtransform_t::check_lua_function_exists(const std::string &func_name)
+{
+ lua_getglobal(L, func_name.c_str());
+ if (!lua_isfunction(L, -1)) {
+ throw std::runtime_error(
+ (boost::format(
+ "Tag transform style does not contain a function %1%") %
+ func_name)
+ .str());
+ }
+ lua_pop(L, 1);
+}
+
+bool lua_tagtransform_t::filter_tags(osmium::OSMObject const &o, int *polygon,
+ int *roads, export_list const &,
+ taglist_t &out_tags, bool)
+{
+ switch (o.type()) {
+ case osmium::item_type::node:
+ lua_getglobal(L, m_node_func.c_str());
+ break;
+ case osmium::item_type::way:
+ lua_getglobal(L, m_way_func.c_str());
+ break;
+ case osmium::item_type::relation:
+ lua_getglobal(L, m_rel_func.c_str());
+ break;
+ default:
+ throw std::runtime_error("Unknown OSM type");
+ }
+
+ lua_newtable(L); /* key value table */
+
+ lua_Integer sz = 0;
+ for (auto const &t : o.tags()) {
+ lua_pushstring(L, t.key());
+ lua_pushstring(L, t.value());
+ lua_rawset(L, -3);
+ ++sz;
+ }
+ if (m_extra_attributes && o.version() > 0) {
+ taglist_t tags;
+ tags.add_attributes(o);
+ for (auto const &t : tags) {
+ lua_pushstring(L, t.key.c_str());
+ lua_pushstring(L, t.value.c_str());
+ lua_rawset(L, -3);
+ }
+ sz += tags.size();
+ }
+
+ lua_pushinteger(L, sz);
+
+ if (lua_pcall(L, 2, (o.type() == osmium::item_type::way) ? 4 : 2, 0)) {
+ fprintf(stderr,
+ "Failed to execute lua function for basic tag processing: %s\n",
+ lua_tostring(L, -1));
+ /* lua function failed */
+ return 1;
+ }
+
+ if (o.type() == osmium::item_type::way) {
+ if (roads) {
+ *roads = (int)lua_tointeger(L, -1);
+ }
+ lua_pop(L, 1);
+ if (polygon) {
+ *polygon = (int)lua_tointeger(L, -1);
+ }
+ lua_pop(L, 1);
+ }
+
+ lua_pushnil(L);
+ while (lua_next(L, -2) != 0) {
+ const char *key = lua_tostring(L, -2);
+ const char *value = lua_tostring(L, -1);
+ out_tags.emplace_back(key, value);
+ lua_pop(L, 1);
+ }
+
+ bool filter = lua_tointeger(L, -2);
+
+ lua_pop(L, 2);
+
+ return filter;
+}
+
+bool lua_tagtransform_t::filter_rel_member_tags(
+ taglist_t const &rel_tags, osmium::memory::Buffer const &members,
+ rolelist_t const &member_roles, int *member_superseded, int *make_boundary,
+ int *make_polygon, int *roads, export_list const &, taglist_t &out_tags,
+ bool)
+{
+ size_t num_members = member_roles.size();
+ lua_getglobal(L, m_rel_mem_func.c_str());
+
+ lua_newtable(L); /* relations key value table */
+
+ for (const auto &rel_tag : rel_tags) {
+ lua_pushstring(L, rel_tag.key.c_str());
+ lua_pushstring(L, rel_tag.value.c_str());
+ lua_rawset(L, -3);
+ }
+
+ lua_newtable(L); /* member tags table */
+
+ int idx = 1;
+ for (auto const &w : members.select<osmium::Way>()) {
+ lua_pushnumber(L, idx++);
+ lua_newtable(L); /* member key value table */
+ for (auto const &member_tag : w.tags()) {
+ lua_pushstring(L, member_tag.key());
+ lua_pushstring(L, member_tag.value());
+ lua_rawset(L, -3);
+ }
+ lua_rawset(L, -3);
+ }
+
+ lua_newtable(L); /* member roles table */
+
+ for (size_t i = 0; i < num_members; ++i) {
+ lua_pushnumber(L, i + 1);
+ lua_pushstring(L, member_roles[i]);
+ lua_rawset(L, -3);
+ }
+
+ lua_pushnumber(L, num_members);
+
+ if (lua_pcall(L, 4, 6, 0)) {
+ fprintf(
+ stderr,
+ "Failed to execute lua function for relation tag processing: %s\n",
+ lua_tostring(L, -1));
+ /* lua function failed */
+ return 1;
+ }
+
+ *roads = (int)lua_tointeger(L, -1);
+ lua_pop(L, 1);
+ *make_polygon = (int)lua_tointeger(L, -1);
+ lua_pop(L, 1);
+ *make_boundary = (int)lua_tointeger(L, -1);
+ lua_pop(L, 1);
+
+ lua_pushnil(L);
+ for (size_t i = 0; i < num_members; ++i) {
+ if (lua_next(L, -2)) {
+ member_superseded[i] = (int)lua_tointeger(L, -1);
+ lua_pop(L, 1);
+ } else {
+ fprintf(stderr,
+ "Failed to read member_superseded from lua function\n");
+ }
+ }
+ lua_pop(L, 2);
+
+ lua_pushnil(L);
+ while (lua_next(L, -2) != 0) {
+ const char *key = lua_tostring(L, -2);
+ const char *value = lua_tostring(L, -1);
+ out_tags.push_back(tag_t(key, value));
+ lua_pop(L, 1);
+ }
+ lua_pop(L, 1);
+
+ bool filter = lua_tointeger(L, -1);
+
+ lua_pop(L, 1);
+
+ return filter;
+}
diff --git a/tagtransform-lua.hpp b/tagtransform-lua.hpp
new file mode 100644
index 0000000..0d951ac
--- /dev/null
+++ b/tagtransform-lua.hpp
@@ -0,0 +1,38 @@
+#ifndef TAGTRANSFORM_LUA_H
+#define TAGTRANSFORM_LUA_H
+
+#include <string>
+
+#include "tagtransform.hpp"
+
+extern "C" {
+#include <lua.h>
+}
+
+class lua_tagtransform_t : public tagtransform_t
+{
+public:
+ lua_tagtransform_t(options_t const *options);
+ ~lua_tagtransform_t();
+
+ bool filter_tags(osmium::OSMObject const &o, int *polygon, int *roads,
+ export_list const &exlist, taglist_t &out_tags,
+ bool strict = false) override;
+
+ bool filter_rel_member_tags(taglist_t const &rel_tags,
+ osmium::memory::Buffer const &members,
+ rolelist_t const &member_roles,
+ int *member_superseded, int *make_boundary,
+ int *make_polygon, int *roads,
+ export_list const &exlist, taglist_t &out_tags,
+ bool allow_typeless = false) override;
+
+private:
+ void check_lua_function_exists(std::string const &func_name);
+
+ lua_State *L;
+ std::string m_node_func, m_way_func, m_rel_func, m_rel_mem_func;
+ bool m_extra_attributes;
+};
+
+#endif // TAGTRANSFORM_LUA_H
diff --git a/tagtransform.cpp b/tagtransform.cpp
index 0c92986..5e3fdb3 100644
--- a/tagtransform.cpp
+++ b/tagtransform.cpp
@@ -1,673 +1,30 @@
-#include <cstddef>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <stdexcept>
-#include <vector>
-#include <boost/format.hpp>
-
#include "tagtransform.hpp"
-#include "options.hpp"
#include "config.h"
-#include "wildcmp.hpp"
-#include "taginfo_impl.hpp"
-
-#ifdef HAVE_LUA
-extern "C" {
- #include <lualib.h>
- #include <lauxlib.h>
-}
-#endif
-
-
-static const struct {
- int offset;
- const char *highway;
- int roads;
-} layers[] = {
- { 1, "proposed", 0 },
- { 2, "construction", 0 },
- { 10, "steps", 0 },
- { 10, "cycleway", 0 },
- { 10, "bridleway", 0 },
- { 10, "footway", 0 },
- { 10, "path", 0 },
- { 11, "track", 0 },
- { 15, "service", 0 },
-
- { 24, "tertiary_link", 0 },
- { 25, "secondary_link",1 },
- { 27, "primary_link", 1 },
- { 28, "trunk_link", 1 },
- { 29, "motorway_link", 1 },
-
- { 30, "raceway", 0 },
- { 31, "pedestrian", 0 },
- { 32, "living_street", 0 },
- { 33, "road", 0 },
- { 33, "unclassified", 0 },
- { 33, "residential", 0 },
- { 34, "tertiary", 0 },
- { 36, "secondary", 1 },
- { 37, "primary", 1 },
- { 38, "trunk", 1 },
- { 39, "motorway", 1 }
-};
-
-static const unsigned int nLayers = (sizeof(layers)/sizeof(*layers));
-
-namespace {
-void add_z_order(taglist_t &tags, int *roads)
-{
- const std::string *layer = tags.get("layer");
- const std::string *highway = tags.get("highway");
- bool bridge = tags.get_bool("bridge", false);
- bool tunnel = tags.get_bool("tunnel", false);
- const std::string *railway = tags.get("railway");
- const std::string *boundary = tags.get("boundary");
-
- int z_order = 0;
-
- int l = layer ? strtol(layer->c_str(), NULL, 10) : 0;
- z_order = 100 * l;
- *roads = 0;
-
- if (highway) {
- for (unsigned i = 0; i < nLayers; i++) {
- if (!strcmp(layers[i].highway, highway->c_str())) {
- z_order += layers[i].offset;
- *roads = layers[i].roads;
- break;
- }
- }
- }
-
- if (railway && !railway->empty()) {
- z_order += 35;
- *roads = 1;
- }
- /* Administrative boundaries are rendered at low zooms so we prefer to use the roads table */
- if (boundary && *boundary == "administrative")
- *roads = 1;
-
- if (bridge)
- z_order += 100;
-
- if (tunnel)
- z_order -= 100;
-
- char z[13];
- snprintf(z, sizeof(z), "%d", z_order);
- tags.push_back(tag_t("z_order", z));
-}
-
-unsigned int c_filter_rel_member_tags(const taglist_t &rel_tags,
- const multitaglist_t &member_tags, const rolelist_t &member_roles,
- int *member_superseeded, int *make_boundary, int *make_polygon, int *roads,
- const export_list &exlist, taglist_t &out_tags, bool allow_typeless)
-{
- //if it has a relation figure out what kind it is
- const std::string *type = rel_tags.get("type");
- bool is_route = false, is_boundary = false, is_multipolygon = false;
- if (type)
- {
- //what kind of relation is it
- if (*type == "route")
- is_route = true;
- else if (*type == "boundary")
- is_boundary = true;
- else if (*type == "multipolygon")
- is_multipolygon = true;
- }//you didnt have a type and it was required
- else if (!allow_typeless)
- {
- return 1;
- }
-
- /* Clone tags from relation */
- for (const auto& rel_tag: rel_tags) {
- //copy the name tag as "route_name"
- if (is_route && (rel_tag.key == "name"))
- out_tags.push_dedupe(tag_t("route_name", rel_tag.value));
- //copy all other tags except for "type"
- if (rel_tag.key != "type")
- out_tags.push_dedupe(rel_tag);
- }
-
- if (is_route) {
- const std::string *netw = rel_tags.get("network");
- int networknr = -1;
-
- if (netw != nullptr) {
- const std::string *state = rel_tags.get("state");
- std::string statetype("yes");
- if (state) {
- if (*state == "alternate")
- statetype = "alternate";
- else if (*state == "connection")
- statetype = "connection";
- }
- if (*netw == "lcn") {
- networknr = 10;
- out_tags.push_dedupe(tag_t("lcn", statetype));
- } else if (*netw == "rcn") {
- networknr = 11;
- out_tags.push_dedupe(tag_t("rcn", statetype));
- } else if (*netw == "ncn") {
- networknr = 12;
- out_tags.push_dedupe(tag_t("ncn", statetype));
- } else if (*netw == "lwn") {
- networknr = 20;
- out_tags.push_dedupe(tag_t("lwn", statetype));
- } else if (*netw == "rwn") {
- networknr = 21;
- out_tags.push_dedupe(tag_t("rwn", statetype));
- } else if (*netw == "nwn") {
- networknr = 22;
- out_tags.push_dedupe(tag_t("nwn", statetype));
- }
- }
-
- const std::string *prefcol = rel_tags.get("preferred_color");
- if (prefcol != NULL && prefcol->size() == 1) {
- if ((*prefcol)[0] == '0' || (*prefcol)[0] == '1'
- || (*prefcol)[0] == '2' || (*prefcol)[0] == '3'
- || (*prefcol)[0] == '4') {
- out_tags.push_dedupe(tag_t("route_pref_color", *prefcol));
- } else {
- out_tags.push_dedupe(tag_t("route_pref_color", "0"));
- }
- } else {
- out_tags.push_dedupe(tag_t("route_pref_color", "0"));
- }
-
- const std::string *relref = rel_tags.get("ref");
- if (relref != NULL ) {
- if (networknr == 10) {
- out_tags.push_dedupe(tag_t("lcn_ref", *relref));
- } else if (networknr == 11) {
- out_tags.push_dedupe(tag_t("rcn_ref", *relref));
- } else if (networknr == 12) {
- out_tags.push_dedupe(tag_t("ncn_ref", *relref));
- } else if (networknr == 20) {
- out_tags.push_dedupe(tag_t("lwn_ref", *relref));
- } else if (networknr == 21) {
- out_tags.push_dedupe(tag_t("rwn_ref", *relref));
- } else if (networknr == 22) {
- out_tags.push_dedupe(tag_t("nwn_ref", *relref));
- }
- }
- } else if (is_boundary) {
- /* Boundaries will get converted into multiple geometries:
- - Linear features will end up in the line and roads tables (useful for admin boundaries)
- - Polygon features also go into the polygon table (useful for national_forests)
- The edges of the polygon also get treated as linear fetaures allowing these to be rendered seperately. */
- *make_boundary = 1;
- } else if (is_multipolygon && out_tags.contains("boundary")) {
- /* Treat type=multipolygon exactly like type=boundary if it has a boundary tag. */
- *make_boundary = 1;
- } else if (is_multipolygon) {
- *make_polygon = 1;
-
- /* Collect a list of polygon-like tags, these are used later to
- identify if an inner rings looks like it should be rendered separately */
- taglist_t poly_tags;
- for (const auto& tag: out_tags) {
- if (tag.key == "area") {
- poly_tags.push_back(tag);
- } else {
- const std::vector<taginfo> &infos = exlist.get(OSMTYPE_WAY);
- for (const auto& info: infos) {
- if (info.name == tag.key) {
- if (info.flags & FLAG_POLYGON) {
- poly_tags.push_back(tag);
- }
- break;
- }
- }
- }
- }
-
- /* Copy the tags from the outer way(s) if the relation is untagged (with
- * respect to tags that influence its polygon nature. Tags like name or fixme should be fine*/
- if (poly_tags.empty()) {
- int first_outerway = 1;
- for (size_t i = 0; i < member_tags.size(); i++) {
- if (member_roles[i] && *(member_roles[i]) == "inner")
- continue;
-
- /* insert all tags of the first outerway to the potential list of copied tags. */
- if (first_outerway) {
- for (const auto& tag: member_tags[i]) {
- poly_tags.push_back(tag);
- }
- } else {
- /* Check if all of the tags in the list of potential tags are present on this way,
- otherwise remove from the list of potential tags. Tags need to be present on
- all outer ways to be copied over to the relation */
- taglist_t::iterator it = poly_tags.begin();
- while (it != poly_tags.end()) {
- if (!member_tags[i].contains(it->key))
- /* This tag is not present on all member outer ways, so don't copy it over to relation */
- it = poly_tags.erase(it);
- else
- ++it;
- }
- }
- first_outerway = 0;
- }
- /* Copy the list identified outer way tags over to the relation */
- for (const auto& poly_tag: poly_tags) {
- out_tags.push_dedupe(poly_tag);
- }
-
- /* We need to re-check and only keep polygon tags in the list of polytags */
- // TODO what is that for? The list is cleared just below.
- taglist_t::iterator q = poly_tags.begin();
- const std::vector<taginfo> &infos = exlist.get(OSMTYPE_WAY);
- while (q != poly_tags.end()) {
- bool contains_tag = false;
- for (std::vector<taginfo>::const_iterator info = infos.begin();
- info != infos.end(); ++info) {
- if (info->name == q->key) {
- if (info->flags & FLAG_POLYGON) {
- contains_tag = true;
- }
- break;
- }
- }
-
- if (contains_tag)
- ++q;
- else
- q = poly_tags.erase(q);
- }
- }
- } else if(!allow_typeless) {
- /* Unknown type, just exit */
- out_tags.clear();
- return 1;
- }
-
- if (out_tags.empty()) {
- return 1;
- }
-
- /* If we are creating a multipolygon then we
- mark each member so that we can skip them during iterate_ways
- but only if the polygon-tags look the same as the outer ring */
- if (make_polygon) {
- for (size_t i = 0; i < member_tags.size(); i++) {
- member_superseeded[i] = 1;
- for (const auto& member_tag: member_tags[i]) {
- const std::string *v = out_tags.get(member_tag.key);
- if (!v || *v != member_tag.value) {
- /* z_order and osm_ are automatically generated tags, so ignore them */
- if ((member_tag.key != "z_order") && (member_tag.key != "osm_user") &&
- (member_tag.key != "osm_version") && (member_tag.key != "osm_uid") &&
- (member_tag.key != "osm_changeset") && (member_tag.key != "osm_timestamp")) {
- member_superseeded[i] = 0;
- break;
- }
- }
- }
- }
- }
-
- add_z_order(out_tags, roads);
-
- return 0;
-}
-} // anonymous namespace
-
-#ifdef HAVE_LUA
-unsigned tagtransform::lua_filter_rel_member_tags(const taglist_t &rel_tags,
- const multitaglist_t &members_tags, const rolelist_t &member_roles,
- int *member_superseeded, int *make_boundary, int *make_polygon, int *roads,
- taglist_t &out_tags)
-{
- lua_getglobal(L, m_rel_mem_func.c_str());
-
- lua_newtable(L); /* relations key value table */
-
- for (const auto& rel_tag: rel_tags) {
- lua_pushstring(L, rel_tag.key.c_str());
- lua_pushstring(L, rel_tag.value.c_str());
- lua_rawset(L, -3);
- }
-
- lua_newtable(L); /* member tags table */
-
- int idx = 1;
- for (const auto& member_tags: members_tags) {
- lua_pushnumber(L, idx++);
- lua_newtable(L); /* member key value table */
- for (const auto& member_tag: member_tags) {
- lua_pushstring(L, member_tag.key.c_str());
- lua_pushstring(L, member_tag.value.c_str());
- lua_rawset(L, -3);
- }
- lua_rawset(L, -3);
- }
-
- lua_newtable(L); /* member roles table */
-
- for (size_t i = 0; i < member_roles.size(); i++) {
- lua_pushnumber(L, i + 1);
- lua_pushstring(L, member_roles[i]->c_str());
- lua_rawset(L, -3);
- }
-
- lua_pushnumber(L, member_roles.size());
-
- if (lua_pcall(L,4,6,0)) {
- fprintf(stderr, "Failed to execute lua function for relation tag processing: %s\n", lua_tostring(L, -1));
- /* lua function failed */
- return 1;
- }
-
- *roads = lua_tointeger(L, -1);
- lua_pop(L,1);
- *make_polygon = lua_tointeger(L, -1);
- lua_pop(L,1);
- *make_boundary = lua_tointeger(L,-1);
- lua_pop(L,1);
-
- lua_pushnil(L);
- for (size_t i = 0; i < members_tags.size(); i++) {
- if (lua_next(L,-2)) {
- member_superseeded[i] = lua_tointeger(L,-1);
- lua_pop(L,1);
- } else {
- fprintf(stderr, "Failed to read member_superseeded from lua function\n");
- }
- }
- lua_pop(L,2);
-
- lua_pushnil(L);
- while (lua_next(L,-2) != 0) {
- const char *key = lua_tostring(L,-2);
- const char *value = lua_tostring(L,-1);
- out_tags.push_back(tag_t(key, value));
- lua_pop(L,1);
- }
- lua_pop(L,1);
-
- int filter = lua_tointeger(L, -1);
-
- lua_pop(L,1);
-
- return filter;
-}
-
-void tagtransform::check_lua_function_exists(const std::string &func_name)
-{
- lua_getglobal(L, func_name.c_str());
- if (!lua_isfunction (L, -1)) {
- throw std::runtime_error((boost::format("Tag transform style does not contain a function %1%")
- % func_name).str());
- }
- lua_pop(L,1);
-}
-#endif
-
-tagtransform::tagtransform(const options_t *options_)
- : options(options_), transform_method(options_->tag_transform_script)
-#ifdef HAVE_LUA
- , L(NULL)
- , m_node_func( options->tag_transform_node_func. get_value_or("filter_tags_node"))
- , m_way_func( options->tag_transform_way_func. get_value_or("filter_tags_way"))
- , m_rel_func( options->tag_transform_rel_func. get_value_or("filter_basic_tags_rel"))
- , m_rel_mem_func(options->tag_transform_rel_mem_func.get_value_or("filter_tags_relation_member"))
-#endif /* HAVE_LUA */
-{
- if (transform_method) {
- fprintf(stderr, "Using lua based tag processing pipeline with script %s\n", options->tag_transform_script->c_str());
-#ifdef HAVE_LUA
- L = luaL_newstate();
- luaL_openlibs(L);
- luaL_dofile(L, options->tag_transform_script->c_str());
-
- check_lua_function_exists(m_node_func);
- check_lua_function_exists(m_way_func);
- check_lua_function_exists(m_rel_func);
- check_lua_function_exists(m_rel_mem_func);
-#else
- throw std::runtime_error("Error: Could not init lua tag transform, as lua support was not compiled into this version");
-#endif
- } else {
- fprintf(stderr, "Using built-in tag processing pipeline\n");
- }
-}
+#include "options.hpp"
+#include "tagtransform-c.hpp"
-tagtransform::~tagtransform() {
#ifdef HAVE_LUA
- if (transform_method) {
- lua_close(L);
- }
+#include "tagtransform-lua.hpp"
#endif
-}
-
-unsigned int tagtransform::filter_node_tags(const taglist_t &tags, const export_list &exlist,
- taglist_t &out_tags, bool strict)
-{
- if (transform_method) {
- return lua_filter_basic_tags(OSMTYPE_NODE, tags, 0, 0, out_tags);
- } else {
- return c_filter_basic_tags(OSMTYPE_NODE, tags, 0, 0, exlist, out_tags, strict);
- }
-}
-
-/*
- * This function gets called twice during initial import per way. Once from add_way and once from out_way
- */
-unsigned tagtransform::filter_way_tags(const taglist_t &tags, int *polygon, int *roads,
- const export_list &exlist, taglist_t &out_tags, bool strict)
-{
- if (transform_method) {
- return lua_filter_basic_tags(OSMTYPE_WAY, tags, polygon, roads, out_tags);
- } else {
- return c_filter_basic_tags(OSMTYPE_WAY, tags, polygon, roads, exlist, out_tags, strict);
- }
-}
-unsigned tagtransform::filter_rel_tags(const taglist_t &tags, const export_list &exlist,
- taglist_t &out_tags, bool strict)
+std::unique_ptr<tagtransform_t>
+tagtransform_t::make_tagtransform(options_t const *options)
{
- if (transform_method) {
- return lua_filter_basic_tags(OSMTYPE_RELATION, tags, 0, 0, out_tags);
- } else {
- return c_filter_basic_tags(OSMTYPE_RELATION, tags, 0, 0, exlist, out_tags, strict);
- }
-}
-
-unsigned tagtransform::filter_rel_member_tags(const taglist_t &rel_tags,
- const multitaglist_t &member_tags, const rolelist_t &member_roles,
- int *member_superseeded, int *make_boundary, int *make_polygon, int *roads,
- const export_list &exlist, taglist_t &out_tags, bool allow_typeless)
-{
- if (transform_method) {
+ if (options->tag_transform_script) {
#ifdef HAVE_LUA
- return lua_filter_rel_member_tags(rel_tags, member_tags, member_roles, member_superseeded, make_boundary, make_polygon, roads, out_tags);
+ fprintf(stderr,
+ "Using lua based tag processing pipeline with script %s\n",
+ options->tag_transform_script->c_str());
+ return std::unique_ptr<tagtransform_t>(new lua_tagtransform_t(options));
#else
- return 1;
+ throw std::runtime_error("Error: Could not init lua tag transform, as "
+ "lua support was not compiled into this "
+ "version");
#endif
- } else {
- return c_filter_rel_member_tags(rel_tags, member_tags, member_roles, member_superseeded, make_boundary, make_polygon, roads, exlist, out_tags, allow_typeless);
- }
-}
-
-unsigned tagtransform::lua_filter_basic_tags(OsmType type, const taglist_t &tags,
- int *polygon, int *roads, taglist_t &out_tags)
-{
-#ifdef HAVE_LUA
- switch (type) {
- case OSMTYPE_NODE: {
- lua_getglobal(L, m_node_func.c_str());
- break;
- }
- case OSMTYPE_WAY: {
- lua_getglobal(L, m_way_func.c_str());
- break;
- }
- case OSMTYPE_RELATION: {
- lua_getglobal(L, m_rel_func.c_str());
- break;
}
- }
-
- lua_newtable(L); /* key value table */
- for (taglist_t::const_iterator it = tags.begin(); it != tags.end(); ++it) {
- lua_pushstring(L, it->key.c_str());
- lua_pushstring(L, it->value.c_str());
- lua_rawset(L, -3);
- }
-
- lua_pushinteger(L, tags.size());
-
- if (lua_pcall(L,2,type == OSMTYPE_WAY ? 4 : 2,0)) {
- fprintf(stderr, "Failed to execute lua function for basic tag processing: %s\n", lua_tostring(L, -1));
- /* lua function failed */
- return 1;
- }
-
- if (type == OSMTYPE_WAY) {
- assert(roads);
- *roads = lua_tointeger(L, -1);
- lua_pop(L,1);
- assert(polygon);
- *polygon = lua_tointeger(L, -1);
- lua_pop(L,1);
- }
-
- lua_pushnil(L);
- while (lua_next(L,-2) != 0) {
- const char *key = lua_tostring(L,-2);
- const char *value = lua_tostring(L,-1);
- out_tags.push_back(tag_t(key, value));
- lua_pop(L,1);
- }
-
- int filter = lua_tointeger(L, -2);
-
- lua_pop(L,2);
-
- return filter;
-#else
- return 1;
-#endif
+ fprintf(stderr, "Using built-in tag processing pipeline\n");
+ return std::unique_ptr<tagtransform_t>(new c_tagtransform_t(options));
}
-/* Go through the given tags and determine the union of flags. Also remove
- * any tags from the list that we don't know about */
-unsigned int tagtransform::c_filter_basic_tags(OsmType type, const taglist_t &tags, int *polygon,
- int *roads, const export_list &exlist,
- taglist_t &out_tags, bool strict)
-{
- //assume we dont like this set of tags
- int filter = 1;
-
- int flags = 0;
- int add_area_tag = 0;
-
- OsmType export_type;
- if (type == OSMTYPE_RELATION) {
- export_type = OSMTYPE_WAY;
- } else {
- export_type = type;
- }
- const std::vector<taginfo> &infos = exlist.get(export_type);
-
- /* We used to only go far enough to determine if it's a polygon or not,
- but now we go through and filter stuff we don't need
- pop each tag off and keep it in the temp list if we like it */
- for (taglist_t::const_iterator item = tags.begin(); item != tags.end(); ++item) {
- //if we want to do more than the export list says
- if(!strict) {
- if (type == OSMTYPE_RELATION && "type" == item->key) {
- out_tags.push_back(*item);
- filter = 0;
- continue;
- }
- /* Allow named islands to appear as polygons */
- if ("natural" == item->key && "coastline" == item->value) {
- add_area_tag = 1;
-
- /* Discard natural=coastline tags (we render these from a shapefile instead) */
- if (!options->keep_coastlines) {
- continue;
- }
- }
- }
-
- //go through the actual tags found on the item and keep the ones in the export list
- size_t i = 0;
- for (; i < infos.size(); i++) {
- const taginfo &info = infos[i];
- if (info.flags & FLAG_DELETE) {
- if (wildMatch(info.name.c_str(), item->key.c_str())) {
- break;
- }
- } else if (info.name == item->key) {
- filter = 0;
- flags |= info.flags;
-
- out_tags.push_back(*item);
- break;
- }
- }
-
- // if we didn't find any tags that we wanted to export
- // and we aren't strictly adhering to the list
- if (i == infos.size() && !strict) {
- if (options->hstore_mode != HSTORE_NONE) {
- /* with hstore, copy all tags... */
- out_tags.push_back(*item);
- /* ... but if hstore_match_only is set then don't take this
- as a reason for keeping the object */
- if (!options->hstore_match_only && "osm_uid" != item->key
- && "osm_user" != item->key
- && "osm_timestamp" != item->key
- && "osm_version" != item->key
- && "osm_changeset" != item->key)
- filter = 0;
- } else if (options->hstore_columns.size() > 0) {
- /* does this column match any of the hstore column prefixes? */
- size_t j = 0;
- for(; j < options->hstore_columns.size(); ++j) {
- size_t pos = item->key.find(options->hstore_columns[j]);
- if (pos == 0) {
- out_tags.push_back(*item);
- /* ... but if hstore_match_only is set then don't take this
- as a reason for keeping the object */
- if (!options->hstore_match_only
- && "osm_uid" != item->key
- && "osm_user" != item->key
- && "osm_timestamp" != item->key
- && "osm_version" != item->key
- && "osm_changeset" != item->key)
- filter = 0;
- break;
- }
- }
- }
- }
- }
-
- if (polygon) {
- if (add_area_tag) {
- /* If we need to force this as a polygon, append an area tag */
- out_tags.push_dedupe(tag_t("area", "yes"));
- *polygon = 1;
- } else {
- *polygon = tags.get_bool("area", flags & FLAG_POLYGON);
- }
- }
-
- if (roads && !filter && (type == OSMTYPE_WAY)) {
- add_z_order(out_tags, roads);
- }
-
- return filter;
-}
+tagtransform_t::~tagtransform_t() = default;
diff --git a/tagtransform.hpp b/tagtransform.hpp
index cbce2ca..086a8cc 100644
--- a/tagtransform.hpp
+++ b/tagtransform.hpp
@@ -1,59 +1,35 @@
-
#ifndef TAGTRANSFORM_H
#define TAGTRANSFORM_H
-#include "config.h"
-#include "osmtypes.hpp"
-
#include <string>
-struct options_t;
-struct export_list;
-
-#ifdef HAVE_LUA
-extern "C" {
- #include <lua.h>
-}
-#endif
+#include <osmium/memory/buffer.hpp>
+#include "osmtypes.hpp"
+struct options_t;
+struct export_list;
-class tagtransform {
+class tagtransform_t
+{
public:
- tagtransform(const options_t *options_);
- ~tagtransform();
-
- unsigned filter_node_tags(const taglist_t &tags, const export_list &exlist,
- taglist_t &out_tags, bool strict = false);
- unsigned filter_way_tags(const taglist_t &tags, int *polygon, int *roads,
- const export_list &exlist, taglist_t &out_tags, bool strict = false);
- unsigned filter_rel_tags(const taglist_t &tags, const export_list &exlist,
- taglist_t &out_tags, bool strict = false);
- unsigned filter_rel_member_tags(const taglist_t &rel_tags,
- const multitaglist_t &member_tags, const rolelist_t &member_roles,
- int *member_superseeded, int *make_boundary, int *make_polygon, int *roads,
- const export_list &exlist, taglist_t &out_tags, bool allow_typeless = false);
-
-private:
- unsigned lua_filter_basic_tags(OsmType type, const taglist_t &tags,
- int *polygon, int *roads, taglist_t &out_tags);
- unsigned c_filter_basic_tags(OsmType type, const taglist_t &tags, int *polygon,
- int *roads, const export_list &exlist,
- taglist_t &out_tags, bool strict);
- unsigned int lua_filter_rel_member_tags(const taglist_t &rel_tags,
- const multitaglist_t &members_tags, const rolelist_t &member_roles,
- int *member_superseeded, int *make_boundary, int *make_polygon, int *roads,
- taglist_t &out_tags);
- void check_lua_function_exists(const std::string &func_name);
-
-
- const options_t* options;
- const bool transform_method;
-#ifdef HAVE_LUA
- lua_State *L;
- const std::string m_node_func, m_way_func, m_rel_func, m_rel_mem_func;
-#endif
-
+ static std::unique_ptr<tagtransform_t>
+ make_tagtransform(options_t const *options);
+
+ virtual ~tagtransform_t() = 0;
+
+ virtual bool filter_tags(osmium::OSMObject const &o, int *polygon,
+ int *roads, export_list const &exlist,
+ taglist_t &out_tags, bool strict = false) = 0;
+
+ virtual bool filter_rel_member_tags(taglist_t const &rel_tags,
+ osmium::memory::Buffer const &members,
+ rolelist_t const &member_roles,
+ int *member_superseded,
+ int *make_boundary, int *make_polygon,
+ int *roads, export_list const &exlist,
+ taglist_t &out_tags,
+ bool allow_typeless = false) = 0;
};
#endif //TAGTRANSFORM_H
diff --git a/tests/000466354.osc.gz b/tests/000466354.osc.gz
index b5e0333..21b56b8 100644
Binary files a/tests/000466354.osc.gz and b/tests/000466354.osc.gz differ
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 1716ad4..f1dd0f8 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -28,6 +28,7 @@ set(TESTS
test-output-pgsql.cpp
test-parse-diff.cpp
test-parse-xml2.cpp
+ test-persistent-node-cache.cpp
test-pgsql-escape.cpp
test-wildcard-match.cpp
)
diff --git a/tests/common-pg.cpp b/tests/common-pg.cpp
index 7b5d4dd..3f96694 100644
--- a/tests/common-pg.cpp
+++ b/tests/common-pg.cpp
@@ -5,6 +5,7 @@
#include <cstdarg>
#include <cstdio>
#include <cstdlib>
+#include <iostream>
#include <unistd.h>
#include <memory>
@@ -83,8 +84,9 @@ tempdb::tempdb()
: m_postgres_conn(conn::connect("dbname=postgres"))
{
result_ptr res = nullptr;
+ // osm2pgsql doesn't require a db name, but subsequent uses of database_options.db.get() in this file assume one is set
database_options.db = (boost::format("osm2pgsql-test-%1%-%2%") % getpid() % time(nullptr)).str();
- m_postgres_conn->exec(boost::format("DROP DATABASE IF EXISTS \"%1%\"") % database_options.db);
+ m_postgres_conn->exec(boost::format("DROP DATABASE IF EXISTS \"%1%\"") % database_options.db.get());
//tests can be run concurrently which means that this query can collide with other similar ones
//so we implement a simple retry here to get around the case that they do collide if we dont
//we often fail due to both trying to access template1 at the same time
@@ -93,7 +95,7 @@ tempdb::tempdb()
while(status != PGRES_COMMAND_OK && retries++ < 20)
{
sleep(1);
- res = m_postgres_conn->exec(boost::format("CREATE DATABASE \"%1%\" WITH ENCODING 'UTF8'") % database_options.db);
+ res = m_postgres_conn->exec(boost::format("CREATE DATABASE \"%1%\" WITH ENCODING 'UTF8'") % database_options.db.get());
status = PQresultStatus(res->get());
}
if (PQresultStatus(res->get()) != PGRES_COMMAND_OK) {
@@ -216,7 +218,7 @@ void tempdb::assert_has_table(const std::string &table_name) {
tempdb::~tempdb() {
m_conn.reset(); // close the connection to the db
if (m_postgres_conn) {
- m_postgres_conn->exec(boost::format("DROP DATABASE IF EXISTS \"%1%\"") % database_options.db);
+ m_postgres_conn->exec(boost::format("DROP DATABASE IF EXISTS \"%1%\"") % database_options.db.get());
}
}
diff --git a/tests/common.hpp b/tests/common.hpp
index 3f602f6..638160f 100644
--- a/tests/common.hpp
+++ b/tests/common.hpp
@@ -10,8 +10,7 @@ namespace testing {
void parse(const char *filename, const char *format,
const options_t &options, osmdata_t *osmdata)
{
- parse_osmium_t parser(options.extra_attributes, options.bbox,
- options.projection.get(), options.append, osmdata);
+ parse_osmium_t parser(options.bbox, options.append, osmdata);
osmdata->start();
parser.stream_file(filename, format);
diff --git a/tests/liechtenstein-2013-08-03.osm.pbf b/tests/liechtenstein-2013-08-03.osm.pbf
index 2bbc7d8..9032c1c 100644
Binary files a/tests/liechtenstein-2013-08-03.osm.pbf and b/tests/liechtenstein-2013-08-03.osm.pbf differ
diff --git a/tests/middle-tests.cpp b/tests/middle-tests.cpp
index 59c0d6d..f495e41 100644
--- a/tests/middle-tests.cpp
+++ b/tests/middle-tests.cpp
@@ -7,235 +7,280 @@
#include <tuple>
#include "osmtypes.hpp"
+#include "reprojection.hpp"
#include "tests/middle-tests.hpp"
+#include <osmium/builder/attr.hpp>
+#include <osmium/memory/buffer.hpp>
+
#define BLOCK_SHIFT 13
#define PER_BLOCK (((osmid_t)1) << BLOCK_SHIFT)
-struct expected_node {
- osmid_t id;
- double lon;
- double lat;
-
- expected_node() : id(0), lon(NAN), lat(NAN) {}
+// simple osmium buffer to store all the objects in
+osmium::memory::Buffer buffer(4096, osmium::memory::Buffer::auto_grow::yes);
+// do not project anything
+std::shared_ptr<reprojection>
+ proj(reprojection::create_projection(PROJ_LATLONG));
- expected_node(osmid_t id, double x, double y) : id(id), lon(x), lat(y) {}
-};
+#define ALLOWED_ERROR 10e-9
+bool node_okay(osmium::Location loc, osmium::Node const &expected)
+{
+ if ((loc.lat() > expected.location().lat() + ALLOWED_ERROR) ||
+ (loc.lat() < expected.location().lat() - ALLOWED_ERROR)) {
+ std::cerr << "ERROR: Node should have lat=" << expected.location().lat()
+ << ", but got back " << loc.lat() << " from middle.\n";
+ return false;
+ }
+ if ((loc.lon() > expected.location().lon() + ALLOWED_ERROR) ||
+ (loc.lon() < expected.location().lon() - ALLOWED_ERROR)) {
+ std::cerr << "ERROR: Node should have lon=" << expected.location().lon()
+ << ", but got back " << loc.lon() << " from middle.\n";
+ return false;
+ }
+ return true;
+}
-typedef std::vector<expected_node> expected_nodelist_t;
+size_t add_node(osmid_t id, double lat, double lon)
+{
+ using namespace osmium::builder::attr;
+ return osmium::builder::add_node(buffer, _id(id), _location(lon, lat));
+}
-#define ALLOWED_ERROR 10e-9
-bool node_okay(osmNode node, expected_node expected) {
- if ((node.lat > expected.lat + ALLOWED_ERROR) || (node.lat < expected.lat - ALLOWED_ERROR)) {
- std::cerr << "ERROR: Node should have lat=" << expected.lat << ", but got back "
- << node.lat << " from middle.\n";
- return false;
- }
- if ((node.lon > expected.lon + ALLOWED_ERROR) || (node.lon < expected.lon - ALLOWED_ERROR)) {
- std::cerr << "ERROR: Node should have lon=" << expected.lon << ", but got back "
- << node.lon << " from middle.\n";
- return false;
- }
- return true;
+size_t way_with_nodes(std::vector<osmid_t> const &ids)
+{
+ using namespace osmium::builder::attr;
+ return osmium::builder::add_way(buffer, _nodes(ids));
}
int test_node_set(middle_t *mid)
{
- idlist_t ids;
- expected_node expected(1234, 12.3456789, 98.7654321);
- taglist_t tags;
- nodelist_t nodes;
+ buffer.clear();
- // set the node
- mid->nodes_set(expected.id, expected.lat, expected.lon, tags);
+ auto const &node = buffer.get<osmium::Node>(add_node(1234, 12.3456789, 98.7654321));
+ auto &way = buffer.get<osmium::Way>(way_with_nodes({node.id()}));
- // get it back
- ids.push_back(expected.id);
- if (mid->nodes_get_list(nodes, ids) != ids.size()) { std::cerr << "ERROR: Unable to get node list.\n"; return 1; }
- if (nodes.size() != ids.size()) { std::cerr << "ERROR: Mismatch in returned node list size.\n"; return 1; }
+ // set the node
+ mid->nodes_set(node);
- // check that it's the same
- if (!node_okay(nodes[0], expected)) {
- return 1;
- }
+ // get it back
+ if (mid->nodes_get_list(&(way.nodes())) != way.nodes().size()) {
+ std::cerr << "ERROR: Unable to get node list.\n";
+ return 1;
+ }
+
+ // check that it's the same
+ if (!node_okay(way.nodes()[0].location(), node)) {
+ return 1;
+ }
- return 0;
+ return 0;
}
inline double test_lat(osmid_t id) {
- return 1 + 1e-5 * id;
+ return 1 + 1e-5 * id;
}
int test_nodes_comprehensive_set(middle_t *mid)
{
- taglist_t tags;
-
- expected_nodelist_t expected_nodes;
- expected_nodes.reserve(PER_BLOCK*8+1);
-
- // 2 dense blocks, the second partially filled at the star
- for (osmid_t id = 0; id < (PER_BLOCK+(PER_BLOCK >> 1) + 1); ++id)
- {
- expected_nodes.emplace_back(id, test_lat(id), 0.0);
- }
-
- // 1 dense block, 75% filled
- for (osmid_t id = PER_BLOCK*2; id < PER_BLOCK*3; ++id)
- {
- if ((id % 4 == 0) || (id % 4 == 1) || (id % 4 == 2))
- expected_nodes.emplace_back(id, test_lat(id), 0.0);
- }
-
- // 1 dense block, sparsly filled
- for (osmid_t id = PER_BLOCK*3; id < PER_BLOCK*4; ++id)
- {
- if (id % 4 == 0)
- expected_nodes.emplace_back(id, test_lat(id), 0.0);
- }
-
- // A lone sparse node
- expected_nodes.emplace_back(PER_BLOCK*5, test_lat(PER_BLOCK*5), 0.0);
-
- // A dense block of alternating positions of zero/non-zero
- for (osmid_t id = PER_BLOCK*6; id < PER_BLOCK*7; ++id)
- {
- if (id % 2 == 0)
- expected_nodes.emplace_back(id, 0.0, 0.0);
- else
- expected_nodes.emplace_back(id, test_lat(id), 0.0);
- }
- expected_nodes.emplace_back(PER_BLOCK*8, 0.0, 0.0);
- expected_nodes.emplace_back(PER_BLOCK*8+1, 0.0, 0.0);
-
- // Load up the nodes into the middle
- idlist_t ids;
- ids.reserve(expected_nodes.size());
-
- for (expected_nodelist_t::iterator node = expected_nodes.begin(); node != expected_nodes.end(); ++node)
- {
- mid->nodes_set(node->id, node->lat, node->lon, tags);
- ids.push_back(node->id);
- }
-
- nodelist_t nodes;
- if (mid->nodes_get_list(nodes, ids) != ids.size()) { std::cerr << "ERROR: Unable to get node list.\n"; return 1; }
-
- if (nodes.size() != ids.size()) { std::cerr << "ERROR: Mismatch in returned node list size.\n"; return 1; }
-
- for (size_t i = 0; i < nodes.size(); ++i)
- {
- if (!node_okay(nodes[i], expected_nodes[i])) {
- return 1;
- }
- }
- return 0;
+ std::vector<size_t> expected_nodes;
+ expected_nodes.reserve(PER_BLOCK*8+1);
+
+ buffer.clear();
+
+ // 2 dense blocks, the second partially filled at the star
+ for (osmid_t id = 0; id < (PER_BLOCK+(PER_BLOCK >> 1) + 1); ++id)
+ {
+ expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
+ }
+
+ // 1 dense block, 75% filled
+ for (osmid_t id = PER_BLOCK*2; id < PER_BLOCK*3; ++id)
+ {
+ if ((id % 4 == 0) || (id % 4 == 1) || (id % 4 == 2))
+ expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
+ }
+
+ // 1 dense block, sparsly filled
+ for (osmid_t id = PER_BLOCK*3; id < PER_BLOCK*4; ++id)
+ {
+ if (id % 4 == 0)
+ expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
+ }
+
+ // A lone sparse node
+ expected_nodes.emplace_back(add_node(PER_BLOCK*5, test_lat(PER_BLOCK*5), 0.0));
+
+ // A dense block of alternating positions of zero/non-zero
+ for (osmid_t id = PER_BLOCK*6; id < PER_BLOCK*7; ++id)
+ {
+ if (id % 2 == 0)
+ expected_nodes.emplace_back(add_node(id, 0.0, 0.0));
+ else
+ expected_nodes.emplace_back(add_node(id, test_lat(id), 0.0));
+ }
+ expected_nodes.emplace_back(add_node(PER_BLOCK*8, 0.0, 0.0));
+ expected_nodes.emplace_back(add_node(PER_BLOCK*8+1, 0.0, 0.0));
+
+ // Load up the nodes into the middle
+ std::vector<osmid_t> ids;
+
+ for (auto pos : expected_nodes)
+ {
+ auto const &node = buffer.get<osmium::Node>(pos);
+ mid->nodes_set(node);
+ ids.push_back(node.id());
+ }
+
+ auto &way = buffer.get<osmium::Way>(way_with_nodes(ids));
+
+ if (mid->nodes_get_list(&(way.nodes())) != ids.size()) {
+ std::cerr << "ERROR: Unable to get node list.\n";
+ return 1;
+ }
+
+ for (size_t i = 0; i < ids.size(); ++i) {
+ auto const &node = buffer.get<osmium::Node>(expected_nodes[i]);
+ if (!node_okay(way.nodes()[i].location(), node)) {
+ return 1;
+ }
+ }
+
+ return 0;
}
struct test_pending_processor : public middle_t::pending_processor {
test_pending_processor(): pending_ways(), pending_rels() {}
- virtual ~test_pending_processor() {}
- virtual void enqueue_ways(osmid_t id) {
+ ~test_pending_processor() {}
+ void enqueue_ways(osmid_t id) override
+ {
pending_ways.push_back(id);
}
- virtual void process_ways() {
+ void process_ways() override
+ {
pending_ways.clear();
}
- virtual void enqueue_relations(osmid_t id) {
+ void enqueue_relations(osmid_t id) override
+ {
pending_rels.push_back(id);
}
- virtual void process_relations() {
+ void process_relations() override
+ {
pending_rels.clear();
}
- virtual int thread_count() {
- return 0;
- }
- virtual int size() {
- return pending_ways.size() + pending_rels.size();
- }
std::list<osmid_t> pending_ways;
std::list<osmid_t> pending_rels;
};
int test_way_set(middle_t *mid)
{
- osmid_t way_id = 1;
- double lat = 12.3456789;
- double lon = 98.7654321;
- taglist_t tags;
- struct osmNode *node_ptr = nullptr;
- idlist_t nds;
- for (osmid_t i = 1; i <= 10; ++i)
- nds.push_back(i);
-
- // set the nodes
- for (size_t i = 0; i < nds.size(); ++i) {
- mid->nodes_set(nds[i], lat, lon, tags);
- }
-
- // set the way
- mid->ways_set(way_id, nds, tags);
-
- // commit the setup data
- mid->commit();
-
- // get it back
- idlist_t ways, xways;
- ways.push_back(way_id);
- std::vector<taglist_t> xtags;
- multinodelist_t xnodes;
- size_t way_count = mid->ways_get_list(ways, xways, xtags, xnodes);
- if (way_count != 1) { std::cerr << "ERROR: Unable to get way list.\n"; return 1; }
-
- // check that it's the same
- if (xnodes[0].size() != nds.size()) {
- std::cerr << "ERROR: Way should have " << nds.size() << " nodes, but got back "
- << xnodes[0].size() << " from middle.\n";
- return 1;
- }
- if (xways[0] != way_id) {
- std::cerr << "ERROR: Way should have id=" << way_id << ", but got back "
- << xways[0] << " from middle.\n";
- return 1;
- }
- for (size_t i = 0; i < nds.size(); ++i) {
- if (xnodes[0][i].lon != lon) {
- std::cerr << "ERROR: Way node should have lon=" << lon << ", but got back "
- << node_ptr[i].lon << " from middle.\n";
- return 1;
- }
- if (xnodes[0][i].lat != lat) {
- std::cerr << "ERROR: Way node should have lat=" << lat << ", but got back "
- << node_ptr[i].lat << " from middle.\n";
- return 1;
- }
- }
-
- // the way we just inserted should not be pending
- test_pending_processor tpp;
- mid->iterate_ways(tpp);
- if (mid->pending_count() != 0) {
- std::cerr << "ERROR: Was expecting no pending ways, but got "
- << mid->pending_count() << " from middle.\n";
- return 1;
- }
-
- // some middles don't support changing the nodes - they
- // don't have diff update ability. here, we will just
- // skip the test for that.
- if (dynamic_cast<slim_middle_t *>(mid)) {
- slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid);
-
- // finally, try touching a node on a non-pending way. that should
- // make it become pending. we just checked that the way is not
- // pending, so any change must be due to the node changing.
- slim->node_changed(nds[0]);
- slim->iterate_ways(tpp);
- if (slim->pending_count() != 1) {
- std::cerr << "ERROR: Was expecting a single pending way from node update, but got "
- << slim->pending_count() << " from middle.\n";
- return 1;
- }
- }
-
- return 0;
+ buffer.clear();
+
+ osmid_t way_id = 1;
+ double lat = 12.3456789;
+ double lon = 98.7654321;
+ idlist_t nds;
+ std::vector<size_t> nodes;
+
+ // set nodes
+ for (osmid_t i = 1; i <= 10; ++i) {
+ nds.push_back(i);
+ nodes.push_back(add_node(i, lat, lon));
+ auto const &node = buffer.get<osmium::Node>(nodes.back());
+ mid->nodes_set(node);
+ }
+
+ // set the way
+ {
+ using namespace osmium::builder::attr;
+ auto pos = osmium::builder::add_way(buffer, _id(way_id), _nodes(nds));
+ auto const &way = buffer.get<osmium::Way>(pos);
+ mid->ways_set(way);
+ }
+
+ // commit the setup data
+ mid->commit();
+
+ // get it back
+ osmium::memory::Buffer relbuf(4096, osmium::memory::Buffer::auto_grow::yes);
+ {
+ using namespace osmium::builder::attr;
+ osmium::builder::add_relation(
+ relbuf, _id(123), _member(osmium::item_type::node, 132),
+ _member(osmium::item_type::way, way_id, "outer"));
+ }
+
+ auto const &rel = relbuf.get<osmium::Relation>(0);
+
+ auto buf_pos = buffer.committed();
+ rolelist_t roles;
+ size_t way_count = mid->rel_way_members_get(rel, &roles, buffer);
+ if (way_count != 1) { std::cerr << "ERROR: Unable to get way list.\n"; return 1; }
+
+ if (roles.size() != 1) {
+ std::cerr << "Bad length of role ist. Expected 1, got " << roles.size()
+ << ".\n";
+ return 1;
+ }
+
+ if (strcmp(roles[0], "outer") != 0) {
+ std::cerr << "Bad role. Expected 'outer', got '" << roles[0] << "'.\n";
+ return 1;
+ }
+
+ auto &way = buffer.get<osmium::Way>(buf_pos);
+ // check that it's the same
+ if (way.nodes().size() != nds.size()) {
+ std::cerr << "ERROR: Way should have " << nds.size() << " nodes, but got back "
+ << way.nodes().size() << " from middle.\n";
+ return 1;
+ }
+ if (way.id() != way_id) {
+ std::cerr << "ERROR: Way should have id=" << way_id << ", but got back "
+ << way.id() << " from middle.\n";
+ return 1;
+ }
+ mid->nodes_get_list(&(way.nodes()));
+ for (size_t i = 0; i < nds.size(); ++i) {
+ if (way.nodes()[i].location().lon() != lon) {
+ std::cerr << "ERROR: Way node should have lon=" << lon
+ << ", but got back " << way.nodes()[i].location().lon()
+ << " from middle.\n";
+ return 1;
+ }
+ if (way.nodes()[i].location().lat() != lat) {
+ std::cerr << "ERROR: Way node should have lat=" << lat
+ << ", but got back " << way.nodes()[i].location().lat()
+ << " from middle.\n";
+ return 1;
+ }
+ }
+
+ // the way we just inserted should not be pending
+ test_pending_processor tpp;
+ mid->iterate_ways(tpp);
+ if (mid->pending_count() != 0) {
+ std::cerr << "ERROR: Was expecting no pending ways, but got "
+ << mid->pending_count() << " from middle.\n";
+ return 1;
+ }
+
+ // some middles don't support changing the nodes - they
+ // don't have diff update ability. here, we will just
+ // skip the test for that.
+ if (dynamic_cast<slim_middle_t *>(mid)) {
+ slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid);
+
+ // finally, try touching a node on a non-pending way. that should
+ // make it become pending. we just checked that the way is not
+ // pending, so any change must be due to the node changing.
+ slim->node_changed(nds[0]);
+ slim->iterate_ways(tpp);
+ if (slim->pending_count() != 1) {
+ std::cerr << "ERROR: Was expecting a single pending way from node update, but got "
+ << slim->pending_count() << " from middle.\n";
+ return 1;
+ }
+ }
+
+ return 0;
}
diff --git a/tests/mockups.hpp b/tests/mockups.hpp
index 8084840..f5ab58e 100644
--- a/tests/mockups.hpp
+++ b/tests/mockups.hpp
@@ -2,114 +2,95 @@
#define TESTS_MOCKUPS_HPP
#include "middle.hpp"
-#include "output.hpp"
+#include "output-null.hpp"
struct dummy_middle_t : public middle_t {
virtual ~dummy_middle_t() = default;
- void start(const options_t *) { }
- void stop(void) { }
+ void start(const options_t *) override { }
+ void stop(void) override { }
void cleanup(void) { }
- void analyze(void) { }
- void end(void) { }
- void commit(void) { }
-
- void nodes_set(osmid_t, double, double, const taglist_t &) { }
- size_t nodes_get_list(nodelist_t &, const idlist_t) const { return 0; }
-
- void ways_set(osmid_t, const idlist_t &, const taglist_t &) { }
- bool ways_get(osmid_t, taglist_t &, nodelist_t &) const { return true; }
- size_t ways_get_list(const idlist_t &, idlist_t &,
- std::vector<taglist_t> &,
- std::vector<nodelist_t> &) const { return 0; }
+ void analyze(void) override { }
+ void end(void) override { }
+ void commit(void) override { }
+
+ void nodes_set(osmium::Node const &) override {}
+ size_t nodes_get_list(osmium::WayNodeList *) const override { return 0; }
+
+ void ways_set(osmium::Way const &) override { }
+ bool ways_get(osmid_t, osmium::memory::Buffer &) const override { return true; }
+ size_t rel_way_members_get(osmium::Relation const &, rolelist_t *,
+ osmium::memory::Buffer &) const override
+ {
+ return 0;
+ }
- void relations_set(osmid_t, const memberlist_t &, const taglist_t &) { }
- bool relations_get(osmid_t, memberlist_t &, taglist_t &) const { return 0; }
+ void relations_set(osmium::Relation const &) override { }
+ bool relations_get(osmid_t, osmium::memory::Buffer &) const override { return 0; }
- void iterate_ways(pending_processor&) { }
- void iterate_relations(pending_processor&) { }
+ void iterate_ways(pending_processor&) override { }
+ void iterate_relations(pending_processor&) override { }
- virtual size_t pending_count() const { return 0; }
+ virtual size_t pending_count() const override { return 0; }
- std::vector<osmid_t> relations_using_way(osmid_t) const { return std::vector<osmid_t>(); }
+ idlist_t relations_using_way(osmid_t) const override { return idlist_t(); }
- virtual std::shared_ptr<const middle_query_t> get_instance() const {return std::shared_ptr<const middle_query_t>();}
+ std::shared_ptr<const middle_query_t> get_instance() const override
+ {
+ return std::shared_ptr<const middle_query_t>();
+ }
};
struct dummy_slim_middle_t : public slim_middle_t {
virtual ~dummy_slim_middle_t() = default;
- void start(const options_t *) { }
- void stop(void) { }
+ void start(const options_t *) override { }
+ void stop(void) override { }
void cleanup(void) { }
- void analyze(void) { }
- void end(void) { }
- void commit(void) { }
-
- void nodes_set(osmid_t, double, double, const taglist_t &) { }
- size_t nodes_get_list(nodelist_t &, const idlist_t) const { return 0; }
-
- void ways_set(osmid_t, const idlist_t &, const taglist_t &) { }
- bool ways_get(osmid_t, taglist_t &, nodelist_t &) const { return true; }
- size_t ways_get_list(const idlist_t &, idlist_t &,
- std::vector<taglist_t> &,
- std::vector<nodelist_t> &) const { return 0; }
+ void analyze(void) override { }
+ void end(void) override { }
+ void commit(void) override { }
+
+ void nodes_set(osmium::Node const &) override {}
+ size_t nodes_get_list(osmium::WayNodeList *) const override { return 0; }
+
+ void ways_set(osmium::Way const &) override { }
+ bool ways_get(osmid_t, osmium::memory::Buffer &) const override { return true; }
+ size_t rel_way_members_get(osmium::Relation const &, rolelist_t *,
+ osmium::memory::Buffer &) const override
+ {
+ return 0;
+ }
- void relations_set(osmid_t, const memberlist_t &, const taglist_t &) { }
- bool relations_get(osmid_t, memberlist_t &, taglist_t &) const { return 0; }
+ void relations_set(osmium::Relation const &) override { }
+ bool relations_get(osmid_t, osmium::memory::Buffer &) const override { return 0; }
- void iterate_ways(pending_processor&) { }
- void iterate_relations(pending_processor&) { }
+ void iterate_ways(pending_processor&) override { }
+ void iterate_relations(pending_processor&) override { }
- size_t pending_count() const { return 0; }
+ size_t pending_count() const override { return 0; }
- std::vector<osmid_t> relations_using_way(osmid_t) const { return std::vector<osmid_t>(); }
+ idlist_t relations_using_way(osmid_t) const override { return idlist_t(); }
- std::shared_ptr<const middle_query_t> get_instance() const {return std::shared_ptr<const middle_query_t>();}
+ std::shared_ptr<const middle_query_t> get_instance() const override {return std::shared_ptr<const middle_query_t>();}
- void nodes_delete(osmid_t) {};
- void node_changed(osmid_t) {};
+ void nodes_delete(osmid_t) override {};
+ void node_changed(osmid_t) override {};
- void ways_delete(osmid_t) {};
- void way_changed(osmid_t) {};
+ void ways_delete(osmid_t) override {};
+ void way_changed(osmid_t) override {};
- void relations_delete(osmid_t) {};
- void relation_changed(osmid_t) {};
+ void relations_delete(osmid_t) override {};
+ void relation_changed(osmid_t) override {};
};
-struct dummy_output_t : public output_t {
+struct dummy_output_t : public output_null_t {
explicit dummy_output_t(const options_t &options_)
- : output_t(nullptr, options_) {
+ : output_null_t(nullptr, options_) {
}
- virtual ~dummy_output_t() = default;
-
- int node_add(osmid_t, double, double, const taglist_t &) { return 0; }
- int way_add(osmid_t, const idlist_t &, const taglist_t &) { return 0; }
- int relation_add(osmid_t, const memberlist_t &, const taglist_t &) { return 0; }
-
- int start() { return 0; }
- int connect(int) { return 0; }
- void stop() { }
- void commit() { }
- void cleanup(void) { }
- void close(int) { }
-
- void enqueue_ways(pending_queue_t &, osmid_t, size_t, size_t&) { }
- int pending_way(osmid_t, int) { return 0; }
-
- void enqueue_relations(pending_queue_t &, osmid_t, size_t, size_t&) { }
- int pending_relation(osmid_t, int) { return 0; }
-
- int node_modify(osmid_t, double, double, const taglist_t &) { return 0; }
- int way_modify(osmid_t, const idlist_t &, const taglist_t &) { return 0; }
- int relation_modify(osmid_t, const memberlist_t &, const taglist_t &) { return 0; }
-
- int node_delete(osmid_t) { return 0; }
- int way_delete(osmid_t) { return 0; }
- int relation_delete(osmid_t) { return 0; }
-
+ ~dummy_output_t() = default;
};
#endif
diff --git a/tests/regression-test.py b/tests/regression-test.py
index bd9aa36..de4a1df 100755
--- a/tests/regression-test.py
+++ b/tests/regression-test.py
@@ -19,37 +19,37 @@ created_tablespace = 0
#****************************************************************
sql_test_statements=[
( 0, 'Basic point count', 'SELECT count(*) FROM planet_osm_point;', 1342 ),
- ( 1, 'Basic line count', 'SELECT count(*) FROM planet_osm_line;', 3300 ),
+ ( 1, 'Basic line count', 'SELECT count(*) FROM planet_osm_line;', 3231 ),
( 2, 'Basic road count', 'SELECT count(*) FROM planet_osm_roads;', 375 ),
- ( 3, 'Basic polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4128 ),
- ( 4, 'Basic latlon line count', 'SELECT count(*) FROM planet_osm_line;', 3298 ),
+ ( 3, 'Basic polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4127 ),
+ ( 4, 'Basic latlon line count', 'SELECT count(*) FROM planet_osm_line;', 3229 ),
( 5, 'Basic latlon road count', 'SELECT count(*) FROM planet_osm_roads;', 374 ),
( 6, 'Basic post-diff point count', 'SELECT count(*) FROM planet_osm_point;', 1457 ),
- ( 7, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3344 ),
- ( 8, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 381 ),
- ( 9, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4275 ),
+ ( 7, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3274 ),
+ ( 8, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 380 ),
+ ( 9, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4274 ),
( 10, 'Absence of nodes table', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_nodes\'', 0),
( 11, 'Absence of way table', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_ways\'', 0),
( 12, 'Absence of rel line', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_rels\'', 0),
- ( 13, 'Basic polygon area', 'SELECT round(sum(cast(ST_Area(way) as numeric)),0) FROM planet_osm_polygon;', 1223800814),
- ( 14, 'Gazetteer place count', 'SELECT count(*) FROM place', 2837),
+ ( 13, 'Basic polygon area', 'SELECT round(sum(cast(ST_Area(way) as numeric)),0) FROM planet_osm_polygon;', 1210958566),
+ ( 14, 'Gazetteer place count', 'SELECT count(*) FROM place', 2836),
( 15, 'Gazetteer place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 759),
( 16, 'Gazetteer place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 2059),
- ( 17, 'Gazetteer place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 19),
- ( 18, 'Gazetteer post-diff place count', 'SELECT count(*) FROM place', 2878),
+ ( 17, 'Gazetteer place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 18),
+ ( 18, 'Gazetteer post-diff place count', 'SELECT count(*) FROM place', 2877),
( 19, 'Gazetteer post-diff place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 764),
( 20, 'Gazetteer post-diff place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 2095),
- ( 21, 'Gazetteer post-diff place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 19),
- ( 22, 'Gazetteer housenumber count', 'SELECT count(*) FROM place WHERE housenumber is not null', 199),
- ( 23, 'Gazetteer post-diff housenumber count count', 'SELECT count(*) FROM place WHERE housenumber is not null', 199),
- ( 24, 'Gazetteer isin count', 'SELECT count(*) FROM place WHERE isin is not null', 239),
- ( 25, 'Gazetteer post-diff isin count count', 'SELECT count(*) FROM place WHERE isin is not null', 239),
+ ( 21, 'Gazetteer post-diff place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 18),
+ ( 22, 'Gazetteer housenumber count', "SELECT count(*) FROM place WHERE address ? 'housenumber'", 199),
+ ( 23, 'Gazetteer post-diff housenumber count count', "SELECT count(*) FROM place WHERE address ? 'housenumber'", 199),
+ ( 24, 'Gazetteer address count', 'SELECT count(*) FROM place WHERE address is not null', 319),
+ ( 25, 'Gazetteer post-diff address count', 'SELECT count(*) FROM place WHERE address is not null', 319),
( 26, 'Multipolygon basic case (Tags from outer way)',
'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -15 and landuse = \'residential\' and name = \'Name_way\'', 12894),
( 27, 'Multipolygon basic case (Tags from relation)',
'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -1 and landuse = \'residential\' and name = \'Name_rel\'', 12895),
( 28, 'Multipolygon named inner - outer (Tags from way)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -16 and landuse = \'residential\' and name = \'Name_way2\'', 12895),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -16 and landuse = \'residential\' and name = \'Name_way2\'', 12894),
( 29, 'Multipolygon named inner - inner way',
'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 4 and landuse = \'farmland\' and name = \'Name_way3\'', 3144),
( 30, 'Multipolygon named inner - outer (Tags from relation)',
@@ -57,19 +57,19 @@ sql_test_statements=[
( 31, 'Multipolygon named inner - inner way',
'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 5 and landuse = \'farmland\' and name = \'Name_way4\'', 3144),
( 32, 'Multipolygon named same inner - outer (Tags from way)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -17 and landuse = \'residential\' and name = \'Name_way16\'', 12895),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -17 and landuse = \'residential\' and name = \'Name_way16\'', 12894),
( 33, 'Multipolygon named same inner - inner way absent',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 15', 0),
( 34, 'Multipolygon non-area inner - outer (Tags from relation)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -14 and landuse = \'residential\' and name = \'Name_way5\'', 12893),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -14 and landuse = \'residential\' and name = \'Name_way5\'', 12894),
( 35, 'Multipolygon non-area inner - inner (Tags from way)',
'SELECT round(ST_Length(way)) FROM planet_osm_line WHERE osm_id = 6 and highway = \'residential\' and name = \'Name_way6\'', 228),
( 36, 'Multipolygon 2 holes (Tags from way)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -18 and landuse = \'residential\' and name = \'Name_way7\'', 11823),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -18 and landuse = \'residential\' and name = \'Name_way7\'', 11822),
( 37, 'Multipolygon 2 holes (Tags from way)',
'SELECT ST_NumInteriorRing(way) FROM planet_osm_polygon WHERE osm_id = -18 and landuse = \'residential\' and name = \'Name_way7\'', 2),
( 38, 'Multipolygon from multiple outer ways 0 holes (Tags from relation)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -11 and landuse = \'residential\' and name = \'Name_rel6\'', 11528),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -11 and landuse = \'residential\' and name = \'Name_rel6\'', 11529),
( 39, 'Multipolygon from multiple outer and multiple inner ways 2 holes (Tags from relation)',
'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -3 and landuse = \'residential\' and name = \'Name_rel11\'', 9286),
( 40, 'Multipolygon 2 holes (Tags from way)',
@@ -95,21 +95,21 @@ sql_test_statements=[
( 50, 'Multipolygon nested outer ways. Both outer and inner ways are from multiple ways (multigeometry)',
'SELECT ST_NumGeometries(way) FROM planet_osm_polygon WHERE osm_id = -7 and landuse = \'farmland\' and name = \'Name_rel15\'', 2),
( 51, 'Basic hstore point count', 'SELECT count(*) FROM planet_osm_point;', 1360 ),
- ( 52, 'Basic hstore line count', 'SELECT count(*) FROM planet_osm_line;', 3323 ),
+ ( 52, 'Basic hstore line count', 'SELECT count(*) FROM planet_osm_line;', 3254 ),
( 53, 'Basic hstore road count', 'SELECT count(*) FROM planet_osm_roads;', 375 ),
- ( 54, 'Basic hstore polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4128 ),
+ ( 54, 'Basic hstore polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4127 ),
( 55, 'Basic post-diff point count', 'SELECT count(*) FROM planet_osm_point;', 1475 ),
- ( 56, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3367 ),
- ( 57, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 381 ),
- ( 58, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4275 ),
+ ( 56, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3297 ),
+ ( 57, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 380 ),
+ ( 58, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4274 ),
( 59, 'Extra hstore full tags point count',
'SELECT count(*) FROM planet_osm_point WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 1360),
( 60, 'Extra hstore full tags line count',
- 'SELECT count(*) FROM planet_osm_line WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 3323),
+ 'SELECT count(*) FROM planet_osm_line WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 3254),
( 61, 'Extra hstore full tags polygon count',
- 'SELECT count(*) FROM planet_osm_polygon WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 4128),
+ 'SELECT count(*) FROM planet_osm_polygon WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 4127),
( 62, 'Multipolygon copying of tags from outer with extra tags on relation',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -22', 20879),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -22', 20878),
( 63, 'Multipolygon copying of tags from outer with extra tags on relation (abscence of way)',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 84', 0),
( 64, 'Multipolygon non copying of tags from outer with polygon tags on relation',
@@ -117,15 +117,15 @@ sql_test_statements=[
( 65, 'Multipolygon non copying of tags from outer with polygon tags on relation (presence of way)',
'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 83 and "landuse" = \'farmland\'', 24859),
( 66, 'Multipolygon diff moved point of outer way case (Tags from outer way)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -15 and landuse = \'residential\' and name = \'Name_way\'', 24750),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -15 and landuse = \'residential\' and name = \'Name_way\'', 24749),
( 67, 'Multipolygon diff moved point of inner way case (Tags from relation)',
'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -1 and landuse = \'residential\' and name = \'Name_rel\'', 13949),
( 68, 'Multipolygon point of inner way case (Tags from relation)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -25 and landuse = \'farmland\' and name = \'my name\'', 23886),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -25 and landuse = \'farmland\' and name = \'my name\'', 23884),
( 69, 'Multipolygon point of inner way case (Tags from relation)',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 90', 0),
( 70, 'Multipolygon diff remove relation (tagged outer way gets re added)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 90 and landuse = \'farmland\'', 32626),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 90 and landuse = \'farmland\'', 32624),
( 71, 'Multipolygon diff remove relation',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -25', 0),
( 72, 'Multipolygon tags on both inner and outer (presence of relation)',
@@ -141,41 +141,41 @@ sql_test_statements=[
( 77, 'Multipolygon tags on both inner and outer diff change on outer (creation of inner)',
'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 118 and "natural" = \'water\'', 1234),
( 78, 'Multipolygon tags on outer (presence of relation)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -33 and "natural" = \'water\'', 15612),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -33 and "natural" = \'water\'', 15613),
( 79, 'Multipolygon tags on outer (abscence of outer)',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 114', 0),
( 80, 'Multipolygon tags on outer change of way tags (presence of relation)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -33 and "landuse" = \'cemetery\'', 15612),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -33 and "landuse" = \'cemetery\'', 15613),
( 81, 'Multipolygon tags on outer (abscence of old relation)',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -33 and "natural" = \'water\'', 0),
( 82, 'Multipolygon tags on relation two outer (presence of relation)',
- 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -29 and "natural" = \'water\'', 68492),
+ 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -29 and "natural" = \'water\'', 68494),
( 83, 'Multipolygon tags on relation two outer (abscence of outer)',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 109', 0),
( 84, 'Multipolygon tags on relation two outer (abscence of outer)',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 104', 0),
( 85, 'Multipolygon tags on relation two outer diff delete way (presence of relation)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -29 and "natural" = \'water\'', 29154),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -29 and "natural" = \'water\'', 29155),
( 86, 'Multipolygon tags on relation two outer (presence of relation)',
- 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -35 and "natural" = \'water\'', 28730),
+ 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -35 and "natural" = \'water\'', 28731),
( 87, 'Multipolygon tags on relation two outer (abscence of outer)',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 107', 0),
( 88, 'Multipolygon tags on relation two outer (abscence of outer)',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 102', 0),
( 89, 'Multipolygon tags on relation two outer diff remove way from relation (presence of relation)',
- 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -35 and "natural" = \'water\'', 15736),
+ 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -35 and "natural" = \'water\'', 15737),
( 90, 'Multipolygon tags on relation two outer diff remove way from relation (presence of single way)',
'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 102 and "natural" = \'water\'', 12994),
- ( 91, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_line;', 4269394),
+ ( 91, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_line;', 4211350),
( 92, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_roads;', 2032023),
( 93, 'Basic number of hstore points tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_point;', 4228),
( 94, 'Basic number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2317),
- ( 95, 'Basic number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 11134),
- ( 96, 'Basic number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9541),
+ ( 95, 'Basic number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 10387),
+ ( 96, 'Basic number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9531),
( 97, 'Diff import number of hstore points tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_point;', 4352),
- ( 98, 'Diff import number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2341),
- ( 99, 'Diff import number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 11257),
- ( 100, 'Diff import number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9835),
+ ( 98, 'Diff import number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2336),
+ ( 99, 'Diff import number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 10505),
+ ( 100, 'Diff import number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9825),
#**** Tests to check if inner polygon appears when outer tags change after initially identicall inner and outer way tags in a multi-polygon ****
#**** These tests are currently broken and noted in trac ticket #2853 ****
( 101, 'Multipolygon identical tags on inner and outer (presence of relation)',
@@ -198,7 +198,7 @@ sql_test_statements=[
( 109, 'Multipolygon copy outer tags (presence of additionally tagged outer way)',
'SELECT round(sum(ST_length(way))) FROM planet_osm_line WHERE (osm_id = 136 OR osm_id = 132) AND "man_made" = \'pier\'', 407),
( 110, 'Multipolygon copy outer tags (presence of relation)',
- 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -37 and "natural" = \'water\'', 29952),
+ 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -37 and "natural" = \'water\'', 29951),
( 111, 'Multipolygon copy outer tags (absence of partial outer tags)',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -37 and "natural" = \'water\' and "man_made" = \'pier\'', 0),
( 112, 'Multipolygon copy outer tags (absence of multi-polygon tagged outer way)',
@@ -211,7 +211,7 @@ sql_test_statements=[
'SELECT round(sum(ST_length(way))) FROM planet_osm_line WHERE (osm_id = 127 OR osm_id = 122) AND "man_made" = \'pier\'', 318),
#**** Test to check that if polygon tags are on both outer ways and relation, polygons don't get duplicated in the db ****
( 116, 'Multipolygon tags on both outer and relation (presence of relation)',
- 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -39 and "landuse" = \'forest\'', 10378),
+ 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -39 and "landuse" = \'forest\'', 10377),
( 117, 'Multipolygon tags on both outer and relation (absence of outer way)',
'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 138', 0),
( 118, 'Multipolygon tags on both outer and relation with additional tags on relation (presence of relation)',
diff --git a/tests/test-expire-tiles.cpp b/tests/test-expire-tiles.cpp
index 66fe186..5cbc018 100644
--- a/tests/test-expire-tiles.cpp
+++ b/tests/test-expire-tiles.cpp
@@ -33,12 +33,12 @@ void run_test(const char* test_name, void (*testfunc)())
#define ASSERT_EQ(a, b) { if (!((a) == (b))) { throw std::runtime_error((boost::format("Expecting %1% == %2%, but %3% != %4%") % #a % #b % (a) % (b)).str()); } }
struct xyz {
- int z, x, y;
- xyz(int z_, int x_, int y_) : z(z_), x(x_), y(y_) {}
- bool operator==(const xyz &other) const {
- return ((z == other.z) &&
- (x == other.x) &&
- (y == other.y));
+ uint32_t z;
+ int64_t x, y;
+ xyz(uint32_t z_, int64_t x_, int64_t y_) : z(z_), x(x_), y(y_) {}
+ bool operator==(const xyz &other) const
+ {
+ return ((z == other.z) && (x == other.x) && (y == other.y));
}
bool operator<(const xyz &other) const {
return ((z < other.z) ||
@@ -71,98 +71,203 @@ std::ostream &operator<<(std::ostream &out, const xyz &tile) {
return out;
}
-struct tile_output_set : public expire_tiles::tile_output
+struct tile_output_set
{
- tile_output_set(int min) : min_zoom(min) {}
-
- ~tile_output_set() = default;
-
- void output_dirty_tile(int x, int y, int zoom) override
- {
- int y_min, x_iter, y_iter, x_max, y_max, out_zoom, zoom_diff;
-
- if (zoom > min_zoom) out_zoom = zoom;
- else out_zoom = min_zoom;
- zoom_diff = out_zoom - zoom;
- y_min = y << zoom_diff;
- x_max = (x + 1) << zoom_diff;
- y_max = (y + 1) << zoom_diff;
- for (x_iter = x << zoom_diff; x_iter < x_max; x_iter++) {
- for (y_iter = y_min; y_iter < y_max; y_iter++) {
- m_tiles.insert(xyz(out_zoom, x_iter, y_iter));
- }
- }
+ tile_output_set(uint32_t min) : min_zoom(min) {}
+
+ ~tile_output_set() = default;
+
+ void output_dirty_tile(int64_t x, int64_t y, uint32_t zoom)
+ {
+ m_tiles.insert(xyz(zoom, x, y));
}
std::set<xyz> m_tiles;
- int min_zoom;
+ uint32_t min_zoom;
};
+void test_xy_to_quadkey_z3()
+{
+ uint64_t quadkey_expected = 0x27;
+ uint64_t quadkey2 = expire_tiles::xy_to_quadkey(3, 5, 3);
+ ASSERT_EQ(quadkey2, quadkey_expected);
+ xy_coord_t xy = expire_tiles::quadkey_to_xy(quadkey_expected, 3);
+ ASSERT_EQ(xy.x, 3);
+ ASSERT_EQ(xy.y, 5);
+}
+
+void test_xy_to_quadkey_z16()
+{
+ uint64_t quadkey_expected = 0xffffffff;
+ uint64_t quadkey2 = expire_tiles::xy_to_quadkey(65535, 65535, 16);
+ ASSERT_EQ(quadkey2, quadkey_expected);
+ xy_coord_t xy = expire_tiles::quadkey_to_xy(quadkey_expected, 16);
+ ASSERT_EQ(xy.x, 65535);
+ ASSERT_EQ(xy.y, 65535);
+}
+
+/**
+ * This test prevents problems which occur if 32-bit integers are used
+ * instead of 64-bit integers.
+ */
+void test_xy_to_quadkey_z18()
+{
+ uint64_t quadkey_expected = 0xfffffffff;
+ uint64_t quadkey2 = expire_tiles::xy_to_quadkey(262143, 262143, 18);
+ ASSERT_EQ(quadkey2, quadkey_expected);
+ xy_coord_t xy = expire_tiles::quadkey_to_xy(quadkey_expected, 18);
+ ASSERT_EQ(xy.x, 262143);
+ ASSERT_EQ(xy.y, 262143);
+ quadkey_expected = 0x3fffffff0;
+ quadkey2 = expire_tiles::xy_to_quadkey(131068, 131068, 18);
+ ASSERT_EQ(quadkey2, quadkey_expected);
+ xy = expire_tiles::quadkey_to_xy(quadkey_expected, 18);
+ ASSERT_EQ(xy.x, 131068);
+ ASSERT_EQ(xy.y, 131068);
+}
+
void test_expire_simple_z1() {
- expire_tiles et(1, 20000, defproj);
- tile_output_set set(1);
-
- // as big a bbox as possible at the origin to dirty all four
- // quadrants of the world.
- et.from_bbox(-10000, -10000, 10000, 10000);
- et.output_and_destroy(&set);
-
- ASSERT_EQ(set.m_tiles.size(), 4);
- std::set<xyz>::iterator itr = set.m_tiles.begin();
- ASSERT_EQ(*itr, xyz(1, 0, 0)); ++itr;
- ASSERT_EQ(*itr, xyz(1, 0, 1)); ++itr;
- ASSERT_EQ(*itr, xyz(1, 1, 0)); ++itr;
- ASSERT_EQ(*itr, xyz(1, 1, 1)); ++itr;
+ uint32_t minzoom = 1;
+ expire_tiles et(minzoom, 20000, defproj);
+ tile_output_set set(minzoom);
+
+ // as big a bbox as possible at the origin to dirty all four
+ // quadrants of the world.
+ et.from_bbox(-10000, -10000, 10000, 10000);
+ et.output_and_destroy<tile_output_set>(set, minzoom);
+
+ ASSERT_EQ(set.m_tiles.size(), 4);
+ std::set<xyz>::iterator itr = set.m_tiles.begin();
+ ASSERT_EQ(*itr, xyz(1, 0, 0));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(1, 0, 1));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(1, 1, 0));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(1, 1, 1));
+ ++itr;
}
void test_expire_simple_z3() {
- expire_tiles et(3, 20000, defproj);
- tile_output_set set(3);
-
- // as big a bbox as possible at the origin to dirty all four
- // quadrants of the world.
- et.from_bbox(-10000, -10000, 10000, 10000);
- et.output_and_destroy(&set);
-
- ASSERT_EQ(set.m_tiles.size(), 4);
- std::set<xyz>::iterator itr = set.m_tiles.begin();
- ASSERT_EQ(*itr, xyz(3, 3, 3)); ++itr;
- ASSERT_EQ(*itr, xyz(3, 3, 4)); ++itr;
- ASSERT_EQ(*itr, xyz(3, 4, 3)); ++itr;
- ASSERT_EQ(*itr, xyz(3, 4, 4)); ++itr;
+ uint32_t minzoom = 3;
+ expire_tiles et(minzoom, 20000, defproj);
+ tile_output_set set(minzoom);
+
+ // as big a bbox as possible at the origin to dirty all four
+ // quadrants of the world.
+ et.from_bbox(-10000, -10000, 10000, 10000);
+ et.output_and_destroy<tile_output_set>(set, minzoom);
+
+ ASSERT_EQ(set.m_tiles.size(), 4);
+ std::set<xyz>::iterator itr = set.m_tiles.begin();
+ ASSERT_EQ(*itr, xyz(3, 3, 3));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(3, 3, 4));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(3, 4, 3));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(3, 4, 4));
+ ++itr;
}
void test_expire_simple_z18() {
- expire_tiles et(18, 20000, defproj);
- tile_output_set set(18);
-
- // dirty a smaller bbox this time, as at z18 the scale is
- // pretty small.
- et.from_bbox(-1, -1, 1, 1);
- et.output_and_destroy(&set);
-
- ASSERT_EQ(set.m_tiles.size(), 4);
- std::set<xyz>::iterator itr = set.m_tiles.begin();
- ASSERT_EQ(*itr, xyz(18, 131071, 131071)); ++itr;
- ASSERT_EQ(*itr, xyz(18, 131071, 131072)); ++itr;
- ASSERT_EQ(*itr, xyz(18, 131072, 131071)); ++itr;
- ASSERT_EQ(*itr, xyz(18, 131072, 131072)); ++itr;
+ uint32_t minzoom = 18;
+ expire_tiles et(18, 20000, defproj);
+ tile_output_set set(minzoom);
+
+ // dirty a smaller bbox this time, as at z18 the scale is
+ // pretty small.
+ et.from_bbox(-1, -1, 1, 1);
+ et.output_and_destroy(set, minzoom);
+
+ ASSERT_EQ(set.m_tiles.size(), 4);
+ std::set<xyz>::iterator itr = set.m_tiles.begin();
+ ASSERT_EQ(*itr, xyz(18, 131071, 131071));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131071, 131072));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131072, 131071));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131072, 131072));
+ ++itr;
+}
+
+/**
+ * Test tile expiry on two zoom levels.
+ */
+void test_expire_simple_z17_18()
+{
+ uint32_t minzoom = 17;
+ expire_tiles et(18, 20000, defproj);
+ tile_output_set set(minzoom);
+
+ // dirty a smaller bbox this time, as at z18 the scale is
+ // pretty small.
+ et.from_bbox(-1, -1, 1, 1);
+ et.output_and_destroy(set, minzoom);
+
+ ASSERT_EQ(set.m_tiles.size(), 8);
+ std::set<xyz>::iterator itr = set.m_tiles.begin();
+ ASSERT_EQ(*itr, xyz(17, 65535, 65535));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(17, 65535, 65536));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(17, 65536, 65535));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(17, 65536, 65536));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131071, 131071));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131071, 131072));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131072, 131071));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131072, 131072));
+ ++itr;
}
-std::set<xyz> generate_random(int zoom, size_t count) {
- size_t num = 0;
- std::set<xyz> set;
- const int coord_mask = (1 << zoom) - 1;
+/**
+ * Similar to test_expire_simple_z17_18 but now all z18 tiles are children
+ * of the same z17 tile.
+ */
+void test_expire_simple_z17_18_one_superior_tile()
+{
+ uint32_t minzoom = 17;
+ expire_tiles et(18, 20000, defproj);
+ tile_output_set set(minzoom);
+
+ et.from_bbox(-163, 140, -140, 164);
+ et.output_and_destroy(set, minzoom);
+
+ ASSERT_EQ(set.m_tiles.size(), 5);
+ std::set<xyz>::iterator itr = set.m_tiles.begin();
+ ASSERT_EQ(*itr, xyz(17, 65535, 65535));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131070, 131070));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131070, 131071));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131071, 131070));
+ ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131071, 131071));
+ ++itr;
+}
- while (num < count) {
- xyz item(zoom, rand() & coord_mask, rand() & coord_mask);
- if (set.count(item) == 0) {
- set.insert(item);
- ++num;
+std::set<xyz> generate_random(uint32_t zoom, size_t count)
+{
+ size_t num = 0;
+ std::set<xyz> set;
+ const int coord_mask = (1 << zoom) - 1;
+
+ while (num < count) {
+ xyz item(zoom, rand() & coord_mask, rand() & coord_mask);
+ if (set.count(item) == 0) {
+ set.insert(item);
+ ++num;
+ }
}
- }
- return set;
+ return set;
}
void assert_tilesets_equal(const std::set<xyz> &a,
@@ -191,17 +296,17 @@ void expire_centroids(const std::set<xyz> &check_set,
// tests that expiring a set of tile centroids means that
// those tiles get expired.
void test_expire_set() {
- int zoom = 18;
- for (int i = 0; i < 100; ++i) {
- expire_tiles et(zoom, 20000, defproj);
- tile_output_set set(zoom);
+ uint32_t zoom = 18;
+ for (int i = 0; i < 100; ++i) {
+ expire_tiles et(zoom, 20000, defproj);
+ tile_output_set set(zoom);
- std::set<xyz> check_set = generate_random(zoom, 100);
- expire_centroids(check_set, et);
+ std::set<xyz> check_set = generate_random(zoom, 100);
+ expire_centroids(check_set, et);
- et.output_and_destroy(&set);
+ et.output_and_destroy(set, zoom);
- assert_tilesets_equal(set.m_tiles, check_set);
+ assert_tilesets_equal(set.m_tiles, check_set);
}
}
@@ -211,31 +316,31 @@ void test_expire_set() {
// same as if the union of the sets of tiles had been
// expired.
void test_expire_merge() {
- int zoom = 18;
+ uint32_t zoom = 18;
- for (int i = 0; i < 100; ++i) {
- expire_tiles et(zoom, 20000, defproj);
- expire_tiles et1(zoom, 20000, defproj);
- expire_tiles et2(zoom, 20000, defproj);
- tile_output_set set(zoom);
+ for (int i = 0; i < 100; ++i) {
+ expire_tiles et(zoom, 20000, defproj);
+ expire_tiles et1(zoom, 20000, defproj);
+ expire_tiles et2(zoom, 20000, defproj);
+ tile_output_set set(zoom);
- std::set<xyz> check_set1 = generate_random(zoom, 100);
- expire_centroids(check_set1, et1);
+ std::set<xyz> check_set1 = generate_random(zoom, 100);
+ expire_centroids(check_set1, et1);
- std::set<xyz> check_set2 = generate_random(zoom, 100);
- expire_centroids(check_set2, et2);
+ std::set<xyz> check_set2 = generate_random(zoom, 100);
+ expire_centroids(check_set2, et2);
- et.merge_and_destroy(et1);
- et.merge_and_destroy(et2);
+ et.merge_and_destroy(et1);
+ et.merge_and_destroy(et2);
- std::set<xyz> check_set;
- std::set_union(check_set1.begin(), check_set1.end(),
- check_set2.begin(), check_set2.end(),
- std::inserter(check_set, check_set.end()));
+ std::set<xyz> check_set;
+ std::set_union(check_set1.begin(), check_set1.end(), check_set2.begin(),
+ check_set2.end(),
+ std::inserter(check_set, check_set.end()));
- et.output_and_destroy(&set);
+ et.output_and_destroy(set, zoom);
- assert_tilesets_equal(set.m_tiles, check_set);
+ assert_tilesets_equal(set.m_tiles, check_set);
}
}
@@ -245,62 +350,62 @@ void test_expire_merge() {
// skipped by the random tile set in the previous
// test.
void test_expire_merge_same() {
- int zoom = 18;
+ uint32_t zoom = 18;
- for (int i = 0; i < 100; ++i) {
- expire_tiles et(zoom, 20000, defproj);
- expire_tiles et1(zoom, 20000, defproj);
- expire_tiles et2(zoom, 20000, defproj);
- tile_output_set set(zoom);
+ for (int i = 0; i < 100; ++i) {
+ expire_tiles et(zoom, 20000, defproj);
+ expire_tiles et1(zoom, 20000, defproj);
+ expire_tiles et2(zoom, 20000, defproj);
+ tile_output_set set(zoom);
- std::set<xyz> check_set = generate_random(zoom, 100);
- expire_centroids(check_set, et1);
- expire_centroids(check_set, et2);
+ std::set<xyz> check_set = generate_random(zoom, 100);
+ expire_centroids(check_set, et1);
+ expire_centroids(check_set, et2);
- et.merge_and_destroy(et1);
- et.merge_and_destroy(et2);
+ et.merge_and_destroy(et1);
+ et.merge_and_destroy(et2);
- et.output_and_destroy(&set);
+ et.output_and_destroy(set, zoom);
- assert_tilesets_equal(set.m_tiles, check_set);
+ assert_tilesets_equal(set.m_tiles, check_set);
}
}
// makes sure that we're testing the case where some
// tiles are in both.
void test_expire_merge_overlap() {
- int zoom = 18;
+ uint32_t zoom = 18;
- for (int i = 0; i < 100; ++i) {
- expire_tiles et(zoom, 20000, defproj);
- expire_tiles et1(zoom, 20000, defproj);
- expire_tiles et2(zoom, 20000, defproj);
- tile_output_set set(zoom);
+ for (int i = 0; i < 100; ++i) {
+ expire_tiles et(zoom, 20000, defproj);
+ expire_tiles et1(zoom, 20000, defproj);
+ expire_tiles et2(zoom, 20000, defproj);
+ tile_output_set set(zoom);
- std::set<xyz> check_set1 = generate_random(zoom, 100);
- expire_centroids(check_set1, et1);
+ std::set<xyz> check_set1 = generate_random(zoom, 100);
+ expire_centroids(check_set1, et1);
- std::set<xyz> check_set2 = generate_random(zoom, 100);
- expire_centroids(check_set2, et2);
+ std::set<xyz> check_set2 = generate_random(zoom, 100);
+ expire_centroids(check_set2, et2);
- std::set<xyz> check_set3 = generate_random(zoom, 100);
- expire_centroids(check_set3, et1);
- expire_centroids(check_set3, et2);
+ std::set<xyz> check_set3 = generate_random(zoom, 100);
+ expire_centroids(check_set3, et1);
+ expire_centroids(check_set3, et2);
- et.merge_and_destroy(et1);
- et.merge_and_destroy(et2);
+ et.merge_and_destroy(et1);
+ et.merge_and_destroy(et2);
- std::set<xyz> check_set;
- std::set_union(check_set1.begin(), check_set1.end(),
- check_set2.begin(), check_set2.end(),
- std::inserter(check_set, check_set.end()));
- std::set_union(check_set1.begin(), check_set1.end(),
- check_set3.begin(), check_set3.end(),
- std::inserter(check_set, check_set.end()));
+ std::set<xyz> check_set;
+ std::set_union(check_set1.begin(), check_set1.end(), check_set2.begin(),
+ check_set2.end(),
+ std::inserter(check_set, check_set.end()));
+ std::set_union(check_set1.begin(), check_set1.end(), check_set3.begin(),
+ check_set3.end(),
+ std::inserter(check_set, check_set.end()));
- et.output_and_destroy(&set);
+ et.output_and_destroy(set, zoom);
- assert_tilesets_equal(set.m_tiles, check_set);
+ assert_tilesets_equal(set.m_tiles, check_set);
}
}
@@ -308,28 +413,28 @@ void test_expire_merge_overlap() {
// large contiguous areas of tiles (i.e: ensure that we
// handle the "complete" flag correctly).
void test_expire_merge_complete() {
- int zoom = 18;
+ uint32_t zoom = 18;
- for (int i = 0; i < 100; ++i) {
- expire_tiles et(zoom, 20000, defproj);
- expire_tiles et0(zoom, 20000, defproj);
- expire_tiles et1(zoom, 20000, defproj);
- expire_tiles et2(zoom, 20000, defproj);
- tile_output_set set(zoom);
- tile_output_set set0(zoom);
+ for (int i = 0; i < 100; ++i) {
+ expire_tiles et(zoom, 20000, defproj);
+ expire_tiles et0(zoom, 20000, defproj);
+ expire_tiles et1(zoom, 20000, defproj);
+ expire_tiles et2(zoom, 20000, defproj);
+ tile_output_set set(zoom);
+ tile_output_set set0(zoom);
- // et1&2 are two halves of et0's box
- et0.from_bbox(-10000, -10000, 10000, 10000);
- et1.from_bbox(-10000, -10000, 0, 10000);
- et2.from_bbox( 0, -10000, 10000, 10000);
+ // et1&2 are two halves of et0's box
+ et0.from_bbox(-10000, -10000, 10000, 10000);
+ et1.from_bbox(-10000, -10000, 0, 10000);
+ et2.from_bbox(0, -10000, 10000, 10000);
- et.merge_and_destroy(et1);
- et.merge_and_destroy(et2);
+ et.merge_and_destroy(et1);
+ et.merge_and_destroy(et2);
- et.output_and_destroy(&set);
- et0.output_and_destroy(&set0);
+ et.output_and_destroy(set, zoom);
+ et0.output_and_destroy(set0, zoom);
- assert_tilesets_equal(set.m_tiles, set0.m_tiles);
+ assert_tilesets_equal(set.m_tiles, set0.m_tiles);
}
}
@@ -340,9 +445,14 @@ int main(int argc, char *argv[])
srand(0);
//try each test if any fail we will exit
+ RUN_TEST(test_xy_to_quadkey_z3);
+ RUN_TEST(test_xy_to_quadkey_z16);
+ RUN_TEST(test_xy_to_quadkey_z18);
RUN_TEST(test_expire_simple_z1);
RUN_TEST(test_expire_simple_z3);
RUN_TEST(test_expire_simple_z18);
+ RUN_TEST(test_expire_simple_z17_18);
+ RUN_TEST(test_expire_simple_z17_18_one_superior_tile);
RUN_TEST(test_expire_set);
RUN_TEST(test_expire_merge);
RUN_TEST(test_expire_merge_same);
diff --git a/tests/test-hstore-match-only.cpp b/tests/test-hstore-match-only.cpp
index e67d59c..b819fd1 100644
--- a/tests/test-hstore-match-only.cpp
+++ b/tests/test-hstore-match-only.cpp
@@ -55,7 +55,7 @@ int main(int argc, char *argv[]) {
auto out_test = std::make_shared<output_pgsql_t>(mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/hstore-match-only.osm", "xml",
options, &osmdata);
diff --git a/tests/test-middle-flat.cpp b/tests/test-middle-flat.cpp
index 5a4be10..e1773ea 100644
--- a/tests/test-middle-flat.cpp
+++ b/tests/test-middle-flat.cpp
@@ -85,7 +85,6 @@ int main(int argc, char *argv[]) {
try {
options_t options;
options.database_options = db->database_options;
- options.scale = 10000000;
options.cache = 1;
options.num_procs = 1;
options.prefix = "osm2pgsql_test";
diff --git a/tests/test-middle-pgsql.cpp b/tests/test-middle-pgsql.cpp
index a22af1a..537e691 100644
--- a/tests/test-middle-pgsql.cpp
+++ b/tests/test-middle-pgsql.cpp
@@ -73,7 +73,6 @@ int main(int argc, char *argv[]) {
try {
options_t options;
options.database_options = db->database_options;
- options.scale = 10000000;
options.cache = 1;
options.num_procs = 1;
options.prefix = "osm2pgsql_test";
diff --git a/tests/test-middle-ram.cpp b/tests/test-middle-ram.cpp
index 8e51482..dbb8663 100644
--- a/tests/test-middle-ram.cpp
+++ b/tests/test-middle-ram.cpp
@@ -48,7 +48,6 @@ void run_tests(const options_t options, const std::string cache_type) {
int main(int argc, char *argv[]) {
try {
options_t options;
- options.scale = 10000000;
options.cache = 1; // Non-zero cache is needed to test
options.alloc_chunkwise = ALLOC_SPARSE | ALLOC_DENSE; // what you get with optimized
diff --git a/tests/test-options-database.cpp b/tests/test-options-database.cpp
index f0e65ea..0de1d9c 100644
--- a/tests/test-options-database.cpp
+++ b/tests/test-options-database.cpp
@@ -49,25 +49,25 @@ void expect_conninfo(const database_options_t &db, const std::string &expect) {
*/
void test_conninfo() {
database_options_t db;
- expect_conninfo(db, "dbname='gis'");
+ expect_conninfo(db, "fallback_application_name='osm2pgsql'");
db.db = "foo";
- expect_conninfo(db, "dbname='foo'");
+ expect_conninfo(db, "fallback_application_name='osm2pgsql' dbname='foo'");
db = database_options_t();
db.username = "bar";
- expect_conninfo(db, "dbname='gis' user='bar'");
+ expect_conninfo(db, "fallback_application_name='osm2pgsql' user='bar'");
db = database_options_t();
db.password = "bar";
- expect_conninfo(db, "dbname='gis' password='bar'");
+ expect_conninfo(db, "fallback_application_name='osm2pgsql' password='bar'");
db = database_options_t();
db.host = "bar";
- expect_conninfo(db, "dbname='gis' host='bar'");
+ expect_conninfo(db, "fallback_application_name='osm2pgsql' host='bar'");
db = database_options_t();
db.port = "bar";
- expect_conninfo(db, "dbname='gis' port='bar'");
+ expect_conninfo(db, "fallback_application_name='osm2pgsql' port='bar'");
db = database_options_t();
db.db = "foo";
@@ -75,7 +75,7 @@ void test_conninfo() {
db.password = "baz";
db.host = "bzz";
db.port = "123";
- expect_conninfo(db, "dbname='foo' user='bar' password='baz' host='bzz' port='123'");
+ expect_conninfo(db, "fallback_application_name='osm2pgsql' dbname='foo' user='bar' password='baz' host='bzz' port='123'");
}
} // anonymous namespace
diff --git a/tests/test-options-parse.cpp b/tests/test-options-parse.cpp
index 3b280c5..7ece40f 100644
--- a/tests/test-options-parse.cpp
+++ b/tests/test-options-parse.cpp
@@ -132,6 +132,40 @@ void test_outputs()
}
}
+void test_parsing_tile_expiry_zoom_levels()
+{
+ const char *a1[] = {
+ "osm2pgsql", "-e",
+ "8--12", "--style",
+ "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ parse_fail(len(a1), a1, "Invalid maximum zoom level given for tile expiry");
+
+ const char *a2[] = {
+ "osm2pgsql", "-e",
+ "-8-12", "--style",
+ "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ parse_fail(len(a2), a2,
+ "Missing argument for option -e. Zoom levels must be positive.");
+
+ const char *a3[] = {"osm2pgsql", "-e", "--style", "default.style",
+ "tests/liechtenstein-2013-08-03.osm.pbf"};
+ parse_fail(len(a3), a3,
+ "Missing argument for option -e. Zoom levels must be positive.");
+
+ const char *a4[] = {
+ "osm2pgsql", "-e",
+ "a-8", "--style",
+ "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ parse_fail(len(a4), a4, "Missing zoom level for tile expiry.");
+
+ const char *a5[] = {
+ "osm2pgsql", "-e",
+ "6:8", "--style",
+ "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ parse_fail(len(a5), a5, "Minimum and maximum zoom level for tile expiry "
+ "must be separated by '-'.");
+}
+
int get_random_proj(std::vector<std::string>& args)
{
int proj = rand() % 3;
@@ -215,7 +249,9 @@ void test_random_perms()
args.push_back(style);
add_arg_and_val_or_not("--cache", args, options.cache, rand() % 800);
- add_arg_and_val_or_not("--database", args, options.database_options.db.c_str(), get_random_string(6));
+ if (options.database_options.db) {
+ add_arg_and_val_or_not("--database", args, options.database_options.db->c_str(), get_random_string(6));
+ }
if (options.database_options.username) {
add_arg_and_val_or_not("--username", args, options.database_options.username->c_str(), get_random_string(6));
}
@@ -265,7 +301,6 @@ void test_random_perms()
add_arg_or_not("--extra-attributes", args, options.extra_attributes);
add_arg_or_not("--multi-geometry", args, options.enable_multi);
add_arg_or_not("--keep-coastlines", args, options.keep_coastlines);
- add_arg_or_not("--exclude-invalid-polygon", args, options.excludepoly);
//add the input file
args.push_back("tests/liechtenstein-2013-08-03.osm.pbf");
@@ -289,6 +324,8 @@ int main(int argc, char *argv[])
run_test("test_middles", test_middles);
run_test("test_outputs", test_outputs);
run_test("test_random_perms", test_random_perms);
+ run_test("test_parsing_tile_expiry_zoom_levels",
+ test_parsing_tile_expiry_zoom_levels);
//passed
return 0;
diff --git a/tests/test-options-projection.cpp b/tests/test-options-projection.cpp
index 6aadff5..32994ed 100644
--- a/tests/test-options-projection.cpp
+++ b/tests/test-options-projection.cpp
@@ -45,7 +45,7 @@ static void check_tables(pg::tempdb *db, options_t &options,
options.database_options = db->database_options;
auto mid_ram = std::make_shared<middle_ram_t>();
auto out_test = std::make_shared<output_pgsql_t>(mid_ram.get(), options);
- osmdata_t osmdata(mid_ram, out_test);
+ osmdata_t osmdata(mid_ram, out_test, options.projection);
testing::parse("tests/liechtenstein-2013-08-03.osm.pbf", "pbf",
options, &osmdata);
diff --git a/tests/test-output-multi-line-storage.cpp b/tests/test-output-multi-line-storage.cpp
index 53c35d3..0badaa5 100644
--- a/tests/test-output-multi-line-storage.cpp
+++ b/tests/test-output-multi-line-storage.cpp
@@ -51,7 +51,7 @@ int main(int argc, char *argv[]) {
std::vector<std::shared_ptr<output_t> > outputs = output_t::create_outputs(middle.get(), options);
//let osmdata orchestrate between the middle and the outs
- osmdata_t osmdata(middle, outputs);
+ osmdata_t osmdata(middle, outputs, options.projection);
testing::parse("tests/test_output_multi_line_storage.osm", "xml",
options, &osmdata);
diff --git a/tests/test-output-multi-line.cpp b/tests/test-output-multi-line.cpp
index d904200..9208655 100644
--- a/tests/test-output-multi-line.cpp
+++ b/tests/test-output-multi-line.cpp
@@ -44,19 +44,19 @@ int main(int argc, char *argv[]) {
geometry_processor::create("line", &options);
export_list columns;
- { taginfo info; info.name = "highway"; info.type = "text"; columns.add(OSMTYPE_WAY, info); }
+ { taginfo info; info.name = "highway"; info.type = "text"; columns.add(osmium::item_type::way, info); }
// This actually uses the multi-backend with C transforms, not Lua transforms. This is unusual and doesn't reflect real practice
auto out_test = std::make_shared<output_multi_t>("foobar_highways", processor, columns, mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/liechtenstein-2013-08-03.osm.pbf", "pbf",
options, &osmdata);
// start a new connection to run tests on
db->check_count(1, "select count(*) from pg_catalog.pg_class where relname = 'foobar_highways'");
- db->check_count(2752, "select count(*) from foobar_highways");
+ db->check_count(2753, "select count(*) from foobar_highways");
//check that we have the right spread
db->check_count(13, "select count(*) from foobar_highways where highway='bridleway'");
@@ -65,7 +65,7 @@ int main(int argc, char *argv[]) {
db->check_count(249, "select count(*) from foobar_highways where highway='footway'");
db->check_count(18, "select count(*) from foobar_highways where highway='living_street'");
db->check_count(171, "select count(*) from foobar_highways where highway='path'");
- db->check_count(5, "select count(*) from foobar_highways where highway='pedestrian'");
+ db->check_count(6, "select count(*) from foobar_highways where highway='pedestrian'");
db->check_count(81, "select count(*) from foobar_highways where highway='primary'");
db->check_count(842, "select count(*) from foobar_highways where highway='residential'");
db->check_count(3, "select count(*) from foobar_highways where highway='road'");
diff --git a/tests/test-output-multi-point-multi-table.cpp b/tests/test-output-multi-point-multi-table.cpp
index 43c47f0..dc537d6 100644
--- a/tests/test-output-multi-point-multi-table.cpp
+++ b/tests/test-output-multi-point-multi-table.cpp
@@ -42,7 +42,7 @@ int main(int argc, char *argv[]) {
options.slim = true;
export_list columns;
- { taginfo info; info.name = "amenity"; info.type = "text"; columns.add(OSMTYPE_NODE, info); }
+ { taginfo info; info.name = "amenity"; info.type = "text"; columns.add(osmium::item_type::node, info); }
std::vector<std::shared_ptr<output_t> > outputs;
@@ -58,7 +58,7 @@ int main(int argc, char *argv[]) {
outputs.push_back(out_test);
}
- osmdata_t osmdata(mid_pgsql, outputs);
+ osmdata_t osmdata(mid_pgsql, outputs, options.projection);
testing::parse("tests/liechtenstein-2013-08-03.osm.pbf", "pbf",
options, &osmdata);
diff --git a/tests/test-output-multi-point.cpp b/tests/test-output-multi-point.cpp
index e5490d7..fd28b14 100644
--- a/tests/test-output-multi-point.cpp
+++ b/tests/test-output-multi-point.cpp
@@ -45,11 +45,11 @@ int main(int argc, char *argv[]) {
geometry_processor::create("point", &options);
export_list columns;
- { taginfo info; info.name = "amenity"; info.type = "text"; columns.add(OSMTYPE_NODE, info); }
+ { taginfo info; info.name = "amenity"; info.type = "text"; columns.add(osmium::item_type::node, info); }
auto out_test = std::make_shared<output_multi_t>("foobar_amenities", processor, columns, mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/liechtenstein-2013-08-03.osm.pbf", "pbf",
options, &osmdata);
diff --git a/tests/test-output-multi-poly-trivial.cpp b/tests/test-output-multi-poly-trivial.cpp
index e1ade0b..01ef31a 100644
--- a/tests/test-output-multi-poly-trivial.cpp
+++ b/tests/test-output-multi-poly-trivial.cpp
@@ -31,7 +31,7 @@ void run_osm2pgsql(options_t &options) {
std::vector<std::shared_ptr<output_t> > outputs = output_t::create_outputs(middle.get(), options);
//let osmdata orchestrate between the middle and the outs
- osmdata_t osmdata(middle, outputs);
+ osmdata_t osmdata(middle, outputs, options.projection);
testing::parse("tests/test_output_multi_poly_trivial.osm", "xml",
options, &osmdata);
diff --git a/tests/test-output-multi-polygon.cpp b/tests/test-output-multi-polygon.cpp
index f0086f7..1d8e88b 100644
--- a/tests/test-output-multi-polygon.cpp
+++ b/tests/test-output-multi-polygon.cpp
@@ -44,11 +44,11 @@ int main(int argc, char *argv[]) {
std::shared_ptr<geometry_processor> processor = geometry_processor::create("polygon", &options);
export_list columns;
- { taginfo info; info.name = "building"; info.type = "text"; columns.add(OSMTYPE_WAY, info); }
+ { taginfo info; info.name = "building"; info.type = "text"; columns.add(osmium::item_type::way, info); }
auto out_test = std::make_shared<output_multi_t>("foobar_buildings", processor, columns, mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/liechtenstein-2013-08-03.osm.pbf", "pbf",
options, &osmdata);
diff --git a/tests/test-output-multi-tags.cpp b/tests/test-output-multi-tags.cpp
index b8d2b0f..78d8436 100644
--- a/tests/test-output-multi-tags.cpp
+++ b/tests/test-output-multi-tags.cpp
@@ -51,7 +51,7 @@ int main(int argc, char *argv[]) {
std::vector<std::shared_ptr<output_t> > outputs = output_t::create_outputs(middle.get(), options);
//let osmdata orchestrate between the middle and the outs
- osmdata_t osmdata(middle, outputs);
+ osmdata_t osmdata(middle, outputs, options.projection);
testing::parse("tests/test_output_multi_tags.osm", "xml",
options, &osmdata);
diff --git a/tests/test-output-pgsql-area.cpp b/tests/test-output-pgsql-area.cpp
index 5fc23cc..7ab4182 100644
--- a/tests/test-output-pgsql-area.cpp
+++ b/tests/test-output-pgsql-area.cpp
@@ -72,11 +72,10 @@ void test_area_base(bool latlon, bool reproj, double expect_area_poly, double ex
if (reproj) {
options.reproject_area = true;
}
- options.scale = latlon ? 10000000 : 100;
auto out_test = std::make_shared<output_pgsql_t>(mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/test_output_pgsql_area.osm", "xml",
options, &osmdata);
diff --git a/tests/test-output-pgsql-schema.cpp b/tests/test-output-pgsql-schema.cpp
index f744949..de21537 100644
--- a/tests/test-output-pgsql-schema.cpp
+++ b/tests/test-output-pgsql-schema.cpp
@@ -77,7 +77,7 @@ void test_other_output_schema() {
auto out_test = std::make_shared<output_pgsql_t>(mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/test_output_pgsql_z_order.osm", "xml",
options, &osmdata);
diff --git a/tests/test-output-pgsql-tablespace.cpp b/tests/test-output-pgsql-tablespace.cpp
index 6b96ca2..1d897fd 100644
--- a/tests/test-output-pgsql-tablespace.cpp
+++ b/tests/test-output-pgsql-tablespace.cpp
@@ -78,7 +78,7 @@ void test_regression_simple() {
auto out_test = std::make_shared<output_pgsql_t>(mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/liechtenstein-2013-08-03.osm.pbf", "pbf",
options, &osmdata);
@@ -89,9 +89,9 @@ void test_regression_simple() {
db->assert_has_table("osm2pgsql_test_roads");
db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point");
- db->check_count(3300, "SELECT count(*) FROM osm2pgsql_test_line");
+ db->check_count(3231, "SELECT count(*) FROM osm2pgsql_test_line");
db->check_count( 375, "SELECT count(*) FROM osm2pgsql_test_roads");
- db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon");
+ db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon");
}
} // anonymous namespace
diff --git a/tests/test-output-pgsql-validgeom.cpp b/tests/test-output-pgsql-validgeom.cpp
index 280ed82..14056fb 100644
--- a/tests/test-output-pgsql-validgeom.cpp
+++ b/tests/test-output-pgsql-validgeom.cpp
@@ -73,7 +73,7 @@ void test_z_order() {
auto out_test = std::make_shared<output_pgsql_t>(mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/test_output_pgsql_validgeom.osm", "xml",
options, &osmdata);
@@ -83,7 +83,7 @@ void test_z_order() {
db->assert_has_table("osm2pgsql_test_polygon");
db->assert_has_table("osm2pgsql_test_roads");
- db->check_count(6, "SELECT COUNT(*) FROM osm2pgsql_test_polygon");
+ db->check_count(10, "SELECT COUNT(*) FROM osm2pgsql_test_polygon");
db->check_count(0, "SELECT COUNT(*) FROM osm2pgsql_test_polygon WHERE NOT ST_IsValid(way)");
db->check_count(0, "SELECT COUNT(*) FROM osm2pgsql_test_polygon WHERE ST_IsEmpty(way)");
}
diff --git a/tests/test-output-pgsql-z_order.cpp b/tests/test-output-pgsql-z_order.cpp
index 1fc865f..25d6330 100644
--- a/tests/test-output-pgsql-z_order.cpp
+++ b/tests/test-output-pgsql-z_order.cpp
@@ -73,7 +73,7 @@ void test_z_order() {
auto out_test = std::make_shared<output_pgsql_t>(mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/test_output_pgsql_z_order.osm", "xml",
options, &osmdata);
diff --git a/tests/test-output-pgsql.cpp b/tests/test-output-pgsql.cpp
index d324ea9..de1142a 100644
--- a/tests/test-output-pgsql.cpp
+++ b/tests/test-output-pgsql.cpp
@@ -77,7 +77,7 @@ void test_regression_simple() {
auto out_test = std::make_shared<output_pgsql_t>(mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/liechtenstein-2013-08-03.osm.pbf", "pbf",
options, &osmdata);
@@ -88,17 +88,17 @@ void test_regression_simple() {
db->assert_has_table("osm2pgsql_test_roads");
db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point");
- db->check_count(3300, "SELECT count(*) FROM osm2pgsql_test_line");
+ db->check_count(3231, "SELECT count(*) FROM osm2pgsql_test_line");
db->check_count( 375, "SELECT count(*) FROM osm2pgsql_test_roads");
- db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon");
+ db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon");
// Check size of lines
- db->check_number(1696.04, "SELECT ST_Length(way) FROM osm2pgsql_test_line WHERE osm_id = 44822682");
- db->check_number(1151.26, "SELECT ST_Length(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_line WHERE osm_id = 44822682");
+ db->check_number(1696.04, "SELECT ST_Length(way) FROM osm2pgsql_test_line WHERE osm_id = 1101");
+ db->check_number(1151.26, "SELECT ST_Length(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_line WHERE osm_id = 1101");
- db->check_number(311.21, "SELECT way_area FROM osm2pgsql_test_polygon WHERE osm_id = 157261342");
- db->check_number(311.21, "SELECT ST_Area(way) FROM osm2pgsql_test_polygon WHERE osm_id = 157261342");
- db->check_number(143.81, "SELECT ST_Area(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_polygon WHERE osm_id = 157261342");
+ db->check_number(311.289, "SELECT way_area FROM osm2pgsql_test_polygon WHERE osm_id = 3265");
+ db->check_number(311.289, "SELECT ST_Area(way) FROM osm2pgsql_test_polygon WHERE osm_id = 3265");
+ db->check_number(143.845, "SELECT ST_Area(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_polygon WHERE osm_id = 3265");
// Check a point's location
db->check_count(1, "SELECT count(*) FROM osm2pgsql_test_point WHERE ST_DWithin(way, 'SRID=3857;POINT(1062645.12 5972593.4)'::geometry, 0.1)");
@@ -126,11 +126,10 @@ void test_latlong() {
options.style = "default.style";
options.projection.reset(reprojection::create_projection(PROJ_LATLONG));
- options.scale = (options.projection->target_latlon()) ? 10000000 : 100;
auto out_test = std::make_shared<output_pgsql_t>(mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/liechtenstein-2013-08-03.osm.pbf", "pbf",
options, &osmdata);
@@ -141,17 +140,17 @@ void test_latlong() {
db->assert_has_table("osm2pgsql_test_roads");
db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point");
- db->check_count(3298, "SELECT count(*) FROM osm2pgsql_test_line");
+ db->check_count(3229, "SELECT count(*) FROM osm2pgsql_test_line");
db->check_count(374, "SELECT count(*) FROM osm2pgsql_test_roads");
- db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon");
+ db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon");
// Check size of lines
- db->check_number(0.0105343, "SELECT ST_Length(way) FROM osm2pgsql_test_line WHERE osm_id = 44822682");
- db->check_number(1151.26, "SELECT ST_Length(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_line WHERE osm_id = 44822682");
+ db->check_number(0.0105343, "SELECT ST_Length(way) FROM osm2pgsql_test_line WHERE osm_id = 1101");
+ db->check_number(1151.26, "SELECT ST_Length(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_line WHERE osm_id = 1101");
- db->check_number(1.70718e-08, "SELECT way_area FROM osm2pgsql_test_polygon WHERE osm_id = 157261342");
- db->check_number(1.70718e-08, "SELECT ST_Area(way) FROM osm2pgsql_test_polygon WHERE osm_id = 157261342");
- db->check_number(143.845, "SELECT ST_Area(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_polygon WHERE osm_id = 157261342");
+ db->check_number(1.70718e-08, "SELECT way_area FROM osm2pgsql_test_polygon WHERE osm_id = 3265");
+ db->check_number(1.70718e-08, "SELECT ST_Area(way) FROM osm2pgsql_test_polygon WHERE osm_id = 3265");
+ db->check_number(143.845, "SELECT ST_Area(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_polygon WHERE osm_id = 3265");
// Check a point's location
db->check_count(1, "SELECT count(*) FROM osm2pgsql_test_point WHERE ST_DWithin(way, 'SRID=4326;POINT(9.5459035 47.1866494)'::geometry, 0.00001)");
@@ -183,7 +182,7 @@ void test_area_way_simple() {
auto out_test = std::make_shared<output_pgsql_t>(mid_pgsql.get(), options);
- osmdata_t osmdata(mid_pgsql, out_test);
+ osmdata_t osmdata(mid_pgsql, out_test, options.projection);
testing::parse("tests/test_output_pgsql_way_area.osm", "xml",
options, &osmdata);
@@ -222,7 +221,7 @@ void test_route_rel() {
auto out_test = std::make_shared<output_pgsql_t>(mid_ram.get(), options);
- osmdata_t osmdata(mid_ram, out_test);
+ osmdata_t osmdata(mid_ram, out_test, options.projection);
testing::parse("tests/test_output_pgsql_route_rel.osm", "xml",
options, &osmdata);
@@ -267,7 +266,7 @@ void test_clone() {
//std::shared_ptr<middle_t> mid_clone = mid_pgsql->get_instance();
std::shared_ptr<output_t> out_clone = out_test.clone(mid_pgsql.get());
- osmdata_t osmdata(mid_pgsql, out_clone);
+ osmdata_t osmdata(mid_pgsql, out_clone, options.projection);
testing::parse("tests/liechtenstein-2013-08-03.osm.pbf", "pbf",
options, &osmdata);
@@ -278,9 +277,9 @@ void test_clone() {
db->assert_has_table("osm2pgsql_test_roads");
db->check_count(1342, "SELECT count(*) FROM osm2pgsql_test_point");
- db->check_count(3300, "SELECT count(*) FROM osm2pgsql_test_line");
+ db->check_count(3231, "SELECT count(*) FROM osm2pgsql_test_line");
db->check_count( 375, "SELECT count(*) FROM osm2pgsql_test_roads");
- db->check_count(4128, "SELECT count(*) FROM osm2pgsql_test_polygon");
+ db->check_count(4127, "SELECT count(*) FROM osm2pgsql_test_polygon");
}
} // anonymous namespace
diff --git a/tests/test-parse-diff.cpp b/tests/test-parse-diff.cpp
index 144fd99..c3d98a4 100644
--- a/tests/test-parse-diff.cpp
+++ b/tests/test-parse-diff.cpp
@@ -35,52 +35,54 @@ struct test_output_t : public dummy_output_t {
virtual ~test_output_t() = default;
- std::shared_ptr<output_t> clone(const middle_query_t *cloned_middle) const{
+ std::shared_ptr<output_t> clone(const middle_query_t *cloned_middle) const override {
test_output_t *clone = new test_output_t(m_options);
clone->m_mid = cloned_middle;
return std::shared_ptr<output_t>(clone);
}
- int node_add(osmid_t id, double, double, const taglist_t &) {
- assert(id > 0);
+ int node_add(osmium::Node const &n) override
+ {
+ assert(n.id() > 0);
++node.added;
return 0;
}
- int way_add(osmid_t id, const idlist_t &, const taglist_t &) {
- assert(id > 0);
+ int way_add(osmium::Way *w) override {
+ assert(w->id() > 0);
++way.added;
return 0;
}
- int relation_add(osmid_t id, const memberlist_t &, const taglist_t &) {
- assert(id > 0);
+ int relation_add(osmium::Relation const &r) override {
+ assert(r.id() > 0);
++rel.added;
return 0;
}
- int node_modify(osmid_t, double, double, const taglist_t &) {
+ int node_modify(osmium::Node const &) override
+ {
++node.modified;
return 0;
}
- int way_modify(osmid_t, const idlist_t &, const taglist_t &) {
+ int way_modify(osmium::Way *) override {
++way.modified;
return 0;
}
- int relation_modify(osmid_t, const memberlist_t &, const taglist_t &) {
+ int relation_modify(osmium::Relation const &) override {
++rel.modified;
return 0;
}
- int node_delete(osmid_t) {
+ int node_delete(osmid_t) override {
++node.deleted;
return 0;
}
- int way_delete(osmid_t) {
+ int way_delete(osmid_t) override {
++way.deleted;
return 0;
}
- int relation_delete(osmid_t) {
+ int relation_delete(osmid_t) override {
++rel.deleted;
return 0;
}
@@ -105,10 +107,10 @@ int main() {
options.projection = projection;
auto out_test = std::make_shared<test_output_t>(options);
- osmdata_t osmdata(std::make_shared<dummy_slim_middle_t>(), out_test);
+ osmdata_t osmdata(std::make_shared<dummy_slim_middle_t>(), out_test, options.projection);
boost::optional<std::string> bbox;
- parse_osmium_t parser(false, bbox, projection.get(), true, &osmdata);
+ parse_osmium_t parser(bbox, true, &osmdata);
parser.stream_file(inputfile, "");
diff --git a/tests/test-parse-xml2.cpp b/tests/test-parse-xml2.cpp
index 8688637..e9db2ac 100644
--- a/tests/test-parse-xml2.cpp
+++ b/tests/test-parse-xml2.cpp
@@ -11,7 +11,7 @@
#include "options.hpp"
#include "osmdata.hpp"
#include "osmtypes.hpp"
-#include "output.hpp"
+#include "output-null.hpp"
#include "parse-osmium.hpp"
void exit_nicely()
@@ -20,74 +20,53 @@ void exit_nicely()
exit(1);
}
-struct test_output_t : public output_t {
+struct test_output_t : public output_null_t {
uint64_t sum_ids, num_nodes, num_ways, num_relations, num_nds, num_members;
explicit test_output_t(const options_t &options_)
- : output_t(nullptr, options_), sum_ids(0), num_nodes(0), num_ways(0), num_relations(0),
+ : output_null_t(nullptr, options_), sum_ids(0), num_nodes(0), num_ways(0), num_relations(0),
num_nds(0), num_members(0) {
}
explicit test_output_t(const test_output_t &other)
- : output_t(other.m_mid, other.m_options), sum_ids(0), num_nodes(0), num_ways(0), num_relations(0),
+ : output_null_t(other.m_mid, other.m_options), sum_ids(0), num_nodes(0), num_ways(0), num_relations(0),
num_nds(0), num_members(0) {
}
virtual ~test_output_t() {
}
- std::shared_ptr<output_t> clone(const middle_query_t *cloned_middle) const{
+ std::shared_ptr<output_t> clone(const middle_query_t *cloned_middle) const override {
test_output_t *clone = new test_output_t(*this);
clone->m_mid = cloned_middle;
return std::shared_ptr<output_t>(clone);
}
- int node_add(osmid_t id, double lat, double lon, const taglist_t &tags) {
- assert(id > 0);
- sum_ids += id;
+ int node_add(osmium::Node const &node) override
+ {
+ assert(node.id() > 0);
+ sum_ids += (unsigned)node.id();
num_nodes += 1;
return 0;
}
- int way_add(osmid_t id, const idlist_t &nds, const taglist_t &tags) {
- assert(id > 0);
- sum_ids += id;
+ int way_add(osmium::Way *way) override {
+ assert(way->id() > 0);
+ sum_ids += (unsigned) way->id();
num_ways += 1;
- assert(nds.size() >= 0);
- num_nds += uint64_t(nds.size());
+ assert(way->nodes().size() >= 0);
+ num_nds += uint64_t(way->nodes().size());
return 0;
}
- int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) {
- assert(id > 0);
- sum_ids += id;
+ int relation_add(osmium::Relation const &rel) override {
+ assert(rel.id() > 0);
+ sum_ids += (unsigned) rel.id();
num_relations += 1;
- assert(members.size() >= 0);
- num_members += uint64_t(members.size());
+ assert(rel.members().size() >= 0);
+ num_members += uint64_t(rel.members().size());
return 0;
}
-
- int start() { return 0; }
- int connect(int startTransaction) { return 0; }
- void stop() { }
- void commit() { }
- void cleanup(void) { }
- void close(int stopTransaction) { }
-
- void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { }
- int pending_way(osmid_t id, int exists) { return 0; }
-
- void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { }
- int pending_relation(osmid_t id, int exists) { return 0; }
-
- int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags) { return 0; }
- int way_modify(osmid_t id, const idlist_t &nds, const taglist_t &tags) { return 0; }
- int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags) { return 0; }
-
- int node_delete(osmid_t id) { return 0; }
- int way_delete(osmid_t id) { return 0; }
- int relation_delete(osmid_t id) { return 0; }
-
};
@@ -107,10 +86,10 @@ int main(int argc, char *argv[]) {
options.projection = projection;
auto out_test = std::make_shared<test_output_t>(options);
- osmdata_t osmdata(std::make_shared<dummy_middle_t>(), out_test);
+ osmdata_t osmdata(std::make_shared<dummy_middle_t>(), out_test, options.projection);
boost::optional<std::string> bbox;
- parse_osmium_t parser(false, bbox, projection.get(), false, &osmdata);
+ parse_osmium_t parser(bbox, false, &osmdata);
parser.stream_file(inputfile, "");
diff --git a/tests/test-persistent-node-cache.cpp b/tests/test-persistent-node-cache.cpp
new file mode 100644
index 0000000..bc0a7c3
--- /dev/null
+++ b/tests/test-persistent-node-cache.cpp
@@ -0,0 +1,119 @@
+#include <cassert>
+#include <iostream>
+
+#include "node-persistent-cache.hpp"
+#include "options.hpp"
+
+#include "tests/common-cleanup.hpp"
+
+#define FLAT_NODES_FILE_NAME "tests/test_middle_flat.flat.nodes.bin"
+
+template <typename T>
+void assert_equal(T actual, T expected)
+{
+ if (actual != expected) {
+ std::cerr << "Expected " << expected << ", but got " << actual << ".\n";
+ exit(1);
+ }
+}
+
+void write_and_read_location(node_persistent_cache &cache, osmid_t id, double x,
+ double y)
+{
+ cache.set(id, osmium::Location(x, y));
+ assert_equal(osmium::Location(x, y), cache.get(id));
+}
+
+void read_invalid_location(node_persistent_cache &cache, osmid_t id)
+{
+ assert_equal(osmium::Location(), cache.get(id));
+}
+
+void read_location(node_persistent_cache &cache, osmid_t id, double x, double y)
+{
+ assert_equal(osmium::Location(x, y), cache.get(id));
+}
+
+void delete_location(node_persistent_cache &cache, osmid_t id)
+{
+ cache.set(id, osmium::Location());
+ assert_equal(osmium::Location(), cache.get(id));
+}
+
+void test_create()
+{
+ options_t options;
+ options.flat_node_file = boost::optional<std::string>(FLAT_NODES_FILE_NAME);
+
+ auto ram_cache = std::make_shared<node_ram_cache>(0, 0); // empty cache
+
+ node_persistent_cache cache(&options, ram_cache);
+
+ // write in order
+ write_and_read_location(cache, 10, 10.01, -45.3);
+ write_and_read_location(cache, 11, -0.4538, 22.22);
+ write_and_read_location(cache, 1058, 9.4, 9);
+ write_and_read_location(cache, 502754, 0.0, 0.0);
+
+ // write out-of-order
+ write_and_read_location(cache, 9934, -179.999, 89.1);
+
+ // read non-existing in middle
+ read_invalid_location(cache, 0);
+ read_invalid_location(cache, 1111);
+ read_invalid_location(cache, 1);
+
+ // read non-existing after the last node
+ read_invalid_location(cache, 502755);
+ read_invalid_location(cache, 7772947204);
+}
+
+void test_append()
+{
+ options_t options;
+ options.flat_node_file = boost::optional<std::string>(FLAT_NODES_FILE_NAME);
+
+ auto ram_cache = std::make_shared<node_ram_cache>(0, 0); // empty cache
+
+ node_persistent_cache cache(&options, ram_cache);
+
+ // read all previously written locations
+ read_location(cache, 10, 10.01, -45.3);
+ read_location(cache, 11, -0.4538, 22.22);
+ read_location(cache, 1058, 9.4, 9);
+ read_location(cache, 502754, 0.0, 0.0);
+ read_location(cache, 9934, -179.999, 89.1);
+
+ // everything else should still be invalid
+ read_invalid_location(cache, 0);
+ read_invalid_location(cache, 12);
+ read_invalid_location(cache, 1059);
+ read_invalid_location(cache, 1);
+ read_invalid_location(cache, 1057);
+ read_invalid_location(cache, 502753);
+ read_invalid_location(cache, 502755);
+ read_invalid_location(cache, 77729404);
+
+ // write new data in the middle
+ write_and_read_location(cache, 13, 10.01, -45.3);
+ write_and_read_location(cache, 3000, 45, 45);
+
+ // append new data
+ write_and_read_location(cache, 502755, 87, 0.45);
+ write_and_read_location(cache, 502756, 87.12, 0.46);
+ write_and_read_location(cache, 510000, 44, 0.0);
+
+ // delete existing
+ delete_location(cache, 11);
+
+ // delete non-existing
+ delete_location(cache, 21);
+}
+
+int main()
+{
+ cleanup::file flat_nodes_file(FLAT_NODES_FILE_NAME);
+
+ test_create();
+ test_append();
+}
diff --git a/tests/test_output_multi_poly_trivial.lua b/tests/test_output_multi_poly_trivial.lua
index 3bf52a3..5aebd59 100644
--- a/tests/test_output_multi_poly_trivial.lua
+++ b/tests/test_output_multi_poly_trivial.lua
@@ -12,13 +12,13 @@ function test_rels (kv, num_keys)
end
function test_members (kv, member_tags, roles, num_members)
- membersuperseeded = {}
+ membersuperseded = {}
for i = 1, num_members do
- membersuperseeded[i] = 0
+ membersuperseded[i] = 0
end
tags = kv
tags["bar"] = "baz"
- return 0, tags, membersuperseeded, 0, 0, 0
+ return 0, tags, membersuperseded, 0, 0, 0
end
diff --git a/win_fsync.h b/win_fsync.h
deleted file mode 100644
index b03b7c2..0000000
--- a/win_fsync.h
+++ /dev/null
@@ -1,71 +0,0 @@
-#ifndef WIN_FSYNC_H
-#define WIN_FSYNC_H
-
-/* Emulate fsync on platforms that lack it, primarily Windows and
- cross-compilers like MinGW.
-
- This is derived from sqlite3 sources.
- http://www.sqlite.org/cvstrac/rlog?f=sqlite/src/os_win.c
- http://www.sqlite.org/copyright.html
-
- Written by Richard W.M. Jones <rjones.at.redhat.com>
-
- Copyright (C) 2008-2014 Free Software Foundation, Inc.
-
- This library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- This library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public License
- along with this program. If not, see <http://www.gnu.org/licenses/>. */
-
-#include <windows.h>
-#include <io.h>
-#include <errno.h>
-
-inline int fsync (int fd)
-{
- HANDLE h = (HANDLE) _get_osfhandle (fd);
- DWORD err;
-
- if (h == INVALID_HANDLE_VALUE)
- {
- errno = EBADF;
- return -1;
- }
-
- if (!FlushFileBuffers (h))
- {
- /* Translate some Windows errors into rough approximations of Unix
- * errors. MSDN is useless as usual - in this case it doesn't
- * document the full range of errors.
- */
- err = GetLastError ();
- switch (err)
- {
- case ERROR_ACCESS_DENIED:
- /* For a read-only handle, fsync should succeed, even though we have
- no way to sync the access-time changes. */
- return 0;
-
- /* eg. Trying to fsync a tty. */
- case ERROR_INVALID_HANDLE:
- errno = EINVAL;
- break;
-
- default:
- errno = EIO;
- }
- return -1;
- }
-
- return 0;
-}
-
-#endif
\ No newline at end of file
diff --git a/wkb.hpp b/wkb.hpp
new file mode 100644
index 0000000..9815ec4
--- /dev/null
+++ b/wkb.hpp
@@ -0,0 +1,363 @@
+#ifndef OSM2PGSQL_WKB_HPP
+#define OSM2PGSQL_WKB_HPP
+
+#include <cassert>
+#include <cmath>
+#include <cstdint>
+#include <string>
+
+#include <osmium/geom/coordinates.hpp>
+#include <osmium/geom/factory.hpp>
+
+namespace ewkb {
+
+enum geometry_type : uint32_t
+{
+ wkb_point = 1,
+ wkb_line = 2,
+ wkb_polygon = 3,
+ wkb_multi_point = 4,
+ wkb_multi_line = 5,
+ wkb_multi_polygon = 6,
+ wkb_collection = 7,
+
+ wkb_srid = 0x20000000 // SRID-presence flag (EWKB)
+};
+
+enum wkb_byte_order_type_t : uint8_t
+{
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+ Endian = 1 // Little Endian
+#else
+ Endian = 0, // Big Endian
+#endif
+};
+
+/**
+ * Writer for EWKB data suitable for postgres.
+ *
+ * Code has been largely derived from osmium::geom::WKBFactoryImpl.
+ */
+class writer_t
+{
+
+ std::string m_data;
+ int m_srid;
+
+ size_t m_geometry_size_offset = 0;
+ size_t m_multigeometry_size_offset = 0;
+ size_t m_ring_size_offset = 0;
+
+ size_t header(std::string &str, geometry_type type, bool add_length) const
+ {
+ str_push(str, Endian);
+ str_push(str, type | wkb_srid);
+ str_push(str, m_srid);
+
+ const size_t offset = str.size();
+ if (add_length) {
+ str_push(str, static_cast<uint32_t>(0));
+ }
+ return offset;
+ }
+
+ void set_size(const size_t offset, const size_t size)
+ {
+ uint32_t s = static_cast<uint32_t>(size);
+ std::copy_n(reinterpret_cast<char *>(&s), sizeof(uint32_t),
+ &m_data[offset]);
+ }
+
+ template <typename T>
+ inline static void str_push(std::string &str, T data)
+ {
+ str.append(reinterpret_cast<const char *>(&data), sizeof(T));
+ }
+
+public:
+ inline static void write_as_hex(std::string &out, std::string const &wkb)
+ {
+ static char const *lookup_hex = "0123456789ABCDEF";
+
+ for (char c : wkb) {
+ out += lookup_hex[(c >> 4) & 0xf];
+ out += lookup_hex[c & 0xf];
+ }
+ }
+
+ explicit writer_t(int srid) : m_srid(srid) {}
+
+ void add_sub_geometry(std::string const &part) { m_data.append(part); }
+
+ void add_location(const osmium::geom::Coordinates &xy)
+ {
+ str_push(m_data, xy.x);
+ str_push(m_data, xy.y);
+ }
+
+ /* Point */
+
+ std::string make_point(const osmium::geom::Coordinates &xy) const
+ {
+ std::string data;
+ header(data, wkb_point, false);
+ str_push(data, xy.x);
+ str_push(data, xy.y);
+
+ return data;
+ }
+
+ /* LineString */
+
+ void linestring_start()
+ {
+ m_geometry_size_offset = header(m_data, wkb_line, true);
+ }
+
+ std::string linestring_finish(size_t num_points)
+ {
+ set_size(m_geometry_size_offset, num_points);
+ std::string data;
+
+ using std::swap;
+ swap(data, m_data);
+
+ return data;
+ }
+
+ /* MultiLineString */
+
+ void multilinestring_start()
+ {
+ m_multigeometry_size_offset = header(m_data, wkb_multi_line, true);
+ }
+
+ std::string multilinestring_finish(size_t num_lines)
+ {
+ set_size(m_multigeometry_size_offset, num_lines);
+ std::string data;
+
+ using std::swap;
+ swap(data, m_data);
+
+ return data;
+ }
+
+ /* Polygon */
+
+ void polygon_start()
+ {
+ m_geometry_size_offset = header(m_data, wkb_polygon, true);
+ }
+
+ void polygon_ring_start()
+ {
+ m_ring_size_offset = m_data.size();
+ str_push(m_data, static_cast<uint32_t>(0));
+ }
+
+ void polygon_ring_finish(size_t num_points)
+ {
+ set_size(m_ring_size_offset, num_points);
+ }
+
+ std::string polygon_finish(size_t num_rings)
+ {
+ set_size(m_geometry_size_offset, num_rings);
+ std::string data;
+
+ using std::swap;
+ swap(data, m_data);
+
+ return data;
+ }
+
+ /* MultiPolygon */
+
+ void multipolygon_start()
+ {
+ m_multigeometry_size_offset = header(m_data, wkb_multi_polygon, true);
+ }
+
+ std::string multipolygon_finish(size_t num_polygons)
+ {
+ set_size(m_multigeometry_size_offset, num_polygons);
+ std::string data;
+
+ using std::swap;
+ swap(data, m_data);
+
+ return data;
+ }
+};
+
+/**
+ * Class that allows to iterate over the elements of a ewkb geometry.
+ *
+ * Note: this class assumes that the wkb was created by ewkb::writer_t.
+ * It implements the exact opposite decoding.
+ */
+class parser_t
+{
+public:
+ inline static std::string wkb_from_hex(std::string const &wkb)
+ {
+ std::string out;
+
+ bool front = true;
+ char outc;
+ for (char c : wkb) {
+ c -= 48;
+ if (c > 9) {
+ c -= 7;
+ }
+ if (front) {
+ outc = char(c << 4);
+ front = false;
+ } else {
+ out += outc | c;
+ front = true;
+ }
+ }
+
+ if (out[0] != Endian)
+ throw std::runtime_error(
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+ "Geometries in the database are returned in big-endian byte order. "
+#else
+ "Geometries in the database are returned in little-endian byte order. "
+#endif
+ "osm2pgsql can only process geometries in native byte order."
+ );
+
+ return out;
+ }
+
+ explicit parser_t(char const *wkb) : m_wkb(wkb), m_pos(0) {}
+ explicit parser_t(std::string const &wkb) : m_wkb(wkb.c_str()), m_pos(0) {}
+
+ size_t save_pos() const { return m_pos; }
+ void rewind(size_t pos) { m_pos = pos; }
+
+ int read_header()
+ {
+ m_pos += sizeof(uint8_t); // skip endianess
+
+ auto type = read_data<uint32_t>();
+
+ if (type & wkb_srid) {
+ m_pos += sizeof(int); // skip srid
+ }
+
+ return type & 0xff;
+ }
+
+ uint32_t read_length() { return read_data<uint32_t>(); }
+
+ osmium::geom::Coordinates read_point()
+ {
+ auto x = read_data<double>();
+ auto y = read_data<double>();
+
+ return osmium::geom::Coordinates(x, y);
+ }
+
+ void skip_points(size_t num) { m_pos += sizeof(double) * 2 * num; }
+
+ template <typename PROJ>
+ double get_area(PROJ *proj = nullptr)
+ {
+ double total = 0;
+
+ auto type = read_header();
+
+ if (type == wkb_polygon) {
+ total = get_polygon_area(proj);
+ } else if (type == wkb_multi_polygon) {
+ auto num_poly = read_length();
+ for (unsigned i = 0; i < num_poly; ++i) {
+ auto ptype = read_header();
+ (void)ptype;
+ assert(ptype == wkb_polygon);
+
+ total += get_polygon_area(proj);
+ }
+ }
+
+ return total;
+ }
+
+private:
+ template <typename PROJ>
+ double get_polygon_area(PROJ *proj)
+ {
+ auto num_rings = read_length();
+ assert(num_rings > 0);
+
+ double total = get_ring_area(proj);
+
+ for (unsigned i = 1; i < num_rings; ++i) {
+ total -= get_ring_area(proj);
+ }
+
+ return total;
+ }
+
+ template <typename PROJ>
+ double get_ring_area(PROJ *proj)
+ {
+ // Algorithm borrowed from
+ // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon
+ // XXX numerically not stable (useless for latlon)
+ auto num_pts = read_length();
+ assert(num_pts > 3);
+
+ double total = 0;
+
+ auto prev = read_point();
+ proj->target_to_tile(&prev.y, &prev.x);
+ for (unsigned i = 1; i < num_pts; ++i) {
+ auto cur = read_point();
+ proj->target_to_tile(&cur.y, &cur.x);
+ total += prev.x * cur.y - cur.x * prev.y;
+ prev = cur;
+ }
+
+ return std::abs(total) * 0.5;
+ }
+
+ double get_ring_area(osmium::geom::IdentityProjection *)
+ {
+ // Algorithm borrowed from
+ // http://stackoverflow.com/questions/451426/how-do-i-calculate-the-area-of-a-2d-polygon
+ auto num_pts = read_length();
+ assert(num_pts > 3);
+
+ double total = 0;
+
+ auto prev = read_point();
+ for (unsigned i = 1; i < num_pts; ++i) {
+ auto cur = read_point();
+ total += prev.x * cur.y - cur.x * prev.y;
+ prev = cur;
+ }
+
+ return std::abs(total) * 0.5;
+ }
+
+ template <typename T>
+ T read_data()
+ {
+ auto *data = reinterpret_cast<T const *>(m_wkb + m_pos);
+ m_pos += sizeof(T);
+
+ return *data;
+ }
+
+ char const *m_wkb;
+ size_t m_pos;
+};
+
+} // namespace
+
+#endif // OSM2PGSQL_WKB_HPP
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/osm2pgsql.git
More information about the Pkg-grass-devel
mailing list