[osm2pgsql] 02/09: Imported Upstream version 0.87.0
Bas Couwenberg
sebastic at xs4all.nl
Wed Dec 10 19:28:41 UTC 2014
This is an automated email from the git hooks/post-receive script.
sebastic-guest pushed a commit to branch master
in repository osm2pgsql.
commit 1ba40911bfec877e32e0f0c26903c047e9e121a6
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Wed Dec 10 11:53:32 2014 +0100
Imported Upstream version 0.87.0
---
.gitignore | 24 +
.travis.yml | 44 +
Makefile.am | 149 +-
README.md | 38 +-
UTF8sanitizer.c => UTF8sanitizer.cpp | 24 +-
binarysearcharray.c => binarysearcharray.cpp | 13 +-
binarysearcharray.h => binarysearcharray.hpp | 0
build_geometry.h | 48 -
configure.ac | 10 +-
docs/osm2pgsql.1 | 2 +-
docs/usage.md | 6 +-
expire-tiles.c => expire-tiles.cpp | 314 ++--
expire-tiles.h | 14 -
expire-tiles.hpp | 63 +
build_geometry.cpp => geometry-builder.cpp | 573 ++++--
geometry-builder.hpp | 67 +
geometry-processor.cpp | 156 ++
geometry-processor.hpp | 98 ++
id-tracker.cpp | 176 ++
id-tracker.hpp | 27 +
input.c => input.cpp | 144 +-
input.h | 11 -
input.hpp | 13 +
keyvals.c | 346 ----
keyvals.cpp | 266 +++
keyvals.h | 41 -
keyvals.hpp | 50 +
m4/ax_boost_base.m4 | 272 +++
m4/ax_boost_filesystem.m4 | 118 ++
m4/ax_boost_system.m4 | 120 ++
m4/ax_boost_thread.m4 | 149 ++
mapnik-osm-updater.sh | 784 ---------
middle-pgsql.c | 1830 --------------------
middle-pgsql.cpp | 1453 ++++++++++++++++
middle-pgsql.h | 14 -
middle-pgsql.hpp | 107 ++
middle-ram.c | 387 -----
middle-ram.cpp | 364 ++++
middle-ram.h | 14 -
middle-ram.hpp | 86 +
middle.cpp | 26 +
middle.h | 49 -
middle.hpp | 78 +
...he-reader.c => node-persistent-cache-reader.cpp | 70 +-
...persistent-cache.c => node-persistent-cache.cpp | 218 +--
node-persistent-cache.h | 29 -
node-persistent-cache.hpp | 64 +
node-ram-cache.c => node-ram-cache.cpp | 170 +-
node-ram-cache.h | 53 -
node-ram-cache.hpp | 84 +
options.cpp | 543 ++++++
options.hpp | 88 +
osm2pgsql-svn.sh | 17 -
osm2pgsql.c | 796 ---------
osm2pgsql.cpp | 128 ++
osmdata.cpp | 407 +++++
osmdata.hpp | 37 +
osmtypes.h | 82 -
osmtypes.hpp | 37 +
output-gazetteer.c => output-gazetteer.cpp | 609 ++++---
output-gazetteer.h | 8 -
output-gazetteer.hpp | 79 +
output-multi.cpp | 435 +++++
output-multi.hpp | 83 +
output-null.c | 75 -
output-null.cpp | 90 +
output-null.h | 11 -
output-null.hpp | 41 +
output-pgsql.c | 1488 ----------------
output-pgsql.cpp | 762 ++++++++
output-pgsql.h | 27 -
output-pgsql.hpp | 89 +
output.cpp | 161 ++
output.h | 82 -
output.hpp | 79 +
parse-o5m.c => parse-o5m.cpp | 205 ++-
parse-o5m.h => parse-o5m.hpp | 17 +-
parse-pbf.c => parse-pbf.cpp | 323 ++--
parse-pbf.h => parse-pbf.hpp | 28 +-
parse-primitive.c | 486 ------
parse-primitive.h | 30 -
parse-xml2.c | 410 -----
parse-xml2.cpp | 426 +++++
parse-xml2.h | 30 -
parse-xml2.hpp | 63 +
parse.cpp | 229 +++
parse.hpp | 91 +
pgsql-id-tracker.cpp | 151 ++
pgsql-id-tracker.hpp | 28 +
pgsql.c | 133 --
pgsql.cpp | 180 ++
pgsql.h | 14 -
pgsql.hpp | 26 +
processor-line.cpp | 19 +
processor-line.hpp | 16 +
processor-point.cpp | 23 +
processor-point.hpp | 16 +
processor-polygon.cpp | 26 +
processor-polygon.hpp | 18 +
rb.c => rb.cpp | 53 +-
rb.h => rb.hpp | 2 +-
reprojection.c => reprojection.cpp | 132 +-
reprojection.h | 27 -
reprojection.hpp | 53 +
sanitizer.h => sanitizer.hpp | 0
sprompt.c => sprompt.cpp | 21 +-
sprompt.h => sprompt.hpp | 0
table.cpp | 558 ++++++
table.hpp | 90 +
taginfo.cpp | 215 +++
taginfo.hpp | 10 +
taginfo_impl.hpp | 50 +
tagtransform.c => tagtransform.cpp | 899 +++++-----
tagtransform.h | 29 -
tagtransform.hpp | 45 +
tests/common-pg.cpp | 173 ++
tests/common-pg.hpp | 69 +
tests/middle-tests.cpp | 176 ++
tests/middle-tests.hpp | 13 +
tests/regression-test.py | 96 +-
tests/regression-test.sh | 5 +
tests/test-expire-tiles.cpp | 365 ++++
tests/test-middle-pgsql.cpp | 80 +
tests/test-middle-ram.cpp | 50 +
tests/test-output-multi-line.cpp | 126 ++
tests/test-output-multi-point-multi-table.cpp | 147 ++
tests/test-output-multi-point.cpp | 121 ++
tests/test-output-multi-polygon.cpp | 126 ++
tests/test-output-pgsql.cpp | 311 ++++
tests/test-parse-options.cpp | 297 ++++
tests/test-parse-xml2.cpp | 162 ++
tests/test-pgsql-escape.cpp | 7 +
tests/test_output_pgsql_route_rel.osm | 15 +
tests/test_output_pgsql_way_area.osm | 13 +
text-tree.c => text-tree.cpp | 66 +-
text-tree.h | 21 -
text-tree.hpp | 21 +
util.cpp | 10 +
util.hpp | 19 +
wildcmp.c => wildcmp.cpp | 12 +-
wildcmp.h => wildcmp.hpp | 2 +-
win_fsync.h | 71 +
142 files changed, 14028 insertions(+), 9228 deletions(-)
diff --git a/.gitignore b/.gitignore
index ac18df5..3f51b71 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
*.o
+*~
Makefile.in
aclocal.m4
autom4te.cache/
@@ -36,3 +37,26 @@ libtool
.deps/
stamp-h1
INSTALL
+
+log
+test-driver
+test-suite.log
+tests/.dirstamp
+tests/test-parse-xml2
+tests/test-middle-ram
+tests/test-middle-pgsql
+tests/test-pgsql-escape
+tests/test-parse-options
+tests/test-output-multi-line
+tests/test-output-multi-point
+tests/test-output-multi-point-multi-table
+tests/test-output-multi-polygon
+tests/test-output-pgsql
+tests/test-expire-tiles
+tests/*.log
+tests/*.trs
+tests/test_output_pgsql_area_way.flat.nodes.bin
+
+.libs/
+*.lo
+libosm2pgsql.la
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..52f3c92
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,44 @@
+language: cpp
+compiler:
+ - gcc
+ - clang
+env:
+ - USE_LUA=true BOOST_PPA=false
+ - USE_LUA=true BOOST_PPA=1.54
+ - USE_LUA=true BOOST_PPA=1.55
+# Skip some parts of the matrix for speed
+# - USE_LUA=false BOOST_PPA=false
+# - USE_LUA=false BOOST_PPA=1.54
+ - USE_LUA=false BOOST_PPA=1.55
+before_install:
+ - |-
+ if [ "${BOOST_PPA}" != "false" ]; then
+ sudo add-apt-repository -y ppa:boost-latest/ppa
+ fi
+ - sudo apt-get update -qq
+install:
+ - sudo apt-get install -y -qq autoconf automake libtool make g++ libpq-dev libxml2-dev libbz2-dev libproj0 proj-bin libproj-dev # core deps
+ - |-
+ if [ "${BOOST_PPA}" = "false" ]; then
+ sudo apt-get install -y -qq libboost1.48-all-dev
+ else
+ sudo apt-get install -y -qq boost${BOOST_PPA}
+ fi
+ - sudo apt-get install -y -qq protobuf-c-compiler libprotobuf-c0-dev
+ - sudo apt-get install -y -qq libgeos-3.3.3 libgeos-dev libgeos++-dev
+ - |-
+ if [ "${USE_LUA}" = "true" ]; then
+ sudo apt-get install -y -qq lua5.2 liblua5.2-dev
+ fi
+before_script:
+ - xml2-config --version
+ - geos-config --version
+ - proj | head -n1
+ - |-
+ if [ "${USE_LUA}" = "true" ]; then
+ lua -v
+ fi
+script:
+ ./autogen.sh && ./configure && make -j2
+after_failure:
+ - cat config.log
diff --git a/Makefile.am b/Makefile.am
index 8959351..cd733f7 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,14 +1,128 @@
ACLOCAL_AMFLAGS = -I m4
+AUTOMAKE_OPTIONS = subdir-objects
bin_PROGRAMS = osm2pgsql nodecachefilereader
-
-osm2pgsql_SOURCES = build_geometry.cpp input.c middle.h middle-ram.h output-gazetteer.h output-pgsql.c rb.c sanitizer.h text-tree.h build_geometry.h input.h middle-pgsql.c osm2pgsql.c output.h output-pgsql.h rb.h sprompt.c UTF8sanitizer.c expire-tiles.c keyvals.c middle-pgsql.h osmtypes.h output-null.c parse-o5m.c parse-o5m.h parse-primitive.c parse-primitive.h parse-xml2.c parse-xml2.h pgsql.c reprojection.c sprompt.h expire-tiles.h keyvals.h middle-ram.c output-gazetteer.c output-null. [...]
-
-nodecachefilereader_SOURCES = node-persistent-cache-reader.c node-persistent-cache.c node-ram-cache.c binarysearcharray.c
-
+noinst_LTLIBRARIES = libosm2pgsql.la
+
+osm2pgsql_SOURCES = osm2pgsql.cpp \
+ binarysearcharray.hpp \
+ geometry-builder.hpp \
+ expire-tiles.hpp \
+ input.hpp \
+ keyvals.hpp \
+ middle-pgsql.hpp \
+ middle-ram.hpp \
+ middle.hpp \
+ node-persistent-cache.hpp \
+ node-ram-cache.hpp \
+ options.hpp \
+ osmdata.hpp \
+ osmtypes.hpp \
+ output-gazetteer.hpp \
+ output-null.hpp \
+ output-pgsql.hpp \
+ output.hpp \
+ parse.hpp \
+ parse-o5m.hpp \
+ parse-pbf.hpp \
+ parse-xml2.hpp \
+ pgsql.hpp \
+ rb.hpp \
+ reprojection.hpp \
+ sanitizer.hpp \
+ sprompt.hpp \
+ table.hpp \
+ text-tree.hpp \
+ util.hpp
+
+osm2pgsql_LDADD = libosm2pgsql.la
+
+libosm2pgsql_la_SOURCES = \
+ UTF8sanitizer.cpp \
+ binarysearcharray.cpp \
+ expire-tiles.cpp \
+ geometry-builder.cpp \
+ geometry-processor.cpp \
+ id-tracker.cpp \
+ input.cpp \
+ keyvals.cpp \
+ middle.cpp \
+ middle-pgsql.cpp \
+ middle-ram.cpp \
+ node-persistent-cache.cpp \
+ node-ram-cache.cpp \
+ options.cpp \
+ osmdata.cpp \
+ output.cpp \
+ output-gazetteer.cpp \
+ output-multi.cpp \
+ output-null.cpp \
+ output-pgsql.cpp \
+ parse.cpp \
+ parse-o5m.cpp \
+ parse-pbf.cpp \
+ parse-xml2.cpp \
+ pgsql.cpp \
+ pgsql-id-tracker.cpp \
+ processor-line.cpp \
+ processor-point.cpp \
+ processor-polygon.cpp \
+ rb.cpp \
+ reprojection.cpp \
+ sprompt.cpp \
+ table.cpp \
+ taginfo.cpp \
+ tagtransform.cpp \
+ text-tree.cpp \
+ util.cpp \
+ wildcmp.cpp
+
+nodecachefilereader_SOURCES = node-persistent-cache-reader.cpp
+nodecachefilereader_LDADD = libosm2pgsql.la
+
+check_PROGRAMS = \
+ tests/test-parse-xml2 \
+ tests/test-middle-ram \
+ tests/test-middle-pgsql \
+ tests/test-output-multi-line \
+ tests/test-output-multi-point \
+ tests/test-output-multi-point-multi-table \
+ tests/test-output-multi-polygon \
+ tests/test-output-pgsql \
+ tests/test-pgsql-escape \
+ tests/test-parse-options \
+ tests/test-expire-tiles
+
+tests_test_parse_xml2_SOURCES = tests/test-parse-xml2.cpp
+tests_test_parse_xml2_LDADD = libosm2pgsql.la
+tests_test_middle_ram_SOURCES = tests/test-middle-ram.cpp tests/middle-tests.cpp
+tests_test_middle_ram_LDADD = libosm2pgsql.la
+tests_test_middle_pgsql_SOURCES = tests/test-middle-pgsql.cpp tests/middle-tests.cpp tests/common-pg.cpp
+tests_test_middle_pgsql_LDADD = libosm2pgsql.la
+tests_test_output_multi_line_SOURCES = tests/test-output-multi-line.cpp tests/common-pg.cpp
+tests_test_output_multi_line_LDADD = libosm2pgsql.la
+tests_test_output_multi_point_SOURCES = tests/test-output-multi-point.cpp tests/common-pg.cpp
+tests_test_output_multi_point_LDADD = libosm2pgsql.la
+tests_test_output_multi_point_multi_table_SOURCES = tests/test-output-multi-point-multi-table.cpp tests/common-pg.cpp
+tests_test_output_multi_point_multi_table_LDADD = libosm2pgsql.la
+tests_test_output_multi_polygon_SOURCES = tests/test-output-multi-polygon.cpp tests/common-pg.cpp
+tests_test_output_multi_polygon_LDADD = libosm2pgsql.la
+tests_test_output_pgsql_SOURCES = tests/test-output-pgsql.cpp tests/common-pg.cpp
+tests_test_output_pgsql_LDADD = libosm2pgsql.la
+tests_test_pgsql_escape_SOURCES = tests/test-pgsql-escape.cpp
+tests_test_pgsql_escape_LDADD = libosm2pgsql.la
+tests_test_parse_options_SOURCES = tests/test-parse-options.cpp
+tests_test_parse_options_LDADD = libosm2pgsql.la
+tests_test_expire_tiles_SOURCES = tests/test-expire-tiles.cpp
+tests_test_expire_tiles_LDADD = libosm2pgsql.la
+
+TESTS = $(check_PROGRAMS) tests/regression-test.sh
+TEST_EXTENSIONS = .sh
+SH_LOG_COMPILER = sh
if READER_PBF
-osm2pgsql_SOURCES += parse-pbf.c parse-pbf.h fileformat.pb-c.c fileformat.pb-c.h osmformat.pb-c.c osmformat.pb-c.h
+osm2pgsql_SOURCES += parse-pbf.hpp fileformat.pb-c.h osmformat.pb-c.h
+libosm2pgsql_la_SOURCES += parse-pbf.cpp fileformat.pb-c.c osmformat.pb-c.c
fileformat.pb-c.c: protobuf/fileformat.proto
$(AM_V_GEN) $(PROTOC_C) --proto_path=protobuf --c_out=. $<
@@ -41,9 +155,22 @@ endif
osm2pgsqldir = $(datadir)/osm2pgsql
AM_CFLAGS = @PTHREAD_CFLAGS@ @LFS_CFLAGS@ @POSTGRESQL_CFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ @PROTOBUF_C_CFLAGS@ @ZLIB_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -DVERSION='"@PACKAGE_VERSION@"' @LUA_INCLUDE@
-AM_CPPFLAGS = @PTHREAD_CFLAGS@ @POSTGRESQL_CFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -Igeos-fallback @LUA_INCLUDE@
-
-AM_LDFLAGS = @PTHREAD_CFLAGS@ @ZLIB_LDFLAGS@ @ZLIB_LIBS@ @POSTGRESQL_LDFLAGS@ @POSTGRESQL_LIBS@ @XML2_LDFLAGS@ @BZIP2_LDFLAGS@ @BZIP2_LIBS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ @PROJ_LDFLAGS@ @PROJ_LIBS@ @PROTOBUF_C_LDFLAGS@ @PROTOBUF_C_LIBS@ -L/usr/lib/x86_64-linux-gnu @LUA_LIB@
+AM_CPPFLAGS = @PTHREAD_CFLAGS@ @POSTGRESQL_CFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -Igeos-fallback @LUA_INCLUDE@ @BOOST_CPPFLAGS@
+
+GLOBAL_LDFLAGS = @PTHREAD_CFLAGS@ @ZLIB_LDFLAGS@ @ZLIB_LIBS@ @POSTGRESQL_LDFLAGS@ @POSTGRESQL_LIBS@ @XML2_LDFLAGS@ @BZIP2_LDFLAGS@ @BZIP2_LIBS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ @PROJ_LDFLAGS@ @PROJ_LIBS@ @PROTOBUF_C_LDFLAGS@ @PROTOBUF_C_LIBS@ -L/usr/lib/x86_64-linux-gnu @LUA_LIB@ @BOOST_LDFLAGS@ @BOOST_FILESYSTEM_LIB@ @BOOST_SYSTEM_LIB@ @BOOST_THREAD_LIB@ -lstdc++
+osm2pgsql_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_parse_xml2_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_middle_ram_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_middle_pgsql_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_output_multi_line_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_output_multi_point_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_output_multi_point_multi_table_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_output_multi_polygon_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_output_pgsql_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_pgsql_escape_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_parse_options_LDADD += $(GLOBAL_LDFLAGS)
+tests_test_expire_tiles_LDADD += $(GLOBAL_LDFLAGS)
+nodecachefilereader_LDADD += $(GLOBAL_LDFLAGS)
osm2pgsql_DATA = default.style 900913.sql
@@ -66,6 +193,4 @@ distclean-local:
@rm -f $(PACKAGE).spec
@rm -f config.nice
-test:
- tests/regression-test.py -f tests/liechtenstein-2013-08-03.osm.pbf
- tests/regression-test.py -f tests/liechtenstein-2013-08-03.osm.bz2
+test: check
diff --git a/README.md b/README.md
index a79525b..95be409 100644
--- a/README.md
+++ b/README.md
@@ -49,16 +49,17 @@ compiler.
To install on a Debian or Ubuntu system, first install the prerequisites:
```sh
-sudo apt-get install autoconf automake libtool make g++ libxml2-dev libgeos-dev
- libgeos++-dev libpq-dev libbz2-dev libproj-dev protobuf-c-compiler
- libprotobuf-c0-dev lua5.2 liblua5.2-dev
+sudo apt-get install autoconf automake libtool make g++ libboost-dev \
+ libboost-system-dev libboost-filesystem-dev libboost-thread-dev libxml2-dev \
+ libgeos-dev libgeos++-dev libpq-dev libbz2-dev libproj-dev
+ protobuf-c-compiler libprotobuf-c0-dev lua5.2 liblua5.2-dev
```
To install on a Fedora system, use
```sh
-sudo yum install gcc-c++ libxml2-devel geos-develpostgresql-devel bzip2-devel
- proj-devel protobuf-compiler
+sudo yum install gcc-c++ boost-devel libxml2-devel geos-devel \
+ postgresql-devel bzip2-devel proj-devel protobuf-compiler
```
Then you should be able to bootstrap the build system:
@@ -100,8 +101,8 @@ When importing a large amount of data such as the complete planet, a typical
command line would be
```sh
- osm2pgsql -c -d gis --slim -C <cache size> \
- --flat-nodes <flat nodes> planet-latest.osm.pbf
+osm2pgsql -c -d gis --slim -C <cache size> \
+ --flat-nodes <flat nodes> planet-latest.osm.pbf
```
where
* ``<cache size>`` is 24000 on machines with 32GiB or more RAM
@@ -127,6 +128,29 @@ null backend for testing.
Any questions should be directed at the osm dev list
http://wiki.openstreetmap.org/index.php/Mailing_lists
+## Testing ##
+
+The code also comes with a suite of tests which can be run by
+executing ``make check``.
+
+Some of these tests depend on being able to set up a database and run
+osm2pgsql against it. You need to ensure that PostgreSQL is running
+and that your user is a superuser of that system. To do that, run:
+
+```sh
+sudo -u postgres createuser -s $USER
+sudo mkdir -p /tmp/psql-tablespace
+sudo chown postgres.postgres /tmp/psql-tablespace
+psql -c "CREATE TABLESPACE tablespacetest LOCATION '/tmp/psql-tablespace'" postgres
+```
+
+Once this is all set up, all the tests should run (no SKIPs), and pass
+(no FAILs). If you encounter a failure, you can find more information
+by looking in the `test-suite.log`. If you find something which seems
+to be a bug, please check to see if it is a known issue at
+https://github.com/openstreetmap/osm2pgsql/issues and, if it's not
+already known, report it there.
+
## Contributing ##
We welcome contributions to osm2pgsql. If you would like to report an issue,
diff --git a/UTF8sanitizer.c b/UTF8sanitizer.cpp
similarity index 91%
rename from UTF8sanitizer.c
rename to UTF8sanitizer.cpp
index 4708011..c0f2753 100644
--- a/UTF8sanitizer.c
+++ b/UTF8sanitizer.cpp
@@ -6,8 +6,8 @@
#include <zlib.h>
#include <bzlib.h>
-#include "sanitizer.h"
-#include "input.h"
+#include "sanitizer.hpp"
+#include "input.hpp"
int sanitizerClose(void *context);
int sanitizerProcess(void *context, char *buffer, int len);
@@ -21,7 +21,7 @@ int sanitizerProcess(void *context, char *buffer, int len);
* [ len = 5 ] [len = 1]
* X1 X2 X3 X4 X5 X6
*
- * OUTPUT: nothing is generated for first buffer
+ * OUTPUT: nothing is generated for first buffer
* This will itself cause caller to assume EOF (hopefully normal reader will read >> 5 bytes).
* subsequent read of len=1 whille return all 6 bytes potentially causing output buffer overflow (and overwriting input data)
*
@@ -38,13 +38,13 @@ struct Context {
int out_char[10];
int pend;
int verbose;
- void *file;
+ Input *file;
};
int sanitizerClose(void *context)
{
- struct Context *ctx = context;
+ struct Context *ctx = (struct Context *)context;
int r = inputClose(ctx->file);
if (ctx->verbose) {
@@ -64,7 +64,7 @@ int sanitizerClose(void *context)
xmlTextReaderPtr sanitizerOpen(const char *name)
{
- struct Context *ctx = malloc (sizeof(*ctx));
+ struct Context *ctx = (struct Context *)malloc(sizeof(*ctx));
if (!ctx)
return NULL;
@@ -85,9 +85,9 @@ xmlTextReaderPtr sanitizerOpen(const char *name)
}
-int sanitizerProcess(void *context, char *buffer, int len)
+int sanitizerProcess(void *context, char *buffer, int len)
{
- struct Context *ctx = context;
+ struct Context *ctx = (struct Context *)context;
int current_char, i, out = 0;
while (out < len) {
@@ -99,10 +99,10 @@ int sanitizerProcess(void *context, char *buffer, int len)
current_char=inputGetChar(ctx->file);
if (inputEof(ctx->file))
break;
-
+
if ((current_char & 128) == 0) {
/* Handle_ASCII_char(); */
- if (current_char == '\n')
+ if (current_char == '\n')
ctx->line++;
else
ctx->chars1++;
@@ -111,7 +111,7 @@ int sanitizerProcess(void *context, char *buffer, int len)
fprintf(stderr, "Error at line %lld\n", ctx->line);
buffer[out++] = '_';
ctx->state = 1;
- }
+ }
/* buffer[out++] = current_char; */
ctx->out_char[ctx->pend++] = current_char;
} else if ((current_char & (128+64)) == 128) {
@@ -125,7 +125,7 @@ int sanitizerProcess(void *context, char *buffer, int len)
}
}
} else {
- if (ctx->verbose)
+ if (ctx->verbose)
fprintf(stderr, "Error at line %lld\n", ctx->line);
buffer[out++] = '_';
ctx->state=1;
diff --git a/binarysearcharray.c b/binarysearcharray.cpp
similarity index 89%
rename from binarysearcharray.c
rename to binarysearcharray.cpp
index 237ddfa..e809c34 100644
--- a/binarysearcharray.c
+++ b/binarysearcharray.cpp
@@ -3,8 +3,8 @@
#include <unistd.h>
#include <string.h>
-#include "osmtypes.h"
-#include "binarysearcharray.h"
+#include "osmtypes.hpp"
+#include "binarysearcharray.hpp"
static int binary_search_lookup(struct binary_search_array * array, int key)
{
@@ -95,12 +95,13 @@ void binary_search_add(struct binary_search_array * array, int key,
struct binary_search_array * init_search_array(int capacity)
{
- struct binary_search_array * array = calloc(1,
- sizeof(struct binary_search_array));
- array->array = calloc(capacity + 1, sizeof(struct key_val_tuple));
+ struct binary_search_array * array =
+ (struct binary_search_array *)
+ calloc(1, sizeof(struct binary_search_array));
+ array->array = (struct key_val_tuple *)calloc(capacity + 1, sizeof(struct key_val_tuple));
if (!array->array) {
fprintf(stderr, "Out of memory trying to allocate %li bytes for binary search array\n", ((capacity + 1) * sizeof(struct key_val_tuple)));
- exit_nicely();
+ return NULL;
}
array->capacity = capacity;
array->size = 0;
diff --git a/binarysearcharray.h b/binarysearcharray.hpp
similarity index 100%
rename from binarysearcharray.h
rename to binarysearcharray.hpp
diff --git a/build_geometry.h b/build_geometry.h
deleted file mode 100644
index b13ce62..0000000
--- a/build_geometry.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
-#-----------------------------------------------------------------------------
-# Part of osm2pgsql utility
-#-----------------------------------------------------------------------------
-# By Artem Pavlenko, Copyright 2007
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#-----------------------------------------------------------------------------
-*/
-
-#ifndef BUILD_GEOMETRY_H
-#define BUILD_GEOMETRY_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "osmtypes.h"
-
-int parse_wkt(const char * wkt, struct osmNode *** xnodes, int ** xcount, int * polygon);
-
-char *get_wkt_simple(struct osmNode *, int count, int polygon);
-size_t get_wkt_split(struct osmNode *, int count, int polygon, double split_at);
-
-char* get_wkt(size_t index);
-double get_area(size_t index);
-size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int make_polygon, int enable_multi, double split_at);
-void clear_wkts();
-void exclude_broken_polygon ();
-char *get_multiline_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/configure.ac b/configure.ac
index bdefcd0..31df268 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,5 +1,5 @@
dnl Process this file with autoconf to produce a configure script.
-AC_INIT(osm2pgsql, 0.86.0)
+AC_INIT(osm2pgsql, 0.87.0)
dnl Required autoconf version
AC_PREREQ(2.61)
@@ -121,6 +121,14 @@ fi
dnl Check for pthread library
AX_PTHREAD(,[AC_MSG_ERROR([no])])
+dnl Check for Boost libraries
+AX_BOOST_BASE([1.48], , [AC_MSG_ERROR([cannot find Boost libraries, which are are required for building osm2pgsql. Please install libboost-dev.])])
+
+AX_BOOST_SYSTEM
+AX_BOOST_FILESYSTEM
+AX_BOOST_THREAD
+
+dnl Check for Lua libraries and headers
AX_PROG_LUA([5.0],[],[
AX_LUA_HEADERS([
AX_LUA_LIBS([
diff --git a/docs/osm2pgsql.1 b/docs/osm2pgsql.1
index 9aeffc3..4d96a5d 100644
--- a/docs/osm2pgsql.1
+++ b/docs/osm2pgsql.1
@@ -93,7 +93,7 @@ Prefix for table names (default: planet_osm).
.TP
\fB\-r\fR|\-\-input\-reader format
Select input format reader. Available choices are \fBlibxml2\fR
-(default) and \fBprimitive\fR for OSM XML format files, \fBo5m\fR for o5m formatted file
+(default) for OSM XML format files, \fBo5m\fR for o5m formatted file
and \fBpbf\fR for OSM PBF binary format (may not be available on all platforms).
.TP
\fB\-s\fR|\-\-slim
diff --git a/docs/usage.md b/docs/usage.md
index 1e978c2..b381f02 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -11,8 +11,7 @@ use them.
or adding to an existing one. ``--slim`` is required with ``--append``.
* ``--input-reader`` specifies the parser if the filetype can't be
- automatically detected for some reason. The ``primitive`` parser is **not**
- suggested.
+ automatically detected for some reason.
* ``--output`` specifies if the output backend is the default
[pgsql](pgsql.md), the [gazetteer](gazetteer.md) output used by Nominatim, or
@@ -70,7 +69,8 @@ database if the database server ever crashes, but are faster to import.
* ``--drop`` discards the slim tables when they are no longer needed in the
import, significantly reducing disk requirements and saving the time of
- building slim table indexes. A ``--slim --drop``
+ building slim table indexes. A ``--slim --drop`` import is generally the
+ fastest way to import the planet if updates are not required.
## Output columns options ##
diff --git a/expire-tiles.c b/expire-tiles.cpp
similarity index 51%
rename from expire-tiles.c
rename to expire-tiles.cpp
index f75352b..a5eb66f 100644
--- a/expire-tiles.c
+++ b/expire-tiles.cpp
@@ -13,28 +13,18 @@
#include <stdlib.h>
#include <string.h>
#include <errno.h>
-#include "expire-tiles.h"
-#include "output.h"
-#include "pgsql.h"
-#include "build_geometry.h"
-#include "reprojection.h"
+#include "output.hpp"
+#include "options.hpp"
+#include "geometry-builder.hpp"
+#include "pgsql.hpp"
+#include "reprojection.hpp"
#define EARTH_CIRCUMFERENCE 40075016.68
#define HALF_EARTH_CIRCUMFERENCE (EARTH_CIRCUMFERENCE / 2)
#define TILE_EXPIRY_LEEWAY 0.1 /* How many tiles worth of space to leave either side of a changed feature */
#define EXPIRE_TILES_MAX_BBOX 20000 /* Maximum width or height of a bounding box (metres) */
-struct tile {
- int complete[2][2]; /* Flags */
- struct tile * subtiles[2][2];
-};
-
-int map_width; /* not "static" since used in reprojection.c! */
-static double tile_width;
-static const struct output_options * Options;
-static struct tile * dirty = NULL;
-static int outcount;
-
+namespace {
/*
* We store the dirty tiles in an in-memory tree during runtime
* and dump them out to a file at the end. This allows us to easilly drop
@@ -50,7 +40,7 @@ static int outcount;
* which are easilly accommodated.
*/
-static int calc_complete(struct tile * tile) {
+int calc_complete(struct expire_tiles::tile * tile) {
int c;
c = tile->complete[0][0];
@@ -60,7 +50,7 @@ static int calc_complete(struct tile * tile) {
return c;
}
-static void destroy_tree(struct tile * tree) {
+void destroy_tree(struct expire_tiles::tile * tree) {
if (! tree) return;
if (tree->subtiles[0][0]) destroy_tree(tree->subtiles[0][0]);
if (tree->subtiles[0][1]) destroy_tree(tree->subtiles[0][1]);
@@ -73,13 +63,13 @@ static void destroy_tree(struct tile * tree) {
* Mark a tile as dirty.
* Returns the number of subtiles which have all their children marked as dirty.
*/
-static int _mark_tile(struct tile ** tree, int x, int y, int zoom, int this_zoom) {
+int _mark_tile(struct expire_tiles::tile ** tree, int x, int y, int zoom, int this_zoom) {
int zoom_diff = zoom - this_zoom;
int rel_x;
int rel_y;
int complete;
- if (! *tree) *tree = calloc(1, sizeof(**tree));
+ if (! *tree) *tree = (struct expire_tiles::tile *)calloc(1, sizeof(**tree));
zoom_diff = (zoom - this_zoom) - 1;
rel_x = (x >> zoom_diff) & 1;
rel_y = (y >> zoom_diff) & 1;
@@ -103,11 +93,11 @@ static int _mark_tile(struct tile ** tree, int x, int y, int zoom, int this_zoom
* Mark a tile as dirty.
* Returns the number of subtiles which have all their children marked as dirty.
*/
-static int mark_tile(struct tile ** tree_head, int x, int y, int zoom) {
+int mark_tile(struct expire_tiles::tile ** tree_head, int x, int y, int zoom) {
return _mark_tile(tree_head, x, y, zoom, 0);
}
-static void output_dirty_tile(FILE * outfile, int x, int y, int zoom, int min_zoom) {
+void output_dirty_tile_impl(FILE * outfile, int x, int y, int zoom, int min_zoom, int &outcount) {
int y_min;
int x_iter;
int y_iter;
@@ -125,83 +115,162 @@ static void output_dirty_tile(FILE * outfile, int x, int y, int zoom, int min_zo
for (x_iter = x << zoom_diff; x_iter < x_max; x_iter++) {
for (y_iter = y_min; y_iter < y_max; y_iter++) {
outcount++;
- if ((outcount <= 1) || (! (outcount % 1000))) {
- fprintf(stderr, "\rWriting dirty tile list (%iK)", outcount / 1000);
- fflush(stderr);
- }
- fprintf(outfile, "%i/%i/%i\n", out_zoom, x_iter, y_iter);
+ if ((outcount <= 1) || ((outcount % 1000) == 0)) {
+ fprintf(stderr, "\rWriting dirty tile list (%iK)", outcount / 1000);
+ fflush(stderr);
+ }
+ fprintf(outfile, "%i/%i/%i\n", out_zoom, x_iter, y_iter);
}
}
}
-static void _output_and_destroy_tree(FILE * outfile, struct tile * tree, int x, int y, int this_zoom, int min_zoom) {
+struct tile_output_file : public expire_tiles::tile_output {
+ tile_output_file(const std::string &expire_tiles_filename)
+ : outcount(0)
+ , outfile(fopen(expire_tiles_filename.c_str(), "a")) {
+ if (outfile == NULL) {
+ fprintf(stderr, "Failed to open expired tiles file (%s). Tile expiry list will not be written!\n", strerror(errno));
+ }
+ }
+
+ virtual ~tile_output_file() {
+ if (outfile) {
+ fclose(outfile);
+ }
+ }
+
+ virtual void output_dirty_tile(int x, int y, int zoom, int min_zoom) {
+ output_dirty_tile_impl(outfile, x, y, zoom, min_zoom, outcount);
+ }
+
+private:
+ int outcount;
+ FILE *outfile;
+};
+
+void _output_and_destroy_tree(expire_tiles::tile_output *output, struct expire_tiles::tile * tree, int x, int y, int this_zoom, int min_zoom) {
int sub_x = x << 1;
int sub_y = y << 1;
- FILE * ofile;
+ expire_tiles::tile_output *out;
if (! tree) return;
- ofile = outfile;
- if ((tree->complete[0][0]) && outfile) {
- output_dirty_tile(outfile, sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom);
- ofile = NULL;
+ out = output;
+ if ((tree->complete[0][0]) && output) {
+ output->output_dirty_tile(sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom);
+ out = NULL;
}
- if (tree->subtiles[0][0]) _output_and_destroy_tree(ofile, tree->subtiles[0][0], sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom);
+ if (tree->subtiles[0][0]) _output_and_destroy_tree(out, tree->subtiles[0][0], sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom);
- ofile = outfile;
- if ((tree->complete[0][1]) && outfile) {
- output_dirty_tile(outfile, sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom);
- ofile = NULL;
+ out = output;
+ if ((tree->complete[0][1]) && output) {
+ output->output_dirty_tile(sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom);
+ out = NULL;
}
- if (tree->subtiles[0][1]) _output_and_destroy_tree(ofile, tree->subtiles[0][1], sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom);
+ if (tree->subtiles[0][1]) _output_and_destroy_tree(out, tree->subtiles[0][1], sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom);
- ofile = outfile;
- if ((tree->complete[1][0]) && outfile) {
- output_dirty_tile(outfile, sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom);
- ofile = NULL;
+ out = output;
+ if ((tree->complete[1][0]) && output) {
+ output->output_dirty_tile(sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom);
+ out = NULL;
}
- if (tree->subtiles[1][0]) _output_and_destroy_tree(ofile, tree->subtiles[1][0], sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom);
+ if (tree->subtiles[1][0]) _output_and_destroy_tree(out, tree->subtiles[1][0], sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom);
- ofile = outfile;
- if ((tree->complete[1][1]) && outfile) {
- output_dirty_tile(outfile, sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom);
- ofile = NULL;
+ out = output;
+ if ((tree->complete[1][1]) && output) {
+ output->output_dirty_tile(sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom);
+ out = NULL;
}
- if (tree->subtiles[1][1]) _output_and_destroy_tree(ofile, tree->subtiles[1][1], sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom);
+ if (tree->subtiles[1][1]) _output_and_destroy_tree(out, tree->subtiles[1][1], sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom);
free(tree);
}
-static void output_and_destroy_tree(FILE * outfile, struct tile * tree) {
- _output_and_destroy_tree(outfile, tree, 0, 0, 0, Options->expire_tiles_zoom_min);
+// merge the two trees, destroying b in the process. returns the
+// number of completed subtrees.
+int _tree_merge(struct expire_tiles::tile **a,
+ struct expire_tiles::tile **b) {
+ if (*a == NULL) {
+ *a = *b;
+ *b = NULL;
+
+ } else if (*b != NULL) {
+ for (int x = 0; x < 2; ++x) {
+ for (int y = 0; y < 2; ++y) {
+ // if b is complete on a subtree, then the merged tree must
+ // be complete too.
+ if ((*b)->complete[x][y]) {
+ (*a)->complete[x][y] = (*b)->complete[x][y];
+ destroy_tree((*a)->subtiles[x][y]);
+ (*a)->subtiles[x][y] = NULL;
+
+ // but if a is already complete, don't bother moving across
+ // anything
+ } else if (!(*a)->complete[x][y]) {
+ int complete = _tree_merge(&((*a)->subtiles[x][y]), &((*b)->subtiles[x][y]));
+
+ if (complete >= 4) {
+ (*a)->complete[x][y] = 1;
+ destroy_tree((*a)->subtiles[x][y]);
+ (*a)->subtiles[x][y] = NULL;
+ }
+ }
+
+ destroy_tree((*b)->subtiles[x][y]);
+ (*b)->subtiles[x][y] = NULL;
+ }
+ }
+ }
+
+ // count the number complete, so we can return it
+ int a_complete = 0;
+ for (int x = 0; x < 2; ++x) {
+ for (int y = 0; y < 2; ++y) {
+ if ((*a != NULL) && ((*a)->complete[x][y])) {
+ ++a_complete;
+ }
+ }
+ }
+
+ return a_complete;
}
-void expire_tiles_stop(void) {
- FILE * outfile;
+} // anonymous namespace
- if (Options->expire_tiles_zoom < 0) return;
- outcount = 0;
- if ((outfile = fopen(Options->expire_tiles_filename, "a"))) {
- output_and_destroy_tree(outfile, dirty);
- fclose(outfile);
- } else {
- fprintf(stderr, "Failed to open expired tiles file (%s). Tile expiry list will not be written!\n", strerror(errno));
- }
- dirty = NULL;
+void expire_tiles::output_and_destroy(tile_output *output) {
+ _output_and_destroy_tree(output, dirty, 0, 0, 0, Options->expire_tiles_zoom_min);
+ dirty = NULL;
}
-void expire_tiles_init(const struct output_options *options) {
- Options = options;
+void expire_tiles::output_and_destroy() {
+ if (Options->expire_tiles_zoom >= 0) {
+ tile_output_file output(Options->expire_tiles_filename);
+
+ output_and_destroy(&output);
+ }
+}
+
+expire_tiles::~expire_tiles() {
+ if (dirty != NULL) {
+ destroy_tree(dirty);
+ dirty = NULL;
+ }
+}
+
+expire_tiles::expire_tiles(const struct options_t *options)
+ : Options(options), map_width(0), tile_width(0),
+ dirty(NULL)
+{
if (Options->expire_tiles_zoom < 0) return;
map_width = 1 << Options->expire_tiles_zoom;
tile_width = EARTH_CIRCUMFERENCE / map_width;
}
-static void expire_tile(int x, int y) {
+void expire_tiles::expire_tile(int x, int y) {
mark_tile(&dirty, x, y, Options->expire_tiles_zoom);
}
-static int normalise_tile_x_coord(int x) {
+int expire_tiles::normalise_tile_x_coord(int x) {
x %= map_width;
if (x < 0) x = (map_width - x) + 1;
return x;
@@ -210,7 +279,7 @@ static int normalise_tile_x_coord(int x) {
/*
* Expire tiles that a line crosses
*/
-static void expire_tiles_from_line(double lon_a, double lat_a, double lon_b, double lat_b) {
+void expire_tiles::from_line(double lon_a, double lat_a, double lon_b, double lat_b) {
double tile_x_a;
double tile_y_a;
double tile_x_b;
@@ -231,8 +300,8 @@ static void expire_tiles_from_line(double lon_a, double lat_a, double lon_b, dou
int y;
int norm_x;
- coords_to_tile(&tile_x_a, &tile_y_a, lon_a, lat_a);
- coords_to_tile(&tile_x_b, &tile_y_b, lon_b, lat_b);
+ Options->projection->coords_to_tile(&tile_x_a, &tile_y_a, lon_a, lat_a, map_width);
+ Options->projection->coords_to_tile(&tile_x_b, &tile_y_b, lon_b, lat_b, map_width);
if (tile_x_a > tile_x_b) {
/* We always want the line to go from left to right - swap the ends if it doesn't */
@@ -261,7 +330,7 @@ static void expire_tiles_from_line(double lon_a, double lat_a, double lon_b, dou
hyp_len = sqrt(pow(x_len, 2) + pow(y_len, 2)); /* Pythagoras */
x_step = x_len / hyp_len;
y_step = y_len / hyp_len;
-
+
for (step = 0; step <= hyp_len; step+= 0.4) {
/* Interpolate points 1 tile width apart */
next_step = step + 0.4;
@@ -270,7 +339,7 @@ static void expire_tiles_from_line(double lon_a, double lat_a, double lon_b, dou
y1 = tile_y_a + ((double)step * y_step);
x2 = tile_x_a + ((double)next_step * x_step);
y2 = tile_y_a + ((double)next_step * y_step);
-
+
/* The line (x1,y1),(x2,y2) is up to 1 tile width long
x1 will always be <= x2
We could be smart and figure out the exact tiles intersected,
@@ -293,7 +362,7 @@ static void expire_tiles_from_line(double lon_a, double lat_a, double lon_b, dou
/*
* Expire tiles within a bounding box
*/
-int expire_tiles_from_bbox(double min_lon, double min_lat, double max_lon, double max_lat) {
+int expire_tiles::from_bbox(double min_lon, double min_lat, double max_lon, double max_lat) {
double width;
double height;
int min_tile_x;
@@ -314,8 +383,8 @@ int expire_tiles_from_bbox(double min_lon, double min_lat, double max_lon, doubl
if (width > HALF_EARTH_CIRCUMFERENCE + 1) {
/* Over half the planet's width within the bounding box - assume the
box crosses the international date line and split it into two boxes */
- ret = expire_tiles_from_bbox(-HALF_EARTH_CIRCUMFERENCE, min_lat, min_lon, max_lat);
- ret += expire_tiles_from_bbox(max_lon, min_lat, HALF_EARTH_CIRCUMFERENCE, max_lat);
+ ret = from_bbox(-HALF_EARTH_CIRCUMFERENCE, min_lat, min_lon, max_lat);
+ ret += from_bbox(max_lon, min_lat, HALF_EARTH_CIRCUMFERENCE, max_lat);
return ret;
}
@@ -324,12 +393,12 @@ int expire_tiles_from_bbox(double min_lon, double min_lat, double max_lon, doubl
/* Convert the box's Mercator coordinates into tile coordinates */
- coords_to_tile(&tmp_x, &tmp_y, min_lon, max_lat);
- min_tile_x = tmp_x - TILE_EXPIRY_LEEWAY;
- min_tile_y = tmp_y - TILE_EXPIRY_LEEWAY;
- coords_to_tile(&tmp_x, &tmp_y, max_lon, min_lat);
- max_tile_x = tmp_x + TILE_EXPIRY_LEEWAY;
- max_tile_y = tmp_y + TILE_EXPIRY_LEEWAY;
+ Options->projection->coords_to_tile(&tmp_x, &tmp_y, min_lon, max_lat, map_width);
+ min_tile_x = tmp_x - TILE_EXPIRY_LEEWAY;
+ min_tile_y = tmp_y - TILE_EXPIRY_LEEWAY;
+ Options->projection->coords_to_tile(&tmp_x, &tmp_y, max_lon, min_lat, map_width);
+ max_tile_x = tmp_x + TILE_EXPIRY_LEEWAY;
+ max_tile_y = tmp_y + TILE_EXPIRY_LEEWAY;
if (min_tile_x < 0) min_tile_x = 0;
if (min_tile_y < 0) min_tile_y = 0;
if (max_tile_x > map_width) max_tile_x = map_width;
@@ -343,7 +412,7 @@ int expire_tiles_from_bbox(double min_lon, double min_lat, double max_lon, doubl
return 0;
}
-void expire_tiles_from_nodes_line(struct osmNode * nodes, int count) {
+void expire_tiles::from_nodes_line(const struct osmNode * nodes, int count) {
int i;
double last_lat;
double last_lon;
@@ -353,11 +422,11 @@ void expire_tiles_from_nodes_line(struct osmNode * nodes, int count) {
last_lat = nodes[0].lat;
last_lon = nodes[0].lon;
if (count < 2) {
- expire_tiles_from_bbox(last_lon, last_lat, last_lon, last_lat);
+ from_bbox(last_lon, last_lat, last_lon, last_lat);
return;
}
for (i = 1; i < count; i ++) {
- expire_tiles_from_line(last_lon, last_lat, nodes[i].lon, nodes[i].lat);
+ from_line(last_lon, last_lat, nodes[i].lon, nodes[i].lat);
last_lat = nodes[i].lat;
last_lon = nodes[i].lon;
}
@@ -366,14 +435,14 @@ void expire_tiles_from_nodes_line(struct osmNode * nodes, int count) {
/*
* Calculate a bounding box from a list of nodes and expire all tiles within it
*/
-void expire_tiles_from_nodes_poly(struct osmNode * nodes, int count, osmid_t osm_id) {
+void expire_tiles::from_nodes_poly(const struct osmNode * nodes, int count, osmid_t osm_id) {
int i;
int got_coords = 0;
double min_lon = 0.0;
double min_lat = 0.0;
double max_lon = 0.0;
double max_lat = 0.0;
-
+
if (Options->expire_tiles_zoom < 0) return;
for (i = 0; i < count; i++) {
if ((! got_coords) || (nodes[i].lon < min_lon)) min_lon = nodes[i].lon;
@@ -383,36 +452,36 @@ void expire_tiles_from_nodes_poly(struct osmNode * nodes, int count, osmid_t osm
got_coords = 1;
}
if (got_coords) {
- if (expire_tiles_from_bbox(min_lon, min_lat, max_lon, max_lat)) {
+ if (from_bbox(min_lon, min_lat, max_lon, max_lat)) {
/* Bounding box too big - just expire tiles on the line */
fprintf(stderr, "\rLarge polygon (%.0f x %.0f metres, OSM ID %" PRIdOSMID ") - only expiring perimeter\n", max_lon - min_lon, max_lat - min_lat, osm_id);
- expire_tiles_from_nodes_line(nodes, count);
+ from_nodes_line(nodes, count);
}
}
}
-static void expire_tiles_from_xnodes_poly(struct osmNode ** xnodes, int * xcount, osmid_t osm_id) {
+void expire_tiles::from_xnodes_poly(const struct osmNode * const * xnodes, int * xcount, osmid_t osm_id) {
int i;
- for (i = 0; xnodes[i]; i++) expire_tiles_from_nodes_poly(xnodes[i], xcount[i], osm_id);
+ for (i = 0; xnodes[i]; i++) from_nodes_poly(xnodes[i], xcount[i], osm_id);
}
-static void expire_tiles_from_xnodes_line(struct osmNode ** xnodes, int * xcount) {
+void expire_tiles::from_xnodes_line(const struct osmNode * const * xnodes, int * xcount) {
int i;
- for (i = 0; xnodes[i]; i++) expire_tiles_from_nodes_line(xnodes[i], xcount[i]);
+ for (i = 0; xnodes[i]; i++) from_nodes_line(xnodes[i], xcount[i]);
}
-void expire_tiles_from_wkt(const char * wkt, osmid_t osm_id) {
+void expire_tiles::from_wkt(const char * wkt, osmid_t osm_id) {
struct osmNode ** xnodes;
int * xcount;
int polygon;
int i;
if (Options->expire_tiles_zoom < 0) return;
- if (! parse_wkt(wkt, &xnodes, &xcount, &polygon)) {
- if (polygon) expire_tiles_from_xnodes_poly(xnodes, xcount, osm_id);
- else expire_tiles_from_xnodes_line(xnodes, xcount);
+ if (! geometry_builder::parse_wkt(wkt, &xnodes, &xcount, &polygon)) {
+ if (polygon) from_xnodes_poly(xnodes, xcount, osm_id);
+ else from_xnodes_line(xnodes, xcount);
for (i = 0; xnodes[i]; i++) free(xnodes[i]);
free(xnodes);
free(xcount);
@@ -429,29 +498,38 @@ void expire_tiles_from_wkt(const char * wkt, osmid_t osm_id) {
* of elements that refer to the osm_id.
*/
-int expire_tiles_from_db(PGconn * sql_conn, osmid_t osm_id) {
- PGresult * res;
- char * wkt;
- int i, noElements = 0;
- char const *paramValues[1];
- char tmp[16];
-
- if (Options->expire_tiles_zoom < 0) return -1;
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, osm_id);
- paramValues[0] = tmp;
-
- /* The prepared statement get_wkt will behave differently depending on the sql_conn
- * each table has its own sql_connection with the get_way refering to the approriate table
- */
- res = pgsql_execPrepared(sql_conn, "get_wkt", 1, (const char * const *)paramValues, PGRES_TUPLES_OK);
- noElements = PQntuples(res);
-
- for (i = 0; i < noElements; i++) {
- wkt = PQgetvalue(res, i, 0);
- expire_tiles_from_wkt(wkt, osm_id);
- }
- PQclear(res);
- return noElements;
+int expire_tiles::from_db(table_t* table, osmid_t osm_id) {
+ //bail if we dont care about expiry
+ if (Options->expire_tiles_zoom < 0)
+ return -1;
+
+ //grab the geom for this id
+ boost::shared_ptr<table_t::wkt_reader> wkts = table->get_wkt_reader(osm_id);
+
+ //dirty the stuff
+ const char* wkt = NULL;
+ while((wkt = wkts->get_next()))
+ from_wkt(wkt, osm_id);
+
+ //return how many rows were affected
+ return wkts->get_count();
}
+void expire_tiles::merge_and_destroy(expire_tiles &other) {
+ if (map_width != other.map_width) {
+ throw std::runtime_error((boost::format("Unable to merge tile expiry sets when "
+ "map_width does not match: %1% != %2%.")
+ % map_width % other.map_width).str());
+ }
+ if (tile_width != other.tile_width) {
+ throw std::runtime_error((boost::format("Unable to merge tile expiry sets when "
+ "tile_width does not match: %1% != %2%.")
+ % tile_width % other.tile_width).str());
+ }
+
+ _tree_merge(&dirty, &other.dirty);
+
+ destroy_tree(other.dirty);
+ other.dirty = NULL;
+}
diff --git a/expire-tiles.h b/expire-tiles.h
deleted file mode 100644
index 80d88c0..0000000
--- a/expire-tiles.h
+++ /dev/null
@@ -1,14 +0,0 @@
-#ifndef EXPIRE_TILES_H
-#define EXPIRE_TILES_H
-
-#include "output.h"
-
-void expire_tiles_init(const struct output_options *options);
-void expire_tiles_stop(void);
-int expire_tiles_from_bbox(double min_lon, double min_lat, double max_lon, double max_lat);
-void expire_tiles_from_nodes_line(struct osmNode * nodes, int count);
-void expire_tiles_from_nodes_poly(struct osmNode * nodes, int count, osmid_t osm_id);
-void expire_tiles_from_wkt(const char * wkt, osmid_t osm_id);
-int expire_tiles_from_db(PGconn * sql_conn, osmid_t osm_id);
-
-#endif
diff --git a/expire-tiles.hpp b/expire-tiles.hpp
new file mode 100644
index 0000000..6d36eaf
--- /dev/null
+++ b/expire-tiles.hpp
@@ -0,0 +1,63 @@
+#ifndef EXPIRE_TILES_H
+#define EXPIRE_TILES_H
+
+#include "table.hpp"
+#include "options.hpp"
+
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+
+struct expire_tiles : public boost::noncopyable {
+ explicit expire_tiles(const struct options_t *options);
+ ~expire_tiles();
+
+ //TODO: copy constructor
+
+ int from_bbox(double min_lon, double min_lat, double max_lon, double max_lat);
+ void from_nodes_line(const struct osmNode * nodes, int count);
+ void from_nodes_poly(const struct osmNode * nodes, int count, osmid_t osm_id);
+ void from_wkt(const char * wkt, osmid_t osm_id);
+ int from_db(table_t* table, osmid_t osm_id);
+
+ struct tile {
+ int complete[2][2];
+ struct tile* subtiles[2][2];
+ };
+
+ /* customisable tile output. this can be passed into the
+ * `output_and_destroy` function to override output to a file.
+ * this is primarily useful for testing.
+ */
+ struct tile_output {
+ virtual ~tile_output() {}
+ // dirty a tile at x, y & zoom, and all descendants of that
+ // tile at the given zoom if zoom < min_zoom.
+ virtual void output_dirty_tile(int x, int y, int zoom, int min_zoom) = 0;
+ };
+
+ // output the list of expired tiles to a file. note that this
+ // consumes the list of expired tiles destructively.
+ void output_and_destroy();
+
+ // output the list of expired tiles using a `tile_output`
+ // functor. this consumes the list of expired tiles destructively.
+ void output_and_destroy(tile_output *output);
+
+ // merge the list of expired tiles in the other object into this
+ // object, destroying the list in the other object.
+ void merge_and_destroy(expire_tiles &);
+
+private:
+ void expire_tile(int x, int y);
+ int normalise_tile_x_coord(int x);
+ void from_line(double lon_a, double lat_a, double lon_b, double lat_b);
+ void from_xnodes_poly(const struct osmNode * const * xnodes, int * xcount, osmid_t osm_id);
+ void from_xnodes_line(const struct osmNode * const * xnodes, int * xcount);
+
+ int map_width;
+ double tile_width;
+ const struct options_t *Options;
+ struct tile *dirty;
+};
+
+#endif
diff --git a/build_geometry.cpp b/geometry-builder.cpp
similarity index 56%
rename from build_geometry.cpp
rename to geometry-builder.cpp
index 6bac59a..8ae8020 100644
--- a/build_geometry.cpp
+++ b/geometry-builder.cpp
@@ -8,12 +8,12 @@
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
-#
+#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
-#
+#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
@@ -66,16 +66,50 @@ using namespace geos::operation::linemerge;
using namespace geos;
#endif
-#include "build_geometry.h"
+#include "geometry-builder.hpp"
typedef std::auto_ptr<Geometry> geom_ptr;
-static std::vector<std::string> wkts;
-static std::vector<double> areas;
+namespace {
+
+int coords2nodes(CoordinateSequence * coords, struct osmNode ** nodes) {
+ size_t num_coords;
+ size_t i;
+ Coordinate coord;
-static int excludepoly = 0;
+ num_coords = coords->getSize();
+ *nodes = (struct osmNode *) malloc(num_coords * sizeof(struct osmNode));
+
+ for (i = 0; i < num_coords; i++) {
+ coord = coords->getAt(i);
+ (*nodes)[i].lon = coord.x;
+ (*nodes)[i].lat = coord.y;
+ }
+ return num_coords;
+}
-char *get_wkt_simple(osmNode *nodes, int count, int polygon) {
+struct polygondata
+{
+ Polygon* polygon;
+ LinearRing* ring;
+ double area;
+ int iscontained;
+ unsigned containedbyid;
+};
+
+int polygondata_comparearea(const void* vp1, const void* vp2)
+{
+ const polygondata* p1 = (const polygondata*)vp1;
+ const polygondata* p2 = (const polygondata*)vp2;
+
+ if (p1->area == p2->area) return 0;
+ if (p1->area > p2->area) return -1;
+ return 1;
+}
+} // anonymous namespace
+
+geometry_builder::maybe_wkt_t geometry_builder::get_wkt_simple(const osmNode *nodes, int count, int polygon) const
+{
GeometryFactory gf;
std::auto_ptr<CoordinateSequence> coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
@@ -88,66 +122,54 @@ char *get_wkt_simple(osmNode *nodes, int count, int polygon) {
coords->add(c, 0);
}
+ maybe_wkt_t wkt(new geometry_builder::wkt_t());
geom_ptr geom;
if (polygon && (coords->getSize() >= 4) && (coords->getAt(coords->getSize() - 1).equals2D(coords->getAt(0)))) {
std::auto_ptr<LinearRing> shell(gf.createLinearRing(coords.release()));
geom = geom_ptr(gf.createPolygon(shell.release(), new std::vector<Geometry *>));
if (!geom->isValid()) {
if (excludepoly) {
- return NULL;
- } else {
+ throw std::runtime_error("Excluding broken polygon.");
+ } else {
geom = geom_ptr(geom->buffer(0));
}
}
geom->normalize(); // Fix direction of ring
+ wkt->area = geom->getArea();
} else {
if (coords->getSize() < 2)
- return NULL;
+ throw std::runtime_error("Excluding degenerate line.");
geom = geom_ptr(gf.createLineString(coords.release()));
+ wkt->area = 0;
}
- WKTWriter wktw;
- std::string wkt = wktw.write(geom.get());
- return strdup(wkt.c_str());
+ wkt->geom = WKTWriter().write(geom.get());
+ return wkt;
}
- catch (std::bad_alloc)
+ catch (std::bad_alloc&)
{
std::cerr << std::endl << "Exception caught processing way. You are likelly running out of memory." << std::endl;
std::cerr << "Try in slim mode, using -s parameter." << std::endl;
- return NULL;
+ }
+ catch (std::runtime_error& e)
+ {
+ //std::cerr << std::endl << "Exception caught processing way: " << e.what() << std::endl;
}
catch (...)
{
std::cerr << std::endl << "Exception caught processing way" << std::endl;
- return NULL;
}
-}
-
-// helper method to add the WKT for a geometry to the
-// global wkts list - used primarily for polygons.
-void add_wkt(geom_ptr &geom, double area) {
- WKTWriter wktw;
- std::string wkt = wktw.write(geom.get());
- wkts.push_back(wkt);
- areas.push_back(area);
-}
-// helper method to add the WKT for a line built from a
-// coordinate sequence to the global wkts list.
-void add_wkt_line(GeometryFactory &gf, std::auto_ptr<CoordinateSequence> &segment) {
- WKTWriter wktw;
- geom_ptr geom = geom_ptr(gf.createLineString(segment.release()));
- std::string wkt = wktw.write(geom.get());
- wkts.push_back(wkt);
- areas.push_back(0);
- segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
+ return maybe_wkt_t();
}
-size_t get_wkt_split(osmNode *nodes, int count, int polygon, double split_at) {
+geometry_builder::maybe_wkts_t geometry_builder::get_wkt_split(const osmNode *nodes, int count, int polygon, double split_at) const
+{
GeometryFactory gf;
std::auto_ptr<CoordinateSequence> coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
- double area;
- size_t wkt_size = 0;
+ WKTWriter writer;
+ //TODO: use count to get some kind of hint of how much we should reserve?
+ maybe_wkts_t wkts(new std::vector<geometry_builder::wkt_t>);
try
{
@@ -164,18 +186,22 @@ size_t get_wkt_split(osmNode *nodes, int count, int polygon, double split_at) {
geom = geom_ptr(gf.createPolygon(shell.release(), new std::vector<Geometry *>));
if (!geom->isValid()) {
if (excludepoly) {
- return 0;
+ throw std::runtime_error("Excluding broken polygon.");
} else {
geom = geom_ptr(geom->buffer(0));
}
}
geom->normalize(); // Fix direction of ring
- area = geom->getArea();
- add_wkt(geom, area);
+
+ //copy of an empty one should be cheapest
+ wkts->push_back(geometry_builder::wkt_t());
+ //then we set on the one we already have
+ wkts->back().geom = writer.write(geom.get());
+ wkts->back().area = geom->getArea();
} else {
if (coords->getSize() < 2)
- return 0;
+ throw std::runtime_error("Excluding degenerate line.");
double distance = 0;
std::auto_ptr<CoordinateSequence> segment;
@@ -185,7 +211,7 @@ size_t get_wkt_split(osmNode *nodes, int count, int polygon, double split_at) {
const Coordinate this_pt = coords->getAt(i);
const Coordinate prev_pt = coords->getAt(i-1);
const double delta = this_pt.distance(prev_pt);
- // figure out if the addition of this point would take the total
+ // figure out if the addition of this point would take the total
// length of the line in `segment` over the `split_at` distance.
const size_t splits = std::floor((distance + delta) / split_at);
@@ -198,7 +224,15 @@ size_t get_wkt_split(osmNode *nodes, int count, int polygon, double split_at) {
const Coordinate interpolated(frac * (this_pt.x - prev_pt.x) + prev_pt.x,
frac * (this_pt.y - prev_pt.y) + prev_pt.y);
segment->add(interpolated);
- add_wkt_line(gf, segment);
+ geom_ptr geom = geom_ptr(gf.createLineString(segment.release()));
+
+ //copy of an empty one should be cheapest
+ wkts->push_back(geometry_builder::wkt_t());
+ //then we set on the one we already have
+ wkts->back().geom = writer.write(geom.get());
+ wkts->back().area = 0;
+
+ segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
segment->add(interpolated);
}
// reset the distance based on the final splitting point for
@@ -216,70 +250,36 @@ size_t get_wkt_split(osmNode *nodes, int count, int polygon, double split_at) {
// on the last iteration, close out the line.
if (i == coords->getSize()-1) {
- add_wkt_line(gf, segment);
+ geom_ptr geom = geom_ptr(gf.createLineString(segment.release()));
+
+ //copy of an empty one should be cheapest
+ wkts->push_back(geometry_builder::wkt_t());
+ //then we set on the one we already have
+ wkts->back().geom = writer.write(geom.get());
+ wkts->back().area = 0;
+
+ segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
}
}
}
-
- // ensure the number of wkts in the global list is accurate.
- wkt_size = wkts.size();
}
- catch (std::bad_alloc)
+ catch (std::bad_alloc&)
{
- std::cerr << std::endl << "Exception caught processing way. You are likelly running out of memory." << std::endl;
+ std::cerr << std::endl << "Exception caught processing way. You are likely running out of memory." << std::endl;
std::cerr << "Try in slim mode, using -s parameter." << std::endl;
- wkt_size = 0;
+ }
+ catch (std::runtime_error& e)
+ {
+ //std::cerr << std::endl << "Exception caught processing way: " << e.what() << std::endl;
}
catch (...)
{
std::cerr << std::endl << "Exception caught processing way" << std::endl;
- wkt_size = 0;
}
- return wkt_size;
-}
-
-
-char * get_wkt(size_t index)
-{
-// return wkts[index].c_str();
- char *result;
- result = (char*) std::malloc( wkts[index].length() + 1);
- // At least give some idea of why we about to seg fault
- if (!result)
- std::cerr << std::endl << "Unable to allocate memory: " << (wkts[index].length() + 1) << std::endl;
- else
- std::strcpy(result, wkts[index].c_str());
- return result;
-}
-
-double get_area(size_t index)
-{
- return areas[index];
+ return wkts;
}
-void clear_wkts()
-{
- wkts.clear();
- areas.clear();
-}
-
-static int coords2nodes(CoordinateSequence * coords, struct osmNode ** nodes) {
- size_t num_coords;
- size_t i;
- Coordinate coord;
-
- num_coords = coords->getSize();
- *nodes = (struct osmNode *) malloc(num_coords * sizeof(struct osmNode));
-
- for (i = 0; i < num_coords; i++) {
- coord = coords->getAt(i);
- (*nodes)[i].lon = coord.x;
- (*nodes)[i].lat = coord.y;
- }
- return num_coords;
-}
-
-int parse_wkt(const char * wkt, struct osmNode *** xnodes, int ** xcount, int * polygon) {
+int geometry_builder::parse_wkt(const char * wkt, struct osmNode *** xnodes, int ** xcount, int * polygon) {
GeometryFactory gf;
WKTReader reader(&gf);
std::string wkt_string(wkt);
@@ -289,7 +289,7 @@ int parse_wkt(const char * wkt, struct osmNode *** xnodes, int ** xcount, int *
CoordinateSequence * coords;
size_t num_geometries;
size_t i;
-
+
*polygon = 0;
try {
geometry = reader.read(wkt_string);
@@ -342,40 +342,23 @@ int parse_wkt(const char * wkt, struct osmNode *** xnodes, int ** xcount, int *
return 0;
}
-struct polygondata
-{
- Polygon* polygon;
- LinearRing* ring;
- double area;
- int iscontained;
- unsigned containedbyid;
-};
-
-static int polygondata_comparearea(const void* vp1, const void* vp2)
+geometry_builder::maybe_wkts_t geometry_builder::build_polygons(const osmNode * const * xnodes, const int *xcount, bool enable_multi, osmid_t osm_id) const
{
- const polygondata* p1 = (const polygondata*)vp1;
- const polygondata* p2 = (const polygondata*)vp2;
-
- if (p1->area == p2->area) return 0;
- if (p1->area > p2->area) return -1;
- return 1;
-}
-
-size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int make_polygon, int enable_multi, double split_at) {
- size_t wkt_size = 0;
std::auto_ptr<std::vector<Geometry*> > lines(new std::vector<Geometry*>);
GeometryFactory gf;
geom_ptr geom;
#ifdef HAS_PREPARED_GEOMETRIES
geos::geom::prep::PreparedGeometryFactory pgf;
#endif
+ maybe_wkts_t wkts(new std::vector<geometry_builder::wkt_t>);
+
try
{
for (int c=0; xnodes[c]; c++) {
std::auto_ptr<CoordinateSequence> coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
for (int i = 0; i < xcount[c]; i++) {
- struct osmNode *nodes = xnodes[c];
+ const osmNode *nodes = xnodes[c];
Coordinate c;
c.x = nodes[i].lon;
c.y = nodes[i].lat;
@@ -403,13 +386,13 @@ size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int
for (unsigned i=0 ;i < merged->size(); ++i)
{
std::auto_ptr<LineString> pline ((*merged ) [i]);
- if (make_polygon && pline->getNumPoints() > 3 && pline->isClosed())
+ if (pline->getNumPoints() > 3 && pline->isClosed())
{
polys[totalpolys].polygon = gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0);
polys[totalpolys].ring = gf.createLinearRing(pline->getCoordinates());
polys[totalpolys].area = polys[totalpolys].polygon->getArea();
polys[totalpolys].iscontained = 0;
- polys[totalpolys].containedbyid = 0;
+ polys[totalpolys].containedbyid = 0;
if (polys[totalpolys].area > 0.0)
totalpolys++;
else {
@@ -417,32 +400,6 @@ size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int
delete(polys[totalpolys].ring);
}
}
- else
- {
- //std::cerr << "polygon(" << osm_id << ") is no good: points(" << pline->getNumPoints() << "), closed(" << pline->isClosed() << "). " << writer.write(pline.get()) << std::endl;
- double distance = 0;
- std::auto_ptr<CoordinateSequence> segment;
- segment = std::auto_ptr<CoordinateSequence>(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
- segment->add(pline->getCoordinateN(0));
- for(unsigned i=1; i<pline->getNumPoints(); i++) {
- segment->add(pline->getCoordinateN(i));
- distance += pline->getCoordinateN(i).distance(pline->getCoordinateN(i-1));
- if ((distance >= split_at) || (i == pline->getNumPoints()-1)) {
- geom = geom_ptr(gf.createLineString(segment.release()));
- std::string wkt = writer.write(geom.get());
- wkts.push_back(wkt);
- areas.push_back(0);
- wkt_size++;
- distance=0;
- segment = std::auto_ptr<CoordinateSequence>(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
- segment->add(pline->getCoordinateN(i));
- }
- }
- //std::string text = writer.write(pline.get());
- //wkts.push_back(text);
- //areas.push_back(0.0);
- //wkt_size++;
- }
}
if (totalpolys)
@@ -482,20 +439,19 @@ size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int
}
#if 0
else if (polys[k].polygon->intersects(polys[j].polygon) || polys[k].polygon->touches(polys[j].polygon))
- {
+ {
// FIXME: This code does not work as intended
// It should be setting the polys[k].ring in order to update this object
// but the value of polys[k].polygon calculated is normally NULL
// Add polygon this polygon (j) to k since they intersect
- // Mark ourselfs to be dropped (2), delete the original k
+ // Mark ourselfs to be dropped (2), delete the original k
Geometry* polyunion = polys[k].polygon->Union(polys[j].polygon);
delete(polys[k].polygon);
- polys[k].polygon = dynamic_cast<Polygon*>(polyunion);
- polys[j].iscontained = 2; // Drop
+ polys[k].polygon = dynamic_cast<Polygon*>(polyunion);
+ polys[j].iscontained = 2; // Drop
istoplevelafterall = 2;
break;
-
}
#endif
}
@@ -507,10 +463,10 @@ size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int
}
}
#ifdef HAS_PREPARED_GEOMETRIES
- pgf.destroy(preparedtoplevelpolygon);
+ pgf.destroy(preparedtoplevelpolygon);
#endif
}
- // polys now is a list of ploygons tagged with which ones are inside each other
+ // polys now is a list of polygons tagged with which ones are inside each other
// List of polygons for multipolygon
std::auto_ptr<std::vector<Geometry*> > polygons(new std::vector<Geometry*>);
@@ -529,7 +485,7 @@ size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int
interior->push_back(polys[j].ring);
}
}
-
+
Polygon* poly(gf.createPolygon(polys[i].ring, interior.release()));
poly->normalize();
polygons->push_back(poly);
@@ -546,15 +502,16 @@ size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int
if ((excludepoly == 0) || (multipoly->isValid()))
{
- std::string text = writer.write(multipoly.get());
- wkts.push_back(text);
- areas.push_back(multipoly->getArea());
- wkt_size++;
+ //copy of an empty one should be cheapest
+ wkts->push_back(geometry_builder::wkt_t());
+ //then we set on the one we already have
+ wkts->back().geom = writer.write(multipoly.get());
+ wkts->back().area = multipoly->getArea();
}
}
else
{
- for(unsigned i=0; i<toplevelpolygons; i++)
+ for(unsigned i=0; i<toplevelpolygons; i++)
{
Geometry* poly = dynamic_cast<Geometry*>(polygons->at(i));
if (!poly->isValid() && (excludepoly == 0)) {
@@ -563,10 +520,11 @@ size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int
}
if ((excludepoly == 0) || (poly->isValid()))
{
- std::string text = writer.write(poly);
- wkts.push_back(text);
- areas.push_back(poly->getArea());
- wkt_size++;
+ //copy of an empty one should be cheapest
+ wkts->push_back(geometry_builder::wkt_t());
+ //then we set on the one we already have
+ wkts->back().geom = writer.write(poly);
+ wkts->back().area = poly->getArea();
}
delete(poly);
}
@@ -578,37 +536,81 @@ size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int
delete(polys[i].polygon);
}
delete[](polys);
- }
+ }//TODO: don't show in message id when osm_id == -1
catch (std::exception& e)
- {
- std::cerr << std::endl << "Standard exception processing way_id "<< osm_id << ": " << e.what() << std::endl;
- wkt_size = 0;
- }
+ {
+ std::cerr << std::endl << "Standard exception processing way_id="<< osm_id << ": " << e.what() << std::endl;
+ }
catch (...)
- {
+ {
std::cerr << std::endl << "Exception caught processing way id=" << osm_id << std::endl;
- wkt_size = 0;
- }
+ }
- return wkt_size;
+ return wkts;
}
-void exclude_broken_polygon ()
+geometry_builder::maybe_wkt_t geometry_builder::build_multilines(const osmNode * const * xnodes, const int *xcount, osmid_t osm_id) const
{
- excludepoly = 1;
+ std::auto_ptr<std::vector<Geometry*> > lines(new std::vector<Geometry*>);
+ GeometryFactory gf;
+ geom_ptr geom;
+
+ maybe_wkt_t wkt(new geometry_builder::wkt_t());
+
+ try
+ {
+ for (int c=0; xnodes[c]; c++) {
+ std::auto_ptr<CoordinateSequence> coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
+ for (int i = 0; i < xcount[c]; i++) {
+ const osmNode *nodes = xnodes[c];
+ Coordinate c;
+ c.x = nodes[i].lon;
+ c.y = nodes[i].lat;
+ coords->add(c, 0);
+ }
+ if (coords->getSize() > 1) {
+ geom = geom_ptr(gf.createLineString(coords.release()));
+ lines->push_back(geom.release());
+ }
+ }
+
+ //geom_ptr segment(0);
+ geom_ptr mline (gf.createMultiLineString(lines.release()));
+ //geom_ptr noded (segment->Union(mline.get()));
+
+ WKTWriter writer;
+ wkt->geom = writer.write(mline.get());
+ wkt->area = 0;
+ }//TODO: don't show in message id when osm_id == -1
+ catch (std::exception& e)
+ {
+ std::cerr << std::endl << "Standard exception processing way_id="<< osm_id << ": " << e.what() << std::endl;
+ }
+ catch (...)
+ {
+ std::cerr << std::endl << "Exception caught processing way id=" << osm_id << std::endl;
+ }
+ return wkt;
}
-char *get_multiline_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount) {
+geometry_builder::maybe_wkts_t geometry_builder::build_both(const osmNode * const * xnodes, const int *xcount, int make_polygon,
+ int enable_multi, double split_at, osmid_t osm_id) const
+{
std::auto_ptr<std::vector<Geometry*> > lines(new std::vector<Geometry*>);
GeometryFactory gf;
geom_ptr geom;
+#ifdef HAS_PREPARED_GEOMETRIES
+ geos::geom::prep::PreparedGeometryFactory pgf;
+#endif
+ maybe_wkts_t wkts(new std::vector<geometry_builder::wkt_t>);
+
try
{
for (int c=0; xnodes[c]; c++) {
std::auto_ptr<CoordinateSequence> coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
for (int i = 0; i < xcount[c]; i++) {
- struct osmNode *nodes = xnodes[c];
+ const osmNode *nodes = xnodes[c];
Coordinate c;
c.x = nodes[i].lon;
c.y = nodes[i].lat;
@@ -620,21 +622,222 @@ char *get_multiline_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcoun
}
}
+ //geom_ptr segment(0);
geom_ptr mline (gf.createMultiLineString(lines.release()));
+ //geom_ptr noded (segment->Union(mline.get()));
+ LineMerger merger;
+ //merger.add(noded.get());
+ merger.add(mline.get());
+ std::auto_ptr<std::vector<LineString *> > merged(merger.getMergedLineStrings());
WKTWriter writer;
- std::string wkt = writer.write(mline.get());
- return strdup(wkt.c_str());
- }
+ // Procces ways into lines or simple polygon list
+ polygondata* polys = new polygondata[merged->size()];
+
+ unsigned totalpolys = 0;
+ for (unsigned i=0 ;i < merged->size(); ++i)
+ {
+ std::auto_ptr<LineString> pline ((*merged ) [i]);
+ if (make_polygon && pline->getNumPoints() > 3 && pline->isClosed())
+ {
+ polys[totalpolys].polygon = gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0);
+ polys[totalpolys].ring = gf.createLinearRing(pline->getCoordinates());
+ polys[totalpolys].area = polys[totalpolys].polygon->getArea();
+ polys[totalpolys].iscontained = 0;
+ polys[totalpolys].containedbyid = 0;
+ if (polys[totalpolys].area > 0.0)
+ totalpolys++;
+ else {
+ delete(polys[totalpolys].polygon);
+ delete(polys[totalpolys].ring);
+ }
+ }
+ else
+ {
+ //std::cerr << "polygon(" << osm_id << ") is no good: points(" << pline->getNumPoints() << "), closed(" << pline->isClosed() << "). " << writer.write(pline.get()) << std::endl;
+ double distance = 0;
+ std::auto_ptr<CoordinateSequence> segment;
+ segment = std::auto_ptr<CoordinateSequence>(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
+ segment->add(pline->getCoordinateN(0));
+ for(unsigned i=1; i<pline->getNumPoints(); i++) {
+ segment->add(pline->getCoordinateN(i));
+ distance += pline->getCoordinateN(i).distance(pline->getCoordinateN(i-1));
+ if ((distance >= split_at) || (i == pline->getNumPoints()-1)) {
+ geom_ptr geom = geom_ptr(gf.createLineString(segment.release()));
+
+ //copy of an empty one should be cheapest
+ wkts->push_back(geometry_builder::wkt_t());
+ //then we set on the one we already have
+ wkts->back().geom = writer.write(geom.get());
+ wkts->back().area = 0;
+
+ segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2));
+ distance=0;
+ segment->add(pline->getCoordinateN(i));
+ }
+ }
+ //std::string text = writer.write(pline.get());
+ //wkts.push_back(text);
+ //areas.push_back(0.0);
+ //wkt_size++;
+ }
+ }
+
+ if (totalpolys)
+ {
+ qsort(polys, totalpolys, sizeof(polygondata), polygondata_comparearea);
+
+ unsigned toplevelpolygons = 0;
+ int istoplevelafterall;
+
+ for (unsigned i=0 ;i < totalpolys; ++i)
+ {
+ if (polys[i].iscontained != 0) continue;
+ toplevelpolygons++;
+#ifdef HAS_PREPARED_GEOMETRIES
+ const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon);
+#endif
+
+ for (unsigned j=i+1; j < totalpolys; ++j)
+ {
+#ifdef HAS_PREPARED_GEOMETRIES
+ // Does preparedtoplevelpolygon contain the smaller polygon[j]?
+ if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon))
+#else
+ // Does polygon[i] contain the smaller polygon[j]?
+ if (polys[j].containedbyid == 0 && polys[i].polygon->contains(polys[j].polygon))
+#endif
+ {
+ // are we in a [i] contains [k] contains [j] situation
+ // which would actually make j top level
+ istoplevelafterall = 0;
+ for (unsigned k=i+1; k < j; ++k)
+ {
+ if (polys[k].iscontained && polys[k].containedbyid == i && polys[k].polygon->contains(polys[j].polygon))
+ {
+ istoplevelafterall = 1;
+ break;
+ }
+#if 0
+ else if (polys[k].polygon->intersects(polys[j].polygon) || polys[k].polygon->touches(polys[j].polygon))
+ {
+ // FIXME: This code does not work as intended
+ // It should be setting the polys[k].ring in order to update this object
+ // but the value of polys[k].polygon calculated is normally NULL
+
+ // Add polygon this polygon (j) to k since they intersect
+ // Mark ourselfs to be dropped (2), delete the original k
+ Geometry* polyunion = polys[k].polygon->Union(polys[j].polygon);
+ delete(polys[k].polygon);
+ polys[k].polygon = dynamic_cast<Polygon*>(polyunion);
+ polys[j].iscontained = 2; // Drop
+ istoplevelafterall = 2;
+ break;
+ }
+#endif
+ }
+ if (istoplevelafterall == 0)
+ {
+ polys[j].iscontained = 1;
+ polys[j].containedbyid = i;
+ }
+ }
+ }
+#ifdef HAS_PREPARED_GEOMETRIES
+ pgf.destroy(preparedtoplevelpolygon);
+#endif
+ }
+ // polys now is a list of polygons tagged with which ones are inside each other
+
+ // List of polygons for multipolygon
+ std::auto_ptr<std::vector<Geometry*> > polygons(new std::vector<Geometry*>);
+
+ // For each top level polygon create a new polygon including any holes
+ for (unsigned i=0 ;i < totalpolys; ++i)
+ {
+ if (polys[i].iscontained != 0) continue;
+
+ // List of holes for this top level polygon
+ std::auto_ptr<std::vector<Geometry*> > interior(new std::vector<Geometry*>);
+ for (unsigned j=i+1; j < totalpolys; ++j)
+ {
+ if (polys[j].iscontained == 1 && polys[j].containedbyid == i)
+ {
+ interior->push_back(polys[j].ring);
+ }
+ }
+
+ Polygon* poly(gf.createPolygon(polys[i].ring, interior.release()));
+ poly->normalize();
+ polygons->push_back(poly);
+ }
+
+ // Make a multipolygon if required
+ if ((toplevelpolygons > 1) && enable_multi)
+ {
+ geom_ptr multipoly(gf.createMultiPolygon(polygons.release()));
+ if (!multipoly->isValid() && (excludepoly == 0)) {
+ multipoly = geom_ptr(multipoly->buffer(0));
+ }
+ multipoly->normalize();
+
+ if ((excludepoly == 0) || (multipoly->isValid()))
+ {
+ //copy of an empty one should be cheapest
+ wkts->push_back(geometry_builder::wkt_t());
+ //then we set on the one we already have
+ wkts->back().geom = writer.write(multipoly.get());
+ wkts->back().area = multipoly->getArea();
+ }
+ }
+ else
+ {
+ for(unsigned i=0; i<toplevelpolygons; i++)
+ {
+ Geometry* poly = dynamic_cast<Geometry*>(polygons->at(i));
+ if (!poly->isValid() && (excludepoly == 0)) {
+ poly = dynamic_cast<Geometry*>(poly->buffer(0));
+ poly->normalize();
+ }
+ if ((excludepoly == 0) || (poly->isValid()))
+ {
+ //copy of an empty one should be cheapest
+ wkts->push_back(geometry_builder::wkt_t());
+ //then we set on the one we already have
+ wkts->back().geom = writer.write(poly);
+ wkts->back().area = poly->getArea();
+ }
+ delete(poly);
+ }
+ }
+ }
+
+ for (unsigned i=0; i < totalpolys; ++i)
+ {
+ delete(polys[i].polygon);
+ }
+ delete[](polys);
+ }//TODO: don't show in message id when osm_id == -1
catch (std::exception& e)
- {
- std::cerr << std::endl << "Standard exception processing relation id "<< osm_id << ": " << e.what() << std::endl;
- }
+ {
+ std::cerr << std::endl << "Standard exception processing relation id="<< osm_id << ": " << e.what() << std::endl;
+ }
catch (...)
- {
- std::cerr << std::endl << "Exception caught processing relation id " << osm_id << std::endl;
- }
+ {
+ std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl;
+ }
- return 0;
+ return wkts;
+}
+
+void geometry_builder::set_exclude_broken_polygon(int exclude)
+{
+ excludepoly = exclude;
}
+geometry_builder::geometry_builder()
+ : excludepoly(0) {
+}
+
+geometry_builder::~geometry_builder() {
+}
diff --git a/geometry-builder.hpp b/geometry-builder.hpp
new file mode 100644
index 0000000..e206497
--- /dev/null
+++ b/geometry-builder.hpp
@@ -0,0 +1,67 @@
+/*
+#-----------------------------------------------------------------------------
+# Part of osm2pgsql utility
+#-----------------------------------------------------------------------------
+# By Artem Pavlenko, Copyright 2007
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#-----------------------------------------------------------------------------
+*/
+
+#ifndef GEOMETRY_BUILDER_H
+#define GEOMETRY_BUILDER_H
+
+#include "osmtypes.hpp"
+
+#include <vector>
+#include <string>
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+
+struct geometry_builder : public boost::noncopyable
+{
+ struct wkt_t
+ {
+ wkt_t(const std::string& geom, const double& area):geom(geom),area(area){}
+ wkt_t():geom(""),area(0){}
+ std::string geom;
+ double area;
+ };
+
+ // type to represent an optional return of WKT-encoded geometry
+ typedef boost::shared_ptr<geometry_builder::wkt_t> maybe_wkt_t;
+ typedef boost::shared_ptr<std::vector<geometry_builder::wkt_t> > maybe_wkts_t;
+ typedef std::vector<geometry_builder::wkt_t>::const_iterator wkt_itr;
+
+ geometry_builder();
+ ~geometry_builder();
+
+ static int parse_wkt(const char * wkt, struct osmNode *** xnodes, int ** xcount, int * polygon);
+ maybe_wkt_t get_wkt_simple(const osmNode *, int count, int polygon) const;
+ maybe_wkts_t get_wkt_split(const osmNode *, int count, int polygon, double split_at) const;
+ maybe_wkts_t build_both(const osmNode * const * xnodes, const int *xcount, int make_polygon, int enable_multi,
+ double split_at, osmid_t osm_id = -1) const;
+ maybe_wkts_t build_lines(const osmNode * const * xnodes, const int *xcount, double split_at, osmid_t osm_id = -1) const;
+ maybe_wkts_t build_polygons(const osmNode * const * xnodes, const int *xcount, bool enable_multi, osmid_t osm_id = -1) const;
+ // Used by gazetteer. Outputting a multiline, it only ever returns one WKT
+ maybe_wkt_t build_multilines(const osmNode * const * xnodes, const int *xcount, osmid_t osm_id) const;
+
+ void set_exclude_broken_polygon(int exclude);
+
+private:
+ int excludepoly;
+};
+
+#endif
diff --git a/geometry-processor.cpp b/geometry-processor.cpp
new file mode 100644
index 0000000..71ebee7
--- /dev/null
+++ b/geometry-processor.cpp
@@ -0,0 +1,156 @@
+#include "geometry-processor.hpp"
+#include "processor-line.hpp"
+#include "processor-point.hpp"
+#include "processor-polygon.hpp"
+
+
+#include <boost/make_shared.hpp>
+#include <boost/format.hpp>
+#include <stdexcept>
+
+boost::shared_ptr<geometry_processor> geometry_processor::create(const std::string &type,
+ const options_t *options) {
+ boost::shared_ptr<geometry_processor> ptr;
+ int srid = options->projection->project_getprojinfo()->srs;
+ double scale = options->scale;
+
+ if (type == "point") {
+ ptr = boost::make_shared<processor_point>(srid, scale);
+ }
+ else if (type == "line") {
+ ptr = boost::make_shared<processor_line>(srid);
+ }
+ else if (type == "polygon") {
+ ptr = boost::make_shared<processor_polygon>(srid, options->enable_multi);
+ }
+ else {
+ throw std::runtime_error((boost::format("Unable to construct geometry processor "
+ "because type `%1%' is not known.")
+ % type).str());
+ }
+
+ return ptr;
+}
+
+geometry_processor::geometry_processor(int srid, const std::string &type, unsigned int interests)
+ : m_srid(srid), m_type(type), m_interests(interests) {
+}
+
+geometry_processor::~geometry_processor() {
+}
+
+int geometry_processor::srid() const {
+ return m_srid;
+}
+
+const std::string &geometry_processor::column_type() const {
+ return m_type;
+}
+
+unsigned int geometry_processor::interests() const {
+ return m_interests;
+}
+
+bool geometry_processor::interests(unsigned int interested) const {
+ return (interested & m_interests) == interested;
+}
+
+geometry_builder::maybe_wkt_t geometry_processor::process_node(double lat, double lon) {
+ return geometry_builder::maybe_wkt_t();
+}
+
+geometry_builder::maybe_wkt_t geometry_processor::process_way(const osmNode *nodes, const size_t node_count) {
+ return geometry_builder::maybe_wkt_t();
+}
+
+geometry_builder::maybe_wkts_t geometry_processor::process_relation(const osmNode * const * nodes, const int* node_counts) {
+ return geometry_builder::maybe_wkts_t();
+}
+
+way_helper::way_helper()
+{
+}
+way_helper::~way_helper()
+{
+}
+size_t way_helper::set(const osmid_t *node_ids, size_t node_count, const middle_query_t *mid)
+{
+ //if we don't have enough space already get more
+ if(node_cache.size() < node_count)
+ node_cache.resize(node_count);
+ //get the node data
+ mid->nodes_get_list(&node_cache.front(), node_ids, node_count);
+ return node_cache.size();
+}
+
+relation_helper::relation_helper():members(NULL), member_count(0), way_count(0)
+{
+}
+relation_helper::~relation_helper()
+{
+ //clean up
+ for(size_t i = 0; i < way_count; ++i)
+ {
+ keyval::resetList(&(tags[i]));
+ free(nodes[i]);
+ }
+}
+
+size_t& relation_helper::set(const member* member_list, const int member_list_length, const middle_t* mid)
+{
+ //clean up
+ for(size_t i = 0; i < way_count; ++i)
+ {
+ keyval::resetList(&(tags[i]));
+ free(nodes[i]);
+ }
+
+ //keep a few things
+ members = member_list;
+ member_count = member_list_length;
+
+ //grab the way members' ids
+ input_way_ids.resize(member_count);
+ size_t used = 0;
+ for(size_t i = 0; i < member_count; ++i)
+ if(members[i].type == OSMTYPE_WAY)
+ input_way_ids[used++] = members[i].id;
+
+ //if we didnt end up using any well bail
+ if(used == 0)
+ {
+ way_count = 0;
+ return way_count;
+ }
+
+ //get the nodes of the ways
+ tags.resize(used + 1);
+ node_counts.resize(used + 1);
+ nodes.resize(used + 1);
+ ways.resize(used + 1);
+ //this is mildly abusive treating vectors like arrays but the memory is contiguous so...
+ way_count = mid->ways_get_list(&input_way_ids.front(), used, &ways.front(), &tags.front(), &nodes.front(), &node_counts.front());
+
+ //grab the roles of each way
+ roles.resize(way_count + 1);
+ roles[way_count] = NULL;
+ for (size_t i = 0; i < way_count; ++i)
+ {
+ size_t j = i;
+ for (; j < member_count; ++j)
+ {
+ if (members[j].id == ways[i])
+ {
+ break;
+ }
+ }
+ roles[i] = members[j].role;
+ }
+
+ //mark the ends of each so whoever uses them will know where they end..
+ nodes[way_count] = NULL;
+ node_counts[way_count] = 0;
+ ways[way_count] = 0;
+ superseeded.resize(way_count);
+ return way_count;
+}
diff --git a/geometry-processor.hpp b/geometry-processor.hpp
new file mode 100644
index 0000000..6b57740
--- /dev/null
+++ b/geometry-processor.hpp
@@ -0,0 +1,98 @@
+#ifndef GEOMETRY_PROCESSOR_HPP
+#define GEOMETRY_PROCESSOR_HPP
+
+#include "middle.hpp"
+#include "geometry-builder.hpp"
+#include <string>
+#include <boost/optional.hpp>
+
+struct geometry_processor {
+ // factory method for creating various types of geometry processors either by name or by geometry column type
+ static boost::shared_ptr<geometry_processor> create(const std::string &type,
+ const options_t *options);
+
+ virtual ~geometry_processor();
+
+ enum interest {
+ interest_NONE = 0,
+ interest_node = 1,
+ interest_way = 2,
+ interest_relation = 4,
+ interest_ALL = 7
+ };
+
+ // return bit-mask of the type of elements this processor is
+ // interested in.
+ unsigned int interests() const;
+
+ // return true if provided intrest is an interest of this processor
+ bool interests(unsigned int interested) const;
+
+ // the postgis column type for the kind of geometry (i.e: POINT,
+ // LINESTRING, etc...) that this processor outputs
+ const std::string &column_type() const;
+
+ // process a node, optionally returning a WKT string describing
+ // geometry to be inserted into the table.
+ virtual geometry_builder::maybe_wkt_t process_node(double lat, double lon);
+
+ // process a way
+ // position data and optionally returning WKT-encoded geometry
+ // for insertion into the table.
+ virtual geometry_builder::maybe_wkt_t process_way(const osmNode *nodes, const size_t node_count);
+
+ // process a way, taking a middle query object to get way and
+ // node position data. optionally returns a WKT-encoded geometry
+ // for insertion into the table.
+ virtual geometry_builder::maybe_wkts_t process_relation(const osmNode * const * nodes, const int* node_counts);
+
+ // returns the SRID of the output geometry.
+ int srid() const;
+
+protected:
+ // SRID of the geometry output
+ const int m_srid;
+
+ // WKT type of the geometry output
+ const std::string m_type;
+
+ // mask of elements that this processor is interested in
+ const unsigned int m_interests;
+
+ // constructor for use by implementing classes only
+ geometry_processor(int srid, const std::string &type, unsigned int interests);
+};
+
+
+//various bits for continuous processing of ways
+struct way_helper
+{
+ way_helper();
+ ~way_helper();
+ size_t set(const osmid_t *node_ids, size_t node_count, const middle_query_t *mid);
+
+ std::vector<osmNode> node_cache;
+};
+
+//various bits for continuous processing of members of relations
+struct relation_helper
+{
+ relation_helper();
+ ~relation_helper();
+ size_t& set(const member* member_list, const int member_list_length, const middle_t* mid);
+
+ const member* members;
+ size_t member_count;
+ std::vector<keyval> tags;
+ std::vector<int> node_counts;
+ std::vector<osmNode*> nodes;
+ std::vector<osmid_t> ways;
+ size_t way_count;
+ std::vector<const char*> roles;
+ std::vector<int> superseeded;
+
+private:
+ std::vector<osmid_t> input_way_ids;
+};
+
+#endif /* GEOMETRY_PROCESSOR_HPP */
diff --git a/id-tracker.cpp b/id-tracker.cpp
new file mode 100644
index 0000000..748ae7e
--- /dev/null
+++ b/id-tracker.cpp
@@ -0,0 +1,176 @@
+#include "id-tracker.hpp"
+
+#include <map>
+#include <vector>
+#include <limits>
+#include <algorithm>
+
+#include <boost/optional.hpp>
+
+#define BLOCK_BITS (16)
+#define BLOCK_SIZE (1 << BLOCK_BITS)
+#define BLOCK_MASK (BLOCK_SIZE - 1)
+
+namespace {
+/* block used to be just a std::vector<bool> of fixed size. however,
+ * it seems there's significant overhead in exposing std::vector<bool>::iterator
+ * and so this is now a minimal re-implementation.
+ *
+ * each block is BLOCK_SIZE bits, stored as a vector of uint32_t elements.
+ */
+struct block {
+ block() : bits(BLOCK_SIZE >> 5, 0) {}
+ inline bool operator[](size_t i) const { return (bits[i >> 5] & (1 << (i & 0x1f))) > 0; }
+ //returns true if the value actually caused a bit to flip
+ inline bool set(size_t i, bool value) {
+ uint32_t &bit = bits[i >> 5];
+ uint32_t old = bit;
+ uint32_t mask = 1 << (i & 0x1f);
+ //allow the bit to become 1 if not already
+ if (value) {
+ bit |= mask;
+ }//force the bit to 0 if its not already
+ else {
+ bit &= ~mask;
+ }
+ //did it actually change the value
+ return old != bit;
+ }
+ // find the next bit which is set, starting from an initial offset
+ // of start. this offset is a bit like an iterator, but not fully
+ // supporting iterator movement forwards and backwards.
+ //
+ // returns BLOCK_SIZE if a set bit isn't found
+ size_t next_set(size_t start) const {
+ uint32_t bit_i = start >> 5;
+
+ while ((bit_i < (BLOCK_SIZE >> 5)) && (bits[bit_i] == 0)) {
+ ++bit_i;
+ }
+
+ if (bit_i >= (BLOCK_SIZE >> 5)) { return BLOCK_SIZE; }
+ uint32_t bit = bits[bit_i];
+ size_t idx = bit_i << 5;
+ while ((bit & 1) == 0) { ++idx; bit >>= 1; }
+ return idx;
+ }
+private:
+ std::vector<uint32_t> bits;
+};
+} // anonymous namespace
+
+struct id_tracker::pimpl {
+ pimpl();
+ ~pimpl();
+
+ bool get(osmid_t id) const;
+ bool set(osmid_t id, bool value);
+ osmid_t pop_min();
+
+ typedef std::map<osmid_t, block> map_t;
+ map_t pending;
+ osmid_t old_id;
+ size_t count;
+ // a cache of the next starting point to search for in the block.
+ // this significantly speeds up pop_min() because it doesn't need
+ // to repeatedly search the beginning of the block each time.
+ boost::optional<size_t> next_start;
+};
+
+bool id_tracker::pimpl::get(osmid_t id) const {
+ const osmid_t block = id >> BLOCK_BITS, offset = id & BLOCK_MASK;
+ map_t::const_iterator itr = pending.find(block);
+ bool result = false;
+
+ if (itr != pending.end()) {
+ result = itr->second[offset];
+ }
+
+ return result;
+}
+
+bool id_tracker::pimpl::set(osmid_t id, bool value) {
+ const osmid_t block = id >> BLOCK_BITS, offset = id & BLOCK_MASK;
+ bool flipped = pending[block].set(offset, value);
+ // a set may potentially invalidate a next_start, as the bit
+ // set might be before the position of next_start.
+ if (next_start) { next_start = boost::none; }
+ return flipped;
+}
+
+// find the first element in a block set to true
+osmid_t id_tracker::pimpl::pop_min() {
+ osmid_t id = max();
+
+ while (next_start || !pending.empty()) {
+ map_t::iterator itr = pending.begin();
+ block &b = itr->second;
+ size_t start = next_start.get_value_or(0);
+
+ size_t b_itr = b.next_set(start);
+ if (b_itr != BLOCK_SIZE) {
+ b.set(b_itr, false);
+ id = (itr->first << BLOCK_BITS) | b_itr;
+ next_start = b_itr;
+ break;
+
+ } else {
+ // no elements in this block - might as well delete
+ // the whole thing.
+ pending.erase(itr);
+ // since next_start is relative to the current
+ // block, which is ceasing to exist, then we need to
+ // reset it.
+ next_start = boost::none;
+ }
+ }
+
+ return id;
+}
+
+id_tracker::pimpl::pimpl()
+ : pending(), old_id(min()), count(0), next_start(boost::none) {
+}
+
+id_tracker::pimpl::~pimpl() {
+}
+
+id_tracker::id_tracker(): impl() {
+ impl.reset(new pimpl());
+}
+
+id_tracker::~id_tracker() {
+}
+
+void id_tracker::mark(osmid_t id) {
+ //setting returns true if the id wasn't already marked
+ impl->count += size_t(impl->set(id, true));
+ //we've marked something so we need to be able to pop it
+ //the assert below will fail though if we've already popped
+ //some that were > id so we have to essentially reset to
+ //allow for more pops to take place
+ impl->old_id = min();
+}
+
+bool id_tracker::is_marked(osmid_t id) {
+ return impl->get(id);
+}
+
+osmid_t id_tracker::pop_mark() {
+ osmid_t id = impl->pop_min();
+
+ assert((id > impl->old_id) || !id_tracker::is_valid(id));
+ impl->old_id = id;
+
+ //we just go rid of one (if there were some to get rid of)
+ if(impl->count > 0)
+ impl->count--;
+
+ return id;
+}
+
+size_t id_tracker::size() { return impl->count; }
+
+bool id_tracker::is_valid(osmid_t id) { return id != max(); }
+osmid_t id_tracker::max() { return std::numeric_limits<osmid_t>::max(); }
+osmid_t id_tracker::min() { return std::numeric_limits<osmid_t>::min(); }
diff --git a/id-tracker.hpp b/id-tracker.hpp
new file mode 100644
index 0000000..db11053
--- /dev/null
+++ b/id-tracker.hpp
@@ -0,0 +1,27 @@
+#ifndef ID_TRACKER_HPP
+#define ID_TRACKER_HPP
+
+#include "osmtypes.hpp"
+#include <string>
+#include <boost/noncopyable.hpp>
+#include <boost/scoped_ptr.hpp>
+
+struct id_tracker : public boost::noncopyable {
+ id_tracker();
+ ~id_tracker();
+
+ void mark(osmid_t id);
+ bool is_marked(osmid_t id);
+ osmid_t pop_mark();
+ size_t size();
+
+ static bool is_valid(osmid_t);
+ static osmid_t max();
+ static osmid_t min();
+
+private:
+ struct pimpl;
+ boost::scoped_ptr<pimpl> impl;
+};
+
+#endif /* ID_TRACKER_HPP */
diff --git a/input.c b/input.cpp
similarity index 62%
rename from input.c
rename to input.cpp
index f14463e..87c27e5 100644
--- a/input.c
+++ b/input.cpp
@@ -1,31 +1,32 @@
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE
-#ifdef __MINGW_H
-# include <windows.h>
+#ifdef _WIN32
+#include <windows.h>
+#include <io.h>
+#define STDIN_FILENO 0
#else
#include <stdio.h>
#include <unistd.h>
-#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
-#include <fcntl.h>
-#include <zlib.h>
#endif
+#include <fcntl.h>
+#include <zlib.h>
#include <bzlib.h>
#include <string.h>
-#include "sanitizer.h"
-#include "input.h"
+#include "sanitizer.hpp"
+#include "input.hpp"
struct Input {
char *name;
enum { plainFile, gzipFile, bzip2File } type;
void *fileHandle;
- /* needed by bzip2 when decompressing from multiple streams. other
+ /* needed by bzip2 when decompressing from multiple streams. other
decompressors must ignore it. */
- FILE *systemHandle;
+ FILE *systemHandle;
int eof;
char buf[4096];
int buf_ptr, buf_fill;
@@ -43,12 +44,12 @@ int bzReOpen(struct Input *ctx, int *error) {
BZ2_bzReadGetUnused(error, (BZFILE *)(ctx->fileHandle), &unused_tmp_ptr, &nUnused);
if (*error != BZ_OK) return -1;
-
- /* when bzReadClose is called the unused buffer is deallocated,
+
+ /* when bzReadClose is called the unused buffer is deallocated,
so it needs to be copied somewhere safe first. */
for (i = 0; i < nUnused; ++i)
unused[i] = ((unsigned char *)unused_tmp_ptr)[i];
-
+
BZ2_bzReadClose(error, (BZFILE *)(ctx->fileHandle));
if (*error != BZ_OK) return -1;
@@ -59,40 +60,39 @@ int bzReOpen(struct Input *ctx, int *error) {
return 0;
}
-int readFile(void *context, char * buffer, int len)
+int readFile(struct Input *ctx, char * buffer, int len)
{
- struct Input *ctx = context;
void *f = ctx->fileHandle;
int l = 0, error = 0;
if (ctx->eof || (len == 0))
return 0;
-
+
switch(ctx->type) {
- case plainFile:
- l = read(*(int *)f, buffer, len);
- if (l <= 0) ctx->eof = 1;
- break;
- case gzipFile:
- l = gzread((gzFile)f, buffer, len);
- if (l <= 0) ctx->eof = 1;
- break;
- case bzip2File:
- l = BZ2_bzRead(&error, (BZFILE *)f, buffer, len);
-
- /* error codes BZ_OK and BZ_STREAM_END are both "OK", but the stream
- end means the reader needs to be reset from the original handle. */
- if (error != BZ_OK) {
- /* for stream errors, try re-opening the stream before admitting defeat. */
+ case Input::plainFile:
+ l = read(*(int *)f, buffer, len);
+ if (l <= 0) ctx->eof = 1;
+ break;
+ case Input::gzipFile:
+ l = gzread((gzFile)f, buffer, len);
+ if (l <= 0) ctx->eof = 1;
+ break;
+ case Input::bzip2File:
+ l = BZ2_bzRead(&error, (BZFILE *)f, buffer, len);
+
+ /* error codes BZ_OK and BZ_STREAM_END are both "OK", but the stream
+ end means the reader needs to be reset from the original handle. */
+ if (error != BZ_OK) {
+ /* for stream errors, try re-opening the stream before admitting defeat. */
if (error != BZ_STREAM_END || bzReOpen(ctx, &error) != 0) {
- l = 0;
- ctx->eof = 1;
+ l = 0;
+ ctx->eof = 1;
}
- }
- break;
- default:
- fprintf(stderr, "Bad file type\n");
- break;
+ }
+ break;
+ default:
+ fprintf(stderr, "Bad file type\n");
+ break;
}
if (l < 0) {
@@ -103,12 +103,10 @@ int readFile(void *context, char * buffer, int len)
return l;
}
-char inputGetChar(void *context)
+char inputGetChar(struct Input *ctx)
{
- struct Input *ctx = context;
-
if (ctx->buf_ptr == ctx->buf_fill) {
- ctx->buf_fill = readFile(context, &ctx->buf[0], sizeof(ctx->buf));
+ ctx->buf_fill = readFile(ctx, &ctx->buf[0], sizeof(ctx->buf));
ctx->buf_ptr = 0;
if (ctx->buf_fill == 0)
return 0;
@@ -120,28 +118,28 @@ char inputGetChar(void *context)
return ctx->buf[ctx->buf_ptr++];
}
-int inputEof(void *context)
+int inputEof(struct Input *ctx)
{
- return ((struct Input *)context)->eof;
+ return ctx->eof;
}
-void *inputOpen(const char *name)
+struct Input *inputOpen(const char *name)
{
const char *ext = strrchr(name, '.');
- struct Input *ctx = malloc (sizeof(*ctx));
+ struct Input *ctx = (struct Input *)malloc(sizeof(*ctx));
if (!ctx)
return NULL;
memset(ctx, 0, sizeof(*ctx));
- ctx->name = malloc(strlen(name) + 1);
+ ctx->name = (char *)malloc(strlen(name) + 1);
if (ctx->name) strcpy(ctx->name, name);
if (ext && !strcmp(ext, ".gz")) {
ctx->fileHandle = (void *)gzopen(name, "rb");
- ctx->type = gzipFile;
+ ctx->type = Input::gzipFile;
} else if (ext && !strcmp(ext, ".bz2")) {
int error = 0;
ctx->systemHandle = fopen(name, "rb");
@@ -151,10 +149,10 @@ void *inputOpen(const char *name)
}
ctx->fileHandle = (void *)BZ2_bzReadOpen(&error, ctx->systemHandle, 0, 0, NULL, 0);
- ctx->type = bzip2File;
-
+ ctx->type = Input::bzip2File;
+
} else {
- int *pfd = malloc(sizeof(int));
+ int *pfd = (int *)malloc(sizeof(int));
if (pfd) {
if (!strcmp(name, "-")) {
*pfd = STDIN_FILENO;
@@ -171,7 +169,7 @@ void *inputOpen(const char *name)
}
}
ctx->fileHandle = (void *)pfd;
- ctx->type = plainFile;
+ ctx->type = Input::plainFile;
}
if (!ctx->fileHandle) {
fprintf(stderr, "error while opening file %s\n", name);
@@ -179,28 +177,27 @@ void *inputOpen(const char *name)
}
ctx->buf_ptr = 0;
ctx->buf_fill = 0;
- return (void *)ctx;
+ return ctx;
}
-int inputClose(void *context)
+int inputClose(struct Input *ctx)
{
- struct Input *ctx = context;
void *f = ctx->fileHandle;
switch(ctx->type) {
- case plainFile:
- close(*(int *)f);
- free(f);
- break;
- case gzipFile:
- gzclose((gzFile)f);
- break;
- case bzip2File:
- BZ2_bzclose((BZFILE *)f);
- break;
- default:
- fprintf(stderr, "Bad file type\n");
- break;
+ case Input::plainFile:
+ close(*(int *)f);
+ free(f);
+ break;
+ case Input::gzipFile:
+ gzclose((gzFile)f);
+ break;
+ case Input::bzip2File:
+ BZ2_bzclose((BZFILE *)f);
+ break;
+ default:
+ fprintf(stderr, "Bad file type\n");
+ break;
}
free(ctx->name);
@@ -208,6 +205,17 @@ int inputClose(void *context)
return 0;
}
+// NOTE: need the two function variants below to fit into what
+// libXML expects the function signatures to be.
+
+static int readFileXML(void *context, char *buffer, int len) {
+ return readFile((struct Input *)context, buffer, len);
+}
+
+static int inputCloseXML(void *context) {
+ return inputClose((struct Input *)context);
+}
+
xmlTextReaderPtr inputUTF8(const char *name)
{
void *ctx = inputOpen(name);
@@ -217,5 +225,5 @@ xmlTextReaderPtr inputUTF8(const char *name)
return NULL;
}
- return xmlReaderForIO(readFile, inputClose, (void *)ctx, NULL, NULL, 0);
+ return xmlReaderForIO(readFileXML, inputCloseXML, (void *)ctx, NULL, NULL, 0);
}
diff --git a/input.h b/input.h
deleted file mode 100644
index de5f802..0000000
--- a/input.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#ifndef INPUT_H
-#define INPUT_H
-
-int readFile(void *context, char * buffer, int len);
-int inputClose(void *context);
-void *inputOpen(const char *name);
-char inputGetChar(void *context);
-int inputEof(void *context);
-xmlTextReaderPtr inputUTF8(const char *name);
-
-#endif
diff --git a/input.hpp b/input.hpp
new file mode 100644
index 0000000..6313181
--- /dev/null
+++ b/input.hpp
@@ -0,0 +1,13 @@
+#ifndef INPUT_H
+#define INPUT_H
+
+struct Input;
+
+int readFile(struct Input *context, char * buffer, int len);
+int inputClose(struct Input *context);
+struct Input *inputOpen(const char *name);
+char inputGetChar(struct Input *context);
+int inputEof(struct Input *context);
+xmlTextReaderPtr inputUTF8(const char *name);
+
+#endif
diff --git a/keyvals.c b/keyvals.c
deleted file mode 100644
index b14bc5f..0000000
--- a/keyvals.c
+++ /dev/null
@@ -1,346 +0,0 @@
-/* Common key-value list processing
- *
- * Used as a small general purpose store for
- * tags, segment lists etc
- *
- */
-#define USE_TREE
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <string.h>
-#include "keyvals.h"
-
-#ifdef USE_TREE
-#include "text-tree.h"
-#endif
-
-void initList(struct keyval *head)
-{
- assert(head);
-
- head->next = head;
- head->prev = head;
- head->key = NULL;
- head->value = NULL;
- head->has_column = 0;
-}
-
-void freeItem(struct keyval *p)
-{
- if (!p)
- return;
-
-#ifdef USE_TREE
- text_release(tree_ctx, p->key);
- text_release(tree_ctx, p->value);
-#else
- free(p->key);
- free(p->value);
-#endif
- free(p);
-}
-
-
-unsigned int countList(struct keyval *head)
-{
- struct keyval *p;
- unsigned int count = 0;
-
- if (!head)
- return 0;
-
- p = head->next;
- while(p != head) {
- count++;
- p = p->next;
- }
- return count;
-}
-
-int listHasData(struct keyval *head)
-{
- if (!head)
- return 0;
-
- return (head->next != head);
-}
-
-
-char *getItem(struct keyval *head, const char *name)
-{
- struct keyval *p;
-
- if (!head)
- return NULL;
-
- p = head->next;
- while(p != head) {
- if (!strcmp(p->key, name))
- return p->value;
- p = p->next;
- }
- return NULL;
-}
-
-/* unlike getItem this function gives a pointer to the whole
- list item which can be used to remove the tag from the linked list
- with the removeTag function
-*/
-struct keyval *getTag(struct keyval *head, const char *name)
-{
- struct keyval *p;
-
- if (!head)
- return NULL;
-
- p = head->next;
- while(p != head) {
- if (!strcmp(p->key, name))
- return p;
- p = p->next;
- }
- return NULL;
-}
-
-void removeTag(struct keyval *tag)
-{
- tag->prev->next=tag->next;
- tag->next->prev=tag->prev;
- freeItem(tag);
-}
-
-struct keyval *firstItem(struct keyval *head)
-{
- if (head == NULL || head == head->next)
- return NULL;
-
- return head->next;
-}
-
-struct keyval *nextItem(struct keyval *head, struct keyval *item)
-{
- if (item->next == head)
- return NULL;
-
- return item->next;
-}
-
-/* Pulls all items from list which match this prefix
- * note: they are removed from the original list an returned in a new one
- */
-struct keyval *getMatches(struct keyval *head, const char *name)
-{
- struct keyval *out = NULL;
- struct keyval *p;
-
- if (!head)
- return NULL;
-
- out = malloc(sizeof(struct keyval));
- if (!out)
- return NULL;
-
- initList(out);
- p = head->next;
- while(p != head) {
- struct keyval *next = p->next;
- if (!strncmp(p->key, name, strlen(name))) {
- p->next->prev = p->prev;
- p->prev->next = p->next;
- pushItem(out, p);
- }
- p = next;
- }
-
- if (listHasData(out))
- return out;
-
- free(out);
- return NULL;
-}
-
-void updateItem(struct keyval *head, const char *name, const char *value)
-{
- struct keyval *item;
-
- if (!head)
- return;
-
- item = head->next;
- while(item != head) {
- if (!strcmp(item->key, name)) {
-#ifdef USE_TREE
- text_release(tree_ctx, item->value);
- item->value = (char *)text_get(tree_ctx,value);
-#else
- free(item->value);
- item->value = strdup(value);
-#endif
- return;
- }
- item = item->next;
- }
- addItem(head, name, value, 0);
-}
-
-
-struct keyval *popItem(struct keyval *head)
-{
- struct keyval *p;
-
- if (!head)
- return NULL;
-
- p = head->next;
- if (p == head)
- return NULL;
-
- head->next = p->next;
- p->next->prev = head;
-
- p->next = NULL;
- p->prev = NULL;
-
- return p;
-}
-
-
-void pushItem(struct keyval *head, struct keyval *item)
-{
-
- assert(head);
- assert(item);
-
- item->next = head;
- item->prev = head->prev;
- head->prev->next = item;
- head->prev = item;
-}
-
-int addItem(struct keyval *head, const char *name, const char *value, int noDupe)
-{
- struct keyval *item;
-
- assert(head);
- assert(name);
- assert(value);
-
- if (noDupe) {
- item = head->next;
- while (item != head) {
- if (!strcmp(item->value, value) && !strcmp(item->key, name))
- return 1;
- item = item->next;
- }
- }
-
- item = malloc(sizeof(struct keyval));
-
- if (!item) {
- fprintf(stderr, "Error allocating keyval\n");
- return 2;
- }
-
-#ifdef USE_TREE
- item->key = (char *)text_get(tree_ctx,name);
- item->value = (char *)text_get(tree_ctx,value);
-#else
- item->key = strdup(name);
- item->value = strdup(value);
-#endif
- item->has_column=0;
-
-
-#if 1
- /* Add to head */
- item->next = head->next;
- item->prev = head;
- head->next->prev = item;
- head->next = item;
-#else
- /* Add to tail */
- item->prev = head->prev;
- item->next = head;
- head->prev->next = item;
- head->prev = item;
-#endif
- return 0;
-}
-
-void resetList(struct keyval *head)
-{
- struct keyval *item;
-
- while((item = popItem(head)))
- freeItem(item);
-}
-
-void cloneList( struct keyval *target, struct keyval *source )
-{
- struct keyval *ptr;
- for( ptr = source->next; ptr != source; ptr=ptr->next )
- addItem( target, ptr->key, ptr->value, 0 );
-}
-
-
-/* create an escaped version of the string for hstore table insert */
-/* make shure dst is 2*strlen(src) */
-static void escape4hstore(char *dst, char *src) {
- size_t i,j;
-
- j=0;
- for (i=0;i<strlen(src);i++) {
- switch(src[i]) {
- case '\\':
- dst[j]='\\';j++;dst[j]='\\';j++;dst[j]='\\';j++;dst[j]='\\'; break;
- case '"':
- dst[j]='\\';j++;dst[j]='\\';j++;dst[j]='"'; break;
- case '\t':
- dst[j]='\\';j++;dst[j]='\t'; break;
- case '\r':
- dst[j]='\\';j++;dst[j]='\r'; break;
- case '\n':
- dst[j]='\\';j++;dst[j]='\n'; break;
- default:
- dst[j]=src[i];
- }
- j++;
- }
- dst[j]='\0';
-}
-
-/* print struct keyval in syntax for pgsql hstore import
- \ and " need to be escaped
-*/
-void keyval2hstore(char *hstring, struct keyval *tags)
-{
- keyval2hstore_manual(hstring, tags->key, tags->value);
-}
-
-void keyval2hstore_manual(char *hstring, char *key, char *value)
-{
- static char* str=NULL;
- static size_t stlen=0;
- size_t len;
-
- len=strlen(value);
- if (len>stlen) {
- stlen=len;
- str=realloc(str,1+stlen*2);
- }
-
- len=strlen(key);
- if (len>stlen) {
- stlen=len;
- str=realloc(str,1+stlen*2);
- }
-
- escape4hstore(str,key);
- hstring+=sprintf(hstring,"\"%s\"=>",str);
- escape4hstore(str,value);
- sprintf(hstring,"\"%s\"",str);
-}
-
diff --git a/keyvals.cpp b/keyvals.cpp
new file mode 100644
index 0000000..92f98f3
--- /dev/null
+++ b/keyvals.cpp
@@ -0,0 +1,266 @@
+/* Common key-value list processing
+ *
+ * Used as a small general purpose store for
+ * tags, segment lists etc
+ *
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <string.h>
+#include "keyvals.hpp"
+
+#include <algorithm>
+
+keyval::keyval()
+{
+ tree_ctx.reset(new text_tree());
+ keyval::initList(this);
+}
+
+keyval::~keyval()
+{
+ //keyval::resetList(this);
+}
+
+void keyval::initList(struct keyval *head)
+{
+ assert(head);
+
+ head->next = head;
+ head->prev = head;
+ head->key = NULL;
+ head->value = NULL;
+ head->has_column = 0;
+}
+
+void keyval::freeItem(struct keyval *p)
+{
+ if (!p)
+ return;
+
+ p->tree_ctx->text_release(p->key);
+ p->tree_ctx->text_release(p->value);
+ delete p;
+}
+
+
+unsigned int keyval::countList(const struct keyval *head)
+{
+ struct keyval *p;
+ unsigned int count = 0;
+
+ if (!head)
+ return 0;
+
+ p = head->next;
+ while(p != head) {
+ count++;
+ p = p->next;
+ }
+ return count;
+}
+
+int keyval::listHasData(struct keyval *head)
+{
+ if (!head)
+ return 0;
+
+ return (head->next != head);
+}
+
+
+char *keyval::getItem(const struct keyval *head, const char *name)
+{
+ struct keyval *p;
+
+ if (!head)
+ return NULL;
+
+ p = head->next;
+ while(p != head) {
+ if (!strcmp(p->key, name))
+ return p->value;
+ p = p->next;
+ }
+ return NULL;
+}
+
+/* unlike getItem this function gives a pointer to the whole
+ list item which can be used to remove the tag from the linked list
+ with the removeTag function
+*/
+struct keyval *keyval::getTag(struct keyval *head, const char *name)
+{
+ struct keyval *p;
+
+ if (!head)
+ return NULL;
+
+ p = head->next;
+ while(p != head) {
+ if (!strcmp(p->key, name))
+ return p;
+ p = p->next;
+ }
+ return NULL;
+}
+
+void keyval::removeTag(struct keyval *tag)
+{
+ tag->prev->next=tag->next;
+ tag->next->prev=tag->prev;
+ freeItem(tag);
+}
+
+struct keyval *keyval::firstItem(struct keyval *head)
+{
+ if (head == NULL || head == head->next)
+ return NULL;
+
+ return head->next;
+}
+
+struct keyval *keyval::nextItem(struct keyval *head, struct keyval *item)
+{
+ if (item->next == head)
+ return NULL;
+
+ return item->next;
+}
+
+/* Pulls all items from list which match this prefix
+ * note: they are removed from the original list an returned in a new one
+ */
+struct keyval *keyval::getMatches(struct keyval *head, const char *name)
+{
+ struct keyval *out = NULL;
+ struct keyval *p;
+
+ if (!head)
+ return NULL;
+
+ //TODO: properly copy the tree_ctx from the keyval passed in
+ out = new keyval();
+ if (!out)
+ return NULL;
+
+ p = head->next;
+ while(p != head) {
+ struct keyval *next = p->next;
+ if (!strncmp(p->key, name, strlen(name))) {
+ p->next->prev = p->prev;
+ p->prev->next = p->next;
+ pushItem(out, p);
+ }
+ p = next;
+ }
+
+ if (listHasData(out))
+ return out;
+
+ delete out;
+ return NULL;
+}
+
+void keyval::updateItem(struct keyval *head, const char *name, const char *value)
+{
+ struct keyval *item;
+
+ if (!head)
+ return;
+
+ item = head->next;
+ while(item != head) {
+ if (!strcmp(item->key, name)) {
+ head->tree_ctx->text_release(item->value);
+ item->value = (char *)head->tree_ctx->text_get(value);
+ return;
+ }
+ item = item->next;
+ }
+ addItem(head, name, value, 0);
+}
+
+
+struct keyval *keyval::popItem(struct keyval *head)
+{
+ struct keyval *p;
+
+ if (!head)
+ return NULL;
+
+ p = head->next;
+ if (p == head)
+ return NULL;
+
+ head->next = p->next;
+ p->next->prev = head;
+
+ p->next = NULL;
+ p->prev = NULL;
+
+ return p;
+}
+
+
+void keyval::pushItem(struct keyval *head, struct keyval *item)
+{
+
+ assert(head);
+ assert(item);
+
+ item->next = head;
+ item->prev = head->prev;
+ head->prev->next = item;
+ head->prev = item;
+}
+
+int keyval::addItem(struct keyval *head, const char *name, const char *value, int noDupe)
+{
+ struct keyval *item;
+
+ assert(head);
+ assert(name);
+ assert(value);
+
+ if (noDupe) {
+ item = head->next;
+ while (item != head) {
+ if (!strcmp(item->value, value) && !strcmp(item->key, name))
+ return 1;
+ item = item->next;
+ }
+ }
+
+ //TODO: properly implement a copy constructor and do the
+ //shallow copy there instead of relying on implicit one?
+ item = new keyval(*head);
+
+ item->key = (char *)head->tree_ctx->text_get(name);
+ item->value = (char *)head->tree_ctx->text_get(value);
+ item->has_column=0;
+
+ /* Add to head */
+ item->next = head->next;
+ item->prev = head;
+ head->next->prev = item;
+ head->next = item;
+ return 0;
+}
+
+void keyval::resetList(struct keyval *head)
+{
+ struct keyval *item;
+
+ while((item = popItem(head)))
+ freeItem(item);
+}
+
+void keyval::cloneList( struct keyval *target, struct keyval *source )
+{
+ struct keyval *ptr;
+ for( ptr = source->next; ptr != source; ptr=ptr->next )
+ addItem( target, ptr->key, ptr->value, 0 );
+}
diff --git a/keyvals.h b/keyvals.h
deleted file mode 100644
index 71f7710..0000000
--- a/keyvals.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/* Common key-value list processing
- *
- * Used as a small general purpose store for
- * tags, segment lists etc
- *
- */
-
-#ifndef KEYVAL_H
-#define KEYVAL_H
-
-struct keyval {
- char *key;
- char *value;
- /* if a hstore column is requested we need a flag to store if a key
- has its own column because it should not be added to the hstore
- in this case
- */
- int has_column;
- struct keyval *next;
- struct keyval *prev;
-};
-
-void initList(struct keyval *head);
-void freeItem(struct keyval *p);
-unsigned int countList(struct keyval *head);
-int listHasData(struct keyval *head);
-char *getItem(struct keyval *head, const char *name);
-struct keyval *getTag(struct keyval *head, const char *name);
-void removeTag(struct keyval *tag);
-struct keyval *firstItem(struct keyval *head);
-struct keyval *nextItem(struct keyval *head, struct keyval *item);
-struct keyval *popItem(struct keyval *head);
-void pushItem(struct keyval *head, struct keyval *item);
-int addItem(struct keyval *head, const char *name, const char *value, int noDupe);
-void resetList(struct keyval *head);
-struct keyval *getMatches(struct keyval *head, const char *name);
-void updateItem(struct keyval *head, const char *name, const char *value);
-void cloneList( struct keyval *target, struct keyval *source );
-void keyval2hstore(char *hstring, struct keyval *tags);
-void keyval2hstore_manual(char *hstring, char *key, char *value);
-#endif
diff --git a/keyvals.hpp b/keyvals.hpp
new file mode 100644
index 0000000..12db037
--- /dev/null
+++ b/keyvals.hpp
@@ -0,0 +1,50 @@
+/* Common key-value list processing
+ *
+ * Used as a small general purpose store for
+ * tags, segment lists etc
+ *
+ */
+
+#ifndef KEYVAL_H
+#define KEYVAL_H
+
+#include "text-tree.hpp"
+
+#include <boost/shared_ptr.hpp>
+
+struct keyval {
+ char *key;
+ char *value;
+ /* if a hstore column is requested we need a flag to store if a key
+ has its own column because it should not be added to the hstore
+ in this case
+ */
+ int has_column;
+ struct keyval *next;
+ struct keyval *prev;
+ boost::shared_ptr<text_tree> tree_ctx;
+
+ keyval();
+ ~keyval();
+
+ static void initList(struct keyval *head);
+ static void freeItem(struct keyval *p);
+ static unsigned int countList(const struct keyval *head);
+ static int listHasData(struct keyval *head);
+ static char *getItem(const struct keyval *head, const char *name);
+ static struct keyval *getTag(struct keyval *head, const char *name);
+ static const struct keyval *getTag(const struct keyval *head, const char *name);
+ static void removeTag(struct keyval *tag);
+ static struct keyval *firstItem(struct keyval *head);
+ static struct keyval *nextItem(struct keyval *head, struct keyval *item);
+ static struct keyval *popItem(struct keyval *head);
+ static void pushItem(struct keyval *head, struct keyval *item);
+ static int addItem(struct keyval *head, const char *name, const char *value, int noDupe);
+ static void resetList(struct keyval *head);
+ static struct keyval *getMatches(struct keyval *head, const char *name);
+ static void updateItem(struct keyval *head, const char *name, const char *value);
+ static void cloneList( struct keyval *target, struct keyval *source );
+};
+
+
+#endif
diff --git a/m4/ax_boost_base.m4 b/m4/ax_boost_base.m4
new file mode 100644
index 0000000..8e6ee9a
--- /dev/null
+++ b/m4/ax_boost_base.m4
@@ -0,0 +1,272 @@
+# ===========================================================================
+# http://www.gnu.org/software/autoconf-archive/ax_boost_base.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_BOOST_BASE([MINIMUM-VERSION], [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
+#
+# DESCRIPTION
+#
+# Test for the Boost C++ libraries of a particular version (or newer)
+#
+# If no path to the installed boost library is given the macro searchs
+# under /usr, /usr/local, /opt and /opt/local and evaluates the
+# $BOOST_ROOT environment variable. Further documentation is available at
+# <http://randspringer.de/boost/index.html>.
+#
+# This macro calls:
+#
+# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS)
+#
+# And sets:
+#
+# HAVE_BOOST
+#
+# LICENSE
+#
+# Copyright (c) 2008 Thomas Porschberg <thomas at randspringer.de>
+# Copyright (c) 2009 Peter Adolphs
+#
+# Copying and distribution of this file, with or without modification, are
+# permitted in any medium without royalty provided the copyright notice
+# and this notice are preserved. This file is offered as-is, without any
+# warranty.
+
+#serial 23
+
+AC_DEFUN([AX_BOOST_BASE],
+[
+AC_ARG_WITH([boost],
+ [AS_HELP_STRING([--with-boost@<:@=ARG@:>@],
+ [use Boost library from a standard location (ARG=yes),
+ from the specified location (ARG=<path>),
+ or disable it (ARG=no)
+ @<:@ARG=yes@:>@ ])],
+ [
+ if test "$withval" = "no"; then
+ want_boost="no"
+ elif test "$withval" = "yes"; then
+ want_boost="yes"
+ ac_boost_path=""
+ else
+ want_boost="yes"
+ ac_boost_path="$withval"
+ fi
+ ],
+ [want_boost="yes"])
+
+
+AC_ARG_WITH([boost-libdir],
+ AS_HELP_STRING([--with-boost-libdir=LIB_DIR],
+ [Force given directory for boost libraries. Note that this will override library path detection, so use this parameter only if default library detection fails and you know exactly where your boost libraries are located.]),
+ [
+ if test -d "$withval"
+ then
+ ac_boost_lib_path="$withval"
+ else
+ AC_MSG_ERROR(--with-boost-libdir expected directory name)
+ fi
+ ],
+ [ac_boost_lib_path=""]
+)
+
+if test "x$want_boost" = "xyes"; then
+ boost_lib_version_req=ifelse([$1], ,1.20.0,$1)
+ boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([[0-9]]*\.[[0-9]]*\)'`
+ boost_lib_version_req_major=`expr $boost_lib_version_req : '\([[0-9]]*\)'`
+ boost_lib_version_req_minor=`expr $boost_lib_version_req : '[[0-9]]*\.\([[0-9]]*\)'`
+ boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'`
+ if test "x$boost_lib_version_req_sub_minor" = "x" ; then
+ boost_lib_version_req_sub_minor="0"
+ fi
+ WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor`
+ AC_MSG_CHECKING(for boostlib >= $boost_lib_version_req)
+ succeeded=no
+
+ dnl On 64-bit systems check for system libraries in both lib64 and lib.
+ dnl The former is specified by FHS, but e.g. Debian does not adhere to
+ dnl this (as it rises problems for generic multi-arch support).
+ dnl The last entry in the list is chosen by default when no libraries
+ dnl are found, e.g. when only header-only libraries are installed!
+ libsubdirs="lib"
+ ax_arch=`uname -m`
+ case $ax_arch in
+ x86_64|ppc64|s390x|sparc64|aarch64)
+ libsubdirs="lib64 lib lib64"
+ ;;
+ esac
+
+ dnl allow for real multi-arch paths e.g. /usr/lib/x86_64-linux-gnu. Give
+ dnl them priority over the other paths since, if libs are found there, they
+ dnl are almost assuredly the ones desired.
+ AC_REQUIRE([AC_CANONICAL_HOST])
+ libsubdirs="lib/${host_cpu}-${host_os} $libsubdirs"
+
+ case ${host_cpu} in
+ i?86)
+ libsubdirs="lib/i386-${host_os} $libsubdirs"
+ ;;
+ esac
+
+ dnl first we check the system location for boost libraries
+ dnl this location ist chosen if boost libraries are installed with the --layout=system option
+ dnl or if you install boost with RPM
+ if test "$ac_boost_path" != ""; then
+ BOOST_CPPFLAGS="-I$ac_boost_path/include"
+ for ac_boost_path_tmp in $libsubdirs; do
+ if test -d "$ac_boost_path"/"$ac_boost_path_tmp" ; then
+ BOOST_LDFLAGS="-L$ac_boost_path/$ac_boost_path_tmp"
+ break
+ fi
+ done
+ elif test "$cross_compiling" != yes; then
+ for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do
+ if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then
+ for libsubdir in $libsubdirs ; do
+ if ls "$ac_boost_path_tmp/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi
+ done
+ BOOST_LDFLAGS="-L$ac_boost_path_tmp/$libsubdir"
+ BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include"
+ break;
+ fi
+ done
+ fi
+
+ dnl overwrite ld flags if we have required special directory with
+ dnl --with-boost-libdir parameter
+ if test "$ac_boost_lib_path" != ""; then
+ BOOST_LDFLAGS="-L$ac_boost_lib_path"
+ fi
+
+ CPPFLAGS_SAVED="$CPPFLAGS"
+ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+ export CPPFLAGS
+
+ LDFLAGS_SAVED="$LDFLAGS"
+ LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+ export LDFLAGS
+
+ AC_REQUIRE([AC_PROG_CXX])
+ AC_LANG_PUSH(C++)
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+ @%:@include <boost/version.hpp>
+ ]], [[
+ #if BOOST_VERSION >= $WANT_BOOST_VERSION
+ // Everything is okay
+ #else
+ # error Boost version is too old
+ #endif
+ ]])],[
+ AC_MSG_RESULT(yes)
+ succeeded=yes
+ found_system=yes
+ ],[
+ ])
+ AC_LANG_POP([C++])
+
+
+
+ dnl if we found no boost with system layout we search for boost libraries
+ dnl built and installed without the --layout=system option or for a staged(not installed) version
+ if test "x$succeeded" != "xyes"; then
+ _version=0
+ if test "$ac_boost_path" != ""; then
+ if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then
+ for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do
+ _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'`
+ V_CHECK=`expr $_version_tmp \> $_version`
+ if test "$V_CHECK" = "1" ; then
+ _version=$_version_tmp
+ fi
+ VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'`
+ BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE"
+ done
+ fi
+ else
+ if test "$cross_compiling" != yes; then
+ for ac_boost_path in /usr /usr/local /opt /opt/local ; do
+ if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then
+ for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do
+ _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'`
+ V_CHECK=`expr $_version_tmp \> $_version`
+ if test "$V_CHECK" = "1" ; then
+ _version=$_version_tmp
+ best_path=$ac_boost_path
+ fi
+ done
+ fi
+ done
+
+ VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'`
+ BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE"
+ if test "$ac_boost_lib_path" = ""; then
+ for libsubdir in $libsubdirs ; do
+ if ls "$best_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi
+ done
+ BOOST_LDFLAGS="-L$best_path/$libsubdir"
+ fi
+ fi
+
+ if test "x$BOOST_ROOT" != "x"; then
+ for libsubdir in $libsubdirs ; do
+ if ls "$BOOST_ROOT/stage/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi
+ done
+ if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/$libsubdir" && test -r "$BOOST_ROOT/stage/$libsubdir"; then
+ version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'`
+ stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'`
+ stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'`
+ V_CHECK=`expr $stage_version_shorten \>\= $_version`
+ if test "$V_CHECK" = "1" -a "$ac_boost_lib_path" = "" ; then
+ AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT)
+ BOOST_CPPFLAGS="-I$BOOST_ROOT"
+ BOOST_LDFLAGS="-L$BOOST_ROOT/stage/$libsubdir"
+ fi
+ fi
+ fi
+ fi
+
+ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+ export CPPFLAGS
+ LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+ export LDFLAGS
+
+ AC_LANG_PUSH(C++)
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
+ @%:@include <boost/version.hpp>
+ ]], [[
+ #if BOOST_VERSION >= $WANT_BOOST_VERSION
+ // Everything is okay
+ #else
+ # error Boost version is too old
+ #endif
+ ]])],[
+ AC_MSG_RESULT(yes)
+ succeeded=yes
+ found_system=yes
+ ],[
+ ])
+ AC_LANG_POP([C++])
+ fi
+
+ if test "$succeeded" != "yes" ; then
+ if test "$_version" = "0" ; then
+ AC_MSG_NOTICE([[We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in <boost/version.hpp>. See http://randspringer.de/boost for more documentation.]])
+ else
+ AC_MSG_NOTICE([Your boost libraries seems to old (version $_version).])
+ fi
+ # execute ACTION-IF-NOT-FOUND (if present):
+ ifelse([$3], , :, [$3])
+ else
+ AC_SUBST(BOOST_CPPFLAGS)
+ AC_SUBST(BOOST_LDFLAGS)
+ AC_DEFINE(HAVE_BOOST,,[define if the Boost library is available])
+ # execute ACTION-IF-FOUND (if present):
+ ifelse([$2], , :, [$2])
+ fi
+
+ CPPFLAGS="$CPPFLAGS_SAVED"
+ LDFLAGS="$LDFLAGS_SAVED"
+fi
+
+])
diff --git a/m4/ax_boost_filesystem.m4 b/m4/ax_boost_filesystem.m4
new file mode 100644
index 0000000..f162163
--- /dev/null
+++ b/m4/ax_boost_filesystem.m4
@@ -0,0 +1,118 @@
+# ===========================================================================
+# http://www.gnu.org/software/autoconf-archive/ax_boost_filesystem.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_BOOST_FILESYSTEM
+#
+# DESCRIPTION
+#
+# Test for Filesystem library from the Boost C++ libraries. The macro
+# requires a preceding call to AX_BOOST_BASE. Further documentation is
+# available at <http://randspringer.de/boost/index.html>.
+#
+# This macro calls:
+#
+# AC_SUBST(BOOST_FILESYSTEM_LIB)
+#
+# And sets:
+#
+# HAVE_BOOST_FILESYSTEM
+#
+# LICENSE
+#
+# Copyright (c) 2009 Thomas Porschberg <thomas at randspringer.de>
+# Copyright (c) 2009 Michael Tindal
+# Copyright (c) 2009 Roman Rybalko <libtorrent at romanr.info>
+#
+# Copying and distribution of this file, with or without modification, are
+# permitted in any medium without royalty provided the copyright notice
+# and this notice are preserved. This file is offered as-is, without any
+# warranty.
+
+#serial 26
+
+AC_DEFUN([AX_BOOST_FILESYSTEM],
+[
+ AC_ARG_WITH([boost-filesystem],
+ AS_HELP_STRING([--with-boost-filesystem@<:@=special-lib@:>@],
+ [use the Filesystem library from boost - it is possible to specify a certain library for the linker
+ e.g. --with-boost-filesystem=boost_filesystem-gcc-mt ]),
+ [
+ if test "$withval" = "no"; then
+ want_boost="no"
+ elif test "$withval" = "yes"; then
+ want_boost="yes"
+ ax_boost_user_filesystem_lib=""
+ else
+ want_boost="yes"
+ ax_boost_user_filesystem_lib="$withval"
+ fi
+ ],
+ [want_boost="yes"]
+ )
+
+ if test "x$want_boost" = "xyes"; then
+ AC_REQUIRE([AC_PROG_CC])
+ CPPFLAGS_SAVED="$CPPFLAGS"
+ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+ export CPPFLAGS
+
+ LDFLAGS_SAVED="$LDFLAGS"
+ LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+ export LDFLAGS
+
+ LIBS_SAVED=$LIBS
+ LIBS="$LIBS $BOOST_SYSTEM_LIB"
+ export LIBS
+
+ AC_CACHE_CHECK(whether the Boost::Filesystem library is available,
+ ax_cv_boost_filesystem,
+ [AC_LANG_PUSH([C++])
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/filesystem/path.hpp>]],
+ [[using namespace boost::filesystem;
+ path my_path( "foo/bar/data.txt" );
+ return 0;]])],
+ ax_cv_boost_filesystem=yes, ax_cv_boost_filesystem=no)
+ AC_LANG_POP([C++])
+ ])
+ if test "x$ax_cv_boost_filesystem" = "xyes"; then
+ AC_DEFINE(HAVE_BOOST_FILESYSTEM,,[define if the Boost::Filesystem library is available])
+ BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
+ if test "x$ax_boost_user_filesystem_lib" = "x"; then
+ for libextension in `ls -r $BOOSTLIBDIR/libboost_filesystem* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do
+ ax_lib=${libextension}
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break],
+ [link_filesystem="no"])
+ done
+ if test "x$link_filesystem" != "xyes"; then
+ for libextension in `ls -r $BOOSTLIBDIR/boost_filesystem* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do
+ ax_lib=${libextension}
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break],
+ [link_filesystem="no"])
+ done
+ fi
+ else
+ for ax_lib in $ax_boost_user_filesystem_lib boost_filesystem-$ax_boost_user_filesystem_lib; do
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break],
+ [link_filesystem="no"])
+ done
+
+ fi
+ if test "x$ax_lib" = "x"; then
+ AC_MSG_ERROR(Could not find a version of the library!)
+ fi
+ if test "x$link_filesystem" != "xyes"; then
+ AC_MSG_ERROR(Could not link against $ax_lib !)
+ fi
+ fi
+
+ CPPFLAGS="$CPPFLAGS_SAVED"
+ LDFLAGS="$LDFLAGS_SAVED"
+ LIBS="$LIBS_SAVED"
+ fi
+])
diff --git a/m4/ax_boost_system.m4 b/m4/ax_boost_system.m4
new file mode 100644
index 0000000..c4c4555
--- /dev/null
+++ b/m4/ax_boost_system.m4
@@ -0,0 +1,120 @@
+# ===========================================================================
+# http://www.gnu.org/software/autoconf-archive/ax_boost_system.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_BOOST_SYSTEM
+#
+# DESCRIPTION
+#
+# Test for System library from the Boost C++ libraries. The macro requires
+# a preceding call to AX_BOOST_BASE. Further documentation is available at
+# <http://randspringer.de/boost/index.html>.
+#
+# This macro calls:
+#
+# AC_SUBST(BOOST_SYSTEM_LIB)
+#
+# And sets:
+#
+# HAVE_BOOST_SYSTEM
+#
+# LICENSE
+#
+# Copyright (c) 2008 Thomas Porschberg <thomas at randspringer.de>
+# Copyright (c) 2008 Michael Tindal
+# Copyright (c) 2008 Daniel Casimiro <dan.casimiro at gmail.com>
+#
+# Copying and distribution of this file, with or without modification, are
+# permitted in any medium without royalty provided the copyright notice
+# and this notice are preserved. This file is offered as-is, without any
+# warranty.
+
+#serial 17
+
+AC_DEFUN([AX_BOOST_SYSTEM],
+[
+ AC_ARG_WITH([boost-system],
+ AS_HELP_STRING([--with-boost-system@<:@=special-lib@:>@],
+ [use the System library from boost - it is possible to specify a certain library for the linker
+ e.g. --with-boost-system=boost_system-gcc-mt ]),
+ [
+ if test "$withval" = "no"; then
+ want_boost="no"
+ elif test "$withval" = "yes"; then
+ want_boost="yes"
+ ax_boost_user_system_lib=""
+ else
+ want_boost="yes"
+ ax_boost_user_system_lib="$withval"
+ fi
+ ],
+ [want_boost="yes"]
+ )
+
+ if test "x$want_boost" = "xyes"; then
+ AC_REQUIRE([AC_PROG_CC])
+ AC_REQUIRE([AC_CANONICAL_BUILD])
+ CPPFLAGS_SAVED="$CPPFLAGS"
+ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+ export CPPFLAGS
+
+ LDFLAGS_SAVED="$LDFLAGS"
+ LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+ export LDFLAGS
+
+ AC_CACHE_CHECK(whether the Boost::System library is available,
+ ax_cv_boost_system,
+ [AC_LANG_PUSH([C++])
+ CXXFLAGS_SAVE=$CXXFLAGS
+
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/system/error_code.hpp>]],
+ [[boost::system::system_category]])],
+ ax_cv_boost_system=yes, ax_cv_boost_system=no)
+ CXXFLAGS=$CXXFLAGS_SAVE
+ AC_LANG_POP([C++])
+ ])
+ if test "x$ax_cv_boost_system" = "xyes"; then
+ AC_SUBST(BOOST_CPPFLAGS)
+
+ AC_DEFINE(HAVE_BOOST_SYSTEM,,[define if the Boost::System library is available])
+ BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
+
+ LDFLAGS_SAVE=$LDFLAGS
+ if test "x$ax_boost_user_system_lib" = "x"; then
+ for libextension in `ls -r $BOOSTLIBDIR/libboost_system* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do
+ ax_lib=${libextension}
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break],
+ [link_system="no"])
+ done
+ if test "x$link_system" != "xyes"; then
+ for libextension in `ls -r $BOOSTLIBDIR/boost_system* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do
+ ax_lib=${libextension}
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break],
+ [link_system="no"])
+ done
+ fi
+
+ else
+ for ax_lib in $ax_boost_user_system_lib boost_system-$ax_boost_user_system_lib; do
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break],
+ [link_system="no"])
+ done
+
+ fi
+ if test "x$ax_lib" = "x"; then
+ AC_MSG_ERROR(Could not find a version of the library!)
+ fi
+ if test "x$link_system" = "xno"; then
+ AC_MSG_ERROR(Could not link against $ax_lib !)
+ fi
+ fi
+
+ CPPFLAGS="$CPPFLAGS_SAVED"
+ LDFLAGS="$LDFLAGS_SAVED"
+ fi
+])
diff --git a/m4/ax_boost_thread.m4 b/m4/ax_boost_thread.m4
new file mode 100644
index 0000000..79e12cd
--- /dev/null
+++ b/m4/ax_boost_thread.m4
@@ -0,0 +1,149 @@
+# ===========================================================================
+# http://www.gnu.org/software/autoconf-archive/ax_boost_thread.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_BOOST_THREAD
+#
+# DESCRIPTION
+#
+# Test for Thread library from the Boost C++ libraries. The macro requires
+# a preceding call to AX_BOOST_BASE. Further documentation is available at
+# <http://randspringer.de/boost/index.html>.
+#
+# This macro calls:
+#
+# AC_SUBST(BOOST_THREAD_LIB)
+#
+# And sets:
+#
+# HAVE_BOOST_THREAD
+#
+# LICENSE
+#
+# Copyright (c) 2009 Thomas Porschberg <thomas at randspringer.de>
+# Copyright (c) 2009 Michael Tindal
+#
+# Copying and distribution of this file, with or without modification, are
+# permitted in any medium without royalty provided the copyright notice
+# and this notice are preserved. This file is offered as-is, without any
+# warranty.
+
+#serial 27
+
+AC_DEFUN([AX_BOOST_THREAD],
+[
+ AC_ARG_WITH([boost-thread],
+ AS_HELP_STRING([--with-boost-thread@<:@=special-lib@:>@],
+ [use the Thread library from boost - it is possible to specify a certain library for the linker
+ e.g. --with-boost-thread=boost_thread-gcc-mt ]),
+ [
+ if test "$withval" = "no"; then
+ want_boost="no"
+ elif test "$withval" = "yes"; then
+ want_boost="yes"
+ ax_boost_user_thread_lib=""
+ else
+ want_boost="yes"
+ ax_boost_user_thread_lib="$withval"
+ fi
+ ],
+ [want_boost="yes"]
+ )
+
+ if test "x$want_boost" = "xyes"; then
+ AC_REQUIRE([AC_PROG_CC])
+ AC_REQUIRE([AC_CANONICAL_BUILD])
+ CPPFLAGS_SAVED="$CPPFLAGS"
+ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+ export CPPFLAGS
+
+ LDFLAGS_SAVED="$LDFLAGS"
+ LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+ export LDFLAGS
+
+ AC_CACHE_CHECK(whether the Boost::Thread library is available,
+ ax_cv_boost_thread,
+ [AC_LANG_PUSH([C++])
+ CXXFLAGS_SAVE=$CXXFLAGS
+
+ if test "x$host_os" = "xsolaris" ; then
+ CXXFLAGS="-pthreads $CXXFLAGS"
+ elif test "x$host_os" = "xmingw32" ; then
+ CXXFLAGS="-mthreads $CXXFLAGS"
+ else
+ CXXFLAGS="-pthread $CXXFLAGS"
+ fi
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/thread/thread.hpp>]],
+ [[boost::thread_group thrds;
+ return 0;]])],
+ ax_cv_boost_thread=yes, ax_cv_boost_thread=no)
+ CXXFLAGS=$CXXFLAGS_SAVE
+ AC_LANG_POP([C++])
+ ])
+ if test "x$ax_cv_boost_thread" = "xyes"; then
+ if test "x$host_os" = "xsolaris" ; then
+ BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS"
+ elif test "x$host_os" = "xmingw32" ; then
+ BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS"
+ else
+ BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS"
+ fi
+
+ AC_SUBST(BOOST_CPPFLAGS)
+
+ AC_DEFINE(HAVE_BOOST_THREAD,,[define if the Boost::Thread library is available])
+ BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
+
+ LDFLAGS_SAVE=$LDFLAGS
+ case "x$host_os" in
+ *bsd* )
+ LDFLAGS="-pthread $LDFLAGS"
+ break;
+ ;;
+ esac
+ if test "x$ax_boost_user_thread_lib" = "x"; then
+ for libextension in `ls -r $BOOSTLIBDIR/libboost_thread* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'`; do
+ ax_lib=${libextension}
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break],
+ [link_thread="no"])
+ done
+ if test "x$link_thread" != "xyes"; then
+ for libextension in `ls -r $BOOSTLIBDIR/boost_thread* 2>/dev/null | sed 's,.*/,,' | sed 's,\..*,,'`; do
+ ax_lib=${libextension}
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break],
+ [link_thread="no"])
+ done
+ fi
+
+ else
+ for ax_lib in $ax_boost_user_thread_lib boost_thread-$ax_boost_user_thread_lib; do
+ AC_CHECK_LIB($ax_lib, exit,
+ [BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break],
+ [link_thread="no"])
+ done
+
+ fi
+ if test "x$ax_lib" = "x"; then
+ AC_MSG_ERROR(Could not find a version of the library!)
+ fi
+ if test "x$link_thread" = "xno"; then
+ AC_MSG_ERROR(Could not link against $ax_lib !)
+ else
+ case "x$host_os" in
+ *bsd* )
+ BOOST_LDFLAGS="-pthread $BOOST_LDFLAGS"
+ break;
+ ;;
+ esac
+
+ fi
+ fi
+
+ CPPFLAGS="$CPPFLAGS_SAVED"
+ LDFLAGS="$LDFLAGS_SAVED"
+ fi
+])
diff --git a/mapnik-osm-updater.sh b/mapnik-osm-updater.sh
deleted file mode 100755
index e3b4ae8..0000000
--- a/mapnik-osm-updater.sh
+++ /dev/null
@@ -1,784 +0,0 @@
-#!/bin/bash
-
-export osm_username="osm"
-export database_name="gis"
-export planet_dir="/home/$osm_username/osm/planet"
-export planet_file="$planet_dir/planet.osm.bz2"
-export sql_dump="$planet_dir/planet.osm.sql.bz2"
-export log_dir=/var/log
-
-export geoinfodb_file="/usr/share/icons/map-icons/geoinfo.db"
-export osmdb_file="/usr/share/gpsdrive/osm.db"
-
-export osm2pgsql_cmd=`which osm2pgsql`
-test -x "$osm2pgsql_cmd" || echo "Missing osm2pgsql in PATH"
-test -x "$osm2pgsql_cmd" || osm2pgsql_cmd="$HOME/svn.openstreetmap.org/applications/utils/export/osm2pgsql/osm2pgsql"
-test -x "$osm2pgsql_cmd" || echo "Missing osm2pgsql"
-
-export cmd_osm2poidb=`which osm2poidb`
-test -x "$cmd_osm2poidb" || echo "Missing osm2poidb in PATH"
-test -x "$cmd_osm2poidb" || cmd_osm2poidb="`dirname $0`/../osm2poidb/build/osm2poidb"
-test -x "$cmd_osm2poidb" || cmd_osm2poidb="$HOME/svn.openstreetmap.org/applications/utils/export/osm2poidb/build/osm2poidb"
-test -x "$cmd_osm2poidb" || echo "Missing osm2poidb"
-
-osm_planet_mirror_cmd=`which osm-planet-mirror`
-test -x "$osm_planet_mirror_cmd" || echo "Missing planet-mirror.pl in PATH"
-test -x "$osm_planet_mirror_cmd" || osm_planet_mirror_cmd="`dirname $0`/../../planet-mirror/planet-mirror.pl"
-test -x "$osm_planet_mirror_cmd" || osm_planet_mirror_cmd="$HOME/svn.openstreetmap.org/applications/utils/planet-mirror/planet-mirror.pl"
-test -x "$osm_planet_mirror_cmd" || osm_planet_mirror_cmd="`dirname ../../planet-mirror/planet-mirror.pl`"
-test -x "$osm_planet_mirror_cmd" || echo "Missing planet-mirror.pl"
-
-test -n "$1" || help=1
-quiet=" -q "
-verbose=1
-
-for arg in "$@" ; do
- case $arg in
- --all-planet) # Do all the creation steps listed below from planet file
- create_osm_user=1
- mirror=1
- check_newer_planet=
- drop=1
- create_db=1
- db_table_create=1
- create_db=1
- create_db_user=1
- db_add_900913=1
- db_add_spatial_ref_sys=1
- grant_all_rights_to_user_osm=1
- planet_fill=1
- db_add_gpsdrive_poitypes=1
- create_db_users=${create_db_users:-*}
- grant_db_users=${grant_db_users:-*}
- ;;
-
- --all-planet-geofabrik=\?) # Use Planet Extract from Frederics GeoFabrik.de Page as planet File and import
- # Use ? for a list of possible files
- dir_country=${arg#*=}
- country=`basename $dir_country`
- planet_file="$planet_dir/${country}.osm.bz2"
- mirror_geofabrik=${dir_country}
- mirror=
- ;;
-
- --all-planet-geofabrik=*) # Use Planet Extract from Frederics GeoFabrik.de Page as planet File and import
- # Use ? for a list of possible files
- # Example: europe/germany/baden-wuerttemberg
- dir_country=${arg#*=}
- country=`basename $dir_country`
- planet_file="$planet_dir/${country}.osm.bz2"
- mirror_geofabrik=${dir_country}
- create_osm_user=1
- mirror=
- check_newer_planet=
- drop=1
- create_db=1
- db_table_create=1
- db_add_900913=1
- db_add_spatial_ref_sys=1
- create_db_user=1
- grant_all_rights_to_user_osm=1
- planet_fill=1
- db_add_gpsdrive_poitypes=1
- create_db_users=${create_db_users:-*}
- grant_db_users=${grant_db_users:-*}
- ;;
-
- --all-planet-update) # Do all the creation steps listed below from planet file with up to date checking
- create_osm_user=1
- mirror=1
- check_newer_planet=1
- drop=1
- create_db=1
- db_add_900913=1
- db_add_spatial_ref_sys=1
- create_db_user=1
- grant_all_rights_to_user_osm=1
- planet_fill=1
- db_add_gpsdrive_poitypes=1
- create_db_users=${create_db_users:-*}
- grant_db_users=${grant_db_users:-*}
- ;;
-
- --all-from-dump) # Do all the creation steps listed below
- # from planet-dump file
- # !!! all-from-dump is not completely tested yet
- create_osm_user=1
- mirror_dump=1
- drop=1
- create_db=1
- db_add_900913=1
- db_add_spatial_ref_sys=1
- create_db_user=1
- grant_all_rights_to_user_osm=1
- create_db_users=${create_db_users:-*}
- fill_from_dump="$sql_dump"
- grant_db_users=${grant_db_users:-*}
- db_add_gpsdrive_poitypes=1
- ;;
-
- --all-create) # Do all the creation steps listed below only no data
- # import and no planet mirroring
- create_osm_user=1
- drop=1
- create_db=1
- db_add_900913=1
- db_add_spatial_ref_sys=1
- create_db_user=1
- grant_all_rights_to_user_osm=1
- create_db_users=${create_db_users:-*}
- grant_db_users=${grant_db_users:-*}
- ;;
-
- --create-osm-user) # create the osm-user needed
- # This means creating a user 'osm' and his home directory
- # with useradd, mkdir, chmod and chown
- create_osm_user=1
- ;;
-
- --mirror) # mirror planet File (http://planet.openstreetmap.org/)
- mirror=1
- ;;
-
- --no-mirror) # do not mirror planet File
- mirror=
- ;;
-
- --check-newer-planet) # Check if Planet File is newer then stampfile.
- # If yes: Continue
- check_newer_planet=1
- ;;
-
- --drop) # drop the old Database (gis) and Database-user (osm)
- drop=1
- ;;
-
- --create-db) # create the database (gis)
- # with this command only the database is created,
- # but no tables inside it
- create_db=1
- ;;
-
- --create-db-user) # create the database-user (osm)
- create_db_user=1
- ;;
-
- --grant-all2osm-user) # grant all rights for the database to the DB-User osm
- grant_all_rights_to_user_osm=1
- ;;
-
- --create-db-users=*) #Create a Database user for all users specified.
- # To create a db-user for all available system-user
- # specify *. (Except root))
- create_db_users=${arg#*=}
- ;;
-
- --grant-db-users=*) # Grant database-users all rights (including write, ...)
- # to the gis Database !!! This has to be changed in the
- # future, normally only the osm user needs update rights
- grant_db_users=${arg#*=}
- ;;
-
- --add-gpsdrive-types) # add GpsDrive POI-Types to points table
- db_add_gpsdrive_poitypes=1
- ;;
-
- --planet-fill) # fill database from planet File
- planet_fill=1
- ;;
-
- --mirror-dump) # mirror the planet.sql dump File
- mirror_dump=1
- ;;
-
- --no-mirror-dump) # Do not mirror the planet.sql dump File
- mirror_dump=
- ;;
-
- --fill-from-dump=*) # fill database from Dump File
- fill_from_dump=${arg#*=}
- ;;
-
- --mapnik-dump=*) # Dump Content of Mapnik Database to a File (.sql|.sql.bz))
- postgis_mapnik_dump=${arg#*=}
- ;;
-
- --db-table-create) # Create tables in Database with osm2pgsql
- db_table_create=1
- ;;
-
- --db-add-srid-900913) # Add SRID 900913
- db_add_900913=1
- ;;
-
- --db-add-spatial_ref_sys) # Add SRIDs to spatial_ref_sys
- db_add_spatial_ref_sys=1
- ;;
-
- --count-db) # Count entries in Database. This is to check
- # if the database really contains entries
- # if you set an empty user with the option osm_username=''
- # the current user is used
- count_db=1
- ;;
-
- -h)
- help=1
- ;;
-
- --help)
- help=1
- ;;
-
- -help)
- help=1
- ;;
-
- --debug) # switch on debugging
- debug=1
- verbose=1
- quiet=""
- ;;
-
- -debug)
- debug=1
- verbose=1
- quiet=""
- ;;
-
-
- --nv) # be a little bit less verbose
- verbose=''
- ;;
-
- --planet-dir=*) # define Directory for Planet-File
- planet_dir=${arg#*=}
- planet_file="$planet_dir/planet.osm.bz2"
- ;;
-
- --planet-file=*) # define Planet-File including Directory
- planet_file=${arg#*=}
- ;;
-
- --poi-file=*) # define POI Database file including Directory
- osmdb_file=${arg#*=}
- ;;
-
- --geoinfo-file=*) # define geoinfo database file containing poi-types
- geoinfodb_file=${arg#*=}
- ;;
-
- --osm-username=*) # Define username to use for DB creation and planet
- # download
- # !! You shouldn't use your username or root as the
- # !! download and install user.
- # This username is the download and install user.
- # The osm-user normally only should have the planet files
- # in hishome directory and nothing else. By default
- # the osm-username is 'osm'
- osm_username=${arg#*=}
-
- if [ "$osm_username" = "$USER" ] ; then
- echo
- echo "!!!!!! ERROR: Don't use your own login account as the osm_username!!" 1>&2
- echo
- exit 1
- fi
-
- if [ "$osm_username" = "root" ] ; then
- echo
- echo "!!!!!! ERROR: Don't use the root account as the osm_username!!" 1>&2
- echo
- exit 1
- fi
-
- planet_dir="/home/$osm_username/osm/planet"
- planet_file="$planet_dir/planet.osm.bz2"
- ;;
-
- --osm2pgsql-cmd=*) # The path to the osm2pgsql command
- # It can be found at
- # svn.openstreetmap.org/applications/utils/export/osm2pgsql/
- # and has to be compiled. Alternatively you can install
- # the Debian Package openstreetmap-utils
- osm2pgsql_cmd=${arg#*=}
- if ! [ -x "$osm2pgsql_cmd" ]; then
- echo "!!!!!! ERROR: Cannot execute '$osm2pgsql_cmd'" 1>&2
- exit -1
- fi
- ;;
-
- --database-name=*) # use this name for the database default is 'gis'
- database_name=${arg#*=}
- ;;
-
- *)
- echo ""
- echo "!!!!!!!!! Unknown option $arg"
- echo ""
- help=1
- ;;
- esac
-done
-
-if [ -n "$help" ] ; then
- # extract options from case commands above
- options=`grep -E -e esac -e '\s*--.*\).*#' $0 | sed '/esac/,$d;s/.*--/ [--/; s/=\*)/=val]/; s/)[\s ]/]/; s/#.*\s*//; s/[\n/]//g;'`
- options=`for a in $options; do echo -n " $a" ; done`
- echo "$0 $options"
- echo "
-!!! Warning: This Script is for now a quick hack to make setting up
-!!! Warning: My databases easier. Please check if it really works for you!!
-!!! Warning: Especially when using different Database names or username, ...
-!!! Warning: not every combination of values except the default is tested.
-
- This script tries to install the mapnik database.
- For this it first creates a new user osm on the system
- and mirrors the current planet to his home directory.
- Then this planet is imported into the postgis Database from a
- newly created user named osm
-
- This script uses sudo. So you either have to have sudo right or you'll
- have to start the script as root. The users needed will be postgres and osm
- "
- # extract options + description from case commands above
- grep -E -e esac -e '--.*\).*#' -e '^[\t\s ]+#' $0 | \
- grep -v /bin/bash | sed '/esac/,$d;s/.*--/ --/;s/=\*)/=val/;s/)//;s/#//;s/\\//;'
- exit;
-fi
-
-
-if [ -n "$osm_username" ] ; then
- sudo_cmd="sudo -u $osm_username"
-else
- sudo_cmd=''
-fi
-
-export import_stamp_file=${log_dir}/osm2pgsql_postgis-$database_name.stamp
-export import_log=${log_dir}/osm2pgsql_postgis-$database_name.log
-
-
-if [ -n "$debug" ] ; then
- echo "Planet File: `ls -l $planet_file`"
- echo "Import Stamp : `ls -l $import_stamp_file`"
-fi
-
-
-############################################
-# Create a user on the system
-############################################
-if [ -n "$create_osm_user" ] ; then
- test -n "$verbose" && echo "----- Check if we already have an user '$osm_username'"
-
- if ! id "$osm_username" >/dev/null; then
- echo "create '$osm_username' User"
- useradd "$osm_username"
- fi
-
- mkdir -p "/home/$osm_username/osm/planet"
- # The user itself should be allowed to read/write all his own files
- # in the ~/osm/ Directory
- chown "$osm_username" "/home/$osm_username"
- chown -R "$osm_username" "/home/$osm_username/osm"
- chmod +rwX "/home/$osm_username"
- chmod -R +rwX "/home/$osm_username/osm"
-
- # Everyone on the system is allowed to read the planet.osm Files
- chmod -R a+rX "/home/$osm_username/osm"
-fi
-
-
-############################################
-# Mirror the planet-dump File for Europe
-############################################
-if [ -n "$mirror_geofabrik" ] ; then
- geofabrik_basedir="http://download.geofabrik.de/osm"
- if [ "$mirror_geofabrik" = "?" ]; then
-
- echo "Retreiving available planet extracts from GeoFabrik ..."
- # Find all Subdirs in the first 3 levels
- wget_out=`wget --no-convert-links -q --level=0 -O - "http://download.geofabrik.de/osm" | grep DIR | grep -v -i Parent `
- sub_dirs=`echo "$wget_out" | perl -ne 'm,href="(.*)/",;print "$1 "'`
-
- for level in 1 2 3; do
- for sub_dir in $sub_dirs ; do
- #echo "Get dirs in Subdir: $sub_dir"
- wget_out=`wget -q --level=0 -O - "$geofabrik_basedir/$sub_dir" | grep 'DIR' | grep -v Parent `
- new_dirs="$new_dirs `echo "$wget_out" | perl -ne 'm,href="(.*)/", && print "'$sub_dir'/$1 "'`"
- # echo "WGET: '$wget_out'"
- done
- sub_dirs="$sub_dirs $new_dirs"
- done
- sub_dirs=`for dir in $sub_dirs; do echo $dir; done | sort -u`
-
-
- # Printout content of all $sub_dirs
-
- echo "Possible Values are:"
- for sub_dir in "" $sub_dirs ; do
- wget -q --level=0 -O - "$geofabrik_basedir/$sub_dir" | grep 'OpenStreetMap data' | \
- perl -ne 'm/.*href="([^"]+)\.osm.bz2"/;print " '$sub_dir/'$1\n"'
- done
- exit 1
- fi
- planet_source_file="${geofabrik_basedir}/${mirror_geofabrik}.osm.bz2"
- if [ -n "$mirror" ] ; then
- test -n "$verbose" && echo "----- Mirroring planet File $planet_source_file"
- wget -v --mirror "$planet_source_file" \
- --no-directories --directory-prefix=$planet_dir/
- fi
-fi
-
-
-############################################
-# Mirror the newest planet File from planet.openstreetmap.org
-############################################
-if [ -n "$mirror" ] ; then
- test -n "$verbose" && echo "----- Mirroring planet File"
- if ! [ -x "$osm_planet_mirror_cmd" ]; then
- echo "!!!!!! ERROR: Cannot execute '$osm_planet_mirror_cmd'" 1>&2
- exit -1
- fi
- if ! $sudo_cmd $osm_planet_mirror_cmd -v -v --planet-dir=$planet_dir ; then
- echo "!!!!!! ERROR: Cannot Mirror Planet File" 1>&2
- exit 1
- fi
- if ! [ -s $planet_file ] ; then
- echo "!!!!!! ERROR: File $planet_file is missing"
- exit -1
- fi
-
-
-fi
-
-############################################
-# Check if Planet File is newer than import Stamp
-############################################
-if [ -n "$check_newer_planet" ] ; then
- if [ "$planet_file" -nt "$import_stamp_file" ] ; then
- if [ -n "$verbose" ] ; then
- echo "----- New File needs updating"
- echo "Planet File: `ls -l $planet_file`"
- echo "Import Stamp : `ls -l $import_stamp_file`"
- fi
- else
- echo "Postgis Database already Up To Date"
- echo "`ls -l $import_stamp_file`"
- exit 0
- fi
-fi
-
-############################################
-# Drop the old Database and Database-user
-############################################
-if [ -n "$drop" ] ; then
- test -n "$verbose" && echo "----- Drop complete Database '$database_name' and user '$osm_username'"
- echo "CHECKPOINT" | sudo -u postgres psql $quiet
- sudo -u postgres dropdb $quiet -Upostgres "$database_name"
- sudo -u postgres dropuser $quiet -Upostgres "$osm_username"
-fi
-
-############################################
-# Create db
-############################################
-if [ -n "$create_db" ] ; then
- test -n "$verbose" && echo
- test -n "$verbose" && echo "----- Create Database '$database_name'"
- if ! sudo -u postgres createdb -Upostgres $quiet -EUTF8 "$database_name"; then
- echo "!!!!!! ERROR: Creation of '$database_name' Failed"
- exit -1
- fi
- if ! sudo -u postgres createlang plpgsql "$database_name"; then
- echo "!!!!!! ERROR: Creation Failed"
- exit -1
- fi
-
- lwpostgis="/usr/share/postgresql-8.4-postgis/lwpostgis.sql"
- test -s $lwpostgis || lwpostgis="/usr/share/postgresql-8.3-postgis/lwpostgis.sql"
- test -s $lwpostgis || lwpostgis="/usr/share/postgresql-8.2-postgis/lwpostgis.sql"
- test -s $lwpostgis || lwpostgis="`ls /usr/share/postgresql-*-postgis/lwpostgis.sql| sort -n | head 1`"
- if [ ! -s $lwpostgis ] ; then
- echo "!!!!!! ERROR: Cannot find $lwpostgis"
- exit -1
- fi
- if sudo -u postgres psql $quiet -Upostgres "$database_name" <${lwpostgis} ; then
- echo "Enabling spacial Extentions done with '$lwpostgis'"
- else
- echo "!!!!!! ERROR: Creation with '$lwpostgis' Failed"
- exit -1
- fi
-fi
-
-############################################
-# Create db-user
-############################################
-if [ -n "$create_db_user" ] ; then
- test -n "$verbose" && echo "----- Create Database-user '$osm_username'"
- sudo -u postgres createuser -Upostgres $quiet -S -D -R "$osm_username" || exit -1
-fi
-
-if [ -n "$grant_all_rights_to_user_osm" ] ; then
- test -n "$verbose" && echo
- test -n "$verbose" && echo "----- Grant rights on Database '$database_name' for '$osm_username'"
- (
- echo "GRANT ALL ON SCHEMA PUBLIC TO \"$osm_username\";"
- echo "GRANT ALL on geometry_columns TO \"$osm_username\";"
- echo "GRANT ALL on spatial_ref_sys TO \"$osm_username\";"
- echo "GRANT ALL ON SCHEMA PUBLIC TO \"$osm_username\";"
- ) | sudo -u postgres psql $quiet -Upostgres "$database_name"
-fi
-
-############################################
-# Create a Database user for all users specified (*) or available on the system. Except root
-############################################
-if [ -n "$create_db_users" ] ; then
-
- if [ "$create_db_users" = "*" ] ; then
- echo "Create DB User for every USER"
- create_db_users=''
- # try to see if all users above uid=1000 are interesting
- all_users=`cat /etc/passwd | sed 's/:/ /g' | while read user pwd uid rest ; do test "$uid" -ge "1000" || continue; echo $user; done`
- echo "all_users: $all_users"
- for user in $all_users ; do
- echo $user | grep -q -e root && continue
- echo $user | grep -q -e "$osm_username" && continue
- echo $user | grep -q -e "nobody" && continue
- echo "$create_db_users" | grep -q " $user " && continue
- create_db_users=" $create_db_users $user "
- done
- fi
-
-# This is not good; this probably broke my postgres installation
-# dpkg --purge postgresql-8.2
-# Stopping PostgreSQL 8.2 database server: main* Error: The cluster is owned by user id 107 which does not exist any more
-# apt-get -f install postgresql-8.2
-# Starting PostgreSQL 8.2 database server: main* Error: The cluster is owned by user id 107 which does not exist any more
-#if false ; then
- for user in $create_db_users; do
- echo " Create DB User for $user"
- sudo -u postgres createuser $quiet -Upostgres --no-superuser --no-createdb --no-createrole "$user"
- done
-#fi
-
-fi
-
-############################################
-# Create Database tables with osm2pgsql
-############################################
-if [ -n "$db_table_create" ] ; then
- if ! [ -x "$osm2pgsql_cmd" ]; then
- echo "!!!!!! ERROR: Cannot execute '$osm2pgsql_cmd'" 1>&2
- exit -1
- fi
- echo ""
- echo "--------- Unpack and import $planet_file"
- cd /usr/share/openstreetmap/
- $sudo_cmd $osm2pgsql_cmd --create "$database_name"
-fi
-
-
-############################################
-# Add SRID spatial_ref_sys
-############################################
-if [ -n "$db_add_spatial_ref_sys" ] ; then
- test -s "$srid_spatial_ref_sys" || srid_spatial_ref_sys="/usr/share/postgresql-8.4-postgis/spatial_ref_sys.sql"
- test -s "$srid_spatial_ref_sys" || srid_spatial_ref_sys="/usr/share/postgresql-8.3-postgis/spatial_ref_sys.sql"
- test -s "$srid_spatial_ref_sys" || srid_spatial_ref_sys="/usr/share/postgresql-8.2-postgis/spatial_ref_sys.sql"
- test -s "$srid_spatial_ref_sys" || srid_spatial_ref_sys="/usr/share/postgresql-8.*-postgis/spatial_ref_sys.sql"
- test -s "$srid_spatial_ref_sys" || srid_spatial_ref_sys="/usr/share/postgresql-*-postgis/spatial_ref_sys.sql"
- if [ ! -s $srid_spatial_ref_sys ] ; then
- echo "!!!!!! ERROR: Cannot find $srid_spatial_ref_sys"
- exit -1
- fi
- if sudo -u postgres psql $quiet -Upostgres "$database_name" <${srid_spatial_ref_sys} ; then
- echo "Adding '$srid_spatial_ref_sys'"
- else
- echo "!!!!!! ERROR: Creation Failed"
- exit -1
- fi
-fi
-
-
-############################################
-# Add SRID 900913
-############################################
-if [ -n "$db_add_900913" ] ; then
-
- test -s "$srid_900913" || srid_900913="`dirname $0`/900913.sql"
- test -s "$srid_900913" || srid_900913="$HOME/svn.openstreetmap.org/applications/utils/export/osm2pgsql/900913.sql"
- test -s "$srid_900913" || srid_900913="/usr/share/mapnik/900913.sql"
- if [ ! -s $srid_900913 ] ; then
- echo "!!!!!! ERROR: Cannot find $srid_900913"
- exit -1
- fi
- if sudo -u postgres psql $quiet -Upostgres "$database_name" <${srid_900913} ; then
- echo "Adding '$srid_900913'"
- else
- echo "!!!!!! ERROR: Creation Failed"
- exit -1
- fi
-fi
-
-
-############################################
-# Grant all rights on the gis Database to all system users or selected users in the system
-############################################
-if [ -n "$grant_db_users" ] ; then
-
- if [ "$grant_db_users" = "*" ] ; then
- echo "-------- GRANT Rights to every USER"
- grant_db_users=''
- for user in `users` ; do
- echo "$user" | grep -q "root" && continue
- echo " $grant_db_users " | grep -q " $user " && continue
- grant_db_users="$grant_db_users $user"
- done
- fi
-
- test -n "$verbose" && echo "Granting rights to users: '$grant_db_users'"
-
- for user in $grant_db_users; do
- echo "Granting all rights to user '$user' for Database '$database_name'"
- (
- echo "GRANT ALL on geometry_columns TO \"$user\";"
- echo "GRANT ALL ON SCHEMA PUBLIC TO \"$user\";"
- echo "GRANT ALL on spatial_ref_sys TO \"$user\";"
- echo "GRANT ALL on TABLE planet_osm_line TO \"$user\";"
- echo "GRANT ALL on TABLE planet_osm_point TO \"$user\";"
- echo "GRANT ALL on TABLE planet_osm_roads TO \"$user\";"
- echo "GRANT ALL on TABLE planet_osm_polygon TO \"$user\";"
- )| sudo -u postgres psql $quiet -Upostgres "$database_name" || true
- done
-fi
-
-
-############################################
-# Fill Database from planet File
-############################################
-if [ -n "$planet_fill" ] ; then
- if ! [ -x "$osm2pgsql_cmd" ]; then
- echo "!!!!!! ERROR: Cannot execute '$osm2pgsql_cmd'" 1>&2
- exit -1
- fi
- echo ""
- echo "--------- Unpack and import $planet_file"
- echo "Import started: `date`" >>"$import_log"
- cd /usr/share/openstreetmap/
- $sudo_cmd $osm2pgsql_cmd --database "$database_name" $planet_file
- rc=$?
- if [ "$rc" -gt "0" ]; then
- echo "`date`: Import With Error $rc:" >> "$import_log"
- echo "`ls -l $planet_file` import --> rc($rc)" >> "$import_log"
- echo "!!!!!!!! ERROR while running '$sudo_cmd $osm2pgsql_cmd --database "$database_name" $planet_file'"
- echo "Creation with for Database "$database_name" from planet-file '$planet_file' with '$osm2pgsql_cmd' Failed"
- echo "see Logfile for more Information:"
- echo "less $import_log"
- exit -1
- fi
- echo "`date`: Import Done: `ls -l $planet_file` import --> $rc" >> "$import_log"
- echo "`date`: `ls -l $planet_file` import --> $rc" >>$import_stamp_file
- touch --reference=$planet_file $import_stamp_file
-fi
-
-
-############################################
-# Create GpsDrive POI-Database
-############################################
-if [ -n "$db_add_gpsdrive_poitypes" ] ; then
- if ! [ -x "$cmd_osm2poidb" ]; then
- echo "!!!!!! ERROR: Cannot execute gpsdrive_poitypes: '$cmd_osm2poidb'" 1>&2
- exit -1
- fi
- echo ""
- echo "--------- Create GpsDrive POI-Database $osmdb_file"
- bunzip2 -c $planet_file | sudo $cmd_osm2poidb -w -f $geoinfodb_file -o $osmdb_file STDIN
- rc=$?
- if [ "$rc" -ne "0" ]; then
- echo "!!!!!!! ERROR: cannot create POI Database"
- exit -1
- fi
-fi
-
-
-############################################
-# Dump the complete Database
-############################################
-if [ -n "$postgis_mapnik_dump" ] ; then
- # get Database Content with Dump
- postgis_mapnik_dump_dir=`dirname $postgis_mapnik_dump`
- mkdir -p "$postgis_mapnik_dump_dir"
- case "$postgis_mapnik_dump" in
- *.bz2)
- $sudo_cmd pg_dump --data-only -U "$osm_username" "$database_name" | bzip2 >"$postgis_mapnik_dump"
- ;;
- *.gz)
- $sudo_cmd pg_dump --data-only -U "$osm_username" "$database_name" | gzip >"$postgis_mapnik_dump"
- ;;
- *)
- $sudo_cmd pg_dump --data-only -U "$osm_username" "$database_name" >"$postgis_mapnik_dump"
- ;;
- esac
- if [ "$?" -gt "0" ]; then
- echo "Error While dumping Database"
- fi
-fi
-
-############################################
-# Mirror the planet-dump File from planet.openstreetmap.de
-############################################
-if [ -n "$mirror_dump" ] ; then
- test -n "$verbose" && echo "----- Mirroring planet-dump File"
- wget -v --mirror http://planet.openstreetmap.de/planet.osm.sql.bz2 \
- --no-directories --directory-prefix=$planet_dir/
-fi
-
-
-############################################
-# Fill Database from Dump File
-############################################
-if [ -n "$fill_from_dump" ] ; then
- echo ""
- echo "--------- Import from Dump '$fill_from_dump'"
- sudo -u postgres createdb -T template0 $database_name
- case "$fill_from_dump" in
- *.bz2)
- test -n "$verbose" && echo "Uncompress File ..."
- bzip2 -dc "$fill_from_dump" | $sudo_cmd psql $quiet "$database_name"
- ;;
- *.gz)
- test -n "$verbose" && echo "Uncompress File ..."
- gzip -dc "$fill_from_dump" | $sudo_cmd psql $quiet "$database_name"
- ;;
- *)
- test -n "$verbose" && echo "Import uncompressed File ..."
- $sudo_cmd psql $quiet "$database_name" <"$fill_from_dump"
- ;;
- esac
- if [ "$?" -gt "0" ]; then
- echo "Error While reding Dump into Database"
- fi
-fi
-
-
-############################################
-# Check number of entries in Database
-############################################
-if [ -n "$count_db" ] ; then
- echo ""
- echo "--------- Check Number of lines in Database '$database_name'"
-
- # Get the Table names
- if [ -n "$osm_username" ]; then
- table_owner=" AND tableowner ='$osm_username' ";
- fi
- table_names=`echo "SELECT tablename from pg_catalog.pg_tables where schemaname = 'public' $tableowner;" | \
- $sudo_cmd psql "$database_name" -h /var/run/postgresql | grep -E -e '^ planet'`
-
- echo "Counting entries in all Tables (" $table_names ")"
- for table in $table_names; do
- echo -n "Table $table = "
- echo "SELECT COUNT(*) from $table;" | \
- $sudo_cmd psql gis -h /var/run/postgresql | grep -v -e count -e '------' -e '1 row' | head -1
- done
-fi
-
diff --git a/middle-pgsql.c b/middle-pgsql.c
deleted file mode 100644
index 5eaa8be..0000000
--- a/middle-pgsql.c
+++ /dev/null
@@ -1,1830 +0,0 @@
-/* Implements the mid-layer processing for osm2pgsql
- * using several PostgreSQL tables
- *
- * This layer stores data read in from the planet.osm file
- * and is then read by the backend processing code to
- * emit the final geometry-enabled output formats
-*/
-
-#include "config.h"
-
-#include <stdio.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <math.h>
-#include <time.h>
-#include <errno.h>
-
-#ifdef HAVE_PTHREAD
-#include <pthread.h>
-#endif
-
-#ifdef HAVE_SYS_WAIT_H
-#include <sys/wait.h>
-#endif
-
-#ifdef HAVE_MMAP
-#include <sys/mman.h>
-#ifndef MAP_ANONYMOUS
-#ifdef MAP_ANON
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-#endif
-#endif
-
-#include <libpq-fe.h>
-
-#include "osmtypes.h"
-#include "middle.h"
-#include "middle-pgsql.h"
-#include "output-pgsql.h"
-#include "node-ram-cache.h"
-#include "node-persistent-cache.h"
-#include "pgsql.h"
-
-struct progress_info {
- time_t start;
- time_t end;
- int count;
- int finished;
-};
-
-enum table_id {
- t_node, t_way, t_rel
-} ;
-
-struct table_desc {
- const char *name;
- const char *start;
- const char *create;
- const char *create_index;
- const char *prepare;
- const char *prepare_intarray;
- const char *copy;
- const char *analyze;
- const char *stop;
- const char *array_indexes;
-
- int copyMode; /* True if we are in copy mode */
- int transactionMode; /* True if we are in an extended transaction */
- PGconn *sql_conn;
-};
-
-static struct table_desc tables [] = {
- {
- /*table = t_node,*/
- .name = "%p_nodes",
- .start = "BEGIN;\n",
-#ifdef FIXED_POINT
- .create = "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat int4 not null, lon int4 not null, tags text[]) {TABLESPACE %t};\n",
- .prepare = "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", int4, int4, text[]) AS INSERT INTO %p_nodes VALUES ($1,$2,$3,$4);\n"
-#else
- .create = "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat double precision not null, lon double precision not null, tags text[]) {TABLESPACE %t};\n",
- .prepare = "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", double precision, double precision, text[]) AS INSERT INTO %p_nodes VALUES ($1,$2,$3,$4);\n"
-#endif
- "PREPARE get_node (" POSTGRES_OSMID_TYPE ") AS SELECT lat,lon,tags FROM %p_nodes WHERE id = $1 LIMIT 1;\n"
- "PREPARE get_node_list(" POSTGRES_OSMID_TYPE "[]) AS SELECT id, lat, lon FROM %p_nodes WHERE id = ANY($1::" POSTGRES_OSMID_TYPE "[]);\n"
- "PREPARE delete_node (" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_nodes WHERE id = $1;\n",
- .copy = "COPY %p_nodes FROM STDIN;\n",
- .analyze = "ANALYZE %p_nodes;\n",
- .stop = "COMMIT;\n"
- },
- {
- /*table = t_way,*/
- .name = "%p_ways",
- .start = "BEGIN;\n",
- .create = "CREATE %m TABLE %p_ways (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, nodes " POSTGRES_OSMID_TYPE "[] not null, tags text[], pending boolean not null) {TABLESPACE %t};\n",
- .create_index = "CREATE INDEX %p_ways_idx ON %p_ways (id) {TABLESPACE %i} WHERE pending;\n",
-.array_indexes = "CREATE INDEX %p_ways_nodes ON %p_ways USING gin (nodes) {TABLESPACE %i};\n",
- .prepare = "PREPARE insert_way (" POSTGRES_OSMID_TYPE ", " POSTGRES_OSMID_TYPE "[], text[], boolean) AS INSERT INTO %p_ways VALUES ($1,$2,$3,$4);\n"
- "PREPARE get_way (" POSTGRES_OSMID_TYPE ") AS SELECT nodes, tags, array_upper(nodes,1) FROM %p_ways WHERE id = $1;\n"
- "PREPARE get_way_list (" POSTGRES_OSMID_TYPE "[]) AS SELECT id, nodes, tags, array_upper(nodes,1) FROM %p_ways WHERE id = ANY($1::" POSTGRES_OSMID_TYPE "[]);\n"
- "PREPARE way_done(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_ways SET pending = false WHERE id = $1;\n"
- "PREPARE pending_ways AS SELECT id FROM %p_ways WHERE pending;\n"
- "PREPARE delete_way(" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_ways WHERE id = $1;\n",
-.prepare_intarray = "PREPARE node_changed_mark(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_ways SET pending = true WHERE nodes && ARRAY[$1] AND NOT pending;\n"
- "PREPARE rel_delete_mark(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_ways SET pending = true WHERE id IN (SELECT unnest(parts[way_off+1:rel_off]) FROM %p_rels WHERE id = $1) AND NOT pending;\n",
- .copy = "COPY %p_ways FROM STDIN;\n",
- .analyze = "ANALYZE %p_ways;\n",
- .stop = "COMMIT;\n"
- },
- {
- /*table = t_rel,*/
- .name = "%p_rels",
- .start = "BEGIN;\n",
- .create = "CREATE %m TABLE %p_rels(id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, way_off int2, rel_off int2, parts " POSTGRES_OSMID_TYPE "[], members text[], tags text[], pending boolean not null) {TABLESPACE %t};\n",
- .create_index = "CREATE INDEX %p_rels_idx ON %p_rels (id) {TABLESPACE %i} WHERE pending;\n",
-.array_indexes = "CREATE INDEX %p_rels_parts ON %p_rels USING gin (parts) {TABLESPACE %i};\n",
- .prepare = "PREPARE insert_rel (" POSTGRES_OSMID_TYPE ", int2, int2, " POSTGRES_OSMID_TYPE "[], text[], text[]) AS INSERT INTO %p_rels VALUES ($1,$2,$3,$4,$5,$6,false);\n"
- "PREPARE get_rel (" POSTGRES_OSMID_TYPE ") AS SELECT members, tags, array_upper(members,1)/2 FROM %p_rels WHERE id = $1;\n"
- "PREPARE rel_done(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_rels SET pending = false WHERE id = $1;\n"
- "PREPARE pending_rels AS SELECT id FROM %p_rels WHERE pending;\n"
- "PREPARE delete_rel(" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_rels WHERE id = $1;\n",
-.prepare_intarray =
- "PREPARE node_changed_mark(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_rels SET pending = true WHERE parts && ARRAY[$1] AND parts[1:way_off] && ARRAY[$1] AND NOT pending;\n"
- "PREPARE way_changed_mark(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_rels SET pending = true WHERE parts && ARRAY[$1] AND parts[way_off+1:rel_off] && ARRAY[$1] AND NOT pending;\n"
- "PREPARE rel_changed_mark(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_rels SET pending = true WHERE parts && ARRAY[$1] AND parts[rel_off+1:array_length(parts,1)] && ARRAY[$1] AND NOT pending;\n",
- .copy = "COPY %p_rels FROM STDIN;\n",
- .analyze = "ANALYZE %p_rels;\n",
- .stop = "COMMIT;\n"
- }
-};
-
-static const int num_tables = sizeof(tables)/sizeof(tables[0]);
-static struct table_desc *node_table = &tables[t_node];
-static struct table_desc *way_table = &tables[t_way];
-static struct table_desc *rel_table = &tables[t_rel];
-
-static int Append;
-
-const struct output_options *out_options;
-
-#define HELPER_STATE_UNINITIALIZED -1
-#define HELPER_STATE_FORKED -2
-#define HELPER_STATE_RUNNING 0
-#define HELPER_STATE_FINISHED 1
-#define HELPER_STATE_CONNECTED 2
-#define HELPER_STATE_FAILED 3
-
-static int pgsql_connect(const struct output_options *options) {
- int i;
- /* We use a connection per table to enable the use of COPY */
- for (i=0; i<num_tables; i++) {
- PGconn *sql_conn;
- sql_conn = PQconnectdb(options->conninfo);
-
- /* Check to see that the backend connection was successfully made */
- if (PQstatus(sql_conn) != CONNECTION_OK) {
- fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(sql_conn));
- return 1;
- }
- tables[i].sql_conn = sql_conn;
-
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;");
-
- if (tables[i].prepare) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare);
- }
-
- if (tables[i].prepare_intarray) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare_intarray);
- }
-
- }
- return 0;
-}
-
-static void pgsql_cleanup(void)
-{
- int i;
-
- for (i=0; i<num_tables; i++) {
- if (tables[i].sql_conn) {
- PQfinish(tables[i].sql_conn);
- tables[i].sql_conn = NULL;
- }
- }
-}
-
-char *pgsql_store_nodes(osmid_t *nds, int nd_count)
-{
- static char *buffer;
- static int buflen;
-
- char *ptr;
- int i, first;
-
- if( buflen <= nd_count * 10 )
- {
- buflen = ((nd_count * 10) | 4095) + 1; /* Round up to next page */
- buffer = realloc( buffer, buflen );
- }
-_restart:
-
- ptr = buffer;
- first = 1;
- *ptr++ = '{';
- for( i=0; i<nd_count; i++ )
- {
- if( !first ) *ptr++ = ',';
- ptr += sprintf(ptr, "%" PRIdOSMID, nds[i] );
-
- if( (ptr-buffer) > (buflen-20) ) /* Almost overflowed? */
- {
- buflen <<= 1;
- buffer = realloc( buffer, buflen );
-
- goto _restart;
- }
- first = 0;
- }
-
- *ptr++ = '}';
- *ptr++ = 0;
-
- return buffer;
-}
-
-/* Special escape routine for escaping strings in array constants: double quote, backslash,newline, tab*/
-static char *escape_tag( char *ptr, const char *in, int escape )
-{
- while( *in )
- {
- switch(*in)
- {
- case '"':
- if( escape ) *ptr++ = '\\';
- *ptr++ = '\\';
- *ptr++ = '"';
- break;
- case '\\':
- if( escape ) *ptr++ = '\\';
- if( escape ) *ptr++ = '\\';
- *ptr++ = '\\';
- *ptr++ = '\\';
- break;
- case '\n':
- if( escape ) *ptr++ = '\\';
- *ptr++ = '\\';
- *ptr++ = 'n';
- break;
- case '\r':
- if( escape ) *ptr++ = '\\';
- *ptr++ = '\\';
- *ptr++ = 'r';
- break;
- case '\t':
- if( escape ) *ptr++ = '\\';
- *ptr++ = '\\';
- *ptr++ = 't';
- break;
- default:
- *ptr++ = *in;
- break;
- }
- in++;
- }
- return ptr;
-}
-
-/* escape means we return '\N' for copy mode, otherwise we return just NULL */
-char *pgsql_store_tags(struct keyval *tags, int escape)
-{
- static char *buffer;
- static int buflen;
-
- char *ptr;
- struct keyval *i;
- int first;
-
- int countlist = countList(tags);
- if( countlist == 0 )
- {
- if( escape )
- return "\\N";
- else
- return NULL;
- }
-
- if( buflen <= countlist * 24 ) /* LE so 0 always matches */
- {
- buflen = ((countlist * 24) | 4095) + 1; /* Round up to next page */
- buffer = realloc( buffer, buflen );
- }
-_restart:
-
- ptr = buffer;
- first = 1;
- *ptr++ = '{';
- /* The lists are circular, exit when we reach the head again */
- for( i=tags->next; i->key; i = i->next )
- {
- int maxlen = (strlen(i->key) + strlen(i->value)) * 4;
- if( (ptr+maxlen-buffer) > (buflen-20) ) /* Almost overflowed? */
- {
- buflen <<= 1;
- buffer = realloc( buffer, buflen );
-
- goto _restart;
- }
- if( !first ) *ptr++ = ',';
- *ptr++ = '"';
- ptr = escape_tag( ptr, i->key, escape );
- *ptr++ = '"';
- *ptr++ = ',';
- *ptr++ = '"';
- ptr = escape_tag( ptr, i->value, escape );
- *ptr++ = '"';
-
- first=0;
- }
-
- *ptr++ = '}';
- *ptr++ = 0;
-
- return buffer;
-}
-
-/* Decodes a portion of an array literal from postgres */
-/* Argument should point to beginning of literal, on return points to delimiter */
-static const char *decode_upto( const char *src, char *dst )
-{
- int quoted = (*src == '"');
- if( quoted ) src++;
-
- while( quoted ? (*src != '"') : (*src != ',' && *src != '}') )
- {
- if( *src == '\\' )
- {
- switch( src[1] )
- {
- case 'n': *dst++ = '\n'; break;
- case 't': *dst++ = '\t'; break;
- default: *dst++ = src[1]; break;
- }
- src+=2;
- }
- else
- *dst++ = *src++;
- }
- if( quoted ) src++;
- *dst = 0;
- return src;
-}
-
-static void pgsql_parse_tags( const char *string, struct keyval *tags )
-{
- char key[1024];
- char val[1024];
-
- if( *string == '\0' )
- return;
-
- if( *string++ != '{' )
- return;
- while( *string != '}' )
- {
- string = decode_upto( string, key );
- /* String points to the comma */
- string++;
- string = decode_upto( string, val );
- /* String points to the comma or closing '}' */
- addItem( tags, key, val, 0 );
- if( *string == ',' )
- string++;
- }
-}
-
-/* Parses an array of integers */
-static void pgsql_parse_nodes(const char *src, osmid_t *nds, int nd_count )
-{
- int count = 0;
- const char *string = src;
-
- if( *string++ != '{' )
- return;
- while( *string != '}' )
- {
- char *ptr;
- nds[count] = strtoosmid( string, &ptr, 10 );
- string = ptr;
- if( *string == ',' )
- string++;
- count++;
- }
- if( count != nd_count )
- {
- fprintf( stderr, "parse_nodes problem: '%s' expected %d got %d\n", src, nd_count, count );
- exit_nicely();
- }
-}
-
-static int pgsql_endCopy( struct table_desc *table)
-{
- PGresult *res;
- PGconn *sql_conn;
- int stop;
- /* Terminate any pending COPY */
- if (table->copyMode) {
- sql_conn = table->sql_conn;
- stop = PQputCopyEnd(sql_conn, NULL);
- if (stop != 1) {
- fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
- exit_nicely();
- }
-
- res = PQgetResult(sql_conn);
- if (PQresultStatus(res) != PGRES_COMMAND_OK) {
- fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
- PQclear(res);
- exit_nicely();
- }
- PQclear(res);
- table->copyMode = 0;
- }
- return 0;
-}
-
-static int pgsql_nodes_set(osmid_t id, double lat, double lon, struct keyval *tags)
-{
- /* Four params: id, lat, lon, tags */
- char *paramValues[4];
- char *buffer;
-
- if( node_table->copyMode )
- {
- char *tag_buf = pgsql_store_tags(tags,1);
- int length = strlen(tag_buf) + 64;
- buffer = alloca( length );
-#ifdef FIXED_POINT
- if( snprintf( buffer, length, "%" PRIdOSMID "\t%d\t%d\t%s\n", id, DOUBLE_TO_FIX(lat), DOUBLE_TO_FIX(lon), tag_buf ) > (length-10) )
- { fprintf( stderr, "buffer overflow node id %" PRIdOSMID "\n", id); return 1; }
-#else
- if( snprintf( buffer, length, "%" PRIdOSMID "\t%.10f\t%.10f\t%s\n", id, lat, lon, tag_buf ) > (length-10) )
- { fprintf( stderr, "buffer overflow node id %" PRIdOSMID "\n", id); return 1; }
-#endif
- return pgsql_CopyData(__FUNCTION__, node_table->sql_conn, buffer);
- }
- buffer = alloca(64);
- paramValues[0] = buffer;
- paramValues[1] = paramValues[0] + sprintf( paramValues[0], "%" PRIdOSMID, id ) + 1;
-#ifdef FIXED_POINT
- paramValues[2] = paramValues[1] + sprintf( paramValues[1], "%d", DOUBLE_TO_FIX(lat) ) + 1;
- sprintf( paramValues[2], "%d", DOUBLE_TO_FIX(lon) );
-#else
- paramValues[2] = paramValues[1] + sprintf( paramValues[1], "%.10f", lat ) + 1;
- sprintf( paramValues[2], "%.10f", lon );
-#endif
- paramValues[3] = pgsql_store_tags(tags,0);
- pgsql_execPrepared(node_table->sql_conn, "insert_node", 4, (const char * const *)paramValues, PGRES_COMMAND_OK);
- return 0;
-}
-
-static int middle_nodes_set(osmid_t id, double lat, double lon, struct keyval *tags) {
- ram_cache_nodes_set( id, lat, lon, tags );
-
- return (out_options->flat_node_cache_enabled) ? persistent_cache_nodes_set(id, lat, lon) : pgsql_nodes_set(id, lat, lon, tags);
-}
-
-
-#if 0
-static int pgsql_nodes_get(struct osmNode *out, osmid_t id)
-{
- PGresult *res;
- char tmp[16];
- char const *paramValues[1];
- PGconn *sql_conn = node_table->sql_conn;
-
- /* Make sure we're out of copy mode */
- pgsql_endCopy( node_table );
-
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
- paramValues[0] = tmp;
-
- res = pgsql_execPrepared(sql_conn, "get_node", 1, paramValues, PGRES_TUPLES_OK);
-
- if (PQntuples(res) != 1) {
- PQclear(res);
- return 1;
- }
-
-#ifdef FIXED_POINT
- out->lat = FIX_TO_DOUBLE(strtol(PQgetvalue(res, 0, 0), NULL, 10));
- out->lon = FIX_TO_DOUBLE(strtol(PQgetvalue(res, 0, 1), NULL, 10));
-#else
- out->lat = strtod(PQgetvalue(res, 0, 0), NULL);
- out->lon = strtod(PQgetvalue(res, 0, 1), NULL);
-#endif
- PQclear(res);
- return 0;
-}
-#endif
-
-/* Currently not used
-static int middle_nodes_get(struct osmNode *out, osmid_t id)
-{
- / * Check cache first * /
- if( ram_cache_nodes_get( out, id ) == 0 )
- return 0;
-
- return (out_options->flat_node_cache_enabled) ? persistent_cache_nodes_get(out, id) : pgsql_nodes_get(out, id);
-}*/
-
-
-/* This should be made more efficient by using an IN(ARRAY[]) construct */
-static int pgsql_nodes_get_list(struct osmNode *nodes, osmid_t *ndids, int nd_count)
-{
- char tmp[16];
- char *tmp2;
- int count, countDB, countPG, i,j;
- osmid_t *ndidspg;
- struct osmNode *nodespg;
- char const *paramValues[1];
-
- PGresult *res;
- PGconn *sql_conn = node_table->sql_conn;
-
- count = 0; countDB = 0;
-
- tmp2 = malloc(sizeof(char)*nd_count*16);
- if (tmp2 == NULL) return 0; /*failed to allocate memory, return */
-
- /* create a list of ids in tmp2 to query the database */
- sprintf(tmp2, "{");
- for( i=0; i<nd_count; i++ ) {
- /* Check cache first */
- if( ram_cache_nodes_get( &nodes[i], ndids[i]) == 0 ) {
- count++;
- continue;
- }
- countDB++;
- /* Mark nodes as needing to be fetched from the DB */
- nodes[i].lat = NAN;
- nodes[i].lon = NAN;
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", ndids[i]);
- strncat(tmp2, tmp, sizeof(char)*(nd_count*16 - 2));
- }
- tmp2[strlen(tmp2) - 1] = '}'; /* replace last , with } to complete list of ids*/
-
- if (countDB == 0) {
- free(tmp2);
- return count; /* All ids where in cache, so nothing more to do */
- }
-
- pgsql_endCopy(node_table);
-
- paramValues[0] = tmp2;
- res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues, PGRES_TUPLES_OK);
- countPG = PQntuples(res);
-
- ndidspg = malloc(sizeof(osmid_t)*countPG);
- nodespg = malloc(sizeof(struct osmNode)*countPG);
-
- if ((ndidspg == NULL) || (nodespg == NULL)) {
- free(tmp2);
- free(ndidspg);
- free(nodespg);
- PQclear(res);
- return 0;
- }
-
- for (i = 0; i < countPG; i++) {
- ndidspg[i] = strtoosmid(PQgetvalue(res, i, 0), NULL, 10);
-#ifdef FIXED_POINT
- nodespg[i].lat = FIX_TO_DOUBLE(strtol(PQgetvalue(res, i, 1), NULL, 10));
- nodespg[i].lon = FIX_TO_DOUBLE(strtol(PQgetvalue(res, i, 2), NULL, 10));
-#else
- nodespg[i].lat = strtod(PQgetvalue(res, i, 1), NULL);
- nodespg[i].lon = strtod(PQgetvalue(res, i, 2), NULL);
-#endif
- }
-
-
- /* The list of results coming back from the db is in a different order to the list of nodes in the way.
- Match the results back to the way node list */
-
- for (i=0; i<nd_count; i++ ) {
- if ((isnan(nodes[i].lat)) || (isnan(nodes[i].lon))) {
- /* TODO: implement an O(log(n)) algorithm to match node ids */
- for (j = 0; j < countPG; j++) {
- if (ndidspg[j] == ndids[i]) {
- nodes[i].lat = nodespg[j].lat;
- nodes[i].lon = nodespg[j].lon;
- count++;
- break;
- }
- }
- }
- }
-
- /* If some of the nodes in the way don't exist, the returning list has holes.
- As the rest of the code expects a continuous list, it needs to be re-compacted */
- if (count != nd_count) {
- j = 0;
- for (i = 0; i < nd_count; i++) {
- if ( !isnan(nodes[i].lat)) {
- nodes[j].lat = nodes[i].lat;
- nodes[j].lon = nodes[i].lon;
- j++;
- }
- }
- }
-
- PQclear(res);
- free(tmp2);
- free(ndidspg);
- free(nodespg);
-
- return count;
-}
-
-static int middle_nodes_get_list(struct osmNode *nodes, osmid_t *ndids, int nd_count)
-{
- return (out_options->flat_node_cache_enabled) ? persistent_cache_nodes_get_list(nodes, ndids, nd_count) : pgsql_nodes_get_list(nodes, ndids, nd_count);
-}
-
-static int pgsql_nodes_delete(osmid_t osm_id)
-{
- char const *paramValues[1];
- char buffer[64];
- /* Make sure we're out of copy mode */
- pgsql_endCopy( node_table );
-
- sprintf( buffer, "%" PRIdOSMID, osm_id );
- paramValues[0] = buffer;
- pgsql_execPrepared(node_table->sql_conn, "delete_node", 1, paramValues, PGRES_COMMAND_OK );
- return 0;
-}
-
-static int middle_nodes_delete(osmid_t osm_id)
-{
- return ((out_options->flat_node_cache_enabled) ? persistent_cache_nodes_set(osm_id, NAN, NAN) : pgsql_nodes_delete(osm_id));
-}
-
-static int pgsql_node_changed(osmid_t osm_id)
-{
- char const *paramValues[1];
- char buffer[64];
- /* Make sure we're out of copy mode */
- pgsql_endCopy( way_table );
- pgsql_endCopy( rel_table );
-
- sprintf( buffer, "%" PRIdOSMID, osm_id );
- paramValues[0] = buffer;
- pgsql_execPrepared(way_table->sql_conn, "node_changed_mark", 1, paramValues, PGRES_COMMAND_OK );
- pgsql_execPrepared(rel_table->sql_conn, "node_changed_mark", 1, paramValues, PGRES_COMMAND_OK );
- return 0;
-}
-
-static int pgsql_ways_set(osmid_t way_id, osmid_t *nds, int nd_count, struct keyval *tags, int pending)
-{
- /* Three params: id, nodes, tags, pending */
- char *paramValues[4];
- char *buffer;
-
- if( way_table->copyMode )
- {
- char *tag_buf = pgsql_store_tags(tags,1);
- char *node_buf = pgsql_store_nodes(nds, nd_count);
- int length = strlen(tag_buf) + strlen(node_buf) + 64;
- buffer = alloca(length);
- if( snprintf( buffer, length, "%" PRIdOSMID "\t%s\t%s\t%c\n",
- way_id, node_buf, tag_buf, pending?'t':'f' ) > (length-10) )
- { fprintf( stderr, "buffer overflow way id %" PRIdOSMID "\n", way_id); return 1; }
- return pgsql_CopyData(__FUNCTION__, way_table->sql_conn, buffer);
- }
- buffer = alloca(64);
- paramValues[0] = buffer;
- paramValues[3] = paramValues[0] + sprintf( paramValues[0], "%" PRIdOSMID, way_id ) + 1;
- sprintf( paramValues[3], "%c", pending?'t':'f' );
- paramValues[1] = pgsql_store_nodes(nds, nd_count);
- paramValues[2] = pgsql_store_tags(tags,0);
- pgsql_execPrepared(way_table->sql_conn, "insert_way", 4, (const char * const *)paramValues, PGRES_COMMAND_OK);
- return 0;
-}
-
-/* Caller is responsible for freeing nodesptr & resetList(tags) */
-static int pgsql_ways_get(osmid_t id, struct keyval *tags, struct osmNode **nodes_ptr, int *count_ptr)
-{
- PGresult *res;
- char tmp[16];
- char const *paramValues[1];
- PGconn *sql_conn = way_table->sql_conn;
- int num_nodes;
- osmid_t *list;
-
- /* Make sure we're out of copy mode */
- pgsql_endCopy( way_table );
-
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
- paramValues[0] = tmp;
-
- res = pgsql_execPrepared(sql_conn, "get_way", 1, paramValues, PGRES_TUPLES_OK);
-
- if (PQntuples(res) != 1) {
- PQclear(res);
- return 1;
- }
-
- pgsql_parse_tags( PQgetvalue(res, 0, 1), tags );
-
- num_nodes = strtol(PQgetvalue(res, 0, 2), NULL, 10);
- list = alloca(sizeof(osmid_t)*num_nodes );
- *nodes_ptr = malloc(sizeof(struct osmNode) * num_nodes);
- pgsql_parse_nodes( PQgetvalue(res, 0, 0), list, num_nodes);
-
- *count_ptr = out_options->flat_node_cache_enabled ?
- persistent_cache_nodes_get_list(*nodes_ptr, list, num_nodes) :
- pgsql_nodes_get_list( *nodes_ptr, list, num_nodes);
- PQclear(res);
- return 0;
-}
-
-static int pgsql_ways_get_list(osmid_t *ids, int way_count, osmid_t **way_ids, struct keyval *tags, struct osmNode **nodes_ptr, int *count_ptr) {
-
- char tmp[16];
- char *tmp2;
- int count, countPG, i, j;
- osmid_t *wayidspg;
- char const *paramValues[1];
- int num_nodes;
- osmid_t *list;
-
- PGresult *res;
- PGconn *sql_conn = way_table->sql_conn;
-
- *way_ids = malloc( sizeof(osmid_t) * (way_count + 1));
- if (way_count == 0) return 0;
-
- tmp2 = malloc(sizeof(char)*way_count*16);
- if (tmp2 == NULL) return 0; /*failed to allocate memory, return */
-
- /* create a list of ids in tmp2 to query the database */
- sprintf(tmp2, "{");
- for( i=0; i<way_count; i++ ) {
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", ids[i]);
- strncat(tmp2,tmp, sizeof(char)*(way_count*16 - 2));
- }
- tmp2[strlen(tmp2) - 1] = '}'; /* replace last , with } to complete list of ids*/
-
- pgsql_endCopy(way_table);
-
- paramValues[0] = tmp2;
- res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues, PGRES_TUPLES_OK);
- countPG = PQntuples(res);
-
- wayidspg = malloc(sizeof(osmid_t)*countPG);
-
- if (wayidspg == NULL) return 0; /*failed to allocate memory, return */
-
- for (i = 0; i < countPG; i++) {
- wayidspg[i] = strtoosmid(PQgetvalue(res, i, 0), NULL, 10);
- }
-
-
- /* Match the list of ways coming from postgres in a different order
- back to the list of ways given by the caller */
- count = 0;
- initList(&(tags[count]));
- for (i = 0; i < way_count; i++) {
- for (j = 0; j < countPG; j++) {
- if (ids[i] == wayidspg[j]) {
- (*way_ids)[count] = ids[i];
- pgsql_parse_tags( PQgetvalue(res, j, 2), &(tags[count]) );
-
- num_nodes = strtol(PQgetvalue(res, j, 3), NULL, 10);
- list = alloca(sizeof(osmid_t)*num_nodes );
- nodes_ptr[count] = malloc(sizeof(struct osmNode) * num_nodes);
- pgsql_parse_nodes( PQgetvalue(res, j, 1), list, num_nodes);
-
- count_ptr[count] = out_options->flat_node_cache_enabled ?
- persistent_cache_nodes_get_list(nodes_ptr[count], list, num_nodes) :
- pgsql_nodes_get_list( nodes_ptr[count], list, num_nodes);
-
- count++;
- initList(&(tags[count]));
- }
- }
- }
-
- PQclear(res);
- free(tmp2);
- free(wayidspg);
-
- return count;
-}
-
-static int pgsql_ways_done(osmid_t id)
-{
- char tmp[16];
- char const *paramValues[1];
- PGconn *sql_conn = way_table->sql_conn;
-
- /* Make sure we're out of copy mode */
- pgsql_endCopy( way_table );
-
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
- paramValues[0] = tmp;
-
- pgsql_execPrepared(sql_conn, "way_done", 1, paramValues, PGRES_COMMAND_OK);
-
- return 0;
-}
-
-static int pgsql_ways_delete(osmid_t osm_id)
-{
- char const *paramValues[1];
- char buffer[64];
- /* Make sure we're out of copy mode */
- pgsql_endCopy( way_table );
-
- sprintf( buffer, "%" PRIdOSMID, osm_id );
- paramValues[0] = buffer;
- pgsql_execPrepared(way_table->sql_conn, "delete_way", 1, paramValues, PGRES_COMMAND_OK );
- return 0;
-}
-
-static void pgsql_iterate_ways(int (*callback)(osmid_t id, struct keyval *tags, struct osmNode *nodes, int count, int exists))
-{
- int noProcs = out_options->num_procs;
- int pid = 0;
- PGresult *res_ways;
- int i, p, count = 0;
- /* The flag we pass to indicate that the way in question might exist already in the database */
- int exists = Append;
-
- time_t start, end;
- time(&start);
-#if HAVE_MMAP
- struct progress_info *info = 0;
- if(noProcs > 1) {
- info = mmap(0, sizeof(struct progress_info)*noProcs, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0);
- info[0].finished = HELPER_STATE_CONNECTED;
- for (i = 1; i < noProcs; i++) {
- info[i].finished = HELPER_STATE_UNINITIALIZED; /* Register that the process was not yet initialised; */
- }
- }
-#endif
- fprintf(stderr, "\nGoing over pending ways...\n");
-
- /* Make sure we're out of copy mode */
- pgsql_endCopy( way_table );
-
- if (out_options->flat_node_cache_enabled) shutdown_node_persistent_cache();
-
- res_ways = pgsql_execPrepared(way_table->sql_conn, "pending_ways", 0, NULL, PGRES_TUPLES_OK);
-
- fprintf(stderr, "\t%i ways are pending\n", PQntuples(res_ways));
-
-
- /**
- * To speed up processing of pending ways, fork noProcs worker processes
- * each of which independently goes through an equal subset of the pending ways array
- */
- fprintf(stderr, "\nUsing %i helper-processes\n", noProcs);
-#ifdef HAVE_FORK
- for (p = 1; p < noProcs; p++) {
- pid=fork();
- if (pid==0) {
-#if HAVE_MMAP
- info[p].finished = HELPER_STATE_FORKED;
-#endif
- break;
- }
- if (pid==-1) {
-#if HAVE_MMAP
- info[p].finished = HELPER_STATE_FAILED;
- fprintf(stderr,"WARNING: Failed to fork helper process %i: %s. Trying to recover.\n", p, strerror(errno));
-#else
- fprintf(stderr,"ERROR: Failed to fork helper process %i: %s. Can't recover!\n", p, strerror(errno));
- exit_nicely();
-#endif
- }
- }
-#endif
- if ((pid == 0) && (noProcs > 1)) {
- /* After forking, need to reconnect to the postgresql db */
- if ((pgsql_connect(out_options) != 0) || (out_options->out->connect(out_options, 1) != 0)) {
-#if HAVE_MMAP
- info[p].finished = HELPER_STATE_FAILED;
-#else
- fprintf(stderr,"\n\n!!!FATAL: Helper process failed, but can't compensate. Your DB will be broken and corrupt!!!!\n\n");
-#endif
- exit_nicely();
- };
- } else {
- p = 0;
- }
-
- if (out_options->flat_node_cache_enabled) init_node_persistent_cache(out_options,1); /* at this point we always want to be in append mode, to not delete and recreate the node cache file */
-
- /* Only start an extended transaction on the ways table,
- * which should cover the bulk of the update statements.
- * The nodes table should not be written to in this phase.
- * The relations table can't be wrapped in an extended
- * transaction, as with prallel processing it may deadlock.
- * Updating a way will trigger an update of the pending status
- * on connected relations. This should not be as many updates,
- * so in combination with the synchronous_comit = off it should be fine.
- *
- */
- if (tables[t_way].start) {
- pgsql_endCopy(&tables[t_way]);
- pgsql_exec(tables[t_way].sql_conn, PGRES_COMMAND_OK, "%s", tables[t_way].start);
- tables[t_way].transactionMode = 1;
- }
-
-#if HAVE_MMAP
- if (noProcs > 1) {
- info[p].finished = HELPER_STATE_CONNECTED;
- /* Syncronize all processes to make sure they have all run through the initialisation steps */
- int all_processes_initialised = 0;
- while (all_processes_initialised == 0) {
- all_processes_initialised = 1;
- for (i = 0; i < noProcs; i++) {
- if (info[i].finished < 0) {
- all_processes_initialised = 0;
- sleep(1);
- }
- }
- }
-
- /* As we process the pending ways in steps of noProcs,
- we need to make sure that all processes correctly forked
- and have connected to the db. Otherwise we need to readjust
- the step size of going through the pending ways array */
- int noProcsTmp = noProcs;
- int pTmp = p;
- for (i = 0; i < noProcs; i++) {
- if (info[i].finished == HELPER_STATE_FAILED) {
- noProcsTmp--;
- if (i < p) pTmp--;
- }
- }
- info[p].finished = HELPER_STATE_RUNNING;
- p = pTmp; /* reset the process number to account for failed processes */
-
- /* As we have potentially changed the process number assignment,
- we need to synchronize on all processes having performed the reassignment
- as otherwise multiple process might have the same number and overwrite
- the info fields incorrectly.
- */
- all_processes_initialised = 0;
- while (all_processes_initialised == 0) {
- all_processes_initialised = 1;
- for (i = 0; i < noProcs; i++) {
- if (info[i].finished == HELPER_STATE_CONNECTED) {
- /* Process is connected, but hasn't performed the re-assignment of p. */
- all_processes_initialised = 0;
- sleep(1);
- break;
- }
- }
- }
- noProcs = noProcsTmp;
- }
-#endif
-
- /* some spaces at end, so that processings outputs get cleaned if already existing */
- fprintf(stderr, "\rHelper process %i out of %i initialised \n", p, noProcs);
- /* Use a stride length of the number of worker processes,
- starting with an offset for each worker process p */
- for (i = p; i < PQntuples(res_ways); i+= noProcs) {
- osmid_t id = strtoosmid(PQgetvalue(res_ways, i, 0), NULL, 10);
- struct keyval tags;
- struct osmNode *nodes;
- int nd_count;
-
- if (count++ %1000 == 0) {
- time(&end);
-#if HAVE_MMAP
- if(info)
- {
- double rate = 0;
- int n, total = 0, finished = 0;
- struct progress_info f;
-
- f.start = start;
- f.end = end;
- f.count = count;
- f.finished = HELPER_STATE_RUNNING;
- info[p] = f;
- for(n = 0; n < noProcs; ++n)
- {
- f = info[n];
- total += f.count;
- finished += f.finished;
- if(f.end > f.start)
- rate += (double)f.count / (double)(f.end - f.start);
- }
- fprintf(stderr, "\rprocessing way (%dk) at %.2fk/s (done %d of %d)", total/1000, rate/1000.0, finished, noProcs);
- }
- else
-#endif
- {
- fprintf(stderr, "\rprocessing way (%dk) at %.2fk/s", count/1000,
- end > start ? ((double)count / 1000.0 / (double)(end - start)) : 0);
- }
- }
-
- initList(&tags);
- if( pgsql_ways_get(id, &tags, &nodes, &nd_count) )
- continue;
-
- callback(id, &tags, nodes, nd_count, exists);
- pgsql_ways_done( id );
-
- free(nodes);
- resetList(&tags);
- }
-
- if (tables[t_way].stop && tables[t_way].transactionMode) {
- pgsql_exec(tables[t_way].sql_conn, PGRES_COMMAND_OK, "%s", tables[t_way].stop);
- tables[t_way].transactionMode = 0;
- }
-
- time(&end);
-#if HAVE_MMAP
- if(info)
- {
- struct progress_info f;
- f.start = start;
- f.end = end;
- f.count = count;
- f.finished = 1;
- info[p] = f;
- }
-#endif
- fprintf(stderr, "\rProcess %i finished processing %i ways in %i sec\n", p, count, (int)(end - start));
-
- if ((pid == 0) && (noProcs > 1)) {
- pgsql_cleanup();
- out_options->out->close(1);
- if (out_options->flat_node_cache_enabled) shutdown_node_persistent_cache();
- exit(0);
- } else {
- for (p = 0; p < noProcs; p++) wait(NULL);
- fprintf(stderr, "\nAll child processes exited\n");
- }
-
-#if HAVE_MMAP
- munmap(info, sizeof(struct progress_info)*noProcs);
-#endif
-
- fprintf(stderr, "\n");
- time(&end);
- if (end - start > 0)
- fprintf(stderr, "%i Pending ways took %ds at a rate of %.2f/s\n",PQntuples(res_ways), (int)(end - start),
- ((double)PQntuples(res_ways) / (double)(end - start)));
- PQclear(res_ways);
-}
-
-static int pgsql_way_changed(osmid_t osm_id)
-{
- char const *paramValues[1];
- char buffer[64];
- /* Make sure we're out of copy mode */
- pgsql_endCopy( rel_table );
-
- sprintf( buffer, "%" PRIdOSMID, osm_id );
- paramValues[0] = buffer;
- pgsql_execPrepared(rel_table->sql_conn, "way_changed_mark", 1, paramValues, PGRES_COMMAND_OK );
- return 0;
-}
-
-static int pgsql_rels_set(osmid_t id, struct member *members, int member_count, struct keyval *tags)
-{
- /* Params: id, way_off, rel_off, parts, members, tags */
- char *paramValues[6];
- char *buffer;
- int i;
- struct keyval member_list;
- char buf[64];
-
- osmid_t node_parts[member_count],
- way_parts[member_count],
- rel_parts[member_count];
- int node_count = 0, way_count = 0, rel_count = 0;
-
- osmid_t all_parts[member_count];
- int all_count = 0;
- initList( &member_list );
- for( i=0; i<member_count; i++ )
- {
- char tag = 0;
- switch( members[i].type )
- {
- case OSMTYPE_NODE: node_parts[node_count++] = members[i].id; tag = 'n'; break;
- case OSMTYPE_WAY: way_parts[way_count++] = members[i].id; tag = 'w'; break;
- case OSMTYPE_RELATION: rel_parts[rel_count++] = members[i].id; tag = 'r'; break;
- default: fprintf( stderr, "Internal error: Unknown member type %d\n", members[i].type ); exit_nicely();
- }
- sprintf( buf, "%c%" PRIdOSMID, tag, members[i].id );
- addItem( &member_list, buf, members[i].role, 0 );
- }
- memcpy( all_parts+all_count, node_parts, node_count*sizeof(osmid_t) ); all_count+=node_count;
- memcpy( all_parts+all_count, way_parts, way_count*sizeof(osmid_t) ); all_count+=way_count;
- memcpy( all_parts+all_count, rel_parts, rel_count*sizeof(osmid_t) ); all_count+=rel_count;
-
- if( rel_table->copyMode )
- {
- char *tag_buf = strdup(pgsql_store_tags(tags,1));
- char *member_buf = pgsql_store_tags(&member_list,1);
- char *parts_buf = pgsql_store_nodes(all_parts, all_count);
- int length = strlen(member_buf) + strlen(tag_buf) + strlen(parts_buf) + 64;
- buffer = alloca(length);
- if( snprintf( buffer, length, "%" PRIdOSMID "\t%d\t%d\t%s\t%s\t%s\tf\n",
- id, node_count, node_count+way_count, parts_buf, member_buf, tag_buf ) > (length-10) )
- { fprintf( stderr, "buffer overflow relation id %" PRIdOSMID "\n", id); return 1; }
- free(tag_buf);
- resetList(&member_list);
- return pgsql_CopyData(__FUNCTION__, rel_table->sql_conn, buffer);
- }
- buffer = alloca(64);
- paramValues[0] = buffer;
- paramValues[1] = paramValues[0] + sprintf( paramValues[0], "%" PRIdOSMID, id ) + 1;
- paramValues[2] = paramValues[1] + sprintf( paramValues[1], "%d", node_count ) + 1;
- sprintf( paramValues[2], "%d", node_count+way_count );
- paramValues[3] = pgsql_store_nodes(all_parts, all_count);
- paramValues[4] = pgsql_store_tags(&member_list,0);
- if( paramValues[4] )
- paramValues[4] = strdup(paramValues[4]);
- paramValues[5] = pgsql_store_tags(tags,0);
- pgsql_execPrepared(rel_table->sql_conn, "insert_rel", 6, (const char * const *)paramValues, PGRES_COMMAND_OK);
- if( paramValues[4] )
- free(paramValues[4]);
- resetList(&member_list);
- return 0;
-}
-
-/* Caller is responsible for freeing members & resetList(tags) */
-static int pgsql_rels_get(osmid_t id, struct member **members, int *member_count, struct keyval *tags)
-{
- PGresult *res;
- char tmp[16];
- char const *paramValues[1];
- PGconn *sql_conn = rel_table->sql_conn;
- struct keyval member_temp;
- char tag;
- int num_members;
- struct member *list;
- int i=0;
- struct keyval *item;
-
- /* Make sure we're out of copy mode */
- pgsql_endCopy( rel_table );
-
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
- paramValues[0] = tmp;
-
- res = pgsql_execPrepared(sql_conn, "get_rel", 1, paramValues, PGRES_TUPLES_OK);
- /* Fields are: members, tags, member_count */
-
- if (PQntuples(res) != 1) {
- PQclear(res);
- return 1;
- }
-
- pgsql_parse_tags( PQgetvalue(res, 0, 1), tags );
- initList(&member_temp);
- pgsql_parse_tags( PQgetvalue(res, 0, 0), &member_temp );
-
- num_members = strtol(PQgetvalue(res, 0, 2), NULL, 10);
- list = malloc( sizeof(struct member)*num_members );
-
- while( (item = popItem(&member_temp)) )
- {
- if( i >= num_members )
- {
- fprintf(stderr, "Unexpected member_count reading relation %" PRIdOSMID "\n", id);
- exit_nicely();
- }
- tag = item->key[0];
- list[i].type = (tag == 'n')?OSMTYPE_NODE:(tag == 'w')?OSMTYPE_WAY:(tag == 'r')?OSMTYPE_RELATION:-1;
- list[i].id = strtoosmid(item->key+1, NULL, 10 );
- list[i].role = strdup( item->value );
- freeItem(item);
- i++;
- }
- *members = list;
- *member_count = num_members;
- PQclear(res);
- return 0;
-}
-
-static int pgsql_rels_done(osmid_t id)
-{
- char tmp[16];
- char const *paramValues[1];
- PGconn *sql_conn = rel_table->sql_conn;
-
- /* Make sure we're out of copy mode */
- pgsql_endCopy( rel_table );
-
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
- paramValues[0] = tmp;
-
- pgsql_execPrepared(sql_conn, "rel_done", 1, paramValues, PGRES_COMMAND_OK);
-
- return 0;
-}
-
-static int pgsql_rels_delete(osmid_t osm_id)
-{
- char const *paramValues[1];
- char buffer[64];
- /* Make sure we're out of copy mode */
- pgsql_endCopy( way_table );
- pgsql_endCopy( rel_table );
-
- sprintf( buffer, "%" PRIdOSMID, osm_id );
- paramValues[0] = buffer;
- pgsql_execPrepared(way_table->sql_conn, "rel_delete_mark", 1, paramValues, PGRES_COMMAND_OK );
- pgsql_execPrepared(rel_table->sql_conn, "delete_rel", 1, paramValues, PGRES_COMMAND_OK );
- return 0;
-}
-
-static void pgsql_iterate_relations(int (*callback)(osmid_t id, struct member *members, int member_count, struct keyval *tags, int exists))
-{
- PGresult *res_rels;
- int noProcs = out_options->num_procs;
- int pid;
- int i, p, count = 0;
- /* The flag we pass to indicate that the way in question might exist already in the database */
- int exists = Append;
-
- time_t start, end;
- time(&start);
-#if HAVE_MMAP
- struct progress_info *info = 0;
- if(noProcs > 1) {
- info = mmap(0, sizeof(struct progress_info)*noProcs, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0);
- info[0].finished = HELPER_STATE_CONNECTED;
- for (i = 1; i < noProcs; i++) {
- info[i].finished = HELPER_STATE_UNINITIALIZED; /* Register that the process was not yet initialised; */
- }
- }
-#endif
- fprintf(stderr, "\nGoing over pending relations...\n");
-
- /* Make sure we're out of copy mode */
- pgsql_endCopy( rel_table );
-
- if (out_options->flat_node_cache_enabled) shutdown_node_persistent_cache();
-
- res_rels = pgsql_execPrepared(rel_table->sql_conn, "pending_rels", 0, NULL, PGRES_TUPLES_OK);
-
- fprintf(stderr, "\t%i relations are pending\n", PQntuples(res_rels));
-
- fprintf(stderr, "\nUsing %i helper-processes\n", noProcs);
- pid = 0;
-#ifdef HAVE_FORK
- for (p = 1; p < noProcs; p++) {
- pid=fork();
- if (pid==0) {
-#if HAVE_MMAP
- info[p].finished = HELPER_STATE_FORKED;
-#endif
- break;
- }
- if (pid==-1) {
-#if HAVE_MMAP
- info[p].finished = HELPER_STATE_FAILED;
- fprintf(stderr,"WARNING: Failed to fork helper processes %i. Trying to recover.\n", p);
-#else
- fprintf(stderr,"ERROR: Failed to fork helper processes. Can't recover! \n");
- exit_nicely();
-#endif
- }
- }
-#endif
- if ((pid == 0) && (noProcs > 1)) {
- if ((out_options->out->connect(out_options, 0) != 0) || (pgsql_connect(out_options) != 0)) {
-#if HAVE_MMAP
- info[p].finished = HELPER_STATE_FAILED;
-#endif
- exit_nicely();
- };
- } else {
- p = 0;
- }
-
- if (out_options->flat_node_cache_enabled) init_node_persistent_cache(out_options, 1); /* at this point we always want to be in append mode, to not delete and recreate the node cache file */
-
-#if HAVE_MMAP
- if (noProcs > 1) {
- info[p].finished = HELPER_STATE_CONNECTED;
- /* Syncronize all processes to make sure they have all run through the initialisation steps */
- int all_processes_initialised = 0;
- while (all_processes_initialised == 0) {
- all_processes_initialised = 1;
- for (i = 0; i < noProcs; i++) {
- if (info[i].finished < 0) {
- all_processes_initialised = 0;
- sleep(1);
- }
- }
- }
-
- /* As we process the pending ways in steps of noProcs,
- we need to make sure that all processes correctly forked
- and have connected to the db. Otherwise we need to readjust
- the step size of going through the pending ways array */
- int noProcsTmp = noProcs;
- int pTmp = p;
- for (i = 0; i < noProcs; i++) {
- if (info[i].finished == HELPER_STATE_FAILED) {
- noProcsTmp--;
- if (i < p) pTmp--;
- }
- }
- info[p].finished = HELPER_STATE_RUNNING;
- p = pTmp; /* reset the process number to account for failed processes */
-
- /* As we have potentially changed the process number assignment,
- we need to synchronize on all processes having performed the reassignment
- as otherwise multiple process might have the same number and overwrite
- the info fields incorrectly.
- */
- all_processes_initialised = 0;
- while (all_processes_initialised == 0) {
- all_processes_initialised = 1;
- for (i = 0; i < noProcs; i++) {
- if (info[i].finished == HELPER_STATE_CONNECTED) {
- /* Process is connected, but hasn't performed the re-assignment of p. */
- all_processes_initialised = 0;
- sleep(1);
- break;
- }
- }
- }
- noProcs = noProcsTmp;
- }
-#endif
-
- for (i = p; i < PQntuples(res_rels); i+= noProcs) {
- osmid_t id = strtoosmid(PQgetvalue(res_rels, i, 0), NULL, 10);
- struct keyval tags;
- struct member *members;
- int member_count;
-
- if (count++ %10 == 0) {
- time(&end);
-#if HAVE_MMAP
- if(info)
- {
- double rate = 0;
- int n, total = 0, finished = 0;
- struct progress_info f;
-
- f.start = start;
- f.end = end;
- f.count = count;
- f.finished = HELPER_STATE_RUNNING;
- info[p] = f;
- for(n = 0; n < noProcs; ++n)
- {
- f = info[n];
- total += f.count;
- finished += f.finished;
- if(f.end > f.start)
- rate += (double)f.count / (double)(f.end - f.start);
- }
- fprintf(stderr, "\rprocessing relation (%d) at %.2f/s (done %d of %d)", total, rate, finished, noProcs);
- }
- else
-#endif
- {
- fprintf(stderr, "\rprocessing relation (%d) at %.2f/s", count,
- end > start ? ((double)count / (double)(end - start)) : 0);
- }
- }
-
- initList(&tags);
- if( pgsql_rels_get(id, &members, &member_count, &tags) )
- continue;
-
- callback(id, members, member_count, &tags, exists);
- pgsql_rels_done( id );
-
- free(members);
- resetList(&tags);
- }
- time(&end);
-#if HAVE_MMAP
- if(info)
- {
- struct progress_info f;
- f.start = start;
- f.end = end;
- f.count = count;
- f.finished = 1;
- info[p] = f;
- }
-#endif
- fprintf(stderr, "\rProcess %i finished processing %i relations in %i sec\n", p, count, (int)(end - start));
-
- if ((pid == 0) && (noProcs > 1)) {
- pgsql_cleanup();
- out_options->out->close(0);
- if (out_options->flat_node_cache_enabled) shutdown_node_persistent_cache();
- exit(0);
- } else {
- for (p = 0; p < noProcs; p++) wait(NULL);
- fprintf(stderr, "\nAll child processes exited\n");
- }
-
-#if HAVE_MMAP
- munmap(info, sizeof(struct progress_info)*noProcs);
-#endif
- time(&end);
- if (end - start > 0)
- fprintf(stderr, "%i Pending relations took %ds at a rate of %.2f/s\n",PQntuples(res_rels), (int)(end - start), ((double)PQntuples(res_rels) / (double)(end - start)));
- PQclear(res_rels);
- fprintf(stderr, "\n");
-
-}
-
-static int pgsql_rel_changed(osmid_t osm_id)
-{
- char const *paramValues[1];
- char buffer[64];
- /* Make sure we're out of copy mode */
- pgsql_endCopy( rel_table );
-
- sprintf( buffer, "%" PRIdOSMID, osm_id );
- paramValues[0] = buffer;
- pgsql_execPrepared(rel_table->sql_conn, "rel_changed_mark", 1, paramValues, PGRES_COMMAND_OK );
- return 0;
-}
-
-static void pgsql_analyze(void)
-{
- int i;
-
- for (i=0; i<num_tables; i++) {
- PGconn *sql_conn = tables[i].sql_conn;
-
- if (tables[i].analyze) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].analyze);
- }
- }
-}
-
-static void pgsql_end(void)
-{
- int i;
-
- for (i=0; i<num_tables; i++) {
- PGconn *sql_conn = tables[i].sql_conn;
-
- /* Commit transaction */
- if (tables[i].stop && tables[i].transactionMode) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].stop);
- tables[i].transactionMode = 0;
- }
-
- }
-}
-
-/**
- * Helper to create SQL queries.
- *
- * The input string is mangled as follows:
- * %p replaced by the content of the "prefix" option
- * %i replaced by the content of the "tblsslim_data" option
- * %t replaced by the content of the "tblssslim_index" option
- * %m replaced by "UNLOGGED" if the "unlogged" option is set
- * other occurrences of the "%" char are treated normally.
- * any occurrence of { or } will be ignored (not copied to output string);
- * anything inside {} is only copied if it contained at least one of
- * %p, %i, %t, %m that was not NULL.
- *
- * So, the input string
- * Hello{ dear %i}!
- * will, if i is set to "John", translate to
- * Hello dear John!
- * but if i is unset, translate to
- * Hello!
- *
- * This is used for constructing SQL queries with proper tablespace settings.
- */
-static void set_prefix_and_tbls(const struct output_options *options, const char **string)
-{
- char buffer[1024];
- const char *source;
- char *dest;
- char *openbrace = NULL;
- int copied = 0;
-
- if (*string == NULL) return;
- source = *string;
- dest = buffer;
-
- while (*source) {
- if (*source == '{') {
- openbrace = dest;
- copied = 0;
- source++;
- continue;
- } else if (*source == '}') {
- if (!copied && openbrace) dest = openbrace;
- source++;
- continue;
- } else if (*source == '%') {
- if (*(source+1) == 'p') {
- if (options->prefix) {
- strcpy(dest, options->prefix);
- dest += strlen(options->prefix);
- copied = 1;
- }
- source+=2;
- continue;
- } else if (*(source+1) == 't') {
- if (options->tblsslim_data) {
- strcpy(dest, options->tblsslim_data);
- dest += strlen(options->tblsslim_data);
- copied = 1;
- }
- source+=2;
- continue;
- } else if (*(source+1) == 'i') {
- if (options->tblsslim_index) {
- strcpy(dest, options->tblsslim_index);
- dest += strlen(options->tblsslim_index);
- copied = 1;
- }
- source+=2;
- continue;
- } else if (*(source+1) == 'm') {
- if (options->unlogged) {
- strcpy(dest, "UNLOGGED");
- dest += 8;
- copied = 1;
- }
- source+=2;
- continue;
- }
- }
- *(dest++) = *(source++);
- }
- *dest = 0;
- *string = strdup(buffer);
-}
-
-static int build_indexes;
-
-
-static int pgsql_start(const struct output_options *options)
-{
- PGresult *res;
- int i;
- int dropcreate = !options->append;
- char * sql;
-
- scale = options->scale;
- Append = options->append;
-
- out_options = options;
-
- init_node_ram_cache( options->alloc_chunkwise | ALLOC_LOSSY, options->cache, scale);
- if (options->flat_node_cache_enabled) init_node_persistent_cache(options, options->append);
-
- fprintf(stderr, "Mid: pgsql, scale=%d cache=%d\n", scale, options->cache);
-
- /* We use a connection per table to enable the use of COPY */
- for (i=0; i<num_tables; i++) {
- PGconn *sql_conn;
-
- set_prefix_and_tbls(options, &(tables[i].name));
- set_prefix_and_tbls(options, &(tables[i].start));
- set_prefix_and_tbls(options, &(tables[i].create));
- set_prefix_and_tbls(options, &(tables[i].create_index));
- set_prefix_and_tbls(options, &(tables[i].prepare));
- set_prefix_and_tbls(options, &(tables[i].prepare_intarray));
- set_prefix_and_tbls(options, &(tables[i].copy));
- set_prefix_and_tbls(options, &(tables[i].analyze));
- set_prefix_and_tbls(options, &(tables[i].stop));
- set_prefix_and_tbls(options, &(tables[i].array_indexes));
-
- fprintf(stderr, "Setting up table: %s\n", tables[i].name);
- sql_conn = PQconnectdb(options->conninfo);
-
- /* Check to see that the backend connection was successfully made */
- if (PQstatus(sql_conn) != CONNECTION_OK) {
- fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(sql_conn));
- exit_nicely();
- }
- tables[i].sql_conn = sql_conn;
-
- /*
- * To allow for parallelisation, the second phase (iterate_ways), cannot be run
- * in an extended transaction and each update statement is its own transaction.
- * Therefore commit rate of postgresql is very important to ensure high speed.
- * If fsync is enabled to ensure safe transactions, the commit rate can be very low.
- * To compensate for this, one can set the postgresql parameter synchronous_commit
- * to off. This means an update statement returns to the client as success before the
- * transaction is saved to disk via fsync, which in return allows to bunch up multiple
- * transactions into a single fsync. This may result in some data loss in the case of a
- * database crash. However, as we don't currently have the ability to restart a full osm2pgsql
- * import session anyway, this is fine. Diff imports are also not effected, as the next
- * diff import would simply deal with all pending ways that were not previously finished.
- * This parameter does not effect safety from data corruption on the back-end.
- */
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;");
-
- /* Not really the right place for this test, but we need a live
- * connection that not used for anything else yet, and we'd like to
- * warn users *before* we start doing mountains of work */
- if (i == t_node)
- {
- res = PQexec(sql_conn, "select 1 from pg_opclass where opcname='gist__intbig_ops'" );
- if(PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1)
- {
- /* intarray is problematic now; causes at least postgres 8.4
- * to not use the index on nodes[]/parts[] which slows diff
- * updates to a crawl!
- * If someone find a way to fix this rather than bow out here,
- * please do.*/
-
- fprintf(stderr,
- "\n"
- "The target database has the intarray contrib module loaded.\n"
- "While required for earlier versions of osm2pgsql, intarray \n"
- "is now unnecessary and will interfere with osm2pgsql's array\n"
- "handling. Please use a database without intarray.\n\n");
- exit_nicely();
- }
- PQclear(res);
-
- if (options->append)
- {
- sql = malloc (2048);
- snprintf(sql, 2047, "SELECT id FROM %s LIMIT 1", tables[t_node].name);
- res = PQexec(sql_conn, sql );
- free(sql);
- sql = NULL;
- if(PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1)
- {
- int size = PQfsize(res, 0);
- if (size != sizeof(osmid_t))
- {
- fprintf(stderr,
- "\n"
- "The target database has been created with %dbit ID fields,\n"
- "but this version of osm2pgsql has been compiled to use %ldbit IDs.\n"
- "You cannot append data to this database with this program.\n"
- "Either re-create the database or use a matching osm2pgsql.\n\n",
- size * 8, sizeof(osmid_t) * 8);
- exit_nicely();
- }
- }
- PQclear(res);
- }
-
- if(!options->append)
- build_indexes = 1;
- }
- if (dropcreate) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS %s", tables[i].name);
- }
-
- if (tables[i].start) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].start);
- tables[i].transactionMode = 1;
- }
-
- if (dropcreate && tables[i].create) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].create);
- if (tables[i].create_index) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].create_index);
- }
- }
-
-
- if (tables[i].prepare) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare);
- }
-
- if (Append && tables[i].prepare_intarray) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare_intarray);
- }
-
- if (tables[i].copy) {
- pgsql_exec(sql_conn, PGRES_COPY_IN, "%s", tables[i].copy);
- tables[i].copyMode = 1;
- }
- }
-
- return 0;
-}
-
-static void pgsql_commit(void) {
- int i;
- for (i=0; i<num_tables; i++) {
- PGconn *sql_conn = tables[i].sql_conn;
- pgsql_endCopy(&tables[i]);
- if (tables[i].stop && tables[i].transactionMode) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].stop);
- tables[i].transactionMode = 0;
- }
- }
-}
-
-static void *pgsql_stop_one(void *arg)
-{
- time_t start, end;
-
- struct table_desc *table = arg;
- PGconn *sql_conn = table->sql_conn;
-
- fprintf(stderr, "Stopping table: %s\n", table->name);
- pgsql_endCopy(table);
- time(&start);
- if (!out_options->droptemp)
- {
- if (build_indexes && table->array_indexes) {
- char *buffer = (char *) malloc(strlen(table->array_indexes) + 99);
- /* we need to insert before the TABLESPACE setting, if any */
- char *insertpos = strstr(table->array_indexes, "TABLESPACE");
- if (!insertpos) insertpos = strchr(table->array_indexes, ';');
-
- /* automatically insert FASTUPDATE=OFF when creating,
- indexes for PostgreSQL 8.4 and higher
- see http://lists.openstreetmap.org/pipermail/dev/2011-January/021704.html */
- if (insertpos && PQserverVersion(sql_conn) >= 80400) {
- char old = *insertpos;
- fprintf(stderr, "Building index on table: %s (fastupdate=off)\n", table->name);
- *insertpos = 0; /* temporary null byte for following strcpy operation */
- strcpy(buffer, table->array_indexes);
- *insertpos = old; /* restore old content */
- strcat(buffer, " WITH (FASTUPDATE=OFF)");
- strcat(buffer, insertpos);
- } else {
- fprintf(stderr, "Building index on table: %s\n", table->name);
- strcpy(buffer, table->array_indexes);
- }
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", buffer);
- free(buffer);
- }
- }
- else
- {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "drop table %s", table->name);
- }
- PQfinish(sql_conn);
- table->sql_conn = NULL;
- time(&end);
- fprintf(stderr, "Stopped table: %s in %is\n", table->name, (int)(end - start));
- return NULL;
-}
-
-static void pgsql_stop(void)
-{
- int i;
-#ifdef HAVE_PTHREAD
- pthread_t threads[num_tables];
-#endif
-
- free_node_ram_cache();
- if (out_options->flat_node_cache_enabled) shutdown_node_persistent_cache();
-
-#ifdef HAVE_PTHREAD
- for (i=0; i<num_tables; i++) {
- int ret = pthread_create(&threads[i], NULL, pgsql_stop_one, &tables[i]);
- if (ret) {
- fprintf(stderr, "pthread_create() returned an error (%d)", ret);
- exit_nicely();
- }
- }
- for (i=0; i<num_tables; i++) {
- int ret = pthread_join(threads[i], NULL);
- if (ret) {
- fprintf(stderr, "pthread_join() returned an error (%d)", ret);
- exit_nicely();
- }
- }
-#else
- for (i=0; i<num_tables; i++)
- pgsql_stop_one(&tables[i]);
-#endif
-}
-
-struct middle_t mid_pgsql = {
- .start = pgsql_start,
- .stop = pgsql_stop,
- .cleanup = pgsql_cleanup,
- .analyze = pgsql_analyze,
- .end = pgsql_end,
- .commit = pgsql_commit,
-
- .nodes_set = middle_nodes_set,
-#if 0
- .nodes_get = middle_nodes_get,
-#endif
- .nodes_get_list = middle_nodes_get_list,
- .nodes_delete = middle_nodes_delete,
- .node_changed = pgsql_node_changed,
-
- .ways_set = pgsql_ways_set,
- .ways_get = pgsql_ways_get,
- .ways_get_list = pgsql_ways_get_list,
- .ways_done = pgsql_ways_done,
- .ways_delete = pgsql_ways_delete,
- .way_changed = pgsql_way_changed,
-
- .relations_set = pgsql_rels_set,
-#if 0
- .relations_get = pgsql_rels_get,
-#endif
- .relations_done = pgsql_rels_done,
- .relations_delete = pgsql_rels_delete,
- .relation_changed = pgsql_rel_changed,
-#if 0
- .iterate_nodes = pgsql_iterate_nodes,
-#endif
- .iterate_ways = pgsql_iterate_ways,
- .iterate_relations = pgsql_iterate_relations
-};
diff --git a/middle-pgsql.cpp b/middle-pgsql.cpp
new file mode 100644
index 0000000..2065edb
--- /dev/null
+++ b/middle-pgsql.cpp
@@ -0,0 +1,1453 @@
+/* Implements the mid-layer processing for osm2pgsql
+ * using several PostgreSQL tables
+ *
+ * This layer stores data read in from the planet.osm file
+ * and is then read by the backend processing code to
+ * emit the final geometry-enabled output formats
+*/
+
+#include "config.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <math.h>
+#include <time.h>
+#include <errno.h>
+
+#ifdef HAVE_PTHREAD
+#include <pthread.h>
+#endif
+
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+
+#ifdef HAVE_MMAP
+#include <sys/mman.h>
+#ifndef MAP_ANONYMOUS
+#ifdef MAP_ANON
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+#endif
+#endif
+
+#ifdef _WIN32
+using namespace std;
+#endif
+
+#ifdef _MSC_VER
+#define alloca _alloca
+#endif
+
+#include <libpq-fe.h>
+
+#include "osmtypes.hpp"
+#include "middle-pgsql.hpp"
+#include "output-pgsql.hpp"
+#include "options.hpp"
+#include "node-ram-cache.hpp"
+#include "node-persistent-cache.hpp"
+#include "pgsql.hpp"
+#include "util.hpp"
+
+#include <stdexcept>
+#include <boost/format.hpp>
+#include <boost/unordered_map.hpp>
+
+struct progress_info {
+ time_t start;
+ time_t end;
+ int count;
+ int finished;
+};
+
+enum table_id {
+ t_node, t_way, t_rel
+} ;
+
+middle_pgsql_t::table_desc::table_desc(const char *name_,
+ const char *start_,
+ const char *create_,
+ const char *create_index_,
+ const char *prepare_,
+ const char *prepare_intarray_,
+ const char *copy_,
+ const char *analyze_,
+ const char *stop_,
+ const char *array_indexes_)
+ : name(name_),
+ start(start_),
+ create(create_),
+ create_index(create_index_),
+ prepare(prepare_),
+ prepare_intarray(prepare_intarray_),
+ copy(copy_),
+ analyze(analyze_),
+ stop(stop_),
+ array_indexes(array_indexes_),
+ copyMode(0),
+ transactionMode(0),
+ sql_conn(NULL)
+{}
+
+#define HELPER_STATE_UNINITIALIZED -1
+#define HELPER_STATE_FORKED -2
+#define HELPER_STATE_RUNNING 0
+#define HELPER_STATE_FINISHED 1
+#define HELPER_STATE_CONNECTED 2
+#define HELPER_STATE_FAILED 3
+
+namespace {
+char *pgsql_store_nodes(const osmid_t *nds, const int& nd_count)
+{
+ static char *buffer;
+ static int buflen;
+
+ char *ptr;
+ int i, first;
+
+ if( buflen <= nd_count * 10 )
+ {
+ buflen = ((nd_count * 10) | 4095) + 1; // Round up to next page */
+ buffer = (char *)realloc( buffer, buflen );
+ }
+_restart:
+
+ ptr = buffer;
+ first = 1;
+ *ptr++ = '{';
+ for( i=0; i<nd_count; i++ )
+ {
+ if( !first ) *ptr++ = ',';
+ ptr += sprintf(ptr, "%" PRIdOSMID, nds[i] );
+
+ if( (ptr-buffer) > (buflen-20) ) // Almost overflowed? */
+ {
+ buflen <<= 1;
+ buffer = (char *)realloc( buffer, buflen );
+
+ goto _restart;
+ }
+ first = 0;
+ }
+
+ *ptr++ = '}';
+ *ptr++ = 0;
+
+ return buffer;
+}
+
+// Special escape routine for escaping strings in array constants: double quote, backslash,newline, tab*/
+inline char *escape_tag( char *ptr, const char *in, const int& escape )
+{
+ while( *in )
+ {
+ switch(*in)
+ {
+ case '"':
+ if( escape ) *ptr++ = '\\';
+ *ptr++ = '\\';
+ *ptr++ = '"';
+ break;
+ case '\\':
+ if( escape ) *ptr++ = '\\';
+ if( escape ) *ptr++ = '\\';
+ *ptr++ = '\\';
+ *ptr++ = '\\';
+ break;
+ case '\n':
+ if( escape ) *ptr++ = '\\';
+ *ptr++ = '\\';
+ *ptr++ = 'n';
+ break;
+ case '\r':
+ if( escape ) *ptr++ = '\\';
+ *ptr++ = '\\';
+ *ptr++ = 'r';
+ break;
+ case '\t':
+ if( escape ) *ptr++ = '\\';
+ *ptr++ = '\\';
+ *ptr++ = 't';
+ break;
+ default:
+ *ptr++ = *in;
+ break;
+ }
+ in++;
+ }
+ return ptr;
+}
+
+// escape means we return '\N' for copy mode, otherwise we return just NULL */
+const char *pgsql_store_tags(const struct keyval *tags, const int& escape)
+{
+ static char *buffer;
+ static int buflen;
+
+ char *ptr;
+ struct keyval *i;
+ int first;
+
+ int countlist = keyval::countList(tags);
+ if( countlist == 0 )
+ {
+ if( escape )
+ return "\\N";
+ else
+ return NULL;
+ }
+
+ if( buflen <= countlist * 24 ) // LE so 0 always matches */
+ {
+ buflen = ((countlist * 24) | 4095) + 1; // Round up to next page */
+ buffer = (char *)realloc( buffer, buflen );
+ }
+_restart:
+
+ ptr = buffer;
+ first = 1;
+ *ptr++ = '{';
+ // The lists are circular, exit when we reach the head again */
+ for( i=tags->next; i->key; i = i->next )
+ {
+ int maxlen = (strlen(i->key) + strlen(i->value)) * 4;
+ if( (ptr+maxlen-buffer) > (buflen-20) ) // Almost overflowed? */
+ {
+ buflen <<= 1;
+ buffer = (char *)realloc( buffer, buflen );
+
+ goto _restart;
+ }
+ if( !first ) *ptr++ = ',';
+ *ptr++ = '"';
+ ptr = escape_tag( ptr, i->key, escape );
+ *ptr++ = '"';
+ *ptr++ = ',';
+ *ptr++ = '"';
+ ptr = escape_tag( ptr, i->value, escape );
+ *ptr++ = '"';
+
+ first=0;
+ }
+
+ *ptr++ = '}';
+ *ptr++ = 0;
+
+ return buffer;
+}
+
+// Decodes a portion of an array literal from postgres */
+// Argument should point to beginning of literal, on return points to delimiter */
+inline const char *decode_upto( const char *src, char *dst )
+{
+ int quoted = (*src == '"');
+ if( quoted ) src++;
+
+ while( quoted ? (*src != '"') : (*src != ',' && *src != '}') )
+ {
+ if( *src == '\\' )
+ {
+ switch( src[1] )
+ {
+ case 'n': *dst++ = '\n'; break;
+ case 't': *dst++ = '\t'; break;
+ default: *dst++ = src[1]; break;
+ }
+ src+=2;
+ }
+ else
+ *dst++ = *src++;
+ }
+ if( quoted ) src++;
+ *dst = 0;
+ return src;
+}
+
+void pgsql_parse_tags( const char *string, struct keyval *tags )
+{
+ char key[1024];
+ char val[1024];
+
+ if( *string == '\0' )
+ return;
+
+ if( *string++ != '{' )
+ return;
+ while( *string != '}' )
+ {
+ string = decode_upto( string, key );
+ // String points to the comma */
+ string++;
+ string = decode_upto( string, val );
+ // String points to the comma or closing '}' */
+ keyval::addItem( tags, key, val, 0 );
+ if( *string == ',' )
+ string++;
+ }
+}
+
+// Parses an array of integers */
+void pgsql_parse_nodes(const char *src, osmid_t *nds, const int& nd_count )
+{
+ int count = 0;
+ const char *string = src;
+
+ if( *string++ != '{' )
+ return;
+ while( *string != '}' )
+ {
+ char *ptr;
+ nds[count] = strtoosmid( string, &ptr, 10 );
+ string = ptr;
+ if( *string == ',' )
+ string++;
+ count++;
+ }
+ if( count != nd_count )
+ {
+ fprintf( stderr, "parse_nodes problem: '%s' expected %d got %d\n", src, nd_count, count );
+ util::exit_nicely();
+ }
+}
+
+int pgsql_endCopy( struct middle_pgsql_t::table_desc *table)
+{
+ PGresult *res;
+ PGconn *sql_conn;
+ int stop;
+ // Terminate any pending COPY */
+ if (table->copyMode) {
+ sql_conn = table->sql_conn;
+ stop = PQputCopyEnd(sql_conn, NULL);
+ if (stop != 1) {
+ fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
+ util::exit_nicely();
+ }
+
+ res = PQgetResult(sql_conn);
+ if (PQresultStatus(res) != PGRES_COMMAND_OK) {
+ fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn));
+ PQclear(res);
+ util::exit_nicely();
+ }
+ PQclear(res);
+ table->copyMode = 0;
+ }
+ return 0;
+}
+} // anonymous namespace
+
+int middle_pgsql_t::local_nodes_set(const osmid_t& id, const double& lat, const double& lon, const struct keyval *tags)
+{
+ // Four params: id, lat, lon, tags */
+ const char *paramValues[4];
+ char *buffer;
+
+ if( node_table->copyMode )
+ {
+ const char *tag_buf = pgsql_store_tags(tags,1);
+ int length = strlen(tag_buf) + 64;
+ buffer = (char *)alloca( length );
+#ifdef FIXED_POINT
+ if( snprintf( buffer, length, "%" PRIdOSMID "\t%d\t%d\t%s\n", id, util::double_to_fix(lat, out_options->scale), util::double_to_fix(lon, out_options->scale), tag_buf ) > (length-10) )
+ { fprintf( stderr, "buffer overflow node id %" PRIdOSMID "\n", id); return 1; }
+#else
+ if( snprintf( buffer, length, "%" PRIdOSMID "\t%.10f\t%.10f\t%s\n", id, lat, lon, tag_buf ) > (length-10) )
+ { fprintf( stderr, "buffer overflow node id %" PRIdOSMID "\n", id); return 1; }
+#endif
+ pgsql_CopyData(__FUNCTION__, node_table->sql_conn, buffer);
+ return 0;
+ }
+ buffer = (char *)alloca(64);
+ char *ptr = buffer;
+ paramValues[0] = ptr;
+ ptr += sprintf( ptr, "%" PRIdOSMID, id ) + 1;
+ paramValues[1] = ptr;
+#ifdef FIXED_POINT
+ ptr += sprintf( ptr, "%d", util::double_to_fix(lat, out_options->scale) ) + 1;
+ paramValues[2] = ptr;
+ sprintf( ptr, "%d", util::double_to_fix(lon, out_options->scale) );
+#else
+ ptr += sprintf( ptr, "%.10f", lat ) + 1;
+ paramValues[2] = ptr;
+ sprintf( ptr, "%.10f", lon );
+#endif
+ paramValues[3] = pgsql_store_tags(tags,0);
+ pgsql_execPrepared(node_table->sql_conn, "insert_node", 4, (const char * const *)paramValues, PGRES_COMMAND_OK);
+ return 0;
+}
+
+// This should be made more efficient by using an IN(ARRAY[]) construct */
+int middle_pgsql_t::local_nodes_get_list(struct osmNode *nodes, const osmid_t *ndids, const int& nd_count) const
+{
+ char tmp[16];
+ char *tmp2;
+ int count, countDB, countPG, i,j;
+ char const *paramValues[1];
+
+ PGresult *res;
+ PGconn *sql_conn = node_table->sql_conn;
+
+ count = 0; countDB = 0;
+
+ tmp2 = (char *)malloc(sizeof(char)*nd_count*16);
+ if (tmp2 == NULL) return 0; //failed to allocate memory, return */
+
+ // create a list of ids in tmp2 to query the database */
+ sprintf(tmp2, "{");
+ for( i=0; i<nd_count; i++ ) {
+ // Check cache first */
+ if( cache->get( &nodes[i], ndids[i]) == 0 ) {
+ count++;
+ continue;
+ }
+ countDB++;
+ // Mark nodes as needing to be fetched from the DB */
+ nodes[i].lat = NAN;
+ nodes[i].lon = NAN;
+ snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", ndids[i]);
+ strncat(tmp2, tmp, sizeof(char)*(nd_count*16 - 2));
+ }
+ tmp2[strlen(tmp2) - 1] = '}'; // replace last , with } to complete list of ids*/
+
+ if (countDB == 0) {
+ free(tmp2);
+ return count; // All ids where in cache, so nothing more to do */
+ }
+
+ pgsql_endCopy(node_table);
+
+ paramValues[0] = tmp2;
+ res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues, PGRES_TUPLES_OK);
+ countPG = PQntuples(res);
+
+ //store the pg results in a hashmap and telling it how many we expect
+ boost::unordered_map<osmid_t, osmNode> pg_nodes(countPG);
+
+ for (i = 0; i < countPG; i++) {
+ osmid_t id = strtoosmid(PQgetvalue(res, i, 0), NULL, 10);
+ osmNode node;
+#ifdef FIXED_POINT
+ node.lat = util::fix_to_double(strtol(PQgetvalue(res, i, 1), NULL, 10), out_options->scale);
+ node.lon = util::fix_to_double(strtol(PQgetvalue(res, i, 2), NULL, 10), out_options->scale);
+#else
+ node.lat = strtod(PQgetvalue(res, i, 1), NULL);
+ node.lon = strtod(PQgetvalue(res, i, 2), NULL);
+#endif
+ pg_nodes.emplace(id, node);
+ }
+
+ //copy the nodes back out of the hashmap to the output
+ for(i = 0; i < nd_count; ++i){
+ //if we can find a matching id
+ boost::unordered_map<osmid_t, osmNode>::iterator found = pg_nodes.find(ndids[i]);
+ if(found != pg_nodes.end()) {
+ nodes[i] = boost::move(found->second); //this trashes whats in the hashmap but who cares
+ count++;
+ }
+ }
+
+ // If some of the nodes in the way don't exist, the returning list has holes.
+ // As the rest of the code expects a continuous list, it needs to be re-compacted */
+ if (count != nd_count) {
+ j = 0;
+ for (i = 0; i < nd_count; i++) {
+ if ( !std::isnan(nodes[i].lat)) {
+ nodes[j] = nodes[i];
+ j++;
+ }
+ }
+ }
+
+ PQclear(res);
+ free(tmp2);
+ return count;
+}
+
+void middle_pgsql_t::cleanup(void)
+{
+ int i;
+
+ for (i=0; i<num_tables; i++) {
+ if (tables[i].sql_conn) {
+ PQfinish(tables[i].sql_conn);
+ tables[i].sql_conn = NULL;
+ }
+ }
+}
+
+int middle_pgsql_t::nodes_set(osmid_t id, double lat, double lon, struct keyval *tags) {
+ cache->set( id, lat, lon, tags );
+
+ return (out_options->flat_node_cache_enabled) ? persistent_cache->set(id, lat, lon) : local_nodes_set(id, lat, lon, tags);
+}
+
+int middle_pgsql_t::nodes_get_list(struct osmNode *nodes, const osmid_t *ndids, int nd_count) const
+{
+ return (out_options->flat_node_cache_enabled) ? persistent_cache->get_list(nodes, ndids, nd_count) : local_nodes_get_list(nodes, ndids, nd_count);
+}
+
+int middle_pgsql_t::local_nodes_delete(osmid_t osm_id)
+{
+ char const *paramValues[1];
+ char buffer[64];
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( node_table );
+
+ sprintf( buffer, "%" PRIdOSMID, osm_id );
+ paramValues[0] = buffer;
+ pgsql_execPrepared(node_table->sql_conn, "delete_node", 1, paramValues, PGRES_COMMAND_OK );
+ return 0;
+}
+
+int middle_pgsql_t::nodes_delete(osmid_t osm_id)
+{
+ return ((out_options->flat_node_cache_enabled) ? persistent_cache->set(osm_id, NAN, NAN) : local_nodes_delete(osm_id));
+}
+
+int middle_pgsql_t::node_changed(osmid_t osm_id)
+{
+ char const *paramValues[1];
+ char buffer[64];
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( way_table );
+ pgsql_endCopy( rel_table );
+
+ sprintf( buffer, "%" PRIdOSMID, osm_id );
+ paramValues[0] = buffer;
+
+ //keep track of whatever ways and rels these nodes intersect
+ //TODO: dont need to stop the copy above since we are only reading?
+ PGresult* res = pgsql_execPrepared(way_table->sql_conn, "mark_ways_by_node", 1, paramValues, PGRES_TUPLES_OK );
+ for(int i = 0; i < PQntuples(res); ++i)
+ {
+ char *end;
+ osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10);
+ ways_pending_tracker->mark(marked);
+ }
+ PQclear(res);
+
+ //do the rels too
+ res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels_by_node", 1, paramValues, PGRES_TUPLES_OK );
+ for(int i = 0; i < PQntuples(res); ++i)
+ {
+ char *end;
+ osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10);
+ rels_pending_tracker->mark(marked);
+ }
+ PQclear(res);
+
+ return 0;
+}
+
+int middle_pgsql_t::ways_set(osmid_t way_id, osmid_t *nds, int nd_count, struct keyval *tags)
+{
+ // Three params: id, nodes, tags */
+ const char *paramValues[4];
+ char *buffer;
+
+ if( way_table->copyMode )
+ {
+ const char *tag_buf = pgsql_store_tags(tags,1);
+ char *node_buf = pgsql_store_nodes(nds, nd_count);
+ int length = strlen(tag_buf) + strlen(node_buf) + 64;
+ buffer = (char *)alloca(length);
+ if( snprintf( buffer, length, "%" PRIdOSMID "\t%s\t%s\n",
+ way_id, node_buf, tag_buf ) > (length-10) )
+ { fprintf( stderr, "buffer overflow way id %" PRIdOSMID "\n", way_id); return 1; }
+ pgsql_CopyData(__FUNCTION__, way_table->sql_conn, buffer);
+ return 0;
+ }
+ buffer = (char *)alloca(64);
+ char *ptr = buffer;
+ paramValues[0] = ptr;
+ sprintf(ptr, "%" PRIdOSMID, way_id);
+ paramValues[1] = pgsql_store_nodes(nds, nd_count);
+ paramValues[2] = pgsql_store_tags(tags,0);
+ pgsql_execPrepared(way_table->sql_conn, "insert_way", 3, (const char * const *)paramValues, PGRES_COMMAND_OK);
+ return 0;
+}
+
+// Caller is responsible for freeing nodesptr & keyval::resetList(tags) */
+int middle_pgsql_t::ways_get(osmid_t id, struct keyval *tags, struct osmNode **nodes_ptr, int *count_ptr) const
+{
+ PGresult *res;
+ char tmp[16];
+ char const *paramValues[1];
+ PGconn *sql_conn = way_table->sql_conn;
+ int num_nodes;
+ osmid_t *list;
+
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( way_table );
+
+ snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
+ paramValues[0] = tmp;
+
+ res = pgsql_execPrepared(sql_conn, "get_way", 1, paramValues, PGRES_TUPLES_OK);
+
+ if (PQntuples(res) != 1) {
+ PQclear(res);
+ return 1;
+ }
+
+ pgsql_parse_tags( PQgetvalue(res, 0, 1), tags );
+
+ num_nodes = strtol(PQgetvalue(res, 0, 2), NULL, 10);
+ list = (osmid_t *)alloca(sizeof(osmid_t)*num_nodes );
+ *nodes_ptr = (struct osmNode *)malloc(sizeof(struct osmNode) * num_nodes);
+ pgsql_parse_nodes( PQgetvalue(res, 0, 0), list, num_nodes);
+
+ *count_ptr = nodes_get_list(*nodes_ptr, list, num_nodes);
+ PQclear(res);
+ return 0;
+}
+
+int middle_pgsql_t::ways_get_list(const osmid_t *ids, int way_count, osmid_t *way_ids, struct keyval *tags, struct osmNode **nodes_ptr, int *count_ptr) const {
+
+ char tmp[16];
+ char *tmp2;
+ int count, countPG, i, j;
+ osmid_t *wayidspg;
+ char const *paramValues[1];
+ int num_nodes;
+ osmid_t *list;
+
+ PGresult *res;
+ PGconn *sql_conn = way_table->sql_conn;
+
+ if (way_count == 0) return 0;
+
+ tmp2 = (char *)malloc(sizeof(char)*way_count*16);
+ if (tmp2 == NULL) return 0; //failed to allocate memory, return */
+
+ // create a list of ids in tmp2 to query the database */
+ sprintf(tmp2, "{");
+ for( i=0; i<way_count; i++ ) {
+ snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", ids[i]);
+ strncat(tmp2,tmp, sizeof(char)*(way_count*16 - 2));
+ }
+ tmp2[strlen(tmp2) - 1] = '}'; // replace last , with } to complete list of ids*/
+
+ pgsql_endCopy(way_table);
+
+ paramValues[0] = tmp2;
+ res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues, PGRES_TUPLES_OK);
+ countPG = PQntuples(res);
+
+ wayidspg = (osmid_t *)malloc(sizeof(osmid_t)*countPG);
+
+ if (wayidspg == NULL) return 0; //failed to allocate memory, return */
+
+ for (i = 0; i < countPG; i++) {
+ wayidspg[i] = strtoosmid(PQgetvalue(res, i, 0), NULL, 10);
+ }
+
+
+ // Match the list of ways coming from postgres in a different order
+ // back to the list of ways given by the caller */
+ count = 0;
+ keyval::initList(&(tags[count]));
+ for (i = 0; i < way_count; i++) {
+ for (j = 0; j < countPG; j++) {
+ if (ids[i] == wayidspg[j]) {
+ way_ids[count] = ids[i];
+ pgsql_parse_tags( PQgetvalue(res, j, 2), &(tags[count]) );
+
+ num_nodes = strtol(PQgetvalue(res, j, 3), NULL, 10);
+ list = (osmid_t *)alloca(sizeof(osmid_t)*num_nodes );
+ nodes_ptr[count] = (struct osmNode *)malloc(sizeof(struct osmNode) * num_nodes);
+ pgsql_parse_nodes( PQgetvalue(res, j, 1), list, num_nodes);
+
+ count_ptr[count] = nodes_get_list(nodes_ptr[count], list, num_nodes);
+
+ count++;
+ keyval::initList(&(tags[count]));
+ }
+ }
+ }
+
+ PQclear(res);
+ free(tmp2);
+ free(wayidspg);
+
+ return count;
+}
+
+
+int middle_pgsql_t::ways_delete(osmid_t osm_id)
+{
+ char const *paramValues[1];
+ char buffer[64];
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( way_table );
+
+ sprintf( buffer, "%" PRIdOSMID, osm_id );
+ paramValues[0] = buffer;
+ pgsql_execPrepared(way_table->sql_conn, "delete_way", 1, paramValues, PGRES_COMMAND_OK );
+ return 0;
+}
+
+void middle_pgsql_t::iterate_ways(middle_t::pending_processor& pf)
+{
+
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( way_table );
+
+ // at this point we always want to be in append mode, to not delete and recreate the node cache file */
+ if (out_options->flat_node_cache_enabled) persistent_cache.reset(new node_persistent_cache(out_options,1,cache));
+
+ // enqueue the jobs
+ osmid_t id;
+ while(id_tracker::is_valid(id = ways_pending_tracker->pop_mark()))
+ {
+ pf.enqueue_ways(id);
+ }
+ // in case we had higher ones than the middle
+ pf.enqueue_ways(id_tracker::max());
+
+ //let the threads work on them
+ pf.process_ways();
+}
+
+int middle_pgsql_t::way_changed(osmid_t osm_id)
+{
+ char const *paramValues[1];
+ char buffer[64];
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( rel_table );
+
+ sprintf( buffer, "%" PRIdOSMID, osm_id );
+ paramValues[0] = buffer;
+
+ //keep track of whatever rels this way intersects
+ //TODO: dont need to stop the copy above since we are only reading?
+ PGresult* res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels_by_way", 1, paramValues, PGRES_TUPLES_OK );
+ for(int i = 0; i < PQntuples(res); ++i)
+ {
+ char *end;
+ osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10);
+ rels_pending_tracker->mark(marked);
+ }
+ PQclear(res);
+
+ return 0;
+}
+
+int middle_pgsql_t::relations_set(osmid_t id, struct member *members, int member_count, struct keyval *tags)
+{
+ // Params: id, way_off, rel_off, parts, members, tags */
+ const char *paramValues[6];
+ char *buffer;
+ int i;
+ struct keyval member_list;
+ char buf[64];
+
+ int node_count = 0, way_count = 0, rel_count = 0;
+
+ std::vector<osmid_t> all_parts(member_count), node_parts, way_parts, rel_parts;
+ node_parts.reserve(member_count);
+ way_parts.reserve(member_count);
+ rel_parts.reserve(member_count);
+
+ keyval::initList( &member_list );
+
+ for( i=0; i<member_count; i++ )
+ {
+ char tag = 0;
+ switch( members[i].type )
+ {
+ case OSMTYPE_NODE: node_count++; node_parts.push_back(members[i].id); tag = 'n'; break;
+ case OSMTYPE_WAY: way_count++; way_parts.push_back(members[i].id); tag = 'w'; break;
+ case OSMTYPE_RELATION: rel_count++; rel_parts.push_back(members[i].id); tag = 'r'; break;
+ default: fprintf( stderr, "Internal error: Unknown member type %d\n", members[i].type ); util::exit_nicely();
+ }
+ sprintf( buf, "%c%" PRIdOSMID, tag, members[i].id );
+ keyval::addItem( &member_list, buf, members[i].role, 0 );
+ }
+
+ int all_count = 0;
+ std::copy( node_parts.begin(), node_parts.end(), all_parts.begin() );
+ std::copy( way_parts.begin(), way_parts.end(), all_parts.begin() + node_count );
+ std::copy( rel_parts.begin(), rel_parts.end(), all_parts.begin() + node_count + way_count);
+ all_count = node_count + way_count + rel_count;
+
+ if( rel_table->copyMode )
+ {
+ char *tag_buf = strdup(pgsql_store_tags(tags,1));
+ const char *member_buf = pgsql_store_tags(&member_list,1);
+ char *parts_buf = pgsql_store_nodes(&all_parts[0], all_count);
+ int length = strlen(member_buf) + strlen(tag_buf) + strlen(parts_buf) + 64;
+ buffer = (char *)alloca(length);
+ if( snprintf( buffer, length, "%" PRIdOSMID "\t%d\t%d\t%s\t%s\t%s\n",
+ id, node_count, node_count+way_count, parts_buf, member_buf, tag_buf ) > (length-10) )
+ { fprintf( stderr, "buffer overflow relation id %" PRIdOSMID "\n", id); return 1; }
+ free(tag_buf);
+ keyval::resetList(&member_list);
+ pgsql_CopyData(__FUNCTION__, rel_table->sql_conn, buffer);
+ return 0;
+ }
+ buffer = (char *)alloca(64);
+ char *ptr = buffer;
+ paramValues[0] = ptr;
+ ptr += sprintf(ptr, "%" PRIdOSMID, id ) + 1;
+ paramValues[1] = ptr;
+ ptr += sprintf(ptr, "%d", node_count ) + 1;
+ paramValues[2] = ptr;
+ sprintf( ptr, "%d", node_count+way_count );
+ paramValues[3] = pgsql_store_nodes(&all_parts[0], all_count);
+ paramValues[4] = pgsql_store_tags(&member_list,0);
+ if( paramValues[4] )
+ paramValues[4] = strdup(paramValues[4]);
+ paramValues[5] = pgsql_store_tags(tags,0);
+ pgsql_execPrepared(rel_table->sql_conn, "insert_rel", 6, (const char * const *)paramValues, PGRES_COMMAND_OK);
+ if( paramValues[4] )
+ free((void *)paramValues[4]);
+ keyval::resetList(&member_list);
+ return 0;
+}
+
+// Caller is responsible for freeing members & keyval::resetList(tags) */
+int middle_pgsql_t::relations_get(osmid_t id, struct member **members, int *member_count, struct keyval *tags) const
+{
+ PGresult *res;
+ char tmp[16];
+ char const *paramValues[1];
+ PGconn *sql_conn = rel_table->sql_conn;
+ struct keyval member_temp;
+ char tag;
+ int num_members;
+ struct member *list;
+ int i=0;
+ struct keyval *item;
+
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( rel_table );
+
+ snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
+ paramValues[0] = tmp;
+
+ res = pgsql_execPrepared(sql_conn, "get_rel", 1, paramValues, PGRES_TUPLES_OK);
+ // Fields are: members, tags, member_count */
+
+ if (PQntuples(res) != 1) {
+ PQclear(res);
+ return 1;
+ }
+
+ pgsql_parse_tags( PQgetvalue(res, 0, 1), tags );
+ keyval::initList(&member_temp);
+ pgsql_parse_tags( PQgetvalue(res, 0, 0), &member_temp );
+
+ num_members = strtol(PQgetvalue(res, 0, 2), NULL, 10);
+ list = (struct member *)malloc( sizeof(struct member)*num_members );
+
+ while( (item = keyval::popItem(&member_temp)) )
+ {
+ if( i >= num_members )
+ {
+ fprintf(stderr, "Unexpected member_count reading relation %" PRIdOSMID "\n", id);
+ util::exit_nicely();
+ }
+ tag = item->key[0];
+ list[i].type = (tag == 'n')?OSMTYPE_NODE:(tag == 'w')?OSMTYPE_WAY:(tag == 'r')?OSMTYPE_RELATION:((OsmType)-1);
+ list[i].id = strtoosmid(item->key+1, NULL, 10 );
+ list[i].role = strdup( item->value );
+ keyval::freeItem(item);
+ i++;
+ }
+ *members = list;
+ *member_count = num_members;
+ PQclear(res);
+ return 0;
+}
+
+int middle_pgsql_t::relations_delete(osmid_t osm_id)
+{
+ char const *paramValues[1];
+ char buffer[64];
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( way_table );
+ pgsql_endCopy( rel_table );
+
+ sprintf( buffer, "%" PRIdOSMID, osm_id );
+ paramValues[0] = buffer;
+ pgsql_execPrepared(rel_table->sql_conn, "delete_rel", 1, paramValues, PGRES_COMMAND_OK );
+
+ //keep track of whatever ways this relation interesects
+ //TODO: dont need to stop the copy above since we are only reading?
+ PGresult* res = pgsql_execPrepared(way_table->sql_conn, "mark_ways_by_rel", 1, paramValues, PGRES_TUPLES_OK );
+ for(int i = 0; i < PQntuples(res); ++i)
+ {
+ char *end;
+ osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10);
+ ways_pending_tracker->mark(marked);
+ }
+ PQclear(res);
+ return 0;
+}
+
+void middle_pgsql_t::iterate_relations(pending_processor& pf)
+{
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( rel_table );
+
+ // at this point we always want to be in append mode, to not delete and recreate the node cache file */
+ if (out_options->flat_node_cache_enabled) persistent_cache.reset(new node_persistent_cache(out_options, 1, cache));
+
+ // enqueue the jobs
+ osmid_t id;
+ while(id_tracker::is_valid(id = rels_pending_tracker->pop_mark()))
+ {
+ pf.enqueue_relations(id);
+ }
+ // in case we had higher ones than the middle
+ pf.enqueue_relations(id_tracker::max());
+
+ //let the threads work on them
+ pf.process_relations();
+}
+
+int middle_pgsql_t::relation_changed(osmid_t osm_id)
+{
+ char const *paramValues[1];
+ char buffer[64];
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( rel_table );
+
+ sprintf( buffer, "%" PRIdOSMID, osm_id );
+ paramValues[0] = buffer;
+
+ //keep track of whatever ways and rels these nodes intersect
+ //TODO: dont need to stop the copy above since we are only reading?
+ //TODO: can we just mark the id without querying? the where clause seems intersect reltable.parts with the id
+ PGresult* res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels", 1, paramValues, PGRES_TUPLES_OK );
+ for(int i = 0; i < PQntuples(res); ++i)
+ {
+ char *end;
+ osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10);
+ rels_pending_tracker->mark(marked);
+ }
+ PQclear(res);
+ return 0;
+}
+
+std::vector<osmid_t> middle_pgsql_t::relations_using_way(osmid_t way_id) const
+{
+ char const *paramValues[1];
+ char buffer[64];
+ // Make sure we're out of copy mode */
+ pgsql_endCopy( rel_table );
+
+ sprintf(buffer, "%" PRIdOSMID, way_id);
+ paramValues[0] = buffer;
+
+ PGresult *result = pgsql_execPrepared(rel_table->sql_conn, "rels_using_way",
+ 1, paramValues, PGRES_TUPLES_OK );
+ const int ntuples = PQntuples(result);
+ std::vector<osmid_t> rel_ids(ntuples);
+ for (int i = 0; i < ntuples; ++i) {
+ rel_ids[i] = strtoosmid(PQgetvalue(result, i, 0), NULL, 10);
+ }
+ PQclear(result);
+
+ return rel_ids;
+}
+
+void middle_pgsql_t::analyze(void)
+{
+ int i;
+
+ for (i=0; i<num_tables; i++) {
+ PGconn *sql_conn = tables[i].sql_conn;
+
+ if (tables[i].analyze) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].analyze);
+ }
+ }
+}
+
+void middle_pgsql_t::end(void)
+{
+ int i;
+
+ for (i=0; i<num_tables; i++) {
+ PGconn *sql_conn = tables[i].sql_conn;
+
+ // Commit transaction */
+ if (tables[i].stop && tables[i].transactionMode) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].stop);
+ tables[i].transactionMode = 0;
+ }
+
+ }
+}
+
+/**
+ * Helper to create SQL queries.
+ *
+ * The input string is mangled as follows:
+ * %p replaced by the content of the "prefix" option
+ * %i replaced by the content of the "tblsslim_data" option
+ * %t replaced by the content of the "tblssslim_index" option
+ * %m replaced by "UNLOGGED" if the "unlogged" option is set
+ * other occurrences of the "%" char are treated normally.
+ * any occurrence of { or } will be ignored (not copied to output string);
+ * anything inside {} is only copied if it contained at least one of
+ * %p, %i, %t, %m that was not NULL.
+ *
+ * So, the input string
+ * Hello{ dear %i}!
+ * will, if i is set to "John", translate to
+ * Hello dear John!
+ * but if i is unset, translate to
+ * Hello!
+ *
+ * This is used for constructing SQL queries with proper tablespace settings.
+ */
+static void set_prefix_and_tbls(const struct options_t *options, const char **string)
+{
+ char buffer[1024];
+ const char *source;
+ char *dest;
+ char *openbrace = NULL;
+ int copied = 0;
+
+ if (*string == NULL) return;
+ source = *string;
+ dest = buffer;
+
+ while (*source) {
+ if (*source == '{') {
+ openbrace = dest;
+ copied = 0;
+ source++;
+ continue;
+ } else if (*source == '}') {
+ if (!copied && openbrace) dest = openbrace;
+ source++;
+ continue;
+ } else if (*source == '%') {
+ if (*(source+1) == 'p') {
+ if (!options->prefix.empty()) {
+ strcpy(dest, options->prefix.c_str());
+ dest += strlen(options->prefix.c_str());
+ copied = 1;
+ }
+ source+=2;
+ continue;
+ } else if (*(source+1) == 't') {
+ if (options->tblsslim_data) {
+ strcpy(dest, options->tblsslim_data->c_str());
+ dest += strlen(options->tblsslim_data->c_str());
+ copied = 1;
+ }
+ source+=2;
+ continue;
+ } else if (*(source+1) == 'i') {
+ if (options->tblsslim_index) {
+ strcpy(dest, options->tblsslim_index->c_str());
+ dest += strlen(options->tblsslim_index->c_str());
+ copied = 1;
+ }
+ source+=2;
+ continue;
+ } else if (*(source+1) == 'm') {
+ if (options->unlogged) {
+ strcpy(dest, "UNLOGGED");
+ dest += 8;
+ copied = 1;
+ }
+ source+=2;
+ continue;
+ }
+ }
+ *(dest++) = *(source++);
+ }
+ *dest = 0;
+ *string = strdup(buffer);
+}
+
+int middle_pgsql_t::connect(table_desc& table) {
+ PGconn *sql_conn;
+
+ set_prefix_and_tbls(out_options, &(table.name));
+ set_prefix_and_tbls(out_options, &(table.start));
+ set_prefix_and_tbls(out_options, &(table.create));
+ set_prefix_and_tbls(out_options, &(table.create_index));
+ set_prefix_and_tbls(out_options, &(table.prepare));
+ set_prefix_and_tbls(out_options, &(table.prepare_intarray));
+ set_prefix_and_tbls(out_options, &(table.copy));
+ set_prefix_and_tbls(out_options, &(table.analyze));
+ set_prefix_and_tbls(out_options, &(table.stop));
+ set_prefix_and_tbls(out_options, &(table.array_indexes));
+
+ fprintf(stderr, "Setting up table: %s\n", table.name);
+ sql_conn = PQconnectdb(out_options->conninfo.c_str());
+
+ // Check to see that the backend connection was successfully made */
+ if (PQstatus(sql_conn) != CONNECTION_OK) {
+ fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(sql_conn));
+ return 1;
+ }
+ table.sql_conn = sql_conn;
+ return 0;
+}
+
+int middle_pgsql_t::start(const options_t *out_options_)
+{
+ out_options = out_options_;
+ PGresult *res;
+ int i;
+ int dropcreate = !out_options->append;
+ char * sql;
+
+ ways_pending_tracker.reset(new id_tracker());
+ rels_pending_tracker.reset(new id_tracker());
+
+ Append = out_options->append;
+ // reset this on every start to avoid options from last run
+ // staying set for the second.
+ build_indexes = 0;
+
+ cache.reset(new node_ram_cache( out_options->alloc_chunkwise | ALLOC_LOSSY, out_options->cache, out_options->scale));
+ if (out_options->flat_node_cache_enabled) persistent_cache.reset(new node_persistent_cache(out_options, out_options->append, cache));
+
+ fprintf(stderr, "Mid: pgsql, scale=%d cache=%d\n", out_options->scale, out_options->cache);
+
+ // We use a connection per table to enable the use of COPY */
+ for (i=0; i<num_tables; i++) {
+ //bomb if you cant connect
+ if(connect(tables[i]))
+ util::exit_nicely();
+ PGconn* sql_conn = tables[i].sql_conn;
+
+ /*
+ * To allow for parallelisation, the second phase (iterate_ways), cannot be run
+ * in an extended transaction and each update statement is its own transaction.
+ * Therefore commit rate of postgresql is very important to ensure high speed.
+ * If fsync is enabled to ensure safe transactions, the commit rate can be very low.
+ * To compensate for this, one can set the postgresql parameter synchronous_commit
+ * to off. This means an update statement returns to the client as success before the
+ * transaction is saved to disk via fsync, which in return allows to bunch up multiple
+ * transactions into a single fsync. This may result in some data loss in the case of a
+ * database crash. However, as we don't currently have the ability to restart a full osm2pgsql
+ * import session anyway, this is fine. Diff imports are also not effected, as the next
+ * diff import would simply deal with all pending ways that were not previously finished.
+ * This parameter does not effect safety from data corruption on the back-end.
+ */
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;");
+
+ /* Not really the right place for this test, but we need a live
+ * connection that not used for anything else yet, and we'd like to
+ * warn users *before* we start doing mountains of work */
+ if (i == t_node)
+ {
+ res = PQexec(sql_conn, "select 1 from pg_opclass where opcname='gist__intbig_ops'" );
+ if(PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1)
+ {
+ /* intarray is problematic now; causes at least postgres 8.4
+ * to not use the index on nodes[]/parts[] which slows diff
+ * updates to a crawl!
+ * If someone find a way to fix this rather than bow out here,
+ * please do.*/
+
+ fprintf(stderr,
+ "\n"
+ "The target database has the intarray contrib module loaded.\n"
+ "While required for earlier versions of osm2pgsql, intarray \n"
+ "is now unnecessary and will interfere with osm2pgsql's array\n"
+ "handling. Please use a database without intarray.\n\n");
+ util::exit_nicely();
+ }
+ PQclear(res);
+
+ if (out_options->append)
+ {
+ sql = (char *)malloc (2048);
+ snprintf(sql, 2047, "SELECT id FROM %s LIMIT 1", tables[t_node].name);
+ res = PQexec(sql_conn, sql );
+ free(sql);
+ sql = NULL;
+ if(PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1)
+ {
+ int size = PQfsize(res, 0);
+ if (size != sizeof(osmid_t))
+ {
+ fprintf(stderr,
+ "\n"
+ "The target database has been created with %dbit ID fields,\n"
+ "but this version of osm2pgsql has been compiled to use %ldbit IDs.\n"
+ "You cannot append data to this database with this program.\n"
+ "Either re-create the database or use a matching osm2pgsql.\n\n",
+ size * 8, sizeof(osmid_t) * 8);
+ util::exit_nicely();
+ }
+ }
+ PQclear(res);
+ }
+
+ if(!out_options->append)
+ build_indexes = 1;
+ }
+
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET client_min_messages = WARNING");
+ if (dropcreate) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS %s", tables[i].name);
+ }
+
+ if (tables[i].start) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].start);
+ tables[i].transactionMode = 1;
+ }
+
+ if (dropcreate && tables[i].create) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].create);
+ if (tables[i].create_index) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].create_index);
+ }
+ }
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "RESET client_min_messages");
+
+ if (tables[i].prepare) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare);
+ }
+
+ if (Append && tables[i].prepare_intarray) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare_intarray);
+ }
+
+ if (tables[i].copy) {
+ pgsql_exec(sql_conn, PGRES_COPY_IN, "%s", tables[i].copy);
+ tables[i].copyMode = 1;
+ }
+ }
+
+ return 0;
+}
+
+void middle_pgsql_t::commit(void) {
+ int i;
+ for (i=0; i<num_tables; i++) {
+ PGconn *sql_conn = tables[i].sql_conn;
+ pgsql_endCopy(&tables[i]);
+ if (tables[i].stop && tables[i].transactionMode) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].stop);
+ tables[i].transactionMode = 0;
+ }
+ }
+}
+
+void *middle_pgsql_t::pgsql_stop_one(void *arg)
+{
+ time_t start, end;
+
+ struct table_desc *table = (struct table_desc *)arg;
+ PGconn *sql_conn = table->sql_conn;
+
+ fprintf(stderr, "Stopping table: %s\n", table->name);
+ pgsql_endCopy(table);
+ time(&start);
+ if (!out_options->droptemp)
+ {
+ if (build_indexes && table->array_indexes) {
+ char *buffer = (char *) malloc(strlen(table->array_indexes) + 99);
+ // we need to insert before the TABLESPACE setting, if any */
+ const char *insertpos = strstr(table->array_indexes, "TABLESPACE");
+ if (!insertpos) insertpos = strchr(table->array_indexes, ';');
+
+ /* automatically insert FASTUPDATE=OFF when creating,
+ indexes for PostgreSQL 8.4 and higher
+ see http://lists.openstreetmap.org/pipermail/dev/2011-January/021704.html */
+ if (insertpos && PQserverVersion(sql_conn) >= 80400) {
+ fprintf(stderr, "Building index on table: %s (fastupdate=off)\n", table->name);
+ size_t n_chars = insertpos - table->array_indexes;
+ strncpy(buffer, table->array_indexes, n_chars);
+ // strncpy doesn't necessarily null-terminate, so we need to add that
+ buffer[n_chars] = '\0';
+ strcat(buffer, " WITH (FASTUPDATE=OFF)");
+ strcat(buffer, insertpos);
+ } else {
+ fprintf(stderr, "Building index on table: %s\n", table->name);
+ strcpy(buffer, table->array_indexes);
+ }
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", buffer);
+ free(buffer);
+ }
+ }
+ else
+ {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "drop table %s", table->name);
+ }
+ PQfinish(sql_conn);
+ table->sql_conn = NULL;
+ time(&end);
+ fprintf(stderr, "Stopped table: %s in %is\n", table->name, (int)(end - start));
+ return NULL;
+}
+
+namespace {
+/* Using pthreads requires us to shoe-horn everything into various void*
+ * pointers. Improvement for the future: just use boost::thread. */
+struct pthread_thunk {
+ middle_pgsql_t *obj;
+ void *ptr;
+};
+
+extern "C" void *pthread_middle_pgsql_stop_one(void *arg) {
+ pthread_thunk *thunk = static_cast<pthread_thunk *>(arg);
+ return thunk->obj->pgsql_stop_one(thunk->ptr);
+};
+} // anonymous namespace
+
+void middle_pgsql_t::stop(void)
+{
+ int i;
+#ifdef HAVE_PTHREAD
+ pthread_t threads[num_tables];
+#endif
+
+ cache.reset();
+ if (out_options->flat_node_cache_enabled) persistent_cache.reset();
+
+#ifdef HAVE_PTHREAD
+ pthread_thunk thunks[num_tables];
+ for (i=0; i<num_tables; i++) {
+ thunks[i].obj = this;
+ thunks[i].ptr = &tables[i];
+ }
+
+ for (i=0; i<num_tables; i++) {
+ int ret = pthread_create(&threads[i], NULL, pthread_middle_pgsql_stop_one, &thunks[i]);
+ if (ret) {
+ fprintf(stderr, "pthread_create() returned an error (%d)", ret);
+ util::exit_nicely();
+ }
+ }
+
+ for (i=0; i<num_tables; i++) {
+ int ret = pthread_join(threads[i], NULL);
+ if (ret) {
+ fprintf(stderr, "pthread_join() returned an error (%d)", ret);
+ util::exit_nicely();
+ }
+ }
+#else
+ for (i=0; i<num_tables; i++)
+ pgsql_stop_one(&tables[i]);
+#endif
+}
+
+middle_pgsql_t::middle_pgsql_t()
+ : tables(), num_tables(0), node_table(NULL), way_table(NULL), rel_table(NULL),
+ Append(0), cache(), persistent_cache(), build_indexes(0)
+{
+ /*table = t_node,*/
+ tables.push_back(table_desc(
+ /*name*/ "%p_nodes",
+ /*start*/ "BEGIN;\n",
+#ifdef FIXED_POINT
+ /*create*/ "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat int4 not null, lon int4 not null, tags text[]) {TABLESPACE %t};\n",
+ /*create_index*/ NULL,
+ /*prepare*/ "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", int4, int4, text[]) AS INSERT INTO %p_nodes VALUES ($1,$2,$3,$4);\n"
+#else
+ /*create*/ "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat double precision not null, lon double precision not null, tags text[]) {TABLESPACE %t};\n",
+ /*create_index*/ NULL,
+ /*prepare*/ "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", double precision, double precision, text[]) AS INSERT INTO %p_nodes VALUES ($1,$2,$3,$4);\n"
+#endif
+ "PREPARE get_node (" POSTGRES_OSMID_TYPE ") AS SELECT lat,lon,tags FROM %p_nodes WHERE id = $1 LIMIT 1;\n"
+ "PREPARE get_node_list(" POSTGRES_OSMID_TYPE "[]) AS SELECT id, lat, lon FROM %p_nodes WHERE id = ANY($1::" POSTGRES_OSMID_TYPE "[]);\n"
+ "PREPARE delete_node (" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_nodes WHERE id = $1;\n",
+/*prepare_intarray*/ NULL,
+ /*copy*/ "COPY %p_nodes FROM STDIN;\n",
+ /*analyze*/ "ANALYZE %p_nodes;\n",
+ /*stop*/ "COMMIT;\n"
+ ));
+ tables.push_back(table_desc(
+ /*table t_way,*/
+ /*name*/ "%p_ways",
+ /*start*/ "BEGIN;\n",
+ /*create*/ "CREATE %m TABLE %p_ways (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, nodes " POSTGRES_OSMID_TYPE "[] not null, tags text[]) {TABLESPACE %t};\n",
+ /*create_index*/ NULL,
+ /*prepare*/ "PREPARE insert_way (" POSTGRES_OSMID_TYPE ", " POSTGRES_OSMID_TYPE "[], text[]) AS INSERT INTO %p_ways VALUES ($1,$2,$3);\n"
+ "PREPARE get_way (" POSTGRES_OSMID_TYPE ") AS SELECT nodes, tags, array_upper(nodes,1) FROM %p_ways WHERE id = $1;\n"
+ "PREPARE get_way_list (" POSTGRES_OSMID_TYPE "[]) AS SELECT id, nodes, tags, array_upper(nodes,1) FROM %p_ways WHERE id = ANY($1::" POSTGRES_OSMID_TYPE "[]);\n"
+ "PREPARE delete_way(" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_ways WHERE id = $1;\n",
+/*prepare_intarray*/
+ "PREPARE mark_ways_by_node(" POSTGRES_OSMID_TYPE ") AS select id from %p_ways WHERE nodes && ARRAY[$1];\n"
+ "PREPARE mark_ways_by_rel(" POSTGRES_OSMID_TYPE ") AS select id from %p_ways WHERE id IN (SELECT unnest(parts[way_off+1:rel_off]) FROM %p_rels WHERE id = $1);\n",
+
+ /*copy*/ "COPY %p_ways FROM STDIN;\n",
+ /*analyze*/ "ANALYZE %p_ways;\n",
+ /*stop*/ "COMMIT;\n",
+ /*array_indexes*/ "CREATE INDEX %p_ways_nodes ON %p_ways USING gin (nodes) {TABLESPACE %i};\n"
+ ));
+ tables.push_back(table_desc(
+ /*table = t_rel,*/
+ /*name*/ "%p_rels",
+ /*start*/ "BEGIN;\n",
+ /*create*/ "CREATE %m TABLE %p_rels(id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, way_off int2, rel_off int2, parts " POSTGRES_OSMID_TYPE "[], members text[], tags text[]) {TABLESPACE %t};\n",
+ /*create_index*/ NULL,
+ /*prepare*/ "PREPARE insert_rel (" POSTGRES_OSMID_TYPE ", int2, int2, " POSTGRES_OSMID_TYPE "[], text[], text[]) AS INSERT INTO %p_rels VALUES ($1,$2,$3,$4,$5,$6);\n"
+ "PREPARE get_rel (" POSTGRES_OSMID_TYPE ") AS SELECT members, tags, array_upper(members,1)/2 FROM %p_rels WHERE id = $1;\n"
+ "PREPARE delete_rel(" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_rels WHERE id = $1;\n",
+/*prepare_intarray*/
+ "PREPARE rels_using_way(" POSTGRES_OSMID_TYPE ") AS SELECT id FROM %p_rels WHERE parts && ARRAY[$1] AND parts[way_off+1:rel_off] && ARRAY[$1];\n"
+ "PREPARE mark_rels_by_node(" POSTGRES_OSMID_TYPE ") AS select id from %p_ways WHERE nodes && ARRAY[$1];\n"
+ "PREPARE mark_rels_by_way(" POSTGRES_OSMID_TYPE ") AS select id from %p_rels WHERE parts && ARRAY[$1] AND parts[way_off+1:rel_off] && ARRAY[$1];\n"
+ "PREPARE mark_rels(" POSTGRES_OSMID_TYPE ") AS select id from %p_rels WHERE parts && ARRAY[$1] AND parts[rel_off+1:array_length(parts,1)] && ARRAY[$1];\n",
+
+ /*copy*/ "COPY %p_rels FROM STDIN;\n",
+ /*analyze*/ "ANALYZE %p_rels;\n",
+ /*stop*/ "COMMIT;\n",
+ /*array_indexes*/ "CREATE INDEX %p_rels_parts ON %p_rels USING gin (parts) {TABLESPACE %i};\n"
+ ));
+
+ // set up the rest of the variables from the tables.
+ num_tables = tables.size();
+ assert(num_tables == 3);
+
+ node_table = &tables[0];
+ way_table = &tables[1];
+ rel_table = &tables[2];
+}
+
+middle_pgsql_t::~middle_pgsql_t() {
+}
+
+boost::shared_ptr<const middle_query_t> middle_pgsql_t::get_instance() const {
+ middle_pgsql_t* mid = new middle_pgsql_t();
+ mid->out_options = out_options;
+ mid->Append = out_options->append;
+
+ //NOTE: this is thread safe for use in pending async processing only because
+ //during that process they are only read from
+ mid->cache = cache;
+ mid->persistent_cache = persistent_cache;
+
+ // We use a connection per table to enable the use of COPY */
+ for(int i=0; i<num_tables; i++) {
+ //bomb if you cant connect
+ if(mid->connect(mid->tables[i]))
+ util::exit_nicely();
+ PGconn* sql_conn = mid->tables[i].sql_conn;
+
+ if (tables[i].prepare) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare);
+ }
+
+ if (Append && tables[i].prepare_intarray) {
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare_intarray);
+ }
+ }
+
+ return boost::shared_ptr<const middle_query_t>(mid);
+}
+
+size_t middle_pgsql_t::pending_count() const {
+ return ways_pending_tracker->size() + rels_pending_tracker->size();
+}
diff --git a/middle-pgsql.h b/middle-pgsql.h
deleted file mode 100644
index fc32b15..0000000
--- a/middle-pgsql.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* Implements the mid-layer processing for osm2pgsql
- * using several PostgreSQL tables
- *
- * This layer stores data read in from the planet.osm file
- * and is then read by the backend processing code to
- * emit the final geometry-enabled output formats
-*/
-
-#ifndef MIDDLE_PGSQL_H
-#define MIDDLE_PGSQL_H
-
-extern struct middle_t mid_pgsql;
-
-#endif
diff --git a/middle-pgsql.hpp b/middle-pgsql.hpp
new file mode 100644
index 0000000..377ab26
--- /dev/null
+++ b/middle-pgsql.hpp
@@ -0,0 +1,107 @@
+/* Implements the mid-layer processing for osm2pgsql
+ * using several PostgreSQL tables
+ *
+ * This layer stores data read in from the planet.osm file
+ * and is then read by the backend processing code to
+ * emit the final geometry-enabled output formats
+*/
+
+#ifndef MIDDLE_PGSQL_H
+#define MIDDLE_PGSQL_H
+
+#include "middle.hpp"
+#include "node-ram-cache.hpp"
+#include "node-persistent-cache.hpp"
+#include "id-tracker.hpp"
+#include <memory>
+#include <vector>
+#include <boost/shared_ptr.hpp>
+
+struct middle_pgsql_t : public slim_middle_t {
+ middle_pgsql_t();
+ virtual ~middle_pgsql_t();
+
+ int start(const options_t *out_options_);
+ void stop(void);
+ void cleanup(void);
+ void analyze(void);
+ void end(void);
+ void commit(void);
+
+ int nodes_set(osmid_t id, double lat, double lon, struct keyval *tags);
+ int nodes_get_list(struct osmNode *out, const osmid_t *nds, int nd_count) const;
+ int nodes_delete(osmid_t id);
+ int node_changed(osmid_t id);
+
+ int ways_set(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags);
+ int ways_get(osmid_t id, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) const;
+ int ways_get_list(const osmid_t *ids, int way_count, osmid_t *way_ids, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) const;
+
+ int ways_delete(osmid_t id);
+ int way_changed(osmid_t id);
+
+ int relations_get(osmid_t id, struct member **members, int *member_count, struct keyval *tags) const;
+ int relations_set(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+ int relations_delete(osmid_t id);
+ int relation_changed(osmid_t id);
+
+ void iterate_ways(middle_t::pending_processor& pf);
+ void iterate_relations(pending_processor& pf);
+
+ size_t pending_count() const;
+
+ std::vector<osmid_t> relations_using_way(osmid_t way_id) const;
+
+ void *pgsql_stop_one(void *arg);
+
+ struct table_desc {
+ table_desc(const char *name_ = NULL,
+ const char *start_ = NULL,
+ const char *create_ = NULL,
+ const char *create_index_ = NULL,
+ const char *prepare_ = NULL,
+ const char *prepare_intarray_ = NULL,
+ const char *copy_ = NULL,
+ const char *analyze_ = NULL,
+ const char *stop_ = NULL,
+ const char *array_indexes_ = NULL);
+
+ const char *name;
+ const char *start;
+ const char *create;
+ const char *create_index;
+ const char *prepare;
+ const char *prepare_intarray;
+ const char *copy;
+ const char *analyze;
+ const char *stop;
+ const char *array_indexes;
+
+ int copyMode; /* True if we are in copy mode */
+ int transactionMode; /* True if we are in an extended transaction */
+ struct pg_conn *sql_conn;
+ };
+
+ virtual boost::shared_ptr<const middle_query_t> get_instance() const;
+private:
+
+ int connect(table_desc& table);
+ int local_nodes_set(const osmid_t& id, const double& lat, const double& lon, const struct keyval *tags);
+ int local_nodes_get_list(struct osmNode *nodes, const osmid_t *ndids, const int& nd_count) const;
+ int local_nodes_delete(osmid_t osm_id);
+
+ std::vector<table_desc> tables;
+ int num_tables;
+ struct table_desc *node_table, *way_table, *rel_table;
+
+ int Append;
+
+ boost::shared_ptr<node_ram_cache> cache;
+ boost::shared_ptr<node_persistent_cache> persistent_cache;
+
+ boost::shared_ptr<id_tracker> ways_pending_tracker, rels_pending_tracker;
+
+ int build_indexes;
+};
+
+#endif
diff --git a/middle-ram.c b/middle-ram.c
deleted file mode 100644
index 27d50c2..0000000
--- a/middle-ram.c
+++ /dev/null
@@ -1,387 +0,0 @@
-/* Implements the mid-layer processing for osm2pgsql
- * using several arrays in RAM. This is fastest if you
- * have sufficient RAM+Swap.
- *
- * This layer stores data read in from the planet.osm file
- * and is then read by the backend processing code to
- * emit the final geometry-enabled output formats
-*/
-
-#include <stdio.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <libpq-fe.h>
-
-#include "osmtypes.h"
-#include "middle.h"
-#include "middle-ram.h"
-#include "node-ram-cache.h"
-
-#include "output-pgsql.h"
-
-/* Store +-20,000km Mercator co-ordinates as fixed point 32bit number with maximum precision */
-/* Scale is chosen such that 40,000 * SCALE < 2^32 */
-#define FIXED_POINT
-
-struct ramWay {
- struct keyval *tags;
- osmid_t *ndids;
- int pending;
-};
-
-struct ramRel {
- struct keyval *tags;
- struct member *members;
- int member_count;
-};
-
-/* Object storage now uses 2 levels of storage arrays.
- *
- * - Low level storage of 2^16 (~65k) objects in an indexed array
- * These are allocated dynamically when we need to first store data with
- * an ID in this block
- *
- * - Fixed array of 2^(32 - 16) = 65k pointers to the dynamically allocated arrays.
- *
- * This allows memory usage to be efficient and scale dynamically without needing to
- * hard code maximum IDs. We now support an ID range of -2^31 to +2^31.
- * The negative IDs often occur in non-uploaded JOSM data or other data import scripts.
- *
- */
-
-#define BLOCK_SHIFT 10
-#define PER_BLOCK (1 << BLOCK_SHIFT)
-#define NUM_BLOCKS (1 << (32 - BLOCK_SHIFT))
-
-static struct ramWay *ways[NUM_BLOCKS];
-static struct ramRel *rels[NUM_BLOCKS];
-
-static int way_blocks;
-
-static int way_out_count;
-static int rel_out_count;
-
-static osmid_t id2block(osmid_t id)
-{
- /* + NUM_BLOCKS/2 allows for negative IDs */
- return (id >> BLOCK_SHIFT) + NUM_BLOCKS/2;
-}
-
-static osmid_t id2offset(osmid_t id)
-{
- return id & (PER_BLOCK-1);
-}
-
-static int block2id(int block, int offset)
-{
- return ((block - NUM_BLOCKS/2) << BLOCK_SHIFT) + offset;
-}
-
-#define UNUSED __attribute__ ((unused))
-
-static int ram_ways_set(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags, int pending)
-{
- int block = id2block(id);
- int offset = id2offset(id);
- struct keyval *p;
-
- if (!ways[block]) {
- ways[block] = calloc(PER_BLOCK, sizeof(struct ramWay));
- if (!ways[block]) {
- fprintf(stderr, "Error allocating ways\n");
- exit_nicely();
- }
- way_blocks++;
- }
-
- if (ways[block][offset].ndids) {
- free(ways[block][offset].ndids);
- ways[block][offset].ndids = NULL;
- }
-
- /* Copy into length prefixed array */
- ways[block][offset].ndids = malloc( (nd_count+1)*sizeof(osmid_t) );
- memcpy( ways[block][offset].ndids+1, nds, nd_count*sizeof(osmid_t) );
- ways[block][offset].ndids[0] = nd_count;
- ways[block][offset].pending = pending;
-
- if (!ways[block][offset].tags) {
- p = malloc(sizeof(struct keyval));
- if (p) {
- initList(p);
- ways[block][offset].tags = p;
- } else {
- fprintf(stderr, "%s malloc failed\n", __FUNCTION__);
- exit_nicely();
- }
- } else
- resetList(ways[block][offset].tags);
-
- cloneList(ways[block][offset].tags, tags);
-
- return 0;
-}
-
-static int ram_relations_set(osmid_t id, struct member *members, int member_count, struct keyval *tags)
-{
- struct keyval *p;
- struct member *ptr;
- int block = id2block(id);
- int offset = id2offset(id);
- if (!rels[block]) {
- rels[block] = calloc(PER_BLOCK, sizeof(struct ramRel));
- if (!rels[block]) {
- fprintf(stderr, "Error allocating rels\n");
- exit_nicely();
- }
- }
-
- if (!rels[block][offset].tags) {
- p = malloc(sizeof(struct keyval));
- if (p) {
- initList(p);
- rels[block][offset].tags = p;
- } else {
- fprintf(stderr, "%s malloc failed\n", __FUNCTION__);
- exit_nicely();
- }
- } else
- resetList(rels[block][offset].tags);
-
- cloneList(rels[block][offset].tags, tags);
-
- if (!rels[block][offset].members)
- free( rels[block][offset].members );
-
- ptr = malloc(sizeof(struct member) * member_count);
- if (ptr) {
- memcpy( ptr, members, sizeof(struct member) * member_count );
- rels[block][offset].member_count = member_count;
- rels[block][offset].members = ptr;
- } else {
- fprintf(stderr, "%s malloc failed\n", __FUNCTION__);
- exit_nicely();
- }
-
- return 0;
-}
-
-static int ram_nodes_get_list(struct osmNode *nodes, osmid_t *ndids, int nd_count)
-{
- int i, count;
-
- count = 0;
- for( i=0; i<nd_count; i++ )
- {
- if (ram_cache_nodes_get(&nodes[count], ndids[i]))
- continue;
-
- count++;
- }
- return count;
-}
-
-static void ram_iterate_relations(int (*callback)(osmid_t id, struct member *members, int member_count, struct keyval *tags, int))
-{
- int block, offset;
-
- fprintf(stderr, "\n");
- for(block=NUM_BLOCKS-1; block>=0; block--) {
- if (!rels[block])
- continue;
-
- for (offset=0; offset < PER_BLOCK; offset++) {
- if (rels[block][offset].members) {
- osmid_t id = block2id(block, offset);
- rel_out_count++;
- if (rel_out_count % 10 == 0)
- fprintf(stderr, "\rWriting relation (%u)", rel_out_count);
-
- callback(id, rels[block][offset].members, rels[block][offset].member_count, rels[block][offset].tags, 0);
- }
- free(rels[block][offset].members);
- rels[block][offset].members = NULL;
- resetList(rels[block][offset].tags);
- free(rels[block][offset].tags);
- rels[block][offset].tags=NULL;
- }
- free(rels[block]);
- rels[block] = NULL;
- }
-
- fprintf(stderr, "\rWriting relation (%u)\n", rel_out_count);
-}
-
-static void ram_iterate_ways(int (*callback)(osmid_t id, struct keyval *tags, struct osmNode *nodes, int count, int exists))
-{
- int block, offset, ndCount = 0;
- struct osmNode *nodes;
-
- fprintf(stderr, "\n");
- for(block=NUM_BLOCKS-1; block>=0; block--) {
- if (!ways[block])
- continue;
-
- for (offset=0; offset < PER_BLOCK; offset++) {
- if (ways[block][offset].ndids) {
- way_out_count++;
- if (way_out_count % 1000 == 0)
- fprintf(stderr, "\rWriting way (%uk)", way_out_count/1000);
-
- if (ways[block][offset].pending) {
- /* First element contains number of nodes */
- nodes = malloc( sizeof(struct osmNode) * ways[block][offset].ndids[0]);
- ndCount = ram_nodes_get_list(nodes, ways[block][offset].ndids+1, ways[block][offset].ndids[0]);
-
- if (nodes) {
- osmid_t id = block2id(block, offset);
- callback(id, ways[block][offset].tags, nodes, ndCount, 0);
- free(nodes);
- }
-
- ways[block][offset].pending = 0;
- }
-
- if (ways[block][offset].tags) {
- resetList(ways[block][offset].tags);
- free(ways[block][offset].tags);
- ways[block][offset].tags = NULL;
- }
- if (ways[block][offset].ndids) {
- free(ways[block][offset].ndids);
- ways[block][offset].ndids = NULL;
- }
- }
- }
- }
- fprintf(stderr, "\rWriting way (%uk)\n", way_out_count/1000);
-}
-
-/* Caller must free nodes_ptr and resetList(tags_ptr) */
-static int ram_ways_get(osmid_t id, struct keyval *tags_ptr, struct osmNode **nodes_ptr, int *count_ptr)
-{
- int block = id2block(id), offset = id2offset(id), ndCount = 0;
- struct osmNode *nodes;
-
- if (!ways[block])
- return 1;
-
- if (ways[block][offset].ndids) {
- /* First element contains number of nodes */
- nodes = malloc( sizeof(struct osmNode) * ways[block][offset].ndids[0]);
- ndCount = ram_nodes_get_list(nodes, ways[block][offset].ndids+1, ways[block][offset].ndids[0]);
-
- if (ndCount) {
- cloneList( tags_ptr, ways[block][offset].tags );
- *nodes_ptr = nodes;
- *count_ptr = ndCount;
- return 0;
- }
- free(nodes);
- }
- return 1;
-}
-
-static int ram_ways_get_list(osmid_t *ids, int way_count, osmid_t **way_ids, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) {
- int count = 0;
- int i;
- *way_ids = malloc( sizeof(osmid_t) * (way_count + 1));
- initList(&(tag_ptr[count]));
- for (i = 0; i < way_count; i++) {
-
- if (ram_ways_get(ids[i], &(tag_ptr[count]), &(node_ptr[count]), &(count_ptr[count])) == 0) {
- (*way_ids)[count] = ids[i];
- count++;
- initList(&(tag_ptr[count]));
- }
- }
- return count;
-}
-
-/* Marks the way so that iterate ways skips it */
-static int ram_ways_done(osmid_t id)
-{
- int block = id2block(id), offset = id2offset(id);
-
- if (!ways[block])
- return 1;
-
- ways[block][offset].pending = 0;
- return 0;
-}
-
-static void ram_analyze(void)
-{
- /* No need */
-}
-
-static void ram_end(void)
-{
- /* No need */
-}
-
-static int ram_start(const struct output_options *options)
-{
- /* latlong has a range of +-180, mercator +-20000
- The fixed poing scaling needs adjusting accordingly to
- be stored accurately in an int */
- scale = options->scale;
-
- init_node_ram_cache( options->alloc_chunkwise, options->cache, scale);
-
- fprintf( stderr, "Mid: Ram, scale=%d\n", scale );
-
- return 0;
-}
-
-static void ram_stop(void)
-{
- int i, j;
- free_node_ram_cache();
-
- for (i=0; i<NUM_BLOCKS; i++) {
- if (ways[i]) {
- for (j=0; j<PER_BLOCK; j++) {
- if (ways[i][j].tags) {
- resetList(ways[i][j].tags);
- free(ways[i][j].tags);
- }
- if (ways[i][j].ndids)
- free(ways[i][j].ndids);
- }
- free(ways[i]);
- ways[i] = NULL;
- }
- }
-}
-
-static void ram_commit(void) {
-}
-
-struct middle_t mid_ram = {
- .start = ram_start,
- .stop = ram_stop,
- .end = ram_end,
- .cleanup = ram_stop,
- .analyze = ram_analyze,
- .commit = ram_commit,
- .nodes_set = ram_cache_nodes_set,
-#if 0
- .nodes_get = ram_nodes_get,
-#endif
- .nodes_get_list = ram_nodes_get_list,
- .ways_set = ram_ways_set,
- .ways_get = ram_ways_get,
- .ways_get_list = ram_ways_get_list,
- .ways_done = ram_ways_done,
-
- .relations_set = ram_relations_set,
-#if 0
- .iterate_nodes = ram_iterate_nodes,
-#endif
- .iterate_ways = ram_iterate_ways,
- .iterate_relations = ram_iterate_relations
-};
-
diff --git a/middle-ram.cpp b/middle-ram.cpp
new file mode 100644
index 0000000..89a544f
--- /dev/null
+++ b/middle-ram.cpp
@@ -0,0 +1,364 @@
+/* Implements the mid-layer processing for osm2pgsql
+ * using several arrays in RAM. This is fastest if you
+ * have sufficient RAM+Swap.
+ *
+ * This layer stores data read in from the planet.osm file
+ * and is then read by the backend processing code to
+ * emit the final geometry-enabled output formats
+*/
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <libpq-fe.h>
+
+#include <stdexcept>
+
+#include "osmtypes.hpp"
+#include "middle-ram.hpp"
+#include "node-ram-cache.hpp"
+#include "output-pgsql.hpp"
+#include "options.hpp"
+#include "util.hpp"
+
+/* Store +-20,000km Mercator co-ordinates as fixed point 32bit number with maximum precision */
+/* Scale is chosen such that 40,000 * SCALE < 2^32 */
+#define FIXED_POINT
+
+/* Object storage now uses 2 levels of storage arrays.
+ *
+ * - Low level storage of 2^16 (~65k) objects in an indexed array
+ * These are allocated dynamically when we need to first store data with
+ * an ID in this block
+ *
+ * - Fixed array of 2^(32 - 16) = 65k pointers to the dynamically allocated arrays.
+ *
+ * This allows memory usage to be efficient and scale dynamically without needing to
+ * hard code maximum IDs. We now support an ID range of -2^31 to +2^31.
+ * The negative IDs often occur in non-uploaded JOSM data or other data import scripts.
+ *
+ */
+
+#define BLOCK_SHIFT 10
+#define PER_BLOCK (1 << BLOCK_SHIFT)
+#define NUM_BLOCKS (1 << (32 - BLOCK_SHIFT))
+
+static osmid_t id2block(osmid_t id)
+{
+ /* + NUM_BLOCKS/2 allows for negative IDs */
+ return (id >> BLOCK_SHIFT) + NUM_BLOCKS/2;
+}
+
+static osmid_t id2offset(osmid_t id)
+{
+ return id & (PER_BLOCK-1);
+}
+
+static int block2id(int block, int offset)
+{
+ return ((block - NUM_BLOCKS/2) << BLOCK_SHIFT) + offset;
+}
+
+int middle_ram_t::nodes_set(osmid_t id, double lat, double lon, struct keyval *tags) {
+ return cache->set(id, lat, lon, tags);
+}
+
+int middle_ram_t::ways_set(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags)
+{
+ int block = id2block(id);
+ int offset = id2offset(id);
+
+ if (!ways[block]) {
+ ways[block] = (struct ramWay *)calloc(PER_BLOCK, sizeof(struct ramWay));
+ if (!ways[block]) {
+ fprintf(stderr, "Error allocating ways\n");
+ util::exit_nicely();
+ }
+ way_blocks++;
+ }
+
+ free(ways[block][offset].ndids);
+ ways[block][offset].ndids = NULL;
+
+ /* Copy into length prefixed array */
+ ways[block][offset].ndids = (osmid_t *)malloc( (nd_count+1)*sizeof(osmid_t) );
+ memcpy( ways[block][offset].ndids+1, nds, nd_count*sizeof(osmid_t) );
+ ways[block][offset].ndids[0] = nd_count;
+
+ if (!ways[block][offset].tags) {
+ ways[block][offset].tags = new keyval();
+ keyval::initList(ways[block][offset].tags);
+ } else
+ keyval::resetList(ways[block][offset].tags);
+
+ keyval::cloneList(ways[block][offset].tags, tags);
+
+ return 0;
+}
+
+int middle_ram_t::relations_set(osmid_t id, struct member *members, int member_count, struct keyval *tags)
+{
+ struct member *ptr;
+ int block = id2block(id);
+ int offset = id2offset(id);
+ if (!rels[block]) {
+ rels[block] = (struct ramRel *)calloc(PER_BLOCK, sizeof(struct ramRel));
+ if (!rels[block]) {
+ fprintf(stderr, "Error allocating rels\n");
+ util::exit_nicely();
+ }
+ }
+
+ if (!rels[block][offset].tags) {
+ rels[block][offset].tags = new keyval();
+ keyval::initList(rels[block][offset].tags);
+ } else
+ keyval::resetList(rels[block][offset].tags);
+
+ keyval::cloneList(rels[block][offset].tags, tags);
+
+ free( rels[block][offset].members );
+ rels[block][offset].members = NULL;
+
+ ptr = (struct member *)malloc(sizeof(struct member) * member_count);
+ if (ptr) {
+ memcpy( ptr, members, sizeof(struct member) * member_count );
+ rels[block][offset].member_count = member_count;
+ rels[block][offset].members = ptr;
+ } else {
+ fprintf(stderr, "%s malloc failed\n", __FUNCTION__);
+ util::exit_nicely();
+ }
+
+ return 0;
+}
+
+int middle_ram_t::nodes_get_list(struct osmNode *nodes, const osmid_t *ndids, int nd_count) const
+{
+ int i, count;
+
+ count = 0;
+ for( i=0; i<nd_count; i++ )
+ {
+ if (cache->get(&nodes[count], ndids[i]))
+ continue;
+
+ count++;
+ }
+ return count;
+}
+
+void middle_ram_t::iterate_relations(pending_processor& pf)
+{
+ //TODO: just dont do anything
+
+ //let the outputs enqueue everything they have the non slim middle
+ //has nothing of its own to enqueue as it doesnt have pending anything
+ pf.enqueue_relations(id_tracker::max());
+
+ //let the threads process the relations
+ pf.process_relations();
+}
+
+size_t middle_ram_t::pending_count() const {
+ return 0;
+}
+
+void middle_ram_t::iterate_ways(middle_t::pending_processor& pf)
+{
+ //let the outputs enqueue everything they have the non slim middle
+ //has nothing of its own to enqueue as it doesnt have pending anything
+ pf.enqueue_ways(id_tracker::max());
+
+ //let the threads process the ways
+ pf.process_ways();
+}
+
+void middle_ram_t::release_relations()
+{
+ int block, offset;
+
+ for(block=NUM_BLOCKS-1; block>=0; block--) {
+ if (!rels[block])
+ continue;
+
+ for (offset=0; offset < PER_BLOCK; offset++) {
+ if (rels[block][offset].members) {
+ free(rels[block][offset].members);
+ rels[block][offset].members = NULL;
+ keyval::resetList(rels[block][offset].tags);
+ delete rels[block][offset].tags;
+ rels[block][offset].tags=NULL;
+ }
+ }
+ free(rels[block]);
+ rels[block] = NULL;
+ }
+}
+
+void middle_ram_t::release_ways()
+{
+ int i, j = 0;
+
+ for (i=0; i<NUM_BLOCKS; i++) {
+ if (ways[i]) {
+ for (j=0; j<PER_BLOCK; j++) {
+ if (ways[i][j].tags) {
+ keyval::resetList(ways[i][j].tags);
+ delete ways[i][j].tags;
+ }
+ if (ways[i][j].ndids)
+ free(ways[i][j].ndids);
+ }
+ free(ways[i]);
+ ways[i] = NULL;
+ }
+ }
+}
+
+/* Caller must free nodes_ptr and keyval::resetList(tags_ptr) */
+int middle_ram_t::ways_get(osmid_t id, struct keyval *tags_ptr, struct osmNode **nodes_ptr, int *count_ptr) const
+{
+ int block = id2block(id), offset = id2offset(id);
+ struct osmNode *nodes;
+
+ if (simulate_ways_deleted)
+ return 1;
+
+ if (!ways[block])
+ return 1;
+
+ if (ways[block][offset].ndids) {
+ /* First element contains number of nodes */
+ nodes = (struct osmNode *)malloc( sizeof(struct osmNode) * ways[block][offset].ndids[0]);
+ int ndCount = nodes_get_list(nodes, ways[block][offset].ndids+1, ways[block][offset].ndids[0]);
+
+ if (ndCount) {
+ keyval::cloneList( tags_ptr, ways[block][offset].tags );
+ *nodes_ptr = nodes;
+ *count_ptr = ndCount;
+ return 0;
+ }
+ free(nodes);
+ }
+ return 1;
+}
+
+int middle_ram_t::ways_get_list(const osmid_t *ids, int way_count, osmid_t *way_ids, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) const {
+ int count = 0;
+ int i;
+
+ keyval::initList(&(tag_ptr[count]));
+ for (i = 0; i < way_count; i++) {
+
+ if (ways_get(ids[i], &(tag_ptr[count]), &(node_ptr[count]), &(count_ptr[count])) == 0) {
+ way_ids[count] = ids[i];
+ count++;
+ keyval::initList(&(tag_ptr[count]));
+ }
+ }
+ return count;
+}
+
+/* Caller must free members_ptr and keyval::resetList(tags_ptr).
+ * Note that the members in members_ptr are copied, but the roles
+ * within the members are not, and should not be freed.
+ */
+int middle_ram_t::relations_get(osmid_t id, struct member **members_ptr, int *member_count, struct keyval *tags_ptr) const
+{
+ int block = id2block(id), offset = id2offset(id);
+ struct member *members;
+
+ if (!rels[block])
+ return 1;
+
+ if (rels[block][offset].members) {
+ const size_t member_bytes = sizeof(struct member) * rels[block][offset].member_count;
+ members = (struct member *)malloc(member_bytes);
+ memcpy(members, rels[block][offset].members, member_bytes);
+ keyval::cloneList( tags_ptr, rels[block][offset].tags );
+
+ *members_ptr = members;
+ *member_count = rels[block][offset].member_count;
+ return 0;
+ }
+ return 1;
+}
+
+void middle_ram_t::analyze(void)
+{
+ /* No need */
+}
+
+void middle_ram_t::cleanup(void)
+{
+ /* No need */
+}
+
+void middle_ram_t::end(void)
+{
+ /* No need */
+}
+
+int middle_ram_t::start(const options_t *out_options_)
+{
+ out_options = out_options_;
+ /* latlong has a range of +-180, mercator +-20000
+ The fixed poing scaling needs adjusting accordingly to
+ be stored accurately in an int */
+ cache.reset(new node_ram_cache(out_options->alloc_chunkwise, out_options->cache, out_options->scale));
+
+ fprintf( stderr, "Mid: Ram, scale=%d\n", out_options->scale );
+
+ return 0;
+}
+
+void middle_ram_t::stop(void)
+{
+ cache.reset(NULL);
+
+ release_ways();
+ release_relations();
+}
+
+void middle_ram_t::commit(void) {
+}
+
+middle_ram_t::middle_ram_t():
+ ways(), rels(), way_blocks(0), cache(),
+ simulate_ways_deleted(false)
+{
+ ways.resize(NUM_BLOCKS); memset(&ways[0], 0, NUM_BLOCKS * sizeof ways[0]);
+ rels.resize(NUM_BLOCKS); memset(&rels[0], 0, NUM_BLOCKS * sizeof rels[0]);
+}
+
+middle_ram_t::~middle_ram_t() {
+ //instance.reset();
+}
+
+std::vector<osmid_t> middle_ram_t::relations_using_way(osmid_t way_id) const
+{
+ // this function shouldn't be called - relations_using_way is only used in
+ // slim mode, and a middle_ram_t shouldn't be constructed if the slim mode
+ // option is set.
+ throw std::runtime_error("middle_ram_t::relations_using_way is unimlpemented, and "
+ "should not have been called. This is probably a bug, please "
+ "report it at https://github.com/openstreetmap/osm2pgsql/issues");
+}
+
+namespace {
+
+void no_delete(const middle_ram_t * middle) {
+ // boost::shared_ptr thinks we are going to delete
+ // the middle object, but we are not. Heh heh heh.
+ // So yeah, this is a hack...
+}
+
+}
+
+boost::shared_ptr<const middle_query_t> middle_ram_t::get_instance() const {
+ //shallow copy here because readonly access is thread safe
+ return boost::shared_ptr<const middle_query_t>(this, no_delete);
+}
diff --git a/middle-ram.h b/middle-ram.h
deleted file mode 100644
index 3a5dd4f..0000000
--- a/middle-ram.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* Implements the mid-layer processing for osm2pgsql
- * using several PostgreSQL tables
- *
- * This layer stores data read in from the planet.osm file
- * and is then read by the backend processing code to
- * emit the final geometry-enabled output formats
-*/
-
-#ifndef MIDDLE_RAM_H
-#define MIDDLE_RAM_H
-
-extern struct middle_t mid_ram;
-
-#endif
diff --git a/middle-ram.hpp b/middle-ram.hpp
new file mode 100644
index 0000000..70c3303
--- /dev/null
+++ b/middle-ram.hpp
@@ -0,0 +1,86 @@
+/* Implements the mid-layer processing for osm2pgsql
+ * using data structures in RAM.
+ *
+ * This layer stores data read in from the planet.osm file
+ * and is then read by the backend processing code to
+ * emit the final geometry-enabled output formats
+*/
+
+#ifndef MIDDLE_RAM_H
+#define MIDDLE_RAM_H
+
+#include "middle.hpp"
+#include "node-ram-cache.hpp"
+#include <memory>
+#include <vector>
+
+struct middle_ram_t : public middle_t {
+ middle_ram_t();
+ virtual ~middle_ram_t();
+
+ int start(const options_t *out_options_);
+ void stop(void);
+ void cleanup(void);
+ void analyze(void);
+ void end(void);
+ void commit(void);
+
+ int nodes_set(osmid_t id, double lat, double lon, struct keyval *tags);
+ int nodes_get_list(struct osmNode *out, const osmid_t *nds, int nd_count) const;
+ int nodes_delete(osmid_t id);
+ int node_changed(osmid_t id);
+
+ int ways_set(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags);
+ int ways_get(osmid_t id, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) const;
+ int ways_get_list(const osmid_t *ids, int way_count, osmid_t *way_ids, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) const;
+
+ int ways_delete(osmid_t id);
+ int way_changed(osmid_t id);
+
+ int relations_get(osmid_t id, struct member **members, int *member_count, struct keyval *tags) const;
+ int relations_set(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+ int relations_delete(osmid_t id);
+ int relation_changed(osmid_t id);
+
+ std::vector<osmid_t> relations_using_way(osmid_t way_id) const;
+
+ void iterate_ways(middle_t::pending_processor& pf);
+ void iterate_relations(pending_processor& pf);
+
+ size_t pending_count() const;
+
+ virtual boost::shared_ptr<const middle_query_t> get_instance() const;
+private:
+
+ void release_ways();
+ void release_relations();
+
+ struct ramWay {
+ struct keyval *tags;
+ osmid_t *ndids;
+ };
+
+ struct ramRel {
+ struct keyval *tags;
+ struct member *members;
+ int member_count;
+ };
+
+ std::vector<ramWay *> ways;
+ std::vector<ramRel *> rels;
+
+ int way_blocks;
+
+ std::auto_ptr<node_ram_cache> cache;
+
+ /* the previous behaviour of iterate_ways was to delete all ways as they
+ * were being iterated. this doesn't work now that the output handles its
+ * own "done" status and output-specific "pending" status. however, the
+ * tests depend on the behaviour that ways will be unavailable once
+ * iterate_ways is complete, so this flag emulates that. */
+ bool simulate_ways_deleted;
+};
+
+extern middle_ram_t mid_ram;
+
+#endif
diff --git a/middle.cpp b/middle.cpp
new file mode 100644
index 0000000..63cc0a3
--- /dev/null
+++ b/middle.cpp
@@ -0,0 +1,26 @@
+#include "middle.hpp"
+#include "middle-pgsql.hpp"
+#include "middle-ram.hpp"
+
+#include <boost/make_shared.hpp>
+
+boost::shared_ptr<middle_t> middle_t::create_middle(const bool slim)
+{
+ if(slim)
+ return boost::make_shared<middle_pgsql_t>();
+ else
+ return boost::make_shared<middle_ram_t>();
+}
+
+
+middle_query_t::~middle_query_t() {
+}
+
+middle_t::~middle_t() {
+}
+
+slim_middle_t::~slim_middle_t() {
+}
+
+middle_t::pending_processor::~pending_processor() {
+}
diff --git a/middle.h b/middle.h
deleted file mode 100644
index c645eda..0000000
--- a/middle.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* Common middle layer interface */
-
-/* Each middle layer data store must provide methods for
- * storing and retrieving node and way data.
- */
-
-#ifndef MIDDLE_H
-#define MIDDLE_H
-
-#include "osmtypes.h"
-
-struct keyval;
-struct member;
-struct output_options;
-
-struct middle_t {
- int (*start)(const struct output_options *options);
- void (*stop)(void);
- void (*cleanup)(void);
- void (*analyze)(void);
- void (*end)(void);
- void (*commit)(void);
-
- int (*nodes_set)(osmid_t id, double lat, double lon, struct keyval *tags);
- int (*nodes_get_list)(struct osmNode *out, osmid_t *nds, int nd_count);
- int (*nodes_delete)(osmid_t id);
- int (*node_changed)(osmid_t id);
- /* int (*nodes_get)(struct osmNode *out, osmid_t id);*/
-
- int (*ways_set)(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags, int pending);
- int (*ways_get)(osmid_t id, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr);
- int (*ways_get_list)(osmid_t *ids, int way_count, osmid_t **way_ids, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr);
-
- int (*ways_done)(osmid_t id);
- int (*ways_delete)(osmid_t id);
- int (*way_changed)(osmid_t id);
-
- int (*relations_set)(osmid_t id, struct member *members, int member_count, struct keyval *tags);
- /* int (*relations_get)(osmid_t id, struct member **members, int *member_count, struct keyval *tags); */
- int (*relations_done)(osmid_t id);
- int (*relations_delete)(osmid_t id);
- int (*relation_changed)(osmid_t id);
-
- /* void (*iterate_nodes)(int (*callback)(osmid_t id, struct keyval *tags, double node_lat, double node_lon)); */
- void (*iterate_ways)(int (*callback)(osmid_t id, struct keyval *tags, struct osmNode *nodes, int count, int exists));
- void (*iterate_relations)(int (*callback)(osmid_t id, struct member *, int member_count, struct keyval *rel_tags, int exists));
-};
-
-#endif
diff --git a/middle.hpp b/middle.hpp
new file mode 100644
index 0000000..d5fdda8
--- /dev/null
+++ b/middle.hpp
@@ -0,0 +1,78 @@
+/* Common middle layer interface */
+
+/* Each middle layer data store must provide methods for
+ * storing and retrieving node and way data.
+ */
+
+#ifndef MIDDLE_H
+#define MIDDLE_H
+
+#include "osmtypes.hpp"
+#include "options.hpp"
+#include <vector>
+
+struct keyval;
+struct member;
+
+struct middle_query_t {
+ virtual ~middle_query_t();
+
+ virtual int nodes_get_list(struct osmNode *out, const osmid_t *nds, int nd_count) const = 0;
+
+ virtual int ways_get(osmid_t id, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) const = 0;
+
+ virtual int ways_get_list(const osmid_t *ids, int way_count, osmid_t *way_ids, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) const = 0;
+
+ virtual int relations_get(osmid_t id, struct member **members, int *member_count, struct keyval *tags) const = 0;
+
+ virtual std::vector<osmid_t> relations_using_way(osmid_t way_id) const = 0;
+
+ virtual boost::shared_ptr<const middle_query_t> get_instance() const = 0;
+};
+
+struct middle_t : public middle_query_t {
+ static boost::shared_ptr<middle_t> create_middle(const bool slim);
+
+ virtual ~middle_t();
+
+ virtual int start(const options_t *out_options_) = 0;
+ virtual void stop(void) = 0;
+ virtual void cleanup(void) = 0;
+ virtual void analyze(void) = 0;
+ virtual void end(void) = 0;
+ virtual void commit(void) = 0;
+
+ virtual int nodes_set(osmid_t id, double lat, double lon, struct keyval *tags) = 0;
+ virtual int ways_set(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags) = 0;
+ virtual int relations_set(osmid_t id, struct member *members, int member_count, struct keyval *tags) = 0;
+
+ struct pending_processor {
+ virtual ~pending_processor();
+ virtual void enqueue_ways(osmid_t id) = 0;
+ virtual void process_ways() = 0;
+ virtual void enqueue_relations(osmid_t id) = 0;
+ virtual void process_relations() = 0;
+ };
+
+ virtual void iterate_ways(pending_processor& pf) = 0;
+ virtual void iterate_relations(pending_processor& pf) = 0;
+
+ virtual size_t pending_count() const = 0;
+
+ const options_t* out_options;
+};
+
+struct slim_middle_t : public middle_t {
+ virtual ~slim_middle_t();
+
+ virtual int nodes_delete(osmid_t id) = 0;
+ virtual int node_changed(osmid_t id) = 0;
+
+ virtual int ways_delete(osmid_t id) = 0;
+ virtual int way_changed(osmid_t id) = 0;
+
+ virtual int relations_delete(osmid_t id) = 0;
+ virtual int relation_changed(osmid_t id) = 0;
+};
+
+#endif
diff --git a/node-persistent-cache-reader.c b/node-persistent-cache-reader.cpp
similarity index 66%
rename from node-persistent-cache-reader.c
rename to node-persistent-cache-reader.cpp
index 94b170f..4693880 100644
--- a/node-persistent-cache-reader.c
+++ b/node-persistent-cache-reader.cpp
@@ -11,19 +11,15 @@
#include <sys/wait.h>
#include <sys/time.h>
-#include "osmtypes.h"
-#include "output.h"
-#include "node-persistent-cache.h"
-#include "node-ram-cache.h"
-#include "binarysearcharray.h"
+#include "osmtypes.hpp"
+#include "output.hpp"
+#include "options.hpp"
+#include "node-persistent-cache.hpp"
+#include "node-ram-cache.hpp"
+#include "binarysearcharray.hpp"
-void exit_nicely()
-{
- fprintf(stderr, "Error occurred, cleaning up\n");
- exit(1);
-}
-
-void test_get_node_list(int itterations, int max_size, int process_number) {
+void test_get_node_list(boost::shared_ptr<node_persistent_cache> cache,
+ int itterations, int max_size, int process_number) {
int i, j, node_cnt, node_cnt_total;
struct osmNode *nodes;
struct timeval start, stop;
@@ -37,13 +33,13 @@ void test_get_node_list(int itterations, int max_size, int process_number) {
node_cnt_total += node_cnt;
printf("Process %i: Getting %i nodes....\n", process_number, node_cnt);
- nodes = malloc(sizeof(struct osmNode) * node_cnt);
- osmids = malloc(sizeof(osmid_t) * node_cnt);
+ nodes = (struct osmNode *)malloc(sizeof(struct osmNode) * node_cnt);
+ osmids = (osmid_t *)malloc(sizeof(osmid_t) * node_cnt);
for (j = 0; j < node_cnt; j++) {
osmids[j] = random() % (1 << 31);
}
gettimeofday(&start, NULL);
- persistent_cache_nodes_get_list(nodes,osmids,node_cnt);
+ cache->get_list(nodes,osmids,node_cnt);
gettimeofday(&stop, NULL);
double duration = ((stop.tv_sec - start.tv_sec)*1000000.0 + (stop.tv_usec - start.tv_usec))/1000000.0;
printf("Process %i: Got nodes in %f at a rate of %f/s\n", process_number, duration, node_cnt / duration);
@@ -57,41 +53,41 @@ void test_get_node_list(int itterations, int max_size, int process_number) {
int main(int argc, char *argv[]) {
int i,p;
- struct output_options options;
+ options_t options;
struct osmNode node;
struct osmNode *nodes;
struct timeval start;
osmid_t *osmids;
int node_cnt;
options.append = 1;
- options.scale = 100;
options.flat_node_cache_enabled = 1;
options.flat_node_file = argv[1];
- init_node_ram_cache(0,10,100);
+ boost::shared_ptr<node_ram_cache> ram_cache(new node_ram_cache(0, 10, options.scale));
+ boost::shared_ptr<node_persistent_cache> cache;
if (argc > 3) {
- init_node_persistent_cache(&options, 1);
+ cache.reset(new node_persistent_cache(&options, 1, ram_cache));
node_cnt = argc - 2;
- nodes = malloc(sizeof(struct osmNode) * node_cnt);
- osmids = malloc(sizeof(osmid_t) * node_cnt);
+ nodes = (struct osmNode *)malloc(sizeof(struct osmNode) * node_cnt);
+ osmids = (osmid_t *)malloc(sizeof(osmid_t) * node_cnt);
for (i = 0; i < node_cnt; i++) {
osmids[i] = atoi(argv[2 + i]);
}
- persistent_cache_nodes_get_list(nodes,osmids,node_cnt);
+ cache->get_list(nodes,osmids,node_cnt);
for (i = 0; i < node_cnt; i++) {
printf("lat: %f / lon: %f\n", nodes[i].lat, nodes[i].lon);
}
} else if (argc == 2) {
- char * state = malloc(sizeof(char)* 128);
+ char * state = (char *)malloc(sizeof(char)* 128);
gettimeofday(&start, NULL);
initstate(start.tv_usec, state, 8);
setstate(state);
printf("Testing mode\n");
- init_node_persistent_cache(&options, 1);
- test_get_node_list(10, 200, 0);
- shutdown_node_persistent_cache();
+ cache.reset(new node_persistent_cache(&options, 1, ram_cache));
+ test_get_node_list(cache, 10, 200, 0);
+ cache.reset();
#ifdef HAVE_FORK
printf("Testing using multiple processes\n");
int noProcs = 4;
@@ -109,11 +105,11 @@ int main(int argc, char *argv[]) {
gettimeofday(&start, NULL);
initstate(start.tv_usec, state, 8);
setstate(state);
- init_node_persistent_cache(&options, 1);
- test_get_node_list(10,200,p);
+ cache.reset(new node_persistent_cache(&options, 1, ram_cache));
+ test_get_node_list(cache, 10,200,p);
if (pid == 0) {
- shutdown_node_persistent_cache();
+ cache.reset();
fprintf(stderr,"Exiting process %i\n", p);
exit(0);
} else {
@@ -123,26 +119,26 @@ int main(int argc, char *argv[]) {
fprintf(stderr, "\nAll child processes exited\n");
#endif
} else {
- init_node_persistent_cache(&options, 1);
+ cache.reset(new node_persistent_cache(&options, 1, ram_cache));
if (strstr(argv[2],",") == NULL) {
- persistent_cache_nodes_get(&node, atoi(argv[2]));
+ cache->get(&node, atoi(argv[2]));
printf("lat: %f / lon: %f\n", node.lat, node.lon);
} else {
- char * node_list = malloc(sizeof(char) * (strlen(argv[2]) + 1));
+ char * node_list = (char *)malloc(sizeof(char) * (strlen(argv[2]) + 1));
strcpy(node_list,argv[2]);
node_cnt = 1;
strtok(node_list,",");
while (strtok(NULL,",") != NULL) node_cnt++;
printf("Processing %i nodes\n", node_cnt);
- nodes = malloc(sizeof(struct osmNode) * node_cnt);
- osmids = malloc(sizeof(osmid_t) * node_cnt);
+ nodes = (struct osmNode *)malloc(sizeof(struct osmNode) * node_cnt);
+ osmids = (osmid_t *)malloc(sizeof(osmid_t) * node_cnt);
strcpy(node_list,argv[2]);
osmids[0] = atoi(strtok(node_list,","));
for (i = 1; i < node_cnt; i++) {
char * tmp = strtok(NULL,",");
osmids[i] = atoi(tmp);
}
- persistent_cache_nodes_get_list(nodes,osmids,node_cnt);
+ cache->get_list(nodes,osmids,node_cnt);
for (i = 0; i < node_cnt; i++) {
printf("lat: %f / lon: %f\n", nodes[i].lat, nodes[i].lon);
}
@@ -150,6 +146,8 @@ int main(int argc, char *argv[]) {
}
- shutdown_node_persistent_cache();
+ cache.reset();
+ ram_cache.reset();
+
return 0;
}
diff --git a/node-persistent-cache.c b/node-persistent-cache.cpp
similarity index 82%
rename from node-persistent-cache.c
rename to node-persistent-cache.cpp
index bd1047a..2c26c7f 100644
--- a/node-persistent-cache.c
+++ b/node-persistent-cache.cpp
@@ -13,39 +13,39 @@
#include <fcntl.h>
#include <math.h>
-#include "osmtypes.h"
-#include "output.h"
-#include "node-persistent-cache.h"
-#include "node-ram-cache.h"
-#include "binarysearcharray.h"
-
-#ifdef __APPLE__
- #define lseek64 lseek
+#include "osmtypes.hpp"
+#include "output.hpp"
+#include "options.hpp"
+#include "node-persistent-cache.hpp"
+#include "binarysearcharray.hpp"
+#include "util.hpp"
+
+#include <stdexcept>
+
+#ifdef _WIN32
+ #include "win_fsync.h"
+ #define lseek64 _lseeki64
+ #ifndef S_IRUSR
+ #define S_IRUSR S_IREAD
+ #endif
+ #ifndef S_IWUSR
+ #define S_IWUSR S_IWRITE
+ #endif
#else
+ #ifdef __APPLE__
+ #define lseek64 lseek
+ #else
#ifndef HAVE_LSEEK64
- #if SIZEOF_OFF_T == 8
- #define lseek64 lseek
- #else
- #error Flat nodes cache requires a 64 bit capable seek
- #endif
+ #if SIZEOF_OFF_T == 8
+ #define lseek64 lseek
+ #else
+ #error Flat nodes cache requires a 64 bit capable seek
+ #endif
#endif
+ #endif
#endif
-static int node_cache_fd;
-static const char * node_cache_fname;
-static int append_mode;
-
-struct persistentCacheHeader cacheHeader;
-static struct ramNodeBlock writeNodeBlock; /* larger node block for more efficient initial sequential writing of node cache */
-static struct ramNodeBlock * readNodeBlockCache;
-static struct binary_search_array * readNodeBlockCacheIdx;
-
-
-static int scale;
-static int cache_already_written = 0;
-
-
-static void writeout_dirty_nodes(osmid_t id)
+void node_persistent_cache::writeout_dirty_nodes(osmid_t id)
{
int i;
@@ -57,8 +57,8 @@ static void writeout_dirty_nodes(osmid_t id)
+ sizeof(struct persistentCacheHeader), SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
-
+ util::exit_nicely();
+
};
if (write(node_cache_fd, writeNodeBlock.nodes,
WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode))
@@ -66,7 +66,7 @@ static void writeout_dirty_nodes(osmid_t id)
{
fprintf(stderr, "Failed to write out node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
cacheHeader.max_initialised_id = ((writeNodeBlock.block_offset + 1)
<< WRITE_NODE_BLOCK_SHIFT) - 1;
@@ -75,7 +75,7 @@ static void writeout_dirty_nodes(osmid_t id)
if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
};
if (write(node_cache_fd, &cacheHeader,
sizeof(struct persistentCacheHeader))
@@ -83,7 +83,7 @@ static void writeout_dirty_nodes(osmid_t id)
{
fprintf(stderr, "Failed to update persistent cache header: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
if (fsync(node_cache_fd) < 0) {
fprintf(stderr, "Info: Node cache could not be guaranteeded to be made durable. fsync failed: %s\n",
@@ -104,7 +104,7 @@ static void writeout_dirty_nodes(osmid_t id)
SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
};
if (write(node_cache_fd, readNodeBlockCache[i].nodes,
READ_NODE_BLOCK_SIZE * sizeof(struct ramNode))
@@ -112,7 +112,7 @@ static void writeout_dirty_nodes(osmid_t id)
{
fprintf(stderr, "Failed to write out node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
}
readNodeBlockCache[i].dirty = 0;
@@ -139,7 +139,7 @@ static void ramNodes_clear(struct ramNode * nodes, int size)
/**
* Find the cache block with the lowest usage count for replacement
*/
-static int persistent_cache_replace_block()
+int node_persistent_cache::replace_block()
{
int min_used = INT_MAX;
int block_id = -1;
@@ -169,7 +169,7 @@ static int persistent_cache_replace_block()
/**
* Find cache block number by block_offset
*/
-static int persistent_cache_find_block(osmid_t block_offset)
+int node_persistent_cache::find_block(osmid_t block_offset)
{
int idx = binary_search_get(readNodeBlockCacheIdx, block_offset);
return idx;
@@ -178,14 +178,14 @@ static int persistent_cache_find_block(osmid_t block_offset)
/**
* Initialise the persistent cache with NaN values to identify which IDs are valid or not
*/
-static void persistent_cache_expand_cache(osmid_t block_offset)
+void node_persistent_cache::expand_cache(osmid_t block_offset)
{
osmid_t i;
- struct ramNode * dummyNodes = malloc(
+ struct ramNode * dummyNodes = (struct ramNode *)malloc(
READ_NODE_BLOCK_SIZE * sizeof(struct ramNode));
if (!dummyNodes) {
fprintf(stderr, "Out of memory: Could not allocate node structure during cache expansion\n");
- exit_nicely();
+ util::exit_nicely();
}
ramNodes_clear(dummyNodes, READ_NODE_BLOCK_SIZE);
/* Need to expand the persistent node cache */
@@ -194,7 +194,7 @@ static void persistent_cache_expand_cache(osmid_t block_offset)
+ sizeof(struct persistentCacheHeader), SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
};
for (i = cacheHeader.max_initialised_id >> READ_NODE_BLOCK_SHIFT;
i <= block_offset; i++)
@@ -205,7 +205,7 @@ static void persistent_cache_expand_cache(osmid_t block_offset)
{
fprintf(stderr, "Failed to expand persistent node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
}
cacheHeader.max_initialised_id = ((block_offset + 1)
@@ -213,26 +213,26 @@ static void persistent_cache_expand_cache(osmid_t block_offset)
if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
};
if (write(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader))
!= sizeof(struct persistentCacheHeader))
{
fprintf(stderr, "Failed to update persistent cache header: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
free(dummyNodes);
fsync(node_cache_fd);
}
-static void persistent_cache_nodes_prefetch_async(osmid_t id)
+void node_persistent_cache::nodes_prefetch_async(osmid_t id)
{
#ifdef HAVE_POSIX_FADVISE
osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT;
- osmid_t block_id = persistent_cache_find_block(block_offset);
+ osmid_t block_id = find_block(block_offset);
if (block_id < 0)
{ /* The needed block isn't in cache already, so initiate loading */
@@ -241,7 +241,7 @@ static void persistent_cache_nodes_prefetch_async(osmid_t id)
/* Make sure the node cache is correctly initialised for the block that will be read */
if (cacheHeader.max_initialised_id
< ((block_offset + 1) << READ_NODE_BLOCK_SHIFT))
- persistent_cache_expand_cache(block_offset);
+ expand_cache(block_offset);
if (posix_fadvise(node_cache_fd, (block_offset << READ_NODE_BLOCK_SHIFT) * sizeof(struct ramNode)
+ sizeof(struct persistentCacheHeader), READ_NODE_BLOCK_SIZE * sizeof(struct ramNode),
@@ -256,10 +256,10 @@ static void persistent_cache_nodes_prefetch_async(osmid_t id)
/**
* Load block offset in a synchronous way.
*/
-static int persistent_cache_load_block(osmid_t block_offset)
+int node_persistent_cache::load_block(osmid_t block_offset)
{
- int block_id = persistent_cache_replace_block();
+ int block_id = replace_block();
if (readNodeBlockCache[block_id].dirty)
{
@@ -269,7 +269,7 @@ static int persistent_cache_load_block(osmid_t block_offset)
+ sizeof(struct persistentCacheHeader), SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
};
if (write(node_cache_fd, readNodeBlockCache[block_id].nodes,
READ_NODE_BLOCK_SIZE * sizeof(struct ramNode))
@@ -277,7 +277,7 @@ static int persistent_cache_load_block(osmid_t block_offset)
{
fprintf(stderr, "Failed to write out node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
readNodeBlockCache[block_id].dirty = 0;
}
@@ -292,7 +292,7 @@ static int persistent_cache_load_block(osmid_t block_offset)
if (cacheHeader.max_initialised_id
< ((block_offset + 1) << READ_NODE_BLOCK_SHIFT))
{
- persistent_cache_expand_cache(block_offset);
+ expand_cache(block_offset);
}
/* Read the block into cache */
@@ -301,7 +301,7 @@ static int persistent_cache_load_block(osmid_t block_offset)
+ sizeof(struct persistentCacheHeader), SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
};
if (read(node_cache_fd, readNodeBlockCache[block_id].nodes,
READ_NODE_BLOCK_SIZE * sizeof(struct ramNode))
@@ -317,7 +317,7 @@ static int persistent_cache_load_block(osmid_t block_offset)
return block_id;
}
-static void persisten_cache_nodes_set_create_writeout_block()
+void node_persistent_cache::nodes_set_create_writeout_block()
{
if (write(node_cache_fd, writeNodeBlock.nodes,
WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode))
@@ -325,7 +325,7 @@ static void persisten_cache_nodes_set_create_writeout_block()
{
fprintf(stderr, "Failed to write out node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
#ifdef HAVE_SYNC_FILE_RANGE
/* writing out large files can cause trouble on some operating systems.
@@ -351,7 +351,7 @@ static void persisten_cache_nodes_set_create_writeout_block()
SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER) < 0) {
fprintf(stderr, "Info: Sync_file_range block has an issue. This shouldn't be anything to worry about.: %s\n",
strerror(errno));
-
+
}
#ifdef HAVE_POSIX_FADVISE
if (posix_fadvise(node_cache_fd, (writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode) +
@@ -364,7 +364,7 @@ static void persisten_cache_nodes_set_create_writeout_block()
#endif
}
-static int persistent_cache_nodes_set_create(osmid_t id, double lat, double lon)
+int node_persistent_cache::set_create(osmid_t id, double lat, double lon)
{
osmid_t block_offset = id >> WRITE_NODE_BLOCK_SHIFT;
int i;
@@ -376,7 +376,7 @@ static int persistent_cache_nodes_set_create(osmid_t id, double lat, double lon)
{
if (writeNodeBlock.dirty)
{
- persisten_cache_nodes_set_create_writeout_block();
+ nodes_set_create_writeout_block();
writeNodeBlock.used = 0;
writeNodeBlock.dirty = 0;
/* After writing out the node block, the file pointer is at the next block level */
@@ -389,14 +389,14 @@ static int persistent_cache_nodes_set_create(osmid_t id, double lat, double lon)
fprintf(stderr,
"ERROR: Block_offset not in sequential order: %" PRIdOSMID "%" PRIdOSMID "\n",
writeNodeBlock.block_offset, block_offset);
- exit_nicely();
+ util::exit_nicely();
}
/* We need to fill the intermediate node cache with node nodes to identify which nodes are valid */
for (i = writeNodeBlock.block_offset; i < block_offset; i++)
{
ramNodes_clear(writeNodeBlock.nodes, WRITE_NODE_BLOCK_SIZE);
- persisten_cache_nodes_set_create_writeout_block();
+ nodes_set_create_writeout_block();
}
ramNodes_clear(writeNodeBlock.nodes, WRITE_NODE_BLOCK_SIZE);
@@ -404,8 +404,8 @@ static int persistent_cache_nodes_set_create(osmid_t id, double lat, double lon)
writeNodeBlock.block_offset = block_offset;
}
#ifdef FIXED_POINT
- writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK].lat = DOUBLE_TO_FIX(lat);
- writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK].lon = DOUBLE_TO_FIX(lon);
+ writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK].lat = util::double_to_fix(lat, scale_);
+ writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK].lon = util::double_to_fix(lon, scale_);
#else
writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK].lat = lat;
writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK].lon = lon;
@@ -416,14 +416,14 @@ static int persistent_cache_nodes_set_create(osmid_t id, double lat, double lon)
return 0;
}
-static int persistent_cache_nodes_set_append(osmid_t id, double lat, double lon)
+int node_persistent_cache::set_append(osmid_t id, double lat, double lon)
{
osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT;
- int block_id = persistent_cache_find_block(block_offset);
+ int block_id = find_block(block_offset);
if (block_id < 0)
- block_id = persistent_cache_load_block(block_offset);
+ block_id = load_block(block_offset);
#ifdef FIXED_POINT
if (isnan(lat) && isnan(lon))
@@ -436,9 +436,9 @@ static int persistent_cache_nodes_set_append(osmid_t id, double lat, double lon)
else
{
readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat =
- DOUBLE_TO_FIX(lat);
+ util::double_to_fix(lat, scale_);
readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon =
- DOUBLE_TO_FIX(lon);
+ util::double_to_fix(lon, scale_);
}
#else
readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat = lat;
@@ -450,23 +450,23 @@ static int persistent_cache_nodes_set_append(osmid_t id, double lat, double lon)
return 1;
}
-int persistent_cache_nodes_set(osmid_t id, double lat, double lon)
+int node_persistent_cache::set(osmid_t id, double lat, double lon)
{
return append_mode ?
- persistent_cache_nodes_set_append(id, lat, lon) :
- persistent_cache_nodes_set_create(id, lat, lon);
+ set_append(id, lat, lon) :
+ set_create(id, lat, lon);
}
-int persistent_cache_nodes_get(struct osmNode *out, osmid_t id)
+int node_persistent_cache::get(struct osmNode *out, osmid_t id)
{
osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT;
- osmid_t block_id = persistent_cache_find_block(block_offset);
+ osmid_t block_id = find_block(block_offset);
if (block_id < 0)
{
writeout_dirty_nodes(id);
- block_id = persistent_cache_load_block(block_offset);
+ block_id = load_block(block_offset);
}
readNodeBlockCache[block_id].used++;
@@ -482,9 +482,9 @@ int persistent_cache_nodes_get(struct osmNode *out, osmid_t id)
else
{
out->lat =
- FIX_TO_DOUBLE(readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat);
+ util::fix_to_double(readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat, scale_);
out->lon =
- FIX_TO_DOUBLE(readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon);
+ util::fix_to_double(readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon, scale_);
return 0;
}
#else
@@ -504,7 +504,7 @@ int persistent_cache_nodes_get(struct osmNode *out, osmid_t id)
return 0;
}
-int persistent_cache_nodes_get_list(struct osmNode *nodes, osmid_t *ndids,
+int node_persistent_cache::get_list(struct osmNode *nodes, const osmid_t *ndids,
int nd_count)
{
int count = 0;
@@ -512,7 +512,7 @@ int persistent_cache_nodes_get_list(struct osmNode *nodes, osmid_t *ndids,
for (i = 0; i < nd_count; i++)
{
/* Check cache first */
- if (ram_cache_nodes_get(&nodes[i], ndids[i]) == 0)
+ if (ram_cache && (ram_cache->get(&nodes[i], ndids[i]) == 0))
{
count++;
}
@@ -530,12 +530,12 @@ int persistent_cache_nodes_get_list(struct osmNode *nodes, osmid_t *ndids,
/* In order to have a higher OS level I/O queue depth
issue posix_fadvise(WILLNEED) requests for all I/O */
if (isnan(nodes[i].lat) && isnan(nodes[i].lon))
- persistent_cache_nodes_prefetch_async(ndids[i]);
+ nodes_prefetch_async(ndids[i]);
}
for (i = 0; i < nd_count; i++)
{
if ((isnan(nodes[i].lat) && isnan(nodes[i].lon))
- && (persistent_cache_nodes_get(&(nodes[i]), ndids[i]) == 0))
+ && (get(&(nodes[i]), ndids[i]) == 0))
count++;
}
@@ -561,17 +561,31 @@ int persistent_cache_nodes_get_list(struct osmNode *nodes, osmid_t *ndids,
return count;
}
-void init_node_persistent_cache(const struct output_options *options, int append)
+node_persistent_cache::node_persistent_cache(const options_t *options, int append,
+ boost::shared_ptr<node_ram_cache> ptr)
+ : node_cache_fd(0), node_cache_fname(NULL), append_mode(0), cacheHeader(),
+ writeNodeBlock(), readNodeBlockCache(NULL), readNodeBlockCacheIdx(NULL),
+ scale_(0), cache_already_written(0), ram_cache(ptr)
{
int i, err;
- scale = options->scale;
+ scale_ = options->scale;
append_mode = append;
- node_cache_fname = options->flat_node_file;
+ if (options->flat_node_file) {
+ node_cache_fname = options->flat_node_file->c_str();
+ } else {
+ throw std::runtime_error("Unable to set up persistent cache: the name "
+ "of the flat node file was not set.");
+ }
fprintf(stderr, "Mid: loading persistent node cache from %s\n",
node_cache_fname);
readNodeBlockCacheIdx = init_search_array(READ_NODE_CACHE_SIZE);
-
+ if (readNodeBlockCacheIdx == NULL)
+ {
+ fprintf(stderr, "Unable to initialise binary search array\n");
+ util::exit_nicely();
+ }
+
/* Setup the file for the node position cache */
if (append_mode)
{
@@ -580,7 +594,7 @@ void init_node_persistent_cache(const struct output_options *options, int append
{
fprintf(stderr, "Failed to open node cache file: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
}
else
@@ -599,12 +613,12 @@ void init_node_persistent_cache(const struct output_options *options, int append
{
fprintf(stderr, "Failed to create node cache file: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
};
if (cache_already_written == 0)
{
@@ -622,15 +636,15 @@ void init_node_persistent_cache(const struct output_options *options, int append
}
close(node_cache_fd);
- exit_nicely();
+ util::exit_nicely();
}
fprintf(stderr, "Allocated space for persistent node cache file\n");
#endif
- writeNodeBlock.nodes = malloc(
+ writeNodeBlock.nodes = (struct ramNode *)malloc(
WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode));
if (!writeNodeBlock.nodes) {
fprintf(stderr, "Out of memory: Failed to allocate node writeout buffer\n");
- exit_nicely();
+ util::exit_nicely();
}
ramNodes_clear(writeNodeBlock.nodes, WRITE_NODE_BLOCK_SIZE);
writeNodeBlock.block_offset = 0;
@@ -642,7 +656,7 @@ void init_node_persistent_cache(const struct output_options *options, int append
if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
};
if (write(node_cache_fd, &cacheHeader,
sizeof(struct persistentCacheHeader))
@@ -650,7 +664,7 @@ void init_node_persistent_cache(const struct output_options *options, int append
{
fprintf(stderr, "Failed to write persistent cache header: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
}
@@ -658,42 +672,42 @@ void init_node_persistent_cache(const struct output_options *options, int append
if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
};
if (read(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader))
!= sizeof(struct persistentCacheHeader))
{
fprintf(stderr, "Failed to read persistent cache header: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
if (cacheHeader.format_version != PERSISTENT_CACHE_FORMAT_VERSION)
{
fprintf(stderr, "Persistent cache header is wrong version\n");
- exit_nicely();
+ util::exit_nicely();
}
if (cacheHeader.id_size != sizeof(osmid_t))
{
fprintf(stderr, "Persistent cache header is wrong id type\n");
- exit_nicely();
+ util::exit_nicely();
}
fprintf(stderr,"Maximum node in persistent node cache: %" PRIdOSMID "\n", cacheHeader.max_initialised_id);
- readNodeBlockCache = malloc(
+ readNodeBlockCache = (struct ramNodeBlock *)malloc(
READ_NODE_CACHE_SIZE * sizeof(struct ramNodeBlock));
if (!readNodeBlockCache) {
fprintf(stderr, "Out of memory: Failed to allocate node read cache\n");
- exit_nicely();
+ util::exit_nicely();
}
for (i = 0; i < READ_NODE_CACHE_SIZE; i++)
{
- readNodeBlockCache[i].nodes = malloc(
+ readNodeBlockCache[i].nodes = (struct ramNode *)malloc(
READ_NODE_BLOCK_SIZE * sizeof(struct ramNode));
if (!readNodeBlockCache[i].nodes) {
fprintf(stderr, "Out of memory: Failed to allocate node read cache\n");
- exit_nicely();
+ util::exit_nicely();
}
readNodeBlockCache[i].block_offset = -1;
readNodeBlockCache[i].used = 0;
@@ -701,7 +715,7 @@ void init_node_persistent_cache(const struct output_options *options, int append
}
}
-void shutdown_node_persistent_cache()
+node_persistent_cache::~node_persistent_cache()
{
int i;
writeout_dirty_nodes(-1);
@@ -709,14 +723,14 @@ void shutdown_node_persistent_cache()
if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) {
fprintf(stderr, "Failed to seek to correct position in node cache: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
};
if (write(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader))
!= sizeof(struct persistentCacheHeader))
{
fprintf(stderr, "Failed to update persistent cache header: %s\n",
strerror(errno));
- exit_nicely();
+ util::exit_nicely();
}
fprintf(stderr,"Maximum node in persistent node cache: %" PRIdOSMID "\n", cacheHeader.max_initialised_id);
diff --git a/node-persistent-cache.h b/node-persistent-cache.h
deleted file mode 100644
index 6d30d71..0000000
--- a/node-persistent-cache.h
+++ /dev/null
@@ -1,29 +0,0 @@
-#ifndef NODE_PERSISTENT_CACHE_H
-#define NODE_PERSISTENT_CACHE_H
-
-#define MAXIMUM_INITIAL_ID 2600000000
-
-#define READ_NODE_CACHE_SIZE 10000
-#define READ_NODE_BLOCK_SHIFT 10l
-#define READ_NODE_BLOCK_SIZE (1l << READ_NODE_BLOCK_SHIFT)
-#define READ_NODE_BLOCK_MASK 0x03FFl
-
-#define WRITE_NODE_BLOCK_SHIFT 20l
-#define WRITE_NODE_BLOCK_SIZE (1l << WRITE_NODE_BLOCK_SHIFT)
-#define WRITE_NODE_BLOCK_MASK 0x0FFFFFl
-
-#define PERSISTENT_CACHE_FORMAT_VERSION 1
-
-struct persistentCacheHeader {
- int format_version;
- int id_size;
- osmid_t max_initialised_id;
-};
-
-int persistent_cache_nodes_set(osmid_t id, double lat, double lon);
-int persistent_cache_nodes_get(struct osmNode *out, osmid_t id);
-int persistent_cache_nodes_get_list(struct osmNode *nodes, osmid_t *ndids, int nd_count);
-void init_node_persistent_cache(const struct output_options *options, const int append);
-void shutdown_node_persistent_cache();
-
-#endif
diff --git a/node-persistent-cache.hpp b/node-persistent-cache.hpp
new file mode 100644
index 0000000..0c9ee9b
--- /dev/null
+++ b/node-persistent-cache.hpp
@@ -0,0 +1,64 @@
+#ifndef NODE_PERSISTENT_CACHE_H
+#define NODE_PERSISTENT_CACHE_H
+
+#include "node-ram-cache.hpp"
+#include <boost/shared_ptr.hpp>
+
+#define MAXIMUM_INITIAL_ID 2600000000
+
+#define READ_NODE_CACHE_SIZE 10000
+#define READ_NODE_BLOCK_SHIFT 10l
+#define READ_NODE_BLOCK_SIZE (1l << READ_NODE_BLOCK_SHIFT)
+#define READ_NODE_BLOCK_MASK 0x03FFl
+
+#define WRITE_NODE_BLOCK_SHIFT 20l
+#define WRITE_NODE_BLOCK_SIZE (1l << WRITE_NODE_BLOCK_SHIFT)
+#define WRITE_NODE_BLOCK_MASK 0x0FFFFFl
+
+#define PERSISTENT_CACHE_FORMAT_VERSION 1
+
+struct persistentCacheHeader {
+ int format_version;
+ int id_size;
+ osmid_t max_initialised_id;
+};
+
+struct node_persistent_cache : public boost::noncopyable
+{
+ node_persistent_cache(const struct options_t *options, const int append,
+ boost::shared_ptr<node_ram_cache> ptr);
+ ~node_persistent_cache();
+
+ int set(osmid_t id, double lat, double lon);
+ int get(struct osmNode *out, osmid_t id);
+ int get_list(struct osmNode *nodes, const osmid_t *ndids, int nd_count);
+
+private:
+
+ int set_append(osmid_t id, double lat, double lon);
+ int set_create(osmid_t id, double lat, double lon);
+
+ void writeout_dirty_nodes(osmid_t id);
+ int replace_block();
+ int find_block(osmid_t block_offset);
+ void expand_cache(osmid_t block_offset);
+ void nodes_prefetch_async(osmid_t id);
+ int load_block(osmid_t block_offset);
+ void nodes_set_create_writeout_block();
+
+ int node_cache_fd;
+ const char * node_cache_fname;
+ int append_mode;
+
+ struct persistentCacheHeader cacheHeader;
+ struct ramNodeBlock writeNodeBlock; /* larger node block for more efficient initial sequential writing of node cache */
+ struct ramNodeBlock * readNodeBlockCache;
+ struct binary_search_array * readNodeBlockCacheIdx;
+
+ int scale_;
+ int cache_already_written;
+
+ boost::shared_ptr<node_ram_cache> ram_cache;
+};
+
+#endif
diff --git a/node-ram-cache.c b/node-ram-cache.cpp
similarity index 80%
rename from node-ram-cache.c
rename to node-ram-cache.cpp
index d237760..09d33fd 100644
--- a/node-ram-cache.c
+++ b/node-ram-cache.cpp
@@ -2,7 +2,7 @@
* It uses two different storage methods, one optimized for dense
* nodes (with respect to id) and the other for sparse representations.
*/
-
+
#include "config.h"
#include <stdio.h>
@@ -11,9 +11,10 @@
#include <string.h>
#include <assert.h>
-#include "osmtypes.h"
-#include "middle.h"
-#include "node-ram-cache.h"
+#include "osmtypes.hpp"
+#include "middle.hpp"
+#include "node-ram-cache.hpp"
+#include "util.hpp"
@@ -58,36 +59,12 @@
-static int allocStrategy = ALLOC_DENSE;
-
#define BLOCK_SHIFT 10
#define PER_BLOCK (((osmid_t)1) << BLOCK_SHIFT)
#define NUM_BLOCKS (((osmid_t)1) << (36 - BLOCK_SHIFT))
#define SAFETY_MARGIN 1024*PER_BLOCK*sizeof(struct ramNode)
-static struct ramNodeBlock *blocks;
-static int usedBlocks;
-/* Note: maxBlocks *must* be odd, to make sure the priority queue has no nodes with only one child */
-static int maxBlocks = 0;
-static void *blockCache = NULL;
-
-static struct ramNodeBlock **queue;
-
-
-static struct ramNodeID *sparseBlock;
-static int64_t maxSparseTuples = 0;
-static int64_t sizeSparseTuples = 0;
-
-
-static int64_t cacheUsed, cacheSize;
-static osmid_t storedNodes, totalNodes;
-int nodesCacheHits, nodesCacheLookups;
-
-static int warn_node_order;
-
-static int ram_cache_nodes_get_sparse(struct osmNode *out, osmid_t id);
-
static int id2block(osmid_t id)
{
/* + NUM_BLOCKS/2 allows for negative IDs */
@@ -106,7 +83,7 @@ static osmid_t block2id(int block, int offset)
#define Swap(a,b) { struct ramNodeBlock * __tmp = a; a = b; b = __tmp; }
-static void percolate_up( int pos )
+void node_ram_cache::percolate_up( int pos )
{
int i = pos;
while( i > 0 )
@@ -122,33 +99,33 @@ static void percolate_up( int pos )
}
}
-static void *next_chunk(size_t count, size_t size) {
+struct ramNode *node_ram_cache::next_chunk(size_t count, size_t size) {
if ( (allocStrategy & ALLOC_DENSE_CHUNK) == 0 ) {
static size_t pos = 0;
- void *result;
+ char *result;
pos += count * size;
result = blockCache + cacheSize - pos + SAFETY_MARGIN;
-
- return result;
+
+ return (struct ramNode *)result;
} else {
- return calloc(PER_BLOCK, sizeof(struct ramNode));
+ return (struct ramNode *)calloc(PER_BLOCK, sizeof(struct ramNode));
}
}
-static int ram_cache_nodes_set_sparse(osmid_t id, double lat, double lon, struct keyval *tags UNUSED) {
+int node_ram_cache::set_sparse(osmid_t id, double lat, double lon, struct keyval *tags) {
if ((sizeSparseTuples > maxSparseTuples) || ( cacheUsed > cacheSize)) {
if ((allocStrategy & ALLOC_LOSSY) > 0)
return 1;
else {
fprintf(stderr, "\nNode cache size is too small to fit all nodes. Please increase cache size\n");
- exit_nicely();
+ util::exit_nicely();
}
}
sparseBlock[sizeSparseTuples].id = id;
#ifdef FIXED_POINT
- sparseBlock[sizeSparseTuples].coord.lat = DOUBLE_TO_FIX(lat);
- sparseBlock[sizeSparseTuples].coord.lon = DOUBLE_TO_FIX(lon);
+ sparseBlock[sizeSparseTuples].coord.lat = util::double_to_fix(lat, scale_);
+ sparseBlock[sizeSparseTuples].coord.lon = util::double_to_fix(lon, scale_);
#else
sparseBlock[sizeSparseTuples].coord.lat = lat;
sparseBlock[sizeSparseTuples].coord.lon = lon;
@@ -159,7 +136,7 @@ static int ram_cache_nodes_set_sparse(osmid_t id, double lat, double lon, struct
return 0;
}
-static int ram_cache_nodes_set_dense(osmid_t id, double lat, double lon, struct keyval *tags UNUSED) {
+int node_ram_cache::set_dense(osmid_t id, double lat, double lon, struct keyval *tags) {
int block = id2block(id);
int offset = id2offset(id);
int i = 0;
@@ -174,10 +151,10 @@ static int ram_cache_nodes_set_dense(osmid_t id, double lat, double lon, struct
/* if usedBlocks > 0 then the previous block is used up. Need to correctly handle it. */
if ( usedBlocks > 0 ) {
/* If sparse allocation is also set, then check if the previous block has sufficient density
- * to store it in dense representation. If not, push all elements of the block
+ * to store it in dense representation. If not, push all elements of the block
* to the sparse node cache and reuse memory of the previous block for the current block */
if ( ((allocStrategy & ALLOC_SPARSE) == 0) ||
- ((queue[usedBlocks - 1]->used / (double)(1<< BLOCK_SHIFT)) >
+ ((queue[usedBlocks - 1]->used / (double)(1<< BLOCK_SHIFT)) >
(sizeof(struct ramNode) / (double)sizeof(struct ramNodeID)))) {
/* Block has reached the level to keep it in dense representation */
/* We've just finished with the previous block, so we need to percolate it up the queue to its correct position */
@@ -188,13 +165,13 @@ static int ram_cache_nodes_set_dense(osmid_t id, double lat, double lon, struct
/* previous block was not dense enough, so push it into the sparse node cache instead */
for (i = 0; i < (1 << BLOCK_SHIFT); i++) {
if (queue[usedBlocks -1]->nodes[i].lat || queue[usedBlocks -1]->nodes[i].lon) {
- ram_cache_nodes_set_sparse(block2id(queue[usedBlocks - 1]->block_offset,i),
+ set_sparse(block2id(queue[usedBlocks - 1]->block_offset,i),
#ifdef FIXED_POINT
- FIX_TO_DOUBLE(queue[usedBlocks -1]->nodes[i].lat),
- FIX_TO_DOUBLE(queue[usedBlocks -1]->nodes[i].lon),
+ util::fix_to_double(queue[usedBlocks -1]->nodes[i].lat, scale_),
+ util::fix_to_double(queue[usedBlocks -1]->nodes[i].lon, scale_),
#else
queue[usedBlocks -1]->nodes[i].lat,
- queue[usedBlocks -1]->nodes[i].lon,
+ queue[usedBlocks -1]->nodes[i].lon,
#endif
NULL);
}
@@ -204,23 +181,22 @@ static int ram_cache_nodes_set_dense(osmid_t id, double lat, double lon, struct
blocks[block].nodes = queue[usedBlocks - 1]->nodes;
blocks[queue[usedBlocks - 1]->block_offset].nodes = NULL;
memset( blocks[block].nodes, 0, PER_BLOCK * sizeof(struct ramNode) );
- usedBlocks--;
+ usedBlocks--;
cacheUsed -= PER_BLOCK * sizeof(struct ramNode);
}
} else {
blocks[block].nodes = next_chunk(PER_BLOCK, sizeof(struct ramNode));
}
-
+
blocks[block].used = 0;
blocks[block].block_offset = block;
if (!blocks[block].nodes) {
fprintf(stderr, "Error allocating nodes\n");
- exit_nicely();
+ util::exit_nicely();
}
queue[usedBlocks] = &blocks[block];
usedBlocks++;
cacheUsed += PER_BLOCK * sizeof(struct ramNode);
-
/* If we've just used up the last possible block we enter the
* transition and we change the invariant. To do this we percolate
@@ -230,12 +206,12 @@ static int ram_cache_nodes_set_dense(osmid_t id, double lat, double lon, struct
} else {
if ((allocStrategy & ALLOC_LOSSY) == 0) {
fprintf(stderr, "\nNode cache size is too small to fit all nodes. Please increase cache size\n");
- exit_nicely();
+ util::exit_nicely();
}
/* We've reached the maximum number of blocks, so now we push the
* current head of the tree down to the right level to restore the
* priority queue invariant. Upto log(maxBlocks) iterations */
-
+
i=0;
while( 2*i+1 < usedBlocks - 1 ) {
if( queue[2*i+1]->used <= queue[2*i+2]->used ) {
@@ -257,7 +233,7 @@ static int ram_cache_nodes_set_dense(osmid_t id, double lat, double lon, struct
blocks[block].nodes = queue[0]->nodes;
blocks[block].used = 0;
memset( blocks[block].nodes, 0, PER_BLOCK * sizeof(struct ramNode) );
-
+
/* Clear old head block and point to new block */
storedNodes -= queue[0]->used;
queue[0]->nodes = NULL;
@@ -268,13 +244,13 @@ static int ram_cache_nodes_set_dense(osmid_t id, double lat, double lon, struct
/* Insert into an existing block. We can't allow this in general or it
* will break the invariant. However, it will work fine if all the
* nodes come in numerical order, which is the common case */
-
+
int expectedpos;
if (( usedBlocks < maxBlocks ) && (cacheUsed < cacheSize))
expectedpos = usedBlocks-1;
else
expectedpos = 0;
-
+
if( queue[expectedpos] != &blocks[block] ) {
if (!warn_node_order) {
fprintf( stderr, "WARNING: Found Out of order node %" PRIdOSMID " (%d,%d) - this will impact the cache efficiency\n", id, block, offset );
@@ -283,10 +259,10 @@ static int ram_cache_nodes_set_dense(osmid_t id, double lat, double lon, struct
return 1;
}
}
-
+
#ifdef FIXED_POINT
- blocks[block].nodes[offset].lat = DOUBLE_TO_FIX(lat);
- blocks[block].nodes[offset].lon = DOUBLE_TO_FIX(lon);
+ blocks[block].nodes[offset].lat = util::double_to_fix(lat, scale_);
+ blocks[block].nodes[offset].lon = util::double_to_fix(lon, scale_);
#else
blocks[block].nodes[offset].lat = lat;
blocks[block].nodes[offset].lon = lon;
@@ -297,7 +273,7 @@ static int ram_cache_nodes_set_dense(osmid_t id, double lat, double lon, struct
}
-static int ram_cache_nodes_get_sparse(struct osmNode *out, osmid_t id) {
+int node_ram_cache::get_sparse(struct osmNode *out, osmid_t id) {
int64_t pivotPos = sizeSparseTuples >> 1;
int64_t minPos = 0;
int64_t maxPos = sizeSparseTuples;
@@ -305,8 +281,8 @@ static int ram_cache_nodes_get_sparse(struct osmNode *out, osmid_t id) {
while (minPos <= maxPos) {
if ( sparseBlock[pivotPos].id == id ) {
#ifdef FIXED_POINT
- out->lat = FIX_TO_DOUBLE(sparseBlock[pivotPos].coord.lat);
- out->lon = FIX_TO_DOUBLE(sparseBlock[pivotPos].coord.lon);
+ out->lat = util::fix_to_double(sparseBlock[pivotPos].coord.lat, scale_);
+ out->lon = util::fix_to_double(sparseBlock[pivotPos].coord.lon, scale_);
#else
out->lat = sparseBlock[pivotPos].coord.lat;
out->lon = sparseBlock[pivotPos].coord.lon;
@@ -323,33 +299,38 @@ static int ram_cache_nodes_get_sparse(struct osmNode *out, osmid_t id) {
pivotPos = minPos + ((maxPos - minPos) >> 1);
}
}
-
- return 1;
+
+ return 1;
}
-static int ram_cache_nodes_get_dense(struct osmNode *out, osmid_t id) {
+int node_ram_cache::get_dense(struct osmNode *out, osmid_t id) {
int block = id2block(id);
int offset = id2offset(id);
-
+
if (!blocks[block].nodes)
return 1;
-
+
if (!blocks[block].nodes[offset].lat && !blocks[block].nodes[offset].lon)
return 1;
-
+
#ifdef FIXED_POINT
- out->lat = FIX_TO_DOUBLE(blocks[block].nodes[offset].lat);
- out->lon = FIX_TO_DOUBLE(blocks[block].nodes[offset].lon);
+ out->lat = util::fix_to_double(blocks[block].nodes[offset].lat, scale_);
+ out->lon = util::fix_to_double(blocks[block].nodes[offset].lon, scale_);
#else
out->lat = blocks[block].nodes[offset].lat;
out->lon = blocks[block].nodes[offset].lon;
#endif
-
+
return 0;
}
-void init_node_ram_cache( int strategy, int cacheSizeMB, int fixpointscale ) {
+node_ram_cache::node_ram_cache( int strategy, int cacheSizeMB, int fixpointscale )
+ : allocStrategy(ALLOC_DENSE), blocks(NULL), usedBlocks(0),
+ maxBlocks(0), blockCache(NULL), queue(NULL), scale_(fixpointscale), sparseBlock(NULL),
+ maxSparseTuples(0), sizeSparseTuples(0), cacheUsed(0),
+ cacheSize(0), storedNodes(0), totalNodes(0), nodesCacheHits(0),
+ nodesCacheLookups(0), warn_node_order(0) {
blockCache = 0;
cacheUsed = 0;
@@ -357,18 +338,17 @@ void init_node_ram_cache( int strategy, int cacheSizeMB, int fixpointscale ) {
/* How much we can fit, and make sure it's odd */
maxBlocks = (cacheSize/(PER_BLOCK*sizeof(struct ramNode)));
maxSparseTuples = (cacheSize/sizeof(struct ramNodeID))+1;
-
+
allocStrategy = strategy;
- scale = fixpointscale;
-
+
if ((allocStrategy & ALLOC_DENSE) > 0 ) {
fprintf(stderr, "Allocating memory for dense node cache\n");
- blocks = calloc(NUM_BLOCKS,sizeof(struct ramNodeBlock));
+ blocks = (struct ramNodeBlock *)calloc(NUM_BLOCKS,sizeof(struct ramNodeBlock));
if (!blocks) {
fprintf(stderr, "Out of memory for node cache dense index, try using \"--cache-strategy sparse\" instead \n");
- exit_nicely();
+ util::exit_nicely();
}
- queue = calloc( maxBlocks,sizeof(struct ramNodeBlock *) );
+ queue = (struct ramNodeBlock **)calloc( maxBlocks,sizeof(struct ramNodeBlock *) );
/* Use this method of allocation if virtual memory is limited,
* or if OS allocs physical memory right away, rather than page by page
* once it is needed.
@@ -377,14 +357,14 @@ void init_node_ram_cache( int strategy, int cacheSizeMB, int fixpointscale ) {
fprintf(stderr, "Allocating dense node cache in block sized chunks\n");
if (!queue) {
fprintf(stderr, "Out of memory, reduce --cache size\n");
- exit_nicely();
+ util::exit_nicely();
}
} else {
fprintf(stderr, "Allocating dense node cache in one big chunk\n");
- blockCache = calloc(maxBlocks + 1024,PER_BLOCK * sizeof(struct ramNode));
+ blockCache = (char *)calloc(maxBlocks + 1024,PER_BLOCK * sizeof(struct ramNode));
if (!queue || !blockCache) {
fprintf(stderr, "Out of memory for dense node cache, reduce --cache size\n");
- exit_nicely();
+ util::exit_nicely();
}
}
}
@@ -395,31 +375,31 @@ void init_node_ram_cache( int strategy, int cacheSizeMB, int fixpointscale ) {
* lazy allocation of physical RAM. Extra accounting during setting of nodes is done
* to ensure physical RAM usage should roughly be no more than --cache
*/
-
+
if ((allocStrategy & ALLOC_SPARSE) > 0 ) {
fprintf(stderr, "Allocating memory for sparse node cache\n");
if (!blockCache) {
- sparseBlock = calloc(maxSparseTuples,sizeof(struct ramNodeID));
+ sparseBlock = (struct ramNodeID *)calloc(maxSparseTuples,sizeof(struct ramNodeID));
} else {
fprintf(stderr, "Sharing dense sparse\n");
- sparseBlock = blockCache;
+ sparseBlock = (struct ramNodeID *)blockCache;
}
if (!sparseBlock) {
fprintf(stderr, "Out of memory for sparse node cache, reduce --cache size\n");
- exit_nicely();
+ util::exit_nicely();
}
}
#ifdef __MINGW_H
- fprintf( stderr, "Node-cache: cache=%ldMB, maxblocks=%d*%d, allocation method=%i\n", (cacheSize >> 20), maxBlocks, PER_BLOCK*sizeof(struct ramNode), allocStrategy );
+ fprintf( stderr, "Node-cache: cache=%ldMB, maxblocks=%d*%d, allocation method=%i\n", (cacheSize >> 20), maxBlocks, PER_BLOCK*sizeof(struct ramNode), allocStrategy );
#else
fprintf( stderr, "Node-cache: cache=%ldMB, maxblocks=%d*%zd, allocation method=%i\n", (cacheSize >> 20), maxBlocks, PER_BLOCK*sizeof(struct ramNode), allocStrategy );
#endif
}
-void free_node_ram_cache() {
+node_ram_cache::~node_ram_cache() {
int i;
- fprintf( stderr, "node cache: stored: %" PRIdOSMID "(%.2f%%), storage efficiency: %.2f%% (dense blocks: %i, sparse nodes: %li), hit rate: %.2f%%\n",
+ fprintf( stderr, "node cache: stored: %" PRIdOSMID "(%.2f%%), storage efficiency: %.2f%% (dense blocks: %i, sparse nodes: %li), hit rate: %.2f%%\n",
storedNodes, 100.0f*storedNodes/totalNodes, 100.0f*storedNodes*sizeof(struct ramNode)/cacheUsed,
usedBlocks, sizeSparseTuples,
100.0f*nodesCacheHits/nodesCacheLookups );
@@ -442,33 +422,33 @@ void free_node_ram_cache() {
}
}
-int ram_cache_nodes_set(osmid_t id, double lat, double lon, struct keyval *tags UNUSED) {
+int node_ram_cache::set(osmid_t id, double lat, double lon, struct keyval *tags) {
totalNodes++;
- /* if ALLOC_DENSE and ALLOC_SPARSE are set, send it through
+ /* if ALLOC_DENSE and ALLOC_SPARSE are set, send it through
* ram_nodes_set_dense. If a block is non dense, it will automatically
* get pushed to the sparse cache if a block is sparse and ALLOC_SPARSE is set
*/
if ( (allocStrategy & ALLOC_DENSE) > 0 ) {
- return ram_cache_nodes_set_dense(id, lat, lon, tags);
+ return set_dense(id, lat, lon, tags);
}
if ( (allocStrategy & ALLOC_SPARSE) > 0 )
- return ram_cache_nodes_set_sparse(id, lat, lon, tags);
+ return set_sparse(id, lat, lon, tags);
return 1;
}
-int ram_cache_nodes_get(struct osmNode *out, osmid_t id) {
+int node_ram_cache::get(struct osmNode *out, osmid_t id) {
nodesCacheLookups++;
if ((allocStrategy & ALLOC_DENSE) > 0) {
- if (ram_cache_nodes_get_dense(out,id) == 0) {
+ if (get_dense(out,id) == 0) {
nodesCacheHits++;
- return 0;
+ return 0;
}
}
if ((allocStrategy & ALLOC_SPARSE) > 0) {
- if (ram_cache_nodes_get_sparse(out,id) == 0) {
+ if (get_sparse(out,id) == 0) {
nodesCacheHits++;
- return 0;
+ return 0;
}
}
diff --git a/node-ram-cache.h b/node-ram-cache.h
deleted file mode 100644
index 14e7692..0000000
--- a/node-ram-cache.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/* Implements the node cache in ram.
- *
- * There are two different storage strategies, either optimised
- * for dense storage of node ids, or for sparse storage as well as
- * a strategy to combine both in an optimal way.
-*/
-
-#ifndef NODE_RAM_CACHE_H
-#define NODE_RAM_CACHE_H
-
-#define ALLOC_SPARSE 1
-#define ALLOC_DENSE 2
-#define ALLOC_DENSE_CHUNK 4
-#define ALLOC_LOSSY 8
-
-/* Store +-20,000km Mercator co-ordinates as fixed point 32bit number with maximum precision */
-/* Scale is chosen such that 40,000 * SCALE < 2^32 */
-#define FIXED_POINT
-static int scale = 100;
-#define DOUBLE_TO_FIX(x) ((int)((x) * scale + 0.4))
-#define FIX_TO_DOUBLE(x) (((double)x) / scale)
-
-#define UNUSED __attribute__ ((unused))
-
-struct ramNode {
-#ifdef FIXED_POINT
- int lon;
- int lat;
-#else
- double lon;
- double lat;
-#endif
-};
-
-struct ramNodeID {
- osmid_t id;
- struct ramNode coord;
-};
-
-struct ramNodeBlock {
- struct ramNode *nodes;
- osmid_t block_offset;
- int used;
- int dirty;
-};
-
-
-void init_node_ram_cache(int strategy, int cacheSizeMB, int fixpointscale);
-void free_node_ram_cache();
-int ram_cache_nodes_set(osmid_t id, double lat, double lon, struct keyval *tags UNUSED);
-int ram_cache_nodes_get(struct osmNode *out, osmid_t id);
-
-#endif
diff --git a/node-ram-cache.hpp b/node-ram-cache.hpp
new file mode 100644
index 0000000..d54d504
--- /dev/null
+++ b/node-ram-cache.hpp
@@ -0,0 +1,84 @@
+/* Implements the node cache in ram.
+ *
+ * There are two different storage strategies, either optimised
+ * for dense storage of node ids, or for sparse storage as well as
+ * a strategy to combine both in an optimal way.
+*/
+
+#ifndef NODE_RAM_CACHE_H
+#define NODE_RAM_CACHE_H
+
+#include "osmtypes.hpp"
+
+#include <boost/noncopyable.hpp>
+
+#define ALLOC_SPARSE 1
+#define ALLOC_DENSE 2
+#define ALLOC_DENSE_CHUNK 4
+#define ALLOC_LOSSY 8
+
+/* Store +-20,000km Mercator co-ordinates as fixed point 32bit number with maximum precision */
+#define FIXED_POINT
+
+struct ramNode {
+#ifdef FIXED_POINT
+ int lon;
+ int lat;
+#else
+ double lon;
+ double lat;
+#endif
+};
+
+struct ramNodeID {
+ osmid_t id;
+ struct ramNode coord;
+};
+
+struct ramNodeBlock {
+ struct ramNode *nodes;
+ osmid_t block_offset;
+ int used;
+ int dirty;
+};
+
+struct node_ram_cache : public boost::noncopyable
+{
+ node_ram_cache(int strategy, int cacheSizeMB, int fixpointscale);
+ ~node_ram_cache();
+
+ int set(osmid_t id, double lat, double lon, struct keyval *tags);
+ int get(struct osmNode *out, osmid_t id);
+
+private:
+ void percolate_up( int pos );
+ struct ramNode *next_chunk(size_t count, size_t size);
+ int set_sparse(osmid_t id, double lat, double lon, struct keyval *tags);
+ int set_dense(osmid_t id, double lat, double lon, struct keyval *tags);
+ int get_sparse(struct osmNode *out, osmid_t id);
+ int get_dense(struct osmNode *out, osmid_t id);
+
+ int allocStrategy;
+
+ struct ramNodeBlock *blocks;
+ int usedBlocks;
+ /* Note: maxBlocks *must* be odd, to make sure the priority queue has no nodes with only one child */
+ int maxBlocks;
+ char *blockCache;
+
+ struct ramNodeBlock **queue;
+
+ int scale_;
+
+ struct ramNodeID *sparseBlock;
+ int64_t maxSparseTuples;
+ int64_t sizeSparseTuples;
+
+ int64_t cacheUsed, cacheSize;
+ osmid_t storedNodes, totalNodes;
+ int nodesCacheHits, nodesCacheLookups;
+
+ int warn_node_order;
+};
+
+#endif
diff --git a/options.cpp b/options.cpp
new file mode 100644
index 0000000..3a0a877
--- /dev/null
+++ b/options.cpp
@@ -0,0 +1,543 @@
+#include "options.hpp"
+#include "sprompt.hpp"
+#include "parse.hpp"
+
+#include <getopt.h>
+#ifdef HAVE_LIBGEN_H
+#include <libgen.h>
+#else
+#define basename /*SKIP IT*/
+#endif
+#include <stdio.h>
+#include <string.h>
+#include <stdexcept>
+#include <sstream>
+#include <boost/format.hpp>
+
+namespace
+{
+ const char * short_options = "ab:cd:KhlmMp:suvU:WH:P:i:IE:C:S:e:o:O:xkjGz:r:V";
+ const struct option long_options[] =
+ {
+ {"append", 0, 0, 'a'},
+ {"bbox", 1, 0, 'b'},
+ {"create", 0, 0, 'c'},
+ {"database", 1, 0, 'd'},
+ {"latlong", 0, 0, 'l'},
+ {"verbose", 0, 0, 'v'},
+ {"slim", 0, 0, 's'},
+ {"prefix", 1, 0, 'p'},
+ {"proj", 1, 0, 'E'},
+ {"merc", 0, 0, 'm'},
+ {"oldmerc", 0, 0, 'M'},
+ {"utf8-sanitize", 0, 0, 'u'},
+ {"cache", 1, 0, 'C'},
+ {"username", 1, 0, 'U'},
+ {"password", 0, 0, 'W'},
+ {"host", 1, 0, 'H'},
+ {"port", 1, 0, 'P'},
+ {"tablespace-index", 1, 0, 'i'},
+ {"tablespace-slim-data", 1, 0, 200},
+ {"tablespace-slim-index", 1, 0, 201},
+ {"tablespace-main-data", 1, 0, 202},
+ {"tablespace-main-index", 1, 0, 203},
+ {"help", 0, 0, 'h'},
+ {"style", 1, 0, 'S'},
+ {"expire-tiles", 1, 0, 'e'},
+ {"expire-output", 1, 0, 'o'},
+ {"output", 1, 0, 'O'},
+ {"extra-attributes", 0, 0, 'x'},
+ {"hstore", 0, 0, 'k'},
+ {"hstore-all", 0, 0, 'j'},
+ {"hstore-column", 1, 0, 'z'},
+ {"hstore-match-only", 0, 0, 208},
+ {"hstore-add-index",0,0,211},
+ {"multi-geometry", 0, 0, 'G'},
+ {"keep-coastlines", 0, 0, 'K'},
+ {"input-reader", 1, 0, 'r'},
+ {"version", 0, 0, 'V'},
+ {"disable-parallel-indexing", 0, 0, 'I'},
+ {"cache-strategy", 1, 0, 204},
+ {"number-processes", 1, 0, 205},
+ {"drop", 0, 0, 206},
+ {"unlogged", 0, 0, 207},
+ {"flat-nodes",1,0,209},
+ {"exclude-invalid-polygon",0,0,210},
+ {"tag-transform-script",1,0,212},
+ {0, 0, 0, 0}
+ };
+
+ void short_usage(char *arg0)
+ {
+ throw std::runtime_error((boost::format("Usage error. For further information see:\n\t%1% -h|--help\n") % basename(arg0)).str());
+ }
+
+ void long_usage(char *arg0, bool verbose = false)
+ {
+ const char *name = basename(arg0);
+
+ printf("Usage:\n");
+ printf("\t%s [options] planet.osm\n", name);
+ printf("\t%s [options] planet.osm.{pbf,gz,bz2}\n", name);
+ printf("\t%s [options] file1.osm file2.osm file3.osm\n", name);
+ printf("\nThis will import the data from the OSM file(s) into a PostgreSQL database\n");
+ printf("suitable for use by the Mapnik renderer.\n\n");
+
+ printf("%s", "\
+ Common options:\n\
+ -a|--append Add the OSM file into the database without removing\n\
+ existing data.\n\
+ -c|--create Remove existing data from the database. This is the\n\
+ default if --append is not specified.\n\
+ -l|--latlong Store data in degrees of latitude & longitude.\n\
+ -m|--merc Store data in proper spherical mercator (default).\n\
+ -E|--proj num Use projection EPSG:num.\n\
+ -s|--slim Store temporary data in the database. This greatly\n\
+ reduces the RAM usage but is much slower. This switch is\n\
+ required if you want to update with --append later.\n\
+ -S|--style Location of the style file. Defaults to\n");
+ printf(" %s/default.style.\n", OSM2PGSQL_DATADIR);
+ printf("%s", "\
+ -C|--cache Use up to this many MB for caching nodes (default: 800)\n\
+ \n\
+ Database options:\n\
+ -d|--database The name of the PostgreSQL database to connect\n\
+ to (default: gis).\n\
+ -U|--username PostgreSQL user name (specify passsword in PGPASS\n\
+ environment variable or use -W).\n\
+ -W|--password Force password prompt.\n\
+ -H|--host Database server host name or socket location.\n\
+ -P|--port Database server port.\n");
+
+ if (verbose)
+ {
+ printf("%s", "\
+ Hstore options:\n\
+ -k|--hstore Add tags without column to an additional hstore\n\
+ (key/value) column\n\
+ --hstore-match-only Only keep objects that have a value in one of\n\
+ the columns (default with --hstore is to keep all objects)\n\
+ -j|--hstore-all Add all tags to an additional hstore (key/value) column\n\
+ -z|--hstore-column Add an additional hstore (key/value) column containing\n\
+ all tags that start with the specified string, eg\n\
+ --hstore-column \"name:\" will produce an extra hstore\n\
+ column that contains all name:xx tags\n\
+ --hstore-add-index Add index to hstore column.\n\
+ \n\
+ Obsolete options:\n\
+ -u|--utf8-sanitize Repair bad UTF8 input data (present in planet\n\
+ dumps prior to August 2007). Adds about 10% overhead.\n\
+ -M|--oldmerc Store data in the legacy OSM mercator format\n\
+ \n\
+ Performance options:\n\
+ -i|--tablespace-index The name of the PostgreSQL tablespace where\n\
+ all indexes will be created.\n\
+ The following options allow more fine-grained control:\n\
+ --tablespace-main-data tablespace for main tables\n\
+ --tablespace-main-index tablespace for main table indexes\n\
+ --tablespace-slim-data tablespace for slim mode tables\n\
+ --tablespace-slim-index tablespace for slim mode indexes\n\
+ (if unset, use db's default; -i is equivalent to setting\n\
+ --tablespace-main-index and --tablespace-slim-index)\n\
+ --drop only with --slim: drop temporary tables after import \n\
+ (no updates are possible).\n\
+ --number-processes Specifies the number of parallel processes \n\
+ used for certain operations (default is 1).\n\
+ -I|--disable-parallel-indexing Disable indexing all tables concurrently.\n\
+ --unlogged Use unlogged tables (lost on crash but faster). \n\
+ Requires PostgreSQL 9.1.\n\
+ --cache-strategy Specifies the method used to cache nodes in ram.\n\
+ Available options are:\n\
+ dense: caching strategy optimised for full planet import\n\
+ chunk: caching strategy optimised for non-contiguous \n\
+ memory allocation\n\
+ sparse: caching strategy optimised for small extracts\n\
+ optimized: automatically combines dense and sparse \n\
+ strategies for optimal storage efficiency. This may\n\
+ us twice as much virtual memory, but no more physical \n\
+ memory.\n");
+ #ifdef __amd64__
+ printf(" The default is \"optimized\"\n");
+ #else
+ /* use "chunked" as a default in 32 bit compilations, as it is less wasteful of virtual memory than "optimized"*/
+ printf(" The default is \"sparse\"\n");
+ #endif
+ printf("%s", "\
+ --flat-nodes Specifies the flat file to use to persistently store node \n\
+ information in slim mode instead of in PostgreSQL.\n\
+ This file is a single > 16Gb large file. Only recommended\n\
+ for full planet imports. Default is disabled.\n\
+ \n\
+ Expiry options:\n\
+ -e|--expire-tiles [min_zoom-]max_zoom Create a tile expiry list.\n\
+ -o|--expire-output filename Output filename for expired tiles list.\n\
+ \n\
+ Other options:\n\
+ -b|--bbox Apply a bounding box filter on the imported data\n\
+ Must be specified as: minlon,minlat,maxlon,maxlat\n\
+ e.g. --bbox -0.5,51.25,0.5,51.75\n\
+ -p|--prefix Prefix for table names (default planet_osm)\n\
+ -r|--input-reader Input frontend.\n\
+ libxml2 - Parse XML using libxml2. (default)\n");
+ #ifdef BUILD_READER_PBF
+ printf(" pbf - OSM binary format.\n");
+ #endif
+ printf("\
+ -O|--output Output backend.\n\
+ pgsql - Output to a PostGIS database. (default)\n\
+ gazetteer - Output to a PostGIS database for Nominatim\n\
+ null - No output. Useful for testing.\n");
+ #ifdef HAVE_LUA
+ printf("\
+ --tag-transform-script Specify a lua script to handle tag filtering and normalisation\n\
+ The script contains callback functions for nodes, ways and relations, which each\n\
+ take a set of tags and returns a transformed, filtered set of tags which are then\n\
+ written to the database.\n");
+ #endif
+ printf("\
+ -x|--extra-attributes\n\
+ Include attributes for each object in the database.\n\
+ This includes the username, userid, timestamp and version.\n\
+ Requires additional entries in your style file.\n\
+ -G|--multi-geometry Generate multi-geometry features in postgresql tables.\n\
+ -K|--keep-coastlines Keep coastline data rather than filtering it out.\n\
+ By default natural=coastline tagged data will be discarded\n\
+ because renderers usually have shape files for them.\n\
+ --exclude-invalid-polygon do not import polygons with invalid geometries.\n\
+ -h|--help Help information.\n\
+ -v|--verbose Verbose output.\n");
+ }
+ else
+ {
+ printf("\n");
+ printf("A typical command to import a full planet is\n");
+ printf(" %s -c -d gis --slim -C <cache size> -k \\\n", name);
+ printf(" --flat-nodes <flat nodes> planet-latest.osm.pbf\n");
+ printf("where\n");
+ printf(" <cache size> is 20000 on machines with 24GB or more RAM \n");
+ printf(" or about 75%% of memory in MB on machines with less\n");
+ printf(" <flat nodes> is a location where a 19GB file can be saved.\n");
+ printf("\n");
+ printf("A typical command to update a database imported with the above command is\n");
+ printf(" osmosis --rri workingDirectory=<osmosis dir> --simc --wx - \\\n");
+ printf(" | %s -a -d gis --slim -k --flat-nodes <flat nodes> \n", name);
+ printf("where\n");
+ printf(" <flat nodes> is the same location as above.\n");
+ printf(" <osmosis dir> is the location osmosis replication was initialized to.\n");
+ printf("\nRun %s --help --verbose (-h -v) for a full list of options.\n", name);
+ }
+
+ }
+
+std::string build_conninfo(const std::string &db,
+ const boost::optional<std::string> &username,
+ const boost::optional<std::string> &password,
+ const boost::optional<std::string> &host,
+ const std::string &port)
+{
+ std::ostringstream out;
+
+ out << "dbname='" << db << "'";
+
+ if (username) {
+ out << " user='" << *username << "'";
+ }
+ if (password) {
+ out << " password='" << *password << "'";
+ }
+ if (host) {
+ out << " host='" << *host << "'";
+ }
+ out << " port='" << port << "'";
+
+ return out.str();
+}
+} // anonymous namespace
+
+
+options_t::options_t():
+ conninfo(""), prefix("planet_osm"), scale(DEFAULT_SCALE), projection(new reprojection(PROJ_SPHERE_MERC)), append(0), slim(0),
+ cache(800), tblsmain_index(boost::none), tblsslim_index(boost::none), tblsmain_data(boost::none), tblsslim_data(boost::none), style(OSM2PGSQL_DATADIR "/default.style"),
+ expire_tiles_zoom(-1), expire_tiles_zoom_min(-1), expire_tiles_filename("dirty_tiles"), hstore_mode(HSTORE_NONE), enable_hstore_index(0),
+ enable_multi(false), hstore_columns(), keep_coastlines(0), parallel_indexing(1),
+ #ifdef __amd64__
+ alloc_chunkwise(ALLOC_SPARSE | ALLOC_DENSE),
+ #else
+ alloc_chunkwise(ALLOC_SPARSE),
+ #endif
+ num_procs(1), droptemp(0), unlogged(0), hstore_match_only(0), flat_node_cache_enabled(0), excludepoly(0), flat_node_file(boost::none),
+ tag_transform_script(boost::none), tag_transform_node_func(boost::none), tag_transform_way_func(boost::none),
+ tag_transform_rel_func(boost::none), tag_transform_rel_mem_func(boost::none),
+ create(0), sanitize(0), long_usage_bool(0), pass_prompt(0), db("gis"), username(boost::none), host(boost::none),
+ password(boost::none), port("5432"), output_backend("pgsql"), input_reader("auto"), bbox(boost::none), extra_attributes(0), verbose(0)
+{
+
+}
+
+options_t::~options_t()
+{
+}
+
+options_t options_t::parse(int argc, char *argv[])
+{
+ options_t options;
+ const char *temparg;
+ int c;
+
+ //keep going while there are args left to handle
+ // note: optind would seem to need to be set to 1, but that gives valgrind
+ // errors - setting it to zero seems to work, though. see
+ // http://stackoverflow.com/questions/15179963/is-it-possible-to-repeat-getopt#15179990
+ optind = 0;
+ while(-1 != (c = getopt_long(argc, argv, short_options, long_options, NULL))) {
+
+ //handle the current arg
+ switch (c) {
+ case 'a':
+ options.append = 1;
+ break;
+ case 'b':
+ options.bbox = optarg;
+ break;
+ case 'c':
+ options.create = 1;
+ break;
+ case 'v':
+ options.verbose = 1;
+ break;
+ case 's':
+ options.slim = 1;
+ break;
+ case 'K':
+ options.keep_coastlines = 1;
+ break;
+ case 'u':
+ options.sanitize = 1;
+ break;
+ case 'l':
+ options.projection.reset(new reprojection(PROJ_LATLONG));
+ break;
+ case 'm':
+ options.projection.reset(new reprojection(PROJ_SPHERE_MERC));
+ break;
+ case 'M':
+ options.projection.reset(new reprojection(PROJ_MERC));
+ break;
+ case 'E':
+ options.projection.reset(new reprojection(-atoi(optarg)));
+ break;
+ case 'p':
+ options.prefix = optarg;
+ break;
+ case 'd':
+ options.db = optarg;
+ break;
+ case 'C':
+ options.cache = atoi(optarg);
+ break;
+ case 'U':
+ options.username = optarg;
+ break;
+ case 'W':
+ options.pass_prompt = 1;
+ break;
+ case 'H':
+ options.host = optarg;
+ break;
+ case 'P':
+ options.port = optarg;
+ break;
+ case 'S':
+ options.style = optarg;
+ break;
+ case 'i':
+ options.tblsmain_index = options.tblsslim_index = optarg;
+ break;
+ case 200:
+ options.tblsslim_data = optarg;
+ break;
+ case 201:
+ options.tblsslim_index = optarg;
+ break;
+ case 202:
+ options.tblsmain_data = optarg;
+ break;
+ case 203:
+ options.tblsmain_index = optarg;
+ break;
+ case 'e':
+ options.expire_tiles_zoom_min = atoi(optarg);
+ temparg = strchr(optarg, '-');
+ if (temparg)
+ options.expire_tiles_zoom = atoi(temparg + 1);
+ if (options.expire_tiles_zoom < options.expire_tiles_zoom_min)
+ options.expire_tiles_zoom = options.expire_tiles_zoom_min;
+ break;
+ case 'o':
+ options.expire_tiles_filename = optarg;
+ break;
+ case 'O':
+ options.output_backend = optarg;
+ break;
+ case 'x':
+ options.extra_attributes = 1;
+ break;
+ case 'k':
+ if (options.hstore_mode != HSTORE_NONE) {
+ throw std::runtime_error("ERROR: You can not specify both --hstore (-k) and --hstore-all (-j)\n");
+ }
+ options.hstore_mode = HSTORE_NORM;
+ break;
+ case 208:
+ options.hstore_match_only = 1;
+ break;
+ case 'j':
+ if (options.hstore_mode != HSTORE_NONE) {
+ throw std::runtime_error("ERROR: You can not specify both --hstore (-k) and --hstore-all (-j)\n");
+ }
+ options.hstore_mode = HSTORE_ALL;
+ break;
+ case 'z':
+ options.hstore_columns.push_back(optarg);
+ break;
+ case 'G':
+ options.enable_multi = true;
+ break;
+ case 'r':
+ options.input_reader = optarg;
+ break;
+ case 'h':
+ options.long_usage_bool = 1;
+ break;
+ case 'I':
+#ifdef HAVE_PTHREAD
+ options.parallel_indexing = 0;
+#endif
+ break;
+ case 204:
+ if (strcmp(optarg, "dense") == 0)
+ options.alloc_chunkwise = ALLOC_DENSE;
+ else if (strcmp(optarg, "chunk") == 0)
+ options.alloc_chunkwise = ALLOC_DENSE | ALLOC_DENSE_CHUNK;
+ else if (strcmp(optarg, "sparse") == 0)
+ options.alloc_chunkwise = ALLOC_SPARSE;
+ else if (strcmp(optarg, "optimized") == 0)
+ options.alloc_chunkwise = ALLOC_DENSE | ALLOC_SPARSE;
+ else {
+ throw std::runtime_error((boost::format("ERROR: Unrecognized cache strategy %1%.\n") % optarg).str());
+ }
+ break;
+ case 205:
+#ifdef HAVE_FORK
+ options.num_procs = atoi(optarg);
+#else
+ fprintf(stderr, "WARNING: osm2pgsql was compiled without fork, only using one process!\n");
+#endif
+ break;
+ case 206:
+ options.droptemp = 1;
+ break;
+ case 207:
+ options.unlogged = 1;
+ break;
+ case 209:
+ options.flat_node_cache_enabled = 1;
+ options.flat_node_file = optarg;
+ break;
+ case 210:
+ options.excludepoly = 1;
+ break;
+ case 211:
+ options.enable_hstore_index = 1;
+ break;
+ case 212:
+ options.tag_transform_script = optarg;
+ break;
+ case 'V':
+ exit (EXIT_SUCCESS);
+ break;
+ case '?':
+ default:
+ short_usage(argv[0]);
+ break;
+ }
+ } //end while
+
+ //they were looking for usage info
+ if (options.long_usage_bool) {
+ long_usage(argv[0], options.verbose);
+ }
+
+ //we require some input files!
+ if (argc == optind) {
+ short_usage(argv[0]);
+ }
+
+ //get the input files
+ while (optind < argc) {
+ options.input_files.push_back(std::string(argv[optind]));
+ optind++;
+ }
+
+ if (options.append && options.create) {
+ throw std::runtime_error("Error: --append and --create options can not be used at the same time!\n");
+ }
+
+ if (options.droptemp && !options.slim) {
+ throw std::runtime_error("Error: --drop only makes sense with --slim.\n");
+ }
+
+ if (options.unlogged && !options.create) {
+ fprintf(stderr, "Warning: --unlogged only makes sense with --create; ignored.\n");
+ options.unlogged = 0;
+ }
+
+ if (options.hstore_mode == HSTORE_NONE && options.hstore_columns.size() == 0 && options.hstore_match_only) {
+ fprintf(stderr, "Warning: --hstore-match-only only makes sense with --hstore, --hstore-all, or --hstore-column; ignored.\n");
+ options.hstore_match_only = 0;
+ }
+
+ if (options.enable_hstore_index && options.hstore_mode == HSTORE_NONE && options.hstore_columns.size() == 0) {
+ fprintf(stderr, "Warning: --hstore-add-index only makes sense with hstore enabled.\n");
+ options.enable_hstore_index = 0;
+ }
+
+ if (options.cache < 0)
+ options.cache = 0;
+
+ if (options.cache == 0) {
+ fprintf(stderr, "WARNING: ram cache is disabled. This will likely slow down processing a lot.\n\n");
+ }
+ if (sizeof(int*) == 4 && options.slim != 1) {
+ fprintf(stderr, "\n!! You are running this on 32bit system, so at most\n");
+ fprintf(stderr, "!! 3GB of RAM can be used. If you encounter unexpected\n");
+ fprintf(stderr, "!! exceptions during import, you should try running in slim\n");
+ fprintf(stderr, "!! mode using parameter -s.\n");
+ }
+
+ if (options.pass_prompt) {
+ char *prompt = simple_prompt("Password:", 100, 0);
+ if (prompt == NULL) {
+ options.password = boost::none;
+ } else {
+ options.password = std::string(prompt);
+ }
+ } else {
+ char *pgpass = getenv("PGPASS");
+ if (pgpass == NULL) {
+ options.password = boost::none;
+ } else {
+ options.password = std::string(pgpass);
+ }
+ }
+
+ if (options.num_procs < 1)
+ options.num_procs = 1;
+
+ //NOTE: this is hugely important if you set it inappropriately and are are caching nodes
+ //you could get overflow when working with larger coordinates (mercator) and larger scales
+ options.scale = (options.projection->get_proj_id() == PROJ_LATLONG) ? 10000000 : 100;
+ options.conninfo = build_conninfo(options.db, options.username, options.password, options.host, options.port);
+
+ return options;
+}
diff --git a/options.hpp b/options.hpp
new file mode 100644
index 0000000..84cb9cf
--- /dev/null
+++ b/options.hpp
@@ -0,0 +1,88 @@
+#ifndef OPTION_H
+#define OPTION_H
+
+#include "keyvals.hpp"
+#include "node-ram-cache.hpp"
+#include "reprojection.hpp"
+
+#include <vector>
+#include <string>
+#include <boost/shared_ptr.hpp>
+#include <boost/optional.hpp>
+
+/* Variants for generation of hstore column */
+/* No hstore column */
+#define HSTORE_NONE 0
+/* create a hstore column for all tags which do not have an exclusive column */
+#define HSTORE_NORM 1
+/* create a hstore column for all tags */
+#define HSTORE_ALL 2
+
+/* Scale is chosen such that 40,000 * SCALE < 2^32 */
+static const int DEFAULT_SCALE = 100;
+
+//TODO: GO THROUGH AND UPDATE TO BOOL WHERE MEMBER DENOTES ONLY ON OR OFF OPTION
+struct options_t {
+public:
+ /* construct with sensible defaults */
+ options_t();
+ virtual ~options_t();
+
+ static options_t parse(int argc, char *argv[]);
+
+ std::string conninfo; /* Connection info string */
+ std::string prefix; /* prefix for table names */
+ int scale; /* scale for converting coordinates to fixed point */
+ boost::shared_ptr<reprojection> projection; /* SRS of projection */
+ bool append; /* Append to existing data */
+ bool slim; /* In slim mode */
+ int cache; /* Memory usable for cache in MB */
+ boost::optional<std::string> tblsmain_index; /* Pg Tablespace to store indexes on main tables (no default TABLESPACE)*/
+ boost::optional<std::string> tblsslim_index; /* Pg Tablespace to store indexes on slim tables (no default TABLESPACE)*/
+ boost::optional<std::string> tblsmain_data; /* Pg Tablespace to store main tables (no default TABLESPACE)*/
+ boost::optional<std::string> tblsslim_data; /* Pg Tablespace to store slim tables (no default TABLESPACE)*/
+ std::string style; /* style file to use */
+ int expire_tiles_zoom; /* Zoom level for tile expiry list */
+ int expire_tiles_zoom_min; /* Minimum zoom level for tile expiry list */
+ std::string expire_tiles_filename; /* File name to output expired tiles list to */
+ int hstore_mode; /* add an additional hstore column with objects key/value pairs */
+ int enable_hstore_index; /* add an index on the hstore column */
+ bool enable_multi; /* Output multi-geometries intead of several simple geometries */
+ std::vector<std::string> hstore_columns; /* list of columns that should be written into their own hstore column */
+ int keep_coastlines;
+ int parallel_indexing;
+ int alloc_chunkwise;
+ int num_procs;
+ int droptemp; /* drop slim mode temp tables after act */
+ int unlogged; /* use unlogged tables where possible */
+ int hstore_match_only; /* only copy rows that match an explicitly listed key */
+ int flat_node_cache_enabled;
+ int excludepoly;
+ boost::optional<std::string> flat_node_file;
+ boost::optional<std::string> tag_transform_script,
+ tag_transform_node_func, // these options allow you to control the name of the
+ tag_transform_way_func, // Lua functions which get called in the tag transform
+ tag_transform_rel_func, // script. this is mostly useful in with the "multi"
+ tag_transform_rel_mem_func; // output so that a single script file can be used.
+
+ int create;
+ int sanitize;
+ int long_usage_bool;
+ int pass_prompt;
+ std::string db;
+ boost::optional<std::string> username;
+ boost::optional<std::string> host;
+ boost::optional<std::string> password;
+ std::string port;
+ std::string output_backend ;
+ std::string input_reader;
+ boost::optional<std::string> bbox;
+ int extra_attributes;
+ int verbose;
+
+ std::vector<std::string> input_files;
+private:
+
+};
+
+#endif
diff --git a/osm2pgsql-svn.sh b/osm2pgsql-svn.sh
deleted file mode 100755
index 12c5486..0000000
--- a/osm2pgsql-svn.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/sh
-
-DATE=$(date +%Y%m%d)
-MODULE="$(basename $0 -svn.sh)"
-SVNROOT=http://svn.openstreetmap.org/applications/utils/export/osm2pgsql/
-
-set -x
-rm -rf $MODULE
-
-svn export $SVNROOT $MODULE/
-
-## tar it up
-tar cjf $MODULE-${DATE}svn.tar.bz2 $MODULE
-
-## cleanup
-rm -rf $MODULE
-
diff --git a/osm2pgsql.c b/osm2pgsql.c
deleted file mode 100644
index 87a400d..0000000
--- a/osm2pgsql.c
+++ /dev/null
@@ -1,796 +0,0 @@
-/*
-#-----------------------------------------------------------------------------
-# osm2pgsql - converts planet.osm file into PostgreSQL
-# compatible output suitable to be rendered by mapnik
-# Use: osm2pgsql planet.osm.bz2
-#-----------------------------------------------------------------------------
-# Original Python implementation by Artem Pavlenko
-# Re-implementation by Jon Burgess, Copyright 2006
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#-----------------------------------------------------------------------------
-*/
-
-#include "config.h"
-
-#include <stdio.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <getopt.h>
-#include <libgen.h>
-#include <time.h>
-
-#include <libpq-fe.h>
-
-#include <libxml/xmlstring.h>
-#include <libxml/xmlreader.h>
-
-#include "osmtypes.h"
-#include "build_geometry.h"
-#include "middle-pgsql.h"
-#include "middle-ram.h"
-#include "node-ram-cache.h"
-#include "output-pgsql.h"
-#include "output-gazetteer.h"
-#include "output-null.h"
-#include "sanitizer.h"
-#include "reprojection.h"
-#include "text-tree.h"
-#include "input.h"
-#include "sprompt.h"
-#include "parse-xml2.h"
-#include "parse-primitive.h"
-#include "parse-o5m.h"
-
-#ifdef BUILD_READER_PBF
-# include "parse-pbf.h"
-#endif
-
-#define INIT_MAX_MEMBERS 64
-#define INIT_MAX_NODES 4096
-
-int verbose;
-
-/* Data structure carrying all parsing related variables */
-static struct osmdata_t osmdata = {
- .filetype = FILETYPE_NONE,
- .action = ACTION_NONE,
- .bbox = NULL
-};
-
-
-static int parse_bbox(struct osmdata_t *osmdata)
-{
- int n;
-
- if (!osmdata->bbox)
- return 0;
-
- n = sscanf(osmdata->bbox, "%lf,%lf,%lf,%lf", &(osmdata->minlon), &(osmdata->minlat), &(osmdata->maxlon), &(osmdata->maxlat));
- if (n != 4) {
- fprintf(stderr, "Bounding box must be specified like: minlon,minlat,maxlon,maxlat\n");
- return 1;
- }
- if (osmdata->maxlon <= osmdata->minlon) {
- fprintf(stderr, "Bounding box failed due to maxlon <= minlon\n");
- return 1;
- }
- if (osmdata->maxlat <= osmdata->minlat) {
- fprintf(stderr, "Bounding box failed due to maxlat <= minlat\n");
- return 1;
- }
- fprintf(stderr, "Applying Bounding box: %f,%f to %f,%f\n", osmdata->minlon, osmdata->minlat, osmdata->maxlon, osmdata->maxlat);
- return 0;
-}
-
-
-
-void exit_nicely()
-{
- fprintf(stderr, "Error occurred, cleaning up\n");
- osmdata.out->cleanup();
- exit(1);
-}
-
-static void short_usage(char *arg0)
-{
- const char *name = basename(arg0);
-
- fprintf(stderr, "Usage error. For further information see:\n");
- fprintf(stderr, "\t%s -h|--help\n", name);
-}
-
-static void long_usage(char *arg0)
-{
- const char *name = basename(arg0);
-
- printf("Usage:\n");
- printf("\t%s [options] planet.osm\n", name);
- printf("\t%s [options] planet.osm.{pbf,gz,bz2}\n", name);
- printf("\t%s [options] file1.osm file2.osm file3.osm\n", name);
- printf("\nThis will import the data from the OSM file(s) into a PostgreSQL database\n");
- printf("suitable for use by the Mapnik renderer.\n\n");
-
- printf("%s", "\
-Common options:\n\
- -a|--append Add the OSM file into the database without removing\n\
- existing data.\n\
- -c|--create Remove existing data from the database. This is the\n\
- default if --append is not specified.\n\
- -l|--latlong Store data in degrees of latitude & longitude.\n\
- -m|--merc Store data in proper spherical mercator (default).\n\
- -E|--proj num Use projection EPSG:num.\n\
- -s|--slim Store temporary data in the database. This greatly\n\
- reduces the RAM usage but is much slower. This switch is\n\
- required if you want to update with --append later.\n\
- -S|--style Location of the style file. Defaults to\n");
- printf(" %s/default.style.\n", OSM2PGSQL_DATADIR);
- printf("%s", "\
- -C|--cache Use up to this many MB for caching nodes (default: 800)\n\
-\n\
-Database options:\n\
- -d|--database The name of the PostgreSQL database to connect\n\
- to (default: gis).\n\
- -U|--username PostgreSQL user name (specify passsword in PGPASS\n\
- environment variable or use -W).\n\
- -W|--password Force password prompt.\n\
- -H|--host Database server host name or socket location.\n\
- -P|--port Database server port.\n");
-
- if (verbose)
- {
- printf("%s", "\
-Hstore options:\n\
- -k|--hstore Add tags without column to an additional hstore\n\
- (key/value) column\n\
- --hstore-match-only Only keep objects that have a value in one of\n\
- the columns (default with --hstore is to keep all objects)\n\
- -j|--hstore-all Add all tags to an additional hstore (key/value) column\n\
- -z|--hstore-column Add an additional hstore (key/value) column containing\n\
- all tags that start with the specified string, eg\n\
- --hstore-column \"name:\" will produce an extra hstore\n\
- column that contains all name:xx tags\n\
- --hstore-add-index Add index to hstore column.\n\
-\n\
-Obsolete options:\n\
- -u|--utf8-sanitize Repair bad UTF8 input data (present in planet\n\
- dumps prior to August 2007). Adds about 10% overhead.\n\
- -M|--oldmerc Store data in the legacy OSM mercator format\n\
-\n\
-Performance options:\n\
- -i|--tablespace-index The name of the PostgreSQL tablespace where\n\
- all indexes will be created.\n\
- The following options allow more fine-grained control:\n\
- --tablespace-main-data tablespace for main tables\n\
- --tablespace-main-index tablespace for main table indexes\n\
- --tablespace-slim-data tablespace for slim mode tables\n\
- --tablespace-slim-index tablespace for slim mode indexes\n\
- (if unset, use db's default; -i is equivalent to setting\n\
- --tablespace-main-index and --tablespace-slim-index)\n\
- --drop only with --slim: drop temporary tables after import \n\
- (no updates are possible).\n\
- --number-processes Specifies the number of parallel processes \n\
- used for certain operations (default is 1).\n\
- -I|--disable-parallel-indexing Disable indexing all tables concurrently.\n\
- --unlogged Use unlogged tables (lost on crash but faster). \n\
- Requires PostgreSQL 9.1.\n\
- --cache-strategy Specifies the method used to cache nodes in ram.\n\
- Available options are:\n\
- dense: caching strategy optimised for full planet import\n\
- chunk: caching strategy optimised for non-contiguous \n\
- memory allocation\n\
- sparse: caching strategy optimised for small extracts\n\
- optimized: automatically combines dense and sparse \n\
- strategies for optimal storage efficiency. This may\n\
- us twice as much virtual memory, but no more physical \n\
- memory.\n");
-#ifdef __amd64__
- printf(" The default is \"optimized\"\n");
-#else
- /* use "chunked" as a default in 32 bit compilations, as it is less wasteful of virtual memory than "optimized"*/
- printf(" The default is \"sparse\"\n");
-#endif
- printf("%s", "\
- --flat-nodes Specifies the flat file to use to persistently store node \n\
- information in slim mode instead of in PostgreSQL.\n\
- This file is a single > 16Gb large file. Only recommended\n\
- for full planet imports. Default is disabled.\n\
-\n\
-Expiry options:\n\
- -e|--expire-tiles [min_zoom-]max_zoom Create a tile expiry list.\n\
- -o|--expire-output filename Output filename for expired tiles list.\n\
-\n\
-Other options:\n\
- -b|--bbox Apply a bounding box filter on the imported data\n\
- Must be specified as: minlon,minlat,maxlon,maxlat\n\
- e.g. --bbox -0.5,51.25,0.5,51.75\n\
- -p|--prefix Prefix for table names (default planet_osm)\n\
- -r|--input-reader Input frontend.\n\
- libxml2 - Parse XML using libxml2. (default)\n\
- primitive - Primitive XML parsing.\n");
-#ifdef BUILD_READER_PBF
- printf(" pbf - OSM binary format.\n");
-#endif
- printf("\
- -O|--output Output backend.\n\
- pgsql - Output to a PostGIS database. (default)\n\
- gazetteer - Output to a PostGIS database for Nominatim\n\
- null - No output. Useful for testing.\n");
-#ifdef HAVE_LUA
- printf("\
- --tag-transform-script Specify a lua script to handle tag filtering and normalisation\n\
- The script contains callback functions for nodes, ways and relations, which each\n\
- take a set of tags and returns a transformed, filtered set of tags which are then\n\
- written to the database.\n");
-#endif
- printf("\
- -x|--extra-attributes\n\
- Include attributes for each object in the database.\n\
- This includes the username, userid, timestamp and version.\n\
- Requires additional entries in your style file.\n\
- -G|--multi-geometry Generate multi-geometry features in postgresql tables.\n\
- -K|--keep-coastlines Keep coastline data rather than filtering it out.\n\
- By default natural=coastline tagged data will be discarded\n\
- because renderers usually have shape files for them.\n\
- --exclude-invalid-polygon do not import polygons with invalid geometries.\n\
- -h|--help Help information.\n\
- -v|--verbose Verbose output.\n");
- }
- else
- {
- printf("\n");
- printf("A typical command to import a full planet is\n");
- printf(" %s -c -d gis --slim -C <cache size> -k \\\n", name);
- printf(" --flat-nodes <flat nodes> planet-latest.osm.pbf\n");
- printf("where\n");
- printf(" <cache size> is 20000 on machines with 24GB or more RAM \n");
- printf(" or about 75%% of memory in MB on machines with less\n");
- printf(" <flat nodes> is a location where a 19GB file can be saved.\n");
- printf("\n");
- printf("A typical command to update a database imported with the above command is\n");
- printf(" osmosis --rri workingDirectory=<osmosis dir> --simc --wx - \\\n");
- printf(" | %s -a -d gis --slim -k --flat-nodes <flat nodes> \n", name);
- printf("where\n");
- printf(" <flat nodes> is the same location as above.\n");
- printf(" <osmosis dir> is the location osmosis replication was initialized to.\n");
- printf("\nRun %s --help --verbose (-h -v) for a full list of options.\n", name);
- }
-
-}
-
-const char *build_conninfo(const char *db, const char *username, const char *password, const char *host, const char *port)
-{
- static char conninfo[1024];
-
- conninfo[0]='\0';
- strcat(conninfo, "dbname='");
- strcat(conninfo, db);
- strcat(conninfo, "'");
-
- if (username) {
- strcat(conninfo, " user='");
- strcat(conninfo, username);
- strcat(conninfo, "'");
- }
- if (password) {
- strcat(conninfo, " password='");
- strcat(conninfo, password);
- strcat(conninfo, "'");
- }
- if (host) {
- strcat(conninfo, " host='");
- strcat(conninfo, host);
- strcat(conninfo, "'");
- }
- if (port) {
- strcat(conninfo, " port='");
- strcat(conninfo, port);
- strcat(conninfo, "'");
- }
-
- return conninfo;
-}
-
-void realloc_nodes(struct osmdata_t *osmdata)
-{
- if( osmdata->nd_max == 0 )
- osmdata->nd_max = INIT_MAX_NODES;
- else
- osmdata->nd_max <<= 1;
-
- osmdata->nds = realloc( osmdata->nds, osmdata->nd_max * sizeof( osmdata->nds[0] ) );
- if( !osmdata->nds )
- {
- fprintf( stderr, "Failed to expand node list to %d\n", osmdata->nd_max );
- exit_nicely();
- }
-}
-
-void realloc_members(struct osmdata_t *osmdata)
-{
- if( osmdata->member_max == 0 )
- osmdata->member_max = INIT_MAX_NODES;
- else
- osmdata->member_max <<= 1;
-
- osmdata->members = realloc( osmdata->members, osmdata->member_max * sizeof( osmdata->members[0] ) );
- if( !osmdata->members )
- {
- fprintf( stderr, "Failed to expand member list to %d\n", osmdata->member_max );
- exit_nicely();
- }
-}
-
-void resetMembers(struct osmdata_t *osmdata)
-{
- unsigned i;
- for(i = 0; i < osmdata->member_count; i++ )
- free( osmdata->members[i].role );
-}
-
-void printStatus(struct osmdata_t *osmdata)
-{
- time_t now;
- time_t end_nodes;
- time_t end_way;
- time_t end_rel;
- time(&now);
- end_nodes = osmdata->start_way > 0 ? osmdata->start_way : now;
- end_way = osmdata->start_rel > 0 ? osmdata->start_rel : now;
- end_rel = now;
- fprintf(stderr, "\rProcessing: Node(%" PRIdOSMID "k %.1fk/s) Way(%" PRIdOSMID "k %.2fk/s) Relation(%" PRIdOSMID " %.2f/s)",
- osmdata->count_node/1000,
- (double)osmdata->count_node/1000.0/((int)(end_nodes - osmdata->start_node) > 0 ? (double)(end_nodes - osmdata->start_node) : 1.0),
- osmdata->count_way/1000,
- osmdata->count_way > 0 ? (double)osmdata->count_way/1000.0/
- ((double)(end_way - osmdata->start_way) > 0.0 ? (double)(end_way - osmdata->start_way) : 1.0) : 0.0,
- osmdata->count_rel,
- osmdata->count_rel > 0 ? (double)osmdata->count_rel/
- ((double)(end_rel - osmdata->start_rel) > 0.0 ? (double)(end_rel - osmdata->start_rel) : 1.0) : 0.0);
-}
-
-int node_wanted(struct osmdata_t *osmdata, double lat, double lon)
-{
- if (!osmdata->bbox)
- return 1;
-
- if (lat < osmdata->minlat || lat > osmdata->maxlat)
- return 0;
- if (lon < osmdata->minlon || lon > osmdata->maxlon)
- return 0;
- return 1;
-}
-
-int main(int argc, char *argv[])
-{
- int append=0;
- int create=0;
- int slim=0;
- int sanitize=0;
- int long_usage_bool=0;
- int pass_prompt=0;
- int projection = PROJ_SPHERE_MERC;
- int expire_tiles_zoom = -1;
- int expire_tiles_zoom_min = -1;
- int enable_hstore = HSTORE_NONE;
- int enable_hstore_index = 0;
- int hstore_match_only = 0;
- int enable_multi = 0;
- int parallel_indexing = 1;
- int flat_node_cache_enabled = 0;
-#ifdef __amd64__
- int alloc_chunkwise = ALLOC_SPARSE | ALLOC_DENSE;
-#else
- int alloc_chunkwise = ALLOC_SPARSE;
-#endif
- int num_procs = 1;
- int droptemp = 0;
- int unlogged = 0;
- int excludepoly = 0;
- time_t start, end;
- time_t overall_start, overall_end;
- time_t now;
- time_t end_nodes;
- time_t end_way;
- time_t end_rel;
- const char *expire_tiles_filename = "dirty_tiles";
- const char *db = "gis";
- const char *username=NULL;
- const char *host=NULL;
- const char *password=NULL;
- const char *port = "5432";
- const char *tblsmain_index = NULL; /* no default TABLESPACE for index on main tables */
- const char *tblsmain_data = NULL; /* no default TABLESPACE for main tables */
- const char *tblsslim_index = NULL; /* no default TABLESPACE for index on slim mode tables */
- const char *tblsslim_data = NULL; /* no default TABLESPACE for slim mode tables */
- const char *conninfo = NULL;
- const char *prefix = "planet_osm";
- const char *style = OSM2PGSQL_DATADIR "/default.style";
- const char *temparg;
- const char *output_backend = "pgsql";
- const char *input_reader = "auto";
- const char **hstore_columns = NULL;
- const char *flat_nodes_file = NULL;
- const char *tag_transform_script = NULL;
- int n_hstore_columns = 0;
- int keep_coastlines=0;
- int cache = 800;
- struct output_options options;
- PGconn *sql_conn;
-
- int (*streamFile)(char *, int, struct osmdata_t *);
-
- fprintf(stderr, "osm2pgsql SVN version %s (%lubit id space)\n\n", VERSION, 8 * sizeof(osmid_t));
-
- while (1) {
- int c, option_index = 0;
- static struct option long_options[] = {
- {"append", 0, 0, 'a'},
- {"bbox", 1, 0, 'b'},
- {"create", 0, 0, 'c'},
- {"database", 1, 0, 'd'},
- {"latlong", 0, 0, 'l'},
- {"verbose", 0, 0, 'v'},
- {"slim", 0, 0, 's'},
- {"prefix", 1, 0, 'p'},
- {"proj", 1, 0, 'E'},
- {"merc", 0, 0, 'm'},
- {"oldmerc", 0, 0, 'M'},
- {"utf8-sanitize", 0, 0, 'u'},
- {"cache", 1, 0, 'C'},
- {"username", 1, 0, 'U'},
- {"password", 0, 0, 'W'},
- {"host", 1, 0, 'H'},
- {"port", 1, 0, 'P'},
- {"tablespace-index", 1, 0, 'i'},
- {"tablespace-slim-data", 1, 0, 200},
- {"tablespace-slim-index", 1, 0, 201},
- {"tablespace-main-data", 1, 0, 202},
- {"tablespace-main-index", 1, 0, 203},
- {"help", 0, 0, 'h'},
- {"style", 1, 0, 'S'},
- {"expire-tiles", 1, 0, 'e'},
- {"expire-output", 1, 0, 'o'},
- {"output", 1, 0, 'O'},
- {"extra-attributes", 0, 0, 'x'},
- {"hstore", 0, 0, 'k'},
- {"hstore-all", 0, 0, 'j'},
- {"hstore-column", 1, 0, 'z'},
- {"hstore-match-only", 0, 0, 208},
- {"hstore-add-index",0,0,211},
- {"multi-geometry", 0, 0, 'G'},
- {"keep-coastlines", 0, 0, 'K'},
- {"input-reader", 1, 0, 'r'},
- {"version", 0, 0, 'V'},
- {"disable-parallel-indexing", 0, 0, 'I'},
- {"cache-strategy", 1, 0, 204},
- {"number-processes", 1, 0, 205},
- {"drop", 0, 0, 206},
- {"unlogged", 0, 0, 207},
- {"flat-nodes",1,0,209},
- {"exclude-invalid-polygon",0,0,210},
- {"tag-transform-script",1,0,212},
- {0, 0, 0, 0}
- };
-
- c = getopt_long (argc, argv, "ab:cd:KhlmMp:suvU:WH:P:i:IE:C:S:e:o:O:xkjGz:r:V", long_options, &option_index);
- if (c == -1)
- break;
-
- switch (c) {
- case 'a': append=1; break;
- case 'b': osmdata.bbox=optarg; break;
- case 'c': create=1; break;
- case 'v': verbose=1; break;
- case 's': slim=1; break;
- case 'K': keep_coastlines=1; break;
- case 'u': sanitize=1; break;
- case 'l': projection=PROJ_LATLONG; break;
- case 'm': projection=PROJ_SPHERE_MERC; break;
- case 'M': projection=PROJ_MERC; break;
- case 'E': projection=-atoi(optarg); break;
- case 'p': prefix=optarg; break;
- case 'd': db=optarg; break;
- case 'C': cache = atoi(optarg); break;
- case 'U': username=optarg; break;
- case 'W': pass_prompt=1; break;
- case 'H': host=optarg; break;
- case 'P': port=optarg; break;
- case 'S': style=optarg; break;
- case 'i': tblsmain_index=tblsslim_index=optarg; break;
- case 200: tblsslim_data=optarg; break;
- case 201: tblsslim_index=optarg; break;
- case 202: tblsmain_data=optarg; break;
- case 203: tblsmain_index=optarg; break;
- case 'e':
- expire_tiles_zoom_min = atoi(optarg);
- temparg = strchr(optarg, '-');
- if (temparg) expire_tiles_zoom = atoi(temparg + 1);
- if (expire_tiles_zoom < expire_tiles_zoom_min) expire_tiles_zoom = expire_tiles_zoom_min;
- break;
- case 'o': expire_tiles_filename=optarg; break;
- case 'O': output_backend = optarg; break;
- case 'x': osmdata.extra_attributes=1; break;
- case 'k': if (enable_hstore != HSTORE_NONE) { fprintf(stderr, "ERROR: You can not specify both --hstore (-k) and --hstore-all (-j)\n"); exit (EXIT_FAILURE); }
- enable_hstore=HSTORE_NORM; break;
- case 208: hstore_match_only = 1; break;
- case 'j': if (enable_hstore != HSTORE_NONE) { fprintf(stderr, "ERROR: You can not specify both --hstore (-k) and --hstore-all (-j)\n"); exit (EXIT_FAILURE); }
- enable_hstore=HSTORE_ALL; break;
- case 'z':
- n_hstore_columns++;
- hstore_columns = (const char**)realloc(hstore_columns, sizeof(char *) * n_hstore_columns);
- hstore_columns[n_hstore_columns-1] = optarg;
- break;
- case 'G': enable_multi=1; break;
- case 'r': input_reader = optarg; break;
- case 'h': long_usage_bool=1; break;
- case 'I':
-#ifdef HAVE_PTHREAD
- parallel_indexing=0;
-#endif
- break;
- case 204:
- if (strcmp(optarg,"dense") == 0) alloc_chunkwise = ALLOC_DENSE;
- else if (strcmp(optarg,"chunk") == 0) alloc_chunkwise = ALLOC_DENSE | ALLOC_DENSE_CHUNK;
- else if (strcmp(optarg,"sparse") == 0) alloc_chunkwise = ALLOC_SPARSE;
- else if (strcmp(optarg,"optimized") == 0) alloc_chunkwise = ALLOC_DENSE | ALLOC_SPARSE;
- else {fprintf(stderr, "ERROR: Unrecognized cache strategy %s.\n", optarg); exit(EXIT_FAILURE); }
- break;
- case 205:
-#ifdef HAVE_FORK
- num_procs = atoi(optarg);
-#else
- fprintf(stderr, "WARNING: osm2pgsql was compiled without fork, only using one process!\n");
-#endif
- break;
- case 206: droptemp = 1; break;
- case 207: unlogged = 1; break;
- case 209:
- flat_node_cache_enabled = 1;
- flat_nodes_file = optarg;
- break;
- case 210: excludepoly = 1; exclude_broken_polygon(); break;
- case 211: enable_hstore_index = 1; break;
- case 212: tag_transform_script = optarg; break;
- case 'V': exit(EXIT_SUCCESS);
- case '?':
- default:
- short_usage(argv[0]);
- exit(EXIT_FAILURE);
- }
- }
-
- if (long_usage_bool) {
- long_usage(argv[0]);
- exit(EXIT_SUCCESS);
- }
-
- if (argc == optind) { /* No non-switch arguments */
- short_usage(argv[0]);
- exit(EXIT_FAILURE);
- }
-
- if (append && create) {
- fprintf(stderr, "Error: --append and --create options can not be used at the same time!\n");
- exit(EXIT_FAILURE);
- }
-
- if (droptemp && !slim) {
- fprintf(stderr, "Error: --drop only makes sense with --slim.\n");
- exit(EXIT_FAILURE);
- }
-
- if (unlogged && !create) {
- fprintf(stderr, "Warning: --unlogged only makes sense with --create; ignored.\n");
- unlogged = 0;
- }
-
- if (enable_hstore == HSTORE_NONE && !n_hstore_columns && hstore_match_only)
- {
- fprintf(stderr, "Warning: --hstore-match-only only makes sense with --hstore, --hstore-all, or --hstore-column; ignored.\n");
- hstore_match_only = 0;
- }
-
- if (enable_hstore_index && enable_hstore == HSTORE_NONE && !n_hstore_columns) {
- fprintf(stderr, "Warning: --hstore-add-index only makes sense with hstore enabled.\n");
- enable_hstore_index = 0;
- }
-
- if (cache < 0) cache = 0;
-
- if (cache == 0) {
- fprintf(stderr, "WARNING: ram cache is disabled. This will likely slow down processing a lot.\n\n");
- }
-
- if (num_procs < 1) num_procs = 1;
-
- if (pass_prompt)
- password = simple_prompt("Password:", 100, 0);
- else {
- password = getenv("PGPASS");
- }
-
-
-
- conninfo = build_conninfo(db, username, password, host, port);
- sql_conn = PQconnectdb(conninfo);
- if (PQstatus(sql_conn) != CONNECTION_OK) {
- fprintf(stderr, "Error: Connection to database failed: %s\n", PQerrorMessage(sql_conn));
- exit(EXIT_FAILURE);
- }
- if (unlogged && PQserverVersion(sql_conn) < 90100) {
- fprintf(stderr, "Error: --unlogged works only with PostgreSQL 9.1 and above, but\n");
- fprintf(stderr, "you are using PostgreSQL %d.%d.%d.\n", PQserverVersion(sql_conn) / 10000, (PQserverVersion(sql_conn) / 100) % 100, PQserverVersion(sql_conn) % 100);
- exit(EXIT_FAILURE);
- }
-
- PQfinish(sql_conn);
-
- text_init();
- initList(&osmdata.tags);
-
- osmdata.count_node = osmdata.max_node = 0;
- osmdata.count_way = osmdata.max_way = 0;
- osmdata.count_rel = osmdata.max_rel = 0;
-
- LIBXML_TEST_VERSION
-
- project_init(projection);
- fprintf(stderr, "Using projection SRS %d (%s)\n",
- project_getprojinfo()->srs, project_getprojinfo()->descr );
-
- if (parse_bbox(&osmdata))
- return 1;
-
- options.conninfo = conninfo;
- options.prefix = prefix;
- options.append = append;
- options.slim = slim;
- options.projection = project_getprojinfo()->srs;
- options.scale = (projection==PROJ_LATLONG)?10000000:100;
- options.mid = slim ? &mid_pgsql : &mid_ram;
- options.cache = cache;
- options.style = style;
- options.tblsmain_index = tblsmain_index;
- options.tblsmain_data = tblsmain_data;
- options.tblsslim_index = tblsslim_index;
- options.tblsslim_data = tblsslim_data;
- options.expire_tiles_zoom = expire_tiles_zoom;
- options.expire_tiles_zoom_min = expire_tiles_zoom_min;
- options.expire_tiles_filename = expire_tiles_filename;
- options.enable_multi = enable_multi;
- options.enable_hstore = enable_hstore;
- options.enable_hstore_index = enable_hstore_index;
- options.hstore_match_only = hstore_match_only;
- options.hstore_columns = hstore_columns;
- options.n_hstore_columns = n_hstore_columns;
- options.keep_coastlines = keep_coastlines;
- options.parallel_indexing = parallel_indexing;
- options.alloc_chunkwise = alloc_chunkwise;
- options.num_procs = num_procs;
- options.droptemp = droptemp;
- options.unlogged = unlogged;
- options.flat_node_cache_enabled = flat_node_cache_enabled;
- options.flat_node_file = flat_nodes_file;
- options.excludepoly = excludepoly;
- options.tag_transform_script = tag_transform_script;
-
- if (strcmp("pgsql", output_backend) == 0) {
- osmdata.out = &out_pgsql;
- } else if (strcmp("gazetteer", output_backend) == 0) {
- osmdata.out = &out_gazetteer;
- } else if (strcmp("null", output_backend) == 0) {
- osmdata.out = &out_null;
- } else {
- fprintf(stderr, "Output backend `%s' not recognised. Should be one of [pgsql, gazetteer, null].\n", output_backend);
- exit(EXIT_FAILURE);
- }
- options.out = osmdata.out;
-
- if (strcmp("auto", input_reader) != 0) {
- if (strcmp("libxml2", input_reader) == 0) {
- streamFile = &streamFileXML2;
- } else if (strcmp("primitive", input_reader) == 0) {
- streamFile = &streamFilePrimitive;
-#ifdef BUILD_READER_PBF
- } else if (strcmp("pbf", input_reader) == 0) {
- streamFile = &streamFilePbf;
-#endif
- } else if (strcmp("o5m", input_reader) == 0) {
- streamFile = &streamFileO5m;
- } else {
- fprintf(stderr, "Input parser `%s' not recognised. Should be one of [libxml2, primitive, o5m"
-#ifdef BUILD_READER_PBF
- ", pbf"
-#endif
- "].\n", input_reader);
- exit(EXIT_FAILURE);
- }
- }
-
- time(&overall_start);
- osmdata.out->start(&options);
-
- realloc_nodes(&osmdata);
- realloc_members(&osmdata);
-
- if (sizeof(int*) == 4 && options.slim != 1) {
- fprintf(stderr, "\n!! You are running this on 32bit system, so at most\n");
- fprintf(stderr, "!! 3GB of RAM can be used. If you encounter unexpected\n");
- fprintf(stderr, "!! exceptions during import, you should try running in slim\n");
- fprintf(stderr, "!! mode using parameter -s.\n");
- }
-
- while (optind < argc) {
- /* if input_reader is not forced by -r switch try to auto-detect it
- by file extension */
- if (strcmp("auto", input_reader) == 0) {
-
- if (strcasecmp(".pbf",argv[optind]+strlen(argv[optind])-4) == 0) {
-#ifdef BUILD_READER_PBF
- streamFile = &streamFilePbf;
-#else
- fprintf(stderr, "ERROR: PBF support has not been compiled into this version of osm2pgsql, please either compile it with pbf support or use one of the other input formats\n");
- exit(EXIT_FAILURE);
-#endif
- } else if (strcasecmp(".o5m",argv[optind]+strlen(argv[optind])-4) == 0 || strcasecmp(".o5c",argv[optind]+strlen(argv[optind])-4) == 0) {
- streamFile = &streamFileO5m;
- } else {
- streamFile = &streamFileXML2;
- }
- }
- fprintf(stderr, "\nReading in file: %s\n", argv[optind]);
- time(&start);
- if (streamFile(argv[optind], sanitize, &osmdata) != 0)
- exit_nicely();
- time(&end);
- fprintf(stderr, " parse time: %ds\n", (int)(end - start));
- optind++;
- }
-
- xmlCleanupParser();
- xmlMemoryDump();
-
- if (osmdata.count_node || osmdata.count_way || osmdata.count_rel) {
- time(&now);
- end_nodes = osmdata.start_way > 0 ? osmdata.start_way : now;
- end_way = osmdata.start_rel > 0 ? osmdata.start_rel : now;
- end_rel = now;
- fprintf(stderr, "\n");
- fprintf(stderr, "Node stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", osmdata.count_node, osmdata.max_node,
- osmdata.count_node > 0 ? (int)(end_nodes - osmdata.start_node) : 0);
- fprintf(stderr, "Way stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", osmdata.count_way, osmdata.max_way,
- osmdata.count_way > 0 ? (int)(end_way - osmdata.start_way) : 0);
- fprintf(stderr, "Relation stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", osmdata.count_rel, osmdata.max_rel,
- osmdata.count_rel > 0 ? (int)(end_rel - osmdata.start_rel) : 0);
- }
- osmdata.out->stop();
-
- free(osmdata.nds);
- free(osmdata.members);
-
- /* free the column pointer buffer */
- free(hstore_columns);
-
- project_exit();
- text_exit();
- fprintf(stderr, "\n");
- time(&overall_end);
- fprintf(stderr, "Osm2pgsql took %ds overall\n", (int)(overall_end - overall_start));
-
- return 0;
-}
diff --git a/osm2pgsql.cpp b/osm2pgsql.cpp
new file mode 100644
index 0000000..99a0f39
--- /dev/null
+++ b/osm2pgsql.cpp
@@ -0,0 +1,128 @@
+/*
+#-----------------------------------------------------------------------------
+# osm2pgsql - converts planet.osm file into PostgreSQL
+# compatible output suitable to be rendered by mapnik
+# Use: osm2pgsql planet.osm.bz2
+#-----------------------------------------------------------------------------
+# Original Python implementation by Artem Pavlenko
+# Re-implementation by Jon Burgess, Copyright 2006
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#-----------------------------------------------------------------------------
+*/
+
+#include "config.h"
+#include "options.hpp"
+#include "parse.hpp"
+#include "middle.hpp"
+#include "output.hpp"
+#include "osmdata.hpp"
+#include "util.hpp"
+#include "text-tree.hpp"
+
+#include <unistd.h>
+#include <assert.h>
+#include <time.h>
+#include <stdexcept>
+
+#include <libpq-fe.h>
+#include <boost/format.hpp>
+
+void check_db(const char* conninfo, const int unlogged)
+{
+ PGconn *sql_conn = PQconnectdb(conninfo);
+
+ //make sure you can connect
+ if (PQstatus(sql_conn) != CONNECTION_OK) {
+ throw std::runtime_error((boost::format("Error: Connection to database failed: %1%\n") % PQerrorMessage(sql_conn)).str());
+ }
+
+ //make sure unlogged it is supported by your database if you want it
+ if (unlogged && PQserverVersion(sql_conn) < 90100) {
+ throw std::runtime_error((
+ boost::format("Error: --unlogged works only with PostgreSQL 9.1 and above, but\n you are using PostgreSQL %1%.%2%.%3%.\n")
+ % (PQserverVersion(sql_conn) / 10000)
+ % ((PQserverVersion(sql_conn) / 100) % 100)
+ % (PQserverVersion(sql_conn) % 100)).str());
+ }
+
+ PQfinish(sql_conn);
+}
+
+int main(int argc, char *argv[])
+{
+ fprintf(stderr, "osm2pgsql SVN version %s (%lubit id space)\n\n", VERSION, 8 * sizeof(osmid_t));
+ try
+ {
+ //parse the args into the different options members
+ options_t options = options_t::parse(argc, argv);
+ if(options.long_usage_bool)
+ return 0;
+
+ //setup the front (input)
+ parse_delegate_t parser(options.extra_attributes, options.bbox, options.projection);
+
+ //setup the middle
+ boost::shared_ptr<middle_t> middle = middle_t::create_middle(options.slim);
+
+ //setup the backend (output)
+ std::vector<boost::shared_ptr<output_t> > outputs = output_t::create_outputs(middle.get(), options);
+
+ //let osmdata orchestrate between the middle and the outs
+ osmdata_t osmdata(middle, outputs);
+
+ //check the database
+ check_db(options.conninfo.c_str(), options.unlogged);
+
+ fprintf(stderr, "Using projection SRS %d (%s)\n",
+ options.projection->project_getprojinfo()->srs,
+ options.projection->project_getprojinfo()->descr );
+
+ //start it up
+ time_t overall_start = time(NULL);
+ osmdata.start();
+
+ /* Processing
+ * In this phase the input file(s) are read and parsed, populating some of the
+ * tables. Not all ways can be handled before relations are processed, so they're
+ * set as pending, to be handled in the next stage.
+ */
+ //read in the input files one by one
+ for(std::vector<std::string>::const_iterator filename = options.input_files.begin(); filename != options.input_files.end(); ++filename)
+ {
+ //read the actual input
+ fprintf(stderr, "\nReading in file: %s\n", filename->c_str());
+ time_t start = time(NULL);
+ if (parser.streamFile(options.input_reader.c_str(), filename->c_str(), options.sanitize, &osmdata) != 0)
+ util::exit_nicely();
+ fprintf(stderr, " parse time: %ds\n", (int)(time(NULL) - start));
+ }
+
+ //show stats
+ parser.printSummary();
+
+ //Process pending ways, relations, cluster, and create indexes
+ osmdata.stop();
+
+ fprintf(stderr, "\nOsm2pgsql took %ds overall\n", (int)(time(NULL) - overall_start));
+
+ return 0;
+ }//something went wrong along the way
+ catch(std::runtime_error& e)
+ {
+ fprintf(stderr, "Osm2pgsql failed due to ERROR: %s\n", e.what());
+ exit(EXIT_FAILURE);
+ }
+}
diff --git a/osmdata.cpp b/osmdata.cpp
new file mode 100644
index 0000000..66d0459
--- /dev/null
+++ b/osmdata.cpp
@@ -0,0 +1,407 @@
+#include "osmdata.hpp"
+
+#include <boost/foreach.hpp>
+#include <boost/make_shared.hpp>
+#include <boost/thread/thread.hpp>
+#include <boost/unordered_map.hpp>
+#include <boost/thread.hpp>
+#include <boost/version.hpp>
+
+#include <stdexcept>
+#include <utility>
+
+#if BOOST_VERSION < 105300
+#else
+#include <boost/atomic.hpp>
+#endif
+
+osmdata_t::osmdata_t(boost::shared_ptr<middle_t> mid_, const boost::shared_ptr<output_t>& out_): mid(mid_)
+{
+ outs.push_back(out_);
+}
+
+osmdata_t::osmdata_t(boost::shared_ptr<middle_t> mid_, const std::vector<boost::shared_ptr<output_t> > &outs_)
+ : mid(mid_), outs(outs_)
+{
+ if (outs.empty()) {
+ throw std::runtime_error("Must have at least one output, but none have "
+ "been configured.");
+ }
+}
+
+osmdata_t::~osmdata_t()
+{
+}
+
+int osmdata_t::node_add(osmid_t id, double lat, double lon, struct keyval *tags) {
+ mid->nodes_set(id, lat, lon, tags);
+
+ int status = 0;
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ status |= out->node_add(id, lat, lon, tags);
+ }
+ return status;
+}
+
+int osmdata_t::way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags) {
+ mid->ways_set(id, nodes, node_count, tags);
+
+ int status = 0;
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ status |= out->way_add(id, nodes, node_count, tags);
+ }
+ return status;
+}
+
+int osmdata_t::relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags) {
+ mid->relations_set(id, members, member_count, tags);
+
+ int status = 0;
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ status |= out->relation_add(id, members, member_count, tags);
+ }
+ return status;
+}
+
+int osmdata_t::node_modify(osmid_t id, double lat, double lon, struct keyval *tags) {
+ slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid.get());
+
+ slim->nodes_delete(id);
+ slim->nodes_set(id, lat, lon, tags);
+
+ int status = 0;
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ status |= out->node_modify(id, lat, lon, tags);
+ }
+
+ slim->node_changed(id);
+
+ return status;
+}
+
+int osmdata_t::way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags) {
+ slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid.get());
+
+ slim->ways_delete(id);
+ slim->ways_set(id, nodes, node_count, tags);
+
+ int status = 0;
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ status |= out->way_modify(id, nodes, node_count, tags);
+ }
+
+ slim->way_changed(id);
+
+ return status;
+}
+
+int osmdata_t::relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags) {
+ slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid.get());
+
+ slim->relations_delete(id);
+ slim->relations_set(id, members, member_count, tags);
+
+ int status = 0;
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ status |= out->relation_modify(id, members, member_count, tags);
+ }
+
+ slim->relation_changed(id);
+
+ return status;
+}
+
+int osmdata_t::node_delete(osmid_t id) {
+ slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid.get());
+
+ int status = 0;
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ status |= out->node_delete(id);
+ }
+
+ slim->nodes_delete(id);
+
+ return status;
+}
+
+int osmdata_t::way_delete(osmid_t id) {
+ slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid.get());
+
+ int status = 0;
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ status |= out->way_delete(id);
+ }
+
+ slim->ways_delete(id);
+
+ return status;
+}
+
+int osmdata_t::relation_delete(osmid_t id) {
+ slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid.get());
+
+ int status = 0;
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ status |= out->relation_delete(id);
+ }
+
+ slim->relations_delete(id);
+
+ return status;
+}
+
+void osmdata_t::start() {
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ out->start();
+ }
+ mid->start(outs[0]->get_options());
+}
+
+namespace {
+
+//TODO: have the main thread using the main middle to query the middle for batches of ways (configurable number)
+//and stuffing those into the work queue, so we have a single producer multi consumer threaded queue
+//since the fetching from middle should be faster than the processing in each backend.
+
+struct pending_threaded_processor : public middle_t::pending_processor {
+ typedef std::vector<boost::shared_ptr<output_t> > output_vec_t;
+ typedef std::pair<boost::shared_ptr<const middle_query_t>, output_vec_t> clone_t;
+
+#if BOOST_VERSION < 105300
+ static void do_jobs(output_vec_t const& outputs, pending_queue_t& queue, size_t& ids_done, boost::mutex& mutex, int append, bool ways) {
+ while (true) {
+ //get the job off the queue synchronously
+ pending_job_t job;
+ mutex.lock();
+ if(queue.empty()) {
+ mutex.unlock();
+ break;
+ }
+ else {
+ job = queue.top();
+ queue.pop();
+ }
+ mutex.unlock();
+
+ //process it
+ if(ways)
+ outputs.at(job.second)->pending_way(job.first, append);
+ else
+ outputs.at(job.second)->pending_relation(job.first, append);
+
+ mutex.lock();
+ ++ids_done;
+ mutex.unlock();
+ }
+ }
+#else
+ static void do_jobs(output_vec_t const& outputs, pending_queue_t& queue, boost::atomic_size_t& ids_done, int append, bool ways) {
+ pending_job_t job;
+ while (queue.pop(job)) {
+ if(ways)
+ outputs.at(job.second)->pending_way(job.first, append);
+ else
+ outputs.at(job.second)->pending_relation(job.first, append);
+ ++ids_done;
+ }
+ }
+#endif
+
+ //starts up count threads and works on the queue
+ pending_threaded_processor(boost::shared_ptr<middle_query_t> mid, const output_vec_t& outs, size_t thread_count, size_t job_count, int append)
+#if BOOST_VERSION < 105300
+ //note that we cant hint to the stack how large it should be ahead of time
+ //we could use a different datastructure like a deque or vector but then
+ //the outputs the enqueue jobs would need the version check for the push(_back) method
+ : outs(outs), ids_queued(0), append(append), queue(), ids_done(0) {
+#else
+ : outs(outs), ids_queued(0), append(append), queue(job_count), ids_done(0) {
+#endif
+
+ //clone all the things we need
+ clones.reserve(thread_count);
+ for (size_t i = 0; i < thread_count; ++i) {
+ //clone the middle
+ boost::shared_ptr<const middle_query_t> mid_clone = mid->get_instance();
+
+ //clone the outs
+ output_vec_t out_clones;
+ BOOST_FOREACH(const boost::shared_ptr<output_t>& out, outs) {
+ out_clones.push_back(out->clone(mid_clone.get()));
+ }
+
+ //keep the clones for a specific thread to use
+ clones.push_back(clone_t(mid_clone, out_clones));
+ }
+ }
+
+ ~pending_threaded_processor() {}
+
+ void enqueue_ways(osmid_t id) {
+ for(size_t i = 0; i < outs.size(); ++i) {
+ outs[i]->enqueue_ways(queue, id, i, ids_queued);
+ }
+ }
+
+ //waits for the completion of all outstanding jobs
+ void process_ways() {
+ //reset the number we've done
+ ids_done = 0;
+
+ fprintf(stderr, "\nGoing over pending ways...\n");
+ fprintf(stderr, "\t%zu ways are pending\n", ids_queued);
+ fprintf(stderr, "\nUsing %zu helper-processes\n", clones.size());
+ time_t start = time(NULL);
+
+
+ //make the threads and start them
+ for (size_t i = 0; i < clones.size(); ++i) {
+#if BOOST_VERSION < 105300
+ workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), boost::ref(mutex), append, true));
+#else
+ workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), append, true));
+#endif
+ }
+
+ //TODO: print out partial progress
+
+ //wait for them to really be done
+ workers.join_all();
+
+ time_t finish = time(NULL);
+ fprintf(stderr, "\rFinished processing %zu ways in %i sec\n\n", ids_queued, (int)(finish - start));
+ if (finish - start > 0)
+ fprintf(stderr, "%zu Pending ways took %ds at a rate of %.2f/s\n", ids_queued, (int)(finish - start),
+ ((double)ids_queued / (double)(finish - start)));
+ ids_queued = 0;
+ ids_done = 0;
+
+ //collect all the new rels that became pending from each
+ //output in each thread back to their respective main outputs
+ BOOST_FOREACH(const clone_t& clone, clones) {
+ //for each clone/original output
+ for(output_vec_t::const_iterator original_output = outs.begin(), clone_output = clone.second.begin();
+ original_output != outs.end() && clone_output != clone.second.end(); ++original_output, ++clone_output) {
+ //done copying ways for now
+ clone_output->get()->commit();
+ //merge the pending from this threads copy of output back
+ original_output->get()->merge_pending_relations(*clone_output);
+ }
+ }
+ }
+
+ void enqueue_relations(osmid_t id) {
+ for(size_t i = 0; i < outs.size(); ++i) {
+ outs[i]->enqueue_relations(queue, id, i, ids_queued);
+ }
+ }
+
+ void process_relations() {
+ //reset the number we've done
+ ids_done = 0;
+
+ fprintf(stderr, "\nGoing over pending relations...\n");
+ fprintf(stderr, "\t%zu relations are pending\n", ids_queued);
+ fprintf(stderr, "\nUsing %zu helper-processes\n", clones.size());
+ time_t start = time(NULL);
+
+ //make the threads and start them
+ for (size_t i = 0; i < clones.size(); ++i) {
+#if BOOST_VERSION < 105300
+ workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), boost::ref(mutex), append, false));
+#else
+ workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), append, false));
+#endif
+ }
+
+ //TODO: print out partial progress
+
+ //wait for them to really be done
+ workers.join_all();
+
+ time_t finish = time(NULL);
+ fprintf(stderr, "\rFinished processing %zu relations in %i sec\n\n", ids_queued, (int)(finish - start));
+ if (finish - start > 0)
+ fprintf(stderr, "%zu Pending relations took %ds at a rate of %.2f/s\n", ids_queued, (int)(finish - start),
+ ((double)ids_queued / (double)(finish - start)));
+ ids_queued = 0;
+ ids_done = 0;
+
+ //collect all expiry tree informations together into one
+ BOOST_FOREACH(const clone_t& clone, clones) {
+ //for each clone/original output
+ for(output_vec_t::const_iterator original_output = outs.begin(), clone_output = clone.second.begin();
+ original_output != outs.end() && clone_output != clone.second.end(); ++original_output, ++clone_output) {
+ //done copying rels for now
+ clone_output->get()->commit();
+ //merge the expire tree from this threads copy of output back
+ original_output->get()->merge_expire_trees(*clone_output);
+ }
+ }
+ }
+
+private:
+ //middle and output copies
+ std::vector<clone_t> clones;
+ output_vec_t outs; //would like to move ownership of outs to osmdata_t and middle passed to output_t instead of owned by it
+ //actual threads
+ boost::thread_group workers;
+ //how many jobs do we have in the queue to start with
+ size_t ids_queued;
+ //appending to output that is already there (diff processing)
+ int append;
+ //job queue
+ pending_queue_t queue;
+
+#if BOOST_VERSION < 105300
+ //how many ids within the job have been processed
+ size_t ids_done;
+ //so the threads can manage some of the shared state
+ boost::mutex mutex;
+#else
+ boost::atomic_size_t ids_done;
+#endif
+};
+
+} // anonymous namespace
+
+void osmdata_t::stop() {
+ /* Commit the transactions, so that multiple processes can
+ * access the data simultanious to process the rest in parallel
+ * as well as see the newly created tables.
+ */
+ size_t pending_count = mid->pending_count();
+ mid->commit();
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ //TODO: each of the outs can be in parallel
+ out->commit();
+ pending_count += out->pending_count();
+ }
+
+ // should be the same for all outputs
+ const int append = outs[0]->get_options()->append;
+
+ //threaded pending processing
+ pending_threaded_processor ptp(mid, outs, outs[0]->get_options()->num_procs, pending_count, append);
+
+ if (!outs.empty()) {
+ //This stage takes ways which were processed earlier, but might be
+ //involved in a multipolygon relation. They could also be ways that
+ //were modified in diff processing.
+ mid->iterate_ways( ptp );
+
+ //This is like pending ways, except there aren't pending relations
+ //on import, only on update.
+ //TODO: Can we skip this on import?
+ mid->iterate_relations( ptp );
+ }
+
+ //Clustering, index creation, and cleanup.
+ //All the intensive parts of this are long-running PostgreSQL commands
+ boost::thread_group threads;
+ BOOST_FOREACH(boost::shared_ptr<output_t>& out, outs) {
+ threads.add_thread(new boost::thread(boost::bind( &output_t::stop, out.get() )));
+ }
+ threads.add_thread(new boost::thread(boost::bind( &middle_t::stop, mid.get() )));
+ threads.join_all();
+}
diff --git a/osmdata.hpp b/osmdata.hpp
new file mode 100644
index 0000000..0dd04bf
--- /dev/null
+++ b/osmdata.hpp
@@ -0,0 +1,37 @@
+#ifndef OSMDATA_H
+#define OSMDATA_H
+
+// when __cplusplus is defined, we need to define this macro as well
+// to get the print format specifiers in the inttypes.h header.
+#include <config.h>
+#include <vector>
+
+#include "output.hpp"
+
+class osmdata_t {
+public:
+ osmdata_t(boost::shared_ptr<middle_t> mid_, const boost::shared_ptr<output_t>& out_);
+ osmdata_t(boost::shared_ptr<middle_t> mid_, const std::vector<boost::shared_ptr<output_t> > &outs_);
+ ~osmdata_t();
+
+ void start();
+ void stop();
+
+ int node_add(osmid_t id, double lat, double lon, struct keyval *tags);
+ int way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
+ int relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+
+ int node_modify(osmid_t id, double lat, double lon, struct keyval *tags);
+ int way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
+ int relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+
+ int node_delete(osmid_t id);
+ int way_delete(osmid_t id);
+ int relation_delete(osmid_t id);
+
+private:
+ boost::shared_ptr<middle_t> mid;
+ std::vector<boost::shared_ptr<output_t> > outs;
+};
+
+#endif
diff --git a/osmtypes.h b/osmtypes.h
deleted file mode 100644
index 6f0e5ea..0000000
--- a/osmtypes.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/* Data types to hold OSM node, segment, way data */
-
-#ifndef OSMTYPES_H
-#define OSMTYPES_H
-
-#include <inttypes.h>
-#include <time.h>
-#include <config.h>
-
-/* Use ./configure --enable-64bit-ids to build a version that supports 64bit IDs. */
-
-#ifdef OSMID64
-typedef int64_t osmid_t;
-#define strtoosmid strtoll
-#define PRIdOSMID PRId64
-#define POSTGRES_OSMID_TYPE "int8"
-#else
-typedef int32_t osmid_t;
-#define strtoosmid strtol
-#define PRIdOSMID PRId32
-#define POSTGRES_OSMID_TYPE "int4"
-#endif
-
-#include "keyvals.h"
-
-enum OsmType { OSMTYPE_WAY, OSMTYPE_NODE, OSMTYPE_RELATION };
-
-struct osmNode {
- double lon;
- double lat;
-};
-
-struct member {
- enum OsmType type;
- osmid_t id;
- char *role;
-};
-
-typedef enum { FILETYPE_NONE, FILETYPE_OSM, FILETYPE_OSMCHANGE, FILETYPE_PLANETDIFF } filetypes_t;
-typedef enum { ACTION_NONE, ACTION_CREATE, ACTION_MODIFY, ACTION_DELETE } actions_t;
-
-struct osmdata_t {
- osmid_t count_node, max_node;
- osmid_t count_way, max_way;
- osmid_t count_rel, max_rel;
- time_t start_node, start_way, start_rel;
-
- struct output_t *out;
-
-/* Since {node,way} elements are not nested we can guarantee the
- values in an end tag must match those of the corresponding
- start tag and can therefore be cached.
-*/
- double node_lon, node_lat;
- struct keyval tags;
- osmid_t *nds;
- int nd_count, nd_max;
- struct member *members;
- int member_count, member_max;
- osmid_t osm_id;
- filetypes_t filetype;
- actions_t action;
- int extra_attributes;
-
- /* Bounding box to filter imported data */
- const char *bbox;
-
- double minlon, minlat, maxlon, maxlat;
-
- int parallel_indexing;
-};
-
-void realloc_nodes(struct osmdata_t *osmdata);
-void realloc_members(struct osmdata_t *osmdata);
-void resetMembers(struct osmdata_t *osmdata);
-void printStatus(struct osmdata_t *osmdata);
-int node_wanted(struct osmdata_t *osmdata, double lat, double lon);
-
-/* exit_nicely - called to cleanup after fatal error */
-void exit_nicely(void);
-
-#endif
diff --git a/osmtypes.hpp b/osmtypes.hpp
new file mode 100644
index 0000000..c884b3c
--- /dev/null
+++ b/osmtypes.hpp
@@ -0,0 +1,37 @@
+/* Data types to hold OSM node, segment, way data */
+
+#ifndef OSMTYPES_H
+#define OSMTYPES_H
+
+// when __cplusplus is defined, we need to define this macro as well
+// to get the print format specifiers in the inttypes.h header.
+#define __STDC_FORMAT_MACROS
+#include <inttypes.h>
+#include <config.h>
+
+#ifdef OSMID64
+typedef int64_t osmid_t;
+#define strtoosmid strtoll
+#define PRIdOSMID PRId64
+#define POSTGRES_OSMID_TYPE "int8"
+#else
+typedef int32_t osmid_t;
+#define strtoosmid strtol
+#define PRIdOSMID PRId32
+#define POSTGRES_OSMID_TYPE "int4"
+#endif
+
+enum OsmType { OSMTYPE_WAY, OSMTYPE_NODE, OSMTYPE_RELATION };
+
+struct osmNode {
+ double lon;
+ double lat;
+};
+
+struct member {
+ enum OsmType type;
+ osmid_t id;
+ char *role;
+};
+
+#endif
diff --git a/output-gazetteer.c b/output-gazetteer.cpp
similarity index 68%
rename from output-gazetteer.c
rename to output-gazetteer.cpp
index d90b612..552b517 100644
--- a/output-gazetteer.c
+++ b/output-gazetteer.cpp
@@ -3,18 +3,18 @@
#include <string.h>
#include <libpq-fe.h>
+#include <boost/make_shared.hpp>
+#include <boost/algorithm/string/predicate.hpp>
-#include "osmtypes.h"
-#include "middle.h"
-#include "output.h"
-#include "output-gazetteer.h"
-#include "pgsql.h"
-#include "reprojection.h"
-#include "build_geometry.h"
+#include "osmtypes.hpp"
+#include "middle.hpp"
+#include "pgsql.hpp"
+#include "reprojection.hpp"
+#include "output-gazetteer.hpp"
+#include "options.hpp"
+#include "util.hpp"
-#define BUFFER_SIZE 4096
-
-#define SRID (project_getprojinfo()->srs)
+#define SRID (reproj->project_getprojinfo()->srs)
#define CREATE_KEYVALUETYPE_TYPE \
"CREATE TYPE keyvalue AS (" \
@@ -24,11 +24,11 @@
#define CREATE_WORDSCORE_TYPE \
"CREATE TYPE wordscore AS (" \
- " word TEXT," \
- " score FLOAT" \
+ " word TEXT," \
+ " score FLOAT" \
")"
-#define CREATE_PLACE_TABLE \
+#define CREATE_PLACE_TABLE \
"CREATE TABLE place (" \
" osm_type CHAR(1) NOT NULL," \
" osm_id " POSTGRES_OSMID_TYPE " NOT NULL," \
@@ -38,7 +38,7 @@
" admin_level INTEGER," \
" housenumber TEXT," \
" street TEXT," \
- " addr_place TEXT," \
+ " addr_place TEXT," \
" isin TEXT," \
" postcode TEXT," \
" country_code VARCHAR(2)," \
@@ -54,31 +54,18 @@
#define TAGINFO_WAY 0x2u
#define TAGINFO_AREA 0x4u
-
-static const struct output_options *Options = NULL;
-static PGconn *Connection = NULL;
-static int CopyActive = 0;
-static char Buffer[BUFFER_SIZE];
-static unsigned int BufferLen = 0;
-
-static PGconn *ConnectionDelete = NULL;
-
-static PGconn *ConnectionError = NULL;
-
-static FILE * hLog = NULL;
-
-static void require_slim_mode(void)
+void output_gazetteer_t::require_slim_mode(void)
{
- if (!Options->slim)
+ if (!m_options.slim)
{
fprintf(stderr, "Cannot apply diffs unless in slim mode\n");
- exit_nicely();
+ util::exit_nicely();
}
return;
}
-static void copy_data(const char *sql)
+void output_gazetteer_t::copy_data(const char *sql)
{
unsigned int sqlLen = strlen(sql);
@@ -122,7 +109,7 @@ static void copy_data(const char *sql)
return;
}
-static void stop_copy(void)
+void output_gazetteer_t::stop_copy(void)
{
PGresult *res;
@@ -133,7 +120,7 @@ static void stop_copy(void)
if (PQputCopyEnd(Connection, NULL) != 1)
{
fprintf(stderr, "COPY_END for place failed: %s\n", PQerrorMessage(Connection));
- exit_nicely();
+ util::exit_nicely();
}
/* Check the result */
@@ -142,7 +129,7 @@ static void stop_copy(void)
{
fprintf(stderr, "COPY_END for place failed: %s\n", PQerrorMessage(Connection));
PQclear(res);
- exit_nicely();
+ util::exit_nicely();
}
/* Discard the result */
@@ -159,8 +146,6 @@ static void copy_error_data(const char *sql)
{
unsigned int sqlLen = strlen(sql);
- if (hLog) fprintf(hLog, "%s", sql);
-
/* Make sure we have an active copy */
if (!CopyErrorActive)
{
@@ -212,7 +197,7 @@ static void stop_error_copy(void)
if (PQputCopyEnd(ConnectionError, NULL) != 1)
{
fprintf(stderr, "COPY_END for import_polygon_error failed: %s\n", PQerrorMessage(ConnectionError));
- exit_nicely();
+ util::exit_nicely();
}
/* Check the result */
@@ -221,7 +206,7 @@ static void stop_error_copy(void)
{
fprintf(stderr, "COPY_END for import_polygon_error failed: %s\n", PQerrorMessage(ConnectionError));
PQclear(res);
- exit_nicely();
+ util::exit_nicely();
}
/* Discard the result */
@@ -234,8 +219,12 @@ static void stop_error_copy(void)
}
#endif
-static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *names, struct keyval *places, struct keyval *extratags,
- int* admin_level, struct keyval ** housenumber, struct keyval ** street, struct keyval ** addr_place, char ** isin, struct keyval ** postcode, struct keyval ** countrycode)
+static int split_tags(struct keyval *tags, unsigned int flags,
+ struct keyval *names, struct keyval *places,
+ struct keyval *extratags, int* admin_level,
+ struct keyval ** housenumber, struct keyval ** street,
+ struct keyval ** addr_place, char ** isin,
+ struct keyval ** postcode, struct keyval ** countrycode)
{
size_t subval;
int placehouse = 0;
@@ -261,12 +250,12 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
streetnumber = 0;
/* Initialise the result lists */
- initList(names);
- initList(places);
- initList(extratags);
+ keyval::initList(names);
+ keyval::initList(places);
+ keyval::initList(extratags);
/* Loop over the tags */
- while ((item = popItem(tags)) != NULL)
+ while ((item = keyval::popItem(tags)) != NULL)
{
/* If this is a name tag, add it to the name list */
@@ -290,20 +279,20 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
strcmp(item->key, "name") == 0 ||
(strncmp(item->key, "name:", 5) == 0) ||
strcmp(item->key, "int_name") == 0 ||
- (strncmp(item->key, "int_name:", 9) == 0) ||
+ (strncmp(item->key, "int_name:", 9) == 0) ||
strcmp(item->key, "nat_name") == 0 ||
- (strncmp(item->key, "nat_name:", 9) == 0) ||
+ (strncmp(item->key, "nat_name:", 9) == 0) ||
strcmp(item->key, "reg_name") == 0 ||
- (strncmp(item->key, "reg_name:", 9) == 0) ||
+ (strncmp(item->key, "reg_name:", 9) == 0) ||
strcmp(item->key, "loc_name") == 0 ||
- (strncmp(item->key, "loc_name:", 9) == 0) ||
+ (strncmp(item->key, "loc_name:", 9) == 0) ||
strcmp(item->key, "old_name") == 0 ||
- (strncmp(item->key, "old_name:", 9) == 0) ||
+ (strncmp(item->key, "old_name:", 9) == 0) ||
strcmp(item->key, "alt_name") == 0 ||
- (strncmp(item->key, "alt_name_", 9) == 0) ||
- (strncmp(item->key, "alt_name:", 9) == 0) ||
+ (strncmp(item->key, "alt_name_", 9) == 0) ||
+ (strncmp(item->key, "alt_name:", 9) == 0) ||
strcmp(item->key, "official_name") == 0 ||
- (strncmp(item->key, "official_name:", 14) == 0) ||
+ (strncmp(item->key, "official_name:", 14) == 0) ||
strcmp(item->key, "commonname") == 0 ||
(strncmp(item->key, "commonname:", 11) == 0) ||
strcmp(item->key, "common_name") == 0 ||
@@ -316,11 +305,11 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
{
if (strcmp(item->key, "name:prefix") == 0)
{
- pushItem(extratags, item);
+ keyval::pushItem(extratags, item);
}
else
{
- pushItem(names, item);
+ keyval::pushItem(names, item);
}
}
else if (strcmp(item->key, "emergency") == 0 ||
@@ -331,11 +320,11 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
{
if (strcmp(item->value, "no") && strcmp(item->value, "yes"))
{
- pushItem(places, item);
+ keyval::pushItem(places, item);
}
else
{
- freeItem(item);
+ keyval::freeItem(item);
}
}
else if (strcmp(item->key, "highway") == 0)
@@ -347,11 +336,11 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
strcmp(item->value, "noexit") &&
strcmp(item->value, "crossing"))
{
- pushItem(places, item);
+ keyval::pushItem(places, item);
}
else
{
- freeItem(item);
+ keyval::freeItem(item);
}
}
else if (strcmp(item->key, "aerialway") == 0 ||
@@ -368,7 +357,7 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
{
if (strcmp(item->value, "no"))
{
- pushItem(places, item);
+ keyval::pushItem(places, item);
if (strcmp(item->key, "boundary") == 0 && strcmp(item->value, "administrative") == 0)
{
placeadmin = 1;
@@ -376,27 +365,27 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
}
else
{
- freeItem(item);
+ keyval::freeItem(item);
}
}
else if (strcmp(item->key, "waterway") == 0 &&
strcmp(item->value, "riverbank") != 0)
{
- pushItem(places, item);
+ keyval::pushItem(places, item);
}
- else if (strcmp(item->key, "place") == 0)
+ else if (strcmp(item->key, "place") == 0)
{
place = item;
}
else if (strcmp(item->key, "addr:housename") == 0)
{
- pushItem(names, item);
+ keyval::pushItem(names, item);
placehouse = 1;
}
else if (strcmp(item->key, "landuse") == 0)
{
if (strcmp(item->value, "cemetery") == 0)
- pushItem(places, item);
+ keyval::pushItem(places, item);
else
landuse = item;
}
@@ -408,7 +397,7 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
strcmp(item->key, "tiger:zip_right") == 0)
{
if (*postcode)
- freeItem(item);
+ keyval::freeItem(item);
else
*postcode = item;
}
@@ -420,17 +409,17 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
{
*addr_place = item;
}
- else if ((strcmp(item->key, "country_code_iso3166_1_alpha_2") == 0 ||
- strcmp(item->key, "country_code_iso3166_1") == 0 ||
- strcmp(item->key, "country_code_iso3166") == 0 ||
- strcmp(item->key, "country_code") == 0 ||
- strcmp(item->key, "iso3166-1:alpha2") == 0 ||
- strcmp(item->key, "iso3166-1") == 0 ||
- strcmp(item->key, "ISO3166-1") == 0 ||
- strcmp(item->key, "iso3166") == 0 ||
- strcmp(item->key, "is_in:country_code") == 0 ||
+ else if ((strcmp(item->key, "country_code_iso3166_1_alpha_2") == 0 ||
+ strcmp(item->key, "country_code_iso3166_1") == 0 ||
+ strcmp(item->key, "country_code_iso3166") == 0 ||
+ strcmp(item->key, "country_code") == 0 ||
+ strcmp(item->key, "iso3166-1:alpha2") == 0 ||
+ strcmp(item->key, "iso3166-1") == 0 ||
+ strcmp(item->key, "ISO3166-1") == 0 ||
+ strcmp(item->key, "iso3166") == 0 ||
+ strcmp(item->key, "is_in:country_code") == 0 ||
strcmp(item->key, "addr:country") == 0 ||
- strcmp(item->key, "addr:country_code") == 0)
+ strcmp(item->key, "addr:country_code") == 0)
&& strlen(item->value) == 2)
{
*countrycode = item;
@@ -439,7 +428,7 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
{
/* house number can be far more complex than just a single house number - leave for postgresql to deal with */
if (*housenumber)
- freeItem(item);
+ keyval::freeItem(item);
else {
*housenumber = item;
placehouse = 1;
@@ -448,7 +437,7 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
else if (strcmp(item->key, "addr:conscriptionnumber") == 0)
{
if (conscriptionnumber)
- freeItem(item);
+ keyval::freeItem(item);
else {
conscriptionnumber = item;
placehouse = 1;
@@ -457,7 +446,7 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
else if (strcmp(item->key, "addr:streetnumber") == 0)
{
if (streetnumber)
- freeItem(item);
+ keyval::freeItem(item);
else {
streetnumber = item;
placehouse = 1;
@@ -467,10 +456,10 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
{
/* house number can be far more complex than just a single house number - leave for postgresql to deal with */
if (*housenumber) {
- freeItem(item);
+ keyval::freeItem(item);
} else {
- *housenumber = item;
- addItem(places, "place", "houses", 1);
+ *housenumber = item;
+ keyval::addItem(places, "place", "houses", 1);
}
}
else if (strcmp(item->key, "tiger:county") == 0)
@@ -480,12 +469,12 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
with the same name.
*/
subval = strcspn(item->value, ",");
- *isin = realloc(*isin, isinsize + 9 + subval);
+ *isin = (char *)realloc(*isin, isinsize + 9 + subval);
*(*isin+isinsize) = ',';
strncpy(*isin+1+isinsize, item->value, subval);
strcpy(*isin+1+isinsize+subval, " county");
isinsize += 8 + subval;
- freeItem(item);
+ keyval::freeItem(item);
}
else if (strcmp(item->key, "is_in") == 0 ||
(strncmp(item->key, "is_in:", 5) == 0) ||
@@ -495,16 +484,16 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
strcmp(item->key, "addr:state_code") == 0 ||
strcmp(item->key, "addr:state") == 0)
{
- *isin = realloc(*isin, isinsize + 2 + strlen(item->value));
+ *isin = (char *)realloc(*isin, isinsize + 2 + strlen(item->value));
*(*isin+isinsize) = ',';
strcpy(*isin+1+isinsize, item->value);
isinsize += 1 + strlen(item->value);
- freeItem(item);
+ keyval::freeItem(item);
}
else if (strcmp(item->key, "admin_level") == 0)
{
*admin_level = atoi(item->value);
- freeItem(item);
+ keyval::freeItem(item);
}
else if (strcmp(item->key, "tracktype") == 0 ||
strcmp(item->key, "traffic_calming") == 0 ||
@@ -580,20 +569,20 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
(strncmp(item->key, "wikipedia:", 10) == 0)
)
{
- pushItem(extratags, item);
+ keyval::pushItem(extratags, item);
}
else if (strcmp(item->key, "building") == 0)
{
placebuilding = 1;
- freeItem(item);
+ keyval::freeItem(item);
}
else if (strcmp(item->key, "mountain_pass") == 0)
{
- pushItem(places, item);
+ keyval::pushItem(places, item);
}
else
{
- freeItem(item);
+ keyval::freeItem(item);
}
}
@@ -607,31 +596,31 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
{
if (*housenumber)
{
- freeItem(*housenumber);
+ keyval::freeItem(*housenumber);
}
if (!conscriptionnumber)
{
- addItem(tags, "addr:housenumber", streetnumber->value, 0);
- freeItem(streetnumber);
- *housenumber = popItem(tags);
+ keyval::addItem(tags, "addr:housenumber", streetnumber->value, 0);
+ keyval::freeItem(streetnumber);
+ *housenumber = keyval::popItem(tags);
}
if (!streetnumber)
{
- addItem(tags, "addr:housenumber", conscriptionnumber->value, 10);
- freeItem(conscriptionnumber);
- *housenumber = popItem(tags);
+ keyval::addItem(tags, "addr:housenumber", conscriptionnumber->value, 10);
+ keyval::freeItem(conscriptionnumber);
+ *housenumber = keyval::popItem(tags);
}
if (conscriptionnumber && streetnumber)
{
char * completenumber = strdup(conscriptionnumber->value);
size_t completenumberlength = strlen(completenumber);
- completenumber = realloc(completenumber, completenumberlength + 2 + strlen(streetnumber->value));
+ completenumber = (char *)realloc(completenumber, completenumberlength + 2 + strlen(streetnumber->value));
*(completenumber + completenumberlength) = '/';
strcpy(completenumber + completenumberlength + 1, streetnumber->value);
- freeItem(conscriptionnumber);
- freeItem(streetnumber);
- addItem(tags, "addr:housenumber", completenumber, 0);
- *housenumber = popItem(tags);
+ keyval::freeItem(conscriptionnumber);
+ keyval::freeItem(streetnumber);
+ keyval::addItem(tags, "addr:housenumber", completenumber, 0);
+ *housenumber = keyval::popItem(tags);
free(completenumber);
}
}
@@ -640,40 +629,40 @@ static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *na
{
if (placeadmin)
{
- pushItem(extratags, place);
- }
+ keyval::pushItem(extratags, place);
+ }
else
{
- pushItem(places, place);
+ keyval::pushItem(places, place);
}
}
- if (placehouse && !listHasData(places))
+ if (placehouse && !keyval::listHasData(places))
{
- addItem(places, "place", "house", 1);
+ keyval::addItem(places, "place", "house", 1);
}
/* Fallback place types - only used if we didn't create something more specific already */
- if (placebuilding && !listHasData(places) && (listHasData(names) || *housenumber || *postcode))
+ if (placebuilding && !keyval::listHasData(places) && (keyval::listHasData(names) || *housenumber || *postcode))
{
- addItem(places, "building", "yes", 1);
+ keyval::addItem(places, "building", "yes", 1);
}
if (landuse)
{
- if (!listHasData(places) && listHasData(names))
+ if (!keyval::listHasData(places) && keyval::listHasData(names))
{
- pushItem(places, landuse);
+ keyval::pushItem(places, landuse);
}
else
{
- freeItem(landuse);
+ keyval::freeItem(landuse);
}
}
- if (*postcode && !listHasData(places))
+ if (*postcode && !keyval::listHasData(places))
{
- addItem(places, "place", "postcode", 1);
+ keyval::addItem(places, "place", "postcode", 1);
}
/* Try to convert everything to an area */
@@ -691,10 +680,10 @@ void escape_array_record(char *out, int len, const char *in)
while(*in && count < len-3) {
switch(*in) {
case '\\': *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; count+= 8; break;
- case '\n':
- case '\r':
- case '\t':
- case '"':
+ case '\n':
+ case '\r':
+ case '\t':
+ case '"':
/* This is a bit naughty - we know that nominatim ignored these characters so just drop them now for simplicity */
*out++ = ' '; count++; break;
default: *out++ = *in; count++; break;
@@ -707,14 +696,14 @@ void escape_array_record(char *out, int len, const char *in)
fprintf(stderr, "%s truncated at %d chars: %s\n%s\n", __FUNCTION__, count, old_in, old_out);
}
-static void delete_unused_classes(char osm_type, osmid_t osm_id, struct keyval *places) {
+void output_gazetteer_t::delete_unused_classes(char osm_type, osmid_t osm_id, struct keyval *places) {
int i,sz, slen;
PGresult *res;
char tmp[16];
char tmp2[2];
char *cls, *clslist = 0;
char const *paramValues[2];
-
+
tmp2[0] = osm_type; tmp2[1] = '\0';
paramValues[0] = tmp2;
snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, osm_id);
@@ -730,14 +719,14 @@ static void delete_unused_classes(char osm_type, osmid_t osm_id, struct keyval *
} else {
for (i = 0; i < sz; i++) {
cls = PQgetvalue(res, i, 0);
- if (!getItem(places, cls)) {
+ if (!keyval::getItem(places, cls)) {
if (!clslist) {
- clslist = malloc(strlen(cls)+3);
+ clslist = (char *)malloc(strlen(cls)+3);
sprintf(clslist, "'%s'", cls);
} else {
slen = strlen(clslist);
- clslist = realloc(clslist, slen + 4 + strlen(cls));
- sprintf(&(clslist[slen]), ",'%s'", cls);
+ clslist = (char *)realloc(clslist, slen + 4 + strlen(cls));
+ sprintf(&(clslist[slen]), ",'%s'", cls);
}
}
}
@@ -756,7 +745,7 @@ static void delete_unused_classes(char osm_type, osmid_t osm_id, struct keyval *
}
}
-static void add_place(char osm_type, osmid_t osm_id, const char *class, const char *type, struct keyval *names, struct keyval *extratags,
+void output_gazetteer_t::add_place(char osm_type, osmid_t osm_id, const char *key_class, const char *type, struct keyval *names, struct keyval *extratags,
int adminlevel, struct keyval *housenumber, struct keyval *street, struct keyval *addr_place, const char *isin, struct keyval *postcode, struct keyval *countrycode, const char *wkt)
{
int first;
@@ -767,7 +756,7 @@ static void add_place(char osm_type, osmid_t osm_id, const char *class, const ch
sprintf(sql, "%c\t%" PRIdOSMID "\t", osm_type, osm_id);
copy_data(sql);
- escape(sql, sizeof(sql), class);
+ escape(sql, sizeof(sql), key_class);
copy_data(sql);
copy_data("\t");
@@ -776,10 +765,10 @@ static void add_place(char osm_type, osmid_t osm_id, const char *class, const ch
copy_data("\t");
/* start name array */
- if (listHasData(names))
+ if (keyval::listHasData(names))
{
first = 1;
- for (name = firstItem(names); name; name = nextItem(names, name))
+ for (name = keyval::firstItem(names); name; name = keyval::nextItem(names, name))
{
if (first) first = 0;
else copy_data(", ");
@@ -874,10 +863,10 @@ static void add_place(char osm_type, osmid_t osm_id, const char *class, const ch
}
/* extra tags array */
- if (listHasData(extratags))
+ if (keyval::listHasData(extratags))
{
first = 1;
- for (name = firstItem(extratags); name; name = nextItem(extratags, name))
+ for (name = keyval::firstItem(extratags); name; name = keyval::nextItem(extratags, name))
{
if (first) first = 0;
else copy_data(", ");
@@ -912,8 +901,10 @@ static void add_place(char osm_type, osmid_t osm_id, const char *class, const ch
}
#if 0
-static void add_polygon_error(char osm_type, osmid_t osm_id, const char *class, const char *type,
- struct keyval *names, const char *countrycode, const char *wkt)
+static void add_polygon_error(char osm_type, osmid_t osm_id,
+ const char *key_class, const char *type,
+ struct keyval *names, const char *countrycode,
+ const char *wkt)
{
int first;
struct keyval *name;
@@ -923,7 +914,7 @@ static void add_polygon_error(char osm_type, osmid_t osm_id, const char *class,
sprintf(sql, "%c\t%" PRIdOSMID "\t", osm_type, osm_id);
copy_error_data(sql);
- escape(sql, sizeof(sql), class);
+ escape(sql, sizeof(sql), key_class);
copy_error_data(sql);
copy_error_data("\t");
@@ -932,10 +923,10 @@ static void add_polygon_error(char osm_type, osmid_t osm_id, const char *class,
copy_error_data("\t");
/* start name array */
- if (listHasData(names))
+ if (keyval::listHasData(names))
{
first = 1;
- for (name = firstItem(names); name; name = nextItem(names, name))
+ for (name = keyval::firstItem(names); name; name = keyval::nextItem(names, name))
{
if (first) first = 0;
else copy_error_data(", ");
@@ -984,7 +975,7 @@ static void add_polygon_error(char osm_type, osmid_t osm_id, const char *class,
#endif
-static void delete_place(char osm_type, osmid_t osm_id)
+void output_gazetteer_t::delete_place(char osm_type, osmid_t osm_id)
{
/* Stop any active copy */
stop_copy();
@@ -995,26 +986,43 @@ static void delete_place(char osm_type, osmid_t osm_id)
return;
}
-static int gazetteer_out_start(const struct output_options *options)
-{
- /* Save option handle */
- Options = options;
+int output_gazetteer_t::connect() {
+ /* Connection to the database */
+ Connection = PQconnectdb(m_options.conninfo.c_str());
- /* Connection to the database */
- Connection = PQconnectdb(options->conninfo);
+ /* Check to see that the backend connection was successfully made */
+ if (PQstatus(Connection) != CONNECTION_OK)
+ {
+ fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(Connection));
+ return 1;
+ }
- /* Check to see that the backend connection was successfully made */
- if (PQstatus(Connection) != CONNECTION_OK)
- {
- fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(Connection));
- exit_nicely();
- }
+ if(m_options.append) {
+ ConnectionDelete = PQconnectdb(m_options.conninfo.c_str());
+ if (PQstatus(ConnectionDelete) != CONNECTION_OK)
+ {
+ fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(ConnectionDelete));
+ return 1;
+ }
+
+ pgsql_exec(ConnectionDelete, PGRES_COMMAND_OK, "PREPARE get_classes (CHAR(1), " POSTGRES_OSMID_TYPE ") AS SELECT class FROM place WHERE osm_type = $1 and osm_id = $2");
+ }
+ return 0;
+}
+
+int output_gazetteer_t::start()
+{
+ reproj = m_options.projection;
+ builder.set_exclude_broken_polygon(m_options.excludepoly);
+
+ if(connect())
+ util::exit_nicely();
/* Start a transaction */
pgsql_exec(Connection, PGRES_COMMAND_OK, "BEGIN");
/* (Re)create the table unless we are appending */
- if (!Options->append)
+ if (!m_options.append)
{
/* Drop any existing table */
pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS place");
@@ -1029,18 +1037,19 @@ static int gazetteer_out_start(const struct output_options *options)
pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_WORDSCORE_TYPE);
/* Create the new table */
- if (Options->tblsmain_data)
+ if (m_options.tblsmain_data)
{
pgsql_exec(Connection, PGRES_COMMAND_OK,
- CREATE_PLACE_TABLE, "TABLESPACE", Options->tblsmain_data);
+ CREATE_PLACE_TABLE, "TABLESPACE", m_options.tblsmain_data->c_str());
}
else
{
pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_TABLE, "", "");
}
- if (Options->tblsmain_index)
+ if (m_options.tblsmain_index)
{
- pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_ID_INDEX, "TABLESPACE", Options->tblsmain_index);
+ pgsql_exec(Connection, PGRES_COMMAND_OK,
+ CREATE_PLACE_ID_INDEX, "TABLESPACE", m_options.tblsmain_index->c_str());
}
else
{
@@ -1049,36 +1058,33 @@ static int gazetteer_out_start(const struct output_options *options)
pgsql_exec(Connection, PGRES_TUPLES_OK, "SELECT AddGeometryColumn('place', 'geometry', %d, 'GEOMETRY', 2)", SRID);
pgsql_exec(Connection, PGRES_COMMAND_OK, "ALTER TABLE place ALTER COLUMN geometry SET NOT NULL");
- } else {
- ConnectionDelete = PQconnectdb(options->conninfo);
- if (PQstatus(ConnectionDelete) != CONNECTION_OK)
- {
- fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(ConnectionDelete));
- exit_nicely();
- }
-
- pgsql_exec(ConnectionDelete, PGRES_COMMAND_OK, "PREPARE get_classes (CHAR(1), " POSTGRES_OSMID_TYPE ") AS SELECT class FROM place WHERE osm_type = $1 and osm_id = $2");
}
- /* Setup middle layer */
- options->mid->start(options);
-
- hLog = fopen("log", "w");
-
return 0;
}
-static void gazetteer_out_stop(void)
+void output_gazetteer_t::commit()
{
- /* Process any remaining ways and relations */
+}
- /* No longer need to access middle layer */
- Options->mid->commit();
- Options->mid->stop();
+void output_gazetteer_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+}
+
+int output_gazetteer_t::pending_way(osmid_t id, int exists) {
+ return 0;
+}
+
+void output_gazetteer_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+}
+
+int output_gazetteer_t::pending_relation(osmid_t id, int exists) {
+ return 0;
+}
+void output_gazetteer_t::stop()
+{
/* Stop any active copy */
stop_copy();
- if (hLog) fclose(hLog);
/* Commit transaction */
pgsql_exec(Connection, PGRES_COMMAND_OK, "COMMIT");
@@ -1093,12 +1099,7 @@ static void gazetteer_out_stop(void)
return;
}
-static void gazetteer_out_cleanup(void)
-{
- return;
-}
-
-static int gazetteer_process_node(osmid_t id, double lat, double lon, struct keyval *tags, int delete_old)
+int output_gazetteer_t::gazetteer_process_node(osmid_t id, double lat, double lon, struct keyval *tags, int delete_old)
{
struct keyval names;
struct keyval places;
@@ -1117,43 +1118,40 @@ static int gazetteer_process_node(osmid_t id, double lat, double lon, struct key
/* Split the tags */
split_tags(tags, TAGINFO_NODE, &names, &places, &extratags, &adminlevel, &housenumber, &street, &addr_place, &isin, &postcode, &countrycode);
- /* Feed this node to the middle layer */
- Options->mid->nodes_set(id, lat, lon, tags);
-
if (delete_old)
delete_unused_classes('N', id, &places);
/* Are we interested in this item? */
- if (listHasData(&places))
+ if (keyval::listHasData(&places))
{
sprintf(wkt, "POINT(%.15g %.15g)", lon, lat);
- for (place = firstItem(&places); place; place = nextItem(&places, place))
+ for (place = keyval::firstItem(&places); place; place = keyval::nextItem(&places, place))
{
add_place('N', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, addr_place, isin, postcode, countrycode, wkt);
}
}
- if (housenumber) freeItem(housenumber);
- if (street) freeItem(street);
- if (addr_place) freeItem(addr_place);
+ if (housenumber) keyval::freeItem(housenumber);
+ if (street) keyval::freeItem(street);
+ if (addr_place) keyval::freeItem(addr_place);
if (isin) free(isin);
- if (postcode) freeItem(postcode);
- if (countrycode) freeItem(countrycode);
+ if (postcode) keyval::freeItem(postcode);
+ if (countrycode) keyval::freeItem(countrycode);
/* Free tag lists */
- resetList(&names);
- resetList(&places);
- resetList(&extratags);
+ keyval::resetList(&names);
+ keyval::resetList(&places);
+ keyval::resetList(&extratags);
return 0;
}
-static int gazetteer_add_node(osmid_t id, double lat, double lon, struct keyval *tags)
+int output_gazetteer_t::node_add(osmid_t id, double lat, double lon, struct keyval *tags)
{
return gazetteer_process_node(id, lat, lon, tags, 0);
}
-static int gazetteer_process_way(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags, int delete_old)
+int output_gazetteer_t::gazetteer_process_way(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags, int delete_old)
{
struct keyval names;
struct keyval places;
@@ -1172,60 +1170,55 @@ static int gazetteer_process_way(osmid_t id, osmid_t *ndv, int ndc, struct keyva
/* Split the tags */
area = split_tags(tags, TAGINFO_WAY, &names, &places, &extratags, &adminlevel, &housenumber, &street, &addr_place, &isin, &postcode, &countrycode);
- /* Feed this way to the middle layer */
- Options->mid->ways_set(id, ndv, ndc, tags, 0);
-
if (delete_old)
delete_unused_classes('W', id, &places);
/* Are we interested in this item? */
- if (listHasData(&places))
+ if (keyval::listHasData(&places))
{
struct osmNode *nodev;
int nodec;
- char *wkt;
-
+
/* Fetch the node details */
- nodev = malloc(ndc * sizeof(struct osmNode));
- nodec = Options->mid->nodes_get_list(nodev, ndv, ndc);
+ nodev = (struct osmNode *)malloc(ndc * sizeof(struct osmNode));
+ nodec = m_mid->nodes_get_list(nodev, ndv, ndc);
/* Get the geometry of the object */
- if ((wkt = get_wkt_simple(nodev, nodec, area)) != NULL && strlen(wkt) > 0)
+ geometry_builder::maybe_wkt_t wkt = builder.get_wkt_simple(nodev, nodec, area);
+ if (wkt)
{
- for (place = firstItem(&places); place; place = nextItem(&places, place))
+ for (place = keyval::firstItem(&places); place; place = keyval::nextItem(&places, place))
{
- add_place('W', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, addr_place, isin, postcode, countrycode, wkt);
+ add_place('W', id, place->key, place->value, &names, &extratags, adminlevel,
+ housenumber, street, addr_place, isin, postcode, countrycode, wkt->geom.c_str());
}
}
- /* Free the geometry */
- free(wkt);
-
/* Free the nodes */
free(nodev);
}
- if (housenumber) freeItem(housenumber);
- if (street) freeItem(street);
- if (addr_place) freeItem(addr_place);
+ if (housenumber) keyval::freeItem(housenumber);
+ if (street) keyval::freeItem(street);
+ if (addr_place) keyval::freeItem(addr_place);
if (isin) free(isin);
- if (postcode) freeItem(postcode);
- if (countrycode) freeItem(countrycode);
+ if (postcode) keyval::freeItem(postcode);
+ if (countrycode) keyval::freeItem(countrycode);
/* Free tag lists */
- resetList(&names);
- resetList(&places);
- resetList(&extratags);
+ keyval::resetList(&names);
+ keyval::resetList(&places);
+ keyval::resetList(&extratags);
return 0;
}
-static int gazetteer_add_way(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags)
+int output_gazetteer_t::way_add(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags)
{
return gazetteer_process_way(id, ndv, ndc, tags, 0);
}
-static int gazetteer_process_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags, int delete_old)
+int output_gazetteer_t::gazetteer_process_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags, int delete_old)
{
struct keyval names;
struct keyval places;
@@ -1238,13 +1231,12 @@ static int gazetteer_process_relation(osmid_t id, struct member *members, int me
char * isin;
struct keyval * postcode;
struct keyval * countrycode;
- int wkt_size;
const char *type;
int cmp_waterway;
- type = getItem(tags, "type");
+ type = keyval::getItem(tags, "type");
if (!type) {
- if (delete_old) delete_unused_classes('R', id, 0);
+ if (delete_old) delete_unused_classes('R', id, 0);
return 0;
}
@@ -1252,33 +1244,34 @@ static int gazetteer_process_relation(osmid_t id, struct member *members, int me
if (!strcmp(type, "associatedStreet"))
{
- Options->mid->relations_set(id, members, member_count, tags);
- if (delete_old) delete_unused_classes('R', id, 0);
+ if (delete_old) delete_unused_classes('R', id, 0);
return 0;
}
if (strcmp(type, "boundary") && strcmp(type, "multipolygon") && cmp_waterway) {
- if (delete_old) delete_unused_classes('R', id, 0);
+ if (delete_old) delete_unused_classes('R', id, 0);
return 0;
}
- Options->mid->relations_set(id, members, member_count, tags);
-
/* Split the tags */
split_tags(tags, TAGINFO_AREA, &names, &places, &extratags, &adminlevel, &housenumber, &street, &addr_place, &isin, &postcode, &countrycode);
+ /* reset type to NULL because split_tags() consumes the tags
+ * keyval and means that it's pointing to some random stuff
+ * which might be harmful if dereferenced. */
+ type = NULL;
+
if (delete_old)
delete_unused_classes('R', id, &places);
- if (listHasData(&places))
+ if (keyval::listHasData(&places))
{
/* get the boundary path (ways) */
int i, count;
- int *xcount = malloc( (member_count+1) * sizeof(int) );
- struct keyval *xtags = malloc( (member_count+1) * sizeof(struct keyval) );
- struct osmNode **xnodes = malloc( (member_count+1) * sizeof(struct osmNode*) );
- osmid_t *xid;
- osmid_t *xid2 = malloc( (member_count+1) * sizeof(osmid_t) );
+ int *xcount = (int *)malloc( (member_count+1) * sizeof(int) );
+ keyval *xtags = new keyval[member_count+1];
+ struct osmNode **xnodes = (struct osmNode **)malloc( (member_count+1) * sizeof(struct osmNode*) );
+ osmid_t *xid2 = (osmid_t *)malloc( (member_count+1) * sizeof(osmid_t) );
count = 0;
for (i=0; i<member_count; i++)
@@ -1294,84 +1287,83 @@ static int gazetteer_process_relation(osmid_t id, struct member *members, int me
{
if (delete_old) delete_unused_classes('R', id, 0);
free(xcount);
- free(xtags);
+ delete [] xtags;
free(xnodes);
free(xid2);
return 0;
}
- count = Options->mid->ways_get_list(xid2, count, &xid, xtags, xnodes, xcount);
+ osmid_t *xid = (osmid_t *)malloc( sizeof(osmid_t) * (count + 1));
+ count = m_mid->ways_get_list(xid2, count, xid, xtags, xnodes, xcount);
xnodes[count] = NULL;
xcount[count] = 0;
if (cmp_waterway)
{
- wkt_size = build_geometry(id, xnodes, xcount, 1, 1, 1000000);
- for (i=0;i<wkt_size;i++)
- {
- char *wkt = get_wkt(i);
- if (strlen(wkt) && (!strncmp(wkt, "POLYGON", strlen("POLYGON")) || !strncmp(wkt, "MULTIPOLYGON", strlen("MULTIPOLYGON"))))
- {
- for (place = firstItem(&places); place; place = nextItem(&places, place))
- {
- add_place('R', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, addr_place, isin, postcode, countrycode, wkt);
- }
- }
- else
- {
- /* add_polygon_error('R', id, "boundary", "adminitrative", &names, countrycode, wkt); */
- }
- free(wkt);
- }
- clear_wkts();
+ geometry_builder::maybe_wkts_t wkts = builder.build_both(xnodes, xcount, 1, 1, 1000000, id);
+ for (geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt)
+ {
+ if ((boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON")))
+ {
+ for (place = keyval::firstItem(&places); place; place = keyval::nextItem(&places, place))
+ {
+ add_place('R', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, addr_place,
+ isin, postcode, countrycode, wkt->geom.c_str());
+ }
+ }
+ else
+ {
+ /* add_polygon_error('R', id, "boundary", "adminitrative", &names, countrycode, wkt); */
+ }
+ }
} else {
- /* waterways result in multilinestrings */
- char *wkt = get_multiline_geometry(id, xnodes, xcount);
- if (wkt && strlen(wkt))
- {
- for (place = firstItem(&places); place; place = nextItem(&places, place))
- {
- add_place('R', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, addr_place, isin, postcode, countrycode, wkt);
- }
- }
- free(wkt);
+ /* waterways result in multilinestrings */
+ // wkt_t build_multilines(const osmNode * const * xnodes, const int *xcount, osmid_t osm_id) const;
+ geometry_builder::maybe_wkt_t wkt = builder.build_multilines(xnodes, xcount, id);
+ if ((wkt->geom).length() > 0)
+ {
+ for (place = keyval::firstItem(&places); place; place = keyval::nextItem(&places, place))
+ {
+ add_place('R', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, addr_place,
+ isin, postcode, countrycode, wkt->geom.c_str());
+ }
+ }
}
-
for( i=0; i<count; i++ )
{
- resetList( &(xtags[i]) );
+ keyval::resetList( &(xtags[i]) );
free( xnodes[i] );
}
free(xid);
free(xid2);
free(xcount);
- free(xtags);
+ delete [] xtags;
free(xnodes);
}
- if (housenumber) freeItem(housenumber);
- if (street) freeItem(street);
- if (addr_place) freeItem(addr_place);
+ if (housenumber) keyval::freeItem(housenumber);
+ if (street) keyval::freeItem(street);
+ if (addr_place) keyval::freeItem(addr_place);
if (isin) free(isin);
- if (postcode) freeItem(postcode);
- if (countrycode) freeItem(countrycode);
+ if (postcode) keyval::freeItem(postcode);
+ if (countrycode) keyval::freeItem(countrycode);
/* Free tag lists */
- resetList(&names);
- resetList(&places);
- resetList(&extratags);
+ keyval::resetList(&names);
+ keyval::resetList(&places);
+ keyval::resetList(&extratags);
return 0;
}
-static int gazetteer_add_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags)
+int output_gazetteer_t::relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags)
{
return gazetteer_process_relation(id, members, member_count, tags, 0);
}
-static int gazetteer_delete_node(osmid_t id)
+int output_gazetteer_t::node_delete(osmid_t id)
{
/* Make sure we are in slim mode */
require_slim_mode();
@@ -1379,13 +1371,10 @@ static int gazetteer_delete_node(osmid_t id)
/* Delete all references to this node */
delete_place('N', id);
- /* Feed this delete to the middle layer */
- Options->mid->nodes_delete(id);
-
return 0;
}
-static int gazetteer_delete_way(osmid_t id)
+int output_gazetteer_t::way_delete(osmid_t id)
{
/* Make sure we are in slim mode */
require_slim_mode();
@@ -1393,13 +1382,10 @@ static int gazetteer_delete_way(osmid_t id)
/* Delete all references to this way */
delete_place('W', id);
- /* Feed this delete to the middle layer */
- Options->mid->ways_delete(id);
-
return 0;
}
-static int gazetteer_delete_relation(osmid_t id)
+int output_gazetteer_t::relation_delete(osmid_t id)
{
/* Make sure we are in slim mode */
require_slim_mode();
@@ -1407,47 +1393,58 @@ static int gazetteer_delete_relation(osmid_t id)
/* Delete all references to this relation */
delete_place('R', id);
- /* Feed this delete to the middle layer */
- Options->mid->relations_delete(id);
-
return 0;
}
-static int gazetteer_modify_node(osmid_t id, double lat, double lon, struct keyval *tags)
+int output_gazetteer_t::node_modify(osmid_t id, double lat, double lon, struct keyval *tags)
{
require_slim_mode();
- Options->mid->nodes_delete(id);
return gazetteer_process_node(id, lat, lon, tags, 1);
}
-static int gazetteer_modify_way(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags)
+int output_gazetteer_t::way_modify(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags)
{
require_slim_mode();
- Options->mid->ways_delete(id);
return gazetteer_process_way(id, ndv, ndc, tags, 1);
}
-static int gazetteer_modify_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags)
+int output_gazetteer_t::relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags)
{
require_slim_mode();
- Options->mid->relations_delete(id);
return gazetteer_process_relation(id, members, member_count, tags, 1);
}
-struct output_t out_gazetteer = {
- .start = gazetteer_out_start,
- .stop = gazetteer_out_stop,
- .cleanup = gazetteer_out_cleanup,
- .node_add = gazetteer_add_node,
- .way_add = gazetteer_add_way,
- .relation_add = gazetteer_add_relation,
+boost::shared_ptr<output_t> output_gazetteer_t::clone(const middle_query_t* cloned_middle) const {
+ output_gazetteer_t *clone = new output_gazetteer_t(*this);
+ clone->m_mid = cloned_middle;
+ return boost::shared_ptr<output_t>(clone);
+}
+
+output_gazetteer_t::output_gazetteer_t(const middle_query_t* mid_, const options_t &options_)
+ : output_t(mid_, options_),
+ Connection(NULL),
+ ConnectionDelete(NULL),
+ ConnectionError(NULL),
+ CopyActive(0),
+ BufferLen(0)
+{
+ memset(Buffer, 0, BUFFER_SIZE);
+}
- .node_modify = gazetteer_modify_node,
- .way_modify = gazetteer_modify_way,
- .relation_modify = gazetteer_modify_relation,
+output_gazetteer_t::output_gazetteer_t(const output_gazetteer_t& other)
+ : output_t(other.m_mid, other.m_options),
+ Connection(NULL),
+ ConnectionDelete(NULL),
+ ConnectionError(NULL),
+ CopyActive(0),
+ BufferLen(0),
+ reproj(other.reproj)
+{
+ builder.set_exclude_broken_polygon(m_options.excludepoly);
+ memset(Buffer, 0, BUFFER_SIZE);
+ connect();
+}
- .node_delete = gazetteer_delete_node,
- .way_delete = gazetteer_delete_way,
- .relation_delete = gazetteer_delete_relation
-};
+output_gazetteer_t::~output_gazetteer_t() {
+}
diff --git a/output-gazetteer.h b/output-gazetteer.h
deleted file mode 100644
index c678cf5..0000000
--- a/output-gazetteer.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef OUTPUT_GAZETTEER_H
-#define OUTPUT_GAZETTEER_H
-
-#include "output.h"
-
-struct output_t out_gazetteer;
-
-#endif
diff --git a/output-gazetteer.hpp b/output-gazetteer.hpp
new file mode 100644
index 0000000..af18a6d
--- /dev/null
+++ b/output-gazetteer.hpp
@@ -0,0 +1,79 @@
+#ifndef OUTPUT_GAZETTEER_H
+#define OUTPUT_GAZETTEER_H
+
+#include "output.hpp"
+#include "geometry-builder.hpp"
+#include "reprojection.hpp"
+
+#include <boost/shared_ptr.hpp>
+
+class output_gazetteer_t : public output_t {
+public:
+ output_gazetteer_t(const middle_query_t* mid_, const options_t &options_);
+ output_gazetteer_t(const output_gazetteer_t& other);
+ virtual ~output_gazetteer_t();
+
+ virtual boost::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const;
+
+ int start();
+ void stop();
+ void commit();
+
+ void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
+ int pending_way(osmid_t id, int exists);
+
+ void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
+ int pending_relation(osmid_t id, int exists);
+
+ int node_add(osmid_t id, double lat, double lon, struct keyval *tags);
+ int way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
+ int relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+
+ int node_modify(osmid_t id, double lat, double lon, struct keyval *tags);
+ int way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
+ int relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+
+ int node_delete(osmid_t id);
+ int way_delete(osmid_t id);
+ int relation_delete(osmid_t id);
+
+private:
+ static const size_t BUFFER_SIZE = 4096;
+
+ void require_slim_mode(void);
+ void copy_data(const char *sql);
+ void stop_copy(void);
+ void delete_unused_classes(char osm_type, osmid_t osm_id, struct keyval *places);
+ void add_place(char osm_type, osmid_t osm_id, const char *key_class, const char *type,
+ struct keyval *names, struct keyval *extratags, int adminlevel,
+ struct keyval *housenumber, struct keyval *street, struct keyval *addr_place,
+ const char *isin, struct keyval *postcode, struct keyval *countrycode,
+ const char *wkt);
+ void delete_place(char osm_type, osmid_t osm_id);
+ int gazetteer_process_node(osmid_t id, double lat, double lon, struct keyval *tags,
+ int delete_old);
+ int gazetteer_process_way(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags,
+ int delete_old);
+ int gazetteer_process_relation(osmid_t id, struct member *members, int member_count,
+ struct keyval *tags, int delete_old);
+ int connect();
+
+ struct pg_conn *Connection;
+ struct pg_conn *ConnectionDelete;
+ struct pg_conn *ConnectionError;
+
+ int CopyActive;
+ unsigned int BufferLen;
+
+ char Buffer[BUFFER_SIZE];
+
+ geometry_builder builder;
+
+ boost::shared_ptr<reprojection> reproj;
+
+ const static std::string NAME;
+};
+
+extern output_gazetteer_t out_gazetteer;
+
+#endif
diff --git a/output-multi.cpp b/output-multi.cpp
new file mode 100644
index 0000000..eae1799
--- /dev/null
+++ b/output-multi.cpp
@@ -0,0 +1,435 @@
+#include "output-multi.hpp"
+#include "taginfo_impl.hpp"
+
+#include <boost/format.hpp>
+#include <boost/foreach.hpp>
+#include <boost/make_shared.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <vector>
+
+namespace {
+
+std::string mk_column_name(const std::string &name, const options_t &options) {
+ return (boost::format("%1%_%2%") % options.prefix % name).str();
+}
+
+} // anonymous namespace
+
+output_multi_t::output_multi_t(const std::string &name,
+ boost::shared_ptr<geometry_processor> processor_,
+ const struct export_list &export_list_,
+ const middle_query_t* mid_, const options_t &options_)
+ : output_t(mid_, options_),
+ m_tagtransform(new tagtransform(&m_options)),
+ m_export_list(new export_list(export_list_)),
+ m_processor(processor_),
+ //TODO: we could in fact have something that is interested in nodes and ways..
+ m_osm_type(m_processor->interests(geometry_processor::interest_node) ? OSMTYPE_NODE : OSMTYPE_WAY),
+ m_table(new table_t(m_options.conninfo, mk_column_name(name, m_options), m_processor->column_type(),
+ m_export_list->normal_columns(m_osm_type),
+ m_options.hstore_columns, m_processor->srid(), m_options.scale,
+ m_options.append, m_options.slim, m_options.droptemp,
+ m_options.hstore_mode, m_options.enable_hstore_index,
+ m_options.tblsmain_data, m_options.tblsmain_index)),
+ ways_pending_tracker(new id_tracker()), ways_done_tracker(new id_tracker()), rels_pending_tracker(new id_tracker()),
+ m_expire(new expire_tiles(&m_options)) {
+}
+
+output_multi_t::output_multi_t(const output_multi_t& other):
+ output_t(other.m_mid, other.m_options), m_tagtransform(new tagtransform(&m_options)), m_export_list(new export_list(*other.m_export_list)),
+ m_processor(other.m_processor), m_osm_type(other.m_osm_type), m_table(new table_t(*other.m_table)),
+ ways_pending_tracker(new id_tracker()), ways_done_tracker(new id_tracker()), rels_pending_tracker(new id_tracker()),
+ m_expire(new expire_tiles(&m_options)) {
+}
+
+
+output_multi_t::~output_multi_t() {
+}
+
+boost::shared_ptr<output_t> output_multi_t::clone(const middle_query_t* cloned_middle) const{
+ output_multi_t *clone = new output_multi_t(*this);
+ clone->m_mid = cloned_middle;
+ //NOTE: we need to know which ways were used by relations so each thread
+ //must have a copy of the original marked done ways, its read only so its ok
+ clone->ways_done_tracker = ways_done_tracker;
+ return boost::shared_ptr<output_t>(clone);
+}
+
+int output_multi_t::start() {
+ m_table->start();
+ return 0;
+}
+
+size_t output_multi_t::pending_count() const {
+ return ways_pending_tracker->size() + rels_pending_tracker->size();
+}
+
+void output_multi_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+ //make sure we get the one passed in
+ if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
+ job_queue.push(pending_job_t(id, output_id));
+ added++;
+ }
+
+ //grab the first one or bail if its not valid
+ osmid_t popped = ways_pending_tracker->pop_mark();
+ if(!id_tracker::is_valid(popped))
+ return;
+
+ //get all the ones up to the id that was passed in
+ while (popped < id) {
+ if (!ways_done_tracker->is_marked(popped)) {
+ job_queue.push(pending_job_t(popped, output_id));
+ added++;
+ }
+ popped = ways_pending_tracker->pop_mark();
+ }
+
+ //make sure to get this one as well and move to the next
+ if(popped == id) {
+ popped = ways_pending_tracker->pop_mark();
+ }
+ if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
+ job_queue.push(pending_job_t(popped, output_id));
+ added++;
+ }
+}
+
+int output_multi_t::pending_way(osmid_t id, int exists) {
+ keyval tags_int;
+ osmNode *nodes_int;
+ int count_int;
+ int ret = 0;
+
+ keyval::initList(&tags_int);
+ // Try to fetch the way from the DB
+ if (!m_mid->ways_get(id, &tags_int, &nodes_int, &count_int)) {
+ // Output the way
+ ret = reprocess_way(id, nodes_int, count_int, &tags_int, exists);
+ free(nodes_int);
+ }
+ keyval::resetList(&tags_int);
+
+ return ret;
+}
+
+void output_multi_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+ //make sure we get the one passed in
+ if(id_tracker::is_valid(id)) {
+ job_queue.push(pending_job_t(id, output_id));
+ added++;
+ }
+
+ //grab the first one or bail if its not valid
+ osmid_t popped = rels_pending_tracker->pop_mark();
+ if(!id_tracker::is_valid(popped))
+ return;
+
+ //get all the ones up to the id that was passed in
+ while (popped < id) {
+ job_queue.push(pending_job_t(popped, output_id));
+ added++;
+ popped = rels_pending_tracker->pop_mark();
+ }
+
+ //make sure to get this one as well and move to the next
+ if(popped == id) {
+ popped = rels_pending_tracker->pop_mark();
+ }
+ if(id_tracker::is_valid(popped)) {
+ job_queue.push(pending_job_t(popped, output_id));
+ added++;
+ }
+}
+
+int output_multi_t::pending_relation(osmid_t id, int exists) {
+ keyval tags_int;
+ member *members_int;
+ int count_int;
+ int ret = 0;
+
+ keyval::initList(&tags_int);
+ // Try to fetch the relation from the DB
+ if (!m_mid->relations_get(id, &members_int, &count_int, &tags_int)) {
+ ret = process_relation(id, members_int, count_int, &tags_int, exists);
+ free(members_int);
+ }
+ keyval::resetList(&tags_int);
+
+ return ret;
+}
+
+void output_multi_t::stop() {
+ m_table->stop();
+ m_expire->output_and_destroy();
+ m_expire.reset();
+}
+
+void output_multi_t::commit() {
+ m_table->commit();
+}
+
+int output_multi_t::node_add(osmid_t id, double lat, double lon, struct keyval *tags) {
+ if (m_processor->interests(geometry_processor::interest_node)) {
+ return process_node(id, lat, lon, tags);
+ }
+ return 0;
+}
+
+int output_multi_t::way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags) {
+ if (m_processor->interests(geometry_processor::interest_way) && node_count > 1) {
+ return process_way(id, nodes, node_count, tags);
+ }
+ return 0;
+}
+
+
+int output_multi_t::relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags) {
+ if (m_processor->interests(geometry_processor::interest_relation) && member_count > 0) {
+ return process_relation(id, members, member_count, tags, 0);
+ }
+ return 0;
+}
+
+int output_multi_t::node_modify(osmid_t id, double lat, double lon, struct keyval *tags) {
+ if (m_processor->interests(geometry_processor::interest_node)) {
+ // TODO - need to know it's a node?
+ delete_from_output(id);
+
+ // TODO: need to mark any ways or relations using it - depends on what
+ // type of output this is... delegate to the geometry processor??
+ return process_node(id, lat, lon, tags);
+
+ } else {
+ return 0;
+ }
+}
+
+int output_multi_t::way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags) {
+ if (m_processor->interests(geometry_processor::interest_way)) {
+ // TODO - need to know it's a way?
+ delete_from_output(id);
+
+ // TODO: need to mark any relations using it - depends on what
+ // type of output this is... delegate to the geometry processor??
+ return process_way(id, nodes, node_count, tags);
+
+ } else {
+ return 0;
+ }
+}
+
+int output_multi_t::relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags) {
+ if (m_processor->interests(geometry_processor::interest_relation)) {
+ // TODO - need to know it's a relation?
+ delete_from_output(-id);
+
+ // TODO: need to mark any other relations using it - depends on what
+ // type of output this is... delegate to the geometry processor??
+ return process_relation(id, members, member_count, tags, false);
+
+ } else {
+ return 0;
+ }
+}
+
+int output_multi_t::node_delete(osmid_t id) {
+ if (m_processor->interests(geometry_processor::interest_node)) {
+ // TODO - need to know it's a node?
+ delete_from_output(id);
+ }
+ return 0;
+}
+
+int output_multi_t::way_delete(osmid_t id) {
+ if (m_processor->interests(geometry_processor::interest_way)) {
+ // TODO - need to know it's a way?
+ delete_from_output(id);
+ }
+ return 0;
+}
+
+int output_multi_t::relation_delete(osmid_t id) {
+ if (m_processor->interests(geometry_processor::interest_relation)) {
+ // TODO - need to know it's a relation?
+ delete_from_output(-id);
+ }
+ return 0;
+}
+
+int output_multi_t::process_node(osmid_t id, double lat, double lon, struct keyval *tags) {
+ //check if we are keeping this node
+ unsigned int filter = m_tagtransform->filter_node_tags(tags, m_export_list.get(), true);
+ if (!filter) {
+ //grab its geom
+ geometry_builder::maybe_wkt_t wkt = m_processor->process_node(lat, lon);
+ if (wkt) {
+ m_expire->from_bbox(lon, lat, lon, lat);
+ copy_to_table(id, wkt->geom.c_str(), tags);
+ }
+ }
+ return 0;
+}
+
+int output_multi_t::reprocess_way(osmid_t id, const osmNode* nodes, int node_count, struct keyval *tags, bool exists)
+{
+ //if the way could exist already we have to make the relation pending and reprocess it later
+ //but only if we actually care about relations
+ if(m_processor->interests(geometry_processor::interest_relation) && exists) {
+ way_delete(id);
+ const std::vector<osmid_t> rel_ids = m_mid->relations_using_way(id);
+ for (std::vector<osmid_t>::const_iterator itr = rel_ids.begin(); itr != rel_ids.end(); ++itr) {
+ rels_pending_tracker->mark(*itr);
+ }
+ }
+
+ //check if we are keeping this way
+ int polygon = 0, roads = 0;
+ unsigned int filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads, m_export_list.get(), true);
+ if (!filter) {
+ //grab its geom
+ geometry_builder::maybe_wkt_t wkt = m_processor->process_way(nodes, node_count);
+ if (wkt) {
+ //TODO: need to know if we care about polygons or lines for this output
+ //the difference only being that if its a really large bbox for the poly
+ //it downgrades to just invalidating the line/perimeter anyway
+ if(boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON"))
+ m_expire->from_nodes_poly(nodes, node_count, id);
+ else
+ m_expire->from_nodes_line(nodes, node_count);
+ copy_to_table(id, wkt->geom.c_str(), tags);
+ }
+ }
+ return 0;
+}
+
+int output_multi_t::process_way(osmid_t id, const osmid_t* node_ids, int node_count, struct keyval *tags) {
+ //check if we are keeping this way
+ int polygon = 0, roads = 0;
+ unsigned int filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads, m_export_list.get(), true);
+ if (!filter) {
+ //get the geom from the middle
+ if(m_way_helper.set(node_ids, node_count, m_mid) < 1)
+ return 0;
+ //grab its geom
+ geometry_builder::maybe_wkt_t wkt = m_processor->process_way(&m_way_helper.node_cache.front(), m_way_helper.node_cache.size());
+
+ if (wkt) {
+ //if we are also interested in relations we need to mark
+ //this way pending just in case it shows up in one
+ if (m_processor->interests(geometry_processor::interest_relation)) {
+ ways_pending_tracker->mark(id);
+ }//we aren't interested in relations so if it comes in on a relation later we wont keep it
+ else {
+ //TODO: need to know if we care about polygons or lines for this output
+ //the difference only being that if its a really large bbox for the poly
+ //it downgrades to just invalidating the line/perimeter anyway
+ if(boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON"))
+ m_expire->from_nodes_poly(&m_way_helper.node_cache.front(), m_way_helper.node_cache.size(), id);
+ else
+ m_expire->from_nodes_line(&m_way_helper.node_cache.front(), m_way_helper.node_cache.size());
+ copy_to_table(id, wkt->geom.c_str(), tags);
+ }
+ }
+ }
+ return 0;
+}
+
+int output_multi_t::process_relation(osmid_t id, const member *members, int member_count, keyval *tags, bool exists, bool pending) {
+ //if it may exist already, delete it first
+ if(exists)
+ relation_delete(id);
+
+ //does this relation have anything interesting to us
+ unsigned int filter = m_tagtransform->filter_rel_tags(tags, m_export_list.get(), true);
+ if (!filter) {
+ //TODO: move this into geometry processor, figure a way to come back for tag transform
+ //grab ways/nodes of the members in the relation, bail if none were used
+ if(m_relation_helper.set(members, member_count, (middle_t*)m_mid) < 1)
+ return 0;
+
+ //filter the tags on each member because we got them from the middle
+ //and since the middle is no longer tied to the output it no longer
+ //shares any kind of tag transform and therefore has all original tags
+ //so we filter here because each individual outputs cares about different tags
+ int polygon, roads;
+ for(size_t i = 0; i < m_relation_helper.way_count; ++i)
+ {
+ m_tagtransform->filter_way_tags(&m_relation_helper.tags[i], &polygon, &roads, m_export_list.get());
+ //TODO: if the filter says that this member is now not interesting we
+ //should decrement the count and remove his nodes and tags etc. for
+ //now we'll just keep him with no tags so he will get filtered later
+ }
+
+ //do the members of this relation have anything interesting to us
+ //NOTE: make_polygon is preset here this is to force the tag matching/superseeded stuff
+ //normally this wouldnt work but we tell the tag transform to allow typeless relations
+ //this is needed because the type can get stripped off by the rel_tag filter above
+ //if the export list did not include the type tag.
+ //TODO: find a less hacky way to do the matching/superseeded and tag copying stuff without
+ //all this trickery
+ int make_boundary, make_polygon = 1;
+ filter = m_tagtransform->filter_rel_member_tags(tags, m_relation_helper.way_count, &m_relation_helper.tags.front(),
+ &m_relation_helper.roles.front(), &m_relation_helper.superseeded.front(),
+ &make_boundary, &make_polygon, &roads, m_export_list.get(), true);
+ if(!filter)
+ {
+ geometry_builder::maybe_wkts_t wkts = m_processor->process_relation(&m_relation_helper.nodes.front(), &m_relation_helper.node_counts.front());
+ if (wkts) {
+ for(geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt)
+ {
+ //TODO: we actually have the nodes in the m_relation_helper and could use them
+ //instead of having to reparse the wkt in the expiry code
+ m_expire->from_wkt(wkt->geom.c_str(), -id);
+ //what part of the code relies on relation members getting negative ids?
+ copy_to_table(-id, wkt->geom.c_str(), tags);
+ }
+ }
+
+ //TODO: should this loop be inside the if above just in case?
+ //take a look at each member to see if its superseeded (tags on it matched the tags on the relation)
+ for(size_t i = 0; i < m_relation_helper.way_count; ++i) {
+ //tags matched so we are keeping this one with this relation
+ if (m_relation_helper.superseeded[i]) {
+ //just in case it wasnt previously with this relation we get rid of them
+ way_delete(m_relation_helper.ways[i]);
+ //the other option is that we marked them pending in the way processing so here we mark them
+ //done so when we go back over the pendings we can just skip it because its in the done list
+ //TODO: dont do this when working with pending relations to avoid thread races
+ if(!pending)
+ ways_done_tracker->mark(m_relation_helper.ways[i]);
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+void output_multi_t::copy_to_table(osmid_t id, const char *wkt, struct keyval *tags) {
+ m_table->write_wkt(id, tags, wkt);
+}
+
+void output_multi_t::delete_from_output(osmid_t id) {
+ if(m_expire->from_db(m_table.get(), id))
+ m_table->delete_row(id);
+}
+
+void output_multi_t::merge_pending_relations(boost::shared_ptr<output_t> other) {
+ boost::shared_ptr<id_tracker> tracker = other.get()->get_pending_relations();
+ osmid_t id;
+ while(tracker.get() && id_tracker::is_valid((id = tracker->pop_mark()))){
+ rels_pending_tracker->mark(id);
+ }
+}
+
+void output_multi_t::merge_expire_trees(boost::shared_ptr<output_t> other) {
+ if(other->get_expire_tree().get())
+ m_expire->merge_and_destroy(*other.get()->get_expire_tree());
+}
+
+boost::shared_ptr<id_tracker> output_multi_t::get_pending_relations() {
+ return rels_pending_tracker;
+}
+boost::shared_ptr<expire_tiles> output_multi_t::get_expire_tree() {
+ return m_expire;
+}
diff --git a/output-multi.hpp b/output-multi.hpp
new file mode 100644
index 0000000..021f97a
--- /dev/null
+++ b/output-multi.hpp
@@ -0,0 +1,83 @@
+/* One implementation of output-layer processing for osm2pgsql.
+ * Manages a single table, transforming geometry using a
+ * variety of algorithms plus tag transformation for the
+ * database columns.
+ */
+
+#ifndef OUTPUT_MULTI_HPP
+#define OUTPUT_MULTI_HPP
+
+#include "output.hpp"
+#include "tagtransform.hpp"
+#include "table.hpp"
+#include "geometry-processor.hpp"
+#include "id-tracker.hpp"
+#include "expire-tiles.hpp"
+
+#include <vector>
+#include <boost/scoped_ptr.hpp>
+#include <boost/variant.hpp>
+
+class output_multi_t : public output_t {
+public:
+ output_multi_t(const std::string &name,
+ boost::shared_ptr<geometry_processor> processor_,
+ const struct export_list &export_list_,
+ const middle_query_t* mid_, const options_t &options_);
+ output_multi_t(const output_multi_t& other);
+ virtual ~output_multi_t();
+
+ virtual boost::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const;
+
+ int start();
+ void stop();
+ void commit();
+
+ void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
+ int pending_way(osmid_t id, int exists);
+
+ void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
+ int pending_relation(osmid_t id, int exists);
+
+ int node_add(osmid_t id, double lat, double lon, struct keyval *tags);
+ int way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
+ int relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+
+ int node_modify(osmid_t id, double lat, double lon, struct keyval *tags);
+ int way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
+ int relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+
+ int node_delete(osmid_t id);
+ int way_delete(osmid_t id);
+ int relation_delete(osmid_t id);
+
+ size_t pending_count() const;
+
+ void merge_pending_relations(boost::shared_ptr<output_t> other);
+ void merge_expire_trees(boost::shared_ptr<output_t> other);
+ virtual boost::shared_ptr<id_tracker> get_pending_relations();
+ virtual boost::shared_ptr<expire_tiles> get_expire_tree();
+
+protected:
+
+ void delete_from_output(osmid_t id);
+ int process_node(osmid_t id, double lat, double lon, struct keyval *tags);
+ int process_way(osmid_t id, const osmid_t* node_ids, int node_count, struct keyval *tags);
+ int reprocess_way(osmid_t id, const osmNode* nodes, int node_count, struct keyval *tags, bool exists);
+ int process_relation(osmid_t id, const member *members, int member_count, struct keyval *tags, bool exists, bool pending=false);
+ void copy_to_table(osmid_t id, const char *wkt, struct keyval *tags);
+
+ boost::scoped_ptr<tagtransform> m_tagtransform;
+ boost::scoped_ptr<export_list> m_export_list;
+ boost::shared_ptr<geometry_processor> m_processor;
+ const OsmType m_osm_type;
+ boost::scoped_ptr<table_t> m_table;
+ boost::shared_ptr<id_tracker> ways_pending_tracker, ways_done_tracker, rels_pending_tracker;
+ boost::shared_ptr<expire_tiles> m_expire;
+ way_helper m_way_helper;
+ relation_helper m_relation_helper;
+
+ const static std::string NAME;
+};
+
+#endif
diff --git a/output-null.c b/output-null.c
deleted file mode 100644
index 277742b..0000000
--- a/output-null.c
+++ /dev/null
@@ -1,75 +0,0 @@
-#include <stdio.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <errno.h>
-
-#include "osmtypes.h"
-#include "output.h"
-#include "output-null.h"
-
-#define UNUSED __attribute__ ((unused))
-
-static void null_out_cleanup(void) {
-}
-
-static int null_out_start(const struct output_options *opt UNUSED) {
- return 0;
-}
-
-static void null_out_stop() {
-}
-
-static int null_add_node(osmid_t a UNUSED, double b UNUSED, double c UNUSED, struct keyval *k UNUSED) {
- return 0;
-}
-
-static int null_add_way(osmid_t a UNUSED, osmid_t *b UNUSED, int c UNUSED, struct keyval *k UNUSED) {
- return 0;
-}
-
-static int null_add_relation(osmid_t a UNUSED, struct member *b UNUSED, int c UNUSED, struct keyval *k UNUSED) {
- return 0;
-}
-
-static int null_delete_node(osmid_t i UNUSED) {
- return 0;
-}
-
-static int null_delete_way(osmid_t i UNUSED) {
- return 0;
-}
-
-static int null_delete_relation(osmid_t i UNUSED) {
- return 0;
-}
-
-static int null_modify_node(osmid_t a UNUSED, double b UNUSED, double c UNUSED, struct keyval * k UNUSED) {
- return 0;
-}
-
-static int null_modify_way(osmid_t a UNUSED, osmid_t * b UNUSED, int c UNUSED, struct keyval * k UNUSED) {
- return 0;
-}
-
-static int null_modify_relation(osmid_t a UNUSED, struct member * b UNUSED, int c UNUSED, struct keyval * k UNUSED) {
- return 0;
-}
-
-struct output_t out_null = {
- .start = null_out_start,
- .stop = null_out_stop,
- .cleanup = null_out_cleanup,
- .node_add = null_add_node,
- .way_add = null_add_way,
- .relation_add = null_add_relation,
-
- .node_modify = null_modify_node,
- .way_modify = null_modify_way,
- .relation_modify = null_modify_relation,
-
- .node_delete = null_delete_node,
- .way_delete = null_delete_way,
- .relation_delete = null_delete_relation
-};
diff --git a/output-null.cpp b/output-null.cpp
new file mode 100644
index 0000000..5d73859
--- /dev/null
+++ b/output-null.cpp
@@ -0,0 +1,90 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+
+#include <boost/make_shared.hpp>
+
+#include "osmtypes.hpp"
+#include "output-null.hpp"
+#include "options.hpp"
+
+void output_null_t::cleanup() {
+}
+
+int output_null_t::start() {
+ return 0;
+}
+
+void output_null_t::stop() {
+}
+
+void output_null_t::commit() {
+}
+
+void output_null_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+}
+
+int output_null_t::pending_way(osmid_t id, int exists) {
+ return 0;
+}
+
+void output_null_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+}
+
+int output_null_t::pending_relation(osmid_t id, int exists) {
+ return 0;
+}
+
+int output_null_t::node_add(osmid_t a, double b, double c, struct keyval *k) {
+ return 0;
+}
+
+int output_null_t::way_add(osmid_t a, osmid_t *b, int c, struct keyval *k) {
+ return 0;
+}
+
+int output_null_t::relation_add(osmid_t a, struct member *b, int c, struct keyval *k) {
+ return 0;
+}
+
+int output_null_t::node_delete(osmid_t i) {
+ return 0;
+}
+
+int output_null_t::way_delete(osmid_t i) {
+ return 0;
+}
+
+int output_null_t::relation_delete(osmid_t i) {
+ return 0;
+}
+
+int output_null_t::node_modify(osmid_t a, double b, double c, struct keyval * k) {
+ return 0;
+}
+
+int output_null_t::way_modify(osmid_t a, osmid_t *b, int c, struct keyval *k) {
+ return 0;
+}
+
+int output_null_t::relation_modify(osmid_t a, struct member *b, int c, struct keyval *k) {
+ return 0;
+}
+
+boost::shared_ptr<output_t> output_null_t::clone(const middle_query_t* cloned_middle) const {
+ output_null_t *clone = new output_null_t(*this);
+ clone->m_mid = cloned_middle;
+ return boost::shared_ptr<output_t>(clone);
+}
+
+output_null_t::output_null_t(const middle_query_t* mid_, const options_t &options_): output_t(mid_, options_) {
+}
+
+output_null_t::output_null_t(const output_null_t& other): output_t(other.m_mid, other.m_options) {
+}
+
+output_null_t::~output_null_t() {
+}
diff --git a/output-null.h b/output-null.h
deleted file mode 100644
index 95a6bde..0000000
--- a/output-null.h
+++ /dev/null
@@ -1,11 +0,0 @@
-/* Implements dummy output-layer processing for testing.
-*/
-
-#ifndef OUTPUT_NULL_H
-#define OUTPUT_NULL_H
-
-#include "output.h"
-
-extern struct output_t out_null;
-
-#endif
diff --git a/output-null.hpp b/output-null.hpp
new file mode 100644
index 0000000..e735aed
--- /dev/null
+++ b/output-null.hpp
@@ -0,0 +1,41 @@
+/* Implements dummy output-layer processing for testing.
+*/
+
+#ifndef OUTPUT_NULL_H
+#define OUTPUT_NULL_H
+
+#include "output.hpp"
+
+class output_null_t : public output_t {
+public:
+ output_null_t(const middle_query_t* mid_, const options_t &options);
+ output_null_t(const output_null_t& other);
+ virtual ~output_null_t();
+
+ virtual boost::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const;
+
+ int start();
+ void stop();
+ void commit();
+ void cleanup(void);
+
+ void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
+ int pending_way(osmid_t id, int exists);
+
+ void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
+ int pending_relation(osmid_t id, int exists);
+
+ int node_add(osmid_t id, double lat, double lon, struct keyval *tags);
+ int way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
+ int relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+
+ int node_modify(osmid_t id, double lat, double lon, struct keyval *tags);
+ int way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
+ int relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+
+ int node_delete(osmid_t id);
+ int way_delete(osmid_t id);
+ int relation_delete(osmid_t id);
+};
+
+#endif
diff --git a/output-pgsql.c b/output-pgsql.c
deleted file mode 100644
index 827ae54..0000000
--- a/output-pgsql.c
+++ /dev/null
@@ -1,1488 +0,0 @@
-/* Implements the mid-layer processing for osm2pgsql
- * using several PostgreSQL tables
- *
- * This layer stores data read in from the planet.osm file
- * and is then read by the backend processing code to
- * emit the final geometry-enabled output formats
-*/
-
-#include "config.h"
-
-#include <stdio.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <errno.h>
-#include <time.h>
-
-#ifdef HAVE_PTHREAD
-#include <pthread.h>
-#endif
-
-#include <libpq-fe.h>
-
-#include "osmtypes.h"
-#include "output.h"
-#include "reprojection.h"
-#include "output-pgsql.h"
-#include "build_geometry.h"
-#include "middle.h"
-#include "pgsql.h"
-#include "expire-tiles.h"
-#include "wildcmp.h"
-#include "node-ram-cache.h"
-#include "tagtransform.h"
-
-#define SRID (project_getprojinfo()->srs)
-
-/* FIXME: Shouldn't malloc this all to begin with but call realloc()
- as required. The program will most likely segfault if it reads a
- style file with more styles than this */
-#define MAX_STYLES 1000
-
-enum table_id {
- t_point, t_line, t_poly, t_roads
-};
-
-static const struct output_options *Options;
-
-/* enable output of a generated way_area tag to either hstore or its own column */
-static int enable_way_area=1;
-
-/* Tables to output */
-static struct s_table {
- char *name;
- const char *type;
- PGconn *sql_conn;
- char buffer[1024];
- unsigned int buflen;
- int copyMode;
- char *columns;
-} tables [] = {
- { .name = "%s_point", .type = "POINT" },
- { .name = "%s_line", .type = "LINESTRING"},
- { .name = "%s_polygon", .type = "GEOMETRY" }, /* Actually POLGYON & MULTIPOLYGON but no way to limit to just these two */
- { .name = "%s_roads", .type = "LINESTRING"}
-};
-#define NUM_TABLES ((signed)(sizeof(tables) / sizeof(tables[0])))
-
-
-static struct flagsname {
- char *name;
- int flag;
-} tagflags[] = {
- { .name = "polygon", .flag = FLAG_POLYGON },
- { .name = "linear", .flag = FLAG_LINEAR },
- { .name = "nocache", .flag = FLAG_NOCACHE },
- { .name = "delete", .flag = FLAG_DELETE },
- { .name = "phstore", .flag = FLAG_PHSTORE }
-};
-#define NUM_FLAGS ((signed)(sizeof(tagflags) / sizeof(tagflags[0])))
-
-
-
-struct taginfo *exportList[4]; /* Indexed by enum table_id */
-int exportListCount[4];
-
-static int pgsql_delete_way_from_output(osmid_t osm_id);
-static int pgsql_delete_relation_from_output(osmid_t osm_id);
-static int pgsql_process_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags, int exists);
-
-void read_style_file( const char *filename )
-{
- FILE *in;
- int lineno = 0;
- int num_read = 0;
- char osmtype[24];
- char tag[64];
- char datatype[24];
- char flags[128];
- int i;
- char *str;
- int fields;
- struct taginfo temp;
- char buffer[1024];
- int flag = 0;
-
- exportList[OSMTYPE_NODE] = malloc( sizeof(struct taginfo) * MAX_STYLES );
- exportList[OSMTYPE_WAY] = malloc( sizeof(struct taginfo) * MAX_STYLES );
-
- in = fopen( filename, "rt" );
- if( !in )
- {
- fprintf( stderr, "Couldn't open style file '%s': %s\n", filename, strerror(errno) );
- exit_nicely();
- }
-
- while( fgets( buffer, sizeof(buffer), in) != NULL )
- {
- lineno++;
-
- str = strchr( buffer, '#' );
- if( str )
- *str = '\0';
-
- fields = sscanf( buffer, "%23s %63s %23s %127s", osmtype, tag, datatype, flags );
- if( fields <= 0 ) /* Blank line */
- continue;
- if( fields < 3 )
- {
- fprintf( stderr, "Error reading style file line %d (fields=%d)\n", lineno, fields );
- exit_nicely();
- }
- temp.name = strdup(tag);
- temp.type = strdup(datatype);
-
- temp.flags = 0;
- for( str = strtok( flags, ",\r\n" ); str; str = strtok(NULL, ",\r\n") )
- {
- for( i=0; i<NUM_FLAGS; i++ )
- {
- if( strcmp( tagflags[i].name, str ) == 0 )
- {
- temp.flags |= tagflags[i].flag;
- break;
- }
- }
- if( i == NUM_FLAGS )
- fprintf( stderr, "Unknown flag '%s' line %d, ignored\n", str, lineno );
- }
- if ((temp.flags!=FLAG_DELETE) && ((strchr(temp.name,'?') != NULL) || (strchr(temp.name,'*') != NULL))) {
- fprintf( stderr, "wildcard '%s' in non-delete style entry\n",temp.name);
- exit_nicely();
- }
-
- if ((0==strcmp(temp.name,"way_area")) && (temp.flags==FLAG_DELETE)) {
- enable_way_area=0;
- }
-
- temp.count = 0;
- /* printf("%s %s %d %d\n", temp.name, temp.type, temp.polygon, offset ); */
-
- if( strstr( osmtype, "node" ) )
- {
- memcpy( &exportList[ OSMTYPE_NODE ][ exportListCount[ OSMTYPE_NODE ] ], &temp, sizeof(temp) );
- exportListCount[ OSMTYPE_NODE ]++;
- flag = 1;
- }
- if( strstr( osmtype, "way" ) )
- {
- memcpy( &exportList[ OSMTYPE_WAY ][ exportListCount[ OSMTYPE_WAY ] ], &temp, sizeof(temp) );
- exportListCount[ OSMTYPE_WAY ]++;
- flag = 1;
- }
- if( !flag )
- {
- fprintf( stderr, "Weird style line %d\n", lineno );
- exit_nicely();
- }
- num_read++;
- }
- if (ferror(in)) {
- perror(filename);
- exit_nicely();
- }
- if (num_read == 0) {
- fprintf(stderr, "Unable to parse any valid columns from the style file. Aborting.\n");
- exit_nicely();
- }
- fclose(in);
-}
-
-static void free_style_refs(const char *name, const char *type)
-{
- /* Find and remove any other references to these pointers
- This would be way easier if we kept a single list of styles
- Currently this scales with n^2 number of styles */
- int i,j;
-
- for (i=0; i<NUM_TABLES; i++) {
- for(j=0; j<exportListCount[i]; j++) {
- if (exportList[i][j].name == name)
- exportList[i][j].name = NULL;
- if (exportList[i][j].type == type)
- exportList[i][j].type = NULL;
- }
- }
-}
-
-static void free_style(void)
-{
- int i, j;
- for (i=0; i<NUM_TABLES; i++) {
- for(j=0; j<exportListCount[i]; j++) {
- free(exportList[i][j].name);
- free(exportList[i][j].type);
- free_style_refs(exportList[i][j].name, exportList[i][j].type);
- }
- }
- for (i=0; i<NUM_TABLES; i++)
- free(exportList[i]);
-}
-
-/* Handles copying out, but coalesces the data into large chunks for
- * efficiency. PostgreSQL doesn't actually need this, but each time you send
- * a block of data you get 5 bytes of overhead. Since we go column by column
- * with most empty and one byte delimiters, without this optimisation we
- * transfer three times the amount of data necessary.
- */
-void copy_to_table(enum table_id table, const char *sql)
-{
- PGconn *sql_conn = tables[table].sql_conn;
- unsigned int len = strlen(sql);
- unsigned int buflen = tables[table].buflen;
- char *buffer = tables[table].buffer;
-
- /* Return to copy mode if we dropped out */
- if( !tables[table].copyMode )
- {
- pgsql_exec(sql_conn, PGRES_COPY_IN, "COPY %s (%s,way) FROM STDIN", tables[table].name, tables[table].columns);
- tables[table].copyMode = 1;
- }
- /* If the combination of old and new data is too big, flush old data */
- if( (unsigned)(buflen + len) > sizeof( tables[table].buffer )-10 )
- {
- pgsql_CopyData(tables[table].name, sql_conn, buffer);
- buflen = 0;
-
- /* If new data by itself is also too big, output it immediately */
- if( (unsigned)len > sizeof( tables[table].buffer )-10 )
- {
- pgsql_CopyData(tables[table].name, sql_conn, sql);
- len = 0;
- }
- }
- /* Normal case, just append to buffer */
- if( len > 0 )
- {
- strcpy( buffer+buflen, sql );
- buflen += len;
- len = 0;
- }
-
- /* If we have completed a line, output it */
- if( buflen > 0 && buffer[buflen-1] == '\n' )
- {
- pgsql_CopyData(tables[table].name, sql_conn, buffer);
- buflen = 0;
- }
-
- tables[table].buflen = buflen;
-}
-
-
-
-
-static void pgsql_out_cleanup(void)
-{
- int i;
-
- for (i=0; i<NUM_TABLES; i++) {
- if (tables[i].sql_conn) {
- PQfinish(tables[i].sql_conn);
- tables[i].sql_conn = NULL;
- }
- }
-}
-
-/* Escape data appropriate to the type */
-static void escape_type(char *sql, int len, const char *value, const char *type) {
- int items;
- static int tmplen=0;
- static char *tmpstr;
-
- if (len > tmplen) {
- tmpstr=realloc(tmpstr,len);
- tmplen=len;
- }
- strcpy(tmpstr,value);
-
- if ( !strcmp(type, "int4") ) {
- int from, to;
- /* For integers we take the first number, or the average if it's a-b */
- items = sscanf(value, "%d-%d", &from, &to);
- if ( items == 1 ) {
- sprintf(sql, "%d", from);
- } else if ( items == 2 ) {
- sprintf(sql, "%d", (from + to) / 2);
- } else {
- sprintf(sql, "\\N");
- }
- } else {
- /*
- try to "repair" real values as follows:
- * assume "," to be a decimal mark which need to be replaced by "."
- * like int4 take the first number, or the average if it's a-b
- * assume SI unit (meters)
- * convert feet to meters (1 foot = 0.3048 meters)
- * reject anything else
- */
- if ( !strcmp(type, "real") ) {
- int i,slen;
- float from,to;
-
- slen=strlen(value);
- for (i=0;i<slen;i++) if (tmpstr[i]==',') tmpstr[i]='.';
-
- items = sscanf(tmpstr, "%f-%f", &from, &to);
- if ( items == 1 ) {
- if ((tmpstr[slen-2]=='f') && (tmpstr[slen-1]=='t')) {
- from*=0.3048;
- }
- sprintf(sql, "%f", from);
- } else if ( items == 2 ) {
- if ((tmpstr[slen-2]=='f') && (tmpstr[slen-1]=='t')) {
- from*=0.3048;
- to*=0.3048;
- }
- sprintf(sql, "%f", (from + to) / 2);
- } else {
- sprintf(sql, "\\N");
- }
- } else {
- escape(sql, len, value);
- }
- }
-}
-
-static void write_hstore(enum table_id table, struct keyval *tags)
-{
- static char *sql;
- static size_t sqllen=0;
- size_t hlen;
- /* a clone of the tags pointer */
- struct keyval *xtags = tags;
-
- /* sql buffer */
- if (sqllen==0) {
- sqllen=2048;
- sql=malloc(sqllen);
- }
-
- /* while this tags has a follow-up.. */
- while (xtags->next->key != NULL)
- {
-
- /* hard exclude z_order tag and keys which have their own column */
- if ((xtags->next->has_column) || (strcmp("z_order",xtags->next->key)==0)) {
- /* update the tag-pointer to point to the next tag */
- xtags = xtags->next;
- continue;
- }
-
- /*
- hstore ASCII representation looks like
- "<key>"=>"<value>"
-
- we need at least strlen(key)+strlen(value)+6+'\0' bytes
- in theory any single character could also be escaped
- thus we need an additional factor of 2.
- The maximum lenght of a single hstore element is thus
- calcuated as follows:
- */
- hlen=2 * (strlen(xtags->next->key) + strlen(xtags->next->value)) + 7;
-
- /* if the sql buffer is too small */
- if (hlen > sqllen) {
- sqllen = hlen;
- sql = realloc(sql, sqllen);
- }
-
- /* pack the tag with its value into the hstore */
- keyval2hstore(sql, xtags->next);
- copy_to_table(table, sql);
-
- /* update the tag-pointer to point to the next tag */
- xtags = xtags->next;
-
- /* if the tag has a follow up, add a comma to the end */
- if (xtags->next->key != NULL)
- copy_to_table(table, ",");
- }
-
- /* finish the hstore column by placing a TAB into the data stream */
- copy_to_table(table, "\t");
-
- /* the main hstore-column has now been written */
-}
-
-/* write an hstore column to the database */
-static void write_hstore_columns(enum table_id table, struct keyval *tags)
-{
- static char *sql;
- static size_t sqllen=0;
- char *shortkey;
- /* the index of the current hstore column */
- int i_hstore_column;
- int found;
- struct keyval *xtags;
- char *pos;
- size_t hlen;
-
- /* sql buffer */
- if (sqllen==0) {
- sqllen=2048;
- sql=malloc(sqllen);
- }
-
- /* iterate over all configured hstore colums in the options */
- for(i_hstore_column = 0; i_hstore_column < Options->n_hstore_columns; i_hstore_column++)
- {
- /* did this node have a tag that matched the current hstore column */
- found = 0;
-
- /* a clone of the tags pointer */
- xtags = tags;
-
- /* while this tags has a follow-up.. */
- while (xtags->next->key != NULL) {
-
- /* check if the tag's key starts with the name of the hstore column */
- pos = strstr(xtags->next->key, Options->hstore_columns[i_hstore_column]);
-
- /* and if it does.. */
- if(pos == xtags->next->key)
- {
- /* remember we found one */
- found=1;
-
- /* generate the short key name */
- shortkey = xtags->next->key + strlen(Options->hstore_columns[i_hstore_column]);
-
- /* calculate the size needed for this hstore entry */
- hlen=2*(strlen(shortkey)+strlen(xtags->next->value))+7;
-
- /* if the sql buffer is too small */
- if (hlen > sqllen) {
- /* resize it */
- sqllen=hlen;
- sql=realloc(sql,sqllen);
- }
-
- /* and pack the shortkey with its value into the hstore */
- keyval2hstore_manual(sql, shortkey, xtags->next->value);
- copy_to_table(table, sql);
-
- /* update the tag-pointer to point to the next tag */
- xtags=xtags->next;
-
- /* if the tag has a follow up, add a comma to the end */
- if (xtags->next->key != NULL)
- copy_to_table(table, ",");
- }
- else
- {
- /* update the tag-pointer to point to the next tag */
- xtags=xtags->next;
- }
- }
-
- /* if no matching tag has been found, write a NULL */
- if(!found)
- copy_to_table(table, "\\N");
-
- /* finish the hstore column by placing a TAB into the data stream */
- copy_to_table(table, "\t");
- }
-
- /* all hstore-columns have now been written */
-}
-
-
-/* example from: pg_dump -F p -t planet_osm gis
-COPY planet_osm (osm_id, name, place, landuse, leisure, "natural", man_made, waterway, highway, railway, amenity, tourism, learning, building, bridge, layer, way) FROM stdin;
-17959841 \N \N \N \N \N \N \N bus_stop \N \N \N \N \N \N -\N 0101000020E610000030CCA462B6C3D4BF92998C9B38E04940
-17401934 The Horn \N \N \N \N \N \N \N \N pub \N \N \N \N -\N 0101000020E6100000C12FC937140FD5BFB4D2F4FB0CE04940
-...
-
-mine - 01 01000000 48424298424242424242424256427364
-psql - 01 01000020 E6100000 30CCA462B6C3D4BF92998C9B38E04940
- 01 01000020 E6100000 48424298424242424242424256427364
-0x2000_0000 = hasSRID, following 4 bytes = srid, not supported by geos WKBWriter
-Workaround - output SRID=4326;<WKB>
-*/
-
-static int pgsql_out_node(osmid_t id, struct keyval *tags, double node_lat, double node_lon)
-{
-
- int filter = tagtransform_filter_node_tags(tags);
- static char *sql;
- static size_t sqllen=0;
- int i;
- struct keyval *tag;
-
- if (filter) return 1;
-
- if (sqllen==0) {
- sqllen=2048;
- sql=malloc(sqllen);
- }
-
- expire_tiles_from_bbox(node_lon, node_lat, node_lon, node_lat);
- sprintf(sql, "%" PRIdOSMID "\t", id);
- copy_to_table(t_point, sql);
-
- for (i=0; i < exportListCount[OSMTYPE_NODE]; i++) {
- if( exportList[OSMTYPE_NODE][i].flags & FLAG_DELETE )
- continue;
- if( (exportList[OSMTYPE_NODE][i].flags & FLAG_PHSTORE) == FLAG_PHSTORE)
- continue;
- if ((tag = getTag(tags, exportList[OSMTYPE_NODE][i].name)))
- {
- escape_type(sql, sqllen, tag->value, exportList[OSMTYPE_NODE][i].type);
- exportList[OSMTYPE_NODE][i].count++;
- if (HSTORE_NORM==Options->enable_hstore)
- tag->has_column=1;
- }
- else
- sprintf(sql, "\\N");
-
- copy_to_table(t_point, sql);
- copy_to_table(t_point, "\t");
- }
-
- /* hstore columns */
- write_hstore_columns(t_point, tags);
-
- /* check if a regular hstore is requested */
- if (Options->enable_hstore)
- write_hstore(t_point, tags);
-
-#ifdef FIXED_POINT
- // guarantee that we use the same values as in the node cache
- scale = Options->scale;
- node_lon = FIX_TO_DOUBLE(DOUBLE_TO_FIX(node_lon));
- node_lat = FIX_TO_DOUBLE(DOUBLE_TO_FIX(node_lat));
-#endif
-
- sprintf(sql, "SRID=%d;POINT(%.15g %.15g)", SRID, node_lon, node_lat);
- copy_to_table(t_point, sql);
- copy_to_table(t_point, "\n");
-
- return 0;
-}
-
-
-
-static void write_wkts(osmid_t id, struct keyval *tags, const char *wkt, enum table_id table)
-{
-
- static char *sql;
- static size_t sqllen=0;
- int j;
- struct keyval *tag;
-
- if (sqllen==0) {
- sqllen=2048;
- sql=malloc(sqllen);
- }
-
- sprintf(sql, "%" PRIdOSMID "\t", id);
- copy_to_table(table, sql);
-
- for (j=0; j < exportListCount[OSMTYPE_WAY]; j++) {
- if( exportList[OSMTYPE_WAY][j].flags & FLAG_DELETE )
- continue;
- if( (exportList[OSMTYPE_WAY][j].flags & FLAG_PHSTORE) == FLAG_PHSTORE)
- continue;
- if ((tag = getTag(tags, exportList[OSMTYPE_WAY][j].name)))
- {
- exportList[OSMTYPE_WAY][j].count++;
- escape_type(sql, sqllen, tag->value, exportList[OSMTYPE_WAY][j].type);
- if (HSTORE_NORM==Options->enable_hstore)
- tag->has_column=1;
- }
- else
- sprintf(sql, "\\N");
-
- copy_to_table(table, sql);
- copy_to_table(table, "\t");
- }
-
- /* hstore columns */
- write_hstore_columns(table, tags);
-
- /* check if a regular hstore is requested */
- if (Options->enable_hstore)
- write_hstore(table, tags);
-
- sprintf(sql, "SRID=%d;", SRID);
- copy_to_table(table, sql);
- copy_to_table(table, wkt);
- copy_to_table(table, "\n");
-}
-
-/*static int tag_indicates_polygon(enum OsmType type, const char *key)
-{
- int i;
-
- if (!strcmp(key, "area"))
- return 1;
-
- for (i=0; i < exportListCount[type]; i++) {
- if( strcmp( exportList[type][i].name, key ) == 0 )
- return exportList[type][i].flags & FLAG_POLYGON;
- }
-
- return 0;
-}*/
-
-
-
-/*
-COPY planet_osm (osm_id, name, place, landuse, leisure, "natural", man_made, waterway, highway, railway, amenity, tourism, learning, bu
-ilding, bridge, layer, way) FROM stdin;
-198497 Bedford Road \N \N \N \N \N \N residential \N \N \N \N \N \N \N 0102000020E610000004000000452BF702B342D5BF1C60E63BF8DF49406B9C4D470037D5BF5471E316F3DF4940DFA815A6EF35D5BF9AE95E27F5DF4940B41EB
-E4C1421D5BF24D06053E7DF4940
-212696 Oswald Road \N \N \N \N \N \N minor \N \N \N \N \N \N \N 0102000020E610000004000000467D923B6C22D5BFA359D93EE4DF4940B3976DA7AD11D5BF84BBB376DBDF4940997FF44D9A06D5BF4223D8B8FEDF49404D158C4AEA04D
-5BF5BB39597FCDF4940
-*/
-static int pgsql_out_way(osmid_t id, struct keyval *tags, struct osmNode *nodes, int count, int exists)
-{
- int polygon = 0, roads = 0;
- int i, wkt_size;
- double split_at;
- double area;
-
- /* If the flag says this object may exist already, delete it first */
- if(exists) {
- pgsql_delete_way_from_output(id);
- Options->mid->way_changed(id);
- }
-
- if (tagtransform_filter_way_tags(tags, &polygon, &roads))
- return 0;
- /* Split long ways after around 1 degree or 100km */
- if (Options->projection == PROJ_LATLONG)
- split_at = 1;
- else
- split_at = 100 * 1000;
-
- wkt_size = get_wkt_split(nodes, count, polygon, split_at);
-
- for (i=0;i<wkt_size;i++)
- {
- char *wkt = get_wkt(i);
-
- if (wkt && strlen(wkt)) {
- /* FIXME: there should be a better way to detect polygons */
- if (!strncmp(wkt, "POLYGON", strlen("POLYGON")) || !strncmp(wkt, "MULTIPOLYGON", strlen("MULTIPOLYGON"))) {
- expire_tiles_from_nodes_poly(nodes, count, id);
- area = get_area(i);
- if ((area > 0.0) && enable_way_area) {
- char tmp[32];
- snprintf(tmp, sizeof(tmp), "%g", area);
- addItem(tags, "way_area", tmp, 0);
- }
- write_wkts(id, tags, wkt, t_poly);
- } else {
- expire_tiles_from_nodes_line(nodes, count);
- write_wkts(id, tags, wkt, t_line);
- if (roads)
- write_wkts(id, tags, wkt, t_roads);
- }
- }
- free(wkt);
- }
- clear_wkts();
-
- return 0;
-}
-
-static int pgsql_out_relation(osmid_t id, struct keyval *rel_tags, int member_count, struct osmNode **xnodes, struct keyval *xtags, int *xcount, osmid_t *xid, const char **xrole)
-{
- int i, wkt_size;
- int roads = 0;
- int make_polygon = 0;
- int make_boundary = 0;
- int * members_superseeded;
- double split_at;
-
- members_superseeded = calloc(sizeof(int), member_count);
-
- if (member_count == 0) {
- free(members_superseeded);
- return 0;
- }
-
- if (tagtransform_filter_rel_member_tags(rel_tags, member_count, xtags, xrole, members_superseeded, &make_boundary, &make_polygon, &roads)) {
- free(members_superseeded);
- return 0;
- }
-
- /* Split long linear ways after around 1 degree or 100km (polygons not effected) */
- if (Options->projection == PROJ_LATLONG)
- split_at = 1;
- else
- split_at = 100 * 1000;
-
- wkt_size = build_geometry(id, xnodes, xcount, make_polygon, Options->enable_multi, split_at);
-
- if (!wkt_size) {
- free(members_superseeded);
- return 0;
- }
-
- for (i=0;i<wkt_size;i++) {
- char *wkt = get_wkt(i);
-
- if (wkt && strlen(wkt)) {
- expire_tiles_from_wkt(wkt, -id);
- /* FIXME: there should be a better way to detect polygons */
- if (!strncmp(wkt, "POLYGON", strlen("POLYGON")) || !strncmp(wkt, "MULTIPOLYGON", strlen("MULTIPOLYGON"))) {
- double area = get_area(i);
- if ((area > 0.0) && enable_way_area) {
- char tmp[32];
- snprintf(tmp, sizeof(tmp), "%g", area);
- addItem(rel_tags, "way_area", tmp, 0);
- }
- write_wkts(-id, rel_tags, wkt, t_poly);
- } else {
- write_wkts(-id, rel_tags, wkt, t_line);
- if (roads)
- write_wkts(-id, rel_tags, wkt, t_roads);
- }
- }
- free(wkt);
- }
-
- clear_wkts();
-
- /* Tagtransform will have marked those member ways of the relation that
- * have fully been dealt with as part of the multi-polygon entry.
- * Set them in the database as done and delete their entry to not
- * have duplicates */
- if (make_polygon) {
- for (i=0; xcount[i]; i++) {
- if (members_superseeded[i]) {
- Options->mid->ways_done(xid[i]);
- pgsql_delete_way_from_output(xid[i]);
- }
- }
- }
-
- free(members_superseeded);
-
- /* If we are making a boundary then also try adding any relations which form complete rings
- The linear variants will have already been processed above */
- if (make_boundary) {
- wkt_size = build_geometry(id, xnodes, xcount, 1, Options->enable_multi, split_at);
- for (i=0;i<wkt_size;i++)
- {
- char *wkt = get_wkt(i);
-
- if (strlen(wkt)) {
- expire_tiles_from_wkt(wkt, -id);
- /* FIXME: there should be a better way to detect polygons */
- if (!strncmp(wkt, "POLYGON", strlen("POLYGON")) || !strncmp(wkt, "MULTIPOLYGON", strlen("MULTIPOLYGON"))) {
- double area = get_area(i);
- if ((area > 0.0) && enable_way_area) {
- char tmp[32];
- snprintf(tmp, sizeof(tmp), "%g", area);
- addItem(rel_tags, "way_area", tmp, 0);
- }
- write_wkts(-id, rel_tags, wkt, t_poly);
- }
- }
- free(wkt);
- }
- clear_wkts();
- }
-
- return 0;
-}
-
-static int pgsql_out_connect(const struct output_options *options, int startTransaction) {
- int i;
- for (i=0; i<NUM_TABLES; i++) {
- PGconn *sql_conn;
- sql_conn = PQconnectdb(options->conninfo);
-
- /* Check to see that the backend connection was successfully made */
- if (PQstatus(sql_conn) != CONNECTION_OK) {
- fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(sql_conn));
- return 1;
- }
- tables[i].sql_conn = sql_conn;
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;");
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "PREPARE get_wkt (" POSTGRES_OSMID_TYPE ") AS SELECT ST_AsText(way) FROM %s WHERE osm_id = $1;\n", tables[i].name);
- if (startTransaction)
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "BEGIN");
- }
- return 0;
-}
-
-static int pgsql_out_start(const struct output_options *options)
-{
- char *sql, tmp[256];
- PGresult *res;
- int i,j;
- unsigned int sql_len;
- int their_srid;
- int i_hstore_column;
- enum OsmType type;
- int numTags;
- struct taginfo *exportTags;
-
- Options = options;
-
- read_style_file( options->style );
-
- sql_len = 2048;
- sql = malloc(sql_len);
- assert(sql);
-
- for (i=0; i<NUM_TABLES; i++) {
- PGconn *sql_conn;
-
- /* Substitute prefix into name of table */
- {
- char *temp = malloc( strlen(options->prefix) + strlen(tables[i].name) + 1 );
- sprintf( temp, tables[i].name, options->prefix );
- tables[i].name = temp;
- }
- fprintf(stderr, "Setting up table: %s\n", tables[i].name);
- sql_conn = PQconnectdb(options->conninfo);
-
- /* Check to see that the backend connection was successfully made */
- if (PQstatus(sql_conn) != CONNECTION_OK) {
- fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(sql_conn));
- exit_nicely();
- }
- tables[i].sql_conn = sql_conn;
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;");
-
- if (!options->append) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS %s", tables[i].name);
- }
- else
- {
- sprintf(sql, "SELECT srid FROM geometry_columns WHERE f_table_name='%s' AND f_geometry_column='way';", tables[i].name);
- res = PQexec(sql_conn, sql);
- if (!((PQntuples(res) == 1) && (PQnfields(res) == 1)))
- {
- fprintf(stderr, "Problem reading geometry information for table %s - does it exist?\n", tables[i].name);
- exit_nicely();
- }
- their_srid = atoi(PQgetvalue(res, 0, 0));
- PQclear(res);
- if (their_srid != SRID)
- {
- fprintf(stderr, "SRID mismatch: cannot append to table %s (SRID %d) using selected SRID %d\n", tables[i].name, their_srid, SRID);
- exit_nicely();
- }
- }
-
- /* These _tmp tables can be left behind if we run out of disk space */
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS %s_tmp", tables[i].name);
-
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "BEGIN");
-
- type = (i == t_point)?OSMTYPE_NODE:OSMTYPE_WAY;
- numTags = exportListCount[type];
- exportTags = exportList[type];
- if (!options->append) {
- sprintf(sql, "CREATE TABLE %s ( osm_id " POSTGRES_OSMID_TYPE, tables[i].name );
- for (j=0; j < numTags; j++) {
- if( exportTags[j].flags & FLAG_DELETE )
- continue;
- if( (exportTags[j].flags & FLAG_PHSTORE ) == FLAG_PHSTORE)
- continue;
- sprintf(tmp, ",\"%s\" %s", exportTags[j].name, exportTags[j].type);
- if (strlen(sql) + strlen(tmp) + 1 > sql_len) {
- sql_len *= 2;
- sql = realloc(sql, sql_len);
- assert(sql);
- }
- strcat(sql, tmp);
- }
- for(i_hstore_column = 0; i_hstore_column < Options->n_hstore_columns; i_hstore_column++)
- {
- strcat(sql, ",\"");
- strcat(sql, Options->hstore_columns[i_hstore_column]);
- strcat(sql, "\" hstore ");
- }
- if (Options->enable_hstore) {
- strcat(sql, ",tags hstore");
- }
- strcat(sql, ")");
- if (Options->tblsmain_data) {
- sprintf(sql + strlen(sql), " TABLESPACE %s", Options->tblsmain_data);
- }
- strcat(sql, "\n");
-
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", sql);
- pgsql_exec(sql_conn, PGRES_TUPLES_OK, "SELECT AddGeometryColumn('%s', 'way', %d, '%s', 2 );\n",
- tables[i].name, SRID, tables[i].type );
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ALTER TABLE %s ALTER COLUMN way SET NOT NULL;\n", tables[i].name);
- /* slim mode needs this to be able to apply diffs */
- if (Options->slim && !Options->droptemp) {
- sprintf(sql, "CREATE INDEX %s_pkey ON %s USING BTREE (osm_id)", tables[i].name, tables[i].name);
- if (Options->tblsmain_index) {
- sprintf(sql + strlen(sql), " TABLESPACE %s\n", Options->tblsmain_index);
- }
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", sql);
- }
- } else {
- /* Add any new columns referenced in the default.style */
- PGresult *res;
- sprintf(sql, "SELECT * FROM %s LIMIT 0;\n", tables[i].name);
- res = PQexec(sql_conn, sql);
- if (PQresultStatus(res) != PGRES_TUPLES_OK) {
- fprintf(stderr, "Error, failed to query table %s\n%s\n", tables[i].name, sql);
- exit_nicely();
- }
- for (j=0; j < numTags; j++) {
- if( exportTags[j].flags & FLAG_DELETE )
- continue;
- if( (exportTags[j].flags & FLAG_PHSTORE) == FLAG_PHSTORE)
- continue;
- sprintf(tmp, "\"%s\"", exportTags[j].name);
- if (PQfnumber(res, tmp) < 0) {
-#if 0
- fprintf(stderr, "Append failed. Column \"%s\" is missing from \"%s\"\n", exportTags[j].name, tables[i].name);
- exit_nicely();
-#else
- fprintf(stderr, "Adding new column \"%s\" to \"%s\"\n", exportTags[j].name, tables[i].name);
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ALTER TABLE %s ADD COLUMN \"%s\" %s;\n", tables[i].name, exportTags[j].name, exportTags[j].type);
-#endif
- }
- /* Note: we do not verify the type or delete unused columns */
- }
-
- PQclear(res);
-
- /* change the type of the geometry column if needed - this can only change to a more permisive type */
- }
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "PREPARE get_wkt (" POSTGRES_OSMID_TYPE ") AS SELECT ST_AsText(way) FROM %s WHERE osm_id = $1;\n", tables[i].name);
-
- /* Generate column list for COPY */
- strcpy(sql, "osm_id");
- for (j=0; j < numTags; j++) {
- if( exportTags[j].flags & FLAG_DELETE )
- continue;
- if( (exportTags[j].flags & FLAG_PHSTORE ) == FLAG_PHSTORE)
- continue;
- sprintf(tmp, ",\"%s\"", exportTags[j].name);
-
- if (strlen(sql) + strlen(tmp) + 1 > sql_len) {
- sql_len *= 2;
- sql = realloc(sql, sql_len);
- assert(sql);
- }
- strcat(sql, tmp);
- }
-
- for(i_hstore_column = 0; i_hstore_column < Options->n_hstore_columns; i_hstore_column++)
- {
- strcat(sql, ",\"");
- strcat(sql, Options->hstore_columns[i_hstore_column]);
- strcat(sql, "\" ");
- }
-
- if (Options->enable_hstore) strcat(sql,",tags");
-
- tables[i].columns = strdup(sql);
- pgsql_exec(sql_conn, PGRES_COPY_IN, "COPY %s (%s,way) FROM STDIN", tables[i].name, tables[i].columns);
-
- tables[i].copyMode = 1;
- }
- free(sql);
-
- if (tagtransform_init(options)) {
- fprintf(stderr, "Error: Failed to initialise tag processing.\n");
- exit_nicely();
- }
- expire_tiles_init(options);
-
- options->mid->start(options);
-
- return 0;
-}
-
-static void pgsql_pause_copy(struct s_table *table)
-{
- PGresult *res;
- int stop;
-
- if( !table->copyMode )
- return;
-
- /* Terminate any pending COPY */
- stop = PQputCopyEnd(table->sql_conn, NULL);
- if (stop != 1) {
- fprintf(stderr, "COPY_END for %s failed: %s\n", table->name, PQerrorMessage(table->sql_conn));
- exit_nicely();
- }
-
- res = PQgetResult(table->sql_conn);
- if (PQresultStatus(res) != PGRES_COMMAND_OK) {
- fprintf(stderr, "COPY_END for %s failed: %s\n", table->name, PQerrorMessage(table->sql_conn));
- PQclear(res);
- exit_nicely();
- }
- PQclear(res);
- table->copyMode = 0;
-}
-
-static void pgsql_out_close(int stopTransaction) {
- int i;
- for (i=0; i<NUM_TABLES; i++) {
- pgsql_pause_copy(&tables[i]);
- /* Commit transaction */
- if (stopTransaction)
- pgsql_exec(tables[i].sql_conn, PGRES_COMMAND_OK, "COMMIT");
- PQfinish(tables[i].sql_conn);
- tables[i].sql_conn = NULL;
- }
-}
-
-static void pgsql_out_commit(void) {
- int i;
- for (i=0; i<NUM_TABLES; i++) {
- pgsql_pause_copy(&tables[i]);
- /* Commit transaction */
- fprintf(stderr, "Committing transaction for %s\n", tables[i].name);
- pgsql_exec(tables[i].sql_conn, PGRES_COMMAND_OK, "COMMIT");
- }
-}
-
-static void *pgsql_out_stop_one(void *arg)
-{
- int i_column;
- struct s_table *table = arg;
- PGconn *sql_conn = table->sql_conn;
-
- if( table->buflen != 0 )
- {
- fprintf( stderr, "Internal error: Buffer for %s has %d bytes after end copy", table->name, table->buflen );
- exit_nicely();
- }
-
- pgsql_pause_copy(table);
- if (!Options->append)
- {
- time_t start, end;
- time(&start);
- fprintf(stderr, "Sorting data and creating indexes for %s\n", table->name);
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ANALYZE %s;\n", table->name);
- fprintf(stderr, "Analyzing %s finished\n", table->name);
- if (Options->tblsmain_data) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE TABLE %s_tmp "
- "TABLESPACE %s AS SELECT * FROM %s ORDER BY way;\n",
- table->name, Options->tblsmain_data, table->name);
- } else {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE TABLE %s_tmp AS SELECT * FROM %s ORDER BY way;\n", table->name, table->name);
- }
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE %s;\n", table->name);
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ALTER TABLE %s_tmp RENAME TO %s;\n", table->name, table->name);
- fprintf(stderr, "Copying %s to cluster by geometry finished\n", table->name);
- fprintf(stderr, "Creating geometry index on %s\n", table->name);
- if (Options->tblsmain_index) {
- /* Use fillfactor 100 for un-updatable imports */
- if (Options->slim && !Options->droptemp) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_index ON %s USING GIST (way) TABLESPACE %s;\n", table->name, table->name, Options->tblsmain_index);
- } else {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_index ON %s USING GIST (way) WITH (FILLFACTOR=100) TABLESPACE %s;\n", table->name, table->name, Options->tblsmain_index);
- }
- } else {
- if (Options->slim && !Options->droptemp) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_index ON %s USING GIST (way);\n", table->name, table->name);
- } else {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_index ON %s USING GIST (way) WITH (FILLFACTOR=100);\n", table->name, table->name);
- }
- }
-
- /* slim mode needs this to be able to apply diffs */
- if (Options->slim && !Options->droptemp)
- {
- fprintf(stderr, "Creating osm_id index on %s\n", table->name);
- if (Options->tblsmain_index) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_pkey ON %s USING BTREE (osm_id) TABLESPACE %s;\n", table->name, table->name, Options->tblsmain_index);
- } else {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_pkey ON %s USING BTREE (osm_id);\n", table->name, table->name);
- }
- }
- /* Create hstore index if selected */
- if (Options->enable_hstore_index) {
- fprintf(stderr, "Creating hstore indexes on %s\n", table->name);
- if (Options->tblsmain_index) {
- if (HSTORE_NONE != (Options->enable_hstore)) {
- if (Options->slim && !Options->droptemp) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_tags_index ON %s USING GIN (tags) TABLESPACE %s;\n", table->name, table->name, Options->tblsmain_index);
- } else {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_tags_index ON %s USING GIN (tags) TABLESPACE %s;\n", table->name, table->name, Options->tblsmain_index);
- }
- }
- for(i_column = 0; i_column < Options->n_hstore_columns; i_column++) {
- if (Options->slim && !Options->droptemp) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_hstore_%i_index ON %s USING GIN (\"%s\") TABLESPACE %s;\n",
- table->name, i_column,table->name, Options->hstore_columns[i_column], Options->tblsmain_index);
- } else {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_hstore_%i_index ON %s USING GIN (\"%s\") TABLESPACE %s;\n",
- table->name, i_column,table->name, Options->hstore_columns[i_column], Options->tblsmain_index);
- }
- }
- } else {
- if (HSTORE_NONE != (Options->enable_hstore)) {
- if (Options->slim && !Options->droptemp) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_tags_index ON %s USING GIN (tags);\n", table->name, table->name);
- } else {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_tags_index ON %s USING GIN (tags) ;\n", table->name, table->name);
- }
- }
- for(i_column = 0; i_column < Options->n_hstore_columns; i_column++) {
- if (Options->slim && !Options->droptemp) {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_hstore_%i_index ON %s USING GIN (\"%s\");\n", table->name, i_column,table->name, Options->hstore_columns[i_column]);
- } else {
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_hstore_%i_index ON %s USING GIN (\"%s\");\n", table->name, i_column,table->name, Options->hstore_columns[i_column]);
- }
- }
- }
- }
- fprintf(stderr, "Creating indexes on %s finished\n", table->name);
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "GRANT SELECT ON %s TO PUBLIC;\n", table->name);
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ANALYZE %s;\n", table->name);
- time(&end);
- fprintf(stderr, "All indexes on %s created in %ds\n", table->name, (int)(end - start));
- }
- PQfinish(sql_conn);
- table->sql_conn = NULL;
-
- fprintf(stderr, "Completed %s\n", table->name);
- free(table->name);
- free(table->columns);
- return NULL;
-}
-
-static void pgsql_out_stop()
-{
- int i;
-#ifdef HAVE_PTHREAD
- pthread_t threads[NUM_TABLES];
-#endif
-
- /* Commit the transactions, so that multiple processes can
- * access the data simultanious to process the rest in parallel
- * as well as see the newly created tables.
- */
- pgsql_out_commit();
- Options->mid->commit();
- /* To prevent deadlocks in parallel processing, the mid tables need
- * to stay out of a transaction. In this stage output tables are only
- * written to and not read, so they can be processed as several parallel
- * independent transactions
- */
- for (i=0; i<NUM_TABLES; i++) {
- PGconn *sql_conn = tables[i].sql_conn;
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "BEGIN");
- }
- /* Processing any remaing to be processed ways */
- Options->mid->iterate_ways( pgsql_out_way );
- pgsql_out_commit();
- Options->mid->commit();
-
- /* Processing any remaing to be processed relations */
- /* During this stage output tables also need to stay out of
- * extended transactions, as the delete_way_from_output, called
- * from process_relation, can deadlock if using multi-processing.
- */
- Options->mid->iterate_relations( pgsql_process_relation );
-
- tagtransform_shutdown();
-
-#ifdef HAVE_PTHREAD
- if (Options->parallel_indexing) {
- for (i=0; i<NUM_TABLES; i++) {
- int ret = pthread_create(&threads[i], NULL, pgsql_out_stop_one, &tables[i]);
- if (ret) {
- fprintf(stderr, "pthread_create() returned an error (%d)", ret);
- exit_nicely();
- }
- }
-
- /* No longer need to access middle layer -- release memory */
- Options->mid->stop();
-
- for (i=0; i<NUM_TABLES; i++) {
- int ret = pthread_join(threads[i], NULL);
- if (ret) {
- fprintf(stderr, "pthread_join() returned an error (%d)", ret);
- exit_nicely();
- }
- }
- } else {
-#endif
-
- /* No longer need to access middle layer -- release memory */
- Options->mid->stop();
- for (i=0; i<NUM_TABLES; i++)
- pgsql_out_stop_one(&tables[i]);
-
-#ifdef HAVE_PTHREAD
- }
-#endif
-
-
- pgsql_out_cleanup();
- free_style();
-
- expire_tiles_stop();
-}
-
-static int pgsql_add_node(osmid_t id, double lat, double lon, struct keyval *tags)
-{
- Options->mid->nodes_set(id, lat, lon, tags);
- pgsql_out_node(id, tags, lat, lon);
-
- return 0;
-}
-
-static int pgsql_add_way(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags)
-{
- int polygon = 0;
- int roads = 0;
-
-
- /* Check whether the way is: (1) Exportable, (2) Maybe a polygon */
- int filter = tagtransform_filter_way_tags(tags, &polygon, &roads);
-
- /* If this isn't a polygon then it can not be part of a multipolygon
- Hence only polygons are "pending" */
- Options->mid->ways_set(id, nds, nd_count, tags, (!filter && polygon) ? 1 : 0);
-
- if( !polygon && !filter )
- {
- /* Get actual node data and generate output */
- struct osmNode *nodes = malloc( sizeof(struct osmNode) * nd_count );
- int count = Options->mid->nodes_get_list( nodes, nds, nd_count );
- pgsql_out_way(id, tags, nodes, count, 0);
- free(nodes);
- }
- return 0;
-}
-
-/* This is the workhorse of pgsql_add_relation, split out because it is used as the callback for iterate relations */
-static int pgsql_process_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags, int exists)
-{
- int i, j, count, count2;
- osmid_t *xid2 = malloc( (member_count+1) * sizeof(osmid_t) );
- osmid_t *xid;
- const char **xrole = malloc( (member_count+1) * sizeof(const char *) );
- int *xcount = malloc( (member_count+1) * sizeof(int) );
- struct keyval *xtags = malloc( (member_count+1) * sizeof(struct keyval) );
- struct osmNode **xnodes = malloc( (member_count+1) * sizeof(struct osmNode*) );
-
- /* If the flag says this object may exist already, delete it first */
- if(exists)
- pgsql_delete_relation_from_output(id);
-
- if (tagtransform_filter_rel_tags(tags)) {
- free(xid2);
- free(xrole);
- free(xcount);
- free(xtags);
- free(xnodes);
- return 1;
- }
-
- count = 0;
- for( i=0; i<member_count; i++ )
- {
-
- /* Need to handle more than just ways... */
- if( members[i].type != OSMTYPE_WAY )
- continue;
- xid2[count] = members[i].id;
- count++;
- }
-
- count2 = Options->mid->ways_get_list(xid2, count, &xid, xtags, xnodes, xcount);
-
- for (i = 0; i < count2; i++) {
- for (j = i; j < member_count; j++) {
- if (members[j].id == xid[i]) break;
- }
- xrole[i] = members[j].role;
- }
- xnodes[count2] = NULL;
- xcount[count2] = 0;
- xid[count2] = 0;
- xrole[count2] = NULL;
-
- /* At some point we might want to consider storing the retrieved data in the members, rather than as separate arrays */
- pgsql_out_relation(id, tags, count2, xnodes, xtags, xcount, xid, xrole);
-
- for( i=0; i<count2; i++ )
- {
- resetList( &(xtags[i]) );
- free( xnodes[i] );
- }
-
- free(xid2);
- free(xid);
- free(xrole);
- free(xcount);
- free(xtags);
- free(xnodes);
- return 0;
-}
-
-static int pgsql_add_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags)
-{
- const char *type = getItem(tags, "type");
-
- /* Must have a type field or we ignore it */
- if (!type)
- return 0;
-
- /* In slim mode we remember these */
- if(Options->mid->relations_set)
- Options->mid->relations_set(id, members, member_count, tags);
-
- /* Only a limited subset of type= is supported, ignore other */
- if ( (strcmp(type, "route") != 0) && (strcmp(type, "multipolygon") != 0) && (strcmp(type, "boundary") != 0))
- return 0;
-
-
- return pgsql_process_relation(id, members, member_count, tags, 0);
-}
-#define UNUSED __attribute__ ((unused))
-
-/* Delete is easy, just remove all traces of this object. We don't need to
- * worry about finding objects that depend on it, since the same diff must
- * contain the change for that also. */
-static int pgsql_delete_node(osmid_t osm_id)
-{
- if( !Options->slim )
- {
- fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
- exit_nicely();
- }
- pgsql_pause_copy(&tables[t_point]);
- if ( expire_tiles_from_db(tables[t_point].sql_conn, osm_id) != 0)
- pgsql_exec(tables[t_point].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_point].name, osm_id );
-
- Options->mid->nodes_delete(osm_id);
- return 0;
-}
-
-/* Seperated out because we use it elsewhere */
-static int pgsql_delete_way_from_output(osmid_t osm_id)
-{
- /* Optimisation: we only need this is slim mode */
- if( !Options->slim )
- return 0;
- /* in droptemp mode we don't have indices and this takes ages. */
- if (Options->droptemp)
- return 0;
- pgsql_pause_copy(&tables[t_roads]);
- pgsql_pause_copy(&tables[t_line]);
- pgsql_pause_copy(&tables[t_poly]);
- pgsql_exec(tables[t_roads].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_roads].name, osm_id );
- if ( expire_tiles_from_db(tables[t_line].sql_conn, osm_id) != 0)
- pgsql_exec(tables[t_line].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_line].name, osm_id );
- if ( expire_tiles_from_db(tables[t_poly].sql_conn, osm_id) != 0)
- pgsql_exec(tables[t_poly].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_poly].name, osm_id );
- return 0;
-}
-
-static int pgsql_delete_way(osmid_t osm_id)
-{
- if( !Options->slim )
- {
- fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
- exit_nicely();
- }
- pgsql_delete_way_from_output(osm_id);
- Options->mid->ways_delete(osm_id);
- return 0;
-}
-
-/* Relations are identified by using negative IDs */
-static int pgsql_delete_relation_from_output(osmid_t osm_id)
-{
- pgsql_pause_copy(&tables[t_roads]);
- pgsql_pause_copy(&tables[t_line]);
- pgsql_pause_copy(&tables[t_poly]);
- pgsql_exec(tables[t_roads].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_roads].name, -osm_id );
- if ( expire_tiles_from_db(tables[t_line].sql_conn, -osm_id) != 0)
- pgsql_exec(tables[t_line].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_line].name, -osm_id );
- if ( expire_tiles_from_db(tables[t_poly].sql_conn, -osm_id) != 0)
- pgsql_exec(tables[t_poly].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_poly].name, -osm_id );
- return 0;
-}
-
-static int pgsql_delete_relation(osmid_t osm_id)
-{
- if( !Options->slim )
- {
- fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
- exit_nicely();
- }
- pgsql_delete_relation_from_output(osm_id);
- Options->mid->relations_delete(osm_id);
- return 0;
-}
-
-/* Modify is slightly trickier. The basic idea is we simply delete the
- * object and create it with the new parameters. Then we need to mark the
- * objects that depend on this one */
-static int pgsql_modify_node(osmid_t osm_id, double lat, double lon, struct keyval *tags)
-{
- if( !Options->slim )
- {
- fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
- exit_nicely();
- }
- pgsql_delete_node(osm_id);
- pgsql_add_node(osm_id, lat, lon, tags);
- Options->mid->node_changed(osm_id);
- return 0;
-}
-
-static int pgsql_modify_way(osmid_t osm_id, osmid_t *nodes, int node_count, struct keyval *tags)
-{
- if( !Options->slim )
- {
- fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
- exit_nicely();
- }
- pgsql_delete_way(osm_id);
- pgsql_add_way(osm_id, nodes, node_count, tags);
- Options->mid->way_changed(osm_id);
- return 0;
-}
-
-static int pgsql_modify_relation(osmid_t osm_id, struct member *members, int member_count, struct keyval *tags)
-{
- if( !Options->slim )
- {
- fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
- exit_nicely();
- }
- pgsql_delete_relation(osm_id);
- pgsql_add_relation(osm_id, members, member_count, tags);
- Options->mid->relation_changed(osm_id);
- return 0;
-}
-
-struct output_t out_pgsql = {
- .start = pgsql_out_start,
- .connect = pgsql_out_connect,
- .stop = pgsql_out_stop,
- .cleanup = pgsql_out_cleanup,
- .close = pgsql_out_close,
- .node_add = pgsql_add_node,
- .way_add = pgsql_add_way,
- .relation_add = pgsql_add_relation,
-
- .node_modify = pgsql_modify_node,
- .way_modify = pgsql_modify_way,
- .relation_modify = pgsql_modify_relation,
-
- .node_delete = pgsql_delete_node,
- .way_delete = pgsql_delete_way,
- .relation_delete = pgsql_delete_relation
-};
diff --git a/output-pgsql.cpp b/output-pgsql.cpp
new file mode 100644
index 0000000..f72d739
--- /dev/null
+++ b/output-pgsql.cpp
@@ -0,0 +1,762 @@
+/* Implements the mid-layer processing for osm2pgsql
+ * using several PostgreSQL tables
+ *
+ * This layer stores data read in from the planet.osm file
+ * and is then read by the backend processing code to
+ * emit the final geometry-enabled output formats
+*/
+
+#include "config.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <time.h>
+#include <stdexcept>
+
+#ifdef HAVE_PTHREAD
+#include <pthread.h>
+#endif
+
+#include "osmtypes.hpp"
+#include "reprojection.hpp"
+#include "output-pgsql.hpp"
+#include "options.hpp"
+#include "middle.hpp"
+#include "pgsql.hpp"
+#include "expire-tiles.hpp"
+#include "wildcmp.hpp"
+#include "node-ram-cache.hpp"
+#include "taginfo_impl.hpp"
+#include "tagtransform.hpp"
+#include "util.hpp"
+
+#include <boost/bind.hpp>
+#include <boost/format.hpp>
+#include <boost/foreach.hpp>
+#include <boost/make_shared.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <iostream>
+#include <limits>
+#include <stdexcept>
+
+#define SRID (reproj->project_getprojinfo()->srs)
+
+/* FIXME: Shouldn't malloc this all to begin with but call realloc()
+ as required. The program will most likely segfault if it reads a
+ style file with more styles than this */
+#define MAX_STYLES 1000
+
+#define NUM_TABLES (output_pgsql_t::t_MAX)
+
+/* example from: pg_dump -F p -t planet_osm gis
+COPY planet_osm (osm_id, name, place, landuse, leisure, "natural", man_made, waterway, highway, railway, amenity, tourism, learning, building, bridge, layer, way) FROM stdin;
+17959841 \N \N \N \N \N \N \N bus_stop \N \N \N \N \N \N -\N 0101000020E610000030CCA462B6C3D4BF92998C9B38E04940
+17401934 The Horn \N \N \N \N \N \N \N \N pub \N \N \N \N -\N 0101000020E6100000C12FC937140FD5BFB4D2F4FB0CE04940
+...
+
+mine - 01 01000000 48424298424242424242424256427364
+psql - 01 01000020 E6100000 30CCA462B6C3D4BF92998C9B38E04940
+ 01 01000020 E6100000 48424298424242424242424256427364
+0x2000_0000 = hasSRID, following 4 bytes = srid, not supported by geos WKBWriter
+Workaround - output SRID=4326;<WKB>
+*/
+
+int output_pgsql_t::pgsql_out_node(osmid_t id, struct keyval *tags, double node_lat, double node_lon)
+{
+
+ int filter = m_tagtransform->filter_node_tags(tags, m_export_list.get());
+
+ if (filter) return 1;
+
+ expire->from_bbox(node_lon, node_lat, node_lon, node_lat);
+ m_tables[t_point]->write_node(id, tags, node_lat, node_lon);
+
+ return 0;
+}
+
+/*static int tag_indicates_polygon(enum OsmType type, const char *key)
+{
+ int i;
+
+ if (!strcmp(key, "area"))
+ return 1;
+
+ for (i=0; i < exportListCount[type]; i++) {
+ if( strcmp( exportList[type][i].name, key ) == 0 )
+ return exportList[type][i].flags & FLAG_POLYGON;
+ }
+
+ return 0;
+}*/
+
+/*
+COPY planet_osm (osm_id, name, place, landuse, leisure, "natural", man_made, waterway, highway, railway, amenity, tourism, learning, bu
+ilding, bridge, layer, way) FROM stdin;
+198497 Bedford Road \N \N \N \N \N \N residential \N \N \N \N \N \N \N 0102000020E610000004000000452BF702B342D5BF1C60E63BF8DF49406B9C4D470037D5BF5471E316F3DF4940DFA815A6EF35D5BF9AE95E27F5DF4940B41EB
+E4C1421D5BF24D06053E7DF4940
+212696 Oswald Road \N \N \N \N \N \N minor \N \N \N \N \N \N \N 0102000020E610000004000000467D923B6C22D5BFA359D93EE4DF4940B3976DA7AD11D5BF84BBB376DBDF4940997FF44D9A06D5BF4223D8B8FEDF49404D158C4AEA04D
+5BF5BB39597FCDF4940
+*/
+int output_pgsql_t::pgsql_out_way(osmid_t id, struct keyval *tags, const struct osmNode *nodes, int count, int exists)
+{
+ int polygon = 0, roads = 0;
+ double split_at;
+
+ /* If the flag says this object may exist already, delete it first */
+ if(exists) {
+ pgsql_delete_way_from_output(id);
+ // TODO: this now only has an effect when called from the iterate_ways
+ // call-back, so we need some alternative way to trigger this within
+ // osmdata_t.
+ const std::vector<osmid_t> rel_ids = m_mid->relations_using_way(id);
+ for (std::vector<osmid_t>::const_iterator itr = rel_ids.begin();
+ itr != rel_ids.end(); ++itr) {
+ rels_pending_tracker->mark(*itr);
+ }
+ }
+
+ if (m_tagtransform->filter_way_tags(tags, &polygon, &roads, m_export_list.get()))
+ return 0;
+ /* Split long ways after around 1 degree or 100km */
+ if (m_options.projection->get_proj_id() == PROJ_LATLONG)
+ split_at = 1;
+ else
+ split_at = 100 * 1000;
+
+ geometry_builder::maybe_wkts_t wkts = builder.get_wkt_split(nodes, count, polygon, split_at);
+ for(geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt)
+ {
+ /* FIXME: there should be a better way to detect polygons */
+ if (boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON")) {
+ expire->from_nodes_poly(nodes, count, id);
+ if ((wkt->area > 0.0) && m_enable_way_area) {
+ char tmp[32];
+ snprintf(tmp, sizeof(tmp), "%g", wkt->area);
+ keyval::addItem(tags, "way_area", tmp, 0);
+ }
+ m_tables[t_poly]->write_wkt(id, tags, wkt->geom.c_str());
+ } else {
+ expire->from_nodes_line(nodes, count);
+ m_tables[t_line]->write_wkt(id, tags, wkt->geom.c_str());
+ if (roads)
+ m_tables[t_roads]->write_wkt(id, tags, wkt->geom.c_str());
+ }
+ }
+
+ return 0;
+}
+
+int output_pgsql_t::pgsql_out_relation(osmid_t id, struct keyval *rel_tags, int member_count, const struct osmNode * const *xnodes, struct keyval *xtags, const int *xcount, const osmid_t *xid, const char * const *xrole, bool pending)
+{
+ if (member_count == 0)
+ return 0;
+
+ int roads = 0;
+ int make_polygon = 0;
+ int make_boundary = 0;
+ int * members_superseeded;
+ double split_at;
+
+ members_superseeded = (int *)calloc(sizeof(int), member_count);
+
+ //if its a route relation make_boundary and make_polygon will be false otherwise one or the other will be true
+ if (m_tagtransform->filter_rel_member_tags(rel_tags, member_count, xtags, xrole, members_superseeded, &make_boundary, &make_polygon, &roads, m_export_list.get())) {
+ free(members_superseeded);
+ return 0;
+ }
+
+ /* Split long linear ways after around 1 degree or 100km (polygons not effected) */
+ if (m_options.projection->get_proj_id() == PROJ_LATLONG)
+ split_at = 1;
+ else
+ split_at = 100 * 1000;
+
+ //this will either make lines or polygons (unless the lines arent a ring or are less than 3 pts) depending on the tag transform above
+ //TODO: pick one or the other based on which we expect to care about
+ geometry_builder::maybe_wkts_t wkts = builder.build_both(xnodes, xcount, make_polygon, m_options.enable_multi, split_at, id);
+
+ if (!wkts->size()) {
+ free(members_superseeded);
+ return 0;
+ }
+
+ for(geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt)
+ {
+ expire->from_wkt(wkt->geom.c_str(), -id);
+ /* FIXME: there should be a better way to detect polygons */
+ if (boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON")) {
+ if ((wkt->area > 0.0) && m_enable_way_area) {
+ char tmp[32];
+ snprintf(tmp, sizeof(tmp), "%g", wkt->area);
+ keyval::addItem(rel_tags, "way_area", tmp, 0);
+ }
+ m_tables[t_poly]->write_wkt(-id, rel_tags, wkt->geom.c_str());
+ } else {
+ m_tables[t_line]->write_wkt(-id, rel_tags, wkt->geom.c_str());
+ if (roads)
+ m_tables[t_roads]->write_wkt(-id, rel_tags, wkt->geom.c_str());
+ }
+ }
+
+ /* Tagtransform will have marked those member ways of the relation that
+ * have fully been dealt with as part of the multi-polygon entry.
+ * Set them in the database as done and delete their entry to not
+ * have duplicates */
+ //dont do this when working with pending relations as its not needed
+ if (make_polygon) {
+ for (int i=0; xcount[i]; i++) {
+ if (members_superseeded[i]) {
+ pgsql_delete_way_from_output(xid[i]);
+ if(!pending)
+ ways_done_tracker->mark(xid[i]);
+ }
+ }
+ }
+
+ free(members_superseeded);
+
+ // If the tag transform said the polygon looked like a boundary we want to make that as well
+ // If we are making a boundary then also try adding any relations which form complete rings
+ // The linear variants will have already been processed above
+ if (make_boundary) {
+ wkts = builder.build_polygons(xnodes, xcount, m_options.enable_multi, id);
+ for(geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt)
+ {
+ expire->from_wkt(wkt->geom.c_str(), -id);
+ if ((wkt->area > 0.0) && m_enable_way_area) {
+ char tmp[32];
+ snprintf(tmp, sizeof(tmp), "%g", wkt->area);
+ keyval::addItem(rel_tags, "way_area", tmp, 0);
+ }
+ m_tables[t_poly]->write_wkt(-id, rel_tags, wkt->geom.c_str());
+ }
+ }
+
+ return 0;
+}
+
+
+
+namespace {
+/* Using pthreads requires us to shoe-horn everything into various void*
+ * pointers. Improvement for the future: just use boost::thread. */
+struct pthread_thunk {
+ table_t *ptr;
+};
+
+extern "C" void *pthread_output_pgsql_stop_one(void *arg) {
+ pthread_thunk *thunk = static_cast<pthread_thunk *>(arg);
+ thunk->ptr->stop();
+ return NULL;
+};
+} // anonymous namespace
+
+void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+ //make sure we get the one passed in
+ if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) {
+ job_queue.push(pending_job_t(id, output_id));
+ added++;
+ }
+
+ //grab the first one or bail if its not valid
+ osmid_t popped = ways_pending_tracker->pop_mark();
+ if(!id_tracker::is_valid(popped))
+ return;
+
+ //get all the ones up to the id that was passed in
+ while (popped < id) {
+ if (!ways_done_tracker->is_marked(popped)) {
+ job_queue.push(pending_job_t(popped, output_id));
+ added++;
+ }
+ popped = ways_pending_tracker->pop_mark();
+ }
+
+ //make sure to get this one as well and move to the next
+ if(popped == id) {
+ popped = ways_pending_tracker->pop_mark();
+ }
+ if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) {
+ job_queue.push(pending_job_t(popped, output_id));
+ added++;
+ }
+}
+
+int output_pgsql_t::pending_way(osmid_t id, int exists) {
+ keyval tags_int;
+ osmNode *nodes_int;
+ int count_int;
+ int ret = 0;
+
+ keyval::initList(&tags_int);
+ // Try to fetch the way from the DB
+ if (!m_mid->ways_get(id, &tags_int, &nodes_int, &count_int)) {
+ // Output the way
+ //ret = reprocess_way(id, nodes_int, count_int, &tags_int, exists);
+ ret = pgsql_out_way(id, &tags_int, nodes_int, count_int, exists);
+ free(nodes_int);
+ }
+ keyval::resetList(&tags_int);
+
+ return ret;
+}
+
+void output_pgsql_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
+ //make sure we get the one passed in
+ if(id_tracker::is_valid(id)) {
+ job_queue.push(pending_job_t(id, output_id));
+ added++;
+ }
+
+ //grab the first one or bail if its not valid
+ osmid_t popped = rels_pending_tracker->pop_mark();
+ if(!id_tracker::is_valid(popped))
+ return;
+
+ //get all the ones up to the id that was passed in
+ while (popped < id) {
+ job_queue.push(pending_job_t(popped, output_id));
+ added++;
+ popped = rels_pending_tracker->pop_mark();
+ }
+
+ //make sure to get this one as well and move to the next
+ if(popped == id) {
+ popped = rels_pending_tracker->pop_mark();
+ }
+ if(id_tracker::is_valid(popped)) {
+ job_queue.push(pending_job_t(popped, output_id));
+ added++;
+ }
+}
+
+int output_pgsql_t::pending_relation(osmid_t id, int exists) {
+ keyval tags_int;
+ member *members_int;
+ int count_int;
+ int ret = 0;
+
+ keyval::initList(&tags_int);
+ // Try to fetch the relation from the DB
+ if (!m_mid->relations_get(id, &members_int, &count_int, &tags_int)) {
+ ret = pgsql_process_relation(id, members_int, count_int, &tags_int, exists, true);
+ free(members_int);
+ }
+ keyval::resetList(&tags_int);
+
+ return ret;
+}
+
+void output_pgsql_t::commit()
+{
+ for (int i=0; i<NUM_TABLES; i++) {
+ m_tables[i]->commit();
+ }
+}
+
+void output_pgsql_t::stop()
+{
+ int i;
+#ifdef HAVE_PTHREAD
+ pthread_t threads[NUM_TABLES];
+#endif
+
+#ifdef HAVE_PTHREAD
+ if (m_options.parallel_indexing) {
+ pthread_thunk thunks[NUM_TABLES];
+ for (i=0; i<NUM_TABLES; i++) {
+ thunks[i].ptr = m_tables[i].get();
+ }
+
+ for (i=0; i<NUM_TABLES; i++) {
+ int ret = pthread_create(&threads[i], NULL, pthread_output_pgsql_stop_one, &thunks[i]);
+ if (ret) {
+ fprintf(stderr, "pthread_create() returned an error (%d)", ret);
+ util::exit_nicely();
+ }
+ }
+
+ for (i=0; i<NUM_TABLES; i++) {
+ int ret = pthread_join(threads[i], NULL);
+ if (ret) {
+ fprintf(stderr, "pthread_join() returned an error (%d)", ret);
+ util::exit_nicely();
+ }
+ }
+ } else {
+#endif
+
+ /* No longer need to access middle layer -- release memory */
+ //TODO: just let the destructor do this
+ for (i=0; i<NUM_TABLES; i++)
+ m_tables[i]->stop();
+
+#ifdef HAVE_PTHREAD
+ }
+#endif
+
+ expire->output_and_destroy();
+ expire.reset();
+}
+
+int output_pgsql_t::node_add(osmid_t id, double lat, double lon, struct keyval *tags)
+{
+ pgsql_out_node(id, tags, lat, lon);
+
+ return 0;
+}
+
+int output_pgsql_t::way_add(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags)
+{
+ int polygon = 0;
+ int roads = 0;
+
+
+ /* Check whether the way is: (1) Exportable, (2) Maybe a polygon */
+ int filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads, m_export_list.get());
+
+ /* If this isn't a polygon then it can not be part of a multipolygon
+ Hence only polygons are "pending" */
+ if (!filter && polygon) { ways_pending_tracker->mark(id); }
+
+ if( !polygon && !filter )
+ {
+ /* Get actual node data and generate output */
+ struct osmNode *nodes = (struct osmNode *)malloc( sizeof(struct osmNode) * nd_count );
+ int count = m_mid->nodes_get_list( nodes, nds, nd_count );
+ pgsql_out_way(id, tags, nodes, count, 0);
+ free(nodes);
+ }
+ return 0;
+}
+
+
+/* This is the workhorse of pgsql_add_relation, split out because it is used as the callback for iterate relations */
+int output_pgsql_t::pgsql_process_relation(osmid_t id, const struct member *members, int member_count, struct keyval *tags, int exists, bool pending)
+{
+ int i, j, count, count2;
+
+ /* If the flag says this object may exist already, delete it first */
+ if(exists)
+ pgsql_delete_relation_from_output(id);
+
+ if (m_tagtransform->filter_rel_tags(tags, m_export_list.get())) {
+ return 1;
+ }
+
+ osmid_t *xid2 = (osmid_t *)malloc( (member_count+1) * sizeof(osmid_t) );
+ const char **xrole = (const char **)malloc( (member_count+1) * sizeof(const char *) );
+ int *xcount = (int *)malloc( (member_count+1) * sizeof(int) );
+ keyval *xtags = new keyval[member_count+1];
+ struct osmNode **xnodes = (struct osmNode **)malloc( (member_count+1) * sizeof(struct osmNode*) );
+
+ count = 0;
+ for( i=0; i<member_count; i++ )
+ {
+ /* Need to handle more than just ways... */
+ if( members[i].type != OSMTYPE_WAY )
+ continue;
+ xid2[count] = members[i].id;
+ count++;
+ }
+
+ osmid_t *xid = (osmid_t *)malloc( sizeof(osmid_t) * (count + 1));
+ count2 = m_mid->ways_get_list(xid2, count, xid, xtags, xnodes, xcount);
+ int polygon = 0, roads = 0;;
+
+ for (i = 0; i < count2; i++) {
+ for (j = i; j < member_count; j++) {
+ if (members[j].id == xid[i]) {
+ //filter the tags on this member because we got it from the middle
+ //and since the middle is no longer tied to the output it no longer
+ //shares any kind of tag transform and therefore all original tags
+ //will come back and need to be filtered by individual outputs before
+ //using these ways
+ m_tagtransform->filter_way_tags(&xtags[i], &polygon, &roads, m_export_list.get());
+ //TODO: if the filter says that this member is now not interesting we
+ //should decrement the count and remove his nodes and tags etc. for
+ //now we'll just keep him with no tags so he will get filtered later
+ break;
+ }
+ }
+ xrole[i] = members[j].role;
+ }
+ xnodes[count2] = NULL;
+ xcount[count2] = 0;
+ xid[count2] = 0;
+ xrole[count2] = NULL;
+
+ /* At some point we might want to consider storing the retrieved data in the members, rather than as separate arrays */
+ pgsql_out_relation(id, tags, count2, xnodes, xtags, xcount, xid, xrole, pending);
+
+ for( i=0; i<count2; i++ )
+ {
+ keyval::resetList( &(xtags[i]) );
+ free( xnodes[i] );
+ }
+
+ free(xid2);
+ free(xid);
+ free(xrole);
+ free(xcount);
+ delete [] xtags;
+ free(xnodes);
+ return 0;
+}
+
+int output_pgsql_t::relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags)
+{
+ const char *type = keyval::getItem(tags, "type");
+
+ /* Must have a type field or we ignore it */
+ if (!type)
+ return 0;
+
+ /* Only a limited subset of type= is supported, ignore other */
+ if ( (strcmp(type, "route") != 0) && (strcmp(type, "multipolygon") != 0) && (strcmp(type, "boundary") != 0))
+ return 0;
+
+
+ return pgsql_process_relation(id, members, member_count, tags, 0);
+}
+
+/* Delete is easy, just remove all traces of this object. We don't need to
+ * worry about finding objects that depend on it, since the same diff must
+ * contain the change for that also. */
+int output_pgsql_t::node_delete(osmid_t osm_id)
+{
+ if( !m_options.slim )
+ {
+ fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
+ util::exit_nicely();
+ }
+
+ if ( expire->from_db(m_tables[t_point].get(), osm_id) != 0)
+ m_tables[t_point]->delete_row(osm_id);
+
+ return 0;
+}
+
+/* Seperated out because we use it elsewhere */
+int output_pgsql_t::pgsql_delete_way_from_output(osmid_t osm_id)
+{
+ /* Optimisation: we only need this is slim mode */
+ if( !m_options.slim )
+ return 0;
+ /* in droptemp mode we don't have indices and this takes ages. */
+ if (m_options.droptemp)
+ return 0;
+
+ m_tables[t_roads]->delete_row(osm_id);
+ if ( expire->from_db(m_tables[t_line].get(), osm_id) != 0)
+ m_tables[t_line]->delete_row(osm_id);
+ if ( expire->from_db(m_tables[t_poly].get(), osm_id) != 0)
+ m_tables[t_poly]->delete_row(osm_id);
+ return 0;
+}
+
+int output_pgsql_t::way_delete(osmid_t osm_id)
+{
+ if( !m_options.slim )
+ {
+ fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
+ util::exit_nicely();
+ }
+ pgsql_delete_way_from_output(osm_id);
+ return 0;
+}
+
+/* Relations are identified by using negative IDs */
+int output_pgsql_t::pgsql_delete_relation_from_output(osmid_t osm_id)
+{
+ m_tables[t_roads]->delete_row(-osm_id);
+ if ( expire->from_db(m_tables[t_line].get(), -osm_id) != 0)
+ m_tables[t_line]->delete_row(-osm_id);
+ if ( expire->from_db(m_tables[t_poly].get(), -osm_id) != 0)
+ m_tables[t_poly]->delete_row(-osm_id);
+ return 0;
+}
+
+int output_pgsql_t::relation_delete(osmid_t osm_id)
+{
+ if( !m_options.slim )
+ {
+ fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
+ util::exit_nicely();
+ }
+ pgsql_delete_relation_from_output(osm_id);
+ return 0;
+}
+
+/* Modify is slightly trickier. The basic idea is we simply delete the
+ * object and create it with the new parameters. Then we need to mark the
+ * objects that depend on this one */
+int output_pgsql_t::node_modify(osmid_t osm_id, double lat, double lon, struct keyval *tags)
+{
+ if( !m_options.slim )
+ {
+ fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
+ util::exit_nicely();
+ }
+ node_delete(osm_id);
+ node_add(osm_id, lat, lon, tags);
+ return 0;
+}
+
+int output_pgsql_t::way_modify(osmid_t osm_id, osmid_t *nodes, int node_count, struct keyval *tags)
+{
+ if( !m_options.slim )
+ {
+ fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
+ util::exit_nicely();
+ }
+ way_delete(osm_id);
+ way_add(osm_id, nodes, node_count, tags);
+
+ return 0;
+}
+
+int output_pgsql_t::relation_modify(osmid_t osm_id, struct member *members, int member_count, struct keyval *tags)
+{
+ if( !m_options.slim )
+ {
+ fprintf( stderr, "Cannot apply diffs unless in slim mode\n" );
+ util::exit_nicely();
+ }
+ relation_delete(osm_id);
+ relation_add(osm_id, members, member_count, tags);
+ return 0;
+}
+
+int output_pgsql_t::start()
+{
+ for(std::vector<boost::shared_ptr<table_t> >::iterator table = m_tables.begin(); table != m_tables.end(); ++table)
+ {
+ //setup the table in postgres
+ table->get()->start();
+ }
+
+ return 0;
+}
+
+boost::shared_ptr<output_t> output_pgsql_t::clone(const middle_query_t* cloned_middle) const {
+ output_pgsql_t *clone = new output_pgsql_t(*this);
+ clone->m_mid = cloned_middle;
+ //NOTE: we need to know which ways were used by relations so each thread
+ //must have a copy of the original marked done ways, its read only so its ok
+ clone->ways_done_tracker = ways_done_tracker;
+ return boost::shared_ptr<output_t>(clone);
+}
+
+output_pgsql_t::output_pgsql_t(const middle_query_t* mid_, const options_t &options_)
+ : output_t(mid_, options_),
+ ways_pending_tracker(new id_tracker()),
+ ways_done_tracker(new id_tracker()),
+ rels_pending_tracker(new id_tracker()) {
+
+ reproj = m_options.projection;
+ builder.set_exclude_broken_polygon(m_options.excludepoly);
+
+ m_export_list.reset(new export_list());
+
+ m_enable_way_area = read_style_file( m_options.style, m_export_list.get() );
+
+ try {
+ m_tagtransform.reset(new tagtransform(&m_options));
+ }
+ catch(std::runtime_error& e) {
+ fprintf(stderr, "%s\n", e.what());
+ fprintf(stderr, "Error: Failed to initialise tag processing.\n");
+ util::exit_nicely();
+ }
+
+ expire.reset(new expire_tiles(&m_options));
+
+ //for each table
+ m_tables.reserve(NUM_TABLES);
+ for (int i=0; i<NUM_TABLES; i++) {
+
+ //figure out the columns this table needs
+ columns_t columns = m_export_list->normal_columns((i == t_point)?OSMTYPE_NODE:OSMTYPE_WAY);
+
+ //figure out what name we are using for this and what type
+ std::string name = m_options.prefix;
+ std::string type;
+ switch(i)
+ {
+ case t_point:
+ name += "_point";
+ type = "POINT";
+ break;
+ case t_line:
+ name += "_line";
+ type = "LINESTRING";
+ break;
+ case t_poly:
+ name += "_polygon";
+ type = "GEOMETRY"; // Actually POLGYON & MULTIPOLYGON but no way to limit to just these two
+ break;
+ case t_roads:
+ name += "_roads";
+ type = "LINESTRING";
+ break;
+ default:
+ //TODO: error message about coding error
+ util::exit_nicely();
+ }
+
+ //tremble in awe of this massive constructor! seriously we are trying to avoid passing an
+ //options object because we want to make use of the table_t in output_mutli_t which could
+ //have a different tablespace/hstores/etc per table
+ m_tables.push_back(boost::shared_ptr<table_t>(
+ new table_t(
+ m_options.conninfo, name, type, columns, m_options.hstore_columns, SRID, m_options.scale,
+ m_options.append, m_options.slim, m_options.droptemp, m_options.hstore_mode,
+ m_options.enable_hstore_index, m_options.tblsmain_data, m_options.tblsmain_index
+ )
+ ));
+ }
+}
+
+output_pgsql_t::output_pgsql_t(const output_pgsql_t& other):
+ output_t(other.m_mid, other.m_options), m_tagtransform(new tagtransform(&m_options)), m_enable_way_area(other.m_enable_way_area),
+ m_export_list(new export_list(*other.m_export_list)), reproj(other.reproj),
+ ways_pending_tracker(new id_tracker()), ways_done_tracker(new id_tracker()), rels_pending_tracker(new id_tracker()),
+ expire(new expire_tiles(&m_options))
+{
+ builder.set_exclude_broken_polygon(m_options.excludepoly);
+ for(std::vector<boost::shared_ptr<table_t> >::const_iterator t = other.m_tables.begin(); t != other.m_tables.end(); ++t) {
+ //copy constructor will just connect to the already there table
+ m_tables.push_back(boost::shared_ptr<table_t>(new table_t(**t)));
+ }
+}
+
+output_pgsql_t::~output_pgsql_t() {
+}
+
+size_t output_pgsql_t::pending_count() const {
+ return ways_pending_tracker->size() + rels_pending_tracker->size();
+}
+
+void output_pgsql_t::merge_pending_relations(boost::shared_ptr<output_t> other) {
+ boost::shared_ptr<id_tracker> tracker = other->get_pending_relations();
+ osmid_t id;
+ while(tracker.get() && id_tracker::is_valid((id = tracker->pop_mark()))){
+ rels_pending_tracker->mark(id);
+ }
+}
+void output_pgsql_t::merge_expire_trees(boost::shared_ptr<output_t> other) {
+ if(other->get_expire_tree().get())
+ expire->merge_and_destroy(*other->get_expire_tree());
+}
+
+boost::shared_ptr<id_tracker> output_pgsql_t::get_pending_relations() {
+ return rels_pending_tracker;
+}
+boost::shared_ptr<expire_tiles> output_pgsql_t::get_expire_tree() {
+ return expire;
+}
diff --git a/output-pgsql.h b/output-pgsql.h
deleted file mode 100644
index 32af34e..0000000
--- a/output-pgsql.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/* Implements the output-layer processing for osm2pgsql
- * storing the data in several PostgreSQL tables
- * with the final PostGIS geometries for each entity
-*/
-
-#ifndef OUTPUT_PGSQL_H
-#define OUTPUT_PGSQL_H
-
-#include "output.h"
-
-#define FLAG_POLYGON 1 /* For polygon table */
-#define FLAG_LINEAR 2 /* For lines table */
-#define FLAG_NOCACHE 4 /* Optimisation: don't bother remembering this one */
-#define FLAG_DELETE 8 /* These tags should be simply deleted on sight */
-#define FLAG_PHSTORE 17 /* polygons without own column but listed in hstore this implies FLAG_POLYGON */
-
-/* Table columns, representing key= tags */
-struct taginfo {
- char *name;
- char *type;
- int flags;
- int count;
-};
-
-extern struct output_t out_pgsql;
-
-#endif
diff --git a/output-pgsql.hpp b/output-pgsql.hpp
new file mode 100644
index 0000000..fb438f8
--- /dev/null
+++ b/output-pgsql.hpp
@@ -0,0 +1,89 @@
+/* Implements the output-layer processing for osm2pgsql
+ * storing the data in several PostgreSQL tables
+ * with the final PostGIS geometries for each entity
+*/
+
+#ifndef OUTPUT_PGSQL_H
+#define OUTPUT_PGSQL_H
+
+#include "output.hpp"
+#include "tagtransform.hpp"
+#include "geometry-builder.hpp"
+#include "reprojection.hpp"
+#include "expire-tiles.hpp"
+#include "id-tracker.hpp"
+#include "table.hpp"
+
+#include <vector>
+#include <boost/shared_ptr.hpp>
+
+class output_pgsql_t : public output_t {
+public:
+ enum table_id {
+ t_point = 0, t_line, t_poly, t_roads, t_MAX
+ };
+
+ output_pgsql_t(const middle_query_t* mid_, const options_t &options_);
+ virtual ~output_pgsql_t();
+ output_pgsql_t(const output_pgsql_t& other);
+
+ virtual boost::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const;
+
+ int start();
+ void stop();
+ void commit();
+
+ void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
+ int pending_way(osmid_t id, int exists);
+
+ void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
+ int pending_relation(osmid_t id, int exists);
+
+ int node_add(osmid_t id, double lat, double lon, struct keyval *tags);
+ int way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
+ int relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+
+ int node_modify(osmid_t id, double lat, double lon, struct keyval *tags);
+ int way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
+ int relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+
+ int node_delete(osmid_t id);
+ int way_delete(osmid_t id);
+ int relation_delete(osmid_t id);
+
+ size_t pending_count() const;
+
+ void merge_pending_relations(boost::shared_ptr<output_t> other);
+ void merge_expire_trees(boost::shared_ptr<output_t> other);
+ virtual boost::shared_ptr<id_tracker> get_pending_relations();
+ virtual boost::shared_ptr<expire_tiles> get_expire_tree();
+
+protected:
+
+ int pgsql_out_node(osmid_t id, struct keyval *tags, double node_lat, double node_lon);
+ int pgsql_out_way(osmid_t id, struct keyval *tags, const struct osmNode *nodes, int count, int exists);
+ int pgsql_out_relation(osmid_t id, struct keyval *rel_tags, int member_count, const struct osmNode * const * xnodes, struct keyval *xtags, const int *xcount, const osmid_t *xid, const char * const *xrole, bool pending);
+ int pgsql_process_relation(osmid_t id, const struct member *members, int member_count, struct keyval *tags, int exists, bool pending=false);
+ int pgsql_delete_way_from_output(osmid_t osm_id);
+ int pgsql_delete_relation_from_output(osmid_t osm_id);
+
+ boost::scoped_ptr<tagtransform> m_tagtransform;
+
+ //enable output of a generated way_area tag to either hstore or its own column
+ int m_enable_way_area;
+
+ std::vector<boost::shared_ptr<table_t> > m_tables;
+
+ boost::scoped_ptr<export_list> m_export_list;
+
+ geometry_builder builder;
+
+ boost::shared_ptr<reprojection> reproj;
+ boost::shared_ptr<expire_tiles> expire;
+
+ boost::shared_ptr<id_tracker> ways_pending_tracker, ways_done_tracker, rels_pending_tracker;
+
+ const static std::string NAME;
+};
+
+#endif
diff --git a/output.cpp b/output.cpp
new file mode 100644
index 0000000..3a0e335
--- /dev/null
+++ b/output.cpp
@@ -0,0 +1,161 @@
+#include "output.hpp"
+#include "output-pgsql.hpp"
+#include "output-gazetteer.hpp"
+#include "output-null.hpp"
+#include "output-multi.hpp"
+#include "taginfo_impl.hpp"
+
+#include <string.h>
+#include <stdexcept>
+
+#include <boost/make_shared.hpp>
+#include <boost/format.hpp>
+#include <boost/foreach.hpp>
+#include <boost/functional/hash.hpp>
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/json_parser.hpp>
+
+namespace pt = boost::property_tree;
+
+namespace {
+
+template <typename T>
+void override_if(T &t, const std::string &key, const pt::ptree &conf) {
+ boost::optional<T> opt = conf.get_optional<T>(key);
+ if (opt) {
+ t = *opt;
+ }
+}
+
+boost::shared_ptr<output_t> parse_multi_single(const pt::ptree &conf,
+ const middle_query_t *mid,
+ const options_t &options) {
+ options_t new_opts = options;
+
+ std::string name = conf.get<std::string>("name");
+ std::string proc_type = conf.get<std::string>("type");
+
+ new_opts.tag_transform_script = conf.get_optional<std::string>("tagtransform");
+
+ new_opts.tag_transform_node_func = conf.get_optional<std::string>("tagtransform-node-function");
+ new_opts.tag_transform_way_func = conf.get_optional<std::string>("tagtransform-way-function");
+ new_opts.tag_transform_rel_func = conf.get_optional<std::string>("tagtransform-relation-function");
+ new_opts.tag_transform_rel_mem_func = conf.get_optional<std::string>("tagtransform-relation-member-function");
+
+ new_opts.tblsmain_index = conf.get_optional<std::string>("tablespace-index");
+ new_opts.tblsmain_data = conf.get_optional<std::string>("tablespace-data");
+ override_if<int>(new_opts.hstore_mode, "enable-hstore", conf);
+ override_if<int>(new_opts.enable_hstore_index, "enable-hstore-index", conf);
+ override_if<bool>(new_opts.enable_multi, "enable-multi", conf);
+ override_if<int>(new_opts.hstore_match_only, "hstore-match-only", conf);
+
+ hstores_t hstore_columns;
+ boost::optional<const pt::ptree &> hstores = conf.get_child_optional("hstores");
+ if (hstores) {
+ BOOST_FOREACH(const pt::ptree::value_type &val, *hstores) {
+ hstore_columns.push_back(val.second.get_value<std::string>());
+ }
+ }
+ new_opts.hstore_columns = hstore_columns;
+
+ boost::shared_ptr<geometry_processor> processor =
+ geometry_processor::create(proc_type, &new_opts);
+
+ // TODO: we're faking this up, but there has to be a better way?
+ OsmType osm_type = ((processor->interests() & geometry_processor::interest_node) > 0)
+ ? OSMTYPE_NODE : OSMTYPE_WAY;
+
+ export_list columns;
+ const pt::ptree &tags = conf.get_child("tags");
+ BOOST_FOREACH(const pt::ptree::value_type &val, tags) {
+ const pt::ptree &tag = val.second;
+ taginfo info;
+ info.name = tag.get<std::string>("name");
+ info.type = tag.get<std::string>("type");
+ std::string flags = tag.get_optional<std::string>("flags").get_value_or(std::string());
+ // TODO: we fake the line number here - any way to get the right one
+ // from the JSON parser?
+ info.flags = parse_tag_flags(flags.c_str(), -1);
+ // TODO: shouldn't need to specify a type here?
+ columns.add(osm_type, info);
+ }
+
+ return boost::make_shared<output_multi_t>(name, processor, columns, mid, new_opts);
+}
+
+std::vector<boost::shared_ptr<output_t> > parse_multi_config(const middle_query_t *mid, const options_t &options) {
+ std::vector<boost::shared_ptr<output_t> > outputs;
+
+ if (!options.style.empty()) {
+ const std::string file_name(options.style);
+
+ try {
+ pt::ptree conf;
+ pt::read_json(file_name, conf);
+
+ BOOST_FOREACH(const pt::ptree::value_type &val, conf) {
+ outputs.push_back(parse_multi_single(val.second, mid, options));
+ }
+
+ } catch (const std::exception &e) {
+ throw std::runtime_error((boost::format("Unable to parse multi config file `%1%': %2%")
+ % file_name % e.what()).str());
+ }
+ } else {
+ throw std::runtime_error("Style file is required for `multi' backend, but was not specified.");
+ }
+
+ return outputs;
+}
+
+} // anonymous namespace
+
+std::vector<boost::shared_ptr<output_t> > output_t::create_outputs(const middle_query_t *mid, const options_t &options) {
+ std::vector<boost::shared_ptr<output_t> > outputs;
+
+ if (options.output_backend == "pgsql") {
+ outputs.push_back(boost::make_shared<output_pgsql_t>(mid, options));
+
+ } else if (options.output_backend == "gazetteer") {
+ outputs.push_back(boost::make_shared<output_gazetteer_t>(mid, options));
+
+ } else if (options.output_backend == "null") {
+ outputs.push_back(boost::make_shared<output_null_t>(mid, options));
+
+ } else if (options.output_backend == "multi") {
+ outputs = parse_multi_config(mid, options);
+
+ } else {
+ throw std::runtime_error((boost::format("Output backend `%1%' not recognised. Should be one of [pgsql, gazetteer, null, multi].\n") % options.output_backend).str());
+ }
+
+ return outputs;
+}
+
+output_t::output_t(const middle_query_t *mid_, const options_t &options_): m_mid(mid_), m_options(options_) {
+
+}
+
+output_t::~output_t() {
+
+}
+
+size_t output_t::pending_count() const{
+ return 0;
+}
+
+const options_t *output_t::get_options()const {
+ return &m_options;
+}
+
+void output_t::merge_pending_relations(boost::shared_ptr<output_t> other) {
+}
+void output_t::merge_expire_trees(boost::shared_ptr<output_t> other) {
+}
+
+boost::shared_ptr<id_tracker> output_t::get_pending_relations() {
+ return boost::shared_ptr<id_tracker>();
+}
+boost::shared_ptr<expire_tiles> output_t::get_expire_tree() {
+ return boost::shared_ptr<expire_tiles>();
+}
diff --git a/output.h b/output.h
deleted file mode 100644
index 8bfa7d1..0000000
--- a/output.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/* Common output layer interface */
-
-/* Each output layer must provide methods for
- * storing:
- * - Nodes (Points of interest etc)
- * - Way geometries
- * Associated tags: name, type etc.
-*/
-
-#ifndef OUTPUT_H
-#define OUTPUT_H
-
-#include "middle.h"
-#include "keyvals.h"
-
-/* Variants for generation of hstore column */
-/* No hstore column */
-#define HSTORE_NONE 0
-/* create a hstore column for all tags which do not have an exclusive column */
-#define HSTORE_NORM 1
-/* create a hstore column for all tags */
-#define HSTORE_ALL 2
-
-struct output_options {
- const char *conninfo; /* Connection info string */
- const char *prefix; /* prefix for table names */
- int scale; /* scale for converting coordinates to fixed point */
- int projection; /* SRS of projection */
- int append; /* Append to existing data */
- int slim; /* In slim mode */
- int cache; /* Memory usable for cache in MB */
- struct middle_t *mid; /* Mid storage to use */
- struct output_t *out; /* Output type used */
- const char *tblsmain_index; /* Pg Tablespace to store indexes on main tables */
- const char *tblsslim_index; /* Pg Tablespace to store indexes on slim tables */
- const char *tblsmain_data; /* Pg Tablespace to store main tables */
- const char *tblsslim_data; /* Pg Tablespace to store slim tables */
- const char *style; /* style file to use */
- int expire_tiles_zoom; /* Zoom level for tile expiry list */
- int expire_tiles_zoom_min; /* Minimum zoom level for tile expiry list */
- const char *expire_tiles_filename; /* File name to output expired tiles list to */
- int enable_hstore; /* add an additional hstore column with objects key/value pairs */
- int enable_hstore_index; /* add an index on the hstore column */
- int enable_multi; /* Output multi-geometries intead of several simple geometries */
- const char** hstore_columns; /* list of columns that should be written into their own hstore column */
- int n_hstore_columns; /* number of hstore columns */
- int keep_coastlines;
- int parallel_indexing;
- int alloc_chunkwise;
- int num_procs;
- int droptemp; /* drop slim mode temp tables after act */
- int unlogged; /* use unlogged tables where possible */
- int hstore_match_only; /* only copy rows that match an explicitly listed key */
- int flat_node_cache_enabled;
- int excludepoly;
- const char *flat_node_file;
- const char *tag_transform_script;
-};
-
-struct output_t {
- int (*start)(const struct output_options *options);
- int (*connect)(const struct output_options *options, int startTransaction);
- void (*stop)();
- void (*cleanup)(void);
- void (*close)(int stopTransaction);
-
- int (*node_add)(osmid_t id, double lat, double lon, struct keyval *tags);
- int (*way_add)(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
- int (*relation_add)(osmid_t id, struct member *members, int member_count, struct keyval *tags);
-
- int (*node_modify)(osmid_t id, double lat, double lon, struct keyval *tags);
- int (*way_modify)(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
- int (*relation_modify)(osmid_t id, struct member *members, int member_count, struct keyval *tags);
-
- int (*node_delete)(osmid_t id);
- int (*way_delete)(osmid_t id);
- int (*relation_delete)(osmid_t id);
-};
-
-unsigned int pgsql_filter_tags(enum OsmType type, struct keyval *tags, int *polygon);
-
-#endif
diff --git a/output.hpp b/output.hpp
new file mode 100644
index 0000000..5f1388a
--- /dev/null
+++ b/output.hpp
@@ -0,0 +1,79 @@
+/* Common output layer interface */
+
+/* Each output layer must provide methods for
+ * storing:
+ * - Nodes (Points of interest etc)
+ * - Way geometries
+ * Associated tags: name, type etc.
+*/
+
+#ifndef OUTPUT_H
+#define OUTPUT_H
+
+#include "middle.hpp"
+#include "id-tracker.hpp"
+#include "expire-tiles.hpp"
+
+#include <boost/noncopyable.hpp>
+#include <boost/version.hpp>
+
+#include <utility>
+
+typedef std::pair<osmid_t, size_t> pending_job_t;
+#if BOOST_VERSION < 105300
+#include <stack>
+typedef std::stack<pending_job_t> pending_queue_t;
+#else
+#include <boost/lockfree/queue.hpp>
+typedef boost::lockfree::queue<pending_job_t> pending_queue_t;
+#endif
+
+class output_t : public boost::noncopyable {
+public:
+ static std::vector<boost::shared_ptr<output_t> > create_outputs(const middle_query_t *mid, const options_t &options);
+
+ output_t(const middle_query_t *mid, const options_t &options_);
+ virtual ~output_t();
+
+ virtual boost::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const = 0;
+
+ virtual int start() = 0;
+ virtual void stop() = 0;
+ virtual void commit() = 0;
+
+ virtual void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) = 0;
+ virtual int pending_way(osmid_t id, int exists) = 0;
+
+ virtual void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) = 0;
+ virtual int pending_relation(osmid_t id, int exists) = 0;
+
+ virtual int node_add(osmid_t id, double lat, double lon, struct keyval *tags) = 0;
+ virtual int way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags) = 0;
+ virtual int relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags) = 0;
+
+ virtual int node_modify(osmid_t id, double lat, double lon, struct keyval *tags) = 0;
+ virtual int way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags) = 0;
+ virtual int relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags) = 0;
+
+ virtual int node_delete(osmid_t id) = 0;
+ virtual int way_delete(osmid_t id) = 0;
+ virtual int relation_delete(osmid_t id) = 0;
+
+ virtual size_t pending_count() const;
+
+ const options_t *get_options() const;
+
+ virtual void merge_pending_relations(boost::shared_ptr<output_t> other);
+ virtual void merge_expire_trees(boost::shared_ptr<output_t> other);
+ virtual boost::shared_ptr<id_tracker> get_pending_relations();
+ virtual boost::shared_ptr<expire_tiles> get_expire_tree();
+
+protected:
+
+ const middle_query_t* m_mid;
+ const options_t m_options;
+};
+
+unsigned int pgsql_filter_tags(enum OsmType type, struct keyval *tags, int *polygon);
+
+#endif
diff --git a/parse-o5m.c b/parse-o5m.cpp
similarity index 86%
rename from parse-o5m.c
rename to parse-o5m.cpp
index 0f605bf..17eb378 100644
--- a/parse-o5m.c
+++ b/parse-o5m.cpp
@@ -21,11 +21,13 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#-----------------------------------------------------------------------------
*/
-#define _GNU_SOURCE
/* 2011-07-03 02:30
Markus Weber */
+// when __cplusplus is defined, we need to define this macro as well
+// to get the print format specifiers in the inttypes.h header.
+#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
@@ -34,13 +36,15 @@
#include <time.h>
#include <fcntl.h>
-#include "osmtypes.h"
-#include "reprojection.h"
-#include "output.h"
+#ifdef _WIN32
+#include <io.h>
+#endif
+
+#include "parse-o5m.hpp"
+#include "output.hpp"
#define inline
-typedef enum {false= 0,true= 1} bool;
typedef uint8_t byte;
typedef unsigned int uint;
#define isdig(x) isdigit((unsigned char)(x))
@@ -49,7 +53,7 @@ static int loglevel= 0; /* logging to stderr; */
3: extended logging; */
#define DP(f) fprintf(stderr,"- Debug: " #f "\n");
#define DPv(f,...) fprintf(stderr,"- Debug: " #f "\n",__VA_ARGS__);
-#if __WIN32__
+#ifdef _WIN32
#define NL "\r\n" /* use CR/LF as new-line sequence */
#define off_t off64_t
#define lseek lseek64
@@ -115,7 +119,7 @@ static inline void createtimestamp(uint64_t v,char* sp) {
int i;
vtime= v;
- #if __WIN32__
+ #ifdef _WIN32
memcpy(&tm,gmtime(&vtime),sizeof(tm));
#else
gmtime_r(&vtime,&tm);
@@ -143,7 +147,7 @@ static inline void createtimestamp(uint64_t v,char* sp) {
/*------------------------------------------------------------
Module pbf_ protobuf conversions module
------------------------------------------------------------
-
+
this module provides procedures for conversions from
protobuf formats to regular numbers;
as usual, all identifiers of a module have the same prefix,
@@ -294,7 +298,7 @@ static inline void pbf_intjump(byte** pp) {
/*------------------------------------------------------------
Module read_ OSM file read module
------------------------------------------------------------
-
+
this module provides procedures for buffered reading of
standard input;
as usual, all identifiers of a module have the same prefix,
@@ -341,7 +345,7 @@ static int read_open(const char* filename) {
read_infop: handle of the file;
note that you should close every opened file with read_close()
before the program ends;
-
+
save status of presently processed input file (if any) */
if(read_infop!=NULL) {
read_infop->bufp= read_bufp;
@@ -398,7 +402,7 @@ static void read_close() {
return;
fd= read_infop->fd;
if(loglevel>=1) { /* verbose */
- fprintf(stderr,"osm2pgsql: Number of bytes read: %"PRIu64"\n",
+ fprintf(stderr,"osm2pgsql: Number of bytes read: %"PRIu64"\n",
read_infop->read__counter);
}
if(loglevel>=2) {
@@ -473,7 +477,7 @@ static inline bool read_input() {
/*------------------------------------------------------------
Module str_ string read module
------------------------------------------------------------
-
+
this module provides procedures for conversions from
strings which have been stored in data stream objects to
c-formatted strings;
@@ -485,7 +489,7 @@ static inline bool read_input() {
by a horizontal line: ---- */
#define str__tabM (15000+4000)
-/* +4000 because it might happen that an object has a lot of
+/* +4000 because it might happen that an object has a lot of
key/val pairs or refroles which are not stored already; */
#define str__tabstrM 250 /* must be < row size of str__rab[] */
typedef struct str__info_struct {
@@ -604,11 +608,11 @@ static void str_read(byte** pp,char** s1p,char** s2p) {
else { /* string (pair) given by reference */
ref= pbf_uint32(pp);
if(ref>str__infop->tabn) { /* string reference invalid */
- WARNv("invalid .o5m string reference: %i->%i",
- str__infop->tabn,ref)
- *s1p= "(invalid)";
- if(s2p!=NULL) /* caller wants a string pair */
- *s2p= "(invalid)";
+ WARNv("invalid .o5m string reference: %i->%i",
+ str__infop->tabn,ref);
+ *s1p= strdup("(invalid)");
+ if(s2p!=NULL) /* caller wants a string pair */
+ *s2p= strdup("(invalid)");
} /* end string reference invalid */
else { /* string reference valid */
ref= str__infop->tabi-ref;
@@ -624,9 +628,19 @@ static void str_read(byte** pp,char** s1p,char** s2p) {
end Module str_ string read module
------------------------------------------------------------ */
+parse_o5m_t::parse_o5m_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr<reprojection>& projection_,
+ const double minlon, const double minlat, const double maxlon, const double maxlat):
+ parse_t(extra_attributes_, bbox_, projection_, minlon, minlat, maxlon, maxlat)
+{
+
+}
+
+parse_o5m_t::~parse_o5m_t()
+{
+}
-int streamFileO5m(char *filename,int sanitize,struct osmdata_t *osmdata) {
+int parse_o5m_t::streamFile(const char *filename, const int sanitize, osmdata_t *osmdata) {
/* open and parse an .o5m file; */
/* return: ==0: ok; !=0: error; */
int otype; /* type of currently processed object; */
@@ -669,7 +683,7 @@ return 1;
endoffile= false;
/* determine file type */ {
- char* p;
+ const char* p = NULL;
read_input();
if(*read_bufp!=0xff) { /* cannot be an .o5m file, nor an .o5c file */
@@ -678,18 +692,18 @@ return 1;
}
p= strchr(filename,0)-4; /* get end of filename */
if(memcmp(read_bufp,"\xff\xe0\0x04""o5m2",7)==0)
- osmdata->filetype= FILETYPE_OSM;
+ filetype= FILETYPE_OSM;
else if(memcmp(read_bufp,"\xff\xe0\0x04""o5c2",7)==0)
- osmdata->filetype= FILETYPE_OSMCHANGE;
+ filetype= FILETYPE_OSMCHANGE;
else if(p>=filename && strcmp(p,".o5m")==0)
- osmdata->filetype= FILETYPE_OSM;
+ filetype= FILETYPE_OSM;
else if(p>=filename && (strcmp(p,".o5c")==0 || strcmp(p,".o5h")==0))
- osmdata->filetype= FILETYPE_OSMCHANGE;
+ filetype= FILETYPE_OSMCHANGE;
else {
WARN("File type not specified. Assuming .o5m")
- osmdata->filetype= FILETYPE_OSM;
+ filetype= FILETYPE_OSM;
}
- if(osmdata->filetype==FILETYPE_OSM)
+ if(filetype==FILETYPE_OSM)
PINFO("Processing .o5m file (not a change file).")
else
PINFO("Processing .o5c change file.")
@@ -747,44 +761,44 @@ return 1;
histime= 0;
hiscset= 0;
hisuid= 0;
- hisuser= "";
- osmdata->nd_count= 0;
- osmdata->member_count= 0;
+ hisuser= NULL;
+ nd_count= 0;
+ member_count= 0;
/* read object id */
bufp++;
l= pbf_uint32(&bufp);
read_bufp= bufe= bufp+l;
- osmdata->osm_id= o5id+= pbf_sint64(&bufp);
+ osm_id= o5id+= pbf_sint64(&bufp);
/* do statistics on object id */
switch(otype) {
case 0: /* node */
- if(osmdata->osm_id>osmdata->max_node)
- osmdata->max_node= osmdata->osm_id;
- if (osmdata->count_node == 0) {
- time(&osmdata->start_node);
+ if(osm_id>max_node)
+ max_node= osm_id;
+ if (count_node == 0) {
+ time(&start_node);
}
- osmdata->count_node++;
- if(osmdata->count_node%10000==0) printStatus(osmdata);
+ count_node++;
+ if(count_node%10000==0) printStatus();
break;
case 1: /* way */
- if(osmdata->osm_id>osmdata->max_way)
- osmdata->max_way= osmdata->osm_id;
- if (osmdata->count_way == 0) {
- time(&osmdata->start_way);
+ if(osm_id>max_way)
+ max_way= osm_id;
+ if (count_way == 0) {
+ time(&start_way);
}
- osmdata->count_way++;
- if(osmdata->count_way%1000==0) printStatus(osmdata);
+ count_way++;
+ if(count_way%1000==0) printStatus();
break;
case 2: /* relation */
- if(osmdata->osm_id>osmdata->max_rel)
- osmdata->max_rel= osmdata->osm_id;
- if (osmdata->count_rel == 0) {
- time(&osmdata->start_rel);
+ if(osm_id>max_rel)
+ max_rel= osm_id;
+ if (count_rel == 0) {
+ time(&start_rel);
}
- osmdata->count_rel++;
- if(osmdata->count_rel%10==0) printStatus(osmdata);
+ count_rel++;
+ if(count_rel%10==0) printStatus();
break;
default: ;
}
@@ -795,18 +809,18 @@ return 1;
hisver= pbf_uint32(&bufp);
uint32toa(hisver,tmpstr);
- addItem(&(osmdata->tags),"osm_version",tmpstr,0);
+ keyval::addItem(&(tags),"osm_version",tmpstr,0);
if(hisver!=0) { /* history information available */
histime= o5histime+= pbf_sint64(&bufp);
createtimestamp(histime,tmpstr);
- addItem(&(osmdata->tags),"osm_timestamp",tmpstr, 0);
+ keyval::addItem(&(tags),"osm_timestamp",tmpstr, 0);
if(histime!=0) {
hiscset= o5hiscset+= pbf_sint32(&bufp); /* (not used) */
str_read(&bufp,&sp,&hisuser);
hisuid= pbf_uint64((byte**)&sp);
uint32toa(hisuid,tmpstr);
- addItem(&(osmdata->tags),"osm_uid",tmpstr,0);
- addItem(&(osmdata->tags),"osm_user",hisuser,0);
+ keyval::addItem(&(tags),"osm_uid",tmpstr,0);
+ keyval::addItem(&(tags),"osm_user",hisuser,0);
}
} /* end history information available */
} /* end read history */
@@ -814,40 +828,40 @@ return 1;
/* perform action */
if(bufp>=bufe) {
/* just the id and history, i.e. this is a delete request */
- osmdata->action= ACTION_DELETE;
+ action= ACTION_DELETE;
switch(otype) {
case 0: /* node */
- osmdata->out->node_delete(osmdata->osm_id);
+ osmdata->node_delete(osm_id);
break;
case 1: /* way */
- osmdata->out->way_delete(osmdata->osm_id);
+ osmdata->way_delete(osm_id);
break;
case 2: /* relation */
- osmdata->out->relation_delete(osmdata->osm_id);
+ osmdata->relation_delete(osm_id);
break;
default: ;
}
- resetList(&(osmdata->tags));
+ keyval::resetList(&(tags));
continue; /* end processing for this object */
} /* end delete request */
else { /* not a delete request */
/* determine action */
- if(osmdata->filetype==FILETYPE_OSMCHANGE && hisver>1)
- osmdata->action= ACTION_MODIFY;
+ if(filetype==FILETYPE_OSMCHANGE && hisver>1)
+ action= ACTION_MODIFY;
else
- osmdata->action= ACTION_CREATE;
+ action= ACTION_CREATE;
/* read coordinates (for nodes only) */
if(otype==0) { /* node */
/* read node body */
- osmdata->node_lon= (double)(o5lon+= pbf_sint32(&bufp))/10000000;
- osmdata->node_lat= (double)(o5lat+= pbf_sint32(&bufp))/10000000;
- if(!node_wanted(osmdata,osmdata->node_lat,osmdata->node_lon)) {
- resetList(&(osmdata->tags));
+ node_lon= (double)(o5lon+= pbf_sint32(&bufp))/10000000;
+ node_lat= (double)(o5lat+= pbf_sint32(&bufp))/10000000;
+ if(!node_wanted(node_lat,node_lon)) {
+ keyval::resetList(&(tags));
continue;
}
- reproject(&(osmdata->node_lat),&(osmdata->node_lon));
+ proj->reproject(&(node_lat),&(node_lon));
} /* end node */
/* read noderefs (for ways only) */
@@ -856,9 +870,9 @@ return 1;
bp= bufp+l;
if(bp>bufe) bp= bufe; /* (format error) */
while(bufp<bp) { /* for all noderefs of this way */
- osmdata->nds[osmdata->nd_count++]= o5rid[0]+= pbf_sint64(&bufp);
- if(osmdata->nd_count>=osmdata->nd_max)
- realloc_nodes(osmdata);
+ nds[nd_count++]= o5rid[0]+= pbf_sint64(&bufp);
+ if(nd_count>=nd_max)
+ realloc_nodes();
} /* end for all noderefs of this way */
} /* end way */
@@ -877,20 +891,20 @@ return 1;
rt= (*rr++ -'0')%3;
switch(rt) {
case 0: /* node */
- osmdata->members[osmdata->member_count].type= OSMTYPE_NODE;
+ members[member_count].type= OSMTYPE_NODE;
break;
case 1: /* way */
- osmdata->members[osmdata->member_count].type= OSMTYPE_WAY;
+ members[member_count].type= OSMTYPE_WAY;
break;
case 2: /* relation */
- osmdata->members[osmdata->member_count].type= OSMTYPE_RELATION;
+ members[member_count].type= OSMTYPE_RELATION;
break;
}
- osmdata->members[osmdata->member_count].id= o5rid[rt]+= ri;
- osmdata->members[osmdata->member_count].role= rr;
- osmdata->member_count++;
- if(osmdata->member_count>=osmdata->member_max)
- realloc_members(osmdata);
+ members[member_count].id= o5rid[rt]+= ri;
+ members[member_count].role= rr;
+ member_count++;
+ if(member_count>=member_max)
+ realloc_members();
} /* end for all references of this relation */
} /* end relation */
@@ -906,49 +920,48 @@ return 1;
/* replace all blanks in key by underlines */
p++;
}
- addItem(&(osmdata->tags),k,v,0);
+ keyval::addItem(&(tags),k,v,0);
}
} /* end for all tags of this object */
/* write object into database */
switch(otype) {
case 0: /* node */
- if(osmdata->action==ACTION_CREATE)
- osmdata->out->node_add(osmdata->osm_id,
- osmdata->node_lat,osmdata->node_lon,&(osmdata->tags));
+ if(action==ACTION_CREATE)
+ osmdata->node_add(osm_id,
+ node_lat,node_lon,&(tags));
else /* ACTION_MODIFY */
- osmdata->out->node_modify(osmdata->osm_id,
- osmdata->node_lat,osmdata->node_lon,&(osmdata->tags));
+ osmdata->node_modify(osm_id,
+ node_lat,node_lon,&(tags));
break;
case 1: /* way */
- if(osmdata->action==ACTION_CREATE)
- osmdata->out->way_add(osmdata->osm_id,
- osmdata->nds,osmdata->nd_count,&(osmdata->tags));
+ if(action==ACTION_CREATE)
+ osmdata->way_add(osm_id,
+ nds,nd_count,&(tags));
else /* ACTION_MODIFY */
- osmdata->out->way_modify(osmdata->osm_id,
- osmdata->nds,osmdata->nd_count,&(osmdata->tags));
+ osmdata->way_modify(osm_id,
+ nds,nd_count,&(tags));
break;
- case 2: /* relation */
- if(osmdata->action==ACTION_CREATE)
- osmdata->out->relation_add(osmdata->osm_id,
- osmdata->members,osmdata->member_count,&(osmdata->tags));
+ case 2: /* relation */
+ if(action==ACTION_CREATE)
+ osmdata->relation_add(osm_id,
+ members,member_count,&(tags));
else /* ACTION_MODIFY */
- osmdata->out->relation_modify(osmdata->osm_id,
- osmdata->members,osmdata->member_count,&(osmdata->tags));
+ osmdata->relation_modify(osm_id,
+ members,member_count,&(tags));
break;
default: ;
}
/* reset temporary storage lists */
- resetList(&(osmdata->tags));
+ keyval::resetList(&(tags));
} /* end not a delete request */
} /* end read input file */
/* close the input file */
- printStatus(osmdata);
+ printStatus();
read_close();
return 0;
} /* streamFileO5m() */
-
diff --git a/parse-o5m.h b/parse-o5m.hpp
similarity index 74%
rename from parse-o5m.h
rename to parse-o5m.hpp
index a97a261..d7d3f2c 100644
--- a/parse-o5m.h
+++ b/parse-o5m.hpp
@@ -25,10 +25,17 @@
#ifndef PARSE_O5M_H
#define PARSE_O5M_H
-int streamFileO5m(char *filename,int sanitize,struct osmdata_t *osmdata);
-
-#endif
-
-
+#include "parse.hpp"
+class parse_o5m_t: public parse_t
+{
+public:
+ parse_o5m_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr<reprojection>& projection_,
+ const double minlon, const double minlat, const double maxlon, const double maxlat);
+ virtual ~parse_o5m_t();
+ virtual int streamFile(const char *filename, const int sanitize, osmdata_t *osmdata);
+protected:
+ parse_o5m_t();
+};
+#endif
diff --git a/parse-pbf.c b/parse-pbf.cpp
similarity index 65%
rename from parse-pbf.c
rename to parse-pbf.cpp
index 247ccaf..348236a 100644
--- a/parse-pbf.c
+++ b/parse-pbf.cpp
@@ -25,19 +25,21 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
+#ifdef _WIN32
+#include <winsock2.h>
+#else
#include <arpa/inet.h>
+#endif
#include <time.h>
#include <zlib.h>
-#include "osmtypes.h"
-#include "output.h"
-#include "reprojection.h"
+#include "config.h"
-#include "fileformat.pb-c.h"
-#include "osmformat.pb-c.h"
+#ifdef BUILD_READER_PBF
-#define UNUSED __attribute__ ((unused))
+#include "parse-pbf.hpp"
+#include "output.hpp"
#define MAX_BLOCK_HEADER_SIZE 64*1024
#define MAX_BLOB_SIZE 32*1024*1024
@@ -68,24 +70,24 @@ static void *realloc_or_free(void *p, size_t len)
#endif
static BlockHeader *read_header(FILE *input, void *buf)
-{
- BlockHeader *header_msg;
+{
+ BlockHeader *header_msg;
size_t read, length = get_length(input);
-
+
if (length < 1 || length > MAX_BLOCK_HEADER_SIZE) {
if (!feof(input)) {
fprintf(stderr, "Invalid blocksize %lu\n", (unsigned long)length);
}
return NULL;
}
-
+
read = fread(buf, length, 1, input);
if (!read) {
perror("parse-pbf: error while reading header data");
return NULL;
}
-
- header_msg = block_header__unpack (NULL, length, buf);
+
+ header_msg = block_header__unpack (NULL, length, (const uint8_t *)buf);
if (header_msg == NULL) {
fprintf(stderr, "Error unpacking BlockHeader message\n");
return NULL;
@@ -102,28 +104,28 @@ static Blob *read_blob(FILE *input, void *buf, int32_t length)
fprintf(stderr, "Blob isn't present or exceeds minimum/maximum size\n");
return NULL;
}
-
+
if(1 != fread(buf, length, 1, input)) {
fprintf(stderr, "error reading blob content\n");
return NULL;
}
-
- blob_msg = blob__unpack (NULL, length, buf);
+
+ blob_msg = blob__unpack (NULL, length, (const uint8_t *)buf);
if (blob_msg == NULL) {
fprintf(stderr, "Error unpacking Blob message\n");
return NULL;
}
-
+
return blob_msg;
}
-static size_t uncompress_blob(Blob *bmsg, void *buf, int32_t max_size)
+static size_t uncompress_blob(Blob *bmsg, void *buf, int32_t max_size)
{
if (bmsg->raw_size > max_size) {
fprintf(stderr, "blob raw size too large\n");
return 0;
}
-
+
if (bmsg->has_raw) {
memcpy(buf, bmsg->raw.data, bmsg->raw.len);
return bmsg->raw.len;
@@ -136,7 +138,7 @@ static size_t uncompress_blob(Blob *bmsg, void *buf, int32_t max_size)
strm.avail_in = bmsg->zlib_data.len;
strm.next_in = bmsg->zlib_data.data;
strm.avail_out = bmsg->raw_size;
- strm.next_out = buf;
+ strm.next_out = (Bytef *)buf;
ret = inflateInit(&strm);
if (ret != Z_OK) {
@@ -145,9 +147,9 @@ static size_t uncompress_blob(Blob *bmsg, void *buf, int32_t max_size)
}
ret = inflate(&strm, Z_NO_FLUSH);
-
+
(void)inflateEnd(&strm);
-
+
if (ret != Z_STREAM_END) {
fprintf(stderr, "Zlib compression failed (code %d, %s)\n", ret, strm.msg);
return 0;
@@ -164,7 +166,7 @@ static size_t uncompress_blob(Blob *bmsg, void *buf, int32_t max_size)
fprintf(stderr, "We cannot handle the %d non-raw bytes yet...\n", bmsg->raw_size);
return 0;
}
-
+
return 0;
}
@@ -173,7 +175,7 @@ int addProtobufItem(struct keyval *head, ProtobufCBinaryData key, ProtobufCBinar
char *keystr, *valstr;
int retval;
- keystr = calloc(key.len + 1, 1);
+ keystr = (char *)calloc(key.len + 1, 1);
memcpy(keystr, key.data, key.len);
/* drop certain keys (matching parse-xml2) */
@@ -182,10 +184,10 @@ int addProtobufItem(struct keyval *head, ProtobufCBinaryData key, ProtobufCBinar
return 0;
}
- valstr = calloc(val.len + 1, 1);
+ valstr = (char *)calloc(val.len + 1, 1);
memcpy(valstr, val.data, val.len);
-
- retval = addItem(head, keystr, valstr, noDupe);
+
+ retval = keyval::addItem(head, keystr, valstr, noDupe);
free(keystr);
free(valstr);
@@ -198,28 +200,28 @@ int addIntItem(struct keyval *head, const char *key, int val, int noDupe)
char buf[100];
sprintf(buf, "%d", val);
- return addItem(head, key, buf, noDupe);
+ return keyval::addItem(head, key, buf, noDupe);
}
int addInfoItems(struct keyval *head, Info *info, StringTable *string_table)
{
if (info->has_version) {
- addIntItem(head, "osm_version", info->version, 0);
+ addIntItem(head, "osm_version", info->version, 0);
}
if (info->has_changeset) {
- addIntItem(head, "osm_changeset", info->changeset, 0);
+ addIntItem(head, "osm_changeset", info->changeset, 0);
}
if (info->has_uid) {
- addIntItem(head, "osm_uid", info->uid, 0);
+ addIntItem(head, "osm_uid", info->uid, 0);
}
if (info->has_user_sid) {
ProtobufCBinaryData user = string_table->s[info->user_sid];
char *username;
- username = calloc(user.len + 1, 1);
+ username = (char *)calloc(user.len + 1, 1);
memcpy(username, user.data, user.len);
- addItem(head, "osm_user", username, 0);
+ keyval::addItem(head, "osm_user", username, 0);
free(username);
}
@@ -230,61 +232,61 @@ int addInfoItems(struct keyval *head, Info *info, StringTable *string_table)
int processOsmHeader(void *data, size_t length)
{
- HeaderBlock *hmsg = header_block__unpack (NULL, length, data);
+ HeaderBlock *hmsg = header_block__unpack (NULL, length, (const uint8_t *)data);
if (hmsg == NULL) {
fprintf(stderr, "Error unpacking HeaderBlock message\n");
return 0;
}
-
+
header_block__free_unpacked (hmsg, NULL);
return 1;
}
-int processOsmDataNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table, double lat_offset, double lon_offset, double granularity)
+int parse_pbf_t::processOsmDataNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table, double lat_offset, double lon_offset, double granularity)
{
unsigned node_id, key_id;
for (node_id = 0; node_id < group->n_nodes; node_id++) {
Node *node = group->nodes[node_id];
double lat, lon;
- resetList(&(osmdata->tags));
+ keyval::resetList(&(tags));
- if (node->info && osmdata->extra_attributes) {
- addInfoItems(&(osmdata->tags), node->info, string_table);
+ if (node->info && extra_attributes) {
+ addInfoItems(&(tags), node->info, string_table);
}
for (key_id = 0; key_id < node->n_keys; key_id++) {
- addProtobufItem(&(osmdata->tags),
- string_table->s[node->keys[key_id]],
- string_table->s[node->vals[key_id]],
+ addProtobufItem(&(tags),
+ string_table->s[node->keys[key_id]],
+ string_table->s[node->vals[key_id]],
0);
}
lat = lat_offset + (node->lat * granularity);
lon = lon_offset + (node->lon * granularity);
- if (node_wanted(osmdata, lat, lon)) {
- reproject(&lat, &lon);
-
- osmdata->out->node_add(node->id, lat, lon, &(osmdata->tags));
-
- if (node->id > osmdata->max_node) {
- osmdata->max_node = node->id;
+ if (node_wanted(lat, lon)) {
+ proj->reproject(&lat, &lon);
+
+ osmdata->node_add(node->id, lat, lon, &(tags));
+
+ if (node->id > max_node) {
+ max_node = node->id;
}
-
- if (osmdata->count_node == 0) {
- time(&osmdata->start_node);
+
+ if (count_node == 0) {
+ time(&start_node);
}
- osmdata->count_node++;
- if (osmdata->count_node%10000 == 0)
- printStatus(osmdata);
+ count_node++;
+ if (count_node%10000 == 0)
+ printStatus();
}
}
return 1;
}
-int processOsmDataDenseNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table, double lat_offset, double lon_offset, double granularity)
+int parse_pbf_t::processOsmDataDenseNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table, double lat_offset, double lon_offset, double granularity)
{
unsigned node_id;
if (group->dense) {
@@ -297,210 +299,210 @@ int processOsmDataDenseNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, S
long int deltauid = 0;
unsigned long int deltauser_sid = 0;
double lat, lon;
-
+
DenseNodes *dense = group->dense;
-
+
for (node_id = 0; node_id < dense->n_id; node_id++) {
- resetList(&(osmdata->tags));
-
+ keyval::resetList(&(tags));
+
deltaid += dense->id[node_id];
deltalat += dense->lat[node_id];
deltalon += dense->lon[node_id];
-
- if (dense->denseinfo && osmdata->extra_attributes) {
+
+ if (dense->denseinfo && extra_attributes) {
DenseInfo *denseinfo = dense->denseinfo;
-
+
deltatimestamp += denseinfo->timestamp[node_id];
deltachangeset += denseinfo->changeset[node_id];
deltauid += denseinfo->uid[node_id];
deltauser_sid += denseinfo->user_sid[node_id];
-
- addIntItem(&(osmdata->tags), "osm_version", denseinfo->version[node_id], 0);
- addIntItem(&(osmdata->tags), "osm_changeset", deltachangeset, 0);
-
+
+ addIntItem(&(tags), "osm_version", denseinfo->version[node_id], 0);
+ addIntItem(&(tags), "osm_changeset", deltachangeset, 0);
+
if (deltauid != -1) { /* osmosis devs failed to read the specs */
char * valstr;
- addIntItem(&(osmdata->tags), "osm_uid", deltauid, 0);
- valstr = calloc(string_table->s[deltauser_sid].len + 1, 1);
+ addIntItem(&(tags), "osm_uid", deltauid, 0);
+ valstr = (char *)calloc(string_table->s[deltauser_sid].len + 1, 1);
memcpy(valstr, string_table->s[deltauser_sid].data, string_table->s[deltauser_sid].len);
- addItem(&(osmdata->tags), "osm_user", valstr, 0);
+ keyval::addItem(&(tags), "osm_user", valstr, 0);
free(valstr);
}
}
-
+
if (l < dense->n_keys_vals) {
while (dense->keys_vals[l] != 0 && l < dense->n_keys_vals) {
- addProtobufItem(&(osmdata->tags),
+ addProtobufItem(&(tags),
string_table->s[dense->keys_vals[l]],
string_table->s[dense->keys_vals[l+1]],
0);
-
+
l += 2;
}
l += 1;
}
-
+
lat = lat_offset + (deltalat * granularity);
lon = lon_offset + (deltalon * granularity);
- if (node_wanted(osmdata, lat, lon)) {
- reproject(&lat, &lon);
-
- osmdata->out->node_add(deltaid, lat, lon, &(osmdata->tags));
-
- if (deltaid > osmdata->max_node) {
- osmdata->max_node = deltaid;
+ if (node_wanted(lat, lon)) {
+ proj->reproject(&lat, &lon);
+
+ osmdata->node_add(deltaid, lat, lon, &(tags));
+
+ if (deltaid > max_node) {
+ max_node = deltaid;
}
-
- if (osmdata->count_node == 0) {
- time(&osmdata->start_node);
+
+ if (count_node == 0) {
+ time(&start_node);
}
- osmdata->count_node++;
- if (osmdata->count_node%10000 == 0)
- printStatus(osmdata);
+ count_node++;
+ if (count_node%10000 == 0)
+ printStatus();
}
}
}
-
+
return 1;
}
-int processOsmDataWays(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table)
+int parse_pbf_t::processOsmDataWays(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table)
{
unsigned way_id, key_id, ref_id;
for (way_id = 0; way_id < group->n_ways; way_id++) {
Way *way = group->ways[way_id];
osmid_t deltaref = 0;
- resetList(&(osmdata->tags));
+ keyval::resetList(&(tags));
- if (way->info && osmdata->extra_attributes) {
- addInfoItems(&(osmdata->tags), way->info, string_table);
+ if (way->info && extra_attributes) {
+ addInfoItems(&(tags), way->info, string_table);
}
- osmdata->nd_count = 0;
+ nd_count = 0;
for (ref_id = 0; ref_id < way->n_refs; ref_id++) {
deltaref += way->refs[ref_id];
-
- osmdata->nds[osmdata->nd_count++] = deltaref;
-
- if( osmdata->nd_count >= osmdata->nd_max )
- realloc_nodes(osmdata);
+
+ nds[nd_count++] = deltaref;
+
+ if( nd_count >= nd_max )
+ realloc_nodes();
}
for (key_id = 0; key_id < way->n_keys; key_id++) {
- addProtobufItem(&(osmdata->tags),
- string_table->s[way->keys[key_id]],
- string_table->s[way->vals[key_id]],
+ addProtobufItem(&(tags),
+ string_table->s[way->keys[key_id]],
+ string_table->s[way->vals[key_id]],
0);
}
- osmdata->out->way_add(way->id,
- osmdata->nds,
- osmdata->nd_count,
- &(osmdata->tags) );
+ osmdata->way_add(way->id,
+ nds,
+ nd_count,
+ &(tags) );
- if (way->id > osmdata->max_way) {
- osmdata->max_way = way->id;
+ if (way->id > max_way) {
+ max_way = way->id;
}
- if (osmdata->count_way == 0) {
- time(&osmdata->start_way);
+ if (count_way == 0) {
+ time(&start_way);
}
- osmdata->count_way++;
- if (osmdata->count_way%1000 == 0)
- printStatus(osmdata);
+ count_way++;
+ if (count_way%1000 == 0)
+ printStatus();
}
return 1;
}
-int processOsmDataRelations(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table)
+int parse_pbf_t::processOsmDataRelations(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table)
{
unsigned rel_id, member_id, key_id;
for (rel_id = 0; rel_id < group->n_relations; rel_id++) {
Relation *relation = group->relations[rel_id];
osmid_t deltamemids = 0;
- resetList(&(osmdata->tags));
+ keyval::resetList(&(tags));
- osmdata->member_count = 0;
+ member_count = 0;
- if (relation->info && osmdata->extra_attributes) {
- addInfoItems(&(osmdata->tags), relation->info, string_table);
+ if (relation->info && extra_attributes) {
+ addInfoItems(&(tags), relation->info, string_table);
}
-
+
for (member_id = 0; member_id < relation->n_memids; member_id++) {
ProtobufCBinaryData role = string_table->s[relation->roles_sid[member_id]];
char *rolestr;
deltamemids += relation->memids[member_id];
- osmdata->members[osmdata->member_count].id = deltamemids;
+ members[member_count].id = deltamemids;
- rolestr = calloc(role.len + 1, 1);
+ rolestr = (char *)calloc(role.len + 1, 1);
memcpy(rolestr, role.data, role.len);
- osmdata->members[osmdata->member_count].role = rolestr;
+ members[member_count].role = rolestr;
switch (relation->types[member_id]) {
case RELATION__MEMBER_TYPE__NODE:
- osmdata->members[osmdata->member_count].type = OSMTYPE_NODE;
+ members[member_count].type = OSMTYPE_NODE;
break;
case RELATION__MEMBER_TYPE__WAY:
- osmdata->members[osmdata->member_count].type = OSMTYPE_WAY;
+ members[member_count].type = OSMTYPE_WAY;
break;
case RELATION__MEMBER_TYPE__RELATION:
- osmdata->members[osmdata->member_count].type = OSMTYPE_RELATION;
+ members[member_count].type = OSMTYPE_RELATION;
break;
default:
fprintf(stderr, "Unsupported type: %u""\n", relation->types[member_id]);
return 0;
}
-
- osmdata->member_count++;
-
- if( osmdata->member_count >= osmdata->member_max ) {
- realloc_members(osmdata);
+
+ member_count++;
+
+ if( member_count >= member_max ) {
+ realloc_members();
}
}
for (key_id = 0; key_id < relation->n_keys; key_id++) {
- addProtobufItem(&(osmdata->tags),
- string_table->s[relation->keys[key_id]],
- string_table->s[relation->vals[key_id]],
+ addProtobufItem(&(tags),
+ string_table->s[relation->keys[key_id]],
+ string_table->s[relation->vals[key_id]],
0);
}
-
- osmdata->out->relation_add(relation->id,
- osmdata->members,
- osmdata->member_count,
- &(osmdata->tags));
-
- for (member_id = 0; member_id < osmdata->member_count; member_id++) {
- free(osmdata->members[member_id].role);
+
+ osmdata->relation_add(relation->id,
+ members,
+ member_count,
+ &(tags));
+
+ for (member_id = 0; member_id < member_count; member_id++) {
+ free(members[member_id].role);
}
- if (relation->id > osmdata->max_rel) {
- osmdata->max_rel = relation->id;
+ if (relation->id > max_rel) {
+ max_rel = relation->id;
}
- if (osmdata->count_rel == 0) {
- time(&osmdata->start_rel);
+ if (count_rel == 0) {
+ time(&start_rel);
}
- osmdata->count_rel++;
- if (osmdata->count_rel%10 == 0)
- printStatus(osmdata);
+ count_rel++;
+ if (count_rel%10 == 0)
+ printStatus();
}
return 1;
}
-int processOsmData(struct osmdata_t *osmdata, void *data, size_t length)
+int parse_pbf_t::processOsmData(struct osmdata_t *osmdata, void *data, size_t length)
{
unsigned int j;
double lat_offset, lon_offset, granularity;
- PrimitiveBlock *pmsg = primitive_block__unpack (NULL, length, data);
+ PrimitiveBlock *pmsg = primitive_block__unpack (NULL, length, (const uint8_t *)data);
if (pmsg == NULL) {
fprintf(stderr, "Error unpacking PrimitiveBlock message\n");
return 0;
@@ -508,7 +510,7 @@ int processOsmData(struct osmdata_t *osmdata, void *data, size_t length)
lat_offset = NANO_DEGREE * pmsg->lat_offset;
lon_offset = NANO_DEGREE * pmsg->lon_offset;
- granularity = NANO_DEGREE * pmsg->granularity;
+ granularity = NANO_DEGREE * pmsg->granularity;
for (j = 0; j < pmsg->n_primitivegroup; j++) {
PrimitiveGroup *group = pmsg->primitivegroup[j];
@@ -525,16 +527,26 @@ int processOsmData(struct osmdata_t *osmdata, void *data, size_t length)
return 1;
}
+parse_pbf_t::parse_pbf_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr<reprojection>& projection_,
+ const double minlon, const double minlat, const double maxlon, const double maxlat):
+ parse_t(extra_attributes_, bbox_, projection_, minlon, minlat, maxlon, maxlat)
+{
+}
-int streamFilePbf(char *filename, int sanitize UNUSED, struct osmdata_t *osmdata)
+parse_pbf_t::~parse_pbf_t()
+{
+
+}
+
+int parse_pbf_t::streamFile(const char *filename, const int, osmdata_t *osmdata)
{
void *header = NULL;
void *blob = NULL;
char *data = NULL;
FILE *input = NULL;
- BlockHeader *header_msg = NULL;
- Blob *blob_msg = NULL;
+ BlockHeader *header_msg = NULL;
+ Blob *blob_msg = NULL;
size_t length;
int exit_status = EXIT_FAILURE;
@@ -550,14 +562,14 @@ int streamFilePbf(char *filename, int sanitize UNUSED, struct osmdata_t *osmdata
goto err;
}
- data = malloc(MAX_BLOB_SIZE);
+ data = (char *)malloc(MAX_BLOB_SIZE);
if (!data) {
fprintf(stderr, "parse-pbf: out of memory allocating data buffer\n");
goto err;
}
input = fopen(filename, "rb");
- if (!input) {
+ if (!input) {
fprintf(stderr, "Unable to open %s\n", filename);
goto err;
}
@@ -601,6 +613,7 @@ int streamFilePbf(char *filename, int sanitize UNUSED, struct osmdata_t *osmdata
if (header) free(header);
if (blob) free(blob);
if (data) free(data);
-
+
return exit_status;
}
+#endif //BUILD_READER_PBF
diff --git a/parse-pbf.h b/parse-pbf.hpp
similarity index 50%
rename from parse-pbf.h
rename to parse-pbf.hpp
index 8783bb0..6382e74 100644
--- a/parse-pbf.h
+++ b/parse-pbf.hpp
@@ -25,6 +25,32 @@
#ifndef PARSE_PBF_H
#define PARSE_PBF_H
-int streamFilePbf(char *filename, int sanitize, struct osmdata_t *osmdata);
+#include "parse.hpp"
+
+#include "config.h"
+
+#ifdef BUILD_READER_PBF
+extern "C" {
+#include "fileformat.pb-c.h"
+#include "osmformat.pb-c.h"
+}
+
+class parse_pbf_t: public parse_t
+{
+public:
+ parse_pbf_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr<reprojection>& projection_,
+ const double minlon, const double minlat, const double maxlon, const double maxlat);
+ virtual ~parse_pbf_t();
+ virtual int streamFile(const char *filename, const int sanitize, osmdata_t *osmdata);
+protected:
+ parse_pbf_t();
+ int processOsmDataNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table, double lat_offset, double lon_offset, double granularity);
+ int processOsmDataDenseNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table, double lat_offset, double lon_offset, double granularity);
+ int processOsmDataWays(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table);
+ int processOsmDataRelations(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table);
+ int processOsmData(struct osmdata_t *osmdata, void *data, size_t length);
+};
+
+#endif //BUILD_READER_PBF
#endif
diff --git a/parse-primitive.c b/parse-primitive.c
deleted file mode 100644
index 235418f..0000000
--- a/parse-primitive.c
+++ /dev/null
@@ -1,486 +0,0 @@
-/*
-#-----------------------------------------------------------------------------
-# osm2pgsql - converts planet.osm file into PostgreSQL
-# compatible output suitable to be rendered by mapnik
-#-----------------------------------------------------------------------------
-# Original Python implementation by Artem Pavlenko
-# Re-implementation by Jon Burgess, Copyright 2006
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#-----------------------------------------------------------------------------
-*/
-
-/*
-
-This is a version of osm2pgsql without proper XML parsing
-it should arrive at the same results as the normal osm2pgsql
-and take an hour less to process the full planet file but
-YMMV. This is just a proof of concept and should not be used
-in a production environment.
-
- */
-
-#define _GNU_SOURCE
-#define UNUSED __attribute__ ((unused))
-
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <ctype.h>
-
-#include "osmtypes.h"
-#include "sanitizer.h"
-#include "reprojection.h"
-#include "input.h"
-#include "output.h"
-
-
-char *extractAttribute(char **token, int tokens, char *attname)
-{
- char buffer[256];
- int cl;
- int i;
- char *in;
- char *out;
- sprintf(buffer, "%s=\"", attname);
- cl = strlen(buffer);
- for (i=0; i<tokens; i++)
- {
- if (!strncmp(token[i], buffer, cl))
- {
- char *quote = strchr(token[i] + cl, '"');
- if (quote == NULL) quote = token[i] + strlen(token[i]) + 1;
- *quote = 0;
- if (strchr(token[i]+cl, '&') == 0) return (token[i] + cl);
-
- for (in=token[i]+cl, out=token[i]+cl; *in; in++)
- {
- if (*in == '&')
- {
- if (!strncmp(in+1, "quot;", 5))
- {
- *out++ = '"';
- in+=5;
- }
- else if (!strncmp(in+1, "lt;", 3))
- {
- *out++ = '<';
- in+=3;
- }
- else if (!strncmp(in+1, "gt;", 3))
- {
- *out++ = '>';
- in+=3;
- }
- else if (!strncmp(in+1, "apos;", 5))
- {
- *out++ = '\'';
- in+=5;
- }
- }
- else
- {
- *out++ = *in;
- }
- }
- *out = 0;
- return (token[i]+cl);
- }
- }
- return NULL;
-}
-
-/* Parses the action="foo" tags in JOSM change files. Obvisouly not useful from osmChange files */
-static actions_t ParseAction(char **token, int tokens, struct osmdata_t *osmdata)
-{
- actions_t new_action;
- char *action;
- if( osmdata->filetype == FILETYPE_OSMCHANGE || osmdata->filetype == FILETYPE_PLANETDIFF )
- return osmdata->action;
- new_action = ACTION_NONE;
- action = extractAttribute(token, tokens, "action");
- if( action == NULL )
- new_action = ACTION_CREATE;
- else if( strcmp((char *)action, "modify") == 0 )
- new_action = ACTION_MODIFY;
- else if( strcmp((char *)action, "delete") == 0 )
- new_action = ACTION_DELETE;
- else
- {
- fprintf( stderr, "Unknown value for action: %s\n", (char*)action );
- exit_nicely();
- }
- return new_action;
-}
-
-static void StartElement(char *name, char *line, struct osmdata_t *osmdata)
-{
- char *xid, *xlat, *xlon, *xk, *xv, *xrole, *xtype;
- char *token[255];
- int tokens = 0;
- int quote = 0;
- char *i;
-
- if (osmdata->filetype == FILETYPE_NONE)
- {
- if (!strcmp(name, "?xml")) return;
- if (!strcmp(name, "osm"))
- {
- osmdata->filetype = FILETYPE_OSM;
- osmdata->action = ACTION_CREATE;
- }
- else if (!strcmp(name, "osmChange"))
- {
- osmdata->filetype = FILETYPE_OSMCHANGE;
- osmdata->action = ACTION_NONE;
- }
- else if (!strcmp(name, "planetdiff"))
- {
- osmdata->filetype = FILETYPE_PLANETDIFF;
- osmdata->action = ACTION_NONE;
- }
- else
- {
- fprintf( stderr, "Unknown XML document type: %s\n", name );
- exit_nicely();
- }
- return;
- }
-
- tokens=1;
- token[0] = line;
- for (i=line; *i; i++)
- {
- if (quote)
- {
- if (*i == '"')
- {
- quote = 0;
- }
- }
- else
- {
- if (*i == '"')
- {
- quote = 1;
- }
- else if (isspace(*i))
- {
- *i = 0;
- token[tokens++] = i + 1;
- }
- }
- }
-
- if (!strcmp(name, "node")) {
- xid = extractAttribute(token, tokens, "id");
- xlon = extractAttribute(token, tokens, "lon");
- xlat = extractAttribute(token, tokens, "lat");
- assert(xid); assert(xlon); assert(xlat);
-
- osmdata->osm_id = strtoosmid((char *)xid, NULL, 10);
- osmdata->node_lon = strtod((char *)xlon, NULL);
- osmdata->node_lat = strtod((char *)xlat, NULL);
- osmdata->action = ParseAction(token, tokens, osmdata);
-
- if (osmdata->osm_id > osmdata->max_node)
- osmdata->max_node = osmdata->osm_id;
-
- if (osmdata->count_node == 0) {
- time(&osmdata->start_node);
- }
-
- osmdata->count_node++;
- if (osmdata->count_node%10000 == 0)
- printStatus(osmdata);
-
- } else if (!strcmp(name, "tag")) {
- xk = extractAttribute(token, tokens, "k");
- assert(xk);
-
- /* 'created_by' and 'source' are common and not interesting to mapnik renderer */
- if (strcmp((char *)xk, "created_by") && strcmp((char *)xk, "source")) {
- char *p;
- xv = extractAttribute(token, tokens, "v");
- assert(xv);
- while ((p = strchr(xk, ' ')))
- *p = '_';
-
- addItem(&(osmdata->tags), xk, (char *)xv, 0);
- }
- } else if (!strcmp(name, "way")) {
-
- xid = extractAttribute(token, tokens, "id");
- assert(xid);
- osmdata->osm_id = strtoosmid((char *)xid, NULL, 10);
- osmdata->action = ParseAction(token, tokens, osmdata);
-
- if (osmdata->osm_id > osmdata->max_way)
- osmdata->max_way = osmdata->osm_id;
-
- if (osmdata->count_way == 0) {
- time(&osmdata->start_way);
- }
-
- osmdata->count_way++;
- if (osmdata->count_way%1000 == 0)
- printStatus(osmdata);
-
- osmdata->nd_count = 0;
- } else if (!strcmp(name, "nd")) {
- xid = extractAttribute(token, tokens, "ref");
- assert(xid);
-
- osmdata->nds[osmdata->nd_count++] = strtoosmid( (char *)xid, NULL, 10 );
-
- if( osmdata->nd_count >= osmdata->nd_max )
- realloc_nodes(osmdata);
- } else if (!strcmp(name, "relation")) {
- xid = extractAttribute(token, tokens, "id");
- assert(xid);
- osmdata->osm_id = strtoosmid((char *)xid, NULL, 10);
- osmdata->action = ParseAction(token, tokens, osmdata);
-
- if (osmdata->osm_id > osmdata->max_rel)
- osmdata->max_rel = osmdata->osm_id;
-
- if (osmdata->count_rel == 0) {
- time(&osmdata->start_rel);
- }
-
- osmdata->count_rel++;
- if (osmdata->count_rel%10 == 0)
- printStatus(osmdata);
-
- osmdata->member_count = 0;
- } else if (!strcmp(name, "member")) {
- xrole = extractAttribute(token, tokens, "role");
- assert(xrole);
-
- xtype = extractAttribute(token, tokens, "type");
- assert(xtype);
-
- xid = extractAttribute(token, tokens, "ref");
- assert(xid);
-
- osmdata->members[osmdata->member_count].id = strtoosmid( (char *)xid, NULL, 0 );
- osmdata->members[osmdata->member_count].role = strdup( (char *)xrole );
-
- /* Currently we are only interested in 'way' members since these form polygons with holes */
- if (!strcmp(xtype, "way"))
- osmdata->members[osmdata->member_count].type = OSMTYPE_WAY;
- else if (!strcmp(xtype, "node"))
- osmdata->members[osmdata->member_count].type = OSMTYPE_NODE;
- else if (!strcmp(xtype, "relation"))
- osmdata->members[osmdata->member_count].type = OSMTYPE_RELATION;
- osmdata->member_count++;
-
- if( osmdata->member_count >= osmdata->member_max )
- realloc_members(osmdata);
- } else if (!strcmp(name, "add") ||
- !strcmp(name, "create")) {
- osmdata->action = ACTION_MODIFY; /* Turns all creates into modifies, makes it resiliant against inconsistant snapshots. */
- } else if (!strcmp(name, "modify")) {
- osmdata->action = ACTION_MODIFY;
- } else if (!strcmp(name, "delete")) {
- osmdata->action = ACTION_DELETE;
- } else if (!strcmp(name, "bound")) {
- /* ignore */
- } else if (!strcmp(name, "bounds")) {
- /* ignore */
- } else if (!strcmp(name, "changeset")) {
- /* ignore */
- } else {
- fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
- }
-
- /* Collect extra attribute information and add as tags */
- if (osmdata->extra_attributes && (!strcmp(name, "node") ||
- !strcmp(name, "way") ||
- !strcmp(name, "relation")))
- {
- char *xtmp;
-
- xtmp = extractAttribute(token, tokens, "user");
- if (xtmp) {
- addItem(&(osmdata->tags), "osm_user", (char *)xtmp, 0);
- }
-
- xtmp = extractAttribute(token, tokens, "uid");
- if (xtmp) {
- addItem(&(osmdata->tags), "osm_uid", (char *)xtmp, 0);
- }
-
- xtmp = extractAttribute(token, tokens, "version");
- if (xtmp) {
- addItem(&(osmdata->tags), "osm_version", (char *)xtmp, 0);
- }
-
- xtmp = extractAttribute(token, tokens, "timestamp");
- if (xtmp) {
- addItem(&(osmdata->tags), "osm_timestamp", (char *)xtmp, 0);
- }
- }
-}
-
-static void EndElement(const char *name, struct osmdata_t *osmdata)
-{
- if (!strcmp(name, "node")) {
- if (node_wanted(osmdata, osmdata->node_lat, osmdata->node_lon)) {
- reproject(&(osmdata->node_lat), &(osmdata->node_lon));
- if( osmdata->action == ACTION_CREATE )
- osmdata->out->node_add(osmdata->osm_id, osmdata->node_lat, osmdata->node_lon, &(osmdata->tags));
- else if( osmdata->action == ACTION_MODIFY )
- osmdata->out->node_modify(osmdata->osm_id, osmdata->node_lat, osmdata->node_lon, &(osmdata->tags));
- else if( osmdata->action == ACTION_DELETE )
- osmdata->out->node_delete(osmdata->osm_id);
- else
- {
- fprintf( stderr, "Don't know action for node %" PRIdOSMID "\n", osmdata->osm_id );
- exit_nicely();
- }
- }
- resetList(&(osmdata->tags));
- } else if (!strcmp(name, "way")) {
- if( osmdata->action == ACTION_CREATE )
- osmdata->out->way_add(osmdata->osm_id, osmdata->nds, osmdata->nd_count, &(osmdata->tags) );
- else if( osmdata->action == ACTION_MODIFY )
- osmdata->out->way_modify(osmdata->osm_id, osmdata->nds, osmdata->nd_count, &(osmdata->tags) );
- else if( osmdata->action == ACTION_DELETE )
- osmdata->out->way_delete(osmdata->osm_id);
- else
- {
- fprintf( stderr, "Don't know action for way %" PRIdOSMID "\n", osmdata->osm_id );
- exit_nicely();
- }
- resetList(&(osmdata->tags));
- } else if (!strcmp(name, "relation")) {
- if( osmdata->action == ACTION_CREATE )
- osmdata->out->relation_add(osmdata->osm_id, osmdata->members, osmdata->member_count, &(osmdata->tags));
- else if( osmdata->action == ACTION_MODIFY )
- osmdata->out->relation_modify(osmdata->osm_id, osmdata->members, osmdata->member_count, &(osmdata->tags));
- else if( osmdata->action == ACTION_DELETE )
- osmdata->out->relation_delete(osmdata->osm_id);
- else
- {
- fprintf( stderr, "Don't know action for relation %" PRIdOSMID "\n", osmdata->osm_id );
- exit_nicely();
- }
- resetList(&(osmdata->tags));
- resetMembers(osmdata);
- } else if (!strcmp(name, "tag")) {
- /* ignore */
- } else if (!strcmp(name, "nd")) {
- /* ignore */
- } else if (!strcmp(name, "member")) {
- /* ignore */
- } else if (!strcmp(name, "osm")) {
- printStatus(osmdata);
- osmdata->filetype = FILETYPE_NONE;
- } else if (!strcmp(name, "osmChange")) {
- printStatus(osmdata);
- osmdata->filetype = FILETYPE_NONE;
- } else if (!strcmp(name, "planetdiff")) {
- printStatus(osmdata);
- osmdata->filetype = FILETYPE_NONE;
- } else if (!strcmp(name, "bound")) {
- /* ignore */
- } else if (!strcmp(name, "bounds")) {
- /* ignore */
- } else if (!strcmp(name, "changeset")) {
- /* ignore */
- resetList(&(osmdata->tags)); /* We may have accumulated some tags even if we ignored the changeset */
- } else if (!strcmp(name, "add")) {
- osmdata->action = ACTION_NONE;
- } else if (!strcmp(name, "create")) {
- osmdata->action = ACTION_NONE;
- } else if (!strcmp(name, "modify")) {
- osmdata->action = ACTION_NONE;
- } else if (!strcmp(name, "delete")) {
- osmdata->action = ACTION_NONE;
- } else {
- fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
- }
-}
-
-static void process(char *line, struct osmdata_t *osmdata) {
- char *lt = strchr(line, '<');
- if (lt)
- {
- char *spc = strchr(lt+1, ' ');
- char *gt = strchr(lt+1, '>');
- char *nx = spc;
- if (*(lt+1) == '/')
- {
- *gt = 0;
- EndElement(lt+2, osmdata);
- }
- else
- {
- int slash = 0;
- if (gt != NULL) {
- *gt-- = 0;
- if (nx == NULL || gt < nx) nx = gt;
- while(gt>lt)
- {
- if (*gt=='/') { slash=1; *gt=0; break; }
- if (!isspace(*gt)) break;
- gt--;
- }
- }
- *nx++ = 0;
- /* printf ("nx=%d, lt+1=#%s#\n", nx-lt,lt+1); */
- StartElement(lt+1, nx, osmdata);
- if (slash) EndElement(lt+1, osmdata);
- }
- }
-}
-
-int streamFilePrimitive(char *filename, int sanitize UNUSED, struct osmdata_t *osmdata) {
- struct Input *i;
- char buffer[65536];
- int bufsz = 0;
- int offset = 0;
- char *nl;
-
- i = inputOpen(filename);
-
- if (i != NULL) {
- while(1)
- {
- bufsz = bufsz + readFile(i, buffer + bufsz, sizeof(buffer) - bufsz - 1);
- buffer[bufsz] = 0;
- nl = strchr(buffer, '\n');
- if (nl == 0) break;
- *nl=0;
- while (nl && nl < buffer + bufsz)
- {
- *nl = 0;
- process(buffer + offset, osmdata);
- offset = nl - buffer + 1;
- nl = strchr(buffer + offset, '\n');
- }
- memcpy(buffer, buffer + offset, bufsz - offset);
- bufsz = bufsz - offset;
- offset = 0;
- }
- } else {
- fprintf(stderr, "Unable to open %s\n", filename);
- return 1;
- }
- inputClose(i);
- return 0;
-}
diff --git a/parse-primitive.h b/parse-primitive.h
deleted file mode 100644
index 6a0a5ae..0000000
--- a/parse-primitive.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
-#-----------------------------------------------------------------------------
-# osm2pgsql - converts planet.osm file into PostgreSQL
-# compatible output suitable to be rendered by mapnik
-#-----------------------------------------------------------------------------
-# Original Python implementation by Artem Pavlenko
-# Re-implementation by Jon Burgess, Copyright 2006
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#-----------------------------------------------------------------------------
-*/
-
-#ifndef PARSE_PRIMITIVE_H
-#define PARSE_PRIMITIVE_H
-
-int streamFilePrimitive(char *filename, int sanitize, struct osmdata_t *osmdata);
-
-#endif
diff --git a/parse-xml2.c b/parse-xml2.c
deleted file mode 100644
index ad31de8..0000000
--- a/parse-xml2.c
+++ /dev/null
@@ -1,410 +0,0 @@
-/*
-#-----------------------------------------------------------------------------
-# osm2pgsql - converts planet.osm file into PostgreSQL
-# compatible output suitable to be rendered by mapnik
-#-----------------------------------------------------------------------------
-# Original Python implementation by Artem Pavlenko
-# Re-implementation by Jon Burgess, Copyright 2006
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#-----------------------------------------------------------------------------
-*/
-
-#define _GNU_SOURCE
-
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-#include <time.h>
-
-#include <libxml/xmlstring.h>
-#include <libxml/xmlreader.h>
-
-#include "osmtypes.h"
-#include "sanitizer.h"
-#include "reprojection.h"
-#include "input.h"
-#include "output.h"
-
-#include "parse-xml2.h"
-
-
-
-/* Parses the action="foo" tags in JOSM change files. Obvisouly not useful from osmChange files */
-static actions_t ParseAction( xmlTextReaderPtr reader, struct osmdata_t *osmdata )
-{
- actions_t new_action;
- xmlChar *action;
- if( osmdata->filetype == FILETYPE_OSMCHANGE || osmdata->filetype == FILETYPE_PLANETDIFF )
- return osmdata->action;
- new_action = ACTION_NONE;
- action = xmlTextReaderGetAttribute( reader, BAD_CAST "action" );
- if( action == NULL )
- new_action = ACTION_CREATE;
- else if( strcmp((char *)action, "modify") == 0 )
- new_action = ACTION_MODIFY;
- else if( strcmp((char *)action, "delete") == 0 )
- new_action = ACTION_DELETE;
- else
- {
- fprintf( stderr, "Unknown value for action: %s\n", (char*)action );
- exit_nicely();
- }
- return new_action;
-}
-
-static void StartElement(xmlTextReaderPtr reader, const xmlChar *name, struct osmdata_t *osmdata)
-{
- xmlChar *xid, *xlat, *xlon, *xk, *xv, *xrole, *xtype;
- char *k;
-
- if (osmdata->filetype == FILETYPE_NONE)
- {
- if (xmlStrEqual(name, BAD_CAST "osm"))
- {
- osmdata->filetype = FILETYPE_OSM;
- osmdata->action = ACTION_CREATE;
- }
- else if (xmlStrEqual(name, BAD_CAST "osmChange"))
- {
- osmdata->filetype = FILETYPE_OSMCHANGE;
- osmdata->action = ACTION_NONE;
- }
- else if (xmlStrEqual(name, BAD_CAST "planetdiff"))
- {
- osmdata->filetype = FILETYPE_PLANETDIFF;
- osmdata->action = ACTION_NONE;
- }
- else
- {
- fprintf( stderr, "Unknown XML document type: %s\n", name );
- exit_nicely();
- }
- return;
- }
-
- if (xmlStrEqual(name, BAD_CAST "node")) {
- xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
- xlon = xmlTextReaderGetAttribute(reader, BAD_CAST "lon");
- xlat = xmlTextReaderGetAttribute(reader, BAD_CAST "lat");
- assert(xid);
-
- osmdata->osm_id = strtoosmid((char *)xid, NULL, 10);
- osmdata->action = ParseAction( reader , osmdata);
-
- if (osmdata->action != ACTION_DELETE) {
- assert(xlon); assert(xlat);
- osmdata->node_lon = strtod((char *)xlon, NULL);
- osmdata->node_lat = strtod((char *)xlat, NULL);
- }
-
- if (osmdata->osm_id > osmdata->max_node)
- osmdata->max_node = osmdata->osm_id;
-
- if (osmdata->count_node == 0) {
- time(&osmdata->start_node);
- }
- osmdata->count_node++;
- if (osmdata->count_node%10000 == 0)
- printStatus(osmdata);
-
- xmlFree(xid);
- xmlFree(xlon);
- xmlFree(xlat);
- } else if (xmlStrEqual(name, BAD_CAST "tag")) {
- xk = xmlTextReaderGetAttribute(reader, BAD_CAST "k");
- assert(xk);
-
- /* 'created_by' and 'source' are common and not interesting to mapnik renderer */
- if (strcmp((char *)xk, "created_by") && strcmp((char *)xk, "source")) {
- char *p;
- xv = xmlTextReaderGetAttribute(reader, BAD_CAST "v");
- assert(xv);
- k = (char *)xmlStrdup(xk);
- while ((p = strchr(k, ' ')))
- *p = '_';
-
- addItem(&(osmdata->tags), k, (char *)xv, 0);
- xmlFree(k);
- xmlFree(xv);
- }
- xmlFree(xk);
- } else if (xmlStrEqual(name, BAD_CAST "way")) {
-
- xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
- assert(xid);
- osmdata->osm_id = strtoosmid((char *)xid, NULL, 10);
- osmdata->action = ParseAction( reader, osmdata );
-
- if (osmdata->osm_id > osmdata->max_way)
- osmdata->max_way = osmdata->osm_id;
-
- if (osmdata->count_way == 0) {
- time(&osmdata->start_way);
- }
- osmdata->count_way++;
- if (osmdata->count_way%1000 == 0)
- printStatus(osmdata);
-
- osmdata->nd_count = 0;
- xmlFree(xid);
-
- } else if (xmlStrEqual(name, BAD_CAST "nd")) {
- xid = xmlTextReaderGetAttribute(reader, BAD_CAST "ref");
- assert(xid);
-
- osmdata->nds[osmdata->nd_count++] = strtoosmid( (char *)xid, NULL, 10 );
-
- if( osmdata->nd_count >= osmdata->nd_max )
- realloc_nodes(osmdata);
- xmlFree(xid);
- } else if (xmlStrEqual(name, BAD_CAST "relation")) {
- xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
- assert(xid);
- osmdata->osm_id = strtoosmid((char *)xid, NULL, 10);
- osmdata->action = ParseAction( reader, osmdata );
-
- if (osmdata->osm_id > osmdata->max_rel)
- osmdata->max_rel = osmdata->osm_id;
-
- if (osmdata->count_rel == 0) {
- time(&osmdata->start_rel);
- }
- osmdata->count_rel++;
- if (osmdata->count_rel%10 == 0)
- printStatus(osmdata);
-
- osmdata->member_count = 0;
- xmlFree(xid);
- } else if (xmlStrEqual(name, BAD_CAST "member")) {
- xrole = xmlTextReaderGetAttribute(reader, BAD_CAST "role");
- assert(xrole);
-
- xtype = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
- assert(xtype);
-
- xid = xmlTextReaderGetAttribute(reader, BAD_CAST "ref");
- assert(xid);
-
- osmdata->members[osmdata->member_count].id = strtoosmid( (char *)xid, NULL, 0 );
- osmdata->members[osmdata->member_count].role = strdup( (char *)xrole );
-
- /* Currently we are only interested in 'way' members since these form polygons with holes */
- if (xmlStrEqual(xtype, BAD_CAST "way"))
- osmdata->members[osmdata->member_count].type = OSMTYPE_WAY;
- if (xmlStrEqual(xtype, BAD_CAST "node"))
- osmdata->members[osmdata->member_count].type = OSMTYPE_NODE;
- if (xmlStrEqual(xtype, BAD_CAST "relation"))
- osmdata->members[osmdata->member_count].type = OSMTYPE_RELATION;
- osmdata->member_count++;
-
- if( osmdata->member_count >= osmdata->member_max )
- realloc_members(osmdata);
- xmlFree(xid);
- xmlFree(xrole);
- xmlFree(xtype);
- } else if (xmlStrEqual(name, BAD_CAST "add") ||
- xmlStrEqual(name, BAD_CAST "create")) {
- osmdata->action = ACTION_MODIFY; /* Turns all creates into modifies, makes it resiliant against inconsistant snapshots. */
- } else if (xmlStrEqual(name, BAD_CAST "modify")) {
- osmdata->action = ACTION_MODIFY;
- } else if (xmlStrEqual(name, BAD_CAST "delete")) {
- osmdata->action = ACTION_DELETE;
- } else if (xmlStrEqual(name, BAD_CAST "bound")) {
- /* ignore */
- } else if (xmlStrEqual(name, BAD_CAST "bounds")) {
- /* ignore */
- } else if (xmlStrEqual(name, BAD_CAST "changeset")) {
- /* ignore */
- } else {
- fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
- }
-
- /* Collect extra attribute information and add as tags */
- if (osmdata->extra_attributes && (xmlStrEqual(name, BAD_CAST "node") ||
- xmlStrEqual(name, BAD_CAST "way") ||
- xmlStrEqual(name, BAD_CAST "relation")))
- {
- xmlChar *xtmp;
-
- xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "user");
- if (xtmp) {
- addItem(&(osmdata->tags), "osm_user", (char *)xtmp, 0);
- xmlFree(xtmp);
- }
-
- xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "uid");
- if (xtmp) {
- addItem(&(osmdata->tags), "osm_uid", (char *)xtmp, 0);
- xmlFree(xtmp);
- }
-
- xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "version");
- if (xtmp) {
- addItem(&(osmdata->tags), "osm_version", (char *)xtmp, 0);
- xmlFree(xtmp);
- }
-
- xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "timestamp");
- if (xtmp) {
- addItem(&(osmdata->tags), "osm_timestamp", (char *)xtmp, 0);
- xmlFree(xtmp);
- }
-
- xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "changeset");
- if (xtmp) {
- addItem(&(osmdata->tags), "osm_changeset", (char *)xtmp, 0);
- xmlFree(xtmp);
- }
- }
-}
-
-
-static void EndElement(const xmlChar *name, struct osmdata_t *osmdata)
-{
- if (xmlStrEqual(name, BAD_CAST "node")) {
- if (node_wanted(osmdata, osmdata->node_lat, osmdata->node_lon)) {
- reproject(&(osmdata->node_lat), &(osmdata->node_lon));
- if( osmdata->action == ACTION_CREATE )
- osmdata->out->node_add(osmdata->osm_id, osmdata->node_lat, osmdata->node_lon, &(osmdata->tags));
- else if( osmdata->action == ACTION_MODIFY )
- osmdata->out->node_modify(osmdata->osm_id, osmdata->node_lat, osmdata->node_lon, &(osmdata->tags));
- else if( osmdata->action == ACTION_DELETE )
- osmdata->out->node_delete(osmdata->osm_id);
- else
- {
- fprintf( stderr, "Don't know action for node %" PRIdOSMID "\n", osmdata->osm_id );
- exit_nicely();
- }
- }
- resetList(&(osmdata->tags));
- } else if (xmlStrEqual(name, BAD_CAST "way")) {
- if( osmdata->action == ACTION_CREATE )
- osmdata->out->way_add(osmdata->osm_id, osmdata->nds, osmdata->nd_count, &(osmdata->tags) );
- else if( osmdata->action == ACTION_MODIFY )
- osmdata->out->way_modify(osmdata->osm_id, osmdata->nds, osmdata->nd_count, &(osmdata->tags) );
- else if( osmdata->action == ACTION_DELETE )
- osmdata->out->way_delete(osmdata->osm_id);
- else
- {
- fprintf( stderr, "Don't know action for way %" PRIdOSMID "\n", osmdata->osm_id );
- exit_nicely();
- }
- resetList(&(osmdata->tags));
- } else if (xmlStrEqual(name, BAD_CAST "relation")) {
- if( osmdata->action == ACTION_CREATE )
- osmdata->out->relation_add(osmdata->osm_id, osmdata->members, osmdata->member_count, &(osmdata->tags));
- else if( osmdata->action == ACTION_MODIFY )
- osmdata->out->relation_modify(osmdata->osm_id, osmdata->members, osmdata->member_count, &(osmdata->tags));
- else if( osmdata->action == ACTION_DELETE )
- osmdata->out->relation_delete(osmdata->osm_id);
- else
- {
- fprintf( stderr, "Don't know action for relation %" PRIdOSMID "\n", osmdata->osm_id );
- exit_nicely();
- }
- resetList(&(osmdata->tags));
- resetMembers(osmdata);
- } else if (xmlStrEqual(name, BAD_CAST "tag")) {
- /* ignore */
- } else if (xmlStrEqual(name, BAD_CAST "nd")) {
- /* ignore */
- } else if (xmlStrEqual(name, BAD_CAST "member")) {
- /* ignore */
- } else if (xmlStrEqual(name, BAD_CAST "osm")) {
- printStatus(osmdata);
- osmdata->filetype = FILETYPE_NONE;
- } else if (xmlStrEqual(name, BAD_CAST "osmChange")) {
- printStatus(osmdata);
- osmdata->filetype = FILETYPE_NONE;
- } else if (xmlStrEqual(name, BAD_CAST "planetdiff")) {
- printStatus(osmdata);
- osmdata->filetype = FILETYPE_NONE;
- } else if (xmlStrEqual(name, BAD_CAST "bound")) {
- /* ignore */
- } else if (xmlStrEqual(name, BAD_CAST "bounds")) {
- /* ignore */
- } else if (xmlStrEqual(name, BAD_CAST "changeset")) {
- /* ignore */
- resetList(&(osmdata->tags)); /* We may have accumulated some tags even if we ignored the changeset */
- } else if (xmlStrEqual(name, BAD_CAST "add")) {
- osmdata->action = ACTION_NONE;
- } else if (xmlStrEqual(name, BAD_CAST "create")) {
- osmdata->action = ACTION_NONE;
- } else if (xmlStrEqual(name, BAD_CAST "modify")) {
- osmdata->action = ACTION_NONE;
- } else if (xmlStrEqual(name, BAD_CAST "delete")) {
- osmdata->action = ACTION_NONE;
- } else {
- fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
- }
-}
-
-static void processNode(xmlTextReaderPtr reader, struct osmdata_t *osmdata) {
- xmlChar *name;
- name = xmlTextReaderName(reader);
- if (name == NULL)
- name = xmlStrdup(BAD_CAST "--");
-
- switch(xmlTextReaderNodeType(reader)) {
- case XML_READER_TYPE_ELEMENT:
- StartElement(reader, name, osmdata);
- if (xmlTextReaderIsEmptyElement(reader))
- EndElement(name, osmdata); /* No end_element for self closing tags! */
- break;
- case XML_READER_TYPE_END_ELEMENT:
- EndElement(name, osmdata);
- break;
- case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
- /* Ignore */
- break;
- default:
- fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
- break;
- }
-
- xmlFree(name);
-}
-
-int streamFileXML2(char *filename, int sanitize, struct osmdata_t *osmdata) {
- xmlTextReaderPtr reader;
- int ret = 0;
-
- if (sanitize)
- reader = sanitizerOpen(filename);
- else
- reader = inputUTF8(filename);
-
- if (reader != NULL) {
- ret = xmlTextReaderRead(reader);
- while (ret == 1) {
- processNode(reader, osmdata);
- ret = xmlTextReaderRead(reader);
- }
-
- if (ret != 0) {
- fprintf(stderr, "%s : failed to parse\n", filename);
- return ret;
- }
-
- xmlFreeTextReader(reader);
- } else {
- fprintf(stderr, "Unable to open %s\n", filename);
- return 1;
- }
- return 0;
-}
-
diff --git a/parse-xml2.cpp b/parse-xml2.cpp
new file mode 100644
index 0000000..fc118ae
--- /dev/null
+++ b/parse-xml2.cpp
@@ -0,0 +1,426 @@
+/*
+#-----------------------------------------------------------------------------
+# osm2pgsql - converts planet.osm file into PostgreSQL
+# compatible output suitable to be rendered by mapnik
+#-----------------------------------------------------------------------------
+# Original Python implementation by Artem Pavlenko
+# Re-implementation by Jon Burgess, Copyright 2006
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#-----------------------------------------------------------------------------
+*/
+
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <time.h>
+
+#include "sanitizer.hpp"
+#include "input.hpp"
+
+#include "parse-xml2.hpp"
+#include "output.hpp"
+#include "util.hpp"
+
+/* Parses the action="foo" tags in JOSM change files. Obvisouly not useful from osmChange files */
+actions_t parse_xml2_t::ParseAction( xmlTextReaderPtr reader)
+{
+ actions_t new_action;
+ xmlChar *action_text;
+ if( filetype == FILETYPE_OSMCHANGE || filetype == FILETYPE_PLANETDIFF )
+ return action;
+ new_action = ACTION_NONE;
+ action_text = xmlTextReaderGetAttribute( reader, BAD_CAST "action" );
+ if( action_text == NULL )
+ new_action = ACTION_CREATE;
+ else if( strcmp((char *)action_text, "modify") == 0 )
+ new_action = ACTION_MODIFY;
+ else if( strcmp((char *)action_text, "delete") == 0 )
+ new_action = ACTION_DELETE;
+ else
+ {
+ fprintf( stderr, "Unknown value for action: %s\n", (char*)action_text );
+ util::exit_nicely();
+ }
+ return new_action;
+}
+
+void parse_xml2_t::SetFiletype(const xmlChar* name, osmdata_t* osmdata)
+{
+ if (xmlStrEqual(name, BAD_CAST "osm"))
+ {
+ filetype = FILETYPE_OSM;
+ action = ACTION_CREATE;
+ }
+ else if (xmlStrEqual(name, BAD_CAST "osmChange"))
+ {
+ filetype = FILETYPE_OSMCHANGE;
+ action = ACTION_NONE;
+ }
+ else if (xmlStrEqual(name, BAD_CAST "planetdiff"))
+ {
+ filetype = FILETYPE_PLANETDIFF;
+ action = ACTION_NONE;
+ }
+ else
+ {
+ fprintf( stderr, "Unknown XML document type: %s\n", name );
+ util::exit_nicely();
+ }
+}
+
+void parse_xml2_t::StartElement(xmlTextReaderPtr reader, const xmlChar *name, struct osmdata_t *osmdata)
+{
+ xmlChar *xid, *xlat, *xlon, *xk, *xv, *xrole, *xtype;
+ char *k;
+
+ //first time in we figure out what kind of data this is
+ if (filetype == FILETYPE_NONE)
+ {
+ SetFiletype(name, osmdata);
+ return;
+ }
+
+ //remember which this was for collecting tags at the end
+ bool can_have_attribs = false;
+
+ if (xmlStrEqual(name, BAD_CAST "node")) {
+ can_have_attribs = true;
+
+ xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
+ xlon = xmlTextReaderGetAttribute(reader, BAD_CAST "lon");
+ xlat = xmlTextReaderGetAttribute(reader, BAD_CAST "lat");
+ assert(xid);
+
+ osm_id = strtoosmid((char *)xid, NULL, 10);
+ action = ParseAction(reader);
+
+ if (action != ACTION_DELETE) {
+ assert(xlon); assert(xlat);
+ node_lon = strtod((char *)xlon, NULL);
+ node_lat = strtod((char *)xlat, NULL);
+ }
+
+ if (osm_id > max_node)
+ max_node = osm_id;
+
+ if (count_node == 0) {
+ time(&start_node);
+ }
+ count_node++;
+ if (count_node%10000 == 0)
+ printStatus();
+
+ xmlFree(xid);
+ xmlFree(xlon);
+ xmlFree(xlat);
+ } else if (xmlStrEqual(name, BAD_CAST "way")) {
+ can_have_attribs = true;
+
+ xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
+ assert(xid);
+ osm_id = strtoosmid((char *)xid, NULL, 10);
+ action = ParseAction( reader );
+
+ if (osm_id > max_way)
+ max_way = osm_id;
+
+ if (count_way == 0) {
+ time(&start_way);
+ }
+ count_way++;
+ if (count_way%1000 == 0)
+ printStatus();
+
+ nd_count = 0;
+ xmlFree(xid);
+
+ } else if (xmlStrEqual(name, BAD_CAST "relation")) {
+ can_have_attribs = true;
+
+ xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
+ assert(xid);
+ osm_id = strtoosmid((char *)xid, NULL, 10);
+ action = ParseAction( reader );
+
+ if (osm_id > max_rel)
+ max_rel = osm_id;
+
+ if (count_rel == 0) {
+ time(&start_rel);
+ }
+ count_rel++;
+ if (count_rel%10 == 0)
+ printStatus();
+
+ member_count = 0;
+ xmlFree(xid);
+ } else if (xmlStrEqual(name, BAD_CAST "tag")) {
+ xk = xmlTextReaderGetAttribute(reader, BAD_CAST "k");
+ assert(xk);
+
+ /* 'created_by' and 'source' are common and not interesting to mapnik renderer */
+ if (strcmp((char *)xk, "created_by") && strcmp((char *)xk, "source")) {
+ char *p;
+ xv = xmlTextReaderGetAttribute(reader, BAD_CAST "v");
+ assert(xv);
+ k = (char *)xmlStrdup(xk);
+ while ((p = strchr(k, ' ')))
+ *p = '_';
+
+ keyval::addItem(&(tags), k, (char *)xv, 0);
+ xmlFree(k);
+ xmlFree(xv);
+ }
+ xmlFree(xk);
+ } else if (xmlStrEqual(name, BAD_CAST "nd")) {
+ xid = xmlTextReaderGetAttribute(reader, BAD_CAST "ref");
+ assert(xid);
+
+ nds[nd_count++] = strtoosmid( (char *)xid, NULL, 10 );
+
+ if( nd_count >= nd_max )
+ realloc_nodes();
+ xmlFree(xid);
+ } else if (xmlStrEqual(name, BAD_CAST "member")) {
+ xrole = xmlTextReaderGetAttribute(reader, BAD_CAST "role");
+ assert(xrole);
+
+ xtype = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
+ assert(xtype);
+
+ xid = xmlTextReaderGetAttribute(reader, BAD_CAST "ref");
+ assert(xid);
+
+ members[member_count].id = strtoosmid((char *) xid, NULL, 0);
+ members[member_count].role = strdup((char *) xrole);
+
+ /* Currently we are only interested in 'way' members since these form polygons with holes */
+ if (xmlStrEqual(xtype, BAD_CAST "way"))
+ members[member_count].type = OSMTYPE_WAY;
+ if (xmlStrEqual(xtype, BAD_CAST "node"))
+ members[member_count].type = OSMTYPE_NODE;
+ if (xmlStrEqual(xtype, BAD_CAST "relation"))
+ members[member_count].type = OSMTYPE_RELATION;
+ member_count++;
+
+ if (member_count >= member_max)
+ realloc_members();
+ xmlFree(xid);
+ xmlFree(xrole);
+ xmlFree(xtype);
+ } else if (xmlStrEqual(name, BAD_CAST "add") ||
+ xmlStrEqual(name, BAD_CAST "create")) {
+ action = ACTION_MODIFY; /* Turns all creates into modifies, makes it resiliant against inconsistant snapshots. */
+ } else if (xmlStrEqual(name, BAD_CAST "modify")) {
+ action = ACTION_MODIFY;
+ } else if (xmlStrEqual(name, BAD_CAST "delete")) {
+ action = ACTION_DELETE;
+ } else if (xmlStrEqual(name, BAD_CAST "bound")) {
+ /* ignore */
+ } else if (xmlStrEqual(name, BAD_CAST "bounds")) {
+ /* ignore */
+ } else if (xmlStrEqual(name, BAD_CAST "changeset")) {
+ /* ignore */
+ } else {
+ fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
+ }
+
+ /* Collect extra attribute information and add as tags */
+ if (extra_attributes && can_have_attribs)
+ {
+ xmlChar *xtmp;
+
+ xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "user");
+ if (xtmp) {
+ keyval::addItem(&(tags), "osm_user", (char *)xtmp, 0);
+ xmlFree(xtmp);
+ }
+
+ xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "uid");
+ if (xtmp) {
+ keyval::addItem(&(tags), "osm_uid", (char *)xtmp, 0);
+ xmlFree(xtmp);
+ }
+
+ xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "version");
+ if (xtmp) {
+ keyval::addItem(&(tags), "osm_version", (char *)xtmp, 0);
+ xmlFree(xtmp);
+ }
+
+ xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "timestamp");
+ if (xtmp) {
+ keyval::addItem(&(tags), "osm_timestamp", (char *)xtmp, 0);
+ xmlFree(xtmp);
+ }
+
+ xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "changeset");
+ if (xtmp) {
+ keyval::addItem(&(tags), "osm_changeset", (char *)xtmp, 0);
+ xmlFree(xtmp);
+ }
+ }
+}
+
+
+void parse_xml2_t::EndElement(const xmlChar *name, struct osmdata_t *osmdata)
+{
+ if (xmlStrEqual(name, BAD_CAST "node")) {
+ if (node_wanted(node_lat, node_lon)) {
+ proj->reproject(&(node_lat), &(node_lon));
+ if( action == ACTION_CREATE )
+ osmdata->node_add(osm_id, node_lat, node_lon, &(tags));
+ else if( action == ACTION_MODIFY )
+ osmdata->node_modify(osm_id, node_lat, node_lon, &(tags));
+ else if( action == ACTION_DELETE )
+ osmdata->node_delete(osm_id);
+ else
+ {
+ fprintf( stderr, "Don't know action for node %" PRIdOSMID "\n", osm_id );
+ util::exit_nicely();
+ }
+ }
+ keyval::resetList(&(tags));
+ } else if (xmlStrEqual(name, BAD_CAST "way")) {
+ if( action == ACTION_CREATE )
+ osmdata->way_add(osm_id, nds, nd_count, &(tags) );
+ else if( action == ACTION_MODIFY )
+ osmdata->way_modify(osm_id, nds, nd_count, &(tags) );
+ else if( action == ACTION_DELETE )
+ osmdata->way_delete(osm_id);
+ else
+ {
+ fprintf( stderr, "Don't know action for way %" PRIdOSMID "\n", osm_id );
+ util::exit_nicely();
+ }
+ keyval::resetList(&(tags));
+ } else if (xmlStrEqual(name, BAD_CAST "relation")) {
+ if( action == ACTION_CREATE )
+ osmdata->relation_add(osm_id, members, member_count, &(tags));
+ else if( action == ACTION_MODIFY )
+ osmdata->relation_modify(osm_id, members, member_count, &(tags));
+ else if( action == ACTION_DELETE )
+ osmdata->relation_delete(osm_id);
+ else
+ {
+ fprintf( stderr, "Don't know action for relation %" PRIdOSMID "\n", osm_id );
+ util::exit_nicely();
+ }
+ keyval::resetList(&(tags));
+ resetMembers();
+ } else if (xmlStrEqual(name, BAD_CAST "tag")) {
+ /* ignore */
+ } else if (xmlStrEqual(name, BAD_CAST "nd")) {
+ /* ignore */
+ } else if (xmlStrEqual(name, BAD_CAST "member")) {
+ /* ignore */
+ } else if (xmlStrEqual(name, BAD_CAST "osm")) {
+ printStatus();
+ filetype = FILETYPE_NONE;
+ } else if (xmlStrEqual(name, BAD_CAST "osmChange")) {
+ printStatus();
+ filetype = FILETYPE_NONE;
+ } else if (xmlStrEqual(name, BAD_CAST "planetdiff")) {
+ printStatus();
+ filetype = FILETYPE_NONE;
+ } else if (xmlStrEqual(name, BAD_CAST "bound")) {
+ /* ignore */
+ } else if (xmlStrEqual(name, BAD_CAST "bounds")) {
+ /* ignore */
+ } else if (xmlStrEqual(name, BAD_CAST "changeset")) {
+ /* ignore */
+ keyval::resetList(&(tags)); /* We may have accumulated some tags even if we ignored the changeset */
+ } else if (xmlStrEqual(name, BAD_CAST "add")) {
+ action = ACTION_NONE;
+ } else if (xmlStrEqual(name, BAD_CAST "create")) {
+ action = ACTION_NONE;
+ } else if (xmlStrEqual(name, BAD_CAST "modify")) {
+ action = ACTION_NONE;
+ } else if (xmlStrEqual(name, BAD_CAST "delete")) {
+ action = ACTION_NONE;
+ } else {
+ fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
+ }
+}
+
+void parse_xml2_t::processNode(xmlTextReaderPtr reader, struct osmdata_t *osmdata) {
+ xmlChar *name;
+ name = xmlTextReaderName(reader);
+ if (name == NULL)
+ name = xmlStrdup(BAD_CAST "--");
+
+ switch(xmlTextReaderNodeType(reader)) {
+ case XML_READER_TYPE_ELEMENT:
+ StartElement(reader, name, osmdata);
+ if (xmlTextReaderIsEmptyElement(reader))
+ EndElement(name, osmdata); /* No end_element for self closing tags! */
+ break;
+ case XML_READER_TYPE_END_ELEMENT:
+ EndElement(name, osmdata);
+ break;
+ case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
+ /* Ignore */
+ break;
+ default:
+ fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
+ break;
+ }
+
+ xmlFree(name);
+}
+
+parse_xml2_t::parse_xml2_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr<reprojection>& projection_,
+ const double minlon, const double minlat, const double maxlon, const double maxlat):
+ parse_t(extra_attributes_, bbox_, projection_, minlon, minlat, maxlon, maxlat)
+{
+ LIBXML_TEST_VERSION;
+}
+
+parse_xml2_t::~parse_xml2_t()
+{
+ xmlCleanupParser();
+ xmlMemoryDump();
+}
+
+int parse_xml2_t::streamFile(const char *filename, const int sanitize, osmdata_t *osmdata) {
+ xmlTextReaderPtr reader;
+ int ret = 0;
+
+ if (sanitize)
+ reader = sanitizerOpen(filename);
+ else
+ reader = inputUTF8(filename);
+
+ if (reader != NULL) {
+ ret = xmlTextReaderRead(reader);
+ while (ret == 1) {
+ processNode(reader, osmdata);
+ ret = xmlTextReaderRead(reader);
+ }
+
+ if (ret != 0) {
+ fprintf(stderr, "%s : failed to parse\n", filename);
+ return ret;
+ }
+
+ xmlFreeTextReader(reader);
+ } else {
+ fprintf(stderr, "Unable to open %s\n", filename);
+ return 1;
+ }
+ return 0;
+}
diff --git a/parse-xml2.h b/parse-xml2.h
deleted file mode 100644
index 3ddd423..0000000
--- a/parse-xml2.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
-#-----------------------------------------------------------------------------
-# osm2pgsql - converts planet.osm file into PostgreSQL
-# compatible output suitable to be rendered by mapnik
-#-----------------------------------------------------------------------------
-# Original Python implementation by Artem Pavlenko
-# Re-implementation by Jon Burgess, Copyright 2006
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#-----------------------------------------------------------------------------
-*/
-
-#ifndef PARSE_XML2_H
-#define PARSE_XML2_H
-
-int streamFileXML2(char *filename, int sanitize, struct osmdata_t *osmdata);
-
-#endif
diff --git a/parse-xml2.hpp b/parse-xml2.hpp
new file mode 100644
index 0000000..10ae1b5
--- /dev/null
+++ b/parse-xml2.hpp
@@ -0,0 +1,63 @@
+/*
+#-----------------------------------------------------------------------------
+# osm2pgsql - converts planet.osm file into PostgreSQL
+# compatible output suitable to be rendered by mapnik
+#-----------------------------------------------------------------------------
+# Original Python implementation by Artem Pavlenko
+# Re-implementation by Jon Burgess, Copyright 2006
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#-----------------------------------------------------------------------------
+*/
+
+#ifndef PARSE_XML2_H
+#define PARSE_XML2_H
+
+/* Open and incrementally read an XML file.
+ *
+ * Parameters:
+ * - filename: the path to the XML file to stream.
+ * - sanitize: if non-zero, use a parser which attempts to sanitize bad UTF-8
+ * characters.
+ * - osmdata: as the file is parsed, it will call functions on `osmdata->out`
+ * based on the action and type of object, e.g: `node_add`, `relation_modify`,
+ * and so on.
+ *
+ * Return value: 0 on success. A non-zero value indicates failure.
+ */
+
+#include "parse.hpp"
+
+#include <libxml/xmlstring.h>
+#include <libxml/xmlreader.h>
+
+class parse_xml2_t: public parse_t
+{
+public:
+ parse_xml2_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr<reprojection>& projection_,
+ const double minlon, const double minlat, const double maxlon, const double maxlat);
+ virtual ~parse_xml2_t();
+ virtual int streamFile(const char *filename, const int sanitize, osmdata_t *osmdata);
+protected:
+ parse_xml2_t();
+ actions_t ParseAction( xmlTextReaderPtr reader);
+ void SetFiletype(const xmlChar* name, osmdata_t* osmdata);
+ void StartElement(xmlTextReaderPtr reader, const xmlChar *name, struct osmdata_t *osmdata);
+ void EndElement(const xmlChar *name, struct osmdata_t *osmdata);
+ void processNode(xmlTextReaderPtr reader, struct osmdata_t *osmdata);
+};
+
+
+#endif
diff --git a/parse.cpp b/parse.cpp
new file mode 100644
index 0000000..8ff775f
--- /dev/null
+++ b/parse.cpp
@@ -0,0 +1,229 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdexcept>
+#include "parse-o5m.hpp"
+#ifdef BUILD_READER_PBF
+# include "parse-pbf.hpp"
+#endif
+#include "parse-xml2.hpp"
+#include "util.hpp"
+
+#define INIT_MAX_MEMBERS 64
+#define INIT_MAX_NODES 4096
+
+#ifdef _MSC_VER
+#define strcasecmp _stricmp
+#endif
+
+
+parse_delegate_t::parse_delegate_t(const int extra_attributes,
+ const boost::optional<std::string> &bbox,
+ boost::shared_ptr<reprojection> projection):
+m_extra_attributes(extra_attributes), m_proj(projection), m_count_node(0), m_max_node(0),
+m_count_way(0), m_max_way(0), m_count_rel(0), m_max_rel(0), m_start_node(0), m_start_way(0), m_start_rel(0)
+{
+ m_bbox = bool(bbox);
+ if (m_bbox) {
+ parse_bbox(*bbox);
+ }
+}
+
+parse_delegate_t::~parse_delegate_t()
+{
+}
+int parse_delegate_t::streamFile(const char* input_reader, const char* filename,const int sanitize, osmdata_t *osmdata)
+{
+ //process the input file with the right parser
+ parse_t* parser = get_input_reader(input_reader, filename);
+ int ret = parser->streamFile(filename, sanitize, osmdata);
+
+ //update statisics
+ m_count_node += parser->count_node;
+ m_count_way += parser->count_way;
+ m_count_rel += parser->count_rel;
+ m_max_node = std::max(parser->max_node, m_max_node);
+ m_max_way = std::max(parser->max_way, m_max_way);
+ m_max_rel = std::max(parser->max_rel, m_max_rel);
+ m_start_node = m_start_node == 0 ? parser->start_node : m_start_node;
+ m_start_way = m_start_way == 0 ? parser->start_way : m_start_way;
+ m_start_rel = m_start_rel == 0 ? parser->start_rel : m_start_rel;
+
+ //done
+ delete parser;
+
+ return ret;
+}
+void parse_delegate_t::printSummary() const
+{
+ time_t now = time(NULL);
+ time_t end_nodes = m_start_way > 0 ? m_start_way : now;
+ time_t end_way = m_start_rel > 0 ? m_start_rel : now;
+ time_t end_rel = now;
+
+ fprintf(stderr,
+ "Node stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n",
+ m_count_node, m_max_node,
+ m_count_node > 0 ? (int) (end_nodes - m_start_node) : 0);
+ fprintf(stderr,
+ "Way stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n",
+ m_count_way, m_max_way,
+ m_count_way > 0 ? (int) (end_way - m_start_way) : 0);
+ fprintf(stderr,
+ "Relation stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n",
+ m_count_rel, m_max_rel,
+ m_count_rel > 0 ? (int) (end_rel - m_start_rel) : 0);
+}
+
+boost::shared_ptr<reprojection> parse_delegate_t::getProjection() const
+{
+ return m_proj;
+}
+
+void parse_delegate_t::parse_bbox(const std::string &bbox_)
+{
+ int n = sscanf(bbox_.c_str(), "%lf,%lf,%lf,%lf", &(m_minlon), &(m_minlat), &(m_maxlon), &(m_maxlat));
+ if (n != 4)
+ throw std::runtime_error("Bounding box must be specified like: minlon,minlat,maxlon,maxlat\n");
+
+ if (m_maxlon <= m_minlon)
+ throw std::runtime_error("Bounding box failed due to maxlon <= minlon\n");
+
+ if (m_maxlat <= m_minlat)
+ throw std::runtime_error("Bounding box failed due to maxlat <= minlat\n");
+
+ fprintf(stderr, "Applying Bounding box: %f,%f to %f,%f\n", m_minlon, m_minlat, m_maxlon, m_maxlat);
+}
+
+parse_t* parse_delegate_t::get_input_reader(const char* input_reader, const char* filename)
+{
+ // if input_reader is forced to a specific iput format
+ if (strcmp("auto", input_reader) != 0) {
+ if (strcmp("libxml2", input_reader) == 0) {
+ return new parse_xml2_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat);
+ } else if (strcmp("primitive", input_reader) == 0) {
+ // The more robust libxml2 parser can be used instead of primitive
+ return new parse_xml2_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat);
+#ifdef BUILD_READER_PBF
+ } else if (strcmp("pbf", input_reader) == 0) {
+ return new parse_pbf_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat);
+#endif
+ } else if (strcmp("o5m", input_reader) == 0) {
+ return new parse_o5m_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat);
+ } else {
+ fprintf(stderr, "Input parser `%s' not recognised. Should be one of [libxml2, o5m"
+#ifdef BUILD_READER_PBF
+ ", pbf"
+#endif
+ "].\n", input_reader);
+ exit(EXIT_FAILURE);
+ }
+ } // if input_reader is not forced by -r switch try to auto-detect it by file extension
+ else {
+ if (strcasecmp(".pbf", filename + strlen(filename) - 4) == 0) {
+#ifdef BUILD_READER_PBF
+ return new parse_pbf_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat);
+#else
+ fprintf(stderr, "ERROR: PBF support has not been compiled into this version of osm2pgsql, please either compile it with pbf support or use one of the other input formats\n");
+ exit(EXIT_FAILURE);
+#endif
+ } else if (strcasecmp(".o5m", filename + strlen(filename) - 4) == 0
+ || strcasecmp(".o5c", filename + strlen(filename) - 4) == 0) {
+ return new parse_o5m_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat);
+ } else {
+ return new parse_xml2_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat);
+ }
+ }
+}
+
+
+
+parse_t::parse_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr<reprojection>& projection_,
+ const double minlon_, const double minlat_, const double maxlon_, const double maxlat_):
+ extra_attributes(extra_attributes_), bbox(bbox_), minlon(minlon_), minlat(minlat_),
+ maxlon(maxlon_), maxlat(maxlat_), proj(projection_)
+{
+ osm_id = nd_max = count_node = max_node = count_way = max_way = count_rel = 0;
+ max_rel = parallel_indexing = start_node = start_way = start_rel = 0;
+ nd_count = nd_max = node_lon = node_lat = member_count = member_max = 0;
+ members = NULL;
+ nds = NULL;
+
+ filetype = FILETYPE_NONE;
+ action = ACTION_NONE;
+
+ realloc_nodes();
+ realloc_members();
+}
+
+parse_t::~parse_t()
+{
+ if(nds != NULL)
+ free(nds);
+ if(members != NULL)
+ free(members);
+}
+
+void parse_t::realloc_nodes()
+{
+ if( nd_max == 0 )
+ nd_max = INIT_MAX_NODES;
+ else
+ nd_max <<= 1;
+
+ nds = (osmid_t *)realloc( nds, nd_max * sizeof( nds[0] ) );
+ if( !nds )
+ {
+ fprintf( stderr, "Failed to expand node list to %d\n", nd_max );
+ util::exit_nicely();
+ }
+}
+
+void parse_t::realloc_members()
+{
+ if( member_max == 0 )
+ member_max = INIT_MAX_NODES;
+ else
+ member_max <<= 1;
+
+ members = (struct member *)realloc( members, member_max * sizeof( members[0] ) );
+ if( !members )
+ {
+ fprintf( stderr, "Failed to expand member list to %d\n", member_max );
+ util::exit_nicely();
+ }
+}
+
+void parse_t::resetMembers()
+{
+ unsigned i;
+ for(i = 0; i < member_count; i++ )
+ free( members[i].role );
+}
+
+void parse_t::printStatus()
+{
+ time_t now = time(NULL);
+ time_t end_nodes = start_way > 0 ? start_way : now;
+ time_t end_way = start_rel > 0 ? start_rel : now;
+ time_t end_rel = now;
+ fprintf(stderr,
+ "\rProcessing: Node(%" PRIdOSMID "k %.1fk/s) Way(%" PRIdOSMID "k %.2fk/s) Relation(%" PRIdOSMID " %.2f/s)",
+ count_node / 1000,
+ (double) count_node / 1000.0 / ((int) (end_nodes - start_node) > 0 ? (double) (end_nodes - start_node) : 1.0),
+ count_way / 1000,
+ count_way > 0 ? (double) count_way / 1000.0 / ((double) (end_way - start_way) > 0.0 ? (double) (end_way - start_way) : 1.0) : 0.0, count_rel,
+ count_rel > 0 ? (double) count_rel / ((double) (end_rel - start_rel) > 0.0 ? (double) (end_rel - start_rel) : 1.0) : 0.0);
+}
+
+int parse_t::node_wanted(double lat, double lon)
+{
+ if (!bbox)
+ return 1;
+
+ if (lat < minlat || lat > maxlat)
+ return 0;
+ if (lon < minlon || lon > maxlon)
+ return 0;
+ return 1;
+}
diff --git a/parse.hpp b/parse.hpp
new file mode 100644
index 0000000..35c754b
--- /dev/null
+++ b/parse.hpp
@@ -0,0 +1,91 @@
+#ifndef PARSE_H
+#define PARSE_H
+
+#include <time.h>
+#include <config.h>
+
+#include "keyvals.hpp"
+#include "reprojection.hpp"
+#include "osmdata.hpp"
+
+#include <boost/shared_ptr.hpp>
+#include <boost/optional.hpp>
+
+typedef enum { FILETYPE_NONE, FILETYPE_OSM, FILETYPE_OSMCHANGE, FILETYPE_PLANETDIFF } filetypes_t;
+typedef enum { ACTION_NONE, ACTION_CREATE, ACTION_MODIFY, ACTION_DELETE } actions_t;
+
+class parse_t;
+
+class parse_delegate_t
+{
+public:
+ parse_delegate_t(const int extra_attributes, const boost::optional<std::string> &bbox, boost::shared_ptr<reprojection> projection);
+ ~parse_delegate_t();
+
+ int streamFile(const char* input_reader, const char* filename, const int sanitize, osmdata_t *osmdata);
+ void printSummary() const;
+ boost::shared_ptr<reprojection> getProjection() const;
+
+private:
+ parse_delegate_t();
+ void parse_bbox(const std::string &bbox);
+ parse_t* get_input_reader(const char* input_reader, const char* filename);
+
+ osmid_t m_count_node, m_max_node;
+ osmid_t m_count_way, m_max_way;
+ osmid_t m_count_rel, m_max_rel;
+ time_t m_start_node, m_start_way, m_start_rel;
+
+ const int m_extra_attributes;
+ boost::shared_ptr<reprojection> m_proj;
+ bool m_bbox;
+ double m_minlon, m_minlat, m_maxlon, m_maxlat;
+};
+
+class parse_t
+{
+ friend class parse_delegate_t;
+
+public:
+ parse_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr<reprojection>& projection_,
+ const double minlon, const double minlat, const double maxlon, const double maxlat);
+ virtual ~parse_t();
+
+ virtual int streamFile(const char *filename, const int sanitize, osmdata_t *osmdata) = 0;
+
+protected:
+ parse_t();
+
+ virtual void realloc_nodes();
+ virtual void realloc_members();
+ virtual void resetMembers();
+ virtual void printStatus();
+ virtual int node_wanted(double lat, double lon);
+
+ osmid_t count_node, max_node;
+ osmid_t count_way, max_way;
+ osmid_t count_rel, max_rel;
+ time_t start_node, start_way, start_rel;
+
+ /* Since {node,way} elements are not nested we can guarantee the
+ values in an end tag must match those of the corresponding
+ start tag and can therefore be cached.
+ */
+ double node_lon, node_lat;
+ keyval tags;
+ osmid_t *nds;
+ int nd_count, nd_max;
+ member *members;
+ int member_count, member_max;
+ osmid_t osm_id;
+ filetypes_t filetype;
+ actions_t action;
+ int parallel_indexing;
+
+ const int extra_attributes;
+ mutable bool bbox;
+ const boost::shared_ptr<reprojection> proj;
+ mutable double minlon, minlat, maxlon, maxlat;
+};
+
+#endif
diff --git a/pgsql-id-tracker.cpp b/pgsql-id-tracker.cpp
new file mode 100644
index 0000000..890eff6
--- /dev/null
+++ b/pgsql-id-tracker.cpp
@@ -0,0 +1,151 @@
+#include "pgsql-id-tracker.hpp"
+
+#include <libpq-fe.h>
+#include <string>
+#include <boost/format.hpp>
+
+#include "osmtypes.hpp"
+#include "pgsql.hpp"
+#include "util.hpp"
+
+struct pgsql_id_tracker::pimpl {
+ pimpl(const std::string &conninfo,
+ const std::string &prefix,
+ const std::string &type,
+ bool owns_table);
+ ~pimpl();
+
+ PGconn *conn;
+ std::string table_name;
+ bool owns_table;
+ osmid_t old_id;
+};
+
+pgsql_id_tracker::pimpl::pimpl(const std::string &conninfo,
+ const std::string &prefix,
+ const std::string &type,
+ bool owns_table_)
+ : conn(PQconnectdb(conninfo.c_str())),
+ table_name((boost::format("%1%_%2%") % prefix % type).str()),
+ owns_table(owns_table_),
+ old_id(0) {
+ if (PQstatus(conn) != CONNECTION_OK) {
+ fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
+ util::exit_nicely();
+ }
+ if (owns_table) {
+ pgsql_exec(conn, PGRES_COMMAND_OK,
+ "DROP TABLE IF EXISTS \"%s\"",
+ table_name.c_str());
+ pgsql_exec(conn, PGRES_COMMAND_OK,
+ "CREATE TABLE \"%s\" (id " POSTGRES_OSMID_TYPE ")",
+ table_name.c_str());
+ }
+ pgsql_exec(conn, PGRES_COMMAND_OK,
+ "PREPARE set_mark(" POSTGRES_OSMID_TYPE ") AS INSERT INTO \"%s\" (id) "
+ "SELECT $1 WHERE NOT EXISTS (SELECT id FROM \"%s\" WHERE id = $1)",
+ table_name.c_str(), table_name.c_str());
+ pgsql_exec(conn, PGRES_COMMAND_OK,
+ "PREPARE get_mark(" POSTGRES_OSMID_TYPE ") AS SELECT id FROM \"%s\" "
+ "WHERE id = $1",
+ table_name.c_str());
+ pgsql_exec(conn, PGRES_COMMAND_OK,
+ "PREPARE get_min AS SELECT min(id) AS id FROM \"%s\"",
+ table_name.c_str());
+ pgsql_exec(conn, PGRES_COMMAND_OK,
+ "PREPARE drop_mark(" POSTGRES_OSMID_TYPE ") AS DELETE FROM \"%s\" "
+ "WHERE id = $1",
+ table_name.c_str());
+ pgsql_exec(conn, PGRES_COMMAND_OK, "BEGIN");
+}
+
+pgsql_id_tracker::pimpl::~pimpl() {
+ if (conn) {
+ pgsql_exec(conn, PGRES_COMMAND_OK, "COMMIT");
+ if (owns_table) {
+ pgsql_exec(conn, PGRES_COMMAND_OK, "DROP TABLE \"%s\"", table_name.c_str());
+ }
+ PQfinish(conn);
+ }
+ conn = NULL;
+}
+
+pgsql_id_tracker::pgsql_id_tracker(const std::string &conninfo,
+ const std::string &prefix,
+ const std::string &type,
+ bool owns_table)
+ : impl() {
+ impl.reset(new pimpl(conninfo, prefix, type, owns_table));
+}
+
+pgsql_id_tracker::~pgsql_id_tracker() {
+}
+
+void pgsql_id_tracker::mark(osmid_t id) {
+ char tmp[16];
+ char const *paramValues[1];
+
+ snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
+ paramValues[0] = tmp;
+
+ pgsql_execPrepared(impl->conn, "set_mark", 1, paramValues, PGRES_COMMAND_OK);
+}
+
+bool pgsql_id_tracker::is_marked(osmid_t id) {
+ char tmp[16];
+ char const *paramValues[1] = {NULL};
+ PGresult *result = NULL;
+
+ snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
+ paramValues[0] = tmp;
+
+ result = pgsql_execPrepared(impl->conn, "get_mark", 1, paramValues, PGRES_TUPLES_OK);
+ bool done = PQntuples(result) > 0;
+ PQclear(result);
+ return done;
+}
+
+osmid_t pgsql_id_tracker::pop_mark() {
+ osmid_t id = std::numeric_limits<osmid_t>::max();
+ PGresult *result = NULL;
+
+ result = pgsql_execPrepared(impl->conn, "get_min", 0, NULL, PGRES_TUPLES_OK);
+ if ((PQntuples(result) == 1) &&
+ (PQgetisnull(result, 0, 0) == 0)) {
+ id = strtoosmid(PQgetvalue(result, 0, 0), NULL, 10);
+ }
+
+ PQclear(result);
+
+ if (id != std::numeric_limits<osmid_t>::max()) {
+ unmark(id);
+ }
+
+ assert((id > impl->old_id) || (id == std::numeric_limits<osmid_t>::max()));
+ impl->old_id = id;
+
+ return id;
+}
+
+void pgsql_id_tracker::unmark(osmid_t id) {
+ char tmp[16];
+ char const *paramValues[1] = {NULL};
+
+ snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
+ paramValues[0] = tmp;
+
+ pgsql_execPrepared(impl->conn, "drop_mark", 1, paramValues, PGRES_COMMAND_OK);
+}
+
+void pgsql_id_tracker::commit() {
+ if (impl->owns_table) {
+ pgsql_exec(impl->conn, PGRES_COMMAND_OK, "CREATE INDEX ON \"%s\" (id)", impl->table_name.c_str());
+ }
+ pgsql_exec(impl->conn, PGRES_COMMAND_OK, "COMMIT");
+ pgsql_exec(impl->conn, PGRES_COMMAND_OK, "BEGIN");
+}
+
+void pgsql_id_tracker::force_release() {
+ impl->owns_table = false;
+ impl->conn = NULL;
+}
diff --git a/pgsql-id-tracker.hpp b/pgsql-id-tracker.hpp
new file mode 100644
index 0000000..1726149
--- /dev/null
+++ b/pgsql-id-tracker.hpp
@@ -0,0 +1,28 @@
+#ifndef PGSQL_ID_TRACKER_HPP
+#define PGSQL_ID_TRACKER_HPP
+
+#include "id-tracker.hpp"
+
+struct pgsql_id_tracker : public id_tracker {
+ pgsql_id_tracker(const std::string &conninfo,
+ const std::string &prefix,
+ const std::string &type,
+ bool owns_table);
+ ~pgsql_id_tracker();
+
+ void mark(osmid_t id);
+ bool is_marked(osmid_t id);
+
+ osmid_t pop_mark();
+
+ void commit();
+ void force_release(); // to avoid brain-damages with fork()
+
+private:
+ void unmark(osmid_t id);
+
+ struct pimpl;
+ boost::scoped_ptr<pimpl> impl;
+};
+
+#endif /* PGSQL_ID_TRACKER_HPP */
diff --git a/pgsql.c b/pgsql.c
deleted file mode 100644
index 566bc1b..0000000
--- a/pgsql.c
+++ /dev/null
@@ -1,133 +0,0 @@
-/* Helper functions for the postgresql connections */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <libpq-fe.h>
-#include "osmtypes.h" /* For exit_nicely() */
-#include "pgsql.h"
-
-void escape(char *out, int len, const char *in)
-{
- /* Apply escaping of TEXT COPY data
- Escape: backslash itself, newline, carriage return, and the current delimiter character (tab)
- file:///usr/share/doc/postgresql-8.1.8/html/sql-copy.html
- */
- int count = 0;
- const char *old_in = in, *old_out = out;
-
- if (!len)
- return;
-
- while(*in && count < len-3) {
- switch(*in) {
- case '\\': *out++ = '\\'; *out++ = '\\'; count+= 2; break;
- /* case 8: *out++ = '\\'; *out++ = '\b'; count+= 2; break; */
- /* case 12: *out++ = '\\'; *out++ = '\f'; count+= 2; break; */
- case '\n': *out++ = '\\'; *out++ = '\n'; count+= 2; break;
- case '\r': *out++ = '\\'; *out++ = '\r'; count+= 2; break;
- case '\t': *out++ = '\\'; *out++ = '\t'; count+= 2; break;
- /* case 11: *out++ = '\\'; *out++ = '\v'; count+= 2; break; */
- default: *out++ = *in; count++; break;
- }
- in++;
- }
- *out = '\0';
-
- if (*in)
- fprintf(stderr, "%s truncated at %d chars: %s\n%s\n", __FUNCTION__, count, old_in, old_out);
-}
-
-int pgsql_exec(PGconn *sql_conn, ExecStatusType expect, const char *fmt, ...)
-{
- PGresult *res;
- va_list ap;
- char *sql, *nsql;
- int n, size = 100;
-
- /* Based on vprintf manual page */
- /* Guess we need no more than 100 bytes. */
-
- if ((sql = malloc(size)) == NULL) {
- fprintf(stderr, "Memory allocation failed\n");
- exit_nicely();
- }
-
- while (1) {
- /* Try to print in the allocated space. */
- va_start(ap, fmt);
- n = vsnprintf(sql, size, fmt, ap);
- va_end(ap);
- /* If that worked, return the string. */
- if (n > -1 && n < size)
- break;
- /* Else try again with more space. */
- if (n > -1) /* glibc 2.1 */
- size = n+1; /* precisely what is needed */
- else /* glibc 2.0 */
- size *= 2; /* twice the old size */
- if ((nsql = realloc (sql, size)) == NULL) {
- free(sql);
- fprintf(stderr, "Memory re-allocation failed\n");
- exit_nicely();
- } else {
- sql = nsql;
- }
- }
-
-#ifdef DEBUG_PGSQL
- fprintf( stderr, "Executing: %s\n", sql );
-#endif
- res = PQexec(sql_conn, sql);
- if (PQresultStatus(res) != expect) {
- fprintf(stderr, "%s failed: %s\n", sql, PQerrorMessage(sql_conn));
- free(sql);
- PQclear(res);
- exit_nicely();
- }
- free(sql);
- PQclear(res);
- return 0;
-}
-
-int pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql)
-{
-#ifdef DEBUG_PGSQL
- fprintf( stderr, "%s>>> %s\n", context, sql );
-#endif
- int r = PQputCopyData(sql_conn, sql, strlen(sql));
- if (r != 1) {
- fprintf(stderr, "%s - bad result during COPY, data %s\n", context, sql);
- exit_nicely();
- }
- return 0;
-}
-
-PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, int nParams, const char *const * paramValues, ExecStatusType expect)
-{
-#ifdef DEBUG_PGSQL
- fprintf( stderr, "ExecPrepared: %s\n", stmtName );
-#endif
- PGresult *res = PQexecPrepared(sql_conn, stmtName, nParams, paramValues, NULL, NULL, 0);
- if (PQresultStatus(res) != expect) {
- fprintf(stderr, "%s failed: %s(%d)\n", stmtName, PQerrorMessage(sql_conn), PQresultStatus(res));
- if( nParams )
- {
- int i;
- fprintf( stderr, "Arguments were: " );
- for( i=0; i<nParams; i++ )
- fprintf( stderr, "%s, ", paramValues[i] );
- fprintf( stderr, "\n");
- }
- PQclear(res);
- exit_nicely();
- }
- if( expect != PGRES_TUPLES_OK )
- {
- PQclear(res);
- res = NULL;
- }
- return res;
-}
-
diff --git a/pgsql.cpp b/pgsql.cpp
new file mode 100644
index 0000000..a7eaea4
--- /dev/null
+++ b/pgsql.cpp
@@ -0,0 +1,180 @@
+/* Helper functions for the postgresql connections */
+#include "pgsql.hpp"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <libpq-fe.h>
+#include <boost/format.hpp>
+
+void escape(const char* src, std::string& dst)
+{
+ //loop over the chars in the input
+ size_t length = strlen(src);
+ for(size_t i = 0; i < length; ++i)
+ {
+ switch(src[i]) {
+ case '\\': dst.append("\\\\"); break;
+ //case 8: dst.append("\\\b"); break;
+ //case 12: dst.append("\\\f"); break;
+ case '\n': dst.append("\\\n"); break;
+ case '\r': dst.append("\\\r"); break;
+ case '\t': dst.append("\\\t"); break;
+ //case 11: dst.append("\\\v"); break;
+ default: dst.push_back(src[i]); break;
+ }
+ }
+}
+
+void escape(char *out, const int len, const char *in)
+{
+ /* Apply escaping of TEXT COPY data
+ Escape: backslash itself, newline, carriage return, and the current delimiter character (tab)
+ file:///usr/share/doc/postgresql-8.1.8/html/sql-copy.html
+ */
+ int count = 0;
+ const char *old_in = in, *old_out = out;
+
+ if (!len)
+ return;
+
+ while(*in && count < len-3) {
+ switch(*in) {
+ case '\\': *out++ = '\\'; *out++ = '\\'; count+= 2; break;
+ /* case 8: *out++ = '\\'; *out++ = '\b'; count+= 2; break; */
+ /* case 12: *out++ = '\\'; *out++ = '\f'; count+= 2; break; */
+ case '\n': *out++ = '\\'; *out++ = '\n'; count+= 2; break;
+ case '\r': *out++ = '\\'; *out++ = '\r'; count+= 2; break;
+ case '\t': *out++ = '\\'; *out++ = '\t'; count+= 2; break;
+ /* case 11: *out++ = '\\'; *out++ = '\v'; count+= 2; break; */
+ default: *out++ = *in; count++; break;
+ }
+ in++;
+ }
+ *out = '\0';
+
+ if (*in)
+ fprintf(stderr, "%s truncated at %d chars: %s\n%s\n", __FUNCTION__, count, old_in, old_out);
+}
+
+boost::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const std::string& sql)
+{
+ return pgsql_exec_simple(sql_conn, expect, sql.c_str());
+}
+
+boost::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const char *sql)
+{
+ PGresult* res;
+#ifdef DEBUG_PGSQL
+ fprintf( stderr, "Executing: %s\n", sql );
+#endif
+ res = PQexec(sql_conn, sql);
+ if (PQresultStatus(res) != expect) {
+ PQclear(res);
+ throw std::runtime_error((boost::format("%1% failed: %2%\n") % sql % PQerrorMessage(sql_conn)).str());
+ }
+ return boost::shared_ptr<PGresult>(res, &PQclear);
+}
+
+int pgsql_exec(PGconn *sql_conn, const ExecStatusType expect, const char *fmt, ...)
+{
+
+ va_list ap;
+ char *sql, *nsql;
+ int n, size = 100;
+
+ /* Based on vprintf manual page */
+ /* Guess we need no more than 100 bytes. */
+
+ if ((sql = (char *)malloc(size)) == NULL)
+ throw std::runtime_error("Memory allocation failed in pgsql_exec");
+
+ while (1) {
+ /* Try to print in the allocated space. */
+ va_start(ap, fmt);
+ n = vsnprintf(sql, size, fmt, ap);
+ va_end(ap);
+ /* If that worked, return the string. */
+ if (n > -1 && n < size)
+ break;
+ /* Else try again with more space. */
+ if (n > -1) /* glibc 2.1 */
+ size = n+1; /* precisely what is needed */
+ else /* glibc 2.0 */
+ size *= 2; /* twice the old size */
+ if ((nsql = (char *)realloc (sql, size)) == NULL) {
+ free(sql);
+ throw std::runtime_error("Memory re-allocation failed in pgsql_exec");
+ } else {
+ sql = nsql;
+ }
+ }
+
+#ifdef DEBUG_PGSQL
+ fprintf( stderr, "Executing: %s\n", sql );
+#endif
+ PGresult* res = PQexec(sql_conn, sql);
+ if (PQresultStatus(res) != expect) {
+ std::string err_msg = (boost::format("%1% failed: %2%") % sql % PQerrorMessage(sql_conn)).str();
+ free(sql);
+ PQclear(res);
+ throw std::runtime_error(err_msg);
+ }
+ free(sql);
+ PQclear(res);
+ return 0;
+}
+
+void pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql)
+{
+#ifdef DEBUG_PGSQL
+ fprintf(stderr, "%s>>> %s\n", context, sql );
+#endif
+ int r = PQputCopyData(sql_conn, sql, strlen(sql));
+ switch(r)
+ {
+ //need to wait for write ready
+ case 0:
+ throw std::runtime_error((boost::format("%1% - bad result during COPY, data %2%") % context % sql % PQerrorMessage(sql_conn)).str());
+ break;
+ //error occurred
+ case -1:
+ throw std::runtime_error((boost::format("%1%: %2% - bad result during COPY, data %3%") % PQerrorMessage(sql_conn) % context % sql).str());
+ break;
+ //other possibility is 1 which means success
+ }
+}
+
+PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, const int nParams, const char *const * paramValues, const ExecStatusType expect)
+{
+#ifdef DEBUG_PGSQL
+ fprintf( stderr, "ExecPrepared: %s\n", stmtName );
+#endif
+ //run the prepared statement
+ PGresult *res = PQexecPrepared(sql_conn, stmtName, nParams, paramValues, NULL, NULL, 0);
+ if(PQresultStatus(res) != expect)
+ {
+ std::string message = (boost::format("%1% failed: %2%(%3%)\n") % stmtName % PQerrorMessage(sql_conn) % PQresultStatus(res)).str();
+ if(nParams)
+ {
+ message += "Arguments were: ";
+ for(int i = 0; i < nParams; i++)
+ {
+ message += paramValues[i];
+ message += ", ";
+ }
+ }
+ PQclear(res);
+ throw std::runtime_error(message);
+ }
+
+ //TODO: this seems a bit strange
+ //if you decided you wanted to expect something other than this you didnt want to use the result?
+ if( expect != PGRES_TUPLES_OK )
+ {
+ PQclear(res);
+ res = NULL;
+ }
+ return res;
+}
diff --git a/pgsql.h b/pgsql.h
deleted file mode 100644
index e21c292..0000000
--- a/pgsql.h
+++ /dev/null
@@ -1,14 +0,0 @@
-/* Helper functions for pgsql access */
-
-/* Current middle and output-pgsql do a lot of things similarly, this should
- * be used to abstract to commonalities */
-
-#ifndef PGSQL_H
-#define PGSQL_H
-
-PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, int nParams, const char *const * paramValues, ExecStatusType expect);
-int pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql);
-int pgsql_exec(PGconn *sql_conn, ExecStatusType expect, const char *fmt, ...) __attribute__ ((format (printf, 3, 4)));
-void escape(char *out, int len, const char *in);
-
-#endif
diff --git a/pgsql.hpp b/pgsql.hpp
new file mode 100644
index 0000000..540def3
--- /dev/null
+++ b/pgsql.hpp
@@ -0,0 +1,26 @@
+/* Helper functions for pgsql access */
+
+/* Current middle and output-pgsql do a lot of things similarly, this should
+ * be used to abstract to commonalities */
+
+#ifndef PGSQL_H
+#define PGSQL_H
+
+#include <string>
+#include <libpq-fe.h>
+#include <boost/shared_ptr.hpp>
+
+PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, const int nParams, const char *const * paramValues, const ExecStatusType expect);
+void pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql);
+boost::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const std::string& sql);
+boost::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const char *sql);
+int pgsql_exec(PGconn *sql_conn, const ExecStatusType expect, const char *fmt, ...)
+#ifndef _MSC_VER
+ __attribute__ ((format (printf, 3, 4)))
+#endif
+;
+
+void escape(const char* src, std::string& dst);
+void escape(char *out, int len, const char *in);
+
+#endif
diff --git a/processor-line.cpp b/processor-line.cpp
new file mode 100644
index 0000000..c6f16ae
--- /dev/null
+++ b/processor-line.cpp
@@ -0,0 +1,19 @@
+#include "processor-line.hpp"
+
+#include <boost/format.hpp>
+
+processor_line::processor_line(int srid) : geometry_processor(srid, "LINESTRING", interest_way)
+{
+}
+
+processor_line::~processor_line()
+{
+}
+
+geometry_builder::maybe_wkt_t processor_line::process_way(const osmNode *nodes, size_t node_count)
+{
+ //have the builder make the wkt
+ geometry_builder::maybe_wkt_t wkt = builder.get_wkt_simple(nodes, node_count, false);
+ //hand back the wkt
+ return wkt;
+}
diff --git a/processor-line.hpp b/processor-line.hpp
new file mode 100644
index 0000000..05f7017
--- /dev/null
+++ b/processor-line.hpp
@@ -0,0 +1,16 @@
+#ifndef PROCESSOR_LINE_HPP
+#define PROCESSOR_LINE_HPP
+
+#include "geometry-processor.hpp"
+
+struct processor_line : public geometry_processor {
+ processor_line(int srid);
+ virtual ~processor_line();
+
+ geometry_builder::maybe_wkt_t process_way(const osmNode *nodes, size_t node_count);
+
+private:
+ geometry_builder builder;
+};
+
+#endif /* PROCESSOR_LINE_HPP */
diff --git a/processor-point.cpp b/processor-point.cpp
new file mode 100644
index 0000000..bd84f06
--- /dev/null
+++ b/processor-point.cpp
@@ -0,0 +1,23 @@
+#include "processor-point.hpp"
+#include "util.hpp"
+
+#include <boost/format.hpp>
+
+processor_point::processor_point(int srid, double scale)
+ : geometry_processor(srid, "POINT", interest_node),
+ m_scale(scale) {
+}
+
+processor_point::~processor_point() {
+}
+
+geometry_builder::maybe_wkt_t processor_point::process_node(double lat, double lon) {
+#ifdef FIXED_POINT
+ // guarantee that we use the same values as in the node cache
+ lon = util::fix_to_double(util::double_to_fix(lon, m_scale), m_scale);
+ lat = util::fix_to_double(util::double_to_fix(lat, m_scale), m_scale);
+#endif
+ geometry_builder::maybe_wkt_t wkt(new geometry_builder::wkt_t());
+ wkt->geom = (boost::format("POINT(%.15g %.15g)") % lon % lat).str();
+ return wkt;
+}
diff --git a/processor-point.hpp b/processor-point.hpp
new file mode 100644
index 0000000..088ef97
--- /dev/null
+++ b/processor-point.hpp
@@ -0,0 +1,16 @@
+#ifndef PROCESSOR_POINT_HPP
+#define PROCESSOR_POINT_HPP
+
+#include "geometry-processor.hpp"
+
+struct processor_point : public geometry_processor {
+ processor_point(int srid, double scale);
+ virtual ~processor_point();
+
+ geometry_builder::maybe_wkt_t process_node(double lat, double lon);
+
+private:
+ double m_scale; // <-- used only when FIXED_POINT is defined, which seems like never?
+};
+
+#endif /* PROCESSOR_POINT_HPP */
diff --git a/processor-polygon.cpp b/processor-polygon.cpp
new file mode 100644
index 0000000..7e705d1
--- /dev/null
+++ b/processor-polygon.cpp
@@ -0,0 +1,26 @@
+#include "processor-polygon.hpp"
+
+#include <boost/format.hpp>
+
+processor_polygon::processor_polygon(int srid, bool enable_multi) : geometry_processor(srid, "GEOMETRY", interest_way | interest_relation), enable_multi(enable_multi)
+{
+}
+
+processor_polygon::~processor_polygon()
+{
+}
+
+geometry_builder::maybe_wkt_t processor_polygon::process_way(const osmNode *nodes, const size_t node_count)
+{
+ //have the builder make the wkt
+ geometry_builder::maybe_wkt_t wkt = builder.get_wkt_simple(nodes, node_count, true);
+ //hand back the wkt
+ return wkt;
+}
+
+geometry_builder::maybe_wkts_t processor_polygon::process_relation(const osmNode * const * nodes, const int* node_counts)
+{
+ //the hard word was already done for us in getting at the node data for each way. at this point just make the geom
+ geometry_builder::maybe_wkts_t wkts = builder.build_polygons(nodes, node_counts, enable_multi, -1);
+ return wkts;
+}
diff --git a/processor-polygon.hpp b/processor-polygon.hpp
new file mode 100644
index 0000000..96bdf42
--- /dev/null
+++ b/processor-polygon.hpp
@@ -0,0 +1,18 @@
+#ifndef PROCESSOR_POLYGON_HPP
+#define PROCESSOR_POLYGON_HPP
+
+#include "geometry-processor.hpp"
+
+struct processor_polygon : public geometry_processor {
+ processor_polygon(int srid, bool enable_multi);
+ virtual ~processor_polygon();
+
+ geometry_builder::maybe_wkt_t process_way(const osmNode *nodes, const size_t node_count);
+ geometry_builder::maybe_wkts_t process_relation(const osmNode * const * nodes, const int* node_counts);
+
+private:
+ bool enable_multi;
+ geometry_builder builder;
+};
+
+#endif /* PROCESSOR_POLYGON_HPP */
diff --git a/rb.c b/rb.cpp
similarity index 94%
rename from rb.c
rename to rb.cpp
index d1339b9..084e7fd 100644
--- a/rb.c
+++ b/rb.cpp
@@ -27,7 +27,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include "rb.h"
+#include "rb.hpp"
/* Creates and returns a new table
with comparison function |compare| using parameter |param|
@@ -44,7 +44,7 @@ rb_create (rb_comparison_func *compare, void *param,
if (allocator == NULL)
allocator = &rb_allocator_default;
- tree = allocator->libavl_malloc (allocator, sizeof *tree);
+ tree = (struct rb_table *)allocator->libavl_malloc (allocator, sizeof *tree);
if (tree == NULL)
return NULL;
@@ -111,7 +111,7 @@ rb_probe (struct rb_table *tree, void *item)
}
n = pa[k - 1]->rb_link[da[k - 1]] =
- tree->rb_alloc->libavl_malloc (tree->rb_alloc, sizeof *n);
+ (struct rb_node *)tree->rb_alloc->libavl_malloc (tree->rb_alloc, sizeof *n);
if (n == NULL)
return NULL;
@@ -729,13 +729,13 @@ rb_t_cur (struct rb_traverser *trav)
|trav| must not have the null item selected.
The new item must not upset the ordering of the tree. */
void *
-rb_t_replace (struct rb_traverser *trav, void *new)
+rb_t_replace (struct rb_traverser *trav, void *new_)
{
void *old;
- assert (trav != NULL && trav->rb_node != NULL && new != NULL);
+ assert (trav != NULL && trav->rb_node != NULL && new_ != NULL);
old = trav->rb_node->rb_data;
- trav->rb_node->rb_data = new;
+ trav->rb_node->rb_data = new_;
return old;
}
@@ -744,13 +744,13 @@ rb_t_replace (struct rb_traverser *trav, void *new)
to null pointers to avoid touching uninitialized data. */
static void
copy_error_recovery (struct rb_node **stack, int height,
- struct rb_table *new, rb_item_func *destroy)
+ struct rb_table *new_, rb_item_func *destroy)
{
- assert (stack != NULL && height >= 0 && new != NULL);
+ assert (stack != NULL && height >= 0 && new_ != NULL);
for (; height > 2; height -= 2)
stack[height - 1]->rb_link[1] = NULL;
- rb_destroy (new, destroy);
+ rb_destroy (new_, destroy);
}
/* Copies |org| to a newly created tree, which is returned.
@@ -769,21 +769,21 @@ rb_copy (const struct rb_table *org, rb_copy_func *copy,
struct rb_node *stack[2 * (RB_MAX_HEIGHT + 1)];
int height = 0;
- struct rb_table *new;
+ struct rb_table *new_;
const struct rb_node *x;
struct rb_node *y;
assert (org != NULL);
- new = rb_create (org->rb_compare, org->rb_param,
+ new_ = rb_create (org->rb_compare, org->rb_param,
allocator != NULL ? allocator : org->rb_alloc);
- if (new == NULL)
+ if (new_ == NULL)
return NULL;
- new->rb_count = org->rb_count;
- if (new->rb_count == 0)
- return new;
+ new_->rb_count = org->rb_count;
+ if (new_->rb_count == 0)
+ return new_;
x = (const struct rb_node *) &org->rb_root;
- y = (struct rb_node *) &new->rb_root;
+ y = (struct rb_node *) &new_->rb_root;
for (;;)
{
while (x->rb_link[0] != NULL)
@@ -791,17 +791,18 @@ rb_copy (const struct rb_table *org, rb_copy_func *copy,
assert (height < 2 * (RB_MAX_HEIGHT + 1));
y->rb_link[0] =
- new->rb_alloc->libavl_malloc (new->rb_alloc,
- sizeof *y->rb_link[0]);
+ (struct rb_node *)
+ new_->rb_alloc->libavl_malloc (new_->rb_alloc,
+ sizeof *y->rb_link[0]);
if (y->rb_link[0] == NULL)
{
- if (y != (struct rb_node *) &new->rb_root)
+ if (y != (struct rb_node *) &new_->rb_root)
{
y->rb_data = NULL;
y->rb_link[1] = NULL;
}
- copy_error_recovery (stack, height, new, destroy);
+ copy_error_recovery (stack, height, new_, destroy);
return NULL;
}
@@ -823,7 +824,7 @@ rb_copy (const struct rb_table *org, rb_copy_func *copy,
if (y->rb_data == NULL)
{
y->rb_link[1] = NULL;
- copy_error_recovery (stack, height, new, destroy);
+ copy_error_recovery (stack, height, new_, destroy);
return NULL;
}
}
@@ -831,11 +832,12 @@ rb_copy (const struct rb_table *org, rb_copy_func *copy,
if (x->rb_link[1] != NULL)
{
y->rb_link[1] =
- new->rb_alloc->libavl_malloc (new->rb_alloc,
- sizeof *y->rb_link[1]);
+ (struct rb_node *)
+ new_->rb_alloc->libavl_malloc (new_->rb_alloc,
+ sizeof *y->rb_link[1]);
if (y->rb_link[1] == NULL)
{
- copy_error_recovery (stack, height, new, destroy);
+ copy_error_recovery (stack, height, new_, destroy);
return NULL;
}
@@ -847,7 +849,7 @@ rb_copy (const struct rb_table *org, rb_copy_func *copy,
y->rb_link[1] = NULL;
if (height <= 2)
- return new;
+ return new_;
y = stack[--height];
x = stack[--height];
@@ -926,4 +928,3 @@ void *
assert (p != NULL);
return p;
}
-
diff --git a/rb.h b/rb.hpp
similarity index 98%
rename from rb.h
rename to rb.hpp
index c8858b5..b16feab 100644
--- a/rb.h
+++ b/rb.hpp
@@ -77,7 +77,7 @@ struct rb_node
{
struct rb_node *rb_link[2]; /* Subtrees. */
void *rb_data; /* Pointer to data. */
- unsigned char rb_color; /* Color. */
+ enum rb_color rb_color; /* Color. */
};
/* RB traverser structure. */
diff --git a/reprojection.c b/reprojection.cpp
similarity index 62%
rename from reprojection.c
rename to reprojection.cpp
index b8f3f17..a2a7ebb 100644
--- a/reprojection.c
+++ b/reprojection.cpp
@@ -1,17 +1,19 @@
/* reprojection.c
*
- * Convert OSM coordinates to another coordinate system for
+ * Convert OSM coordinates to another coordinate system for
* the database (usually convert lat/lon to Spherical Mercator
* so Mapnik doesn't have to).
*/
+#include "config.h"
+
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <proj_api.h>
#include <math.h>
-#include "reprojection.h"
+#include "reprojection.hpp"
#ifndef M_PI
#define M_PI 3.14159265358979323846
@@ -20,64 +22,54 @@
/** must match expire.tiles.c */
#define EARTH_CIRCUMFERENCE 40075016.68
-/** The projection of the source data. Always lat/lon (EPSG:4326). */
-static projPJ pj_source = NULL;
-
-/** The target projection (used in the PostGIS tables). Controlled by the -l/-M/-m/-E options. */
-static projPJ pj_target = NULL;
+Projection_Info::Projection_Info(const char *descr_, const char *proj4text_, int srs_, const char *option_)
+ : descr(descr_), proj4text(proj4text_), srs(srs_), option(option_) {
+}
-/** The projection used for tiles. Currently this is fixed to be Spherical
- * Mercator. You will usually have tiles in the same projection as used
- * for PostGIS, but it is theoretically possible to have your PostGIS data
- * in, say, lat/lon but still create tiles in Spherical Mercator.
- */
-static projPJ pj_tile = NULL;
-
-static int Proj;
-
-const struct Projection_Info Projection_Info[] = {
- [PROJ_LATLONG] = {
- .descr = "Latlong",
- .proj4text = "+init=epsg:4326",
- .srs = 4326,
- .option = "-l" },
- [PROJ_MERC] = {
- .descr = "WGS84 Mercator",
- .proj4text = "+proj=merc +datum=WGS84 +k=1.0 +units=m +over +no_defs",
- .srs = 3395,
- .option = "-M" },
- [PROJ_SPHERE_MERC] = {
- .descr = "Spherical Mercator",
- .proj4text = "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs",
- .srs = 900913,
- .option = "-m" }
+namespace {
+
+const struct Projection_Info Projection_Infos[] = {
+ /*PROJ_LATLONG*/ Projection_Info(
+ /*descr */ "Latlong",
+ /*proj4text*/ "+init=epsg:4326",
+ /*srs */ 4326,
+ /*option */ "-l" ),
+ /*PROJ_MERC*/ Projection_Info(
+ /*descr */ "WGS84 Mercator",
+ /*proj4text*/ "+proj=merc +datum=WGS84 +k=1/*0 +units=m +over +no_defs",
+ /*srs */ 3395,
+ /*option */ "-M" ),
+ /*PROJ_SPHERE_MERC*/ Projection_Info(
+ /*descr */ "Spherical Mercator",
+ /*proj4text*/ "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs",
+ /*srs */ 900913,
+ /*option */ "-m" )
};
-static struct Projection_Info custom_projection;
-/** defined in expire-tiles.c; depends on the zoom level selected for expiry. */
-extern int map_width;
+} // anonymous namespace
/* Positive numbers refer the to the table above, negative numbers are
assumed to refer to EPSG codes and it uses the proj4 to find those. */
-void project_init(int proj)
+reprojection::reprojection(int proj)
+ : Proj(proj), pj_source(NULL), pj_target(NULL), pj_tile(NULL),
+ custom_projection(NULL)
{
char buffer[32];
- Proj = proj;
-
- /* hard-code the source projection to be lat/lon, since OSM XML always
+
+ /* hard-code the source projection to be lat/lon, since OSM XML always
* has coordinates in degrees. */
pj_source = pj_init_plus("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs");
- /* hard-code the tile projection to be spherical mercator always.
+ /* hard-code the tile projection to be spherical mercator always.
* theoretically this could be made selectable but not all projections
* lend themselves well to making tiles; non-spherical mercator tiles
* are uncharted waters in OSM. */
- pj_tile = pj_init_plus(Projection_Info[PROJ_SPHERE_MERC].proj4text);
+ pj_tile = pj_init_plus(Projection_Infos[PROJ_SPHERE_MERC].proj4text);
/* now set the target projection - the only one which is really variable */
if (proj >= 0 && proj < PROJ_COUNT)
{
- pj_target = pj_init_plus(Projection_Info[proj].proj4text);
+ pj_target = pj_init_plus(Projection_Infos[proj].proj4text);
}
else if (proj < 0)
{
@@ -93,48 +85,53 @@ void project_init(int proj)
exit(1);
}
}
-
- if (!pj_source || !pj_target || !pj_tile)
+
+ if (!pj_source || !pj_target || !pj_tile)
{
fprintf(stderr, "Projection code failed to initialise\n");
exit(1);
}
-
+
if (proj >= 0)
return;
- custom_projection.srs = -proj;
- custom_projection.proj4text = pj_get_def(pj_target, 0);
if (snprintf(buffer, sizeof(buffer), "EPSG:%d", -proj) >= (int)sizeof(buffer))
{
fprintf(stderr, "Buffer overflow computing projection description\n");
exit(1);
}
- custom_projection.descr = strdup(buffer);
- custom_projection.option = "-E";
- return;
+ custom_projection = new Projection_Info(
+ strdup(buffer),
+ pj_get_def(pj_target, 0),
+ -proj, "-E");
}
-void project_exit(void)
+reprojection::~reprojection()
{
pj_free(pj_source);
pj_source = NULL;
pj_free(pj_target);
pj_target = NULL;
+ pj_free(pj_tile);
+ pj_tile = NULL;
+
+ if (custom_projection != NULL) {
+ delete custom_projection;
+ }
}
-struct Projection_Info const *project_getprojinfo(void)
+struct Projection_Info const *reprojection::project_getprojinfo(void)
{
if( Proj >= 0 )
- return &Projection_Info[Proj];
+ return &Projection_Infos[Proj];
else
- return &custom_projection;
+ return custom_projection;
}
-void reproject(double *lat, double *lon)
+void reprojection::reproject(double *lat, double *lon)
{
double x[1], y[1], z[1];
-
+
/** Caution: This section is only correct if the source projection is lat/lon;
* so even if it looks like pj_source was just a variable, things break if
* pj_source is something else than lat/lon. */
@@ -144,7 +141,7 @@ void reproject(double *lat, double *lon)
if (Proj == PROJ_SPHERE_MERC)
{
- /* The latitude co-ordinate is clipped at slightly larger than the 900913 'world'
+ /* The latitude co-ordinate is clipped at slightly larger than the 900913 'world'
* extent of +-85.0511 degrees to ensure that the points appear just outside the
* edge of the map. */
@@ -163,23 +160,24 @@ void reproject(double *lat, double *lon)
z[0] = 0;
/** end of "caution" section. */
-
+
pj_transform(pj_source, pj_target, 1, 1, x, y, z);
-
+
*lat = y[0];
*lon = x[0];
}
-/**
+/**
* Converts from (target) coordinates to tile coordinates.
*
- * The zoom level for the coordinates is implicitly given in the global
+ * The zoom level for the coordinates is explicitly given in the
* variable map_width.
*/
-void coords_to_tile(double *tilex, double *tiley, double lon, double lat)
+void reprojection::coords_to_tile(double *tilex, double *tiley, double lon, double lat,
+ int map_width)
{
double x[1], y[1], z[1];
-
+
x[0] = lon;
y[0] = lat;
z[0] = 0;
@@ -190,7 +188,7 @@ void coords_to_tile(double *tilex, double *tiley, double lon, double lat)
y[0] *= DEG_TO_RAD;
}
- /* since pj_tile is always spherical merc, don't bother doing anything if
+ /* since pj_tile is always spherical merc, don't bother doing anything if
* destination proj is the same. */
if (Proj != PROJ_SPHERE_MERC)
@@ -198,7 +196,7 @@ void coords_to_tile(double *tilex, double *tiley, double lon, double lat)
pj_transform(pj_target, pj_tile, 1, 1, x, y, z);
/** FIXME: pj_transform could fail if coordinates are outside +/- 85 degrees latitude */
}
-
+
/* if ever pj_tile were allowed to be PROJ_LATLONG then results would have to
* be divided by DEG_TO_RAD here. */
@@ -206,3 +204,7 @@ void coords_to_tile(double *tilex, double *tiley, double lon, double lat)
*tiley = map_width * (0.5 - y[0] / EARTH_CIRCUMFERENCE);
}
+int reprojection::get_proj_id() const
+{
+ return Proj;
+}
diff --git a/reprojection.h b/reprojection.h
deleted file mode 100644
index fdc9234..0000000
--- a/reprojection.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/* reprojection.h
- *
- * Convert OSM lattitude / longitude from degrees to mercator
- * so that Mapnik does not have to project the data again
- *
- */
-
-#ifndef REPROJECTION_H
-#define REPROJECTION_H
-
-struct Projection_Info {
- char *descr;
- char *proj4text;
- int srs;
- char *option;
-};
-
-enum Projection { PROJ_LATLONG = 0, PROJ_MERC, PROJ_SPHERE_MERC, PROJ_COUNT };
-void project_init(int);
-void project_exit(void);
-struct Projection_Info const* project_getprojinfo(void);
-void reproject(double *lat, double *lon);
-void coords_to_tile(double *tilex, double *tiley, double lon, double lat);
-
-extern const struct Projection_Info Projection_Info[];
-
-#endif
diff --git a/reprojection.hpp b/reprojection.hpp
new file mode 100644
index 0000000..0d9a03c
--- /dev/null
+++ b/reprojection.hpp
@@ -0,0 +1,53 @@
+/* reprojection.h
+ *
+ * Convert OSM lattitude / longitude from degrees to mercator
+ * so that Mapnik does not have to project the data again
+ *
+ */
+
+#ifndef REPROJECTION_H
+#define REPROJECTION_H
+
+#include <boost/noncopyable.hpp>
+
+struct Projection_Info {
+ Projection_Info(const char *descr_, const char *proj4text_, int srs_, const char *option_);
+
+ const char *descr;
+ const char *proj4text;
+ const int srs;
+ const char *option;
+};
+
+enum Projection { PROJ_LATLONG = 0, PROJ_MERC, PROJ_SPHERE_MERC, PROJ_COUNT };
+
+struct reprojection : public boost::noncopyable
+{
+ explicit reprojection(int proj);
+ ~reprojection();
+
+ struct Projection_Info const* project_getprojinfo(void);
+ void reproject(double *lat, double *lon);
+ void coords_to_tile(double *tilex, double *tiley, double lon, double lat, int map_width);
+ int get_proj_id() const;
+
+private:
+ int Proj;
+
+ /** The projection of the source data. Always lat/lon (EPSG:4326). */
+ void *pj_source;
+
+ /** The target projection (used in the PostGIS tables). Controlled by the -l/-M/-m/-E options. */
+ void *pj_target;
+
+ /** The projection used for tiles. Currently this is fixed to be Spherical
+ * Mercator. You will usually have tiles in the same projection as used
+ * for PostGIS, but it is theoretically possible to have your PostGIS data
+ * in, say, lat/lon but still create tiles in Spherical Mercator.
+ */
+ void *pj_tile;
+
+ struct Projection_Info *custom_projection;
+};
+
+#endif
diff --git a/sanitizer.h b/sanitizer.hpp
similarity index 100%
rename from sanitizer.h
rename to sanitizer.hpp
diff --git a/sprompt.c b/sprompt.cpp
similarity index 95%
rename from sprompt.c
rename to sprompt.cpp
index a63c651..63e4f21 100644
--- a/sprompt.c
+++ b/sprompt.cpp
@@ -11,7 +11,7 @@
* $PostgreSQL: pgsql/src/port/sprompt.c,v 1.18 2006/10/04 00:30:14 momjian Exp $
*
*-------------------------------------------------------------------------
- *
+ *
* PostgreSQL Database Management System
* (formerly known as Postgres, then as Postgres95)
*
@@ -62,14 +62,13 @@
#include <libpq-fe.h>
-#ifdef __MINGW_H
-# include <windows.h>
-#else
-# define HAVE_TERMIOS_H
-# include <termios.h>
+#ifdef HAVE_TERMIOS_H
+#include <termios.h>
#endif
-extern char *simple_prompt(const char *prompt, int maxlen, int echo);
+#ifdef _WIN32
+#include <windows.h>
+#endif
char *
simple_prompt(const char *prompt, int maxlen, int echo)
@@ -83,7 +82,7 @@ simple_prompt(const char *prompt, int maxlen, int echo)
struct termios t_orig,
t;
#else
-#ifdef WIN32
+#ifdef _WIN32
HANDLE t = NULL;
LPDWORD t_orig = NULL;
#endif
@@ -100,7 +99,7 @@ simple_prompt(const char *prompt, int maxlen, int echo)
termin = fopen(DEVTTY, "r");
termout = fopen(DEVTTY, "w");
if (!termin || !termout
-#ifdef WIN32
+#ifdef _WIN32
/* See DEVTTY comment for msys */
|| (getenv("OSTYPE") && strcmp(getenv("OSTYPE"), "msys") == 0)
#endif
@@ -123,7 +122,7 @@ simple_prompt(const char *prompt, int maxlen, int echo)
tcsetattr(fileno(termin), TCSAFLUSH, &t);
}
#else
-#ifdef WIN32
+#ifdef _WIN32
if (!echo)
{
/* get a new handle to turn echo off */
@@ -175,7 +174,7 @@ simple_prompt(const char *prompt, int maxlen, int echo)
fflush(termout);
}
#else
-#ifdef WIN32
+#ifdef _WIN32
if (!echo)
{
/* reset to the original console mode */
diff --git a/sprompt.h b/sprompt.hpp
similarity index 100%
rename from sprompt.h
rename to sprompt.hpp
diff --git a/table.cpp b/table.cpp
new file mode 100644
index 0000000..515dce2
--- /dev/null
+++ b/table.cpp
@@ -0,0 +1,558 @@
+#include "table.hpp"
+#include "options.hpp"
+#include "util.hpp"
+
+#include <string.h>
+#include <utility>
+
+using std::string;
+
+#define BUFFER_SEND_SIZE 1024
+
+
+table_t::table_t(const string& conninfo, const string& name, const string& type, const columns_t& columns, const hstores_t& hstore_columns,
+ const int srid, const int scale, const bool append, const bool slim, const bool drop_temp, const int hstore_mode,
+ const bool enable_hstore_index, const boost::optional<string>& table_space, const boost::optional<string>& table_space_index) :
+ conninfo(conninfo), name(name), type(type), sql_conn(NULL), copyMode(false), srid((fmt("%1%") % srid).str()), scale(scale),
+ append(append), slim(slim), drop_temp(drop_temp), hstore_mode(hstore_mode), enable_hstore_index(enable_hstore_index),
+ columns(columns), hstore_columns(hstore_columns), table_space(table_space), table_space_index(table_space_index)
+{
+ //if we dont have any columns
+ if(columns.size() == 0)
+ throw std::runtime_error((fmt("No columns provided for table %1%") % name).str());
+
+ //nothing to copy to start with
+ buffer = "";
+
+ //we use these a lot, so instead of constantly allocating them we predefine these
+ single_fmt = fmt("%1%");
+ point_fmt = fmt("POINT(%.15g %.15g)");
+ del_fmt = fmt("DELETE FROM %1% WHERE osm_id = %2%");
+}
+
+table_t::table_t(const table_t& other):
+ conninfo(other.conninfo), name(other.name), type(other.type), sql_conn(NULL), copyMode(false), buffer(), srid(other.srid), scale(other.scale),
+ append(other.append), slim(other.slim), drop_temp(other.drop_temp), hstore_mode(other.hstore_mode), enable_hstore_index(other.enable_hstore_index),
+ columns(other.columns), hstore_columns(other.hstore_columns), copystr(other.copystr), table_space(other.table_space),
+ table_space_index(other.table_space_index), single_fmt(other.single_fmt), point_fmt(other.point_fmt), del_fmt(other.del_fmt)
+{
+ // if the other table has already started, then we want to execute
+ // the same stuff to get into the same state. but if it hasn't, then
+ // this would be premature.
+ if (other.sql_conn) {
+ connect();
+ //let postgres cache this query as it will presumably happen a lot
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("PREPARE get_wkt (" POSTGRES_OSMID_TYPE ") AS SELECT ST_AsText(way) FROM %1% WHERE osm_id = $1") % name).str());
+ //start the copy
+ begin();
+ pgsql_exec_simple(sql_conn, PGRES_COPY_IN, copystr);
+ copyMode = true;
+ }
+}
+
+table_t::~table_t()
+{
+ teardown();
+}
+
+std::string const& table_t::get_name() {
+ return name;
+}
+
+void table_t::teardown()
+{
+ if(sql_conn != NULL)
+ {
+ PQfinish(sql_conn);
+ sql_conn = NULL;
+ }
+}
+
+void table_t::begin()
+{
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, "BEGIN");
+}
+
+void table_t::commit()
+{
+ stop_copy();
+ fprintf(stderr, "Committing transaction for %s\n", name.c_str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, "COMMIT");
+}
+
+void table_t::connect()
+{
+ //connect
+ PGconn* _conn = PQconnectdb(conninfo.c_str());
+ if (PQstatus(_conn) != CONNECTION_OK)
+ throw std::runtime_error((fmt("Connection to database failed: %1%\n") % PQerrorMessage(_conn)).str());
+ sql_conn = _conn;
+ //let commits happen faster by delaying when they actually occur
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;");
+}
+
+void table_t::start()
+{
+ if(sql_conn)
+ throw std::runtime_error(name + " cannot start, its already started");
+
+ connect();
+ fprintf(stderr, "Setting up table: %s\n", name.c_str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, "SET client_min_messages = WARNING");
+ //we are making a new table
+ if (!append)
+ {
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("DROP TABLE IF EXISTS %1%") % name).str());
+ }//we are checking in append mode that the srid you specified matches whats already there
+ else
+ {
+ boost::shared_ptr<PGresult> res = pgsql_exec_simple(sql_conn, PGRES_TUPLES_OK, (fmt("SELECT srid FROM geometry_columns WHERE f_table_name='%1%';") % name).str());
+ if (!((PQntuples(res.get()) == 1) && (PQnfields(res.get()) == 1)))
+ throw std::runtime_error((fmt("Problem reading geometry information for table %1% - does it exist?\n") % name).str());
+ char* their_srid = PQgetvalue(res.get(), 0, 0);
+ if (srid.compare(their_srid) != 0)
+ throw std::runtime_error((fmt("SRID mismatch: cannot append to table %1% (SRID %2%) using selected SRID %3%\n") % name % their_srid % srid).str());
+ }
+
+ /* These _tmp tables can be left behind if we run out of disk space */
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("DROP TABLE IF EXISTS %1%_tmp") % name).str());
+
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, "RESET client_min_messages");
+ begin();
+
+ //making a new table
+ if (!append)
+ {
+ //define the new table
+ string sql = (fmt("CREATE TABLE %1% (osm_id %2%,") % name % POSTGRES_OSMID_TYPE).str();
+
+ //first with the regular columns
+ for(columns_t::const_iterator column = columns.begin(); column != columns.end(); ++column)
+ sql += (fmt("\"%1%\" %2%,") % column->first % column->second).str();
+
+ //then with the hstore columns
+ for(hstores_t::const_iterator hcolumn = hstore_columns.begin(); hcolumn != hstore_columns.end(); ++hcolumn)
+ sql += (fmt("\"%1%\" hstore,") % (*hcolumn)).str();
+
+ //add tags column
+ if (hstore_mode != HSTORE_NONE)
+ sql += "\"tags\" hstore)";
+ //or remove the last ", " from the end
+ else
+ sql[sql.length() - 1] = ')';
+
+ //add the main table space
+ if (table_space)
+ sql += " TABLESPACE " + table_space.get();
+
+ //create the table
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, sql);
+
+ //add some constraints
+ pgsql_exec_simple(sql_conn, PGRES_TUPLES_OK, (fmt("SELECT AddGeometryColumn('%1%', 'way', %2%, '%3%', 2 )") % name % srid % type).str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ALTER TABLE %1% ALTER COLUMN way SET NOT NULL") % name).str());
+
+ //slim mode needs this to be able to apply diffs
+ if (slim && !drop_temp) {
+ sql = (fmt("CREATE INDEX %1%_pkey ON %2% USING BTREE (osm_id)") % name % name).str();
+ if (table_space_index)
+ sql += " TABLESPACE " + table_space_index.get();
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, sql);
+ }
+
+ }//appending
+ else {
+ //check the columns against those in the existing table
+ boost::shared_ptr<PGresult> res = pgsql_exec_simple(sql_conn, PGRES_TUPLES_OK, (fmt("SELECT * FROM %1% LIMIT 0") % name).str());
+ for(columns_t::const_iterator column = columns.begin(); column != columns.end(); ++column)
+ {
+ if(PQfnumber(res.get(), ('"' + column->first + '"').c_str()) < 0)
+ {
+#if 0
+ throw std::runtime_error((fmt("Append failed. Column \"%1%\" is missing from \"%2%\"\n") % info.name % name).str());
+#else
+ fprintf(stderr, "%s", (fmt("Adding new column \"%1%\" to \"%2%\"\n") % column->first % name).str().c_str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ALTER TABLE %1% ADD COLUMN \"%2%\" %3%") % name % column->first % column->second).str());
+#endif
+ }
+ //Note: we do not verify the type or delete unused columns
+ }
+
+ //TODO: check over hstore columns
+
+ //TODO: change the type of the geometry column if needed - this can only change to a more permissive type
+ }
+
+ //let postgres cache this query as it will presumably happen a lot
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("PREPARE get_wkt (" POSTGRES_OSMID_TYPE ") AS SELECT ST_AsText(way) FROM %1% WHERE osm_id = $1") % name).str());
+
+ //generate column list for COPY
+ string cols = "osm_id,";
+ //first with the regular columns
+ for(columns_t::const_iterator column = columns.begin(); column != columns.end(); ++column)
+ cols += (fmt("\"%1%\",") % column->first).str();
+
+ //then with the hstore columns
+ for(hstores_t::const_iterator hcolumn = hstore_columns.begin(); hcolumn != hstore_columns.end(); ++hcolumn)
+ cols += (fmt("\"%1%\",") % (*hcolumn)).str();
+
+ //add tags column and geom column
+ if (hstore_mode != HSTORE_NONE)
+ cols += "tags,way";
+ //or just the geom column
+ else
+ cols += "way";
+
+ //get into copy mode
+ copystr = (fmt("COPY %1% (%2%) FROM STDIN") % name % cols).str();
+ pgsql_exec_simple(sql_conn, PGRES_COPY_IN, copystr);
+ copyMode = true;
+}
+
+void table_t::stop()
+{
+ stop_copy();
+ if (!append)
+ {
+ time_t start, end;
+ time(&start);
+
+ fprintf(stderr, "Sorting data and creating indexes for %s\n", name.c_str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ANALYZE %1%") % name).str());
+ fprintf(stderr, "Analyzing %s finished\n", name.c_str());
+
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE TABLE %1%_tmp %2% AS SELECT * FROM %3% ORDER BY way") % name % (table_space ? "TABLESPACE " + table_space.get() : "") % name).str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("DROP TABLE %1%") % name).str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ALTER TABLE %1%_tmp RENAME TO %2%") % name % name).str());
+ fprintf(stderr, "Copying %s to cluster by geometry finished\n", name.c_str());
+ fprintf(stderr, "Creating geometry index on %s\n", name.c_str());
+
+ // Use fillfactor 100 for un-updatable imports
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE INDEX %1%_index ON %2% USING GIST (way) %3% %4%") % name % name %
+ (slim && !drop_temp ? "" : "WITH (FILLFACTOR=100)") %
+ (table_space_index ? "TABLESPACE " + table_space_index.get() : "")).str());
+
+ /* slim mode needs this to be able to apply diffs */
+ if (slim && !drop_temp)
+ {
+ fprintf(stderr, "Creating osm_id index on %s\n", name.c_str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE INDEX %1%_pkey ON %2% USING BTREE (osm_id) %3%") % name % name %
+ (table_space_index ? "TABLESPACE " + table_space_index.get() : "")).str());
+ }
+ /* Create hstore index if selected */
+ if (enable_hstore_index) {
+ fprintf(stderr, "Creating hstore indexes on %s\n", name.c_str());
+ if (hstore_mode != HSTORE_NONE) {
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE INDEX %1%_tags_index ON %2% USING GIN (tags) %3%") % name % name %
+ (table_space_index ? "TABLESPACE " + table_space_index.get() : "")).str());
+ }
+ for(size_t i = 0; i < hstore_columns.size(); ++i) {
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE INDEX %1%_hstore_%2%_index ON %3% USING GIN (\"%4%\") %5%") % name % i % name % hstore_columns[i] %
+ (table_space_index ? "TABLESPACE " + table_space_index.get() : "")).str());
+ }
+ }
+ fprintf(stderr, "Creating indexes on %s finished\n", name.c_str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("GRANT SELECT ON %1% TO PUBLIC") % name).str());
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ANALYZE %1%") % name).str());
+ time(&end);
+ fprintf(stderr, "All indexes on %s created in %ds\n", name.c_str(), (int)(end - start));
+ }
+ teardown();
+
+ fprintf(stderr, "Completed %s\n", name.c_str());
+}
+
+void table_t::stop_copy()
+{
+ PGresult* res;
+ int stop;
+
+ //we werent copying anyway
+ if(!copyMode)
+ return;
+ //if there is stuff left over in the copy buffer send it offand copy it before we stop
+ else if(buffer.length() != 0)
+ {
+ pgsql_CopyData(name.c_str(), sql_conn, buffer.c_str());
+ buffer.clear();
+ };
+
+ //stop the copy
+ stop = PQputCopyEnd(sql_conn, NULL);
+ if (stop != 1)
+ throw std::runtime_error((fmt("stop COPY_END for %1% failed: %2%\n") % name % PQerrorMessage(sql_conn)).str());
+
+ //get the result
+ res = PQgetResult(sql_conn);
+ if (PQresultStatus(res) != PGRES_COMMAND_OK)
+ {
+ PQclear(res);
+ throw std::runtime_error((fmt("result COPY_END for %1% failed: %2%\n") % name % PQerrorMessage(sql_conn)).str());
+ }
+ PQclear(res);
+ copyMode = false;
+}
+
+void table_t::write_node(const osmid_t id, struct keyval *tags, double lat, double lon)
+{
+#ifdef FIXED_POINT
+ // guarantee that we use the same values as in the node cache
+ lon = util::fix_to_double(util::double_to_fix(lon, scale), scale);
+ lat = util::fix_to_double(util::double_to_fix(lat, scale), scale);
+#endif
+
+ write_wkt(id, tags, (point_fmt % lon % lat).str().c_str());
+}
+
+void table_t::delete_row(const osmid_t id)
+{
+ stop_copy();
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (del_fmt % name % id).str());
+}
+
+void table_t::write_wkt(const osmid_t id, struct keyval *tags, const char *wkt)
+{
+ //add the osm id
+ buffer.append((single_fmt % id).str());
+ buffer.push_back('\t');
+
+ //get the regular columns' values
+ write_columns(tags, buffer);
+
+ //get the hstore columns' values
+ write_hstore_columns(tags, buffer);
+
+ //get the key value pairs for the tags column
+ if (hstore_mode != HSTORE_NONE)
+ write_tags_column(tags, buffer);
+
+ //give the wkt an srid
+ buffer.append("SRID=");
+ buffer.append(srid);
+ buffer.push_back(';');
+ //add the wkt
+ buffer.append(wkt);
+ //we need \n because we are copying from stdin
+ buffer.push_back('\n');
+
+ //tell the db we are copying if for some reason we arent already
+ if (!copyMode)
+ {
+ pgsql_exec_simple(sql_conn, PGRES_COPY_IN, copystr);
+ copyMode = true;
+ }
+
+ //send all the data to postgres
+ if(buffer.length() > BUFFER_SEND_SIZE)
+ {
+ pgsql_CopyData(name.c_str(), sql_conn, buffer.c_str());
+ buffer.clear();
+ }
+}
+
+void table_t::write_columns(keyval *tags, string& values)
+{
+ //for each column
+ for(columns_t::const_iterator column = columns.begin(); column != columns.end(); ++column)
+ {
+ keyval *tag = NULL;
+ if ((tag = keyval::getTag(tags, column->first.c_str())))
+ {
+ escape_type(tag->value, column->second.c_str(), values);
+ //remember we already used this one so we cant use again later in the hstore column
+ if (hstore_mode == HSTORE_NORM)
+ tag->has_column = 1;
+ }
+ else
+ values.append("\\N");
+ values.push_back('\t');
+ }
+}
+
+void table_t::write_tags_column(keyval *tags, std::string& values)
+{
+ //iterate through the list of tags, first one is always null
+ bool added = false;
+ for (keyval* xtags = tags->next; xtags->key != NULL; xtags = xtags->next)
+ {
+ //skip z_order tag and keys which have their own column
+ if ((xtags->has_column) || (strcmp("z_order", xtags->key) == 0))
+ continue;
+
+ //hstore ASCII representation looks like "key"=>"value"
+ if(added)
+ values.push_back(',');
+ escape4hstore(xtags->key, values);
+ values.append("=>");
+ escape4hstore(xtags->value, values);
+
+ //we did at least one so we need commas from here on out
+ added = true;
+ }
+
+ //finish the hstore column by placing a TAB into the data stream
+ values.push_back('\t');
+}
+
+/* write an hstore column to the database */
+void table_t::write_hstore_columns(keyval *tags, std::string& values)
+{
+ //iterate over all configured hstore columns in the options
+ for(hstores_t::const_iterator hstore_column = hstore_columns.begin(); hstore_column != hstore_columns.end(); ++hstore_column)
+ {
+ //a clone of the tags pointer
+ bool added = false;
+
+ //iterate through the list of tags, first one is always null
+ for (keyval* xtags = tags->next; xtags->key != NULL; xtags = xtags->next)
+ {
+ //check if the tag's key starts with the name of the hstore column
+ if(hstore_column->find(xtags->key) == 0)
+ {
+ //generate the short key name, somehow pointer arithmetic works against this member of the keyval data structure...
+ char* shortkey = xtags->key + hstore_column->size();
+
+ //and pack the shortkey with its value into the hstore
+ //hstore ASCII representation looks like "key"=>"value"
+ if(added)
+ values.push_back(',');
+ escape4hstore(shortkey, values);
+ values.append("=>");
+ escape4hstore(xtags->value, values);
+
+ //we did at least one so we need commas from here on out
+ added = true;
+ }
+ }
+
+ //if you found not matching tags write a NULL
+ if(!added)
+ values.append("\\N");
+
+ //finish the column off with a tab
+ values.push_back('\t');
+ }
+}
+
+//create an escaped version of the string for hstore table insert
+void table_t::escape4hstore(const char *src, string& dst)
+{
+ dst.push_back('"');
+ for (size_t i = 0; i < strlen(src); ++i) {
+ switch (src[i]) {
+ case '\\':
+ dst.append("\\\\\\\\");
+ break;
+ case '"':
+ dst.append("\\\\\"");
+ break;
+ case '\t':
+ dst.append("\\\t");
+ break;
+ case '\r':
+ dst.append("\\\r");
+ break;
+ case '\n':
+ dst.append("\\\n");
+ break;
+ default:
+ dst.push_back(src[i]);
+ break;
+ }
+ }
+ dst.push_back('"');
+}
+
+/* Escape data appropriate to the type */
+void table_t::escape_type(const char *value, const char *type, string& dst) {
+
+ // For integers we take the first number, or the average if it's a-b
+ if (!strcmp(type, "int4")) {
+ int from, to;
+ int items = sscanf(value, "%d-%d", &from, &to);
+ if (items == 1)
+ dst.append((single_fmt % from).str());
+ else if (items == 2)
+ dst.append((single_fmt % ((from + to) / 2)).str());
+ else
+ dst.append("\\N");
+ }
+ /* try to "repair" real values as follows:
+ * assume "," to be a decimal mark which need to be replaced by "."
+ * like int4 take the first number, or the average if it's a-b
+ * assume SI unit (meters)
+ * convert feet to meters (1 foot = 0.3048 meters)
+ * reject anything else
+ */
+ else if (!strcmp(type, "real"))
+ {
+ string escaped(value);
+ std::replace(escaped.begin(), escaped.end(), ',', '.');
+
+ float from, to;
+ int items = sscanf(escaped.c_str(), "%f-%f", &from, &to);
+ if (items == 1)
+ {
+ if (escaped.size() > 1 && escaped.substr(escaped.size() - 2).compare("ft") == 0)
+ from *= 0.3048;
+ dst.append((single_fmt % from).str());
+ }
+ else if (items == 2)
+ {
+ if (escaped.size() > 1 && escaped.substr(escaped.size() - 2).compare("ft") == 0)
+ {
+ from *= 0.3048;
+ to *= 0.3048;
+ }
+ dst.append((single_fmt % ((from + to) / 2)).str());
+ }
+ else
+ dst.append("\\N");
+ }//just a string
+ else
+ escape(value, dst);
+}
+
+boost::shared_ptr<table_t::wkt_reader> table_t::get_wkt_reader(const osmid_t id)
+{
+ //cant get wkt using the prepared statement without stopping the copy first
+ stop_copy();
+
+ char const *paramValues[1];
+ char tmp[16];
+ snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id);
+ paramValues[0] = tmp;
+
+ //the prepared statement get_wkt will behave differently depending on the sql_conn
+ //each table has its own sql_connection with the get_way referring to the appropriate table
+ PGresult* res = pgsql_execPrepared(sql_conn, "get_wkt", 1, (const char * const *)paramValues, PGRES_TUPLES_OK);
+ return boost::shared_ptr<wkt_reader>(new wkt_reader(res));
+}
+
+table_t::wkt_reader::wkt_reader(PGresult* result):result(result), current(0)
+{
+ count = PQntuples(result);
+}
+
+table_t::wkt_reader::~wkt_reader()
+{
+ PQclear(result);
+}
+
+const char* table_t::wkt_reader::get_next()
+{
+ if(current < count)
+ return PQgetvalue(result, current++, 0);
+ return NULL;
+}
+
+size_t table_t::wkt_reader::get_count() const
+{
+ return count;
+}
+
+void table_t::wkt_reader::reset()
+{
+ //NOTE: PQgetvalue doc doesn't say if you can call it multiple times with the same row col
+ current = 0;
+}
diff --git a/table.hpp b/table.hpp
new file mode 100644
index 0000000..b08a3f7
--- /dev/null
+++ b/table.hpp
@@ -0,0 +1,90 @@
+#ifndef TABLE_H
+#define TABLE_H
+
+#include "keyvals.hpp"
+#include "pgsql.hpp"
+#include "osmtypes.hpp"
+
+#include <string>
+#include <vector>
+
+#include <boost/optional.hpp>
+#include <boost/format.hpp>
+
+typedef std::vector<std::string> hstores_t;
+typedef std::vector<std::pair<std::string, std::string> > columns_t;
+typedef boost::format fmt;
+
+class table_t
+{
+ public:
+ table_t(const std::string& conninfo, const std::string& name, const std::string& type, const columns_t& columns, const hstores_t& hstore_columns, const int srid,
+ const int scale, const bool append, const bool slim, const bool droptemp, const int hstore_mode, const bool enable_hstore_index,
+ const boost::optional<std::string>& table_space, const boost::optional<std::string>& table_space_index);
+ table_t(const table_t& other);
+ ~table_t();
+
+ void start();
+ void stop();
+
+ void begin();
+ void commit();
+
+ void write_wkt(const osmid_t id, struct keyval *tags, const char *wkt);
+ void write_node(const osmid_t id, struct keyval *tags, double lat, double lon);
+ void delete_row(const osmid_t id);
+
+ std::string const& get_name();
+
+ //interface from retrieving well known text geometry from the table
+ struct wkt_reader
+ {
+ friend class table_t;
+ public:
+ virtual ~wkt_reader();
+ const char* get_next();
+ size_t get_count() const;
+ void reset();
+ private:
+ wkt_reader(PGresult* result);
+ PGresult* result;
+ size_t count;
+ size_t current;
+ };
+ boost::shared_ptr<wkt_reader> get_wkt_reader(const osmid_t id);
+
+ protected:
+ void connect();
+ void stop_copy();
+ void teardown();
+
+ void write_columns(struct keyval *tags, std::string& values);
+ void write_tags_column(keyval *tags, std::string& values);
+ void write_hstore_columns(keyval *tags, std::string& values);
+
+ void escape4hstore(const char *src, std::string& dst);
+ void escape_type(const char *value, const char *type, std::string& dst);
+
+ std::string conninfo;
+ std::string name;
+ std::string type;
+ pg_conn *sql_conn;
+ bool copyMode;
+ std::string buffer;
+ std::string srid;
+ int scale;
+ bool append;
+ bool slim;
+ bool drop_temp;
+ int hstore_mode;
+ bool enable_hstore_index;
+ columns_t columns;
+ hstores_t hstore_columns;
+ std::string copystr;
+ boost::optional<std::string> table_space;
+ boost::optional<std::string> table_space_index;
+
+ fmt single_fmt, point_fmt, del_fmt;
+};
+
+#endif
diff --git a/taginfo.cpp b/taginfo.cpp
new file mode 100644
index 0000000..1f91472
--- /dev/null
+++ b/taginfo.cpp
@@ -0,0 +1,215 @@
+#include "taginfo_impl.hpp"
+#include "table.hpp"
+#include "util.hpp"
+
+#include <cassert>
+#include <cstring>
+#include <stdexcept>
+#include <boost/format.hpp>
+#include <errno.h>
+
+#ifdef _WIN32
+#ifndef strtok_r
+ #define strtok_r strtok_s
+#endif
+#endif
+
+/* NOTE: section below for flags genuinely is static and
+ * constant, so there's no need to hoist this into a per
+ * class variable. It doesn't get modified, so it's safe
+ * to share across threads and its lifetime is the whole
+ * program.
+ */
+struct flagsname {
+ flagsname(const char *name_, int flag_)
+ : name(strdup(name_)), flag(flag_) {
+ }
+ char *name;
+ int flag;
+};
+
+static const flagsname tagflags[] = {
+ flagsname("polygon", FLAG_POLYGON),
+ flagsname("linear", FLAG_LINEAR),
+ flagsname("nocache", FLAG_NOCACHE),
+ flagsname("delete", FLAG_DELETE),
+ flagsname("phstore", FLAG_PHSTORE)
+};
+#define NUM_FLAGS ((signed)(sizeof(tagflags) / sizeof(tagflags[0])))
+
+taginfo::taginfo()
+ : name(), type(), flags(0) {
+}
+
+taginfo::taginfo(const taginfo &other)
+ : name(other.name), type(other.type),
+ flags(other.flags) {
+}
+
+export_list::export_list()
+ : num_tables(0), exportList() {
+}
+
+void export_list::add(enum OsmType id, const taginfo &info) {
+ std::vector<taginfo> &infos = get(id);
+ infos.push_back(info);
+}
+
+std::vector<taginfo> &export_list::get(enum OsmType id) {
+ if (id >= num_tables) {
+ exportList.resize(id+1);
+ num_tables = id + 1;
+ }
+ return exportList[id];
+}
+
+const std::vector<taginfo> &export_list::get(enum OsmType id) const {
+ // this fakes as if we have infinite taginfo vectors, but
+ // means we don't actually have anything allocated unless
+ // the info object has been assigned.
+ static const std::vector<taginfo> empty;
+
+ if (id < num_tables) {
+ return exportList[id];
+ } else {
+ return empty;
+ }
+}
+
+columns_t export_list::normal_columns(enum OsmType id) const {
+ columns_t columns;
+ const std::vector<taginfo> &infos = get(id);
+ for(std::vector<taginfo>::const_iterator info = infos.begin(); info != infos.end(); ++info)
+ {
+ if( info->flags & FLAG_DELETE )
+ continue;
+ if( (info->flags & FLAG_PHSTORE ) == FLAG_PHSTORE)
+ continue;
+ columns.push_back(std::pair<std::string, std::string>(info->name, info->type));
+ }
+ return columns;
+}
+
+int parse_tag_flags(const char *flags_, int lineno) {
+ int temp_flags = 0;
+ char *str = NULL, *saveptr = NULL;
+ int i = 0;
+
+ // yuck! but strtok requires a non-const char * pointer, and i'm fairly sure it
+ // doesn't actually modify the string.
+ char *flags = const_cast<char *>(flags_);
+
+ //split the flags column on commas and keep track of which flags you've seen in a bit mask
+ for(str = strtok_r(flags, ",\r\n", &saveptr); str != NULL; str = strtok_r(NULL, ",\r\n", &saveptr))
+ {
+ for( i=0; i<NUM_FLAGS; i++ )
+ {
+ if( strcmp( tagflags[i].name, str ) == 0 )
+ {
+ temp_flags |= tagflags[i].flag;
+ break;
+ }
+ }
+ if( i == NUM_FLAGS )
+ fprintf( stderr, "Unknown flag '%s' line %d, ignored\n", str, lineno );
+ }
+
+ return temp_flags;
+}
+
+int read_style_file( const std::string &filename, export_list *exlist )
+{
+ FILE *in;
+ int lineno = 0;
+ int num_read = 0;
+ char osmtype[24];
+ char tag[64];
+ char datatype[24];
+ char flags[128];
+ char *str;
+ int fields;
+ struct taginfo temp;
+ char buffer[1024];
+ int enable_way_area = 1;
+
+ in = fopen( filename.c_str(), "rt" );
+ if( !in )
+ {
+ throw std::runtime_error((boost::format("Couldn't open style file '%1%': %2%")
+ % filename % strerror(errno)).str());
+ }
+
+ //for each line of the style file
+ while( fgets( buffer, sizeof(buffer), in) != NULL )
+ {
+ lineno++;
+
+ //find where a comment starts and terminate the string there
+ str = strchr( buffer, '#' );
+ if( str )
+ *str = '\0';
+
+ //grab the expected fields for this row
+ fields = sscanf( buffer, "%23s %63s %23s %127s", osmtype, tag, datatype, flags );
+ if( fields <= 0 ) /* Blank line */
+ continue;
+ if( fields < 3 )
+ {
+ fprintf( stderr, "Error reading style file line %d (fields=%d)\n", lineno, fields );
+ util::exit_nicely();
+ }
+
+ //place to keep info about this tag
+ temp.name.assign(tag);
+ temp.type.assign(datatype);
+ temp.flags = parse_tag_flags(flags, lineno);
+
+ if ((temp.flags != FLAG_DELETE) &&
+ ((temp.name.find('?') != std::string::npos) ||
+ (temp.name.find('*') != std::string::npos))) {
+ fprintf( stderr, "wildcard '%s' in non-delete style entry\n",temp.name.c_str());
+ util::exit_nicely();
+ }
+
+ if ((temp.name == "way_area") && (temp.flags==FLAG_DELETE)) {
+ enable_way_area=0;
+ }
+
+ /* printf("%s %s %d %d\n", temp.name, temp.type, temp.polygon, offset ); */
+ bool kept = false;
+
+ //keep this tag info if it applies to nodes
+ if( strstr( osmtype, "node" ) )
+ {
+ exlist->add(OSMTYPE_NODE, temp);
+ kept = true;
+ }
+
+ //keep this tag info if it applies to ways
+ if( strstr( osmtype, "way" ) )
+ {
+ exlist->add(OSMTYPE_WAY, temp);
+ kept = true;
+ }
+
+ //do we really want to completely quit on an unusable line?
+ if( !kept )
+ {
+ throw std::runtime_error((boost::format("Weird style line %1%:%2%")
+ % filename % lineno).str());
+ }
+ num_read++;
+ }
+
+
+ if (ferror(in)) {
+ throw std::runtime_error((boost::format("%1%: %2%")
+ % filename % strerror(errno)).str());
+ }
+ if (num_read == 0) {
+ throw std::runtime_error("Unable to parse any valid columns from "
+ "the style file. Aborting.");
+ }
+ fclose(in);
+ return enable_way_area;
+}
diff --git a/taginfo.hpp b/taginfo.hpp
new file mode 100644
index 0000000..9f6d87b
--- /dev/null
+++ b/taginfo.hpp
@@ -0,0 +1,10 @@
+#ifndef TAGINFO_HPP
+#define TAGINFO_HPP
+
+/* Table columns, representing key= tags */
+struct taginfo;
+
+/* list of exported tags */
+struct export_list;
+
+#endif /* TAGINFO_HPP */
diff --git a/taginfo_impl.hpp b/taginfo_impl.hpp
new file mode 100644
index 0000000..5910fac
--- /dev/null
+++ b/taginfo_impl.hpp
@@ -0,0 +1,50 @@
+#ifndef TAGINFO_IMPL_HPP
+#define TAGINFO_IMPL_HPP
+
+#include "taginfo.hpp"
+#include "osmtypes.hpp"
+#include <string>
+#include <vector>
+#include <utility>
+
+#define FLAG_POLYGON 1 /* For polygon table */
+#define FLAG_LINEAR 2 /* For lines table */
+#define FLAG_NOCACHE 4 /* Optimisation: don't bother remembering this one */
+#define FLAG_DELETE 8 /* These tags should be simply deleted on sight */
+#define FLAG_PHSTORE 17 /* polygons without own column but listed in hstore this implies FLAG_POLYGON */
+
+struct taginfo {
+ taginfo();
+ taginfo(const taginfo &);
+
+ std::string name, type;
+ int flags;
+};
+
+struct export_list {
+ export_list();
+
+ void add(enum OsmType id, const taginfo &info);
+ std::vector<taginfo> &get(enum OsmType id);
+ const std::vector<taginfo> &get(enum OsmType id) const;
+
+ std::vector<std::pair<std::string, std::string> > normal_columns(enum OsmType id) const;
+
+ int num_tables;
+ std::vector<std::vector<taginfo> > exportList; /* Indexed by enum OsmType */
+};
+
+/* Parse a comma or whitespace delimited list of tags to apply to
+ * a style file entry, returning the OR-ed set of flags.
+ */
+int parse_tag_flags(const char *flags, int lineno);
+
+/* Parse an osm2pgsql "pgsql" backend format style file, putting
+ * the results in the `exlist` argument.
+ *
+ * Returns 1 if the 'way_area' column should (implicitly) exist, or
+ * 0 if it should be suppressed.
+ */
+int read_style_file( const std::string &filename, export_list *exlist );
+
+#endif /* TAGINFO_IMPL_HPP */
diff --git a/tagtransform.c b/tagtransform.cpp
similarity index 53%
rename from tagtransform.c
rename to tagtransform.cpp
index 5263a2d..119f4f2 100644
--- a/tagtransform.c
+++ b/tagtransform.cpp
@@ -2,18 +2,18 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include "osmtypes.h"
-#include "keyvals.h"
-#include "tagtransform.h"
-#include "output-pgsql.h"
+#include <stdexcept>
+#include "osmtypes.hpp"
+#include "keyvals.hpp"
+#include "tagtransform.hpp"
+#include "output-pgsql.hpp"
+#include "options.hpp"
#include "config.h"
-#include "wildcmp.h"
+#include "wildcmp.hpp"
+#include "taginfo_impl.hpp"
-#ifdef HAVE_LUA
-static lua_State *L;
-#endif
-static struct {
+static const struct {
int offset;
const char *highway;
int roads;
@@ -36,20 +36,14 @@ static struct {
static const unsigned int nLayers = (sizeof(layers)/sizeof(*layers));
-const struct output_options *options;
-
-extern struct taginfo *exportList[4]; /* Indexed by enum table_id */
-extern int exportListCount[4];
-
-int transform_method = 0;
-
-static int add_z_order(struct keyval *tags, int *roads) {
- const char *layer = getItem(tags, "layer");
- const char *highway = getItem(tags, "highway");
- const char *bridge = getItem(tags, "bridge");
- const char *tunnel = getItem(tags, "tunnel");
- const char *railway = getItem(tags, "railway");
- const char *boundary = getItem(tags, "boundary");
+namespace {
+int add_z_order(keyval *tags, int *roads) {
+ const char *layer = keyval::getItem(tags, "layer");
+ const char *highway = keyval::getItem(tags, "highway");
+ const char *bridge = keyval::getItem(tags, "bridge");
+ const char *tunnel = keyval::getItem(tags, "tunnel");
+ const char *railway = keyval::getItem(tags, "railway");
+ const char *boundary = keyval::getItem(tags, "boundary");
int z_order = 0;
int l;
@@ -89,334 +83,54 @@ static int add_z_order(struct keyval *tags, int *roads) {
z_order -= 10;
snprintf(z, sizeof(z), "%d", z_order);
- addItem(tags, "z_order", z, 0);
+ keyval::addItem(tags, "z_order", z, 0);
return 0;
}
-static unsigned int tagtransform_lua_filter_basic_tags(enum OsmType type, struct keyval *tags, int * polygon, int * roads) {
-#ifdef HAVE_LUA
- int idx = 0;
- int filter;
- int count = 0;
- struct keyval *item;
- const char * key, * value;
-
- *polygon = 0; *roads = 0;
-
- switch (type) {
- case OSMTYPE_NODE: {
- lua_getglobal(L, "filter_tags_node");
- break;
- }
- case OSMTYPE_WAY: {
- lua_getglobal(L, "filter_tags_way");
- break;
- }
- case OSMTYPE_RELATION: {
- lua_getglobal(L, "filter_basic_tags_rel");
- break;
- }
- }
-
- lua_newtable(L); /* key value table */
-
- idx = 1;
- while( (item = popItem(tags)) != NULL ) {
- lua_pushstring(L, item->key);
- lua_pushstring(L, item->value);
- lua_rawset(L, -3);
- freeItem(item);
- count++;
- }
-
- //printf("C count %i\n", count);
- lua_pushinteger(L, count);
-
- if (lua_pcall(L,2,type == OSMTYPE_WAY ? 4 : 2,0)) {
- fprintf(stderr, "Failed to execute lua function for basic tag processing: %s\n", lua_tostring(L, -1));
- /* lua function failed */
- return 1;
- }
-
- if (type == OSMTYPE_WAY) {
- *roads = lua_tointeger(L, -1);
- lua_pop(L,1);
- *polygon = lua_tointeger(L, -1);
- lua_pop(L,1);
- }
-
- lua_pushnil(L);
- while (lua_next(L,-2) != 0) {
- key = lua_tostring(L,-2);
- value = lua_tostring(L,-1);
- addItem(tags, key, value, 0);
- lua_pop(L,1);
- }
-
- filter = lua_tointeger(L, -2);
+unsigned int c_filter_rel_member_tags(
+ keyval *rel_tags, const int member_count,
+ keyval *member_tags, const char * const *member_roles,
+ int * member_superseeded, int * make_boundary, int * make_polygon, int * roads,
+ const export_list *exlist, bool allow_typeless) {
- lua_pop(L,2);
-
- return filter;
-#else
- return 1;
-#endif
-}
-
-
-/* Go through the given tags and determine the union of flags. Also remove
- * any tags from the list that we don't know about */
-static unsigned int tagtransform_c_filter_basic_tags(enum OsmType type,
- struct keyval *tags, int *polygon, int * roads) {
- int i, filter = 1;
- int flags = 0;
- int add_area_tag = 0;
- enum OsmType export_type;
-
- const char *area;
- struct keyval *item;
- struct keyval temp;
- initList(&temp);
-
- if (type == OSMTYPE_RELATION) {export_type = OSMTYPE_WAY;} else {export_type = type;}
-
- /* We used to only go far enough to determine if it's a polygon or not, but now we go through and filter stuff we don't need */
- while ((item = popItem(tags)) != NULL ) {
- if (type == OSMTYPE_RELATION && !strcmp("type", item->key)) {
- pushItem(&temp, item);
- item = NULL;
- filter = 0;
- continue;
- }
- /* Allow named islands to appear as polygons */
- if (!strcmp("natural", item->key)
- && !strcmp("coastline", item->value)) {
- add_area_tag = 1;
- }
-
- /* Discard natural=coastline tags (we render these from a shapefile instead) */
- if (!options->keep_coastlines && !strcmp("natural", item->key)
- && !strcmp("coastline", item->value)) {
- freeItem(item);
- item = NULL;
- continue;
- }
-
- for (i = 0; i < exportListCount[export_type]; i++) {
- if (wildMatch(exportList[export_type][i].name, item->key)) {
- if (exportList[export_type][i].flags & FLAG_DELETE) {
- freeItem(item);
- item = NULL;
- break;
- }
-
- filter = 0;
- flags |= exportList[export_type][i].flags;
-
- pushItem(&temp, item);
- item = NULL;
- break;
- }
- }
-
- /** if tag not found in list of exports: */
- if (i == exportListCount[export_type]) {
- if (options->enable_hstore) {
- /* with hstore, copy all tags... */
- pushItem(&temp, item);
- /* ... but if hstore_match_only is set then don't take this
- as a reason for keeping the object */
- if (!options->hstore_match_only && strcmp("osm_uid", item->key)
- && strcmp("osm_user", item->key)
- && strcmp("osm_timestamp", item->key)
- && strcmp("osm_version", item->key)
- && strcmp("osm_changeset", item->key))
- filter = 0;
- } else if (options->n_hstore_columns) {
- /* does this column match any of the hstore column prefixes? */
- int j;
- for (j = 0; j < options->n_hstore_columns; j++) {
- char *pos = strstr(item->key, options->hstore_columns[j]);
- if (pos == item->key) {
- pushItem(&temp, item);
- /* ... but if hstore_match_only is set then don't take this
- as a reason for keeping the object */
- if (!options->hstore_match_only
- && strcmp("osm_uid", item->key)
- && strcmp("osm_user", item->key)
- && strcmp("osm_timestamp", item->key)
- && strcmp("osm_version", item->key)
- && strcmp("osm_changeset", item->key))
- filter = 0;
- break;
- }
- }
- /* if not, skip the tag */
- if (j == options->n_hstore_columns) {
- freeItem(item);
- }
- } else {
- freeItem(item);
- }
- item = NULL;
- }
- }
-
- /* Move from temp list back to original list */
- while ((item = popItem(&temp)) != NULL )
- pushItem(tags, item);
-
- *polygon = flags & FLAG_POLYGON;
-
- /* Special case allowing area= to override anything else */
- if ((area = getItem(tags, "area"))) {
- if (!strcmp(area, "yes") || !strcmp(area, "true") || !strcmp(area, "1"))
- *polygon = 1;
- else if (!strcmp(area, "no") || !strcmp(area, "false")
- || !strcmp(area, "0"))
- *polygon = 0;
- } else {
- /* If we need to force this as a polygon, append an area tag */
- if (add_area_tag) {
- addItem(tags, "area", "yes", 0);
- *polygon = 1;
- }
- }
-
- if (!filter && (type == OSMTYPE_WAY)) {
- add_z_order(tags,roads);
- }
-
- return filter;
-}
-
-
-static unsigned int tagtransform_lua_filter_rel_member_tags(struct keyval *rel_tags, int member_count,
- struct keyval *member_tags,const char **member_role,
- int * member_superseeded, int * make_boundary, int * make_polygon, int * roads) {
-#ifdef HAVE_LUA
-
- int i;
- int idx = 0;
- int filter;
- int count = 0;
- struct keyval *item;
- const char * key, * value;
-
- lua_getglobal(L, "filter_tags_relation_member");
-
- lua_newtable(L); /* relations key value table */
-
- idx = 1;
- while( (item = popItem(rel_tags)) != NULL ) {
- lua_pushstring(L, item->key);
- lua_pushstring(L, item->value);
- lua_rawset(L, -3);
- freeItem(item);
- count++;
- }
-
- lua_newtable(L); /* member tags table */
-
- for (i = 1; i <= member_count; i++) {
- lua_pushnumber(L, i);
- lua_newtable(L); /* member key value table */
- while( (item = popItem(&(member_tags[i - 1]))) != NULL ) {
- lua_pushstring(L, item->key);
- lua_pushstring(L, item->value);
- lua_rawset(L, -3);
- freeItem(item);
- count++;
- }
- lua_rawset(L, -3);
- }
-
- lua_newtable(L); /* member roles table */
-
- for (i = 0; i < member_count; i++) {
- lua_pushnumber(L, i + 1);
- lua_pushstring(L, member_role[i]);
- lua_rawset(L, -3);
- }
-
- lua_pushnumber(L, member_count);
-
- if (lua_pcall(L,4,6,0)) {
- fprintf(stderr, "Failed to execute lua function for relation tag processing: %s\n", lua_tostring(L, -1));
- /* lua function failed */
- return 1;
- }
-
- *roads = lua_tointeger(L, -1);
- lua_pop(L,1);
- *make_polygon = lua_tointeger(L, -1);
- lua_pop(L,1);
- *make_boundary = lua_tointeger(L,-1);
- lua_pop(L,1);
-
- lua_pushnil(L);
- for (i = 0; i < member_count; i++) {
- if (lua_next(L,-2)) {
- member_superseeded[i] = lua_tointeger(L,-1);
- lua_pop(L,1);
- } else {
- fprintf(stderr, "Failed to read member_superseeded from lua function\n");
- }
- }
- lua_pop(L,2);
-
- lua_pushnil(L);
- while (lua_next(L,-2) != 0) {
- key = lua_tostring(L,-2);
- value = lua_tostring(L,-1);
- addItem(rel_tags, key, value, 0);
- lua_pop(L,1);
- }
- lua_pop(L,1);
-
- filter = lua_tointeger(L, -1);
-
- lua_pop(L,1);
-
- return filter;
-#else
- return 1;
-#endif
-}
-
-
-static unsigned int tagtransform_c_filter_rel_member_tags(
- struct keyval *rel_tags, int member_count,
- struct keyval *member_tags, const char **member_role,
- int * member_superseeded, int * make_boundary, int * make_polygon, int * roads) {
char *type;
struct keyval tags, *p, *q, *qq, poly_tags;
int i, j;
int first_outerway, contains_tag;
- /* Get the type, if there's no type we don't care */
- type = getItem(rel_tags, "type");
- if (!type)
+ //if it has a relation figure out what kind it is
+ type = keyval::getItem(rel_tags, "type");
+ bool is_route = false, is_boundary = false, is_multipolygon = false;
+ if(type)
+ {
+ //what kind of relation is it
+ is_route = strcmp(type, "route") == 0;
+ is_boundary = strcmp(type, "boundary") == 0;
+ is_multipolygon = strcmp(type, "multipolygon") == 0;
+ }//you didnt have a type and it was required
+ else if(!allow_typeless)
+ {
return 1;
-
- initList(&tags);
- initList(&poly_tags);
+ }
/* Clone tags from relation */
+ keyval::initList(&tags);
+ keyval::initList(&poly_tags);
p = rel_tags->next;
while (p != rel_tags) {
- /* For routes, we convert name to route_name */
- if ((strcmp(type, "route") == 0) && (strcmp(p->key, "name") == 0))
- addItem(&tags, "route_name", p->value, 1);
- else if (strcmp(p->key, "type")) /* drop type= */
- addItem(&tags, p->key, p->value, 1);
+ //copy the name tag as "route_name"
+ if (is_route && (strcmp(p->key, "name") == 0))
+ keyval::addItem(&tags, "route_name", p->value, 1);
+ //copy all other tags except for "type"
+ else if (strcmp(p->key, "type"))
+ keyval::addItem(&tags, p->key, p->value, 1);
p = p->next;
}
- if (strcmp(type, "route") == 0) {
- const char *state = getItem(rel_tags, "state");
- const char *netw = getItem(rel_tags, "network");
+ if (is_route) {
+ const char *state = keyval::getItem(rel_tags, "state");
+ const char *netw = keyval::getItem(rel_tags, "network");
int networknr = -1;
if (state == NULL ) {
@@ -427,100 +141,99 @@ static unsigned int tagtransform_c_filter_rel_member_tags(
if (strcmp(netw, "lcn") == 0) {
networknr = 10;
if (strcmp(state, "alternate") == 0) {
- addItem(&tags, "lcn", "alternate", 1);
+ keyval::addItem(&tags, "lcn", "alternate", 1);
} else if (strcmp(state, "connection") == 0) {
- addItem(&tags, "lcn", "connection", 1);
+ keyval::addItem(&tags, "lcn", "connection", 1);
} else {
- addItem(&tags, "lcn", "yes", 1);
+ keyval::addItem(&tags, "lcn", "yes", 1);
}
} else if (strcmp(netw, "rcn") == 0) {
networknr = 11;
if (strcmp(state, "alternate") == 0) {
- addItem(&tags, "rcn", "alternate", 1);
+ keyval::addItem(&tags, "rcn", "alternate", 1);
} else if (strcmp(state, "connection") == 0) {
- addItem(&tags, "rcn", "connection", 1);
+ keyval::addItem(&tags, "rcn", "connection", 1);
} else {
- addItem(&tags, "rcn", "yes", 1);
+ keyval::addItem(&tags, "rcn", "yes", 1);
}
} else if (strcmp(netw, "ncn") == 0) {
networknr = 12;
if (strcmp(state, "alternate") == 0) {
- addItem(&tags, "ncn", "alternate", 1);
+ keyval::addItem(&tags, "ncn", "alternate", 1);
} else if (strcmp(state, "connection") == 0) {
- addItem(&tags, "ncn", "connection", 1);
+ keyval::addItem(&tags, "ncn", "connection", 1);
} else {
- addItem(&tags, "ncn", "yes", 1);
+ keyval::addItem(&tags, "ncn", "yes", 1);
}
} else if (strcmp(netw, "lwn") == 0) {
networknr = 20;
if (strcmp(state, "alternate") == 0) {
- addItem(&tags, "lwn", "alternate", 1);
+ keyval::addItem(&tags, "lwn", "alternate", 1);
} else if (strcmp(state, "connection") == 0) {
- addItem(&tags, "lwn", "connection", 1);
+ keyval::addItem(&tags, "lwn", "connection", 1);
} else {
- addItem(&tags, "lwn", "yes", 1);
+ keyval::addItem(&tags, "lwn", "yes", 1);
}
} else if (strcmp(netw, "rwn") == 0) {
networknr = 21;
if (strcmp(state, "alternate") == 0) {
- addItem(&tags, "rwn", "alternate", 1);
+ keyval::addItem(&tags, "rwn", "alternate", 1);
} else if (strcmp(state, "connection") == 0) {
- addItem(&tags, "rwn", "connection", 1);
+ keyval::addItem(&tags, "rwn", "connection", 1);
} else {
- addItem(&tags, "rwn", "yes", 1);
+ keyval::addItem(&tags, "rwn", "yes", 1);
}
} else if (strcmp(netw, "nwn") == 0) {
networknr = 22;
if (strcmp(state, "alternate") == 0) {
- addItem(&tags, "nwn", "alternate", 1);
+ keyval::addItem(&tags, "nwn", "alternate", 1);
} else if (strcmp(state, "connection") == 0) {
- addItem(&tags, "nwn", "connection", 1);
+ keyval::addItem(&tags, "nwn", "connection", 1);
} else {
- addItem(&tags, "nwn", "yes", 1);
+ keyval::addItem(&tags, "nwn", "yes", 1);
}
}
}
- if (getItem(rel_tags, "preferred_color") != NULL ) {
- const char *a = getItem(rel_tags, "preferred_color");
+ if (keyval::getItem(rel_tags, "preferred_color") != NULL ) {
+ const char *a = keyval::getItem(rel_tags, "preferred_color");
if (strcmp(a, "0") == 0 || strcmp(a, "1") == 0
|| strcmp(a, "2") == 0 || strcmp(a, "3") == 0
|| strcmp(a, "4") == 0) {
- addItem(&tags, "route_pref_color", a, 1);
+ keyval::addItem(&tags, "route_pref_color", a, 1);
} else {
- addItem(&tags, "route_pref_color", "0", 1);
+ keyval::addItem(&tags, "route_pref_color", "0", 1);
}
} else {
- addItem(&tags, "route_pref_color", "0", 1);
+ keyval::addItem(&tags, "route_pref_color", "0", 1);
}
- if (getItem(rel_tags, "ref") != NULL ) {
+ if (keyval::getItem(rel_tags, "ref") != NULL ) {
if (networknr == 10) {
- addItem(&tags, "lcn_ref", getItem(rel_tags, "ref"), 1);
+ keyval::addItem(&tags, "lcn_ref", keyval::getItem(rel_tags, "ref"), 1);
} else if (networknr == 11) {
- addItem(&tags, "rcn_ref", getItem(rel_tags, "ref"), 1);
+ keyval::addItem(&tags, "rcn_ref", keyval::getItem(rel_tags, "ref"), 1);
} else if (networknr == 12) {
- addItem(&tags, "ncn_ref", getItem(rel_tags, "ref"), 1);
+ keyval::addItem(&tags, "ncn_ref", keyval::getItem(rel_tags, "ref"), 1);
} else if (networknr == 20) {
- addItem(&tags, "lwn_ref", getItem(rel_tags, "ref"), 1);
+ keyval::addItem(&tags, "lwn_ref", keyval::getItem(rel_tags, "ref"), 1);
} else if (networknr == 21) {
- addItem(&tags, "rwn_ref", getItem(rel_tags, "ref"), 1);
+ keyval::addItem(&tags, "rwn_ref", keyval::getItem(rel_tags, "ref"), 1);
} else if (networknr == 22) {
- addItem(&tags, "nwn_ref", getItem(rel_tags, "ref"), 1);
+ keyval::addItem(&tags, "nwn_ref", keyval::getItem(rel_tags, "ref"), 1);
}
}
- } else if (strcmp(type, "boundary") == 0) {
+ } else if (is_boundary) {
/* Boundaries will get converted into multiple geometries:
- Linear features will end up in the line and roads tables (useful for admin boundaries)
- Polygon features also go into the polygon table (useful for national_forests)
The edges of the polygon also get treated as linear fetaures allowing these to be rendered seperately. */
*make_boundary = 1;
- } else if (strcmp(type, "multipolygon") == 0
- && getItem(&tags, "boundary")) {
+ } else if (is_multipolygon && keyval::getItem(&tags, "boundary")) {
/* Treat type=multipolygon exactly like type=boundary if it has a boundary tag. */
*make_boundary = 1;
- } else if (strcmp(type, "multipolygon") == 0) {
+ } else if (is_multipolygon) {
*make_polygon = 1;
/* Collect a list of polygon-like tags, these are used later to
@@ -528,12 +241,14 @@ static unsigned int tagtransform_c_filter_rel_member_tags(
p = tags.next;
while (p != &tags) {
if (!strcmp(p->key, "area")) {
- addItem(&poly_tags, p->key, p->value, 1);
+ keyval::addItem(&poly_tags, p->key, p->value, 1);
} else {
- for (i = 0; i < exportListCount[OSMTYPE_WAY]; i++) {
- if (strcmp(exportList[OSMTYPE_WAY][i].name, p->key) == 0) {
- if (exportList[OSMTYPE_WAY][i].flags & FLAG_POLYGON) {
- addItem(&poly_tags, p->key, p->value, 1);
+ const std::vector<taginfo> &infos = exlist->get(OSMTYPE_WAY);
+ for (i = 0; i < infos.size(); i++) {
+ const taginfo &info = infos[i];
+ if (strcmp(info.name.c_str(), p->key) == 0) {
+ if (info.flags & FLAG_POLYGON) {
+ keyval::addItem(&poly_tags, p->key, p->value, 1);
}
break;
}
@@ -544,32 +259,32 @@ static unsigned int tagtransform_c_filter_rel_member_tags(
/* Copy the tags from the outer way(s) if the relation is untagged (with
* respect to tags that influence its polygon nature. Tags like name or fixme should be fine*/
- if (!listHasData(&poly_tags)) {
+ if (!keyval::listHasData(&poly_tags)) {
first_outerway = 1;
for (i = 0; i < member_count; i++) {
- if (member_role[i] && !strcmp(member_role[i], "inner"))
+ if (member_roles[i] && !strcmp(member_roles[i], "inner"))
continue;
/* insert all tags of the first outerway to the potential list of copied tags. */
if (first_outerway) {
p = member_tags[i].next;
while (p != &(member_tags[i])) {
- addItem(&poly_tags, p->key, p->value, 1);
+ keyval::addItem(&poly_tags, p->key, p->value, 1);
p = p->next;
}
} else {
/* Check if all of the tags in the list of potential tags are present on this way,
otherwise remove from the list of potential tags. Tags need to be present on
all outer ways to be copied over to the relation */
- q = poly_tags.next;
+ q = poly_tags.next;
while (q != &poly_tags) {
- p = getTag(&(member_tags[i]), q->key);
+ p = keyval::getTag(&(member_tags[i]), q->key);
if ((p != NULL) && (strcmp(q->value, p->value) == 0)) {
q = q->next;
} else {
/* This tag is not present on all member outer ways, so don't copy it over to relation */
qq = q->next;
- removeTag(q);
+ keyval::removeTag(q);
q = qq;
}
}
@@ -577,19 +292,21 @@ static unsigned int tagtransform_c_filter_rel_member_tags(
first_outerway = 0;
}
/* Copy the list identified outer way tags over to the relation */
- q = poly_tags.next;
+ q = poly_tags.next;
while (q != &poly_tags) {
- addItem(&tags, q->key, q->value, 1);
+ keyval::addItem(&tags, q->key, q->value, 1);
q = q->next;
}
/* We need to re-check and only keep polygon tags in the list of polytags */
- q = poly_tags.next;
+ q = poly_tags.next;
while (q != &poly_tags) {
contains_tag = 0;
- for (j = 0; j < exportListCount[OSMTYPE_WAY]; j++) {
- if (strcmp(exportList[OSMTYPE_WAY][j].name, q->key) == 0) {
- if (exportList[OSMTYPE_WAY][j].flags & FLAG_POLYGON) {
+ const std::vector<taginfo> &infos = exlist->get(OSMTYPE_WAY);
+ for (j = 0; j < infos.size(); j++) {
+ const taginfo &info = infos[j];
+ if (strcmp(info.name.c_str(), q->key) == 0) {
+ if (info.flags & FLAG_POLYGON) {
contains_tag = 1;
break;
}
@@ -597,24 +314,24 @@ static unsigned int tagtransform_c_filter_rel_member_tags(
}
if (contains_tag == 0) {
qq = q->next;
- removeTag(q);
+ keyval::removeTag(q);
q = qq;
} else {
q = q->next;
}
}
}
- resetList(&poly_tags);
- } else {
+ keyval::resetList(&poly_tags);
+ } else if(!allow_typeless) {
/* Unknown type, just exit */
- resetList(&tags);
- resetList(&poly_tags);
+ keyval::resetList(&tags);
+ keyval::resetList(&poly_tags);
return 1;
}
- if (!listHasData(&tags)) {
- resetList(&tags);
- resetList(&poly_tags);
+ if (!keyval::listHasData(&tags)) {
+ keyval::resetList(&tags);
+ keyval::resetList(&poly_tags);
return 1;
}
@@ -626,10 +343,10 @@ static unsigned int tagtransform_c_filter_rel_member_tags(
int match = 1;
struct keyval *p = member_tags[i].next;
while (p != &(member_tags[i])) {
- const char *v = getItem(&tags, p->key);
+ const char *v = keyval::getItem(&tags, p->key);
if (!v || strcmp(v, p->value)) {
/* z_order and osm_ are automatically generated tags, so ignore them */
- if ((strcmp(p->key, "z_order") != 0) && (strcmp(p->key, "osm_user") != 0) &&
+ if ((strcmp(p->key, "z_order") != 0) && (strcmp(p->key, "osm_user") != 0) &&
(strcmp(p->key, "osm_version") != 0) && (strcmp(p->key, "osm_uid") != 0) &&
(strcmp(p->key, "osm_changeset")) && (strcmp(p->key, "osm_timestamp") != 0)) {
match = 0;
@@ -646,111 +363,403 @@ static unsigned int tagtransform_c_filter_rel_member_tags(
}
}
- resetList(rel_tags);
- cloneList(rel_tags, &tags);
- resetList(&tags);
+ keyval::resetList(rel_tags);
+ keyval::cloneList(rel_tags, &tags);
+ keyval::resetList(&tags);
add_z_order(rel_tags, roads);
return 0;
}
-static int tagtransform_lua_init() {
#ifdef HAVE_LUA
- L = luaL_newstate();
- luaL_openlibs(L);
- luaL_dofile(L, options->tag_transform_script);
+unsigned int lua_filter_rel_member_tags(lua_State* L, const char* rel_mem_func, keyval *rel_tags, const int member_count,
+ keyval *member_tags,const char * const * member_roles,
+ int * member_superseeded, int * make_boundary, int * make_polygon, int * roads) {
- lua_getglobal(L, "filter_tags_node");
- if (!lua_isfunction (L, -1)) {
- fprintf(stderr,"Tag transform style does not contain a function filter_tags_node\n");
- return 1;
+ int i;
+ int filter;
+ int count = 0;
+ struct keyval *item;
+ const char * key, * value;
+
+ lua_getglobal(L, rel_mem_func);
+
+ lua_newtable(L); /* relations key value table */
+
+ while( (item = keyval::popItem(rel_tags)) != NULL ) {
+ lua_pushstring(L, item->key);
+ lua_pushstring(L, item->value);
+ lua_rawset(L, -3);
+ keyval::freeItem(item);
+ count++;
}
- lua_pop(L,1);
- lua_getglobal(L, "filter_tags_way");
- if (!lua_isfunction (L, -1)) {
- fprintf(stderr,"Tag transform style does not contain a function filter_tags_way\n");
+ lua_newtable(L); /* member tags table */
+
+ for (i = 1; i <= member_count; i++) {
+ lua_pushnumber(L, i);
+ lua_newtable(L); /* member key value table */
+ while( (item = keyval::popItem(&(member_tags[i - 1]))) != NULL ) {
+ lua_pushstring(L, item->key);
+ lua_pushstring(L, item->value);
+ lua_rawset(L, -3);
+ keyval::freeItem(item);
+ count++;
+ }
+ lua_rawset(L, -3);
+ }
+
+ lua_newtable(L); /* member roles table */
+
+ for (i = 0; i < member_count; i++) {
+ lua_pushnumber(L, i + 1);
+ lua_pushstring(L, member_roles[i]);
+ lua_rawset(L, -3);
+ }
+
+ lua_pushnumber(L, member_count);
+
+ if (lua_pcall(L,4,6,0)) {
+ fprintf(stderr, "Failed to execute lua function for relation tag processing: %s\n", lua_tostring(L, -1));
+ /* lua function failed */
return 1;
}
+
+ *roads = lua_tointeger(L, -1);
+ lua_pop(L,1);
+ *make_polygon = lua_tointeger(L, -1);
+ lua_pop(L,1);
+ *make_boundary = lua_tointeger(L,-1);
lua_pop(L,1);
- lua_getglobal(L, "filter_basic_tags_rel");
- if (!lua_isfunction (L, -1)) {
- fprintf(stderr,"Tag transform style does not contain a function filter_basic_tags_rel\n");
- return 1;
+ lua_pushnil(L);
+ for (i = 0; i < member_count; i++) {
+ if (lua_next(L,-2)) {
+ member_superseeded[i] = lua_tointeger(L,-1);
+ lua_pop(L,1);
+ } else {
+ fprintf(stderr, "Failed to read member_superseeded from lua function\n");
+ }
}
+ lua_pop(L,2);
+
+ lua_pushnil(L);
+ while (lua_next(L,-2) != 0) {
+ key = lua_tostring(L,-2);
+ value = lua_tostring(L,-1);
+ keyval::addItem(rel_tags, key, value, 0);
+ lua_pop(L,1);
+ }
+ lua_pop(L,1);
+
+ filter = lua_tointeger(L, -1);
+
+ lua_pop(L,1);
+
+ return filter;
+}
- lua_getglobal(L, "filter_tags_relation_member");
+void check_lua_function_exists(lua_State *L, const std::string &func_name) {
+ lua_getglobal(L, func_name.c_str());
if (!lua_isfunction (L, -1)) {
- fprintf(stderr,"Tag transform style does not contain a function filter_tags_relation_member\n");
- return 1;
+ throw std::runtime_error((boost::format("Tag transform style does not contain a function %1%")
+ % func_name).str());
}
+ lua_pop(L,1);
+}
+#endif
+} // anonymous namespace
- return 0;
+tagtransform::tagtransform(const options_t *options_)
+ : options(options_), transform_method(options_->tag_transform_script)
+#ifdef HAVE_LUA
+ , L(NULL)
+ , m_node_func( options->tag_transform_node_func. get_value_or("filter_tags_node"))
+ , m_way_func( options->tag_transform_way_func. get_value_or("filter_tags_way"))
+ , m_rel_func( options->tag_transform_rel_func. get_value_or("filter_basic_tags_rel"))
+ , m_rel_mem_func(options->tag_transform_rel_mem_func.get_value_or("filter_tags_relation_member"))
+#endif /* HAVE_LUA */
+ {
+ if (transform_method) {
+ fprintf(stderr, "Using lua based tag processing pipeline with script %s\n", options->tag_transform_script->c_str());
+#ifdef HAVE_LUA
+ L = luaL_newstate();
+ luaL_openlibs(L);
+ luaL_dofile(L, options->tag_transform_script->c_str());
+
+ check_lua_function_exists(L, m_node_func);
+ check_lua_function_exists(L, m_way_func);
+ check_lua_function_exists(L, m_rel_func);
+ check_lua_function_exists(L, m_rel_mem_func);
#else
- fprintf(stderr,"Error: Could not init lua tag transform, as lua support was not compiled into this version\n");
- return 1;
+ throw std::runtime_error("Error: Could not init lua tag transform, as lua support was not compiled into this version");
#endif
+ } else {
+ fprintf(stderr, "Using built-in tag processing pipeline\n");
+ }
}
-void tagtransform_lua_shutdown() {
+tagtransform::~tagtransform() {
#ifdef HAVE_LUA
+ if (transform_method)
lua_close(L);
#endif
}
-unsigned int tagtransform_filter_node_tags(struct keyval *tags) {
+unsigned int tagtransform::filter_node_tags(struct keyval *tags, const export_list *exlist, bool strict) {
int poly, roads;
if (transform_method) {
- return tagtransform_lua_filter_basic_tags(OSMTYPE_NODE, tags, &poly, &roads);
+ return lua_filter_basic_tags(OSMTYPE_NODE, tags, &poly, &roads);
} else {
- return tagtransform_c_filter_basic_tags(OSMTYPE_NODE, tags, &poly, &roads);
+ return c_filter_basic_tags(OSMTYPE_NODE, tags, &poly, &roads, exlist, strict);
}
}
/*
* This function gets called twice during initial import per way. Once from add_way and once from out_way
*/
-unsigned int tagtransform_filter_way_tags(struct keyval *tags, int * polygon, int * roads) {
+unsigned int tagtransform::filter_way_tags(struct keyval *tags, int * polygon, int * roads,
+ const export_list *exlist, bool strict) {
if (transform_method) {
- return tagtransform_lua_filter_basic_tags(OSMTYPE_WAY, tags, polygon, roads);
+#ifdef HAVE_LUA
+ return lua_filter_basic_tags(OSMTYPE_WAY, tags, polygon, roads);
+#else
+ return 1;
+#endif
} else {
- return tagtransform_c_filter_basic_tags(OSMTYPE_WAY, tags, polygon, roads);
+ return c_filter_basic_tags(OSMTYPE_WAY, tags, polygon, roads, exlist, strict);
}
}
-unsigned int tagtransform_filter_rel_tags(struct keyval *tags) {
+unsigned int tagtransform::filter_rel_tags(struct keyval *tags, const export_list *exlist, bool strict) {
int poly, roads;
if (transform_method) {
- return tagtransform_lua_filter_basic_tags(OSMTYPE_RELATION, tags, &poly, &roads);
+ return lua_filter_basic_tags(OSMTYPE_RELATION, tags, &poly, &roads);
} else {
- return tagtransform_c_filter_basic_tags(OSMTYPE_RELATION, tags, &poly, &roads);
+ return c_filter_basic_tags(OSMTYPE_RELATION, tags, &poly, &roads, exlist, strict);
}
}
-unsigned int tagtransform_filter_rel_member_tags(struct keyval *rel_tags, int member_count, struct keyval *member_tags,const char **member_role, int * member_superseeded, int * make_boundary, int * make_polygon, int * roads) {
+unsigned int tagtransform::filter_rel_member_tags(struct keyval *rel_tags, int member_count, struct keyval *member_tags,const char * const * member_roles, int * member_superseeded, int * make_boundary, int * make_polygon, int * roads, const export_list *exlist, bool allow_typeless) {
if (transform_method) {
- return tagtransform_lua_filter_rel_member_tags(rel_tags, member_count, member_tags, member_role, member_superseeded, make_boundary, make_polygon, roads);
+#ifdef HAVE_LUA
+ return lua_filter_rel_member_tags(L, m_rel_mem_func.c_str(), rel_tags, member_count, member_tags, member_roles, member_superseeded, make_boundary, make_polygon, roads);
+#else
+ return 1;
+#endif
} else {
- return tagtransform_c_filter_rel_member_tags(rel_tags, member_count, member_tags, member_role, member_superseeded, make_boundary, make_polygon, roads);
+ return c_filter_rel_member_tags(rel_tags, member_count, member_tags, member_roles, member_superseeded, make_boundary, make_polygon, roads, exlist, allow_typeless);
}
}
-int tagtransform_init(const struct output_options *opts) {
- options = opts;
- if (opts->tag_transform_script) {
- transform_method = 1;
- fprintf(stderr, "Using lua based tag processing pipeline with script %s\n", opts->tag_transform_script);
- return tagtransform_lua_init();
- } else {
- transform_method = 0;
- fprintf(stderr, "Using built-in tag processing pipeline\n");
- return 0; //Nothing to initialise
+unsigned int tagtransform::lua_filter_basic_tags(const OsmType type, keyval *tags, int * polygon, int * roads) {
+#ifdef HAVE_LUA
+ int filter;
+ int count = 0;
+ struct keyval *item;
+ const char * key, * value;
+
+ *polygon = 0; *roads = 0;
+
+ switch (type) {
+ case OSMTYPE_NODE: {
+ lua_getglobal(L, m_node_func.c_str());
+ break;
+ }
+ case OSMTYPE_WAY: {
+ lua_getglobal(L, m_way_func.c_str());
+ break;
+ }
+ case OSMTYPE_RELATION: {
+ lua_getglobal(L, m_rel_func.c_str());
+ break;
+ }
+ }
+
+ lua_newtable(L); /* key value table */
+
+ while( (item = keyval::popItem(tags)) != NULL ) {
+ lua_pushstring(L, item->key);
+ lua_pushstring(L, item->value);
+ lua_rawset(L, -3);
+ keyval::freeItem(item);
+ count++;
+ }
+
+ //printf("C count %i\n", count);
+ lua_pushinteger(L, count);
+
+ if (lua_pcall(L,2,type == OSMTYPE_WAY ? 4 : 2,0)) {
+ fprintf(stderr, "Failed to execute lua function for basic tag processing: %s\n", lua_tostring(L, -1));
+ /* lua function failed */
+ return 1;
+ }
+
+ if (type == OSMTYPE_WAY) {
+ *roads = lua_tointeger(L, -1);
+ lua_pop(L,1);
+ *polygon = lua_tointeger(L, -1);
+ lua_pop(L,1);
+ }
+
+ lua_pushnil(L);
+ while (lua_next(L,-2) != 0) {
+ key = lua_tostring(L,-2);
+ value = lua_tostring(L,-1);
+ keyval::addItem(tags, key, value, 0);
+ lua_pop(L,1);
}
+
+ filter = lua_tointeger(L, -2);
+
+ lua_pop(L,2);
+
+ return filter;
+#else
+ return 1;
+#endif
}
-void tagtransform_shutdown() {
- if (transform_method)
- tagtransform_lua_shutdown();
+/* Go through the given tags and determine the union of flags. Also remove
+ * any tags from the list that we don't know about */
+unsigned int tagtransform::c_filter_basic_tags(
+ const OsmType type, keyval *tags, int *polygon, int * roads,
+ const export_list *exlist, bool strict) {
+
+ //assume we dont like this set of tags
+ int filter = 1;
+
+ int flags = 0;
+ int add_area_tag = 0;
+
+ //a place to keep the tags we like as we go
+ struct keyval temp;
+ keyval::initList(&temp);
+
+ enum OsmType export_type;
+ if (type == OSMTYPE_RELATION) {
+ export_type = OSMTYPE_WAY;
+ } else {
+ export_type = type;
+ }
+
+ /* We used to only go far enough to determine if it's a polygon or not, but now we go through and filter stuff we don't need */
+ //pop each tag off and keep it in the temp list if we like it
+ struct keyval *item;
+ while ((item = keyval::popItem(tags)) != NULL ) {
+ //if we want to do more than the export list says
+ if(!strict) {
+ if (type == OSMTYPE_RELATION && !strcmp("type", item->key)) {
+ keyval::pushItem(&temp, item);
+ item = NULL;
+ filter = 0;
+ continue;
+ }
+ /* Allow named islands to appear as polygons */
+ if (!strcmp("natural", item->key)
+ && !strcmp("coastline", item->value)) {
+ add_area_tag = 1;
+ }
+
+ /* Discard natural=coastline tags (we render these from a shapefile instead) */
+ if (!options->keep_coastlines && !strcmp("natural", item->key)
+ && !strcmp("coastline", item->value)) {
+ keyval::freeItem(item);
+ item = NULL;
+ continue;
+ }
+ }
+
+ //go through the actual tags found on the item and keep the ones in the export list
+ const std::vector<taginfo> &infos = exlist->get(export_type);
+ size_t i = 0;
+ for (; i < infos.size(); i++) {
+ const taginfo &info = infos[i];
+ if (wildMatch(info.name.c_str(), item->key)) {
+ if (info.flags & FLAG_DELETE) {
+ keyval::freeItem(item);
+ item = NULL;
+ break;
+ }
+
+ filter = 0;
+ flags |= info.flags;
+
+ keyval::pushItem(&temp, item);
+ item = NULL;
+ break;
+ }
+ }
+
+ //if we didnt find any tags that we wanted to export and we aren't strictly adhering to the list
+ if (i == infos.size() && !strict) {
+ if (options->hstore_mode != HSTORE_NONE) {
+ /* with hstore, copy all tags... */
+ keyval::pushItem(&temp, item);
+ /* ... but if hstore_match_only is set then don't take this
+ as a reason for keeping the object */
+ if (!options->hstore_match_only && strcmp("osm_uid", item->key)
+ && strcmp("osm_user", item->key)
+ && strcmp("osm_timestamp", item->key)
+ && strcmp("osm_version", item->key)
+ && strcmp("osm_changeset", item->key))
+ filter = 0;
+ } else if (options->hstore_columns.size() > 0) {
+ /* does this column match any of the hstore column prefixes? */
+ size_t j = 0;
+ for(; j < options->hstore_columns.size(); ++j) {
+ char *pos = strstr(item->key, options->hstore_columns[j].c_str());
+ if (pos == item->key) {
+ keyval::pushItem(&temp, item);
+ /* ... but if hstore_match_only is set then don't take this
+ as a reason for keeping the object */
+ if (!options->hstore_match_only
+ && strcmp("osm_uid", item->key)
+ && strcmp("osm_user", item->key)
+ && strcmp("osm_timestamp", item->key)
+ && strcmp("osm_version", item->key)
+ && strcmp("osm_changeset", item->key))
+ filter = 0;
+ break;
+ }
+ }
+ /* if not, skip the tag */
+ if (j == options->hstore_columns.size()) {
+ keyval::freeItem(item);
+ }
+ } else {
+ keyval::freeItem(item);
+ }
+ item = NULL;
+ }
+ }
+
+ /* Move from temp list back to original list */
+ while ((item = keyval::popItem(&temp)) != NULL )
+ keyval::pushItem(tags, item);
+
+ *polygon = flags & FLAG_POLYGON;
+
+ /* Special case allowing area= to override anything else */
+ const char *area;
+ if ((area = keyval::getItem(tags, "area"))) {
+ if (!strcmp(area, "yes") || !strcmp(area, "true") || !strcmp(area, "1"))
+ *polygon = 1;
+ else if (!strcmp(area, "no") || !strcmp(area, "false")
+ || !strcmp(area, "0"))
+ *polygon = 0;
+ } else {
+ /* If we need to force this as a polygon, append an area tag */
+ if (add_area_tag) {
+ keyval::addItem(tags, "area", "yes", 0);
+ *polygon = 1;
+ }
+ }
+
+ if (!filter && (type == OSMTYPE_WAY)) {
+ add_z_order(tags,roads);
+ }
+
+ return filter;
}
diff --git a/tagtransform.h b/tagtransform.h
deleted file mode 100644
index 3fe1252..0000000
--- a/tagtransform.h
+++ /dev/null
@@ -1,29 +0,0 @@
-
-#ifndef TAGTRANSFORM_H
-#define TAGTRANSFORM_H
-
-#ifdef HAVE_LUA
-#include <lua.h>
-#include <lualib.h>
-#include <lauxlib.h>
-#endif
-#include "output.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-
-unsigned int tagtransform_filter_node_tags(struct keyval *tags);
-unsigned int tagtransform_filter_way_tags(struct keyval *tags, int * polygon, int * roads);
-unsigned int tagtransform_filter_rel_tags(struct keyval *tags);
-unsigned int tagtransform_filter_rel_member_tags(struct keyval *rel_tags, int member_count, struct keyval *member_tags,const char **member_role, int * member_superseeded, int * make_boundary, int * make_polygon, int * roads);
-
-int tagtransform_init(const struct output_options *options);
-void tagtransform_shutdown();
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif //TAGTRANSFORM_H
diff --git a/tagtransform.hpp b/tagtransform.hpp
new file mode 100644
index 0000000..7ad1b7b
--- /dev/null
+++ b/tagtransform.hpp
@@ -0,0 +1,45 @@
+
+#ifndef TAGTRANSFORM_H
+#define TAGTRANSFORM_H
+
+#include "output.hpp"
+#include "taginfo.hpp"
+
+#ifdef HAVE_LUA
+extern "C" {
+ #include <lua.h>
+ #include <lualib.h>
+ #include <lauxlib.h>
+}
+#endif
+
+
+
+class tagtransform {
+public:
+ tagtransform(const options_t *options_);
+ ~tagtransform();
+
+ unsigned int filter_node_tags(keyval *tags, const export_list *exlist, bool strict = false);
+ unsigned int filter_way_tags(keyval *tags, int * polygon, int * roads, const export_list *exlist, bool strict = false);
+ unsigned int filter_rel_tags(keyval *tags, const export_list *exlist, bool strict = false);
+ unsigned int filter_rel_member_tags(keyval *rel_tags, int member_count,
+ keyval *member_tags, const char * const * member_roles, int * member_superseeded,
+ int * make_boundary, int * make_polygon, int * roads, const export_list *exlist,
+ bool allow_typeless = false);
+
+private:
+ unsigned int lua_filter_basic_tags(const OsmType type, keyval *tags, int * polygon, int * roads);
+ unsigned int c_filter_basic_tags(const OsmType type, keyval *tags, int *polygon, int * roads,
+ const export_list *exlist, bool strict);
+
+ const options_t* options;
+ const bool transform_method;
+#ifdef HAVE_LUA
+ lua_State *L;
+ const std::string m_node_func, m_way_func, m_rel_func, m_rel_mem_func;
+#endif
+
+};
+
+#endif //TAGTRANSFORM_H
diff --git a/tests/common-pg.cpp b/tests/common-pg.cpp
new file mode 100644
index 0000000..48f5633
--- /dev/null
+++ b/tests/common-pg.cpp
@@ -0,0 +1,173 @@
+#include "common-pg.hpp"
+
+#include <sstream>
+#include <cstdarg>
+#include <unistd.h>
+
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/make_shared.hpp>
+#include <boost/format.hpp>
+#include <boost/enable_shared_from_this.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
+#ifdef _MSC_VER
+#include <windows.h>
+#include <process.h>
+#define getpid _getpid
+#define sleep Sleep
+#endif
+
+namespace fs = boost::filesystem;
+
+namespace pg {
+
+conn_ptr conn::connect(const std::string &conninfo) {
+ return boost::shared_ptr<conn>(new conn(conninfo));
+}
+
+result_ptr conn::exec(const std::string &query) {
+ return boost::make_shared<result>(shared_from_this(), query);
+}
+
+result_ptr conn::exec(const boost::format &fmt) {
+ return exec(fmt.str());
+}
+
+PGconn *conn::get() { return m_conn; }
+
+conn::~conn() {
+ if (m_conn != NULL) {
+ PQfinish(m_conn);
+ m_conn = NULL;
+ }
+}
+
+conn::conn(const std::string &conninfo)
+ : m_conn(PQconnectdb(conninfo.c_str())) {
+ if (PQstatus(m_conn) != CONNECTION_OK) {
+ std::ostringstream out;
+ out << "Could not connect to database \"" << conninfo
+ << "\" because: " << PQerrorMessage(m_conn);
+ PQfinish(m_conn);
+ m_conn = NULL;
+ throw std::runtime_error(out.str());
+ }
+}
+
+result::result(conn_ptr conn, const std::string &query)
+ : m_conn(conn), m_result(PQexec(conn->get(), query.c_str())) {
+ if (m_result == NULL) {
+ throw std::runtime_error((boost::format("Unable to run query \"%1%\": NULL result")
+ % query).str());
+ }
+}
+
+PGresult *result::get() { return m_result; }
+
+result::~result() {
+ if (m_result != NULL) {
+ PQclear(m_result);
+ m_result = NULL;
+ }
+}
+
+tempdb::tempdb()
+ : m_conn(conn::connect("dbname=postgres")) {
+ result_ptr res = m_conn->exec("SELECT spcname FROM pg_tablespace WHERE "
+ "spcname = 'tablespacetest'");
+
+ if ((PQresultStatus(res->get()) != PGRES_TUPLES_OK) ||
+ (PQntuples(res->get()) != 1)) {
+ std::ostringstream out;
+ out << "The test needs a temporary tablespace to run in, but it does not "
+ << "exist. Please create the temporary tablespace. On Linux, you can "
+ << "do this by running:\n"
+ << " sudo mkdir -p /tmp/psql-tablespace\n"
+ << " sudo /bin/chown postgres.postgres /tmp/psql-tablespace\n"
+ << " psql -c \"CREATE TABLESPACE tablespacetest LOCATION "
+ << "'/tmp/psql-tablespace'\" postgres\n";
+ throw std::runtime_error(out.str());
+ }
+
+ m_db_name = (boost::format("osm2pgsql-test-%1%-%2%") % getpid() % time(NULL)).str();
+ m_conn->exec(boost::format("DROP DATABASE IF EXISTS \"%1%\"") % m_db_name);
+ //tests can be run concurrently which means that this query can collide with other similar ones
+ //so we implement a simple retry here to get around the case that they do collide if we dont
+ //we often fail due to both trying to access template1 at the same time
+ size_t retries = 0;
+ ExecStatusType status = PGRES_FATAL_ERROR;
+ while(status != PGRES_COMMAND_OK && retries++ < 20)
+ {
+ sleep(1);
+ res = m_conn->exec(boost::format("CREATE DATABASE \"%1%\" WITH ENCODING 'UTF8'") % m_db_name);
+ status = PQresultStatus(res->get());
+ }
+ if (PQresultStatus(res->get()) != PGRES_COMMAND_OK) {
+ throw std::runtime_error((boost::format("Could not create a database: %1%")
+ % PQresultErrorMessage(res->get())).str());
+ }
+
+ m_conninfo = (boost::format("dbname=%1%") % m_db_name).str();
+ conn_ptr db = conn::connect(m_conninfo);
+
+ setup_extension(db, "postgis", "postgis-1.5/postgis.sql", "postgis-1.5/spatial_ref_sys.sql", NULL);
+ setup_extension(db, "hstore", NULL);
+}
+
+tempdb::~tempdb() {
+ if (m_conn) {
+ m_conn->exec(boost::format("DROP DATABASE IF EXISTS \"%1%\"") % m_db_name);
+ }
+}
+
+const std::string &tempdb::conninfo() const {
+ return m_conninfo;
+}
+
+void tempdb::setup_extension(conn_ptr db, const std::string &extension, ...) {
+ // first, try the new way of setting up extensions
+ result_ptr res = db->exec(boost::format("CREATE EXTENSION %1%") % extension);
+ if (PQresultStatus(res->get()) != PGRES_COMMAND_OK) {
+ // if that fails, then fall back to trying to find the files on
+ // the filesystem to load to create the extension.
+ res = db->exec("select regexp_replace(split_part(version(),' ',2),'\\.[0-9]*$','');");
+
+ if ((PQresultStatus(res->get()) != PGRES_TUPLES_OK) ||
+ (PQntuples(res->get()) != 1)) {
+ throw std::runtime_error("Unable to determine PostgreSQL version.");
+ }
+ std::string pg_version(PQgetvalue(res->get(), 0, 0));
+
+ // Guess the directory from the postgres version.
+ // TODO: make the contribdir configurable. Probably
+ // only works on Debian-based distributions at the moment.
+ fs::path contribdir = fs::path("/usr/share/postgresql/") / pg_version / fs::path("contrib");
+ va_list ap;
+ va_start(ap, extension);
+ const char *str = NULL;
+ while ((str = va_arg(ap, const char *)) != NULL) {
+ fs::path sql_file = contribdir / fs::path(str);
+ if (fs::exists(sql_file) && fs::is_regular_file(sql_file)) {
+ size_t size = fs::file_size(sql_file);
+ std::string sql(size + 1, '\0');
+ fs::ifstream in(sql_file);
+ in.read(&sql[0], size);
+ if (in.fail() || in.bad()) {
+ throw std::runtime_error((boost::format("Unable to read file %1% while trying to "
+ "load extension \"%2%\".")
+ % sql_file % extension).str());
+ }
+ res = db->exec(sql);
+ if (PQresultStatus(res->get()) != PGRES_COMMAND_OK) {
+ throw std::runtime_error((boost::format("Could not load extension \"%1%\": %2%")
+ % extension
+ % PQresultErrorMessage(res->get())).str());
+ }
+ }
+ }
+ va_end(ap);
+ }
+}
+} // namespace pg
diff --git a/tests/common-pg.hpp b/tests/common-pg.hpp
new file mode 100644
index 0000000..bb3cb08
--- /dev/null
+++ b/tests/common-pg.hpp
@@ -0,0 +1,69 @@
+#ifndef COMMON_PG_HPP
+#define COMMON_PG_HPP
+
+#include <string>
+#include <boost/shared_ptr.hpp>
+#include <boost/format.hpp>
+#include <boost/noncopyable.hpp>
+#include <boost/enable_shared_from_this.hpp>
+
+#include <libpq-fe.h>
+
+/* Some RAII objects to make writing stuff that needs a temporary database
+ * easier, and to keep track of and free connections and results objects.
+ */
+namespace pg {
+
+struct conn;
+struct result;
+
+typedef boost::shared_ptr<conn> conn_ptr;
+typedef boost::shared_ptr<result> result_ptr;
+
+struct conn
+ : public boost::noncopyable,
+ public boost::enable_shared_from_this<conn> {
+
+ static conn_ptr connect(const std::string &conninfo);
+ result_ptr exec(const std::string &query);
+ result_ptr exec(const boost::format &fmt);
+ PGconn *get();
+
+ ~conn();
+
+private:
+ conn(const std::string &conninfo);
+
+ PGconn *m_conn;
+};
+
+struct result
+ : public boost::noncopyable {
+ result(conn_ptr conn, const std::string &query);
+ PGresult *get();
+ ~result();
+
+private:
+ conn_ptr m_conn;
+ PGresult *m_result;
+};
+
+struct tempdb
+ : public boost::noncopyable {
+
+ tempdb();
+ ~tempdb();
+
+ const std::string &conninfo() const;
+
+private:
+ void setup_extension(conn_ptr db, const std::string &extension, ...);
+
+ conn_ptr m_conn;
+ std::string m_db_name;
+ std::string m_conninfo;
+};
+
+} // namespace pg
+
+#endif /* COMMON_PG_HPP */
diff --git a/tests/middle-tests.cpp b/tests/middle-tests.cpp
new file mode 100644
index 0000000..dc0df94
--- /dev/null
+++ b/tests/middle-tests.cpp
@@ -0,0 +1,176 @@
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <cassert>
+#include <list>
+
+#include "osmtypes.hpp"
+#include "keyvals.hpp"
+#include "tests/middle-tests.hpp"
+
+int test_node_set(middle_t *mid)
+{
+ osmid_t id = 1234;
+ double lat = 12.3456789;
+ double lon = 98.7654321;
+ struct keyval tags;
+ struct osmNode node;
+ int status = 0;
+
+ // set the node
+ status = mid->nodes_set(id, lat, lon, &tags);
+ if (status != 0) { std::cerr << "ERROR: Unable to set node.\n"; return 1; }
+
+ // get it back
+ int count = mid->nodes_get_list(&node, &id, 1);
+ if (count != 1) { std::cerr << "ERROR: Unable to get node list.\n"; return 1; }
+
+ // check that it's the same
+ if (node.lon != lon) {
+ std::cerr << "ERROR: Node should have lon=" << lon << ", but got back "
+ << node.lon << " from middle.\n";
+ return 1;
+ }
+ if (node.lat != lat) {
+ std::cerr << "ERROR: Node should have lat=" << lat << ", but got back "
+ << node.lat << " from middle.\n";
+ return 1;
+ }
+
+ // clean up for next test
+ if (dynamic_cast<slim_middle_t *>(mid)) {
+ dynamic_cast<slim_middle_t *>(mid)->nodes_delete(id);
+ }
+
+ keyval::resetList(&tags);
+
+ return 0;
+}
+
+struct test_pending_processor : public middle_t::pending_processor {
+ test_pending_processor(): pending_ways(), pending_rels() {}
+ virtual ~test_pending_processor() {}
+ virtual void enqueue_ways(osmid_t id) {
+ pending_ways.push_back(id);
+ }
+ virtual void process_ways() {
+ pending_ways.clear();
+ }
+ virtual void enqueue_relations(osmid_t id) {
+ pending_rels.push_back(id);
+ }
+ virtual void process_relations() {
+ pending_rels.clear();
+ }
+ virtual int thread_count() {
+ return 0;
+ }
+ virtual int size() {
+ return pending_ways.size() + pending_rels.size();
+ }
+ std::list<osmid_t> pending_ways;
+ std::list<osmid_t> pending_rels;
+};
+
+int test_way_set(middle_t *mid)
+{
+ osmid_t way_id = 1;
+ double lat = 12.3456789;
+ double lon = 98.7654321;
+ struct keyval tags[2]; /* <-- this is needed because the ways_get_list method calls
+ * keyval::initList() on the `count + 1`th tags element. */
+ struct osmNode *node_ptr = NULL;
+ osmid_t way_ids_ptr;
+ int node_count = 0;
+ int status = 0;
+ osmid_t nds[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
+ const int nd_count = ((sizeof nds) / (sizeof nds[0]));
+
+ // set the nodes
+ for (int i = 0; i < nd_count; ++i) {
+ status = mid->nodes_set(nds[i], lat, lon, &tags[0]);
+ if (status != 0) { std::cerr << "ERROR: Unable to set node " << nds[i] << ".\n"; return 1; }
+ }
+
+ // set the way
+ status = mid->ways_set(way_id, nds, nd_count, &tags[0]);
+ if (status != 0) { std::cerr << "ERROR: Unable to set way.\n"; return 1; }
+
+ // commit the setup data
+ mid->commit();
+
+ // get it back
+ int way_count = mid->ways_get_list(&way_id, 1, &way_ids_ptr, &tags[0], &node_ptr, &node_count);
+ if (way_count != 1) { std::cerr << "ERROR: Unable to get way list.\n"; return 1; }
+
+ // check that it's the same
+ if (node_count != nd_count) {
+ std::cerr << "ERROR: Way should have " << nd_count << " nodes, but got back "
+ << node_count << " from middle.\n";
+ return 1;
+ }
+ if (way_ids_ptr != way_id) {
+ std::cerr << "ERROR: Way should have id=" << way_id << ", but got back "
+ << way_ids_ptr << " from middle.\n";
+ return 1;
+ }
+ for (int i = 0; i < nd_count; ++i) {
+ if (node_ptr[i].lon != lon) {
+ std::cerr << "ERROR: Way node should have lon=" << lon << ", but got back "
+ << node_ptr[i].lon << " from middle.\n";
+ return 1;
+ }
+ if (node_ptr[i].lat != lat) {
+ std::cerr << "ERROR: Way node should have lat=" << lat << ", but got back "
+ << node_ptr[i].lat << " from middle.\n";
+ return 1;
+ }
+ }
+
+ // the way we just inserted should not be pending
+ test_pending_processor tpp;
+ mid->iterate_ways(tpp);
+ if (mid->pending_count() != 0) {
+ std::cerr << "ERROR: Was expecting no pending ways, but got "
+ << mid->pending_count() << " from middle.\n";
+ return 1;
+ }
+
+ // some middles don't support changing the nodes - they
+ // don't have diff update ability. here, we will just
+ // skip the test for that.
+ if (dynamic_cast<slim_middle_t *>(mid)) {
+ slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid);
+
+ // finally, try touching a node on a non-pending way. that should
+ // make it become pending. we just checked that the way is not
+ // pending, so any change must be due to the node changing.
+ status = slim->node_changed(nds[0]);
+ if (status != 0) { std::cerr << "ERROR: Unable to reset node.\n"; return 1; }
+ slim->iterate_ways(tpp);
+ if (slim->pending_count() != 1) {
+ std::cerr << "ERROR: Was expecting a single pending way from node update, but got "
+ << slim->pending_count() << " from middle.\n";
+ return 1;
+ }
+ }
+
+ keyval::resetList(&tags[0]);
+ free(node_ptr);
+
+ // clean up for next test
+ if (dynamic_cast<slim_middle_t *>(mid)) {
+ slim_middle_t *slim = dynamic_cast<slim_middle_t *>(mid);
+
+ for (int i = 0; i < nd_count; ++i) {
+ slim->nodes_delete(nds[i]);
+ }
+ slim->ways_delete(way_id);
+ }
+
+ // commit the torn-down data
+ mid->commit();
+
+ return 0;
+}
diff --git a/tests/middle-tests.hpp b/tests/middle-tests.hpp
new file mode 100644
index 0000000..0193912
--- /dev/null
+++ b/tests/middle-tests.hpp
@@ -0,0 +1,13 @@
+#ifndef TESTS_MIDDLE_TEST_HPP
+#define TESTS_MIDDLE_TEST_HPP
+
+#include "middle.hpp"
+
+// tests that a single node can be set and retrieved. returns 0 on success.
+int test_node_set(middle_t *mid);
+
+// tests that a single way and supporting nodes can be set and retrieved.
+// returns 0 on success.
+int test_way_set(middle_t *mid);
+
+#endif /* TESTS_MIDDLE_TEST_HPP */
diff --git a/tests/regression-test.py b/tests/regression-test.py
index e9c41e3..418a09d 100755
--- a/tests/regression-test.py
+++ b/tests/regression-test.py
@@ -3,7 +3,7 @@
import unittest
import psycopg2
import os
-from pwd import getpwnam
+import fnmatch
import subprocess
full_import_file="tests/liechtenstein-2013-08-03.osm.pbf"
@@ -483,6 +483,22 @@ class BasicGazetteerTestCase(BaseGazetteerTestCase):
self.executeStatements(self.postDiffStatements)
+#****************************************************************
+#****************************************************************
+def findContribSql(filename):
+
+ # Try to get base dir for postgres contrib
+ try:
+ postgis_base_dir = os.popen('pg_config | grep -m 1 "^INCLUDEDIR ="').read().strip().split(' ')[2].split('/include')[0]
+ except:
+ postgis_base_dir = '/usr'
+
+ # Search for the actual sql file
+ for root, dirs, files in os.walk(postgis_base_dir):
+ if 'share' in root and 'postgresql' in root and 'contrib' in root and len(fnmatch.filter(files, filename)) > 0:
+ return '/'.join([root, filename])
+ else:
+ raise Exception('Cannot find %s searching under %s' % (filename, postgis_base_dir))
#****************************************************************
#****************************************************************
@@ -493,21 +509,21 @@ def setupDB():
gen_conn.autocommit = True
except Exception, e:
print "I am unable to connect to the database."
- exit()
+ exit(1)
try:
gen_cur = gen_conn.cursor()
except Exception, e:
gen_conn.close()
print "I am unable to connect to the database."
- exit()
+ exit(1)
try:
gen_cur.execute("""DROP DATABASE IF EXISTS \"osm2pgsql-test\"""")
gen_cur.execute("""CREATE DATABASE \"osm2pgsql-test\" WITH ENCODING 'UTF8'""")
except Exception, e:
print "Failed to create osm2pgsql-test db" + e.pgerror
- exit();
+ exit(1);
finally:
gen_cur.close()
gen_conn.close()
@@ -517,66 +533,56 @@ def setupDB():
test_conn.autocommit = True
except Exception, e:
print "I am unable to connect to the database." + e
- exit()
+ exit(1)
try:
test_cur = test_conn.cursor()
except Exception, e:
print "I am unable to connect to the database." + e
gen_conn.close()
- exit()
+ exit(1)
try:
+
+ # Check the tablespace
try:
global created_tablespace
+ created_tablespace = 0
+
test_cur.execute("""SELECT spcname FROM pg_tablespace WHERE spcname = 'tablespacetest'""")
if test_cur.fetchone():
print "We already have a tablespace, can use that"
- created_tablespace = 0
else:
- print "For the test, we need to create a tablespace. This needs root privileges"
- created_tablespace = 1
- ### This makes postgresql read from /tmp
- ## Does this have security implications like opening this to a possible symlink attack?
- try:
- os.mkdir("/tmp/psql-tablespace")
- returncode = subprocess.call(["/usr/bin/sudo", "/bin/chown", "postgres.postgres", "/tmp/psql-tablespace"])
- test_cur.execute("""CREATE TABLESPACE tablespacetest LOCATION '/tmp/psql-tablespace'""")
- except Exception, e:
- os.rmdir("/tmp/psql-tablespace")
- self.assertEqual(0, 1, "Failed to create tablespace")
+ print "The test needs a temporary tablespace to run in, but it does not exist. Please create the temporary tablespace. On Linux, you can do this by running:"
+ print " sudo mkdir -p /tmp/psql-tablespace"
+ print " sudo /bin/chown postgres.postgres tmp/psql-tablespace"
+ print " psql -c \"CREATE TABLESPACE tablespacetest LOCATION '/tmp/psql-tablespace'\" postgres"
+ exit(77)
except Exception, e:
print "Failed to create directory for tablespace" + str(e)
-
+ # Check for postgis
try:
test_cur.execute("""CREATE EXTENSION postgis;""")
except:
test_conn.rollback()
- # Guess the directory from the postgres version.
- # TODO: make the postgisdir configurable. Probably
- # only works on Debian-based distributions at the moment.
- postgisdir = ('/usr/share/postgresql/%d.%d/contrib' %
- (test_conn.server_version / 10000, (test_conn.server_version / 100) % 100))
- for fl in os.listdir(postgisdir):
- if fl.startswith('postgis'):
- newdir = os.path.join(postgisdir, fl)
- if os.path.isdir(newdir):
- postgisdir = newdir
- break
- else:
- raise Exception('Cannot find postgis directory.')
- pgscript = open(os.path.join(postgisdir, 'postgis.sql'),'r').read()
+
+ pgis = findContribSql('postgis.sql')
+ pgscript = open(pgis).read()
test_cur.execute(pgscript)
- pgscript = open(os.path.join(postgisdir, 'spatial_ref_sys.sql'), 'r').read()
+
+ srs = findContribSql('spatial_ref_sys.sql')
+ pgscript = open(srs).read()
test_cur.execute(pgscript)
+ # Check for hstore support
try:
test_cur.execute("""CREATE EXTENSION hstore;""")
-
except Exception, e:
- print "I am unable to create extensions: " + e.pgerror
- exit()
+ hst = findContribSql('hstore.sql')
+ pgscript = open(hst).read()
+ test_cur.execute(pgscript)
+
finally:
test_cur.close()
test_conn.close()
@@ -589,7 +595,7 @@ def tearDownDB():
gen_cur = gen_conn.cursor()
except Exception, e:
print "I am unable to connect to the database."
- exit()
+ exit(1)
try:
gen_cur.execute("""DROP DATABASE IF EXISTS \"osm2pgsql-test\"""")
@@ -597,7 +603,7 @@ def tearDownDB():
gen_cur.execute("""DROP TABLESPACE IF EXISTS \"tablespacetest\"""")
except Exception, e:
print "Failed to clean up osm2pgsql-test db" + e.pgerror
- exit();
+ exit(1);
gen_cur.close()
gen_conn.close()
@@ -620,9 +626,19 @@ if __name__ == "__main__":
ts2 = CompleteTestSuite()
+success = False
try:
setupDB()
runner = unittest.TextTestRunner()
- runner.run(ts2)
+ result = runner.run(ts2)
+ success = result.wasSuccessful()
+
finally:
tearDownDB()
+
+if success:
+ print "All tests passed :-)"
+ exit(0)
+else:
+ print "Some tests failed :-("
+ exit(1)
diff --git a/tests/regression-test.sh b/tests/regression-test.sh
new file mode 100755
index 0000000..b96f016
--- /dev/null
+++ b/tests/regression-test.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+tests/regression-test.py -f tests/liechtenstein-2013-08-03.osm.pbf &&
+tests/regression-test.py -f tests/liechtenstein-2013-08-03.osm.bz2 &&
+echo "Tests passed :-)"
diff --git a/tests/test-expire-tiles.cpp b/tests/test-expire-tiles.cpp
new file mode 100644
index 0000000..62f8fe5
--- /dev/null
+++ b/tests/test-expire-tiles.cpp
@@ -0,0 +1,365 @@
+#include "expire-tiles.hpp"
+
+#include <stdio.h>
+#include <string.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdexcept>
+#include <boost/format.hpp>
+#include <set>
+
+#define EARTH_CIRCUMFERENCE (40075016.68)
+
+namespace {
+
+void run_test(const char* test_name, void (*testfunc)())
+{
+ try
+ {
+ fprintf(stderr, "%s\n", test_name);
+ testfunc();
+ }
+ catch(std::exception& e)
+ {
+ fprintf(stderr, "%s\n", e.what());
+ fprintf(stderr, "FAIL\n");
+ exit(EXIT_FAILURE);
+ }
+ fprintf(stderr, "PASS\n");
+}
+#define RUN_TEST(x) run_test(#x, &(x))
+#define ASSERT_EQ(a, b) { if (!((a) == (b))) { throw std::runtime_error((boost::format("Expecting %1% == %2%, but %3% != %4%") % #a % #b % (a) % (b)).str()); } }
+
+struct xyz {
+ int z, x, y;
+ xyz(int z_, int x_, int y_) : z(z_), x(x_), y(y_) {}
+ bool operator==(const xyz &other) const {
+ return ((z == other.z) &&
+ (x == other.x) &&
+ (y == other.y));
+ }
+ bool operator<(const xyz &other) const {
+ return ((z < other.z) ||
+ ((z == other.z) &&
+ ((x < other.x) ||
+ ((x == other.x) &&
+ (y < other.y)))));
+ }
+ void to_bbox(double &x0, double &y0,
+ double &x1, double &y1) const {
+ const double datum = 0.5 * (1 << z);
+ const double scale = EARTH_CIRCUMFERENCE / (1 << z);
+ x0 = (x - datum) * scale;
+ y0 = (datum - (y + 1)) * scale;
+ x1 = ((x + 1) - datum) * scale;
+ y1 = (datum - y) * scale;
+ }
+ void to_centroid(double &x0, double &y0) const {
+ const double datum = 0.5 * (1 << z);
+ const double scale = EARTH_CIRCUMFERENCE / (1 << z);
+ x0 = ((x + 0.5) - datum) * scale;
+ y0 = (datum - (y + 0.5)) * scale;
+ }
+};
+
+std::ostream &operator<<(std::ostream &out, const xyz &tile) {
+ out << tile.z << "/" << tile.x << "/" << tile.y;
+ return out;
+}
+
+struct tile_output_set : public expire_tiles::tile_output {
+ tile_output_set() {}
+
+ virtual ~tile_output_set() {}
+
+ virtual void output_dirty_tile(int x, int y, int zoom, int min_zoom) {
+ int y_min, x_iter, y_iter, x_max, y_max, out_zoom, zoom_diff;
+
+ if (zoom > min_zoom) out_zoom = zoom;
+ else out_zoom = min_zoom;
+ zoom_diff = out_zoom - zoom;
+ y_min = y << zoom_diff;
+ x_max = (x + 1) << zoom_diff;
+ y_max = (y + 1) << zoom_diff;
+ for (x_iter = x << zoom_diff; x_iter < x_max; x_iter++) {
+ for (y_iter = y_min; y_iter < y_max; y_iter++) {
+ m_tiles.insert(xyz(out_zoom, x_iter, y_iter));
+ }
+ }
+ }
+
+ std::set<xyz> m_tiles;
+};
+
+void test_expire_simple_z1() {
+ options_t opt;
+ opt.expire_tiles_zoom = 1;
+ opt.expire_tiles_zoom_min = 1;
+
+ expire_tiles et(&opt);
+ tile_output_set set;
+
+ // as big a bbox as possible at the origin to dirty all four
+ // quadrants of the world.
+ et.from_bbox(-10000, -10000, 10000, 10000);
+ et.output_and_destroy(&set);
+
+ ASSERT_EQ(set.m_tiles.size(), 4);
+ std::set<xyz>::iterator itr = set.m_tiles.begin();
+ ASSERT_EQ(*itr, xyz(1, 0, 0)); ++itr;
+ ASSERT_EQ(*itr, xyz(1, 0, 1)); ++itr;
+ ASSERT_EQ(*itr, xyz(1, 1, 0)); ++itr;
+ ASSERT_EQ(*itr, xyz(1, 1, 1)); ++itr;
+}
+
+void test_expire_simple_z3() {
+ options_t opt;
+ opt.expire_tiles_zoom = 3;
+ opt.expire_tiles_zoom_min = 3;
+
+ expire_tiles et(&opt);
+ tile_output_set set;
+
+ // as big a bbox as possible at the origin to dirty all four
+ // quadrants of the world.
+ et.from_bbox(-10000, -10000, 10000, 10000);
+ et.output_and_destroy(&set);
+
+ ASSERT_EQ(set.m_tiles.size(), 4);
+ std::set<xyz>::iterator itr = set.m_tiles.begin();
+ ASSERT_EQ(*itr, xyz(3, 3, 3)); ++itr;
+ ASSERT_EQ(*itr, xyz(3, 3, 4)); ++itr;
+ ASSERT_EQ(*itr, xyz(3, 4, 3)); ++itr;
+ ASSERT_EQ(*itr, xyz(3, 4, 4)); ++itr;
+}
+
+void test_expire_simple_z18() {
+ options_t opt;
+ opt.expire_tiles_zoom = 18;
+ opt.expire_tiles_zoom_min = 18;
+
+ expire_tiles et(&opt);
+ tile_output_set set;
+
+ // dirty a smaller bbox this time, as at z18 the scale is
+ // pretty small.
+ et.from_bbox(-1, -1, 1, 1);
+ et.output_and_destroy(&set);
+
+ ASSERT_EQ(set.m_tiles.size(), 4);
+ std::set<xyz>::iterator itr = set.m_tiles.begin();
+ ASSERT_EQ(*itr, xyz(18, 131071, 131071)); ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131071, 131072)); ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131072, 131071)); ++itr;
+ ASSERT_EQ(*itr, xyz(18, 131072, 131072)); ++itr;
+}
+
+std::set<xyz> generate_random(int zoom, size_t count) {
+ size_t num = 0;
+ std::set<xyz> set;
+ const int coord_mask = (1 << zoom) - 1;
+
+ while (num < count) {
+ xyz item(zoom, rand() & coord_mask, rand() & coord_mask);
+ if (set.count(item) == 0) {
+ set.insert(item);
+ ++num;
+ }
+ }
+
+ return set;
+}
+
+void assert_tilesets_equal(const std::set<xyz> &a,
+ const std::set<xyz> &b) {
+ ASSERT_EQ(a.size(), b.size());
+ std::set<xyz>::const_iterator a_itr = a.begin();
+ std::set<xyz>::const_iterator b_itr = b.begin();
+ while ((a_itr != a.end()) &&
+ (b_itr != b.end())) {
+ ASSERT_EQ(*a_itr, *b_itr);
+ ++a_itr;
+ ++b_itr;
+ }
+}
+
+void expire_centroids(const std::set<xyz> &check_set,
+ expire_tiles &et) {
+ for (std::set<xyz>::const_iterator itr = check_set.begin();
+ itr != check_set.end(); ++itr) {
+ double x0 = 0.0, y0 = 0.0;
+ itr->to_centroid(x0, y0);
+ et.from_bbox(x0, y0, x0, y0);
+ }
+}
+
+// tests that expiring a set of tile centroids means that
+// those tiles get expired.
+void test_expire_set() {
+ options_t opt;
+ int zoom = 18;
+ opt.expire_tiles_zoom = zoom;
+ opt.expire_tiles_zoom_min = zoom;
+
+ for (int i = 0; i < 100; ++i) {
+ expire_tiles et(&opt);
+ tile_output_set set;
+
+ std::set<xyz> check_set = generate_random(zoom, 100);
+ expire_centroids(check_set, et);
+
+ et.output_and_destroy(&set);
+
+ assert_tilesets_equal(set.m_tiles, check_set);
+ }
+}
+
+// this tests that, after expiring a random set of tiles
+// in one expire_tiles object and a different set in
+// another, when they are merged together they are the
+// same as if the union of the sets of tiles had been
+// expired.
+void test_expire_merge() {
+ options_t opt;
+ int zoom = 18;
+ opt.expire_tiles_zoom = zoom;
+ opt.expire_tiles_zoom_min = zoom;
+
+ for (int i = 0; i < 100; ++i) {
+ expire_tiles et(&opt), et1(&opt), et2(&opt);
+ tile_output_set set;
+
+ std::set<xyz> check_set1 = generate_random(zoom, 100);
+ expire_centroids(check_set1, et1);
+
+ std::set<xyz> check_set2 = generate_random(zoom, 100);
+ expire_centroids(check_set2, et2);
+
+ et.merge_and_destroy(et1);
+ et.merge_and_destroy(et2);
+
+ std::set<xyz> check_set;
+ std::set_union(check_set1.begin(), check_set1.end(),
+ check_set2.begin(), check_set2.end(),
+ std::inserter(check_set, check_set.end()));
+
+ et.output_and_destroy(&set);
+
+ assert_tilesets_equal(set.m_tiles, check_set);
+ }
+}
+
+// tests that merging two identical sets results in
+// the same set. this guarantees that we check some
+// pathways of the merging which possibly could be
+// skipped by the random tile set in the previous
+// test.
+void test_expire_merge_same() {
+ options_t opt;
+ int zoom = 18;
+ opt.expire_tiles_zoom = zoom;
+ opt.expire_tiles_zoom_min = zoom;
+
+ for (int i = 0; i < 100; ++i) {
+ expire_tiles et(&opt), et1(&opt), et2(&opt);
+ tile_output_set set;
+
+ std::set<xyz> check_set = generate_random(zoom, 100);
+ expire_centroids(check_set, et1);
+ expire_centroids(check_set, et2);
+
+ et.merge_and_destroy(et1);
+ et.merge_and_destroy(et2);
+
+ et.output_and_destroy(&set);
+
+ assert_tilesets_equal(set.m_tiles, check_set);
+ }
+}
+
+// makes sure that we're testing the case where some
+// tiles are in both.
+void test_expire_merge_overlap() {
+ options_t opt;
+ int zoom = 18;
+ opt.expire_tiles_zoom = zoom;
+ opt.expire_tiles_zoom_min = zoom;
+
+ for (int i = 0; i < 100; ++i) {
+ expire_tiles et(&opt), et1(&opt), et2(&opt);
+ tile_output_set set;
+
+ std::set<xyz> check_set1 = generate_random(zoom, 100);
+ expire_centroids(check_set1, et1);
+
+ std::set<xyz> check_set2 = generate_random(zoom, 100);
+ expire_centroids(check_set2, et2);
+
+ std::set<xyz> check_set3 = generate_random(zoom, 100);
+ expire_centroids(check_set3, et1);
+ expire_centroids(check_set3, et2);
+
+ et.merge_and_destroy(et1);
+ et.merge_and_destroy(et2);
+
+ std::set<xyz> check_set;
+ std::set_union(check_set1.begin(), check_set1.end(),
+ check_set2.begin(), check_set2.end(),
+ std::inserter(check_set, check_set.end()));
+ std::set_union(check_set1.begin(), check_set1.end(),
+ check_set3.begin(), check_set3.end(),
+ std::inserter(check_set, check_set.end()));
+
+ et.output_and_destroy(&set);
+
+ assert_tilesets_equal(set.m_tiles, check_set);
+ }
+}
+
+// checks that the set union still works when we expire
+// large contiguous areas of tiles (i.e: ensure that we
+// handle the "complete" flag correctly).
+void test_expire_merge_complete() {
+ options_t opt;
+ int zoom = 18;
+ opt.expire_tiles_zoom = zoom;
+ opt.expire_tiles_zoom_min = zoom;
+
+ for (int i = 0; i < 100; ++i) {
+ expire_tiles et(&opt), et1(&opt), et2(&opt), et0(&opt);
+ tile_output_set set, set0;
+
+ // et1&2 are two halves of et0's box
+ et0.from_bbox(-10000, -10000, 10000, 10000);
+ et1.from_bbox(-10000, -10000, 0, 10000);
+ et2.from_bbox( 0, -10000, 10000, 10000);
+
+ et.merge_and_destroy(et1);
+ et.merge_and_destroy(et2);
+
+ et.output_and_destroy(&set);
+ et0.output_and_destroy(&set0);
+
+ assert_tilesets_equal(set.m_tiles, set0.m_tiles);
+ }
+}
+
+} // anonymous namespace
+
+int main(int argc, char *argv[])
+{
+ srand(0);
+
+ //try each test if any fail we will exit
+ RUN_TEST(test_expire_simple_z1);
+ RUN_TEST(test_expire_simple_z3);
+ RUN_TEST(test_expire_simple_z18);
+ RUN_TEST(test_expire_set);
+ RUN_TEST(test_expire_merge);
+ RUN_TEST(test_expire_merge_same);
+ RUN_TEST(test_expire_merge_overlap);
+ RUN_TEST(test_expire_merge_complete);
+
+ //passed
+ return 0;
+}
diff --git a/tests/test-middle-pgsql.cpp b/tests/test-middle-pgsql.cpp
new file mode 100644
index 0000000..c926e12
--- /dev/null
+++ b/tests/test-middle-pgsql.cpp
@@ -0,0 +1,80 @@
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <cassert>
+#include <sstream>
+#include <stdexcept>
+#include <memory>
+
+#include "osmtypes.hpp"
+#include "middle.hpp"
+#include "output-null.hpp"
+#include "options.hpp"
+#include "middle-pgsql.hpp"
+
+#include <libpq-fe.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <boost/scoped_ptr.hpp>
+
+#include "tests/middle-tests.hpp"
+#include "tests/common-pg.hpp"
+
+int main(int argc, char *argv[]) {
+ boost::scoped_ptr<pg::tempdb> db;
+
+ try {
+ db.reset(new pg::tempdb);
+ } catch (const std::exception &e) {
+ std::cerr << "Unable to setup database: " << e.what() << "\n";
+ return 77; // <-- code to skip this test.
+ }
+
+ struct middle_pgsql_t mid_pgsql;
+ options_t options;
+ options.conninfo = db->conninfo().c_str();
+ options.scale = 10000000;
+ options.num_procs = 1;
+ options.prefix = "osm2pgsql_test";
+ options.tblsslim_index = "tablespacetest";
+ options.tblsslim_data = "tablespacetest";
+ options.slim = 1;
+
+ struct output_null_t out_test(&mid_pgsql, options);
+
+ try {
+ // start an empty table to make the middle create the
+ // tables it needs. we then run the test in "append" mode.
+ mid_pgsql.start(&options);
+ mid_pgsql.commit();
+ mid_pgsql.stop();
+
+ options.append = 1; /* <- needed because we're going to change the
+ * data and check that the updates fire. */
+
+ mid_pgsql.start(&options);
+
+ int status = 0;
+
+ status = test_node_set(&mid_pgsql);
+ if (status != 0) { mid_pgsql.stop(); throw std::runtime_error("test_node_set failed."); }
+
+ status = test_way_set(&mid_pgsql);
+ if (status != 0) { mid_pgsql.stop(); throw std::runtime_error("test_way_set failed."); }
+
+ mid_pgsql.commit();
+ mid_pgsql.stop();
+
+ return 0;
+
+ } catch (const std::exception &e) {
+ std::cerr << "ERROR: " << e.what() << std::endl;
+
+ } catch (...) {
+ std::cerr << "UNKNOWN ERROR" << std::endl;
+ }
+
+ return 1;
+}
diff --git a/tests/test-middle-ram.cpp b/tests/test-middle-ram.cpp
new file mode 100644
index 0000000..59a2b63
--- /dev/null
+++ b/tests/test-middle-ram.cpp
@@ -0,0 +1,50 @@
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <cassert>
+#include <stdexcept>
+
+#include "osmtypes.hpp"
+#include "middle.hpp"
+#include "output-null.hpp"
+#include "options.hpp"
+#include "middle-ram.hpp"
+#include "node-ram-cache.hpp"
+
+#include "tests/middle-tests.hpp"
+
+int main(int argc, char *argv[]) {
+ try {
+ options_t options;
+ options.scale = 10000000;
+ options.alloc_chunkwise = ALLOC_SPARSE | ALLOC_DENSE;
+ options.cache = 1;
+
+ struct middle_ram_t mid_ram;
+ struct output_null_t out_test(&mid_ram, options);
+
+ mid_ram.start(&options);
+
+ int status = 0;
+
+ status = test_node_set(&mid_ram);
+ if (status != 0) { throw std::runtime_error("test_node_set failed."); }
+
+ status = test_way_set(&mid_ram);
+ if (status != 0) { throw std::runtime_error("test_node_set failed."); }
+
+ mid_ram.commit();
+ mid_ram.stop();
+
+ return 0;
+
+ } catch (const std::exception &e) {
+ std::cerr << "ERROR: " << e.what() << std::endl;
+
+ } catch (...) {
+ std::cerr << "UNKNOWN ERROR" << std::endl;
+ }
+
+ return 1;
+}
diff --git a/tests/test-output-multi-line.cpp b/tests/test-output-multi-line.cpp
new file mode 100644
index 0000000..c55fd1c
--- /dev/null
+++ b/tests/test-output-multi-line.cpp
@@ -0,0 +1,126 @@
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <cassert>
+#include <sstream>
+#include <stdexcept>
+#include <memory>
+
+#include "osmtypes.hpp"
+#include "middle.hpp"
+#include "output-multi.hpp"
+#include "options.hpp"
+#include "middle-pgsql.hpp"
+#include "taginfo_impl.hpp"
+#include "parse.hpp"
+#include "text-tree.hpp"
+
+#include <libpq-fe.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+
+#include "tests/middle-tests.hpp"
+#include "tests/common-pg.hpp"
+
+void check_count(pg::conn_ptr &conn, int expected, const std::string &query) {
+ pg::result_ptr res = conn->exec(query);
+
+ int ntuples = PQntuples(res->get());
+ if (ntuples != 1) {
+ throw std::runtime_error((boost::format("Expected only one tuple from a query "
+ "to check COUNT(*), but got %1%. Query "
+ "was: %2%.")
+ % ntuples % query).str());
+ }
+
+ std::string numstr = PQgetvalue(res->get(), 0, 0);
+ int count = boost::lexical_cast<int>(numstr);
+
+ if (count != expected) {
+ throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running "
+ "query: %3%.")
+ % expected % count % query).str());
+ }
+}
+
+int main(int argc, char *argv[]) {
+ boost::scoped_ptr<pg::tempdb> db;
+
+ try {
+ db.reset(new pg::tempdb);
+ } catch (const std::exception &e) {
+ std::cerr << "Unable to setup database: " << e.what() << "\n";
+ return 77; // <-- code to skip this test.
+ }
+
+ try {
+ boost::shared_ptr<middle_pgsql_t> mid_pgsql(new middle_pgsql_t());
+ options_t options;
+ options.conninfo = db->conninfo().c_str();
+ options.num_procs = 1;
+ options.prefix = "osm2pgsql_test";
+ options.tblsslim_index = "tablespacetest";
+ options.tblsslim_data = "tablespacetest";
+ options.slim = 1;
+
+ boost::shared_ptr<geometry_processor> processor =
+ geometry_processor::create("line", &options);
+
+ export_list columns;
+ { taginfo info; info.name = "highway"; info.type = "text"; columns.add(OSMTYPE_WAY, info); }
+
+ boost::shared_ptr<output_multi_t> out_test(new output_multi_t("foobar_highways", processor, columns, mid_pgsql.get(), options));
+
+ osmdata_t osmdata(mid_pgsql, out_test);
+
+ boost::scoped_ptr<parse_delegate_t> parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection));
+
+ osmdata.start();
+
+ if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) {
+ throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'.");
+ }
+
+ parser.reset(NULL);
+
+ osmdata.stop();
+
+ // start a new connection to run tests on
+ pg::conn_ptr test_conn = pg::conn::connect(db->conninfo());
+
+ check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'osm2pgsql_test_foobar_highways'");
+ check_count(test_conn, 2753, "select count(*) from osm2pgsql_test_foobar_highways");
+
+ //check that we have the right spread
+ check_count(test_conn, 13, "select count(*) from osm2pgsql_test_foobar_highways where highway='bridleway'");
+ check_count(test_conn, 3, "select count(*) from osm2pgsql_test_foobar_highways where highway='construction'");
+ check_count(test_conn, 96, "select count(*) from osm2pgsql_test_foobar_highways where highway='cycleway'");
+ check_count(test_conn, 249, "select count(*) from osm2pgsql_test_foobar_highways where highway='footway'");
+ check_count(test_conn, 18, "select count(*) from osm2pgsql_test_foobar_highways where highway='living_street'");
+ check_count(test_conn, 171, "select count(*) from osm2pgsql_test_foobar_highways where highway='path'");
+ check_count(test_conn, 6, "select count(*) from osm2pgsql_test_foobar_highways where highway='pedestrian'");
+ check_count(test_conn, 81, "select count(*) from osm2pgsql_test_foobar_highways where highway='primary'");
+ check_count(test_conn, 842, "select count(*) from osm2pgsql_test_foobar_highways where highway='residential'");
+ check_count(test_conn, 3, "select count(*) from osm2pgsql_test_foobar_highways where highway='road'");
+ check_count(test_conn, 90, "select count(*) from osm2pgsql_test_foobar_highways where highway='secondary'");
+ check_count(test_conn, 1, "select count(*) from osm2pgsql_test_foobar_highways where highway='secondary_link'");
+ check_count(test_conn, 352, "select count(*) from osm2pgsql_test_foobar_highways where highway='service'");
+ check_count(test_conn, 34, "select count(*) from osm2pgsql_test_foobar_highways where highway='steps'");
+ check_count(test_conn, 33, "select count(*) from osm2pgsql_test_foobar_highways where highway='tertiary'");
+ check_count(test_conn, 597, "select count(*) from osm2pgsql_test_foobar_highways where highway='track'");
+ check_count(test_conn, 164, "select count(*) from osm2pgsql_test_foobar_highways where highway='unclassified'");
+ return 0;
+
+ } catch (const std::exception &e) {
+ std::cerr << "ERROR: " << e.what() << std::endl;
+
+ } catch (...) {
+ std::cerr << "UNKNOWN ERROR" << std::endl;
+ }
+
+ return 1;
+}
diff --git a/tests/test-output-multi-point-multi-table.cpp b/tests/test-output-multi-point-multi-table.cpp
new file mode 100644
index 0000000..ee43b9e
--- /dev/null
+++ b/tests/test-output-multi-point-multi-table.cpp
@@ -0,0 +1,147 @@
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <cassert>
+#include <sstream>
+#include <stdexcept>
+#include <memory>
+
+#include "osmtypes.hpp"
+#include "middle.hpp"
+#include "output-multi.hpp"
+#include "options.hpp"
+#include "middle-pgsql.hpp"
+#include "taginfo_impl.hpp"
+#include "parse.hpp"
+#include "text-tree.hpp"
+
+#include <libpq-fe.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+
+#include "tests/middle-tests.hpp"
+#include "tests/common-pg.hpp"
+
+void check_count(pg::conn_ptr &conn, int expected, const std::string &query) {
+ pg::result_ptr res = conn->exec(query);
+
+ if (PQresultStatus(res->get()) != PGRES_TUPLES_OK) {
+ throw std::runtime_error((boost::format("Query ERROR running %1%: %2%")
+ % query % PQresultErrorMessage(res->get())).str());
+ }
+
+ int ntuples = PQntuples(res->get());
+ if (ntuples != 1) {
+ throw std::runtime_error((boost::format("Expected only one tuple from a query "
+ "to check COUNT(*), but got %1%. Query "
+ "was: %2%.")
+ % ntuples % query).str());
+ }
+
+ std::string numstr = PQgetvalue(res->get(), 0, 0);
+ int count = boost::lexical_cast<int>(numstr);
+
+ if (count != expected) {
+ throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running "
+ "query: %3%.")
+ % expected % count % query).str());
+ }
+}
+
+int main(int argc, char *argv[]) {
+ boost::scoped_ptr<pg::tempdb> db;
+
+ try {
+ db.reset(new pg::tempdb);
+ } catch (const std::exception &e) {
+ std::cerr << "Unable to setup database: " << e.what() << "\n";
+ return 77; // <-- code to skip this test.
+ }
+
+ try {
+ boost::shared_ptr<middle_pgsql_t> mid_pgsql(new middle_pgsql_t());
+ options_t options;
+ options.conninfo = db->conninfo().c_str();
+ options.num_procs = 1;
+ options.prefix = "osm2pgsql_test";
+ options.tblsslim_index = "tablespacetest";
+ options.tblsslim_data = "tablespacetest";
+ options.slim = 1;
+
+ export_list columns;
+ { taginfo info; info.name = "amenity"; info.type = "text"; columns.add(OSMTYPE_NODE, info); }
+
+ std::vector<boost::shared_ptr<output_t> > outputs;
+
+ // let's make lots of tables!
+ for (int i = 0; i < 10; ++i) {
+ std::string name = (boost::format("foobar_%d") % i).str();
+
+ boost::shared_ptr<geometry_processor> processor =
+ geometry_processor::create("point", &options);
+
+ boost::shared_ptr<output_multi_t> out_test(new output_multi_t(name, processor, columns, mid_pgsql.get(), options));
+
+ outputs.push_back(out_test);
+ }
+
+ osmdata_t osmdata(mid_pgsql, outputs);
+
+ boost::scoped_ptr<parse_delegate_t> parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection));
+
+ osmdata.start();
+
+ if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) {
+ throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'.");
+ }
+
+ parser.reset(NULL);
+
+ osmdata.stop();
+
+ // start a new connection to run tests on
+ pg::conn_ptr test_conn = pg::conn::connect(db->conninfo());
+
+ for (int i = 0; i < 10; ++i) {
+ std::string name = (boost::format("foobar_%d") % i).str();
+
+ check_count(test_conn, 1,
+ (boost::format("select count(*) from pg_catalog.pg_class "
+ "where relname = 'osm2pgsql_test_foobar_%d'")
+ % i).str());
+
+ check_count(test_conn, 244,
+ (boost::format("select count(*) from osm2pgsql_test_foobar_%d")
+ % i).str());
+
+ check_count(test_conn, 36,
+ (boost::format("select count(*) from osm2pgsql_test_foobar_%d "
+ "where amenity='parking'")
+ % i).str());
+
+ check_count(test_conn, 34,
+ (boost::format("select count(*) from osm2pgsql_test_foobar_%d "
+ "where amenity='bench'")
+ % i).str());
+
+ check_count(test_conn, 1,
+ (boost::format("select count(*) from osm2pgsql_test_foobar_%d "
+ "where amenity='vending_machine'")
+ % i).str());
+ }
+
+ return 0;
+
+ } catch (const std::exception &e) {
+ std::cerr << "ERROR: " << e.what() << std::endl;
+
+ } catch (...) {
+ std::cerr << "UNKNOWN ERROR" << std::endl;
+ }
+
+ return 1;
+}
diff --git a/tests/test-output-multi-point.cpp b/tests/test-output-multi-point.cpp
new file mode 100644
index 0000000..7996922
--- /dev/null
+++ b/tests/test-output-multi-point.cpp
@@ -0,0 +1,121 @@
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <cassert>
+#include <sstream>
+#include <stdexcept>
+#include <memory>
+
+#include "osmtypes.hpp"
+#include "middle.hpp"
+#include "output-multi.hpp"
+#include "options.hpp"
+#include "middle-pgsql.hpp"
+#include "taginfo_impl.hpp"
+#include "parse.hpp"
+#include "text-tree.hpp"
+
+#include <libpq-fe.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+
+#include "tests/middle-tests.hpp"
+#include "tests/common-pg.hpp"
+
+void check_count(pg::conn_ptr &conn, int expected, const std::string &query) {
+ pg::result_ptr res = conn->exec(query);
+
+ int ntuples = PQntuples(res->get());
+ if (ntuples != 1) {
+ throw std::runtime_error((boost::format("Expected only one tuple from a query "
+ "to check COUNT(*), but got %1%. Query "
+ "was: %2%.")
+ % ntuples % query).str());
+ }
+
+ std::string numstr = PQgetvalue(res->get(), 0, 0);
+ int count = boost::lexical_cast<int>(numstr);
+
+ if (count != expected) {
+ throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running "
+ "query: %3%.")
+ % expected % count % query).str());
+ }
+}
+
+int main(int argc, char *argv[]) {
+ boost::scoped_ptr<pg::tempdb> db;
+
+ try {
+ db.reset(new pg::tempdb);
+ } catch (const std::exception &e) {
+ std::cerr << "Unable to setup database: " << e.what() << "\n";
+ return 77; // <-- code to skip this test.
+ }
+
+ try {
+ boost::shared_ptr<middle_pgsql_t> mid_pgsql(new middle_pgsql_t());
+ options_t options;
+ options.conninfo = db->conninfo().c_str();
+ options.num_procs = 1;
+ options.prefix = "osm2pgsql_test";
+ options.tblsslim_index = "tablespacetest";
+ options.tblsslim_data = "tablespacetest";
+ options.slim = 1;
+
+ boost::shared_ptr<geometry_processor> processor =
+ geometry_processor::create("point", &options);
+
+ export_list columns;
+ { taginfo info; info.name = "amenity"; info.type = "text"; columns.add(OSMTYPE_NODE, info); }
+
+ boost::shared_ptr<output_multi_t> out_test(new output_multi_t("foobar_amenities", processor, columns, mid_pgsql.get(), options));
+
+ osmdata_t osmdata(mid_pgsql, out_test);
+
+ boost::scoped_ptr<parse_delegate_t> parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection));
+
+ osmdata.start();
+
+ if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) {
+ throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'.");
+ }
+
+ parser.reset(NULL);
+
+ osmdata.stop();
+
+ // start a new connection to run tests on
+ pg::conn_ptr test_conn = pg::conn::connect(db->conninfo());
+
+ check_count(test_conn, 1,
+ "select count(*) from pg_catalog.pg_class "
+ "where relname = 'osm2pgsql_test_foobar_amenities'");
+
+ check_count(test_conn, 244,
+ "select count(*) from osm2pgsql_test_foobar_amenities");
+
+ check_count(test_conn, 36,
+ "select count(*) from osm2pgsql_test_foobar_amenities where amenity='parking'");
+
+ check_count(test_conn, 34,
+ "select count(*) from osm2pgsql_test_foobar_amenities where amenity='bench'");
+
+ check_count(test_conn, 1,
+ "select count(*) from osm2pgsql_test_foobar_amenities where amenity='vending_machine'");
+
+ return 0;
+
+ } catch (const std::exception &e) {
+ std::cerr << "ERROR: " << e.what() << std::endl;
+
+ } catch (...) {
+ std::cerr << "UNKNOWN ERROR" << std::endl;
+ }
+
+ return 1;
+}
diff --git a/tests/test-output-multi-polygon.cpp b/tests/test-output-multi-polygon.cpp
new file mode 100644
index 0000000..63eeac1
--- /dev/null
+++ b/tests/test-output-multi-polygon.cpp
@@ -0,0 +1,126 @@
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <cassert>
+#include <sstream>
+#include <stdexcept>
+#include <memory>
+
+#include "osmtypes.hpp"
+#include "middle.hpp"
+#include "output-multi.hpp"
+#include "options.hpp"
+#include "middle-pgsql.hpp"
+#include "taginfo_impl.hpp"
+#include "parse.hpp"
+#include "text-tree.hpp"
+
+#include <libpq-fe.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+
+#include "tests/middle-tests.hpp"
+#include "tests/common-pg.hpp"
+
+void check_count(pg::conn_ptr &conn, int expected, const std::string &query) {
+ pg::result_ptr res = conn->exec(query);
+
+ int ntuples = PQntuples(res->get());
+ if (ntuples != 1) {
+ throw std::runtime_error((boost::format("Expected only one tuple from a query "
+ "to check COUNT(*), but got %1%. Query "
+ "was: %2%.")
+ % ntuples % query).str());
+ }
+
+ std::string numstr = PQgetvalue(res->get(), 0, 0);
+ int count = boost::lexical_cast<int>(numstr);
+
+ if (count != expected) {
+ throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running "
+ "query: %3%.")
+ % expected % count % query).str());
+ }
+}
+
+int main(int argc, char *argv[]) {
+ boost::scoped_ptr<pg::tempdb> db;
+
+ try {
+ db.reset(new pg::tempdb);
+ } catch (const std::exception &e) {
+ std::cerr << "Unable to setup database: " << e.what() << "\n";
+ return 77; // <-- code to skip this test.
+ }
+
+ try {
+ boost::shared_ptr<middle_pgsql_t> mid_pgsql(new middle_pgsql_t());
+ options_t options;
+ options.conninfo = db->conninfo().c_str();
+ options.num_procs = 1;
+ options.prefix = "osm2pgsql_test";
+ options.tblsslim_index = "tablespacetest";
+ options.tblsslim_data = "tablespacetest";
+ options.slim = 1;
+
+ boost::shared_ptr<geometry_processor> processor = geometry_processor::create("polygon", &options);
+
+ export_list columns;
+ { taginfo info; info.name = "building"; info.type = "text"; columns.add(OSMTYPE_WAY, info); }
+
+ boost::shared_ptr<output_multi_t> out_test(new output_multi_t("foobar_buildings", processor, columns, mid_pgsql.get(), options));
+
+ osmdata_t osmdata(mid_pgsql, out_test);
+
+ boost::scoped_ptr<parse_delegate_t> parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection));
+
+ osmdata.start();
+
+ if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) {
+ throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'.");
+ }
+
+ parser.reset(NULL);
+
+ osmdata.stop();
+
+ // start a new connection to run tests on
+ pg::conn_ptr test_conn = pg::conn::connect(db->conninfo());
+
+ check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'osm2pgsql_test_foobar_buildings'");
+ check_count(test_conn, 0, "select count(*) from osm2pgsql_test_foobar_buildings where building is null");
+ check_count(test_conn, 3723, "select count(*) from osm2pgsql_test_foobar_buildings");
+
+ //check that we have the right spread
+ check_count(test_conn, 1, "select count(*) from osm2pgsql_test_foobar_buildings where building='barn'");
+ check_count(test_conn, 1, "select count(*) from osm2pgsql_test_foobar_buildings where building='chapel'");
+ check_count(test_conn, 5, "select count(*) from osm2pgsql_test_foobar_buildings where building='church'");
+ check_count(test_conn, 3, "select count(*) from osm2pgsql_test_foobar_buildings where building='commercial'");
+ check_count(test_conn, 6, "select count(*) from osm2pgsql_test_foobar_buildings where building='farm'");
+ check_count(test_conn, 1, "select count(*) from osm2pgsql_test_foobar_buildings where building='garage'");
+ check_count(test_conn, 2, "select count(*) from osm2pgsql_test_foobar_buildings where building='glasshouse'");
+ check_count(test_conn, 1, "select count(*) from osm2pgsql_test_foobar_buildings where building='greenhouse'");
+ check_count(test_conn, 153, "select count(*) from osm2pgsql_test_foobar_buildings where building='house'");
+ check_count(test_conn, 4, "select count(*) from osm2pgsql_test_foobar_buildings where building='hut'");
+ check_count(test_conn, 8, "select count(*) from osm2pgsql_test_foobar_buildings where building='industrial'");
+ check_count(test_conn, 200, "select count(*) from osm2pgsql_test_foobar_buildings where building='residential'");
+ check_count(test_conn, 6, "select count(*) from osm2pgsql_test_foobar_buildings where building='roof'");
+ check_count(test_conn, 4, "select count(*) from osm2pgsql_test_foobar_buildings where building='school'");
+ check_count(test_conn, 2, "select count(*) from osm2pgsql_test_foobar_buildings where building='station'");
+ check_count(test_conn, 3, "select count(*) from osm2pgsql_test_foobar_buildings where building='warehouse'");
+ check_count(test_conn, 3323, "select count(*) from osm2pgsql_test_foobar_buildings where building='yes'");
+ return 0;
+
+ } catch (const std::exception &e) {
+ std::cerr << "ERROR: " << e.what() << std::endl;
+
+ } catch (...) {
+ std::cerr << "UNKNOWN ERROR" << std::endl;
+ }
+
+ return 1;
+}
diff --git a/tests/test-output-pgsql.cpp b/tests/test-output-pgsql.cpp
new file mode 100644
index 0000000..cf773cf
--- /dev/null
+++ b/tests/test-output-pgsql.cpp
@@ -0,0 +1,311 @@
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <cassert>
+#include <sstream>
+#include <stdexcept>
+#include <memory>
+
+#include "osmtypes.hpp"
+#include "middle.hpp"
+#include "output-pgsql.hpp"
+#include "options.hpp"
+#include "middle-pgsql.hpp"
+#include "middle-ram.hpp"
+#include "taginfo_impl.hpp"
+#include "parse.hpp"
+#include "text-tree.hpp"
+
+#include <libpq-fe.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+
+#include "tests/middle-tests.hpp"
+#include "tests/common-pg.hpp"
+
+namespace {
+
+struct skip_test : public std::exception {
+ const char *what() { return "Test skipped."; }
+};
+
+void run_test(const char* test_name, void (*testfunc)()) {
+ try {
+ fprintf(stderr, "%s\n", test_name);
+ testfunc();
+
+ } catch (const skip_test &) {
+ exit(77); // <-- code to skip this test.
+
+ } catch (const std::exception& e) {
+ fprintf(stderr, "%s\n", e.what());
+ fprintf(stderr, "FAIL\n");
+ exit(EXIT_FAILURE);
+ }
+
+ fprintf(stderr, "PASS\n");
+}
+#define RUN_TEST(x) run_test(#x, &(x))
+
+void check_count(pg::conn_ptr &conn, int expected, const std::string &query) {
+ pg::result_ptr res = conn->exec(query);
+
+ int ntuples = PQntuples(res->get());
+ if (ntuples != 1) {
+ throw std::runtime_error((boost::format("Expected only one tuple from a query "
+ "to check COUNT(*), but got %1%. Query "
+ "was: %2%.")
+ % ntuples % query).str());
+ }
+
+ std::string numstr = PQgetvalue(res->get(), 0, 0);
+ int count = boost::lexical_cast<int>(numstr);
+
+ if (count != expected) {
+ throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running "
+ "query: %3%.")
+ % expected % count % query).str());
+ }
+}
+
+void assert_has_table(pg::conn_ptr &test_conn, const std::string &table_name) {
+ std::string query = (boost::format("select count(*) from pg_catalog.pg_class "
+ "where relname = '%1%'")
+ % table_name).str();
+
+ check_count(test_conn, 1, query);
+}
+
+// "simple" test modeled on the basic regression test from
+// the python script. this is just to check everything is
+// working as expected before we start the complex stuff.
+void test_regression_simple() {
+ boost::scoped_ptr<pg::tempdb> db;
+
+ try {
+ db.reset(new pg::tempdb);
+ } catch (const std::exception &e) {
+ std::cerr << "Unable to setup database: " << e.what() << "\n";
+ throw skip_test();
+ }
+
+ std::string proc_name("test-output-pgsql"), input_file("-");
+ char *argv[] = { &proc_name[0], &input_file[0], NULL };
+
+ boost::shared_ptr<middle_pgsql_t> mid_pgsql(new middle_pgsql_t());
+ options_t options = options_t::parse(2, argv);
+ options.conninfo = db->conninfo().c_str();
+ options.num_procs = 1;
+ options.prefix = "osm2pgsql_test";
+ options.tblsslim_index = "tablespacetest";
+ options.tblsslim_data = "tablespacetest";
+ options.slim = 1;
+ options.style = "default.style";
+
+ boost::shared_ptr<output_pgsql_t> out_test(new output_pgsql_t(mid_pgsql.get(), options));
+
+ osmdata_t osmdata(mid_pgsql, out_test);
+
+ boost::scoped_ptr<parse_delegate_t> parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection));
+
+ osmdata.start();
+
+ if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) {
+ throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'.");
+ }
+
+ parser.reset(NULL);
+
+ osmdata.stop();
+
+ // start a new connection to run tests on
+ pg::conn_ptr test_conn = pg::conn::connect(db->conninfo());
+
+ assert_has_table(test_conn, "osm2pgsql_test_point");
+ assert_has_table(test_conn, "osm2pgsql_test_line");
+ assert_has_table(test_conn, "osm2pgsql_test_polygon");
+ assert_has_table(test_conn, "osm2pgsql_test_roads");
+
+ check_count(test_conn, 1342, "SELECT count(*) FROM osm2pgsql_test_point");
+ check_count(test_conn, 3300, "SELECT count(*) FROM osm2pgsql_test_line");
+ check_count(test_conn, 375, "SELECT count(*) FROM osm2pgsql_test_roads");
+ check_count(test_conn, 4128, "SELECT count(*) FROM osm2pgsql_test_polygon");
+}
+
+void test_area_way_simple() {
+ boost::scoped_ptr<pg::tempdb> db;
+
+ try {
+ db.reset(new pg::tempdb);
+ } catch (const std::exception &e) {
+ std::cerr << "Unable to setup database: " << e.what() << "\n";
+ throw skip_test();
+ }
+
+ std::string proc_name("test-output-pgsql"), input_file("-");
+ char *argv[] = { &proc_name[0], &input_file[0], NULL };
+
+ boost::shared_ptr<middle_pgsql_t> mid_pgsql(new middle_pgsql_t());
+ options_t options = options_t::parse(2, argv);
+ options.conninfo = db->conninfo().c_str();
+ options.num_procs = 1;
+ options.prefix = "osm2pgsql_test";
+ options.slim = 1;
+ options.style = "default.style";
+ options.flat_node_cache_enabled = true;
+ options.flat_node_file = boost::optional<std::string>("tests/test_output_pgsql_area_way.flat.nodes.bin");
+
+ boost::shared_ptr<output_pgsql_t> out_test(new output_pgsql_t(mid_pgsql.get(), options));
+
+ osmdata_t osmdata(mid_pgsql, out_test);
+
+ boost::scoped_ptr<parse_delegate_t> parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection));
+
+ osmdata.start();
+
+ if (parser->streamFile("libxml2", "tests/test_output_pgsql_way_area.osm", options.sanitize, &osmdata) != 0) {
+ throw std::runtime_error("Unable to read input file `tests/test_output_pgsql_way_area.osm'.");
+ }
+
+ parser.reset(NULL);
+
+ osmdata.stop();
+
+ // start a new connection to run tests on
+ pg::conn_ptr test_conn = pg::conn::connect(db->conninfo());
+
+ assert_has_table(test_conn, "osm2pgsql_test_point");
+ assert_has_table(test_conn, "osm2pgsql_test_line");
+ assert_has_table(test_conn, "osm2pgsql_test_polygon");
+ assert_has_table(test_conn, "osm2pgsql_test_roads");
+
+ check_count(test_conn, 0, "SELECT count(*) FROM osm2pgsql_test_point");
+ check_count(test_conn, 0, "SELECT count(*) FROM osm2pgsql_test_line");
+ check_count(test_conn, 0, "SELECT count(*) FROM osm2pgsql_test_roads");
+ check_count(test_conn, 1, "SELECT count(*) FROM osm2pgsql_test_polygon");
+}
+
+void test_route_rel() {
+ boost::scoped_ptr<pg::tempdb> db;
+
+ try {
+ db.reset(new pg::tempdb);
+ } catch (const std::exception &e) {
+ std::cerr << "Unable to setup database: " << e.what() << "\n";
+ throw skip_test();
+ }
+
+ std::string proc_name("test-output-pgsql"), input_file("-");
+ char *argv[] = { &proc_name[0], &input_file[0], NULL };
+
+ boost::shared_ptr<middle_ram_t> mid_ram(new middle_ram_t());
+ options_t options = options_t::parse(2, argv);
+ options.conninfo = db->conninfo().c_str();
+ options.num_procs = 1;
+ options.prefix = "osm2pgsql_test";
+ options.slim = 0;
+ options.style = "default.style";
+
+ boost::shared_ptr<output_pgsql_t> out_test(new output_pgsql_t(mid_ram.get(), options));
+
+ osmdata_t osmdata(mid_ram, out_test);
+
+ boost::scoped_ptr<parse_delegate_t> parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection));
+
+ osmdata.start();
+
+ if (parser->streamFile("libxml2", "tests/test_output_pgsql_route_rel.osm", options.sanitize, &osmdata) != 0) {
+ throw std::runtime_error("Unable to read input file `tests/test_output_pgsql_way_area.osm'.");
+ }
+
+ parser.reset(NULL);
+
+ osmdata.stop();
+
+ // start a new connection to run tests on
+ pg::conn_ptr test_conn = pg::conn::connect(db->conninfo());
+
+ assert_has_table(test_conn, "osm2pgsql_test_point");
+ assert_has_table(test_conn, "osm2pgsql_test_line");
+ assert_has_table(test_conn, "osm2pgsql_test_polygon");
+ assert_has_table(test_conn, "osm2pgsql_test_roads");
+
+ check_count(test_conn, 0, "SELECT count(*) FROM osm2pgsql_test_point");
+ check_count(test_conn, 2, "SELECT count(*) FROM osm2pgsql_test_line");
+ check_count(test_conn, 1, "SELECT count(*) FROM osm2pgsql_test_roads");
+ check_count(test_conn, 0, "SELECT count(*) FROM osm2pgsql_test_polygon");
+}
+
+// test the same, but clone the output. it should
+// behave the same as the original.
+void test_clone() {
+ boost::scoped_ptr<pg::tempdb> db;
+
+ try {
+ db.reset(new pg::tempdb);
+ } catch (const std::exception &e) {
+ std::cerr << "Unable to setup database: " << e.what() << "\n";
+ throw skip_test();
+ }
+
+ std::string proc_name("test-output-pgsql"), input_file("-");
+ char *argv[] = { &proc_name[0], &input_file[0], NULL };
+
+ boost::shared_ptr<middle_pgsql_t> mid_pgsql(new middle_pgsql_t());
+ options_t options = options_t::parse(2, argv);
+ options.conninfo = db->conninfo().c_str();
+ options.num_procs = 1;
+ options.prefix = "osm2pgsql_test";
+ options.tblsslim_index = "tablespacetest";
+ options.tblsslim_data = "tablespacetest";
+ options.slim = 1;
+ options.style = "default.style";
+
+ struct output_pgsql_t out_test(mid_pgsql.get(), options);
+
+ //TODO: make the middle testable too
+ //boost::shared_ptr<middle_t> mid_clone = mid_pgsql->get_instance();
+ boost::shared_ptr<output_t> out_clone = out_test.clone(mid_pgsql.get());
+
+ osmdata_t osmdata(mid_pgsql, out_clone);
+
+ boost::scoped_ptr<parse_delegate_t> parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection));
+
+ osmdata.start();
+
+ if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) {
+ throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'.");
+ }
+
+ parser.reset(NULL);
+
+ osmdata.stop();
+
+ // start a new connection to run tests on
+ pg::conn_ptr test_conn = pg::conn::connect(db->conninfo());
+
+ assert_has_table(test_conn, "osm2pgsql_test_point");
+ assert_has_table(test_conn, "osm2pgsql_test_line");
+ assert_has_table(test_conn, "osm2pgsql_test_polygon");
+ assert_has_table(test_conn, "osm2pgsql_test_roads");
+
+ check_count(test_conn, 1342, "SELECT count(*) FROM osm2pgsql_test_point");
+ check_count(test_conn, 3300, "SELECT count(*) FROM osm2pgsql_test_line");
+ check_count(test_conn, 375, "SELECT count(*) FROM osm2pgsql_test_roads");
+ check_count(test_conn, 4128, "SELECT count(*) FROM osm2pgsql_test_polygon");
+}
+
+} // anonymous namespace
+
+int main(int argc, char *argv[]) {
+ RUN_TEST(test_regression_simple);
+ RUN_TEST(test_clone);
+ RUN_TEST(test_area_way_simple);
+ RUN_TEST(test_route_rel);
+
+ return 0;
+}
diff --git a/tests/test-parse-options.cpp b/tests/test-parse-options.cpp
new file mode 100644
index 0000000..8cc57ee
--- /dev/null
+++ b/tests/test-parse-options.cpp
@@ -0,0 +1,297 @@
+#include "options.hpp"
+#include "middle-pgsql.hpp"
+#include "middle-ram.hpp"
+#include "output-pgsql.hpp"
+#include "output-gazetteer.hpp"
+#include "output-null.hpp"
+
+#include <stdio.h>
+#include <string.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdexcept>
+#include <boost/format.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
+namespace alg = boost::algorithm;
+
+#define len(x) sizeof(x)/sizeof(x[0])
+
+void run_test(const char* test_name, void (*testfunc)())
+{
+ try
+ {
+ fprintf(stderr, "%s\n", test_name);
+ testfunc();
+ }
+ catch(std::exception& e)
+ {
+ fprintf(stderr, "%s\n", e.what());
+ fprintf(stderr, "FAIL\n");
+ exit(EXIT_FAILURE);
+ }
+ fprintf(stderr, "PASS\n");
+}
+
+void parse_fail(const int argc, const char* argv[], const std::string& fail_message)
+{
+ try
+ {
+ options_t options = options_t::parse(argc, const_cast<char **>(argv));
+ throw std::logic_error((boost::format("Expected '%1%'") % fail_message).str());
+ }
+ catch(std::runtime_error& e)
+ {
+ if(!alg::icontains(e.what(), fail_message))
+ throw std::logic_error((boost::format("Expected '%1%' but instead got '%2%'") % fail_message % e.what()).str());
+ }
+}
+
+void test_insufficient_args()
+{
+ const char* argv[] = {"osm2pgsql", "-a", "-c", "--slim"};
+ parse_fail(len(argv), argv, "usage error");
+}
+
+void test_incompatible_args()
+{
+ const char* a1[] = {"osm2pgsql", "-a", "-c", "--slim", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ parse_fail(len(a1), a1, "options can not be used at the same time");
+
+ const char* a2[] = {"osm2pgsql", "--drop", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ parse_fail(len(a2), a2, "drop only makes sense with");
+
+ const char* a3[] = {"osm2pgsql", "-j", "-k", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ parse_fail(len(a3), a3, "you can not specify both");
+}
+
+void test_middles()
+{
+ const char* a1[] = {"osm2pgsql", "--slim", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ options_t options = options_t::parse(len(a1), const_cast<char **>(a1));
+ boost::shared_ptr<middle_t> mid = middle_t::create_middle(options.slim);
+ if(dynamic_cast<middle_pgsql_t *>(mid.get()) == NULL)
+ {
+ throw std::logic_error("Using slim mode we expected a pgsql middle");
+ }
+
+ const char* a2[] = {"osm2pgsql", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ options = options_t::parse(len(a2), const_cast<char **>(a2));
+ mid = middle_t::create_middle(options.slim);
+ if(dynamic_cast<middle_ram_t *>(mid.get()) == NULL)
+ {
+ throw std::logic_error("Using without slim mode we expected a ram middle");
+ }
+}
+
+void test_outputs()
+{
+ const char* a1[] = {"osm2pgsql", "-O", "pgsql", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ options_t options = options_t::parse(len(a1), const_cast<char **>(a1));
+ boost::shared_ptr<middle_t> mid = middle_t::create_middle(options.slim);
+ std::vector<boost::shared_ptr<output_t> > outs = output_t::create_outputs(mid.get(), options);
+ output_t* out = outs.front().get();
+ if(dynamic_cast<output_pgsql_t *>(out) == NULL)
+ {
+ throw std::logic_error("Expected a pgsql output");
+ }
+
+ const char* a2[] = {"osm2pgsql", "-O", "gazetteer", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ options = options_t::parse(len(a2), const_cast<char **>(a2));
+ mid = middle_t::create_middle(options.slim);
+ outs = output_t::create_outputs(mid.get(), options);
+ out = outs.front().get();
+ if(dynamic_cast<output_gazetteer_t *>(out) == NULL)
+ {
+ throw std::logic_error("Expected a gazetteer output");
+ }
+
+ const char* a3[] = {"osm2pgsql", "-O", "null", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ options = options_t::parse(len(a3), const_cast<char **>(a3));
+ mid = middle_t::create_middle(options.slim);
+ outs = output_t::create_outputs(mid.get(), options);
+ out = outs.front().get();
+ if(dynamic_cast<output_null_t *>(out) == NULL)
+ {
+ throw std::logic_error("Expected a null output");
+ }
+
+ const char* a4[] = {"osm2pgsql", "-O", "keine_richtige_ausgabe", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"};
+ options = options_t::parse(len(a4), const_cast<char **>(a4));
+ mid = middle_t::create_middle(options.slim);
+ try
+ {
+ outs = output_t::create_outputs(mid.get(), options);
+ out = outs.front().get();
+ throw std::logic_error("Expected 'not recognised'");
+ }
+ catch(std::runtime_error& e)
+ {
+ if(!alg::icontains(e.what(), "not recognised"))
+ throw std::logic_error((boost::format("Expected 'not recognised' but instead got '%2%'") % e.what()).str());
+ }
+}
+
+int get_random_proj(std::vector<std::string>& args)
+{
+ int proj = rand() % (PROJ_COUNT + 1);
+ switch(proj)
+ {
+ case PROJ_LATLONG:
+ case PROJ_MERC:
+ case PROJ_SPHERE_MERC:
+ args.push_back(reprojection(proj).project_getprojinfo()->option);
+ break;
+ default:
+ args.push_back("--proj");
+ //nice contiguous block of valid epsgs here randomly use one of those..
+ proj = (rand() % (2962 - 2308)) + 2308;
+ args.push_back((boost::format("%1%") % proj).str());
+ proj = -proj;
+ break;
+ }
+ return proj;
+}
+
+std::string get_random_string(const int length)
+{
+ std::string charset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890");
+ std::string result;
+ result.resize(length);
+
+ for (int i = 0; i < length; i++)
+ result[i] = charset[rand() % charset.length()];
+
+ return result;
+}
+
+template<typename T>
+void add_arg_or_not(const char* arg, std::vector<std::string>& args, T& option)
+{
+ if(rand() % 2)
+ {
+ args.push_back(arg);
+ option = 1;
+ }
+ else
+ option = 0;
+}
+
+void add_arg_and_val_or_not(const char* arg, std::vector<std::string>& args, int option, const int val)
+{
+ if(rand() % 2)
+ {
+ args.push_back(arg);
+ args.push_back((boost::format("%1%") % val).str());
+ option = val;
+ }
+}
+
+void add_arg_and_val_or_not(const char* arg, std::vector<std::string>& args, const char *option, std::string val)
+{
+ if(rand() % 2)
+ {
+ args.push_back(arg);
+ args.push_back(val);
+ option = val.c_str();
+ }
+}
+
+void test_random_perms()
+{
+
+ for(int i = 0; i < 5; ++i)
+ {
+ options_t options;
+ std::vector<std::string> args;
+ args.push_back("osm2pgsql");
+
+ //pick a projection
+ options.projection.reset(new reprojection(get_random_proj(args)));
+
+ //pick a style file
+ std::string style = get_random_string(15);
+ options.style = style.c_str();
+ args.push_back("--style");
+ args.push_back(style);
+
+ add_arg_and_val_or_not("--cache", args, options.cache, rand() % 800);
+ add_arg_and_val_or_not("--database", args, options.db.c_str(), get_random_string(6));
+ if (options.username) {
+ add_arg_and_val_or_not("--username", args, options.username->c_str(), get_random_string(6));
+ }
+ if (options.host) {
+ add_arg_and_val_or_not("--host", args, options.host->c_str(), get_random_string(6));
+ }
+ //add_arg_and_val_or_not("--port", args, options.port, rand() % 9999);
+
+ //--hstore-match-only
+ //--hstore-column Add an additional hstore (key/value) column containing all tags that start with the specified string, eg --hstore-column "name:" will produce an extra hstore column that contains all name:xx tags
+
+ add_arg_or_not("--hstore-add-index", args, options.enable_hstore_index);
+ add_arg_or_not("--utf8-sanitize", args, options.sanitize);
+
+ //--tablespace-index The name of the PostgreSQL tablespace where all indexes will be created. The following options allow more fine-grained control:
+ // --tablespace-main-data tablespace for main tables
+ // --tablespace-main-index tablespace for main table indexes
+ // --tablespace-slim-data tablespace for slim mode tables
+ // --tablespace-slim-index tablespace for slim mode indexes
+ // (if unset, use db's default; -i is equivalent to setting
+ // --tablespace-main-index and --tablespace-slim-index)
+
+ add_arg_and_val_or_not("--number-processes", args, options.num_procs, rand() % 12);
+
+ //add_arg_or_not("--disable-parallel-indexing", args, options.parallel_indexing);
+
+ add_arg_or_not("--unlogged", args, options.unlogged);
+
+ //--cache-strategy Specifies the method used to cache nodes in ram. Available options are: dense chunk sparse optimized
+
+ if (options.flat_node_file) {
+ add_arg_and_val_or_not("--flat-nodes", args, options.flat_node_file->c_str(), get_random_string(15));
+ }
+
+ //--expire-tiles [min_zoom-]max_zoom Create a tile expiry list.
+
+ add_arg_and_val_or_not("--expire-output", args, options.expire_tiles_filename.c_str(), get_random_string(15));
+
+ //--bbox Apply a bounding box filter on the imported data Must be specified as: minlon,minlat,maxlon,maxlat e.g. --bbox -0.5,51.25,0.5,51.75
+
+ add_arg_and_val_or_not("--prefix", args, options.prefix.c_str(), get_random_string(15));
+
+ //--input-reader Input frontend. libxml2 - Parse XML using libxml2. (default) primitive - Primitive XML parsing. pbf - OSM binary format.
+
+ if (options.tag_transform_script) {
+ add_arg_and_val_or_not("--tag-transform-script", args, options.tag_transform_script->c_str(), get_random_string(15));
+ }
+ add_arg_or_not("--extra-attributes", args, options.extra_attributes);
+ add_arg_or_not("--multi-geometry", args, options.enable_multi);
+ add_arg_or_not("--keep-coastlines", args, options.keep_coastlines);
+ add_arg_or_not("--exclude-invalid-polygon", args, options.excludepoly);
+
+ //add the input file
+ args.push_back("tests/liechtenstein-2013-08-03.osm.pbf");
+
+ const char** argv = new const char*[args.size() + 1];
+ argv[args.size()] = NULL;
+ for(std::vector<std::string>::const_iterator arg = args.begin(); arg != args.end(); ++arg)
+ argv[arg - args.begin()] = arg->c_str();
+ options_t::parse(args.size(), const_cast<char **>(argv));
+ delete[] argv;
+ }
+}
+
+int main(int argc, char *argv[])
+{
+ srand(0);
+
+ //try each test if any fail we will exit
+ run_test("test_insufficient_args", test_insufficient_args);
+ run_test("test_incompatible_args", test_incompatible_args);
+ run_test("test_middles", test_middles);
+ run_test("test_outputs", test_outputs);
+ run_test("test_random_perms", test_random_perms);
+
+ //passed
+ return 0;
+}
diff --git a/tests/test-parse-xml2.cpp b/tests/test-parse-xml2.cpp
new file mode 100644
index 0000000..44ce7d9
--- /dev/null
+++ b/tests/test-parse-xml2.cpp
@@ -0,0 +1,162 @@
+#include <iostream>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <cassert>
+
+#include <boost/make_shared.hpp>
+
+#include "osmtypes.hpp"
+#include "parse-xml2.hpp"
+#include "output.hpp"
+#include "options.hpp"
+#include "text-tree.hpp"
+#include "keyvals.hpp"
+
+void exit_nicely()
+{
+ fprintf(stderr, "Error occurred, cleaning up\n");
+ exit(1);
+}
+
+struct test_middle_t : public middle_t {
+ virtual ~test_middle_t() {}
+
+ int start(const options_t *out_options_) { return 0; }
+ void stop(void) { }
+ void cleanup(void) { }
+ void analyze(void) { }
+ void end(void) { }
+ void commit(void) { }
+
+ int nodes_set(osmid_t id, double lat, double lon, struct keyval *tags) { return 0; }
+ int nodes_get_list(struct osmNode *out, const osmid_t *nds, int nd_count) const { return 0; }
+
+ int ways_set(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags) { return 0; }
+ int ways_get(osmid_t id, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) const { return 0; }
+ int ways_get_list(const osmid_t *ids, int way_count, osmid_t *way_ids, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) const { return 0; }
+
+ int relations_set(osmid_t id, struct member *members, int member_count, struct keyval *tags) { return 0; }
+ int relations_get(osmid_t id, struct member **members, int *member_count, struct keyval *tags) const { return 0; }
+
+ void iterate_ways(pending_processor& pf) { }
+ void iterate_relations(pending_processor& pf) { }
+
+ virtual size_t pending_count() const { return 0; }
+
+ std::vector<osmid_t> relations_using_way(osmid_t way_id) const { return std::vector<osmid_t>(); }
+
+ virtual boost::shared_ptr<const middle_query_t> get_instance() const {return boost::shared_ptr<const middle_query_t>();}
+};
+
+struct test_output_t : public output_t {
+ uint64_t sum_ids, num_nodes, num_ways, num_relations, num_nds, num_members;
+
+ explicit test_output_t(const options_t &options_)
+ : output_t(NULL, options_), sum_ids(0), num_nodes(0), num_ways(0), num_relations(0),
+ num_nds(0), num_members(0) {
+ }
+
+ explicit test_output_t(const test_output_t &other)
+ : output_t(this->m_mid, this->m_options), sum_ids(0), num_nodes(0), num_ways(0), num_relations(0),
+ num_nds(0), num_members(0) {
+ }
+
+ virtual ~test_output_t() {
+ }
+
+ boost::shared_ptr<output_t> clone(const middle_query_t *cloned_middle) const{
+ test_output_t *clone = new test_output_t(*this);
+ clone->m_mid = cloned_middle;
+ return boost::shared_ptr<output_t>(clone);
+ }
+
+ int node_add(osmid_t id, double lat, double lon, struct keyval *tags) {
+ assert(id > 0);
+ sum_ids += id;
+ num_nodes += 1;
+ return 0;
+ }
+
+ int way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags) {
+ assert(id > 0);
+ sum_ids += id;
+ num_ways += 1;
+ assert(node_count >= 0);
+ num_nds += uint64_t(node_count);
+ return 0;
+ }
+
+ int relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags) {
+ assert(id > 0);
+ sum_ids += id;
+ num_relations += 1;
+ assert(member_count >= 0);
+ num_members += uint64_t(member_count);
+ return 0;
+ }
+
+ int start() { return 0; }
+ int connect(int startTransaction) { return 0; }
+ void stop() { }
+ void commit() { }
+ void cleanup(void) { }
+ void close(int stopTransaction) { }
+
+ void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { }
+ int pending_way(osmid_t id, int exists) { return 0; }
+
+ void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { }
+ int pending_relation(osmid_t id, int exists) { return 0; }
+
+ int node_modify(osmid_t id, double lat, double lon, struct keyval *tags) { return 0; }
+ int way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags) { return 0; }
+ int relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags) { return 0; }
+
+ int node_delete(osmid_t id) { return 0; }
+ int way_delete(osmid_t id) { return 0; }
+ int relation_delete(osmid_t id) { return 0; }
+
+};
+
+
+void assert_equal(uint64_t actual, uint64_t expected) {
+ if (actual != expected) {
+ std::cerr << "Expected " << expected << ", but got " << actual << ".\n";
+ exit(1);
+ }
+}
+
+int main(int argc, char *argv[]) {
+ char *srcdir = getenv("srcdir");
+
+ if (srcdir == NULL) {
+ std::cerr << "$srcdir not set!\n";
+ return 1;
+ }
+
+ std::string inputfile = std::string(srcdir) + std::string("/tests/test_multipolygon.osm");
+
+ options_t options;
+ boost::shared_ptr<reprojection> projection(new reprojection(PROJ_SPHERE_MERC));
+ options.projection = projection;
+
+ boost::shared_ptr<test_output_t> out_test(new test_output_t(options));
+ osmdata_t osmdata(boost::make_shared<test_middle_t>(), out_test);
+
+ parse_xml2_t parser(0, false, projection, 0, 0, 0, 0);
+
+ int ret = parser.streamFile(inputfile.c_str(), 0, &osmdata);
+ if (ret != 0) {
+ return ret;
+ }
+
+ assert_equal(out_test->sum_ids, 73514L);
+ assert_equal(out_test->num_nodes, 353L);
+ assert_equal(out_test->num_ways, 140L);
+ assert_equal(out_test->num_relations, 40L);
+ assert_equal(out_test->num_nds, 495L);
+ assert_equal(out_test->num_members, 146L);
+
+ return 0;
+}
diff --git a/tests/test-pgsql-escape.cpp b/tests/test-pgsql-escape.cpp
new file mode 100644
index 0000000..6c2fbdd
--- /dev/null
+++ b/tests/test-pgsql-escape.cpp
@@ -0,0 +1,7 @@
+#include "pgsql.hpp"
+
+int main(int argc, char *argv[]) {
+ std::string sql;
+ escape("farmland", sql);
+ return sql.compare("farmland") != 0;
+}
diff --git a/tests/test_output_pgsql_route_rel.osm b/tests/test_output_pgsql_route_rel.osm
new file mode 100644
index 0000000..e6298e3
--- /dev/null
+++ b/tests/test_output_pgsql_route_rel.osm
@@ -0,0 +1,15 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<osm version='0.6' generator='hand'>
+ <node id='1' version='1' visible='true' lat='49' lon='-122.5' />
+ <node id='2' version='1' visible='true' lat='49' lon='-122.51' />
+ <way id='1' version='1' visible='true'>
+ <nd ref='1' />
+ <nd ref='2' />
+ <tag k='highway' v='primary' />
+ </way>
+ <relation id='1' version='1' visible='true'>
+ <member type="way" ref="1" role=""/>
+ <tag k="type" v="route"/>
+ <tag k="route" v="road"/>
+ </relation>
+</osm>
diff --git a/tests/test_output_pgsql_way_area.osm b/tests/test_output_pgsql_way_area.osm
new file mode 100644
index 0000000..c5b3612
--- /dev/null
+++ b/tests/test_output_pgsql_way_area.osm
@@ -0,0 +1,13 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<osm version='0.6' generator='hand'>
+ <node id='1' version='1' visible='true' lat='49' lon='-122.5' />
+ <node id='2' version='1' visible='true' lat='49' lon='-122.51' />
+ <node id='3' version='1' visible='true' lat='49.01' lon='-122.5' />
+ <way id='1' version='1' visible='true'>
+ <nd ref='1' />
+ <nd ref='2' />
+ <nd ref='3' />
+ <nd ref='1' />
+ <tag k='leisure' v='playground' />
+ </way>
+</osm>
diff --git a/text-tree.c b/text-tree.cpp
similarity index 50%
rename from text-tree.c
rename to text-tree.cpp
index 7670e11..36616b4 100644
--- a/text-tree.c
+++ b/text-tree.cpp
@@ -3,56 +3,50 @@
* Storage of reference counted text strings
* used by keyvals.c to store the key/value strings
*/
-#define _GNU_SOURCE
+
#include <string.h>
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
-#include "text-tree.h"
-
-struct tree_context *tree_ctx = NULL;
+#include "text-tree.hpp"
-int text_compare(const void *pa, const void *pb, void *rb_param)
+namespace
{
- struct text_node *a = (struct text_node *)pa;
- struct text_node *b = (struct text_node *)pb;
+ int text_compare(const void *pa, const void *pb, void *rb_param)
+ {
+ struct text_node *a = (struct text_node *)pa;
+ struct text_node *b = (struct text_node *)pb;
- rb_param = NULL;
- return strcmp(a->str, b->str);
-}
-
-struct tree_context *text_init(void)
-{
- struct tree_context *context;
- struct rb_table *table = rb_create (text_compare, NULL, NULL);
+ rb_param = NULL;
+ return strcmp(a->str, b->str);
+ }
- assert(table);
- context = calloc(1, sizeof(*context));
- assert(context);
- context->table = table;
- tree_ctx = context;
- return context;
+ void text_free(void *pa, void *rb_param)
+ {
+ struct text_node *a = (struct text_node *)pa;
+ rb_param = NULL;
+ free(a->str);
+ free(a);
+ }
}
-void text_free(void *pa, void *rb_param)
+text_tree::text_tree()
{
- struct text_node *a = (struct text_node *)pa;
- rb_param = NULL;
- free(a->str);
- free(a);
+ table = rb_create (text_compare, NULL, NULL);
+ assert(table);
}
-const char *text_get(struct tree_context *context, const char *text)
+const char *text_tree::text_get(const char *text)
{
struct text_node *node, *dupe;
- node = malloc(sizeof(*node));
+ node = (struct text_node *)malloc(sizeof(*node));
assert(node);
node->str = strdup(text);
assert(node->str);
node->ref = 0;
- dupe = rb_insert(context->table, (void *)node);
+ dupe = (struct text_node *)rb_insert(table, (void *)node);
if (dupe) {
free(node->str);
free(node);
@@ -64,32 +58,28 @@ const char *text_get(struct tree_context *context, const char *text)
}
}
-
-void text_release(struct tree_context *context, const char *text)
+void text_tree::text_release(const char *text)
{
struct text_node *node, find;
find.str = (char *)text;
find.ref = 0;
- node = rb_find(context->table, (void *)&find);
+ node = (struct text_node *)rb_find(table, (void *)&find);
if (!node) {
fprintf(stderr, "failed to find '%s'\n", text);
return;
}
node->ref--;
if (!node->ref) {
- rb_delete (context->table, &find);
+ rb_delete (table, &find);
free(node->str);
free(node);
}
}
-void text_exit(void)
+text_tree::~text_tree()
{
- struct tree_context *context = tree_ctx;
- rb_destroy(context->table, text_free);
- free(context);
- tree_ctx = NULL;
+ rb_destroy(table, text_free);
}
#if 0
int main(int argc, char **argv)
diff --git a/text-tree.h b/text-tree.h
deleted file mode 100644
index c309fb0..0000000
--- a/text-tree.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#ifndef TEXT_TREE_H
-#define TEXT_TREE_H
-
-#include "rb.h"
-
-struct tree_context {
- struct rb_table *table;
-};
-
-extern struct tree_context *tree_ctx;
-
-struct text_node {
- char *str;
- int ref;
-};
-
-struct tree_context *text_init(void);
-void text_exit(void);
-const char *text_get(struct tree_context *context, const char *text);
-void text_release(struct tree_context *context, const char *text);
-#endif
diff --git a/text-tree.hpp b/text-tree.hpp
new file mode 100644
index 0000000..51473d5
--- /dev/null
+++ b/text-tree.hpp
@@ -0,0 +1,21 @@
+#ifndef TEXT_TREE_H
+#define TEXT_TREE_H
+
+#include "rb.hpp"
+
+struct text_tree {
+ struct rb_table *table;
+
+ text_tree();
+ ~text_tree();
+ const char *text_get(const char *text);
+ void text_release(const char *text);
+};
+
+struct text_node {
+ char *str;
+ int ref;
+};
+
+
+#endif
diff --git a/util.cpp b/util.cpp
new file mode 100644
index 0000000..8b834c3
--- /dev/null
+++ b/util.cpp
@@ -0,0 +1,10 @@
+#include "util.hpp"
+
+namespace util {
+
+ void exit_nicely() {
+ fprintf(stderr, "Error occurred, cleaning up\n");
+ exit(EXIT_FAILURE);
+ }
+
+}
diff --git a/util.hpp b/util.hpp
new file mode 100644
index 0000000..87d9069
--- /dev/null
+++ b/util.hpp
@@ -0,0 +1,19 @@
+#ifndef UTIL_H
+#define UTIL_H
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+namespace util {
+ inline int double_to_fix(const double x, const int scale) {
+ return x * scale + 0.4;
+ }
+ inline double fix_to_double(const int x, const int scale) {
+ return (double)x / scale;
+ }
+ void exit_nicely();
+}
+
+
+#endif
diff --git a/wildcmp.c b/wildcmp.cpp
similarity index 93%
rename from wildcmp.c
rename to wildcmp.cpp
index 02785e8..81b302d 100644
--- a/wildcmp.c
+++ b/wildcmp.cpp
@@ -1,12 +1,12 @@
/* Wildcard matching.
heavily based on wildcmp.c copyright 2002 Jim Kent
-
+
*/
#include <ctype.h>
-#include "wildcmp.h"
+#include "wildcmp.hpp"
-static int subMatch(char *str, char *wild)
+static int subMatch(const char *str, const char *wild)
/* Returns number of characters that match between str and wild up
* to the next wildcard in wild (or up to end of string.). */
{
@@ -27,14 +27,14 @@ for(;;)
}
}
-int wildMatch(char *wildCard, char *string)
+int wildMatch(const char *wildCard, const char *string)
/* does a case sensitive wild card match with a string.
* * matches any string or no character.
* ? matches any single character.
* anything else etc must match the character exactly.
returns NO_MATCH, FULL_MATCH or WC_MATCH defined in wildcmp.h
-
+
*/
{
int matchStar = 0;
@@ -75,7 +75,7 @@ NEXT_WILD:
default:
{
if(matchStar)
- {
+ {
for(;;)
{
if(*string == 0) /* if out of string no match */
diff --git a/wildcmp.h b/wildcmp.hpp
similarity index 64%
rename from wildcmp.h
rename to wildcmp.hpp
index 0321f15..ff88287 100644
--- a/wildcmp.h
+++ b/wildcmp.hpp
@@ -5,6 +5,6 @@
#define FULL_MATCH 1
#define WC_MATCH 2
-int wildMatch(char *wildCard, char *string);
+int wildMatch(const char *wildCard, const char *string);
#endif
diff --git a/win_fsync.h b/win_fsync.h
new file mode 100644
index 0000000..b03b7c2
--- /dev/null
+++ b/win_fsync.h
@@ -0,0 +1,71 @@
+#ifndef WIN_FSYNC_H
+#define WIN_FSYNC_H
+
+/* Emulate fsync on platforms that lack it, primarily Windows and
+ cross-compilers like MinGW.
+
+ This is derived from sqlite3 sources.
+ http://www.sqlite.org/cvstrac/rlog?f=sqlite/src/os_win.c
+ http://www.sqlite.org/copyright.html
+
+ Written by Richard W.M. Jones <rjones.at.redhat.com>
+
+ Copyright (C) 2008-2014 Free Software Foundation, Inc.
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */
+
+#include <windows.h>
+#include <io.h>
+#include <errno.h>
+
+inline int fsync (int fd)
+{
+ HANDLE h = (HANDLE) _get_osfhandle (fd);
+ DWORD err;
+
+ if (h == INVALID_HANDLE_VALUE)
+ {
+ errno = EBADF;
+ return -1;
+ }
+
+ if (!FlushFileBuffers (h))
+ {
+ /* Translate some Windows errors into rough approximations of Unix
+ * errors. MSDN is useless as usual - in this case it doesn't
+ * document the full range of errors.
+ */
+ err = GetLastError ();
+ switch (err)
+ {
+ case ERROR_ACCESS_DENIED:
+ /* For a read-only handle, fsync should succeed, even though we have
+ no way to sync the access-time changes. */
+ return 0;
+
+ /* eg. Trying to fsync a tty. */
+ case ERROR_INVALID_HANDLE:
+ errno = EINVAL;
+ break;
+
+ default:
+ errno = EIO;
+ }
+ return -1;
+ }
+
+ return 0;
+}
+
+#endif
\ No newline at end of file
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/osm2pgsql.git
More information about the Pkg-grass-devel
mailing list