[osm2pgsql] 01/08: Imported Upstream version 0.87.2
Sebastiaan Couwenberg
sebastic at moszumanska.debian.org
Tue Feb 24 08:23:19 UTC 2015
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository osm2pgsql.
commit a4edaef0ec1b3341c88ca5a70ed75a2944add6dd
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Tue Feb 24 09:00:35 2015 +0100
Imported Upstream version 0.87.2
---
README.md | 12 +
building.lua | 124 ----
configure.ac | 31 +-
default.style | 14 +-
docs/migrations.md | 22 +
docs/multi.md | 7 +-
docs/usage.md | 7 +-
expire-tiles.cpp | 1 +
expire-tiles.hpp | 3 +-
geometry-builder.cpp | 43 +-
m4/ax_lua.m4 | 234 +++---
middle-pgsql.cpp | 51 +-
multi.lua | 130 ++++
multi.style.json | 41 +
options.cpp | 12 +-
osmdata.cpp | 14 +-
osmtypes.hpp | 7 -
output-gazetteer.cpp | 1847 +++++++++++++++-------------------------------
output-gazetteer.hpp | 267 ++++++-
output-pgsql.cpp | 42 +-
output.hpp | 2 +-
pgsql.cpp | 43 +-
pgsql.hpp | 14 +-
table.cpp | 16 +-
table.hpp | 2 +-
tests/regression-test.py | 12 +-
util.cpp | 5 +-
util.hpp | 4 -
28 files changed, 1338 insertions(+), 1669 deletions(-)
diff --git a/README.md b/README.md
index ce1b0cb..a37391e 100644
--- a/README.md
+++ b/README.md
@@ -62,6 +62,14 @@ sudo yum install gcc-c++ boost-devel libxml2-devel geos-devel \
postgresql-devel bzip2-devel proj-devel protobuf-compiler
```
+To install on a FreeBSD system, use
+
+```sh
+pkg install devel/git devel/autoconf devel/automake devel/gmake devel/libtool \
+ textproc/libxml2 graphics/geos graphics/proj databases/postgresql94-client \
+ devel/boost-libs devel/protobuf-c lang/lua52 devel/pkgconf
+```
+
Then you should be able to bootstrap the build system:
./autogen.sh
@@ -73,6 +81,10 @@ And then run the standard GNU build install:
Please see `./configure --help` for more options on how to control the build
process.
+On FreeBSD instead bootstrap and then run
+
+ LUA_LIB=`pkg-config --libs lua-5.2` ./configure --without-lockfree && gmake && gmake install
+
## Usage ##
Osm2pgsql has one program, the executable itself, which has **43** command line
diff --git a/building.lua b/building.lua
deleted file mode 100644
index 93e002f..0000000
--- a/building.lua
+++ /dev/null
@@ -1,124 +0,0 @@
-tags = { 'building', 'shop', 'amenity' }
-
-function filter_tags_generic(keyvalues, nokeys)
- filter = 0
- tagcount = 0
-
- --if there were no tags passed in, ie keyvalues is empty
- if nokeys == 0 then
- filter = 1
- return filter, keyvalues
- end
-
- --remove anything we dont care about
- for i,k in ipairs(tags) do
- if keyvalues[k] then
- tagcount = tagcount + 1
- end
- end
-
- --if we didnt find any tags we care about
- if tagcount == 0 then
- filter = 1
- end
-
- --tell the caller whether we think we want this feature or not and give back the modified tags
- return filter, keyvalues
-end
-
-function nodes_proc (keyvalues, nokeys)
- --we dont care about nodes at all so filter all of them
- filter = 1
- return filter, keyvalues
-end
-
-function rels_proc (keyvalues, nokeys)
- --let the generic filtering do its job
- filter, keyvalues = filter_tags_generic(keyvalues, nokeys)
- if filter == 1 then
- return filter, keyvalues
- end
-
- --dont keep any relations with types other than multipolygon
- if keyvalues["type"] ~= "multipolygon" then
- filter = 1
- return filter, keyvalues
- end
-
- --let the caller know if its a keeper or not and give back the modified tags
- return filter, keyvalues
-end
-
-function ways_proc (keyvalues, nokeys)
- filter = 0
-
- --let the generic filtering do its job
- filter, keyvalues = filter_tags_generic(keyvalues, nokeys)
- poly = (filter + 1) % 2
- roads = 0
-
- --let the caller know if its a keeper or not and give back the modified tags
- --also tell it whether or not its a polygon or road
- return filter, keyvalues, poly, roads
-end
-
-function rel_members_proc (keyvalues, keyvaluemembers, roles, membercount)
-
- filter = 0
- boundary = 0
- polygon = 0
- roads = 0
-
- --mark each way of the relation to tell the caller if its going
- --to be used in the relation or by itself as its own standalone way
- --we start by assuming each way will not be used as part of the relation
- membersuperseeded = {}
- for i = 1, membercount do
- membersuperseeded[i] = 0
- end
-
- --remember the type on the relation and erase it from the tags
- type = keyvalues["type"]
- keyvalues["type"] = nil
-
- if (type == "multipolygon") and keyvalues["boundary"] == nil then
- --check if this relation has tags we care about
- polygon = 1
- filter, keyvalues = filter_tags_generic(keyvalues, 1)
-
- --if the relation didn't have the tags we need go grab the tags from
- --any members that are marked as outers of the multipolygon
- if (filter == 1) then
- for i = 1,membercount do
- if (roles[i] == "outer") then
- for j,k in ipairs(tags) do
- v = keyvaluemembers[i][k]
- if v then
- keyvalues[k] = v
- filter = 0
- end
- end
- end
- end
- end
- if filter == 1 then
- return filter, keyvalues, membersuperseeded, boundary, polygon, roads
- end
-
- --for each tag of each member if the relation have the tag or has a non matching value for it
- --then we say the member will not be used in the relation and is there for not superseeded
- --ie it is kept as a standalone way
- for i = 1,membercount do
- superseeded = 1
- for k,v in pairs(keyvaluemembers[i]) do
- if ((keyvalues[k] == nil) or (keyvalues[k] ~= v)) then
- superseeded = 0;
- break
- end
- end
- membersuperseeded[i] = superseeded
- end
- end
-
- return filter, keyvalues, membersuperseeded, boundary, polygon, roads
-end
diff --git a/configure.ac b/configure.ac
index 8b6f9bc..3b08a07 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,5 +1,5 @@
dnl Process this file with autoconf to produce a configure script.
-AC_INIT(osm2pgsql, 0.87.1)
+AC_INIT(osm2pgsql, 0.87.2)
dnl Required autoconf version
AC_PREREQ(2.61)
@@ -49,17 +49,6 @@ AC_SUBST(LFS_CFLAGS)
AC_CHECK_FUNC(lseek64,[AC_DEFINE(HAVE_LSEEK64, [1], [lseek64 is present])],[AX_COMPILE_CHECK_SIZEOF(off_t)])
AC_CHECK_FUNCS([posix_fallocate posix_fadvise sync_file_range fork])
-
-dnl legacy 32bit ID mode
-AC_ARG_ENABLE([64bit-ids],
- AS_HELP_STRING([--disable-64bit-ids], [Disable 64bit IDs for OSM IDs]),
- [ if test "$enableval" = "yes"
- then
- AC_DEFINE(OSMID64, [1], [Enable 64bit OSM IDs])
- fi
- ], [ AC_DEFINE(OSMID64, [1], [Enable 64bit OSM IDs])])
-
-
dnl Check for libxml2 library
AX_LIB_XML2
if test "$HAVE_XML2" = "no"
@@ -82,11 +71,7 @@ then
fi
dnl Check for Geos library
-AX_LIB_GEOS
-if test "x$GEOS_VERSION" = "x"
-then
- AC_MSG_ERROR([geos library not found]);
-fi
+AX_LIB_GEOS([3.1])
dnl Check for Proj library
AX_LIB_PROJ
@@ -128,6 +113,18 @@ AX_BOOST_SYSTEM
AX_BOOST_FILESYSTEM
AX_BOOST_THREAD
+dnl Check if Boost is recent enough for lockfree, and if it hasn't been overridden
+AC_ARG_WITH([lockfree],
+ [AS_HELP_STRING([--without-lockfree],
+ [disable lockfree queue])],
+ [],
+ [with_lockfree=yes])
+
+if test "x$with_lockfree" = "xyes"
+then
+ AX_BOOST_BASE([1.53], AC_DEFINE([HAVE_LOCKFREE], [1], [Using lockfree queue]), [])
+fi
+
dnl Check for Lua libraries and headers
AX_PROG_LUA([5.0],[],[
AX_LUA_HEADERS([
diff --git a/default.style b/default.style
index 1c53f73..690f3d1 100644
--- a/default.style
+++ b/default.style
@@ -68,10 +68,14 @@
# way_area - datatype real. The area of the way, in the units of the projection
# (e.g. square mercator meters). Only applies to areas
#
-# osm_user, osm_uid, osm_version, osm_timestamp - datatype text. Used with the
-# --extra-attributes option to include metadata in the database. If importing
-# with both --hstore and --extra-attributes the meta-data will end up in the
-# tags hstore column regardless of the style file.
+# osm_user - datatype text
+# osm_uid - datatype integer
+# osm_version - datatype integer
+# osm_changeset - datatype integer
+# osm_timestamp - datatype timestamptz(0).
+# Used with the --extra-attributes option to include metadata in the database.
+# If importing with both --hstore and --extra-attributes the meta-data will
+# end up in the tags hstore column regardless of the style file.
# OsmType Tag DataType Flags
node,way access text linear
@@ -126,7 +130,7 @@ node,way power_source text linear
node,way public_transport text polygon
node,way railway text linear
node,way ref text linear
-node,way religion text nocache
+node,way religion text
node,way route text linear
node,way service text linear
node,way shop text polygon
diff --git a/docs/migrations.md b/docs/migrations.md
new file mode 100644
index 0000000..10457c8
--- /dev/null
+++ b/docs/migrations.md
@@ -0,0 +1,22 @@
+# Migrations between versions #
+
+Some osm2pgsql changes have slightly changed the database schema it expects. If
+updating an old database, a migration may be needed. The migrations here assume
+the default `planet_osm` prefix.
+
+## 0.87.0 pending removal ##
+
+0.87.0 moved the in-database tracking of pending ways and relations to
+in-memory, for an increase in speed. This requires removal of the pending
+column and a partial index associated with it.
+
+```sql
+ALTER TABLE planet_osm_ways DROP COLUMN pending;
+ALTER TABLE planet_osm_rels DROP COLUMN pending;
+```
+
+## 32 bit to 64 bit ID migration ##
+
+Old databases may have been imported with 32 bit node IDs, while current OSM
+data requires 64 bit IDs. A database this old should not be migrated, but
+reloaded. To migrate, the type of ID columns needs to be changed to `bigint`.
diff --git a/docs/multi.md b/docs/multi.md
index ebffd19..67ebc63 100644
--- a/docs/multi.md
+++ b/docs/multi.md
@@ -47,9 +47,14 @@ following: `tablespace-index`, `tablespace-data`, `enable-hstore`,
may be specified via an array of strings named `hstores`. Finally standard columns
may be specified via an array of objects named `tags` with each object containing
a `name` and a postgres `type`. Note you may also set `flags` on each tag as with
-the standard osm2pgsql style file.`flags` is formated exactly as in the style file
+the standard osm2pgsql style file. `flags` is formated exactly as in the style file
as a string of flag names seprated by commas.
+## Example ##
+An example based on the above is in [multi.lua](../multi.lua) and
+[multi.style.json](../multi.style.json). It creates two tables, one for bus stops
+and one for buildings. Some Lua processing is done to unify tagging values.
+
## Importing ##
See: [Importing](pgsql.md#importing).
diff --git a/docs/usage.md b/docs/usage.md
index b381f02..1cc6c02 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -14,8 +14,11 @@ use them.
automatically detected for some reason.
* ``--output`` specifies if the output backend is the default
- [pgsql](pgsql.md), the [gazetteer](gazetteer.md) output used by Nominatim, or
- null, which emits no output.
+ [`pgsql`](pgsql.md), the [`gazetteer`](gazetteer.md) output used by Nominatim,
+ the new [`multi`](multi.md) backend which allows more customization of tables,
+ or `null`, which emits no output to the backend.
+
+ `null` will create slim tables if ``--slim`` is also used.
## Performance
diff --git a/expire-tiles.cpp b/expire-tiles.cpp
index a5eb66f..5c088f7 100644
--- a/expire-tiles.cpp
+++ b/expire-tiles.cpp
@@ -18,6 +18,7 @@
#include "geometry-builder.hpp"
#include "pgsql.hpp"
#include "reprojection.hpp"
+#include "table.hpp"
#define EARTH_CIRCUMFERENCE 40075016.68
#define HALF_EARTH_CIRCUMFERENCE (EARTH_CIRCUMFERENCE / 2)
diff --git a/expire-tiles.hpp b/expire-tiles.hpp
index 6d36eaf..7dfca61 100644
--- a/expire-tiles.hpp
+++ b/expire-tiles.hpp
@@ -1,12 +1,13 @@
#ifndef EXPIRE_TILES_H
#define EXPIRE_TILES_H
-#include "table.hpp"
#include "options.hpp"
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
+class table_t;
+
struct expire_tiles : public boost::noncopyable {
explicit expire_tiles(const struct options_t *options);
~expire_tiles();
diff --git a/geometry-builder.cpp b/geometry-builder.cpp
index 8ae8020..5603665 100644
--- a/geometry-builder.cpp
+++ b/geometry-builder.cpp
@@ -30,17 +30,8 @@
#define GEOS_INLINE
#endif
-/* Need to know which geos version we have to work out which headers to include */
-#include <geos/version.h>
-
-/* geos (3.0.0+) */
-#if (GEOS_VERSION_MAJOR==3)
-#if (GEOS_VERSION_MINOR>=1)
-/* Prepared geometries are new in 3.1.0 */
-#define HAS_PREPARED_GEOMETRIES
#include <geos/geom/prep/PreparedGeometryFactory.h>
#include <geos/geom/prep/PreparedPolygon.h>
-#endif
#include <geos/geom/GeometryFactory.h>
#include <geos/geom/CoordinateSequenceFactory.h>
#include <geos/geom/Geometry.h>
@@ -58,13 +49,6 @@ using namespace geos::geom;
using namespace geos::io;
using namespace geos::util;
using namespace geos::operation::linemerge;
-#else
-/* geos-2.2.3 */
-#include <geos/geom.h>
-#include <geos/io.h>
-#include <geos/opLinemerge.h>
-using namespace geos;
-#endif
#include "geometry-builder.hpp"
@@ -347,9 +331,8 @@ geometry_builder::maybe_wkts_t geometry_builder::build_polygons(const osmNode *
std::auto_ptr<std::vector<Geometry*> > lines(new std::vector<Geometry*>);
GeometryFactory gf;
geom_ptr geom;
-#ifdef HAS_PREPARED_GEOMETRIES
geos::geom::prep::PreparedGeometryFactory pgf;
-#endif
+
maybe_wkts_t wkts(new std::vector<geometry_builder::wkt_t>);
@@ -413,19 +396,12 @@ geometry_builder::maybe_wkts_t geometry_builder::build_polygons(const osmNode *
{
if (polys[i].iscontained != 0) continue;
toplevelpolygons++;
-#ifdef HAS_PREPARED_GEOMETRIES
const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon);
-#endif
for (unsigned j=i+1; j < totalpolys; ++j)
{
-#ifdef HAS_PREPARED_GEOMETRIES
// Does preparedtoplevelpolygon contain the smaller polygon[j]?
if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon))
-#else
- // Does polygon[i] contain the smaller polygon[j]?
- if (polys[j].containedbyid == 0 && polys[i].polygon->contains(polys[j].polygon))
-#endif
{
// are we in a [i] contains [k] contains [j] situation
// which would actually make j top level
@@ -462,9 +438,7 @@ geometry_builder::maybe_wkts_t geometry_builder::build_polygons(const osmNode *
}
}
}
-#ifdef HAS_PREPARED_GEOMETRIES
- pgf.destroy(preparedtoplevelpolygon);
-#endif
+ pgf.destroy(preparedtoplevelpolygon);
}
// polys now is a list of polygons tagged with which ones are inside each other
@@ -599,9 +573,7 @@ geometry_builder::maybe_wkts_t geometry_builder::build_both(const osmNode * cons
std::auto_ptr<std::vector<Geometry*> > lines(new std::vector<Geometry*>);
GeometryFactory gf;
geom_ptr geom;
-#ifdef HAS_PREPARED_GEOMETRIES
geos::geom::prep::PreparedGeometryFactory pgf;
-#endif
maybe_wkts_t wkts(new std::vector<geometry_builder::wkt_t>);
@@ -694,19 +666,12 @@ geometry_builder::maybe_wkts_t geometry_builder::build_both(const osmNode * cons
{
if (polys[i].iscontained != 0) continue;
toplevelpolygons++;
-#ifdef HAS_PREPARED_GEOMETRIES
const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon);
-#endif
for (unsigned j=i+1; j < totalpolys; ++j)
{
-#ifdef HAS_PREPARED_GEOMETRIES
// Does preparedtoplevelpolygon contain the smaller polygon[j]?
if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon))
-#else
- // Does polygon[i] contain the smaller polygon[j]?
- if (polys[j].containedbyid == 0 && polys[i].polygon->contains(polys[j].polygon))
-#endif
{
// are we in a [i] contains [k] contains [j] situation
// which would actually make j top level
@@ -743,9 +708,7 @@ geometry_builder::maybe_wkts_t geometry_builder::build_both(const osmNode * cons
}
}
}
-#ifdef HAS_PREPARED_GEOMETRIES
- pgf.destroy(preparedtoplevelpolygon);
-#endif
+ pgf.destroy(preparedtoplevelpolygon);
}
// polys now is a list of polygons tagged with which ones are inside each other
diff --git a/m4/ax_lua.m4 b/m4/ax_lua.m4
index c6f5974..1553491 100644
--- a/m4/ax_lua.m4
+++ b/m4/ax_lua.m4
@@ -55,9 +55,6 @@
# version number greater or equal to MINIMUM-VERSION and less than
# TOO-BIG-VERSION will be accepted.
#
-# Version comparisons require the AX_COMPARE_VERSION macro, which is
-# provided by ax_compare_version.m4 from the Autoconf Archive.
-#
# The Lua version number, LUA_VERSION, is found from the interpreter, and
# substituted. LUA_PLATFORM is also found, but not currently supported (no
# standard representation).
@@ -109,6 +106,7 @@
# * /usr/include/lua/X.Y
# * /usr/include/luaXY
# * /usr/local/include/luaX.Y
+# * /usr/local/include/lua-X.Y
# * /usr/local/include/lua/X.Y
# * /usr/local/include/luaXY
#
@@ -153,8 +151,8 @@
#
# LICENSE
#
-# Copyright (c) 2013 Tim Perkins <tprk77 at gmail.com>
-# Copyright (c) 2013 Reuben Thomas <rrt at sc3d.org>
+# Copyright (c) 2014 Reuben Thomas <rrt at sc3d.org>
+# Copyright (c) 2014 Tim Perkins <tprk77 at gmail.com>
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
@@ -182,7 +180,7 @@
# modified version of the Autoconf Macro, you may extend this special
# exception to the GPL to apply to your modified version as well.
-#serial 20
+#serial 36
dnl =========================================================================
dnl AX_PROG_LUA([MINIMUM-VERSION], [TOO-BIG-VERSION],
@@ -190,12 +188,16 @@ dnl [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])
dnl =========================================================================
AC_DEFUN([AX_PROG_LUA],
[
+ dnl Check for required tools.
+ AC_REQUIRE([AC_PROG_GREP])
+ AC_REQUIRE([AC_PROG_SED])
+
dnl Make LUA a precious variable.
AC_ARG_VAR([LUA], [The Lua interpreter, e.g. /usr/bin/lua5.1])
dnl Find a Lua interpreter.
m4_define_default([_AX_LUA_INTERPRETER_LIST],
- [lua lua5.2 lua5.1 lua50])
+ [lua lua5.2 lua52 lua5.1 lua51 lua50])
m4_if([$1], [],
[ dnl No version check is needed. Find any Lua interpreter.
@@ -203,12 +205,14 @@ AC_DEFUN([AX_PROG_LUA],
[AC_PATH_PROGS([LUA], [_AX_LUA_INTERPRETER_LIST], [:])])
ax_display_LUA='lua'
- dnl At least check if this is a Lua interpreter.
- AC_MSG_CHECKING([if $LUA is a Lua interpreter])
- _AX_LUA_CHK_IS_INTRP([$LUA],
- [AC_MSG_RESULT([yes])],
- [ AC_MSG_RESULT([no])
- AC_MSG_ERROR([not a Lua interpreter])
+ AS_IF([test "x$LUA" != 'x:'],
+ [ dnl At least check if this is a Lua interpreter.
+ AC_MSG_CHECKING([if $LUA is a Lua interpreter])
+ _AX_LUA_CHK_IS_INTRP([$LUA],
+ [AC_MSG_RESULT([yes])],
+ [ AC_MSG_RESULT([no])
+ AC_MSG_ERROR([not a Lua interpreter])
+ ])
])
],
[ dnl A version check is needed.
@@ -256,21 +260,26 @@ AC_DEFUN([AX_PROG_LUA],
m4_default([$4], [AC_MSG_ERROR([cannot find suitable Lua interpreter])])
],
[ dnl Query Lua for its version number.
- AC_CACHE_CHECK([for $ax_display_LUA version], [ax_cv_lua_version],
- [ ax_cv_lua_version=`$LUA -e "print(_VERSION)" | \
- sed "s|^Lua \(.*\)|\1|" | \
- grep -o "^@<:@0-9@:>@\+\\.@<:@0-9@:>@\+"`
+ AC_CACHE_CHECK([for $ax_display_LUA version],
+ [ax_cv_lua_version],
+ [ dnl Get the interpreter version in X.Y format. This should work for
+ dnl interpreters version 5.0 and beyond.
+ ax_cv_lua_version=[`$LUA -e '
+ -- return a version number in X.Y format
+ local _, _, ver = string.find(_VERSION, "^Lua (%d+%.%d+)")
+ print(ver)'`]
])
AS_IF([test "x$ax_cv_lua_version" = 'x'],
[AC_MSG_ERROR([invalid Lua version number])])
AC_SUBST([LUA_VERSION], [$ax_cv_lua_version])
- AC_SUBST([LUA_SHORT_VERSION], [`echo "$LUA_VERSION" | sed 's|\.||'`])
+ AC_SUBST([LUA_SHORT_VERSION], [`echo "$LUA_VERSION" | $SED 's|\.||'`])
dnl The following check is not supported:
dnl At times (like when building shared libraries) you may want to know
dnl which OS platform Lua thinks this is.
- AC_CACHE_CHECK([for $ax_display_LUA platform], [ax_cv_lua_platform],
- [ax_cv_lua_platform=`$LUA -e "print('unknown')"`])
+ AC_CACHE_CHECK([for $ax_display_LUA platform],
+ [ax_cv_lua_platform],
+ [ax_cv_lua_platform=[`$LUA -e 'print("unknown")'`]])
AC_SUBST([LUA_PLATFORM], [$ax_cv_lua_platform])
dnl Use the values of $prefix and $exec_prefix for the corresponding
@@ -298,9 +307,9 @@ AC_DEFUN([AX_PROG_LUA],
_AX_LUA_FND_PRFX_PTH([$LUA], [$ax_lua_prefix], [package.path])
AS_IF([test "x$ax_lua_prefixed_path" != 'x'],
[ dnl Fix the prefix.
- _ax_strip_prefix=`echo "$ax_lua_prefix" | sed 's|.|.|g'`
+ _ax_strip_prefix=`echo "$ax_lua_prefix" | $SED 's|.|.|g'`
ax_cv_lua_luadir=`echo "$ax_lua_prefixed_path" | \
- sed "s,^$_ax_strip_prefix,$LUA_PREFIX,"`
+ $SED "s|^$_ax_strip_prefix|$LUA_PREFIX|"`
])
])
AC_SUBST([luadir], [$ax_cv_lua_luadir])
@@ -322,12 +331,12 @@ AC_DEFUN([AX_PROG_LUA],
dnl Try to find a path with the prefix.
_AX_LUA_FND_PRFX_PTH([$LUA],
- [$ax_lua_exec_prefix], [package.cpathd])
+ [$ax_lua_exec_prefix], [package.cpath])
AS_IF([test "x$ax_lua_prefixed_path" != 'x'],
[ dnl Fix the prefix.
- _ax_strip_prefix=`echo "$ax_lua_exec_prefix" | sed 's|.|.|g'`
+ _ax_strip_prefix=`echo "$ax_lua_exec_prefix" | $SED 's|.|.|g'`
ax_cv_lua_luaexecdir=`echo "$ax_lua_prefixed_path" | \
- sed "s,^$_ax_strip_prefix,$LUA_EXEC_PREFIX,"`
+ $SED "s|^$_ax_strip_prefix|$LUA_EXEC_PREFIX|"`
])
])
AC_SUBST([luaexecdir], [$ax_cv_lua_luaexecdir])
@@ -341,7 +350,7 @@ AC_DEFUN([AX_PROG_LUA],
dnl AX_WITH_LUA is now the same thing as AX_PROG_LUA.
AC_DEFUN([AX_WITH_LUA],
[
- AC_MSG_WARN([[$0 is deprecated, please use AX_PROG_LUA]])
+ AC_MSG_WARN([[$0 is deprecated, please use AX_PROG_LUA instead]])
AX_PROG_LUA
])
@@ -351,8 +360,19 @@ dnl _AX_LUA_CHK_IS_INTRP(PROG, [ACTION-IF-TRUE], [ACTION-IF-FALSE])
dnl =========================================================================
AC_DEFUN([_AX_LUA_CHK_IS_INTRP],
[
- dnl Just print _VERSION because all Lua interpreters have this global.
- AS_IF([$1 -e "print('Hello ' .. _VERSION .. '!')" &>/dev/null],
+ dnl A minimal Lua factorial to prove this is an interpreter. This should work
+ dnl for Lua interpreters version 5.0 and beyond.
+ _ax_lua_factorial=[`$1 2>/dev/null -e '
+ -- a simple factorial
+ function fact (n)
+ if n == 0 then
+ return 1
+ else
+ return n * fact(n-1)
+ end
+ end
+ print("fact(5) is " .. fact(5))'`]
+ AS_IF([test "$_ax_lua_factorial" = 'fact(5) is 120'],
[$2], [$3])
])
@@ -363,48 +383,70 @@ dnl [ACTION-IF-TRUE], [ACTION-IF-FALSE])
dnl =========================================================================
AC_DEFUN([_AX_LUA_CHK_VER],
[
- _ax_test_ver=`$1 -e "print(_VERSION)" 2>/dev/null | \
- sed "s|^Lua \(.*\)|\1|" | grep -o "^@<:@0-9@:>@\+\\.@<:@0-9@:>@\+"`
- AS_IF([test "x$_ax_test_ver" = 'x'],
- [_ax_test_ver='0'])
- AX_COMPARE_VERSION([$_ax_test_ver], [ge], [$2])
- m4_if([$3], [], [],
- [ AS_IF([$ax_compare_version],
- [AX_COMPARE_VERSION([$_ax_test_ver], [lt], [$3])])
- ])
- AS_IF([$ax_compare_version], [$4], [$5])
+ dnl Check that the Lua version is within the bounds. Only the major and minor
+ dnl version numbers are considered. This should work for Lua interpreters
+ dnl version 5.0 and beyond.
+ _ax_lua_good_version=[`$1 -e '
+ -- a script to compare versions
+ function verstr2num(verstr)
+ local _, _, majorver, minorver = string.find(verstr, "^(%d+)%.(%d+)")
+ if majorver and minorver then
+ return tonumber(majorver) * 100 + tonumber(minorver)
+ end
+ end
+ local minver = verstr2num("$2")
+ local _, _, trimver = string.find(_VERSION, "^Lua (.*)")
+ local ver = verstr2num(trimver)
+ local maxver = verstr2num("$3") or 1e9
+ if minver <= ver and ver < maxver then
+ print("yes")
+ else
+ print("no")
+ end'`]
+ AS_IF([test "x$_ax_lua_good_version" = "xyes"],
+ [$4], [$5])
])
dnl =========================================================================
-dnl _AX_LUA_FND_PRFX_PTH(PROG, PREFIX, LUA-PATH-VARIABLE)
+dnl _AX_LUA_FND_PRFX_PTH(PROG, PREFIX, SCRIPT-OR-MODULE-DIR)
dnl =========================================================================
AC_DEFUN([_AX_LUA_FND_PRFX_PTH],
[
- dnl Invokes the Lua interpreter PROG to print the path variable
- dnl LUA-PATH-VARIABLE, usually package.path or package.cpath. Paths are
- dnl then matched against PREFIX. The first path to begin with PREFIX is set
- dnl to ax_lua_prefixed_path.
-
- ax_lua_prefixed_path=''
- _ax_package_paths=`$1 -e 'print($3)' 2>/dev/null | sed 's|;|\n|g'`
- dnl Try the paths in order, looking for the prefix.
- for _ax_package_path in $_ax_package_paths; do
- dnl Copy the path, up to the use of a Lua wildcard.
- _ax_path_parts=`echo "$_ax_package_path" | sed 's|/|\n|g'`
- _ax_reassembled=''
- for _ax_path_part in $_ax_path_parts; do
- echo "$_ax_path_part" | grep '\?' >/dev/null && break
- _ax_reassembled="$_ax_reassembled/$_ax_path_part"
- done
- dnl Check the path against the prefix.
- _ax_package_path=$_ax_reassembled
- if echo "$_ax_package_path" | grep "^$2" >/dev/null; then
- dnl Found it.
- ax_lua_prefixed_path=$_ax_package_path
- break
- fi
- done
+ dnl Get the script or module directory by querying the Lua interpreter,
+ dnl filtering on the given prefix, and selecting the shallowest path. If no
+ dnl path is found matching the prefix, the result will be an empty string.
+ dnl The third argument determines the type of search, it can be 'script' or
+ dnl 'module'. Supplying 'script' will perform the search with package.path
+ dnl and LUA_PATH, and supplying 'module' will search with package.cpath and
+ dnl LUA_CPATH. This is done for compatibility with Lua 5.0.
+
+ ax_lua_prefixed_path=[`$1 -e '
+ -- get the path based on search type
+ local searchtype = "$3"
+ local paths = ""
+ if searchtype == "script" then
+ paths = (package and package.path) or LUA_PATH
+ elseif searchtype == "module" then
+ paths = (package and package.cpath) or LUA_CPATH
+ end
+ -- search for the prefix
+ local prefix = "$2"
+ local minpath = ""
+ local mindepth = 1e9
+ string.gsub(paths, "(@<:@^;@:>@+)",
+ function (path)
+ path = string.gsub(path, "%?.*$", "")
+ path = string.gsub(path, "/@<:@^/@:>@*$", "")
+ if string.find(path, prefix) then
+ local depth = string.len(string.gsub(path, "@<:@^/@:>@", ""))
+ if depth < mindepth then
+ minpath = path
+ mindepth = depth
+ end
+ end
+ end)
+ print(minpath)'`]
])
@@ -425,12 +467,14 @@ AC_DEFUN([AX_LUA_HEADERS],
AC_ARG_VAR([LUA_INCLUDE], [The Lua includes, e.g. -I/usr/include/lua5.1])
dnl Some default directories to search.
- LUA_SHORT_VERSION=`echo "$LUA_VERSION" | sed 's|\.||'`
+ LUA_SHORT_VERSION=`echo "$LUA_VERSION" | $SED 's|\.||'`
m4_define_default([_AX_LUA_INCLUDE_LIST],
[ /usr/include/lua$LUA_VERSION \
+ /usr/include/lua-$LUA_VERSION \
/usr/include/lua/$LUA_VERSION \
/usr/include/lua$LUA_SHORT_VERSION \
/usr/local/include/lua$LUA_VERSION \
+ /usr/local/include/lua-$LUA_VERSION \
/usr/local/include/lua/$LUA_VERSION \
/usr/local/include/lua$LUA_SHORT_VERSION \
])
@@ -470,14 +514,17 @@ AC_DEFUN([AX_LUA_HEADERS],
AS_IF([test "x$ac_cv_header_lua_h" = 'xyes'],
[ dnl Make a program to print LUA_VERSION defined in the header.
- dnl TODO This probably shouldn't be a runtime test.
-
- AC_CACHE_CHECK([for Lua header version],
- [ax_cv_lua_header_version],
- [ _ax_lua_saved_cppflags=$CPPFLAGS
- CPPFLAGS="$CPPFLAGS $LUA_INCLUDE"
- AC_RUN_IFELSE(
- [ AC_LANG_SOURCE([[
+ dnl TODO It would be really nice if we could do this without compiling a
+ dnl program, then it would work when cross compiling. But I'm not sure how
+ dnl to do this reliably. For now, assume versions match when cross compiling.
+
+ AS_IF([test "x$cross_compiling" != 'xyes'],
+ [ AC_CACHE_CHECK([for Lua header version],
+ [ax_cv_lua_header_version],
+ [ _ax_lua_saved_cppflags=$CPPFLAGS
+ CPPFLAGS="$CPPFLAGS $LUA_INCLUDE"
+ AC_RUN_IFELSE(
+ [ AC_LANG_SOURCE([[
#include <lua.h>
#include <stdlib.h>
#include <stdio.h>
@@ -487,23 +534,26 @@ int main(int argc, char ** argv)
exit(EXIT_SUCCESS);
}
]])
+ ],
+ [ ax_cv_lua_header_version=`./conftest$EXEEXT p | \
+ $SED -n "s|^Lua \(@<:@0-9@:>@\{1,\}\.@<:@0-9@:>@\{1,\}\).\{0,\}|\1|p"`
+ ],
+ [ax_cv_lua_header_version='unknown'])
+ CPPFLAGS=$_ax_lua_saved_cppflags
+ ])
+
+ dnl Compare this to the previously found LUA_VERSION.
+ AC_MSG_CHECKING([if Lua header version matches $LUA_VERSION])
+ AS_IF([test "x$ax_cv_lua_header_version" = "x$LUA_VERSION"],
+ [ AC_MSG_RESULT([yes])
+ ax_header_version_match='yes'
],
- [ ax_cv_lua_header_version=`./conftest$EXEEXT p | \
- sed "s|^Lua \(.*\)|\1|" | \
- grep -o "^@<:@0-9@:>@\+\\.@<:@0-9@:>@\+"`
- ],
- [ax_cv_lua_header_version='unknown'])
- CPPFLAGS=$_ax_lua_saved_cppflags
- ])
-
- dnl Compare this to the previously found LUA_VERSION.
- AC_MSG_CHECKING([if Lua header version matches $LUA_VERSION])
- AS_IF([test "x$ax_cv_lua_header_version" = "x$LUA_VERSION"],
- [ AC_MSG_RESULT([yes])
- ax_header_version_match='yes'
+ [ AC_MSG_RESULT([no])
+ ax_header_version_match='no'
+ ])
],
- [ AC_MSG_RESULT([no])
- ax_header_version_match='no'
+ [ AC_MSG_WARN([cross compiling so assuming header version number matches])
+ ax_header_version_match='yes'
])
])
@@ -520,7 +570,7 @@ int main(int argc, char ** argv)
dnl AX_LUA_HEADERS_VERSION no longer exists, use AX_LUA_HEADERS.
AC_DEFUN([AX_LUA_HEADERS_VERSION],
[
- AC_MSG_WARN([[$0 is deprecated, please use AX_LUA_HEADERS]])
+ AC_MSG_WARN([[$0 is deprecated, please use AX_LUA_HEADERS instead]])
])
@@ -575,7 +625,13 @@ AC_DEFUN([AX_LUA_LIBS],
dnl Try to find the Lua libs.
_ax_lua_saved_libs=$LIBS
LIBS="$LIBS $LUA_LIB"
- AC_SEARCH_LIBS([lua_load], [lua$LUA_VERSION lua$LUA_SHORT_VERSION lua],
+ AC_SEARCH_LIBS([lua_load],
+ [ lua$LUA_VERSION \
+ lua$LUA_SHORT_VERSION \
+ lua-$LUA_VERSION \
+ lua-$LUA_SHORT_VERSION \
+ lua \
+ ],
[_ax_found_lua_libs='yes'],
[_ax_found_lua_libs='no'],
[$_ax_lua_extra_libs])
diff --git a/middle-pgsql.cpp b/middle-pgsql.cpp
index 6377047..62eeda7 100644
--- a/middle-pgsql.cpp
+++ b/middle-pgsql.cpp
@@ -687,9 +687,6 @@ void middle_pgsql_t::iterate_ways(middle_t::pending_processor& pf)
// Make sure we're out of copy mode */
pgsql_endCopy( way_table );
- // at this point we always want to be in append mode, to not delete and recreate the node cache file */
- if (out_options->flat_node_cache_enabled) persistent_cache.reset(new node_persistent_cache(out_options,1,cache));
-
// enqueue the jobs
osmid_t id;
while(id_tracker::is_valid(id = ways_pending_tracker->pop_mark()))
@@ -882,9 +879,6 @@ void middle_pgsql_t::iterate_relations(pending_processor& pf)
// Make sure we're out of copy mode */
pgsql_endCopy( rel_table );
- // at this point we always want to be in append mode, to not delete and recreate the node cache file */
- if (out_options->flat_node_cache_enabled) persistent_cache.reset(new node_persistent_cache(out_options, 1, cache));
-
// enqueue the jobs
osmid_t id;
while(id_tracker::is_valid(id = rels_pending_tracker->pop_mark()))
@@ -1226,6 +1220,9 @@ void middle_pgsql_t::commit(void) {
tables[i].transactionMode = 0;
}
}
+ // Make sure the flat nodes are committed to disk or there will be
+ // surprises later.
+ if (out_options->flat_node_cache_enabled) persistent_cache.reset();
}
void *middle_pgsql_t::pgsql_stop_one(void *arg)
@@ -1238,37 +1235,16 @@ void *middle_pgsql_t::pgsql_stop_one(void *arg)
fprintf(stderr, "Stopping table: %s\n", table->name);
pgsql_endCopy(table);
time(&start);
- if (!out_options->droptemp)
+ if (out_options->droptemp)
{
- if (build_indexes && table->array_indexes) {
- char *buffer = (char *) malloc(strlen(table->array_indexes) + 99);
- // we need to insert before the TABLESPACE setting, if any */
- const char *insertpos = strstr(table->array_indexes, "TABLESPACE");
- if (!insertpos) insertpos = strchr(table->array_indexes, ';');
-
- /* automatically insert FASTUPDATE=OFF when creating,
- indexes for PostgreSQL 8.4 and higher
- see http://lists.openstreetmap.org/pipermail/dev/2011-January/021704.html */
- if (insertpos && PQserverVersion(sql_conn) >= 80400) {
- fprintf(stderr, "Building index on table: %s (fastupdate=off)\n", table->name);
- size_t n_chars = insertpos - table->array_indexes;
- strncpy(buffer, table->array_indexes, n_chars);
- // strncpy doesn't necessarily null-terminate, so we need to add that
- buffer[n_chars] = '\0';
- strcat(buffer, " WITH (FASTUPDATE=OFF)");
- strcat(buffer, insertpos);
- } else {
- fprintf(stderr, "Building index on table: %s\n", table->name);
- strcpy(buffer, table->array_indexes);
- }
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", buffer);
- free(buffer);
- }
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE %s", table->name);
}
- else
+ else if (build_indexes && table->array_indexes)
{
- pgsql_exec(sql_conn, PGRES_COMMAND_OK, "drop table %s", table->name);
+ fprintf(stderr, "Building index on table: %s\n", table->name);
+ pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", table->array_indexes);
}
+
PQfinish(sql_conn);
table->sql_conn = NULL;
time(&end);
@@ -1370,7 +1346,7 @@ middle_pgsql_t::middle_pgsql_t()
/*copy*/ "COPY %p_ways FROM STDIN;\n",
/*analyze*/ "ANALYZE %p_ways;\n",
/*stop*/ "COMMIT;\n",
- /*array_indexes*/ "CREATE INDEX %p_ways_nodes ON %p_ways USING gin (nodes) {TABLESPACE %i};\n"
+ /*array_indexes*/ "CREATE INDEX %p_ways_nodes ON %p_ways USING gin (nodes) WITH (FASTUPDATE=OFF) {TABLESPACE %i};\n"
));
tables.push_back(table_desc(
/*table = t_rel,*/
@@ -1390,7 +1366,7 @@ middle_pgsql_t::middle_pgsql_t()
/*copy*/ "COPY %p_rels FROM STDIN;\n",
/*analyze*/ "ANALYZE %p_rels;\n",
/*stop*/ "COMMIT;\n",
- /*array_indexes*/ "CREATE INDEX %p_rels_parts ON %p_rels USING gin (parts) {TABLESPACE %i};\n"
+ /*array_indexes*/ "CREATE INDEX %p_rels_parts ON %p_rels USING gin (parts) WITH (FASTUPDATE=OFF) {TABLESPACE %i};\n"
));
// set up the rest of the variables from the tables.
@@ -1419,7 +1395,10 @@ boost::shared_ptr<const middle_query_t> middle_pgsql_t::get_instance() const {
//NOTE: this is thread safe for use in pending async processing only because
//during that process they are only read from
mid->cache = cache;
- mid->persistent_cache = persistent_cache;
+ // The persistent cache on the other hand is not thread-safe for reading,
+ // so we create one per instance.
+ if (out_options->flat_node_cache_enabled)
+ mid->persistent_cache.reset(new node_persistent_cache(out_options,1,cache));
// We use a connection per table to enable the use of COPY */
for(int i=0; i<num_tables; i++) {
diff --git a/multi.lua b/multi.lua
new file mode 100644
index 0000000..f1969c0
--- /dev/null
+++ b/multi.lua
@@ -0,0 +1,130 @@
+-- This is an example Lua transform for a multi style
+-- It is not intended for use directly with --tag-transform-script but
+-- for use from multi.style.json
+--
+-- See docs/lua.md and docs/multi.md
+
+-- A function to determine if the tags make the object "interesting" to the buildings table
+function building_interesting (keyvals)
+ return keyvals["building"] and keyvals["building"] ~= "no"
+end
+
+function building_ways (keyvals, num_keys)
+ return generic_ways(building_interesting, keyvals)
+end
+
+function building_rels (keyvals, num_keys)
+ return generic_rels(building_interesting, keyvals)
+end
+
+function builing_rel_members (keyvals, keyvaluemembers, roles, membercount)
+ return generic_rel_members(building_interesting, keyvals, keyvaluemembers, roles, membercount)
+end
+
+function bus_nodes_proc (keyvals, num_tags)
+ if keyvals["highway"] == "bus_stop" then
+ tags = keyvals
+ -- Turns values into true unless they're no, leaving empty tags as null.
+ -- This lets these columns be boolean, vastly simplifying stylesheet
+ -- logic
+ if tags["shelter"] then
+ -- Checks if the value is no or false, then returns a string that can be turned into a boolean
+ tags["shelter"] = ((tags["shelter"] ~= "no" and tags["shelter"] ~= "false") and "true" or "false")
+ end
+ if tags["bench"] then
+ tags["bench"] = ((tags["bench"] ~= "no" and tags["bench"] ~= "false") and "true" or "false")
+ end
+ if tags["wheelchair"] then
+ tags["wheelchair"] = ((tags["wheelchair"] ~= "no" and tags["wheelchair"] ~= "false") and "true" or "false")
+ end
+ return 0, tags
+ else
+ return 1, {}
+ end
+end
+
+-- This function gets rid of something we don't care about
+function drop_all (...)
+ return 1, {}
+end
+
+-- A generic way to process ways, given a function which determines if tags are interesting
+function generic_ways (f, kv)
+ if f(kv) then
+ tags = kv
+ return 0, tags, 1, 0
+ else
+ return 1, {}, 0, 0
+ end
+end
+
+-- A generic way to process relations, given a function which determines if tags are interesting
+function generic_rels (f, kv)
+ if kv["type"] == "multipolygon" and f(kv) then
+ tags = kv
+ return 0, tags
+ else
+ return 1, {}
+ end
+end
+
+-- Basically taken from style.lua
+function generic_rel_members (f, keyvals, keyvaluemembers, roles, membercount)
+ filter = 0
+ boundary = 0
+ polygon = 0
+ roads = 0
+
+ --mark each way of the relation to tell the caller if its going
+ --to be used in the relation or by itself as its own standalone way
+ --we start by assuming each way will not be used as part of the relation
+ membersuperseeded = {}
+ for i = 1, membercount do
+ membersuperseeded[i] = 0
+ end
+
+ --remember the type on the relation and erase it from the tags
+ type = keyvals["type"]
+ keyvals["type"] = nil
+
+ if (type == "multipolygon") and keyvals["boundary"] == nil then
+ --check if this relation has tags we care about
+ polygon = 1
+ filter = f(keyvals)
+
+ --if the relation didn't have the tags we need go grab the tags from
+ --any members that are marked as outers of the multipolygon
+ if (filter == 1) then
+ for i = 1,membercount do
+ if (roles[i] == "outer") then
+ for j,k in ipairs(tags) do
+ v = keyvaluemembers[i][k]
+ if v then
+ keyvals[k] = v
+ filter = 0
+ end
+ end
+ end
+ end
+ end
+ if filter == 1 then
+ return filter, keyvals, membersuperseeded, boundary, polygon, roads
+ end
+
+ --for each tag of each member if the relation have the tag or has a non matching value for it
+ --then we say the member will not be used in the relation and is there for not superseeded
+ --ie it is kept as a standalone way
+ for i = 1,membercount do
+ superseeded = 1
+ for k,v in pairs(keyvaluemembers[i]) do
+ if ((keyvals[k] == nil) or (keyvals[k] ~= v)) then
+ superseeded = 0;
+ break
+ end
+ end
+ membersuperseeded[i] = superseeded
+ end
+ end
+
+ return filter, keyvals, membersuperseeded, boundary, polygon, roads
+end
diff --git a/multi.style.json b/multi.style.json
new file mode 100644
index 0000000..e063812
--- /dev/null
+++ b/multi.style.json
@@ -0,0 +1,41 @@
+/* This is an example multi-backend style file that comes with osm2pgsql.
+ *
+ * It contains comments so is invalid JSON but the parser doesn't complain.
+ *
+ * To use this file, pass it in as a style file with the --style switch
+ */
+[
+ {
+ "name": "stops",
+ "type": "point",
+ "tagtransform": "multi.lua",
+ "tagtransform-node-function": "bus_nodes_proc",
+ "tagtransform-way-function": "drop_all",
+ "tagtransform-relation-function": "drop_all",
+ "tagtransform-relation-member-function": "drop_all",
+ "tags": [
+ /* We don't need a column for highway, since it's always highway=bus_stop */
+ {"name": "name", "type": "text"},
+ {"name": "ref", "type": "text"},
+ {"name": "operator", "type": "text"},
+ /* By using boolean columns stylesheet processing can be simplified */
+ {"name": "shelter", "type": "boolean"},
+ {"name": "bench", "type": "boolean"},
+ {"name": "wheelchair", "type": "boolean"}
+ ]
+ },
+ {
+ "name": "buildings",
+ "type": "polygon",
+ "tagtransform": "multi.lua",
+ "tagtransform-node-function": "drop_all",
+ "tagtransform-way-function": "building_ways",
+ "tagtransform-relation-function": "building_rels",
+ "tagtransform-relation-member-function": "builing_rel_members",
+ "tags": [
+ {"name": "building", "type": "text"},
+ {"name": "shop", "type": "text"},
+ {"name": "amenity", "type": "text"}
+ ]
+ }
+]
diff --git a/options.cpp b/options.cpp
index 6691ca2..eea61bc 100644
--- a/options.cpp
+++ b/options.cpp
@@ -96,7 +96,8 @@ namespace
reduces the RAM usage but is much slower. This switch is\n\
required if you want to update with --append later.\n\
-S|--style Location of the style file. Defaults to\n");
- printf(" %s/default.style.\n", OSM2PGSQL_DATADIR);
+ printf("\
+ %s/default.style.\n", OSM2PGSQL_DATADIR);
printf("%s", "\
-C|--cache Use up to this many MB for caching nodes (default: 800)\n\
\n\
@@ -157,10 +158,12 @@ namespace
us twice as much virtual memory, but no more physical \n\
memory.\n");
#ifdef __amd64__
- printf(" The default is \"optimized\"\n");
+ printf("\
+ The default is \"optimized\"\n");
#else
/* use "chunked" as a default in 32 bit compilations, as it is less wasteful of virtual memory than "optimized"*/
- printf(" The default is \"sparse\"\n");
+ printf("\
+ The default is \"sparse\"\n");
#endif
printf("%s", "\
--flat-nodes Specifies the flat file to use to persistently store node \n\
@@ -180,7 +183,8 @@ namespace
-r|--input-reader Input frontend.\n\
libxml2 - Parse XML using libxml2. (default)\n");
#ifdef BUILD_READER_PBF
- printf(" pbf - OSM binary format.\n");
+ printf("\
+ pbf - OSM binary format.\n");
#endif
printf("\
-O|--output Output backend.\n\
diff --git a/osmdata.cpp b/osmdata.cpp
index 66d0459..eedf942 100644
--- a/osmdata.cpp
+++ b/osmdata.cpp
@@ -9,9 +9,9 @@
#include <stdexcept>
#include <utility>
+#include <cstdio>
-#if BOOST_VERSION < 105300
-#else
+#ifdef HAVE_LOCKFREE
#include <boost/atomic.hpp>
#endif
@@ -167,7 +167,7 @@ struct pending_threaded_processor : public middle_t::pending_processor {
typedef std::vector<boost::shared_ptr<output_t> > output_vec_t;
typedef std::pair<boost::shared_ptr<const middle_query_t>, output_vec_t> clone_t;
-#if BOOST_VERSION < 105300
+#ifndef HAVE_LOCKFREE
static void do_jobs(output_vec_t const& outputs, pending_queue_t& queue, size_t& ids_done, boost::mutex& mutex, int append, bool ways) {
while (true) {
//get the job off the queue synchronously
@@ -209,7 +209,7 @@ struct pending_threaded_processor : public middle_t::pending_processor {
//starts up count threads and works on the queue
pending_threaded_processor(boost::shared_ptr<middle_query_t> mid, const output_vec_t& outs, size_t thread_count, size_t job_count, int append)
-#if BOOST_VERSION < 105300
+#ifndef HAVE_LOCKFREE
//note that we cant hint to the stack how large it should be ahead of time
//we could use a different datastructure like a deque or vector but then
//the outputs the enqueue jobs would need the version check for the push(_back) method
@@ -256,7 +256,7 @@ struct pending_threaded_processor : public middle_t::pending_processor {
//make the threads and start them
for (size_t i = 0; i < clones.size(); ++i) {
-#if BOOST_VERSION < 105300
+#ifndef HAVE_LOCKFREE
workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), boost::ref(mutex), append, true));
#else
workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), append, true));
@@ -307,7 +307,7 @@ struct pending_threaded_processor : public middle_t::pending_processor {
//make the threads and start them
for (size_t i = 0; i < clones.size(); ++i) {
-#if BOOST_VERSION < 105300
+#ifndef HAVE_LOCKFREE
workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), boost::ref(mutex), append, false));
#else
workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), append, false));
@@ -353,7 +353,7 @@ private:
//job queue
pending_queue_t queue;
-#if BOOST_VERSION < 105300
+#ifndef HAVE_LOCKFREE
//how many ids within the job have been processed
size_t ids_done;
//so the threads can manage some of the shared state
diff --git a/osmtypes.hpp b/osmtypes.hpp
index c884b3c..1684458 100644
--- a/osmtypes.hpp
+++ b/osmtypes.hpp
@@ -9,17 +9,10 @@
#include <inttypes.h>
#include <config.h>
-#ifdef OSMID64
typedef int64_t osmid_t;
#define strtoosmid strtoll
#define PRIdOSMID PRId64
#define POSTGRES_OSMID_TYPE "int8"
-#else
-typedef int32_t osmid_t;
-#define strtoosmid strtol
-#define PRIdOSMID PRId32
-#define POSTGRES_OSMID_TYPE "int4"
-#endif
enum OsmType { OSMTYPE_WAY, OSMTYPE_NODE, OSMTYPE_RELATION };
diff --git a/output-gazetteer.cpp b/output-gazetteer.cpp
index d2d4757..689e0fb 100644
--- a/output-gazetteer.cpp
+++ b/output-gazetteer.cpp
@@ -1,10 +1,7 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
#include <libpq-fe.h>
#include <boost/make_shared.hpp>
#include <boost/algorithm/string/predicate.hpp>
+#include <boost/format.hpp>
#include "osmtypes.hpp"
#include "middle.hpp"
@@ -14,19 +11,9 @@
#include "options.hpp"
#include "util.hpp"
-#define SRID (reproj->project_getprojinfo()->srs)
-
-#define CREATE_KEYVALUETYPE_TYPE \
- "CREATE TYPE keyvalue AS (" \
- " key TEXT," \
- " value TEXT" \
- ")"
+#include <algorithm>
-#define CREATE_WORDSCORE_TYPE \
- "CREATE TYPE wordscore AS (" \
- " word TEXT," \
- " score FLOAT" \
- ")"
+#define SRID (reproj->project_getprojinfo()->srs)
#define CREATE_PLACE_TABLE \
"CREATE TABLE place (" \
@@ -50,919 +37,540 @@
#define CREATE_PLACE_ID_INDEX \
"CREATE INDEX place_id_idx ON place USING BTREE (osm_type, osm_id) %s %s"
-#define TAGINFO_NODE 0x1u
-#define TAGINFO_WAY 0x2u
-#define TAGINFO_AREA 0x4u
-
-void output_gazetteer_t::require_slim_mode(void)
-{
- if (!m_options.slim)
- {
- fprintf(stderr, "Cannot apply diffs unless in slim mode\n");
- util::exit_nicely();
- }
- return;
-}
+enum { BUFFER_SIZE = 4092 };
-void output_gazetteer_t::copy_data(const char *sql)
+void place_tag_processor::clear()
{
- unsigned int sqlLen = strlen(sql);
-
- /* Make sure we have an active copy */
- if (!CopyActive)
- {
- pgsql_exec(Connection, PGRES_COPY_IN, "COPY place (osm_type, osm_id, class, type, name, admin_level, housenumber, street, addr_place, isin, postcode, country_code, extratags, geometry) FROM STDIN");
- CopyActive = 1;
- }
-
- /* If the combination of old and new data is too big, flush old data */
- if (BufferLen + sqlLen > BUFFER_SIZE - 10)
- {
- pgsql_CopyData("place", Connection, Buffer);
- BufferLen = 0;
- }
-
- /*
- * If new data by itself is too big, output it immediately,
- * otherwise just add it to the buffer.
- */
- if (sqlLen > BUFFER_SIZE - 10)
- {
- pgsql_CopyData("Place", Connection, sql);
- sqlLen = 0;
- }
- else if (sqlLen > 0)
- {
- strcpy(Buffer + BufferLen, sql);
- BufferLen += sqlLen;
- sqlLen = 0;
- }
-
- /* If we have completed a line, output it */
- if (BufferLen > 0 && Buffer[BufferLen-1] == '\n')
- {
- pgsql_CopyData("place", Connection, Buffer);
- BufferLen = 0;
- }
-
- return;
-}
-
-void output_gazetteer_t::stop_copy(void)
+ // set members to sane defaults
+ src = NULL;
+ admin_level = ADMINLEVEL_NONE;
+ countrycode = 0;
+ housenumber.assign("\\N");
+ street = 0;
+ addr_place = 0;
+ postcode = 0;
+
+ places.clear();
+ extratags.clear();
+ address.clear();
+ names.clear();
+}
+
+struct UnnamedPredicate
{
- PGresult *res;
-
- /* Do we have a copy active? */
- if (!CopyActive) return;
-
- /* Terminate the copy */
- if (PQputCopyEnd(Connection, NULL) != 1)
- {
- fprintf(stderr, "COPY_END for place failed: %s\n", PQerrorMessage(Connection));
- util::exit_nicely();
- }
-
- /* Check the result */
- res = PQgetResult(Connection);
- if (PQresultStatus(res) != PGRES_COMMAND_OK)
- {
- fprintf(stderr, "COPY_END for place failed: %s\n", PQerrorMessage(Connection));
- PQclear(res);
- util::exit_nicely();
- }
-
- /* Discard the result */
- PQclear(res);
+ bool operator()(const keyval *val) const {
+ return val->key == "natural" ||
+ val->key == "railway" ||
+ val->key == "waterway" ||
+ val->key == "boundary" ||
+ (val->key == "highway" &&
+ (val->value == "traffic_signals" ||
+ val->value == "service" ||
+ val->value == "cycleway" ||
+ val->value == "path" ||
+ val->value == "footway" ||
+ val->value == "steps" ||
+ val->value == "bridleway" ||
+ val->value == "track" ||
+ val->value == "byway" ||
+ boost::ends_with(val->value, "_link")));
+ }
+};
- /* We no longer have an active copy */
- CopyActive = 0;
+void place_tag_processor::process_tags(keyval *tags)
+{
+ bool placeadmin = false;
+ bool placehouse = false;
+ bool placebuilding = false;
+ const keyval *place = 0;
+ const keyval *junction = 0;
+ const keyval *landuse = 0;
+ bool isnamed = false;
+ bool isinterpolation = false;
+ const std::string *house_nr = 0;
+ const std::string *conscr_nr = 0;
+ const std::string *street_nr = 0;
+
+ clear();
+ src = tags;
+
+ for (keyval *item = tags->firstItem(); item; item = tags->nextItem(item)) {
+ if (item->key == "name:prefix") {
+ extratags.push_back(item);
+ } else if (item->key == "ref" ||
+ item->key == "int_ref" ||
+ item->key == "nat_ref" ||
+ item->key == "reg_ref" ||
+ item->key == "loc_ref" ||
+ item->key == "old_ref" ||
+ item->key == "iata" ||
+ item->key == "icao" ||
+ item->key == "operator" ||
+ item->key == "pcode" ||
+ boost::starts_with(item->key, "pcode:")) {
+ names.push_back(item);
+ } else if (item->key == "name" ||
+ boost::starts_with(item->key, "name:") ||
+ item->key == "int_name" ||
+ boost::starts_with(item->key, "int_name:") ||
+ item->key == "nat_name" ||
+ boost::starts_with(item->key, "nat_name:") ||
+ item->key == "reg_name" ||
+ boost::starts_with(item->key, "reg_name:") ||
+ item->key == "loc_name" ||
+ boost::starts_with(item->key, "loc_name:") ||
+ item->key == "old_name" ||
+ boost::starts_with(item->key, "old_name:") ||
+ item->key == "alt_name" ||
+ boost::starts_with(item->key, "alt_name:") ||
+ boost::starts_with(item->key, "alt_name_") ||
+ item->key == "official_name" ||
+ boost::starts_with(item->key, "official_name:") ||
+ item->key == "place_name" ||
+ boost::starts_with(item->key, "place_name:") ||
+ item->key == "short_name" ||
+ boost::starts_with(item->key, "short_name:") ||
+ item->key == "brand") {
+ names.push_back(item);
+ isnamed = true;
+ } else if (item->key == "addr:housename") {
+ names.push_back(item);
+ placehouse = true;
+ } else if (item->key == "emergency") {
+ if (item->value != "fire_hydrant" &&
+ item->value != "yes" &&
+ item->value != "no")
+ places.push_back(item);
+ } else if (item->key == "tourism" ||
+ item->key == "historic" ||
+ item->key == "military") {
+ if (item->value != "no" && item->value != "yes")
+ places.push_back(item);
+ } else if (item->key == "natural") {
+ if (item->value != "no" &&
+ item->value != "yes" &&
+ item->value != "costaline")
+ places.push_back(item);
+ } else if (item->key == "landuse") {
+ if (item->value == "cemetry")
+ places.push_back(item);
+ else
+ landuse = item;
+ } else if (item->key == "highway") {
+ if (item->value != "no" &&
+ item->value != "turning_circle" &&
+ item->value != "mini_roundabout" &&
+ item->value != "noexit" &&
+ item->value != "crossing")
+ places.push_back(item);
+ } else if (item->key == "railway") {
+ if (item->value != "level_crossing" &&
+ item->value != "no")
+ places.push_back(item);
+ } else if (item->key == "man_made") {
+ if (item->value != "survey_point" &&
+ item->value != "cutline")
+ places.push_back(item);
+ } else if (item->key == "aerialway") {
+ if (item->value != "pylon" &&
+ item->value != "no")
+ places.push_back(item);
+ } else if (item->key == "boundary") {
+ if (item->value == "administrative")
+ placeadmin = true;
+ places.push_back(item);
+ } else if (item->key == "aeroway" ||
+ item->key == "amenity" ||
+ item->key == "boundary" ||
+ item->key == "bridge" ||
+ item->key == "craft" ||
+ item->key == "leisure" ||
+ item->key == "office" ||
+ item->key == "shop" ||
+ item->key == "tunnel" ||
+ item->key == "mountain_pass") {
+ if (item->value != "no")
+ {
+ places.push_back(item);
+ }
+ } else if (item->key == "waterway") {
+ if (item->value != "riverbank")
+ places.push_back(item);
+ } else if (item->key == "place") {
+ place = item;
+ } else if (item->key == "junction") {
+ junction = item;
+ } else if (item->key == "addr:interpolation") {
+ housenumber.clear();
+ escape(item->value, housenumber);
+ isinterpolation = true;
+ } else if (item->key == "addr:housenumber") {
+ house_nr = &item->value;
+ placehouse = true;
+ } else if (item->key == "addr:conscriptionnumber") {
+ conscr_nr = &item->value;
+ placehouse = true;
+ } else if (item->key == "addr:streetnumber") {
+ street_nr = &item->value;
+ placehouse = true;
+ } else if (item->key == "addr:street") {
+ street = &item->value;
+ } else if (item->key == "addr:place") {
+ addr_place = &item->value;
+ } else if (item->key == "postal_code" ||
+ item->key == "postcode" ||
+ item->key == "addr:postcode" ||
+ item->key == "tiger:zip_left" ||
+ item->key == "tiger:zip_right") {
+ if (!postcode)
+ postcode = &item->value;
+ } else if (item->key == "country_code" ||
+ item->key == "ISO3166-1" ||
+ item->key == "is_in:country_code" ||
+ item->key == "addr:country" ||
+ item->key == "addr:country_code") {
+ if (item->value.length() == 2)
+ countrycode = &item->value;
+ } else if (boost::starts_with(item->key, "addr:") ||
+ item->key == "is_in" ||
+ boost::starts_with(item->key, "is_in:") ||
+ item->key == "tiger:county") {
+ address.push_back(item);
+ } else if (item->key == "admin_level") {
+ admin_level = atoi(item->value.c_str());
+ if (admin_level <= 0 || admin_level > 100)
+ admin_level = 100;
+ } else if (item->key == "tracktype" ||
+ item->key == "traffic_calming" ||
+ item->key == "service" ||
+ item->key == "cuisine" ||
+ item->key == "capital" ||
+ item->key == "dispensing" ||
+ item->key == "religion" ||
+ item->key == "denomination" ||
+ item->key == "sport" ||
+ item->key == "internet_access" ||
+ item->key == "lanes" ||
+ item->key == "surface" ||
+ item->key == "smoothness" ||
+ item->key == "width" ||
+ item->key == "est_width" ||
+ item->key == "incline" ||
+ item->key == "opening_hours" ||
+ item->key == "collection_times" ||
+ item->key == "service_times" ||
+ item->key == "disused" ||
+ item->key == "wheelchair" ||
+ item->key == "sac_scale" ||
+ item->key == "trail_visibility" ||
+ item->key == "mtb:scale" ||
+ item->key == "mtb:description" ||
+ item->key == "wood" ||
+ item->key == "drive_through" ||
+ item->key == "drive_in" ||
+ item->key == "access" ||
+ item->key == "vehicle" ||
+ item->key == "bicyle" ||
+ item->key == "foot" ||
+ item->key == "goods" ||
+ item->key == "hgv" ||
+ item->key == "motor_vehicle" ||
+ item->key == "motor_car" ||
+ boost::starts_with(item->key, "access:") ||
+ boost::starts_with(item->key, "contact:") ||
+ boost::starts_with(item->key, "drink:") ||
+ item->key == "oneway" ||
+ item->key == "date_on" ||
+ item->key == "date_off" ||
+ item->key == "day_on" ||
+ item->key == "day_off" ||
+ item->key == "hour_on" ||
+ item->key == "hour_off" ||
+ item->key == "maxweight" ||
+ item->key == "maxheight" ||
+ item->key == "maxspeed" ||
+ item->key == "fee" ||
+ item->key == "toll" ||
+ boost::starts_with(item->key, "toll:") ||
+ item->key == "charge" ||
+ item->key == "population" ||
+ item->key == "description" ||
+ item->key == "image" ||
+ item->key == "attribution" ||
+ item->key == "fax" ||
+ item->key == "email" ||
+ item->key == "url" ||
+ item->key == "website" ||
+ item->key == "phone" ||
+ item->key == "real_ale" ||
+ item->key == "smoking" ||
+ item->key == "food" ||
+ item->key == "camera" ||
+ item->key == "brewery" ||
+ item->key == "locality" ||
+ item->key == "wikipedia" ||
+ boost::starts_with(item->key, "wikipedia:")) {
+ extratags.push_back(item);
+ } else if (item->key == "building") {
+ placebuilding = true;
+ }
+ }
- return;
-}
+ // skip some tags, if they don't have a proper name (ref doesn't count)
+ if (!isnamed) {
+ if (!places.empty())
+ places.erase(std::remove_if(places.begin(), places.end(),
+ UnnamedPredicate()),
+ places.end());
+ }
-#if 0
-static void copy_error_data(const char *sql)
-{
- unsigned int sqlLen = strlen(sql);
+ if (isinterpolation) {
+ keyval *b = new keyval("place", "houses");
+ src->pushItem(b); // to make sure it gets deleted
+ places.push_back(b);
+ }
- /* Make sure we have an active copy */
- if (!CopyErrorActive)
- {
- pgsql_exec(ConnectionError, PGRES_COPY_IN, "COPY import_polygon_error (osm_type, osm_id, class, type, name, country_code, updated, errormessage, prevgeometry, newgeometry) FROM stdin;");
- CopyErrorActive = 1;
- }
+ if (place) {
+ if (isinterpolation ||
+ (placeadmin &&
+ place ->value != "island" &&
+ place ->value != "islet"))
+ extratags.push_back(place);
+ else
+ places.push_back(place);
+ }
- /* If the combination of old and new data is too big, flush old data */
- if (BufferErrorLen + sqlLen > BUFFER_SIZE - 10)
- {
- pgsql_CopyData("import_polygon_error", ConnectionError, BufferError);
- BufferErrorLen = 0;
- }
+ if (isnamed && places.empty()) {
+ if (junction)
+ places.push_back(junction);
+ else if (landuse)
+ places.push_back(landuse);
+ }
- /*
- * If new data by itself is too big, output it immediately,
- * otherwise just add it to the buffer.
- */
- if (sqlLen > BUFFER_SIZE - 10)
- {
- pgsql_CopyData("import_polygon_error", ConnectionError, sql);
- sqlLen = 0;
- }
- else if (sqlLen > 0)
- {
- strcpy(BufferError + BufferErrorLen, sql);
- BufferErrorLen += sqlLen;
- sqlLen = 0;
- }
+ if (places.empty()) {
+ if (placebuilding && (!names.empty() || placehouse || postcode)) {
+ keyval *b = new keyval("building", "yes");
+ src->pushItem(b); // to make sure it gets deleted
+ places.push_back(b);
+ } else if (placehouse) {
+ keyval *b = new keyval("place", "house");
+ src->pushItem(b); // to make sure it gets deleted
+ places.push_back(b);
+ } else if (postcode) {
+ keyval *b = new keyval("place", "postcode");
+ src->pushItem(b); // to make sure it gets deleted
+ places.push_back(b);
+ }
+ }
- /* If we have completed a line, output it */
- if (BufferErrorLen > 0 && BufferError[BufferErrorLen-1] == '\n')
- {
- pgsql_CopyData("place", ConnectionError, BufferError);
- BufferErrorLen = 0;
- }
+ // housenumbers
+ if (!isinterpolation) {
+ if (street_nr && conscr_nr) {
+ housenumber.clear();
+ escape(*conscr_nr, housenumber);
+ housenumber.append("/");
+ escape(*street_nr, housenumber);
+ } else if (conscr_nr) {
+ housenumber.clear();
+ escape(*conscr_nr, housenumber);
+ } else if (street_nr) {
+ housenumber.clear();
+ escape(*street_nr, housenumber);
+ } else if (house_nr) {
+ housenumber.clear();
+ escape(*house_nr, housenumber);
+ }
+ }
- return;
}
-static void stop_error_copy(void)
+void place_tag_processor::copy_out(char osm_type, osmid_t osm_id,
+ const std::string &wkt,
+ std::string &buffer)
{
- PGresult *res;
-
- /* Do we have a copy active? */
- if (!CopyErrorActive) return;
-
- /* Terminate the copy */
- if (PQputCopyEnd(ConnectionError, NULL) != 1)
- {
- fprintf(stderr, "COPY_END for import_polygon_error failed: %s\n", PQerrorMessage(ConnectionError));
- util::exit_nicely();
- }
-
- /* Check the result */
- res = PQgetResult(ConnectionError);
- if (PQresultStatus(res) != PGRES_COMMAND_OK)
- {
- fprintf(stderr, "COPY_END for import_polygon_error failed: %s\n", PQerrorMessage(ConnectionError));
- PQclear(res);
- util::exit_nicely();
- }
+ BOOST_FOREACH(const keyval* place, places) {
+ std::string name;
+ if (place->key == "bridge" || place->key == "tunnel") {
+ name = domain_name(place->key);
+ if (name.empty())
+ continue; // don't include unnamed bridges and tunnels
+ }
- /* Discard the result */
- PQclear(res);
+ // osm_type
+ buffer += osm_type;
+ buffer += '\t';
+ // osm_id
+ buffer += (single_fmt % osm_id).str();
+ // class
+ escape(place->key, buffer);
+ buffer += '\t';
+ // type
+ escape(place->value, buffer);
+ buffer += '\t';
+ // names
+ if (!name.empty()) {
+ buffer += name;
+ buffer += '\t';
+ } else if (!names.empty()) {
+ bool first = true;
+ // operator will be ignored on anything but these classes
+ // (amenity for restaurant and fuel)
+ bool shop = (place->key == "shop") ||
+ (place->key == "amenity") ||
+ (place->key == "tourism");
+ BOOST_FOREACH(const keyval *entry, names) {
+ if (!shop && (entry->key == "operator"))
+ continue;
+
+ if (first)
+ first = false;
+ else
+ buffer += ',';
+
+ buffer += "\"";
+ escape_array_record(entry->key, buffer);
+ buffer += "\"=>\"";
+ escape_array_record(entry->value, buffer);
+ buffer += "\"";
+ }
+ buffer += '\t';
+ } else
+ buffer += "\\N\t";
+ // admin_level
+ buffer += (single_fmt % admin_level).str();
+ // house number
+ buffer += housenumber;
+ buffer += '\t';
+ // street
+ copy_opt_string(street, buffer);
+ // addr_place
+ copy_opt_string(addr_place, buffer);
+ // isin
+ if (!address.empty()) {
+ BOOST_FOREACH(const keyval *entry, address) {
+ if (entry->key == "tiger:county") {
+ escape(std::string(entry->value, 0, entry->value.find(",")),
+ buffer);
+ buffer += " county";
+ } else {
+ escape(entry->value, buffer);
+ }
+ buffer += ',';
+ }
+ buffer[buffer.length() - 1] = '\t';
+ } else
+ buffer += "\\N\t";
+ // postcode
+ copy_opt_string(postcode, buffer);
+ // country code
+ copy_opt_string(countrycode, buffer);
+ // extra tags
+ if (extratags.empty()) {
+ buffer += "\\N\t";
+ } else {
+ bool first = true;
+ BOOST_FOREACH(const keyval *entry, extratags) {
+ if (first)
+ first = false;
+ else
+ buffer += ',';
+
+ buffer += "\"";
+ escape_array_record(entry->key, buffer);
+ buffer += "\"=>\"";
+ escape_array_record(entry->value, buffer);
+ buffer += "\"";
+ }
+ buffer += "\t";
+ }
+ // wkt
+ buffer += srid_str;
+ buffer += wkt;
+ buffer += '\n';
+ }
+}
- /* We no longer have an active copy */
- CopyErrorActive = 0;
- return;
-}
-#endif
-
-static int split_tags(struct keyval *tags, unsigned int flags,
- struct keyval *names, struct keyval *places,
- struct keyval *extratags, int* admin_level,
- struct keyval ** housenumber, struct keyval ** street,
- struct keyval ** addr_place, char ** isin,
- struct keyval ** postcode, struct keyval ** countrycode)
+void output_gazetteer_t::stop_copy(void)
{
- size_t subval;
- int placehouse = 0;
- int placebuilding = 0;
- int placeadmin = 0;
- struct keyval *landuse;
- struct keyval *place;
- struct keyval *item;
- struct keyval *conscriptionnumber;
- struct keyval *streetnumber;
-
- *admin_level = ADMINLEVEL_NONE;
- *housenumber = 0;
- *street = 0;
- *addr_place = 0;
- *isin = 0;
- int isinsize = 0;
- *postcode = 0;
- *countrycode = 0;
- landuse = 0;
- place = 0;
- conscriptionnumber = 0;
- streetnumber = 0;
-
- /* Loop over the tags */
- while ((item = tags->popItem()) != NULL)
- {
-
- /* If this is a name tag, add it to the name list */
- if (item->key == "ref" ||
- item->key == "int_ref" ||
- item->key == "nat_ref" ||
- item->key == "reg_ref" ||
- item->key == "loc_ref" ||
- item->key == "old_ref" ||
- item->key == "ncn_ref" ||
- item->key == "rcn_ref" ||
- item->key == "lcn_ref" ||
- item->key == "iata" ||
- item->key == "icao" ||
- item->key == "pcode:1" ||
- item->key == "pcode:2" ||
- item->key == "pcode:3" ||
- item->key == "un:pcode:1" ||
- item->key == "un:pcode:2" ||
- item->key == "un:pcode:3" ||
- item->key == "name" ||
- (strncmp(item->key.c_str(), "name:", 5) == 0) ||
- item->key == "int_name" ||
- (strncmp(item->key.c_str(), "int_name:", 9) == 0) ||
- item->key == "nat_name" ||
- (strncmp(item->key.c_str(), "nat_name:", 9) == 0) ||
- item->key == "reg_name" ||
- (strncmp(item->key.c_str(), "reg_name:", 9) == 0) ||
- item->key == "loc_name" ||
- (strncmp(item->key.c_str(), "loc_name:", 9) == 0) ||
- item->key == "old_name" ||
- (strncmp(item->key.c_str(), "old_name:", 9) == 0) ||
- item->key == "alt_name" ||
- (strncmp(item->key.c_str(), "alt_name_", 9) == 0) ||
- (strncmp(item->key.c_str(), "alt_name:", 9) == 0) ||
- item->key == "official_name" ||
- (strncmp(item->key.c_str(), "official_name:", 14) == 0) ||
- item->key == "commonname" ||
- (strncmp(item->key.c_str(), "commonname:", 11) == 0) ||
- item->key == "common_name" ||
- (strncmp(item->key.c_str(), "common_name:", 12) == 0) ||
- item->key == "place_name" ||
- (strncmp(item->key.c_str(), "place_name:", 11) == 0) ||
- item->key == "short_name" ||
- (strncmp(item->key.c_str(), "short_name:", 11) == 0) ||
- item->key == "operator") /* operator is a bit of an oddity */
- {
- if (item->key == "name:prefix")
- {
- extratags->pushItem(item);
- }
- else
- {
- names->pushItem(item);
- }
- }
- else if (item->key == "emergency" ||
- item->key == "tourism" ||
- item->key == "historic" ||
- item->key == "military" ||
- item->key == "natural")
- {
- if (item->value != "no" && item->value != "yes")
- {
- places->pushItem(item);
- }
- else
- {
- delete(item);
- }
- }
- else if (item->key == "highway")
- {
- if (item->value != "no" &&
- item->value != "turning_circle" &&
- item->value != "traffic_signals" &&
- item->value != "mini_roundabout" &&
- item->value != "noexit" &&
- item->value != "crossing")
- {
- places->pushItem(item);
- }
- else
- {
- delete(item);
- }
- }
- else if (item->key == "aerialway" ||
- item->key == "aeroway" ||
- item->key == "amenity" ||
- item->key == "boundary" ||
- item->key == "bridge" ||
- item->key == "craft" ||
- item->key == "leisure" ||
- item->key == "office" ||
- item->key == "railway" ||
- item->key == "shop" ||
- item->key == "tunnel" )
- {
- if (item->value != "no")
- {
- places->pushItem(item);
- if (item->key == "boundary" && item->value == "administrative")
- {
- placeadmin = 1;
- }
- }
- else
- {
- delete(item);
- }
- }
- else if (item->key == "waterway" && item->value != "riverbank")
- {
- places->pushItem(item);
- }
- else if (item->key == "place")
- {
- place = item;
- }
- else if (item->key == "addr:housename")
- {
- names->pushItem(item);
- placehouse = 1;
- }
- else if (item->key == "landuse")
- {
- if (item->value == "cemetery")
- places->pushItem(item);
- else
- landuse = item;
- }
- else if (item->key == "postal_code" ||
- item->key == "post_code" ||
- item->key == "postcode" ||
- item->key == "addr:postcode" ||
- item->key == "tiger:zip_left" ||
- item->key == "tiger:zip_right")
- {
- if (*postcode)
- delete(item);
- else
- *postcode = item;
- }
- else if (item->key == "addr:street")
- {
- *street = item;
- }
- else if (item->key == "addr:place")
- {
- *addr_place = item;
- }
- else if ((item->key == "country_code_iso3166_1_alpha_2" ||
- item->key == "country_code_iso3166_1" ||
- item->key == "country_code_iso3166" ||
- item->key == "country_code" ||
- item->key == "iso3166-1:alpha2" ||
- item->key == "iso3166-1" ||
- item->key == "ISO3166-1" ||
- item->key == "iso3166" ||
- item->key == "is_in:country_code" ||
- item->key == "addr:country" ||
- item->key == "addr:country_code")
- && item->value.length() == 2)
- {
- *countrycode = item;
- }
- else if (item->key == "addr:housenumber")
- {
- /* house number can be far more complex than just a single house number - leave for postgresql to deal with */
- if (*housenumber)
- delete(item);
- else {
- *housenumber = item;
- placehouse = 1;
- }
- }
- else if (item->key == "addr:conscriptionnumber")
- {
- if (conscriptionnumber)
- delete(item);
- else {
- conscriptionnumber = item;
- placehouse = 1;
- }
- }
- else if (item->key == "addr:streetnumber")
- {
- if (streetnumber)
- delete(item);
- else {
- streetnumber = item;
- placehouse = 1;
- }
- }
- else if (item->key == "addr:interpolation")
- {
- /* house number can be far more complex than just a single house number - leave for postgresql to deal with */
- if (*housenumber) {
- delete(item);
- } else {
- *housenumber = item;
- places->addItem("place", "houses", true);
- }
- }
- else if (item->key == "tiger:county")
- {
- /* strip the state and replace it with a county suffix to ensure that
- the tag only matches against counties and not against some town
- with the same name.
- */
- subval = strcspn(item->value.c_str(), ",");
- *isin = (char *)realloc(*isin, isinsize + 9 + subval);
- *(*isin+isinsize) = ',';
- strncpy(*isin+1+isinsize, item->value.c_str(), subval);
- strcpy(*isin+1+isinsize+subval, " county");
- isinsize += 8 + subval;
- delete(item);
- }
- else if (item->key == "is_in" ||
- (strncmp(item->key.c_str(), "is_in:", 5) == 0) ||
- item->key == "addr:suburb" ||
- item->key == "addr:county" ||
- item->key == "addr:city" ||
- item->key == "addr:state_code" ||
- item->key == "addr:state")
- {
- *isin = (char *)realloc(*isin, isinsize + 2 + item->value.length());
- *(*isin+isinsize) = ',';
- strcpy(*isin+1+isinsize, item->value.c_str());
- isinsize += 1 + item->value.length();
- delete(item);
- }
- else if (item->key == "admin_level")
- {
- *admin_level = atoi(item->value.c_str());
- delete(item);
- }
- else if (item->key == "tracktype" ||
- item->key == "traffic_calming" ||
- item->key == "service" ||
- item->key == "cuisine" ||
- item->key == "capital" ||
- item->key == "dispensing" ||
- item->key == "religion" ||
- item->key == "denomination" ||
- item->key == "sport" ||
- item->key == "internet_access" ||
- item->key == "lanes" ||
- item->key == "surface" ||
- item->key == "smoothness" ||
- item->key == "width" ||
- item->key == "est_width" ||
- item->key == "incline" ||
- item->key == "opening_hours" ||
- item->key == "food_hours" ||
- item->key == "collection_times" ||
- item->key == "service_times" ||
- item->key == "smoking_hours" ||
- item->key == "disused" ||
- item->key == "wheelchair" ||
- item->key == "sac_scale" ||
- item->key == "trail_visibility" ||
- item->key == "mtb:scale" ||
- item->key == "mtb:description" ||
- item->key == "wood" ||
- item->key == "drive_thru" ||
- item->key == "drive_in" ||
- item->key == "access" ||
- item->key == "vehicle" ||
- item->key == "bicyle" ||
- item->key == "foot" ||
- item->key == "goods" ||
- item->key == "hgv" ||
- item->key == "motor_vehicle" ||
- item->key == "motor_car" ||
- (strncmp(item->key.c_str(), "access:", 7) == 0) ||
- (strncmp(item->key.c_str(), "contact:", 8) == 0) ||
- (strncmp(item->key.c_str(), "drink:", 6) == 0) ||
- item->key == "oneway" ||
- item->key == "date_on" ||
- item->key == "date_off" ||
- item->key == "day_on" ||
- item->key == "day_off" ||
- item->key == "hour_on" ||
- item->key == "hour_off" ||
- item->key == "maxweight" ||
- item->key == "maxheight" ||
- item->key == "maxspeed" ||
- item->key == "disused" ||
- item->key == "toll" ||
- item->key == "charge" ||
- item->key == "population" ||
- item->key == "description" ||
- item->key == "image" ||
- item->key == "attribution" ||
- item->key == "fax" ||
- item->key == "email" ||
- item->key == "url" ||
- item->key == "website" ||
- item->key == "phone" ||
- item->key == "tel" ||
- item->key == "real_ale" ||
- item->key == "smoking" ||
- item->key == "food" ||
- item->key == "camera" ||
- item->key == "brewery" ||
- item->key == "locality" ||
- item->key == "wikipedia" ||
- (strncmp(item->key.c_str(), "wikipedia:", 10) == 0)
- )
- {
- extratags->pushItem(item);
- }
- else if (item->key == "building")
- {
- placebuilding = 1;
- delete(item);
- }
- else if (item->key == "mountain_pass")
- {
- places->pushItem(item);
- }
- else
- {
- delete(item);
- }
- }
+ /* Do we have a copy active? */
+ if (!copy_active) return;
- /* Handle Czech/Slovak addresses:
- - if we have just a conscription number or a street number,
- just use the one we have as a house number
- - if we have both of them, concatenate them so users may search
- by any of them
- */
- if (conscriptionnumber || streetnumber)
- {
- if (*housenumber)
- {
- delete(*housenumber);
- }
- if (!conscriptionnumber)
- {
- streetnumber->key.assign("addr:housenumber");
- *housenumber = streetnumber;
- }
- if (!streetnumber)
- {
- conscriptionnumber->key.assign("addr:housenumber");
- *housenumber = conscriptionnumber;
- }
- if (conscriptionnumber && streetnumber)
- {
- conscriptionnumber->key.assign("addr:housenumber");
- conscriptionnumber->value.reserve(conscriptionnumber->value.size() + 1
- + streetnumber->value.size());
- conscriptionnumber->value += '/';
- conscriptionnumber->value += streetnumber->value;
- delete(streetnumber);
- *housenumber = conscriptionnumber;
- }
+ if (buffer.length() > 0)
+ {
+ pgsql_CopyData("place", Connection, buffer);
+ buffer.clear();
}
- if (place)
- {
- if (placeadmin)
- {
- extratags->pushItem(place);
- }
- else
- {
- places->pushItem(place);
- }
- }
-
- if (placehouse && !places->listHasData())
- {
- places->addItem("place", "house", false);
- }
-
- /* Fallback place types - only used if we didn't create something more specific already */
- if (placebuilding && !places->listHasData() && (names->listHasData() || *housenumber || *postcode))
- {
- places->addItem("building", "yes", false);
- }
+ /* Terminate the copy */
+ if (PQputCopyEnd(Connection, NULL) != 1)
+ {
+ std::cerr << "COPY_END for place failed: " << PQerrorMessage(Connection) << "\n";
+ util::exit_nicely();
+ }
- if (landuse)
- {
- if (!places->listHasData() && names->listHasData())
- {
- places->pushItem(landuse);
- }
- else
- {
- delete(landuse);
- }
- }
+ /* Check the result */
+ PGresult *res = PQgetResult(Connection);
+ if (PQresultStatus(res) != PGRES_COMMAND_OK)
+ {
+ std::cerr << "COPY_END for place failed: " << PQerrorMessage(Connection) << "\n";
+ PQclear(res);
+ util::exit_nicely();
+ }
- if (*postcode && !places->listHasData())
- {
- places->addItem("place", "postcode", false);
- }
+ /* Discard the result */
+ PQclear(res);
- /* Try to convert everything to an area */
- return 1;
+ /* We no longer have an active copy */
+ copy_active = false;
}
-void escape_array_record(char *out, int len, const char *in)
-{
- int count = 0;
- const char *old_in = in, *old_out = out;
-
- if (!len)
- return;
-
- while(*in && count < len-3) {
- switch(*in) {
- case '\\': *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; count+= 8; break;
- case '\n':
- case '\r':
- case '\t':
- case '"':
- /* This is a bit naughty - we know that nominatim ignored these characters so just drop them now for simplicity */
- *out++ = ' '; count++; break;
- default: *out++ = *in; count++; break;
- }
- in++;
- }
- *out = '\0';
- if (*in)
- fprintf(stderr, "%s truncated at %d chars: %s\n%s\n", __FUNCTION__, count, old_in, old_out);
-}
-void output_gazetteer_t::delete_unused_classes(char osm_type, osmid_t osm_id, struct keyval *places) {
- int i,sz, slen;
- PGresult *res;
- char tmp[16];
+void output_gazetteer_t::delete_unused_classes(char osm_type, osmid_t osm_id) {
char tmp2[2];
- char *cls, *clslist = 0;
- char const *paramValues[2];
-
tmp2[0] = osm_type; tmp2[1] = '\0';
+ char const *paramValues[2];
paramValues[0] = tmp2;
- snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, osm_id);
- paramValues[1] = tmp;
- res = pgsql_execPrepared(ConnectionDelete, "get_classes", 2, paramValues, PGRES_TUPLES_OK);
+ paramValues[1] = (single_fmt % osm_id).str().c_str();
+ PGresult *res = pgsql_execPrepared(ConnectionDelete, "get_classes", 2,
+ paramValues, PGRES_TUPLES_OK);
- sz = PQntuples(res);
- if (sz > 0 && !places) {
+ int sz = PQntuples(res);
+ if (sz > 0 && !places.has_data()) {
PQclear(res);
- /* uncondtional delete of all places */
- stop_copy();
- pgsql_exec(Connection, PGRES_COMMAND_OK, "DELETE FROM place WHERE osm_type = '%c' AND osm_id = %" PRIdOSMID, osm_type, osm_id);
+ /* unconditional delete of all places */
+ delete_place(osm_type, osm_id);
} else {
- for (i = 0; i < sz; i++) {
- cls = PQgetvalue(res, i, 0);
- if (!places->getItem(cls)) {
- if (!clslist) {
- clslist = (char *)malloc(strlen(cls)+3);
- sprintf(clslist, "'%s'", cls);
- } else {
- slen = strlen(clslist);
- clslist = (char *)realloc(clslist, slen + 4 + strlen(cls));
- sprintf(&(clslist[slen]), ",'%s'", cls);
- }
+ std::string clslist;
+ for (int i = 0; i < sz; i++) {
+ std::string cls(PQgetvalue(res, i, 0));
+ if (!places.has_place(cls)) {
+ clslist.reserve(clslist.length() + cls.length() + 3);
+ if (!clslist.empty())
+ clslist += ',';
+ clslist += '\'';
+ clslist += cls;
+ clslist += '\'';
}
}
PQclear(res);
- if (clslist) {
+ if (!clslist.empty()) {
/* Stop any active copy */
stop_copy();
/* Delete all places for this object */
- pgsql_exec(Connection, PGRES_COMMAND_OK, "DELETE FROM place WHERE osm_type = '%c' AND osm_id = %"
- PRIdOSMID " and class = any(ARRAY[%s])", osm_type, osm_id, clslist);
- free(clslist);
+ pgsql_exec(Connection, PGRES_COMMAND_OK,
+ "DELETE FROM place WHERE osm_type = '%c' AND osm_id = %"
+ PRIdOSMID " and class = any(ARRAY[%s])",
+ osm_type, osm_id, clslist.c_str());
}
}
}
-void output_gazetteer_t::add_place(char osm_type, osmid_t osm_id, const std::string &key_class, const std::string &type, struct keyval *names, struct keyval *extratags,
- int adminlevel, struct keyval *housenumber, struct keyval *street, struct keyval *addr_place, const char *isin, struct keyval *postcode, struct keyval *countrycode, const char *wkt)
-{
- int first;
- struct keyval *name;
- char sql[2048];
-
- /* Output a copy line for this place */
- sprintf(sql, "%c\t%" PRIdOSMID "\t", osm_type, osm_id);
- copy_data(sql);
-
- escape(sql, sizeof(sql), key_class.c_str());
- copy_data(sql);
- copy_data("\t");
-
- escape(sql, sizeof(sql), type.c_str());
- copy_data(sql);
- copy_data("\t");
-
- /* start name array */
- if (names->listHasData())
- {
- first = 1;
- for (name = names->firstItem(); name; name = names->nextItem(name))
- {
- if (first) first = 0;
- else copy_data(", ");
-
- copy_data("\"");
-
- escape_array_record(sql, sizeof(sql), name->key.c_str());
- copy_data(sql);
-
- copy_data("\"=>\"");
-
- escape_array_record(sql, sizeof(sql), name->value.c_str());
- copy_data(sql);
-
- copy_data("\"");
- }
- copy_data("\t");
- }
- else
- {
- copy_data("\\N\t");
- }
-
- sprintf(sql, "%d\t", adminlevel);
- copy_data(sql);
-
- if (housenumber)
- {
- escape(sql, sizeof(sql), housenumber->value.c_str());
- copy_data(sql);
- copy_data("\t");
- }
- else
- {
- copy_data("\\N\t");
- }
-
- if (street)
- {
- escape(sql, sizeof(sql), street->value.c_str());
- copy_data(sql);
- copy_data("\t");
- }
- else
- {
- copy_data("\\N\t");
- }
-
- if (addr_place)
- {
- escape(sql, sizeof(sql), addr_place->value.c_str());
- copy_data(sql);
- copy_data("\t");
- }
- else
- {
- copy_data("\\N\t");
- }
-
- if (isin)
- {
- /* Skip the leading ',' from the contactination */
- escape(sql, sizeof(sql), isin+1);
- copy_data(sql);
- copy_data("\t");
- }
- else
- {
- copy_data("\\N\t");
- }
-
- if (postcode)
- {
- escape(sql, sizeof(sql), postcode->value.c_str());
- copy_data(sql);
- copy_data("\t");
- }
- else
- {
- copy_data("\\N\t");
- }
-
- if (countrycode)
- {
- escape(sql, sizeof(sql), countrycode->value.c_str());
- copy_data(sql);
- copy_data("\t");
- }
- else
- {
- copy_data("\\N\t");
- }
-
- /* extra tags array */
- if (extratags->listHasData())
- {
- first = 1;
- for (name = extratags->firstItem(); name; name = extratags->nextItem(name))
- {
- if (first) first = 0;
- else copy_data(", ");
-
- copy_data("\"");
-
- escape_array_record(sql, sizeof(sql), name->key.c_str());
- copy_data(sql);
-
- copy_data("\"=>\"");
-
- escape_array_record(sql, sizeof(sql), name->value.c_str());
- copy_data(sql);
-
- copy_data("\"");
- }
- copy_data("\t");
- }
- else
- {
- copy_data("\\N\t");
- }
-
- sprintf(sql, "SRID=%d;", SRID);
- copy_data(sql);
- copy_data(wkt);
-
- copy_data("\n");
-
-
- return;
-}
-
-#if 0
-static void add_polygon_error(char osm_type, osmid_t osm_id,
- const char *key_class, const char *type,
- struct keyval *names, const char *countrycode,
- const char *wkt)
-{
- int first;
- struct keyval *name;
- char sql[2048];
-
- /* Output a copy line for this place */
- sprintf(sql, "%c\t%" PRIdOSMID "\t", osm_type, osm_id);
- copy_error_data(sql);
-
- escape(sql, sizeof(sql), key_class);
- copy_error_data(sql);
- copy_error_data("\t");
-
- escape(sql, sizeof(sql), type);
- copy_error_data(sql);
- copy_error_data("\t");
-
- /* start name array */
- if (keyval::listHasData(names))
- {
- first = 1;
- for (name = keyval::firstItem(names); name; name = keyval::nextItem(names, name))
- {
- if (first) first = 0;
- else copy_error_data(", ");
-
- copy_error_data("\"");
-
- escape_array_record(sql, sizeof(sql), name->key);
- copy_error_data(sql);
-
- copy_error_data("\"=>\"");
-
- escape_array_record(sql, sizeof(sql), name->value);
- copy_error_data(sql);
-
- copy_error_data("\"");
- }
- copy_error_data("\t");
- }
- else
- {
- copy_error_data("\\N\t");
- }
-
- if (countrycode)
- {
- escape(sql, sizeof(sql), countrycode);
- copy_error_data(sql);
- copy_error_data("\t");
- }
- else
- {
- copy_error_data("\\N\t");
- }
-
- copy_error_data("now\tNot a polygon\t\\N\t");
-
- sprintf(sql, "SRID=%d;", SRID);
- copy_error_data(sql);
- copy_error_data(wkt);
-
- copy_error_data("\n");
-
-
- return;
-}
-#endif
-
void output_gazetteer_t::delete_place(char osm_type, osmid_t osm_id)
{
@@ -980,17 +588,16 @@ int output_gazetteer_t::connect() {
Connection = PQconnectdb(m_options.conninfo.c_str());
/* Check to see that the backend connection was successfully made */
- if (PQstatus(Connection) != CONNECTION_OK)
- {
- fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(Connection));
+ if (PQstatus(Connection) != CONNECTION_OK) {
+ std::cerr << "Connection to database failed: " << PQerrorMessage(Connection) << "\n";
return 1;
}
- if(m_options.append) {
+ if (m_options.append) {
ConnectionDelete = PQconnectdb(m_options.conninfo.c_str());
if (PQstatus(ConnectionDelete) != CONNECTION_OK)
{
- fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(ConnectionDelete));
+ std::cerr << "Connection to database failed: " << PQerrorMessage(Connection) << "\n";
return 1;
}
@@ -1004,6 +611,8 @@ int output_gazetteer_t::start()
reproj = m_options.projection;
builder.set_exclude_broken_polygon(m_options.excludepoly);
+ places.srid_str = (boost::format("SRID=%1%;") % SRID).str();
+
if(connect())
util::exit_nicely();
@@ -1011,37 +620,21 @@ int output_gazetteer_t::start()
pgsql_exec(Connection, PGRES_COMMAND_OK, "BEGIN");
/* (Re)create the table unless we are appending */
- if (!m_options.append)
- {
+ if (!m_options.append) {
/* Drop any existing table */
pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS place");
- pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TYPE if exists keyvalue cascade");
- pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TYPE if exists wordscore cascade");
- pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TYPE if exists stringlanguagetype cascade");
- pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TYPE if exists keyvaluetype cascade");
- pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP FUNCTION IF EXISTS get_connected_ways(integer[])");
-
- /* Create types and functions */
- pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_KEYVALUETYPE_TYPE);
- pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_WORDSCORE_TYPE);
/* Create the new table */
- if (m_options.tblsmain_data)
- {
+ if (m_options.tblsmain_data) {
pgsql_exec(Connection, PGRES_COMMAND_OK,
CREATE_PLACE_TABLE, "TABLESPACE", m_options.tblsmain_data->c_str());
- }
- else
- {
+ } else {
pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_TABLE, "", "");
}
- if (m_options.tblsmain_index)
- {
+ if (m_options.tblsmain_index) {
pgsql_exec(Connection, PGRES_COMMAND_OK,
CREATE_PLACE_ID_INDEX, "TABLESPACE", m_options.tblsmain_index->c_str());
- }
- else
- {
+ } else {
pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_ID_INDEX, "", "");
}
@@ -1052,23 +645,6 @@ int output_gazetteer_t::start()
return 0;
}
-void output_gazetteer_t::commit()
-{
-}
-
-void output_gazetteer_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
-}
-
-int output_gazetteer_t::pending_way(osmid_t id, int exists) {
- return 0;
-}
-
-void output_gazetteer_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {
-}
-
-int output_gazetteer_t::pending_relation(osmid_t id, int exists) {
- return 0;
-}
void output_gazetteer_t::stop()
{
@@ -1088,351 +664,146 @@ void output_gazetteer_t::stop()
return;
}
-int output_gazetteer_t::gazetteer_process_node(osmid_t id, double lat, double lon, struct keyval *tags, int delete_old)
-{
- struct keyval names;
- struct keyval places;
- struct keyval extratags;
- struct keyval *place;
- int adminlevel;
- struct keyval * housenumber;
- struct keyval * street;
- struct keyval * addr_place;
- char * isin;
- struct keyval * postcode;
- struct keyval * countrycode;
- char wkt[128];
-
-
- /* Split the tags */
- split_tags(tags, TAGINFO_NODE, &names, &places, &extratags, &adminlevel, &housenumber, &street, &addr_place, &isin, &postcode, &countrycode);
-
- if (delete_old)
- delete_unused_classes('N', id, &places);
-
- /* Are we interested in this item? */
- if (places.listHasData())
- {
- sprintf(wkt, "POINT(%.15g %.15g)", lon, lat);
- for (place = places.firstItem(); place; place = places.nextItem(place))
- {
- add_place('N', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, addr_place, isin, postcode, countrycode, wkt);
- }
- }
-
- if (housenumber) delete(housenumber);
- if (street) delete(street);
- if (addr_place) delete(addr_place);
- if (isin) free(isin);
- if (postcode) delete(postcode);
- if (countrycode) delete(countrycode);
-
- /* Free tag lists */
- names.resetList();
- places.resetList();
- extratags.resetList();
-
- return 0;
-}
-
-int output_gazetteer_t::node_add(osmid_t id, double lat, double lon, struct keyval *tags)
-{
- return gazetteer_process_node(id, lat, lon, tags, 0);
-}
-
-int output_gazetteer_t::gazetteer_process_way(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags, int delete_old)
+int output_gazetteer_t::process_node(osmid_t id, double lat, double lon,
+ struct keyval *tags)
{
- struct keyval names;
- struct keyval places;
- struct keyval extratags;
- struct keyval *place;
- int adminlevel;
- struct keyval * housenumber;
- struct keyval * street;
- struct keyval * addr_place;
- char * isin;
- struct keyval * postcode;
- struct keyval * countrycode;
- int area;
-
-
- /* Split the tags */
- area = split_tags(tags, TAGINFO_WAY, &names, &places, &extratags, &adminlevel, &housenumber, &street, &addr_place, &isin, &postcode, &countrycode);
-
- if (delete_old)
- delete_unused_classes('W', id, &places);
-
- /* Are we interested in this item? */
- if (places.listHasData())
- {
- struct osmNode *nodev;
- int nodec;
-
- /* Fetch the node details */
- nodev = (struct osmNode *)malloc(ndc * sizeof(struct osmNode));
- nodec = m_mid->nodes_get_list(nodev, ndv, ndc);
-
- /* Get the geometry of the object */
- geometry_builder::maybe_wkt_t wkt = builder.get_wkt_simple(nodev, nodec, area);
- if (wkt)
- {
- for (place = places.firstItem(); place; place = places.nextItem(place))
- {
- add_place('W', id, place->key, place->value, &names, &extratags, adminlevel,
- housenumber, street, addr_place, isin, postcode, countrycode, wkt->geom.c_str());
- }
- }
-
- /* Free the nodes */
- free(nodev);
- }
+ places.process_tags(tags);
- if (housenumber) delete(housenumber);
- if (street) delete(street);
- if (addr_place) delete(addr_place);
- if (isin) free(isin);
- if (postcode) delete(postcode);
- if (countrycode) delete(countrycode);
+ if (m_options.append)
+ delete_unused_classes('N', id);
- /* Free tag lists */
- names.resetList();
- places.resetList();
- extratags.resetList();
-
- return 0;
-}
+ /* Are we interested in this item? */
+ if (places.has_data()) {
+ std::string wkt = (point_fmt % lon % lat).str();
+ places.copy_out('N', id, wkt, buffer);
+ flush_place_buffer();
+ }
-int output_gazetteer_t::way_add(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags)
-{
- return gazetteer_process_way(id, ndv, ndc, tags, 0);
+ return 0;
}
-int output_gazetteer_t::gazetteer_process_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags, int delete_old)
+int output_gazetteer_t::process_way(osmid_t id, osmid_t *ndv, int ndc,
+ struct keyval *tags)
{
- struct keyval names;
- struct keyval places;
- struct keyval extratags;
- struct keyval *place;
- int adminlevel;
- struct keyval * housenumber;
- struct keyval * street;
- struct keyval * addr_place;
- char * isin;
- struct keyval * postcode;
- struct keyval * countrycode;
- int cmp_waterway;
-
- const std::string *type = tags->getItem("type");
- if (!type) {
- if (delete_old) delete_unused_classes('R', id, 0);
- return 0;
- }
-
- cmp_waterway = type->compare("waterway");
-
- if (!type->compare("associatedStreet"))
- {
- if (delete_old) delete_unused_classes('R', id, 0);
- return 0;
- }
-
- if (type->compare("boundary") && type->compare("multipolygon") && cmp_waterway) {
- if (delete_old) delete_unused_classes('R', id, 0);
- return 0;
- }
-
- /* Split the tags */
- split_tags(tags, TAGINFO_AREA, &names, &places, &extratags, &adminlevel, &housenumber, &street, &addr_place, &isin, &postcode, &countrycode);
-
- /* reset type to NULL because split_tags() consumes the tags
- * keyval and means that it's pointing to some random stuff
- * which might be harmful if dereferenced. */
- type = NULL;
-
- if (delete_old)
- delete_unused_classes('R', id, &places);
-
- if (places.listHasData())
- {
- /* get the boundary path (ways) */
- int i, count;
- int *xcount = (int *)malloc( (member_count+1) * sizeof(int) );
- keyval *xtags = new keyval[member_count+1];
- struct osmNode **xnodes = (struct osmNode **)malloc( (member_count+1) * sizeof(struct osmNode*) );
- osmid_t *xid2 = (osmid_t *)malloc( (member_count+1) * sizeof(osmid_t) );
-
- count = 0;
- for (i=0; i<member_count; i++)
- {
- /* only interested in ways */
- if (members[i].type != OSMTYPE_WAY)
- continue;
- xid2[count] = members[i].id;
- count++;
- }
+ places.process_tags(tags);
- if (count == 0)
- {
- if (delete_old) delete_unused_classes('R', id, 0);
- free(xcount);
- delete [] xtags;
- free(xnodes);
- free(xid2);
- return 0;
- }
+ if (m_options.append)
+ delete_unused_classes('W', id);
- osmid_t *xid = (osmid_t *)malloc( sizeof(osmid_t) * (count + 1));
- count = m_mid->ways_get_list(xid2, count, xid, xtags, xnodes, xcount);
+ /* Are we interested in this item? */
+ if (places.has_data()) {
+ struct osmNode *nodev;
+ int nodec;
- xnodes[count] = NULL;
- xcount[count] = 0;
+ /* Fetch the node details */
+ nodev = (struct osmNode *)malloc(ndc * sizeof(struct osmNode));
+ nodec = m_mid->nodes_get_list(nodev, ndv, ndc);
- if (cmp_waterway)
- {
- geometry_builder::maybe_wkts_t wkts = builder.build_both(xnodes, xcount, 1, 1, 1000000, id);
- for (geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt)
- {
- if ((boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON")))
- {
- for (place = places.firstItem(); place; place = places.nextItem(place))
- {
- add_place('R', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, addr_place,
- isin, postcode, countrycode, wkt->geom.c_str());
- }
- }
- else
- {
- /* add_polygon_error('R', id, "boundary", "adminitrative", &names, countrycode, wkt); */
- }
- }
- } else {
- /* waterways result in multilinestrings */
- // wkt_t build_multilines(const osmNode * const * xnodes, const int *xcount, osmid_t osm_id) const;
- geometry_builder::maybe_wkt_t wkt = builder.build_multilines(xnodes, xcount, id);
- if ((wkt->geom).length() > 0)
- {
- for (place = places.firstItem(); place; place = places.nextItem(place))
- {
- add_place('R', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, addr_place,
- isin, postcode, countrycode, wkt->geom.c_str());
- }
- }
- }
- for( i=0; i<count; i++ )
- {
- xtags[i].resetList();
- free( xnodes[i] );
- }
-
- free(xid);
- free(xid2);
- free(xcount);
- delete [] xtags;
- free(xnodes);
- }
-
- if (housenumber) delete(housenumber);
- if (street) delete(street);
- if (addr_place) delete(addr_place);
- if (isin) free(isin);
- if (postcode) delete(postcode);
- if (countrycode) delete(countrycode);
+ /* Get the geometry of the object */
+ geometry_builder::maybe_wkt_t wkt = builder.get_wkt_simple(nodev, nodec, 1);
+ if (wkt) {
+ places.copy_out('W', id, wkt->geom, buffer);
+ flush_place_buffer();
+ }
- /* Free tag lists */
- names.resetList();
- places.resetList();
- extratags.resetList();
+ /* Free the nodes */
+ free(nodev);
+ }
- return 0;
+ return 0;
}
-int output_gazetteer_t::relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags)
+int output_gazetteer_t::process_relation(osmid_t id, struct member *members,
+ int member_count, struct keyval *tags)
{
- return gazetteer_process_relation(id, members, member_count, tags, 0);
-}
+ int cmp_waterway;
-int output_gazetteer_t::node_delete(osmid_t id)
-{
- /* Make sure we are in slim mode */
- require_slim_mode();
+ const std::string *type = tags->getItem("type");
+ if (!type) {
+ delete_unused_full('R', id);
+ return 0;
+ }
- /* Delete all references to this node */
- delete_place('N', id);
+ cmp_waterway = type->compare("waterway");
- return 0;
-}
+ if (*type == "associatedStreet"
+ || !(*type == "boundary" || *type == "multipolygon" || !cmp_waterway)) {
+ delete_unused_full('R', id);
+ return 0;
+ }
-int output_gazetteer_t::way_delete(osmid_t id)
-{
- /* Make sure we are in slim mode */
- require_slim_mode();
+ places.process_tags(tags);
- /* Delete all references to this way */
- delete_place('W', id);
+ if (m_options.append)
+ delete_unused_classes('R', id);
- return 0;
-}
+ /* Are we interested in this item? */
+ if (!places.has_data())
+ return 0;
-int output_gazetteer_t::relation_delete(osmid_t id)
-{
- /* Make sure we are in slim mode */
- require_slim_mode();
+ /* get the boundary path (ways) */
+ osmid_t *xid2 = new osmid_t[member_count+1];
- /* Delete all references to this relation */
- delete_place('R', id);
+ int count = 0;
+ for (int i=0; i<member_count; ++i) {
+ /* only interested in ways */
+ if (members[i].type != OSMTYPE_WAY)
+ continue;
+ xid2[count] = members[i].id;
+ count++;
+ }
- return 0;
-}
+ if (count == 0) {
+ if (m_options.append)
+ delete_unused_full('R', id);
-int output_gazetteer_t::node_modify(osmid_t id, double lat, double lon, struct keyval *tags)
-{
- require_slim_mode();
- return gazetteer_process_node(id, lat, lon, tags, 1);
-}
+ delete [] xid2;
-int output_gazetteer_t::way_modify(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags)
-{
- require_slim_mode();
- return gazetteer_process_way(id, ndv, ndc, tags, 1);
-}
+ return 0;
+ }
-int output_gazetteer_t::relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags)
-{
- require_slim_mode();
- return gazetteer_process_relation(id, members, member_count, tags, 1);
-}
+ int *xcount = new int[count + 1];
+ keyval *xtags = new keyval[count+1];
+ struct osmNode **xnodes = new osmNode*[count + 1];
+ osmid_t *xid = new osmid_t[count + 1];
+ count = m_mid->ways_get_list(xid2, count, xid, xtags, xnodes, xcount);
+
+ xnodes[count] = NULL;
+ xcount[count] = 0;
+
+ if (cmp_waterway) {
+ geometry_builder::maybe_wkts_t wkts = builder.build_both(xnodes, xcount, 1, 1, 1000000, id);
+ for (geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt) {
+ if (boost::starts_with(wkt->geom, "POLYGON")
+ || boost::starts_with(wkt->geom, "MULTIPOLYGON")) {
+ places.copy_out('R', id, wkt->geom, buffer);
+ flush_place_buffer();
+ } else {
+ /* add_polygon_error('R', id, "boundary", "adminitrative", &names, countrycode, wkt); */
+ }
+ }
+ } else {
+ /* waterways result in multilinestrings */
+ geometry_builder::maybe_wkt_t wkt = builder.build_multilines(xnodes, xcount, id);
+ if ((wkt->geom).length() > 0) {
+ places.copy_out('R', id, wkt->geom, buffer);
+ flush_place_buffer();
+ }
+ }
+ for (int i=0; i<count; ++i)
+ {
+ xtags[i].resetList();
+ free(xnodes[i]);
+ }
-boost::shared_ptr<output_t> output_gazetteer_t::clone(const middle_query_t* cloned_middle) const {
- output_gazetteer_t *clone = new output_gazetteer_t(*this);
- clone->m_mid = cloned_middle;
- return boost::shared_ptr<output_t>(clone);
-}
+ free(xid);
+ delete [] xid2;
+ delete [] xcount;
+ delete [] xtags;
+ delete [] xnodes;
-output_gazetteer_t::output_gazetteer_t(const middle_query_t* mid_, const options_t &options_)
- : output_t(mid_, options_),
- Connection(NULL),
- ConnectionDelete(NULL),
- ConnectionError(NULL),
- CopyActive(0),
- BufferLen(0)
-{
- memset(Buffer, 0, BUFFER_SIZE);
+ return 0;
}
-output_gazetteer_t::output_gazetteer_t(const output_gazetteer_t& other)
- : output_t(other.m_mid, other.m_options),
- Connection(NULL),
- ConnectionDelete(NULL),
- ConnectionError(NULL),
- CopyActive(0),
- BufferLen(0),
- reproj(other.reproj)
-{
- builder.set_exclude_broken_polygon(m_options.excludepoly);
- memset(Buffer, 0, BUFFER_SIZE);
- connect();
-}
-output_gazetteer_t::~output_gazetteer_t() {
-}
diff --git a/output-gazetteer.hpp b/output-gazetteer.hpp
index caa3a1b..4788848 100644
--- a/output-gazetteer.hpp
+++ b/output-gazetteer.hpp
@@ -4,73 +4,276 @@
#include "output.hpp"
#include "geometry-builder.hpp"
#include "reprojection.hpp"
+#include "util.hpp"
#include <boost/shared_ptr.hpp>
+#include <boost/foreach.hpp>
+#include <boost/format.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
+#include <string>
+#include <iostream>
+
+/**
+ * A private class to convert tags.
+ */
+class place_tag_processor
+{
+public:
+ place_tag_processor()
+ : single_fmt("%1%\t")
+ {
+ places.reserve(4);
+ extratags.reserve(15);
+ address.reserve(10);
+ }
+
+ ~place_tag_processor() {}
+
+ void process_tags(keyval *tags);
+
+ bool has_data() const { return !places.empty(); }
+
+ bool has_place(const std::string &cls)
+ {
+ BOOST_FOREACH(const keyval *item, places) {
+ if (cls == item->key)
+ return true;
+ }
+
+ return false;
+ }
+
+ void copy_out(char osm_type, osmid_t osm_id, const std::string &wkt,
+ std::string &buffer);
+
+ void clear();
+
+private:
+ void copy_opt_string(const std::string *val, std::string &buffer)
+ {
+ if (val) {
+ escape(*val, buffer);
+ buffer += "\t";
+ } else {
+ buffer += "\\N\t";
+ }
+ }
+
+ std::string domain_name(const std::string &cls)
+ {
+ std::string ret;
+ bool hasname = false;
+
+ std::string prefix(cls + ":name");
+
+ for (keyval *item = src->firstItem(); item; item = src->nextItem(item)) {
+ if (boost::starts_with(item->key, prefix) &&
+ (item->key.length() == prefix.length()
+ || item->key[prefix.length()] == ':')) {
+ if (!hasname) {
+ ret.reserve(item->key.length() + item->value.length() + 10);
+ hasname = true;
+ } else
+ ret += ",";
+ ret += "\"";
+ escape_array_record(std::string(item->key, cls.length() + 1), ret);
+ ret += "\"=>\"";
+ escape_array_record(item->value, ret);
+ ret += "\"";
+ }
+ }
+
+ return ret;
+ }
+
+
+ void escape_array_record(const std::string &in, std::string &out)
+ {
+ BOOST_FOREACH(const char c, in) {
+ switch(c) {
+ case '\\': out += "\\\\\\\\\\\\\\\\"; break;
+ case '\n':
+ case '\r':
+ case '\t':
+ case '"':
+ /* This is a bit naughty - we know that nominatim ignored these characters so just drop them now for simplicity */
+ out += ' '; break;
+ default: out += c; break;
+ }
+ }
+ }
+
+
+ std::vector<const keyval *> places;
+ std::vector<const keyval *> names;
+ std::vector<const keyval *> extratags;
+ std::vector<const keyval *> address;
+ keyval *src;
+ int admin_level;
+ const std::string *countrycode;
+ std::string housenumber;
+ const std::string *street;
+ const std::string *addr_place;
+ const std::string *postcode;
+
+ boost::format single_fmt;
+public:
+ std::string srid_str;
+};
+
class output_gazetteer_t : public output_t {
public:
- output_gazetteer_t(const middle_query_t* mid_, const options_t &options_);
- output_gazetteer_t(const output_gazetteer_t& other);
- virtual ~output_gazetteer_t();
+ output_gazetteer_t(const middle_query_t* mid_, const options_t &options_)
+ : output_t(mid_, options_),
+ Connection(NULL),
+ ConnectionDelete(NULL),
+ ConnectionError(NULL),
+ copy_active(false),
+ single_fmt("%1%"),
+ point_fmt("POINT(%.15g %.15g)")
+ {
+ buffer.reserve(PLACE_BUFFER_SIZE);
+ }
+
+ output_gazetteer_t(const output_gazetteer_t& other)
+ : output_t(other.m_mid, other.m_options),
+ Connection(NULL),
+ ConnectionDelete(NULL),
+ ConnectionError(NULL),
+ copy_active(false),
+ reproj(other.reproj),
+ single_fmt(other.single_fmt),
+ point_fmt(other.point_fmt)
+ {
+ buffer.reserve(PLACE_BUFFER_SIZE);
+ builder.set_exclude_broken_polygon(m_options.excludepoly);
+ connect();
+ }
- virtual boost::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const;
+ virtual ~output_gazetteer_t() {}
+
+ virtual boost::shared_ptr<output_t> clone(const middle_query_t* cloned_middle) const
+ {
+ output_gazetteer_t *clone = new output_gazetteer_t(*this);
+ clone->m_mid = cloned_middle;
+ return boost::shared_ptr<output_t>(clone);
+ }
int start();
void stop();
- void commit();
+ void commit() {}
+
+ void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {}
+ int pending_way(osmid_t id, int exists) { return 0; }
- void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
- int pending_way(osmid_t id, int exists);
+ void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {}
+ int pending_relation(osmid_t id, int exists) { return 0; }
- void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added);
- int pending_relation(osmid_t id, int exists);
+ int node_add(osmid_t id, double lat, double lon, struct keyval *tags)
+ {
+ return process_node(id, lat, lon, tags);
+ }
- int node_add(osmid_t id, double lat, double lon, struct keyval *tags);
- int way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
- int relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+ int way_add(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags)
+ {
+ return process_way(id, nodes, node_count, tags);
+ }
- int node_modify(osmid_t id, double lat, double lon, struct keyval *tags);
- int way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags);
- int relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags);
+ int relation_add(osmid_t id, struct member *members, int member_count, struct keyval *tags)
+ {
+ return process_relation(id, members, member_count, tags);
+ }
- int node_delete(osmid_t id);
- int way_delete(osmid_t id);
- int relation_delete(osmid_t id);
+ int node_modify(osmid_t id, double lat, double lon, struct keyval *tags)
+ {
+ return process_node(id, lat, lon, tags);
+ }
+
+ int way_modify(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags)
+ {
+ return process_way(id, nodes, node_count, tags);
+ }
+
+ int relation_modify(osmid_t id, struct member *members, int member_count, struct keyval *tags)
+ {
+ return process_relation(id, members, member_count, tags);
+ }
+
+ int node_delete(osmid_t id)
+ {
+ delete_place('N', id);
+ return 0;
+ }
+
+ int way_delete(osmid_t id)
+ {
+ delete_place('W', id);
+ return 0;
+ }
+
+ int relation_delete(osmid_t id)
+ {
+ delete_place('R', id);
+ return 0;
+ }
private:
- static const size_t BUFFER_SIZE = 4096;
+ enum { PLACE_BUFFER_SIZE = 4092 };
- void require_slim_mode(void);
- void copy_data(const char *sql);
void stop_copy(void);
- void delete_unused_classes(char osm_type, osmid_t osm_id, struct keyval *places);
+ void delete_unused_classes(char osm_type, osmid_t osm_id);
void add_place(char osm_type, osmid_t osm_id, const std::string &key_class, const std::string &type,
struct keyval *names, struct keyval *extratags, int adminlevel,
struct keyval *housenumber, struct keyval *street, struct keyval *addr_place,
const char *isin, struct keyval *postcode, struct keyval *countrycode,
- const char *wkt);
+ const std::string &wkt);
void delete_place(char osm_type, osmid_t osm_id);
- int gazetteer_process_node(osmid_t id, double lat, double lon, struct keyval *tags,
- int delete_old);
- int gazetteer_process_way(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags,
- int delete_old);
- int gazetteer_process_relation(osmid_t id, struct member *members, int member_count,
- struct keyval *tags, int delete_old);
+ int process_node(osmid_t id, double lat, double lon, keyval *tags);
+ int process_way(osmid_t id, osmid_t *ndv, int ndc, keyval *tags);
+ int process_relation(osmid_t id, member *members, int member_count, keyval *tags);
int connect();
+ void flush_place_buffer()
+ {
+ if (!copy_active)
+ {
+ pgsql_exec(Connection, PGRES_COPY_IN, "COPY place (osm_type, osm_id, class, type, name, admin_level, housenumber, street, addr_place, isin, postcode, country_code, extratags, geometry) FROM STDIN");
+ copy_active = true;
+ }
+
+ pgsql_CopyData("place", Connection, buffer);
+ buffer.clear();
+ }
+
+ void delete_unused_full(char osm_type, osmid_t osm_id)
+ {
+ if (m_options.append) {
+ places.clear();
+ delete_place(osm_type, osm_id);
+ }
+ }
+
struct pg_conn *Connection;
struct pg_conn *ConnectionDelete;
struct pg_conn *ConnectionError;
- int CopyActive;
- unsigned int BufferLen;
+ bool copy_active;
+ bool append_mode;
- char Buffer[BUFFER_SIZE];
+ std::string buffer;
+ place_tag_processor places;
geometry_builder builder;
boost::shared_ptr<reprojection> reproj;
+ // string formatters
+ // Need to be part of the class, so we have one per thread.
+ boost::format single_fmt;
+ boost::format point_fmt;
+
const static std::string NAME;
};
diff --git a/output-pgsql.cpp b/output-pgsql.cpp
index 8add8a0..0a5939d 100644
--- a/output-pgsql.cpp
+++ b/output-pgsql.cpp
@@ -39,10 +39,20 @@
#include <boost/foreach.hpp>
#include <boost/make_shared.hpp>
#include <boost/algorithm/string/predicate.hpp>
+#include <boost/exception_ptr.hpp>
#include <iostream>
#include <limits>
#include <stdexcept>
+/* make the diagnostic information work with older versions of
+ * boost - the function signature changed at version 1.54.
+ */
+#if BOOST_VERSION >= 105400
+#define BOOST_DIAGNOSTIC_INFO(e) boost::diagnostic_information((e), true)
+#else
+#define BOOST_DIAGNOSTIC_INFO(e) boost::diagnostic_information((e))
+#endif
+
#define SRID (reproj->project_getprojinfo()->srs)
/* FIXME: Shouldn't malloc this all to begin with but call realloc()
@@ -245,13 +255,21 @@ namespace {
/* Using pthreads requires us to shoe-horn everything into various void*
* pointers. Improvement for the future: just use boost::thread. */
struct pthread_thunk {
- table_t *ptr;
+ table_t *ptr;
+ boost::exception_ptr error;
};
extern "C" void *pthread_output_pgsql_stop_one(void *arg) {
- pthread_thunk *thunk = static_cast<pthread_thunk *>(arg);
+ pthread_thunk *thunk = static_cast<pthread_thunk *>(arg);
+
+ try {
thunk->ptr->stop();
- return NULL;
+
+ } catch (...) {
+ thunk->error = boost::current_exception();
+ }
+
+ return NULL;
};
} // anonymous namespace
@@ -368,23 +386,35 @@ void output_pgsql_t::stop()
pthread_thunk thunks[NUM_TABLES];
for (i=0; i<NUM_TABLES; i++) {
thunks[i].ptr = m_tables[i].get();
+ thunks[i].error = boost::exception_ptr();
}
for (i=0; i<NUM_TABLES; i++) {
int ret = pthread_create(&threads[i], NULL, pthread_output_pgsql_stop_one, &thunks[i]);
if (ret) {
- fprintf(stderr, "pthread_create() returned an error (%d)", ret);
+ fprintf(stderr, "pthread_create() returned an error (%d)\n", ret);
util::exit_nicely();
}
}
+ // set a flag if there are any errors - this allows us to collect all the
+ // threads and check them all for errors before shutting down the process.
+ bool thread_had_error = false;
for (i=0; i<NUM_TABLES; i++) {
int ret = pthread_join(threads[i], NULL);
if (ret) {
- fprintf(stderr, "pthread_join() returned an error (%d)", ret);
- util::exit_nicely();
+ fprintf(stderr, "pthread_join() returned an error (%d)\n", ret);
+ thread_had_error = true;
+ }
+ if (thunks[i].error) {
+ std::string error_message = BOOST_DIAGNOSTIC_INFO(thunks[i].error);
+ fprintf(stderr, "pthread_join() returned exception: %s\n", error_message.c_str());
+ thread_had_error = true;
}
}
+ if (thread_had_error) {
+ util::exit_nicely();
+ }
} else {
#endif
diff --git a/output.hpp b/output.hpp
index 5f1388a..851cef9 100644
--- a/output.hpp
+++ b/output.hpp
@@ -20,7 +20,7 @@
#include <utility>
typedef std::pair<osmid_t, size_t> pending_job_t;
-#if BOOST_VERSION < 105300
+#ifndef HAVE_LOCKFREE
#include <stack>
typedef std::stack<pending_job_t> pending_queue_t;
#else
diff --git a/pgsql.cpp b/pgsql.cpp
index ed3435a..9c164b2 100644
--- a/pgsql.cpp
+++ b/pgsql.cpp
@@ -7,12 +7,13 @@
#include <string.h>
#include <libpq-fe.h>
#include <boost/format.hpp>
+#include <boost/foreach.hpp>
-void escape(const char* src, std::string& dst)
+void escape(const std::string &src, std::string &dst)
{
- for(; *src; ++src)
+ BOOST_FOREACH(const char c, src)
{
- switch(*src) {
+ switch(c) {
case '\\': dst.append("\\\\"); break;
//case 8: dst.append("\\\b"); break;
//case 12: dst.append("\\\f"); break;
@@ -20,41 +21,11 @@ void escape(const char* src, std::string& dst)
case '\r': dst.append("\\\r"); break;
case '\t': dst.append("\\\t"); break;
//case 11: dst.append("\\\v"); break;
- default: dst.push_back(*src); break;
+ default: dst.push_back(c); break;
}
}
}
-void escape(char *out, const int len, const char *in)
-{
- /* Apply escaping of TEXT COPY data
- Escape: backslash itself, newline, carriage return, and the current delimiter character (tab)
- file:///usr/share/doc/postgresql-8.1.8/html/sql-copy.html
- */
- int count = 0;
- const char *old_in = in, *old_out = out;
-
- if (!len)
- return;
-
- while(*in && count < len-3) {
- switch(*in) {
- case '\\': *out++ = '\\'; *out++ = '\\'; count+= 2; break;
- /* case 8: *out++ = '\\'; *out++ = '\b'; count+= 2; break; */
- /* case 12: *out++ = '\\'; *out++ = '\f'; count+= 2; break; */
- case '\n': *out++ = '\\'; *out++ = '\n'; count+= 2; break;
- case '\r': *out++ = '\\'; *out++ = '\r'; count+= 2; break;
- case '\t': *out++ = '\\'; *out++ = '\t'; count+= 2; break;
- /* case 11: *out++ = '\\'; *out++ = '\v'; count+= 2; break; */
- default: *out++ = *in; count++; break;
- }
- in++;
- }
- *out = '\0';
-
- if (*in)
- fprintf(stderr, "%s truncated at %d chars: %s\n%s\n", __FUNCTION__, count, old_in, old_out);
-}
boost::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const std::string& sql)
{
@@ -124,12 +95,12 @@ int pgsql_exec(PGconn *sql_conn, const ExecStatusType expect, const char *fmt, .
return 0;
}
-void pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql)
+void pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql, int len)
{
#ifdef DEBUG_PGSQL
fprintf(stderr, "%s>>> %s\n", context, sql );
#endif
- int r = PQputCopyData(sql_conn, sql, strlen(sql));
+ int r = PQputCopyData(sql_conn, sql, len);
switch(r)
{
//need to wait for write ready
diff --git a/pgsql.hpp b/pgsql.hpp
index 540def3..684ad1c 100644
--- a/pgsql.hpp
+++ b/pgsql.hpp
@@ -7,11 +7,12 @@
#define PGSQL_H
#include <string>
+#include <cstring>
#include <libpq-fe.h>
#include <boost/shared_ptr.hpp>
PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, const int nParams, const char *const * paramValues, const ExecStatusType expect);
-void pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql);
+void pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql, int len);
boost::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const std::string& sql);
boost::shared_ptr<PGresult> pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const char *sql);
int pgsql_exec(PGconn *sql_conn, const ExecStatusType expect, const char *fmt, ...)
@@ -20,7 +21,14 @@ int pgsql_exec(PGconn *sql_conn, const ExecStatusType expect, const char *fmt, .
#endif
;
-void escape(const char* src, std::string& dst);
-void escape(char *out, int len, const char *in);
+void escape(const std::string &src, std::string& dst);
+
+inline void pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql) {
+ pgsql_CopyData(context, sql_conn, sql, strlen(sql));
+}
+
+inline void pgsql_CopyData(const char *context, PGconn *sql_conn, const std::string &sql) {
+ pgsql_CopyData(context, sql_conn, sql.c_str(), sql.length());
+}
#endif
diff --git a/table.cpp b/table.cpp
index 7e40f7d..e22109a 100644
--- a/table.cpp
+++ b/table.cpp
@@ -218,10 +218,10 @@ void table_t::stop()
time(&start);
fprintf(stderr, "Sorting data and creating indexes for %s\n", name.c_str());
- pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ANALYZE %1%") % name).str());
- fprintf(stderr, "Analyzing %s finished\n", name.c_str());
- pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE TABLE %1%_tmp %2% AS SELECT * FROM %3% ORDER BY way") % name % (table_space ? "TABLESPACE " + table_space.get() : "") % name).str());
+ // Special handling for empty geometries because geohash chokes on
+ // empty geometries on postgis 1.5.
+ pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE TABLE %1%_tmp %2% AS SELECT * FROM %3% ORDER BY CASE WHEN ST_IsEmpty(way) THEN NULL ELSE ST_GeoHash(ST_Transform(ST_Envelope(way),4326),10) END") % name % (table_space ? "TABLESPACE " + table_space.get() : "") % name).str());
pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("DROP TABLE %1%") % name).str());
pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ALTER TABLE %1%_tmp RENAME TO %2%") % name % name).str());
fprintf(stderr, "Copying %s to cluster by geometry finished\n", name.c_str());
@@ -273,7 +273,7 @@ void table_t::stop_copy()
//if there is stuff left over in the copy buffer send it offand copy it before we stop
else if(buffer.length() != 0)
{
- pgsql_CopyData(name.c_str(), sql_conn, buffer.c_str());
+ pgsql_CopyData(name.c_str(), sql_conn, buffer);
buffer.clear();
};
@@ -345,7 +345,7 @@ void table_t::write_wkt(const osmid_t id, struct keyval *tags, const char *wkt)
//send all the data to postgres
if(buffer.length() > BUFFER_SEND_SIZE)
{
- pgsql_CopyData(name.c_str(), sql_conn, buffer.c_str());
+ pgsql_CopyData(name.c_str(), sql_conn, buffer);
buffer.clear();
}
}
@@ -358,7 +358,7 @@ void table_t::write_columns(keyval *tags, string& values)
keyval *tag = NULL;
if ((tag = tags->getTag(column->first)))
{
- escape_type(tag->value.c_str(), column->second.c_str(), values);
+ escape_type(tag->value, column->second.c_str(), values);
//remember we already used this one so we cant use again later in the hstore column
if (hstore_mode == HSTORE_NORM)
tag->has_column = 1;
@@ -463,12 +463,12 @@ void table_t::escape4hstore(const char *src, string& dst)
}
/* Escape data appropriate to the type */
-void table_t::escape_type(const char *value, const char *type, string& dst) {
+void table_t::escape_type(const string &value, const char *type, string& dst) {
// For integers we take the first number, or the average if it's a-b
if (!strcmp(type, "int4")) {
int from, to;
- int items = sscanf(value, "%d-%d", &from, &to);
+ int items = sscanf(value.c_str(), "%d-%d", &from, &to);
if (items == 1)
dst.append((single_fmt % from).str());
else if (items == 2)
diff --git a/table.hpp b/table.hpp
index b08a3f7..0e38def 100644
--- a/table.hpp
+++ b/table.hpp
@@ -63,7 +63,7 @@ class table_t
void write_hstore_columns(keyval *tags, std::string& values);
void escape4hstore(const char *src, std::string& dst);
- void escape_type(const char *value, const char *type, std::string& dst);
+ void escape_type(const std::string &value, const char *type, std::string& dst);
std::string conninfo;
std::string name;
diff --git a/tests/regression-test.py b/tests/regression-test.py
index 418a09d..fca7b33 100755
--- a/tests/regression-test.py
+++ b/tests/regression-test.py
@@ -30,13 +30,13 @@ sql_test_statements=[
( 11, 'Absence of way table', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_ways\'', 0),
( 12, 'Absence of rel line', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_rels\'', 0),
( 13, 'Basic polygon area', 'SELECT round(sum(cast(ST_Area(way) as numeric)),0) FROM planet_osm_polygon;', 1223800814),
- ( 14, 'Gazetteer place count', 'SELECT count(*) FROM place', 4374),
- ( 15, 'Gazetteer place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 778),
- ( 16, 'Gazetteer place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 3577),
+ ( 14, 'Gazetteer place count', 'SELECT count(*) FROM place', 2837),
+ ( 15, 'Gazetteer place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 759),
+ ( 16, 'Gazetteer place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 2059),
( 17, 'Gazetteer place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 19),
- ( 18, 'Gazetteer post-diff place count', 'SELECT count(*) FROM place', 4428),
- ( 19, 'Gazetteer post-diff place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 787),
- ( 20, 'Gazetteer post-diff place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 3622),
+ ( 18, 'Gazetteer post-diff place count', 'SELECT count(*) FROM place', 2879),
+ ( 19, 'Gazetteer post-diff place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 764),
+ ( 20, 'Gazetteer post-diff place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 2096),
( 21, 'Gazetteer post-diff place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 19),
( 22, 'Gazetteer housenumber count', 'SELECT count(*) FROM place WHERE housenumber is not null', 199),
( 23, 'Gazetteer post-diff housenumber count count', 'SELECT count(*) FROM place WHERE housenumber is not null', 199),
diff --git a/util.cpp b/util.cpp
index 8b834c3..783580b 100644
--- a/util.cpp
+++ b/util.cpp
@@ -1,9 +1,12 @@
#include "util.hpp"
+#include <iostream>
+#include <cstdlib>
+
namespace util {
void exit_nicely() {
- fprintf(stderr, "Error occurred, cleaning up\n");
+ std::cerr << "Error occurred, cleaning up\n";
exit(EXIT_FAILURE);
}
diff --git a/util.hpp b/util.hpp
index 87d9069..06d570f 100644
--- a/util.hpp
+++ b/util.hpp
@@ -1,10 +1,6 @@
#ifndef UTIL_H
#define UTIL_H
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
namespace util {
inline int double_to_fix(const double x, const int scale) {
return x * scale + 0.4;
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/osm2pgsql.git
More information about the Pkg-grass-devel
mailing list