[Git][debian-gis-team/osm2pgsql][upstream] New upstream version 1.4.2+ds
Bas Couwenberg
gitlab at salsa.debian.org
Wed Apr 7 14:49:50 BST 2021
Bas Couwenberg pushed to branch upstream at Debian GIS Project / osm2pgsql
Commits:
42e455ab by Bas Couwenberg at 2021-04-07T15:32:27+02:00
New upstream version 1.4.2+ds
- - - - -
18 changed files:
- .github/actions/ubuntu-prerequisites/action.yml
- CMakeLists.txt
- CONTRIBUTING.md
- README.md
- docs/CMakeLists.txt
- + docs/osm2pgsql-replication.1
- docs/osm2pgsql.1
- + scripts/osm2pgsql-replication
- src/expire-tiles.hpp
- src/middle-pgsql.cpp
- src/node-ram-cache.cpp
- src/output-flex.cpp
- src/pgsql.cpp
- src/progress-display.cpp
- src/table.cpp
- tests/data/test_output_flex_types.lua
- tests/test-middle.cpp
- tests/test-output-flex-types.cpp
Changes:
=====================================
.github/actions/ubuntu-prerequisites/action.yml
=====================================
@@ -6,18 +6,21 @@ runs:
steps:
- name: Remove preinstalled postgresql
run: |
- sudo apt update -qq
+ sudo apt-get update -qq
sudo apt-get remove -yq postgresql*
shell: bash
- name: Install software
- run: sudo apt install -yq --no-install-suggests --no-install-recommends postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION} postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION}-scripts postgresql-client postgresql-contrib-${POSTGRESQL_VERSION} postgresql-${POSTGRESQL_VERSION} libpq-dev libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev pandoc python3-psycopg2 libluajit-5.1-dev
+ run: |
+ sudo apt-get install -yq --no-install-suggests --no-install-recommends postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION} postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION}-scripts postgresql-client postgresql-contrib-${POSTGRESQL_VERSION} postgresql-${POSTGRESQL_VERSION} libpq-dev libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libproj-dev pandoc python3-psycopg2 libluajit-5.1-dev
+ if [ "$CC" = clang-6.0 ]; then sudo apt-get install -yq --no-install-suggests --no-install-recommends clang-6.0; fi
+ if [ "$CC" = clang-8 ]; then sudo apt-get install -yq --no-install-suggests --no-install-recommends clang-8; fi
shell: bash
- name: Install lua
run: |
if [ -n "${LUA_VERSION}" ]; then
- sudo apt install -yq --no-install-suggests --no-install-recommends liblua${LUA_VERSION}-dev lua${LUA_VERSION}
+ sudo apt-get install -yq --no-install-suggests --no-install-recommends liblua${LUA_VERSION}-dev lua${LUA_VERSION}
fi
shell: bash
=====================================
CMakeLists.txt
=====================================
@@ -1,6 +1,6 @@
set(PACKAGE osm2pgsql)
set(PACKAGE_NAME osm2pgsql)
-set(PACKAGE_VERSION 1.4.1)
+set(PACKAGE_VERSION 1.4.2)
cmake_minimum_required(VERSION 2.8.12)
@@ -10,6 +10,15 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+# Do not create install targets when run as a subproject.
+# Currently used by Nominatim which cannot yet rely on installed versions
+# of osm2pgsql.
+if (${PROJECT_NAME} STREQUAL ${CMAKE_PROJECT_NAME})
+ set(ENABLE_INSTALL ON)
+else()
+ set(ENABLE_INSTALL OFF)
+endif()
+
if (WIN32)
set(DEFAULT_STYLE "default.style" CACHE STRING "Default style used unless one is given on the command line")
else()
@@ -324,5 +333,7 @@ add_subdirectory(docs)
# Install
#############################################################
-install(TARGETS osm2pgsql DESTINATION bin)
-install(FILES default.style empty.style DESTINATION share/osm2pgsql)
+if (ENABLE_INSTALL)
+ install(TARGETS osm2pgsql DESTINATION bin)
+ install(FILES default.style empty.style DESTINATION share/osm2pgsql)
+endif()
=====================================
CONTRIBUTING.md
=====================================
@@ -59,8 +59,13 @@ User documentation is available on [the website](https://osm2pgsql.org/), some
is stored in `docs/`. Pages on the OpenStreetMap wiki are known to be
unreliable and outdated.
-The [man page](docs/osm2pgsql.1) can be built from [source](docs/osm2pgsql.md)
-with `make man`. The result should be checked into the repository.
+The [osm2pgsql man page](docs/osm2pgsql.1) can be built from [source](docs/osm2pgsql.md)
+with `make man`. The [osm2pgsql-replication man page](docs/osm2pgsql-replication.1)
+has been built with:
+
+ argparse-manpage --pyfile scripts/osm2pgsql-replication --function get_parser
+
+Results should be checked into the repository.
## Platforms targeted
=====================================
README.md
=====================================
@@ -103,7 +103,7 @@ On Alpine, use
```sh
apk --update-cache add cmake make g++ boost-dev expat-dev \
- bzip2-dev zlib-dev libpq proj-dev lua5.3-dev
+ bzip2-dev zlib-dev libpq proj-dev lua5.3-dev postgresql-dev
```
Once dependencies are installed, use CMake to build the Makefiles in a separate
=====================================
docs/CMakeLists.txt
=====================================
@@ -27,5 +27,6 @@ else()
message(STATUS " Manual page can not be built")
endif()
-install(FILES osm2pgsql.1 DESTINATION share/man/man1)
-
+if (ENABLE_INSTALL)
+ install(FILES osm2pgsql.1 DESTINATION share/man/man1)
+endif()
=====================================
docs/osm2pgsql-replication.1
=====================================
@@ -0,0 +1,183 @@
+.TH osm2pgsql-replication "1" Manual
+.SH NAME
+osm2pgsql-replication
+.SH SYNOPSIS
+.B osm2pgsql-replication
+[-h] {init,update} ...
+.SH DESCRIPTION
+Update an osm2pgsql database with changes from a OSM replication server.
+.br
+
+.br
+This tool initialises the updating process by looking at the import file
+.br
+or the newest object in the database. The state is then saved in a table
+.br
+in the database. Subsequent runs download newly available data and apply
+.br
+it to the database.
+.br
+
+.br
+See the help of the 'init' and 'update' command for more information on
+.br
+how to use osm2pgsql\-replication.
+.SH OPTIONS
+
+
+.SS
+\fBSub-commands\fR
+.TP
+\fBosm2pgsql-replication\fR \fI\,init\/\fR
+ Initialise the replication process.
+.TP
+\fBosm2pgsql-replication\fR \fI\,update\/\fR
+ Download newly available data and apply it to the database.
+.SH OPTIONS 'osm2pgsql-replication init'
+usage: osm2pgsql-replication init [-h] [-q] [-v] [-d DB] [-U NAME] [-H HOST]
+ [-P PORT] [--prefix PREFIX]
+ [--osm-file FILE | --server URL]
+
+Initialise the replication process.
+.br
+
+.br
+There are two ways to initialise the replication process: if you have imported
+.br
+from a file that contains replication source information, then the
+.br
+initialisation process can use this and set up replication from there.
+.br
+Use the command '%(prog)s \-\-osm\-file <filename>' for this.
+.br
+
+.br
+If the file has no replication information or you don't have the initial
+.br
+import file anymore then replication can be set up according to
+.br
+the data found in the database. It checks the planet_osm_way table for the
+.br
+newest way in the database and then queries the OSM API when the way was
+.br
+created. The date is used as the start date for replication. In this mode
+.br
+the minutely diffs from the OSM servers are used as a source. You can change
+.br
+this with the '\-\-server' parameter.
+
+
+
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+Print only error messages
+
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Increase verboseness of output
+
+.TP
+\fB\-d\fR DB, \fB\-\-database\fR DB
+Name of PostgreSQL database to connect to or conninfo string
+
+.TP
+\fB\-U\fR NAME, \fB\-\-username\fR NAME
+PostgreSQL user name
+
+.TP
+\fB\-H\fR HOST, \fB\-\-host\fR HOST
+Database server host name or socket location
+
+.TP
+\fB\-P\fR PORT, \fB\-\-port\fR PORT
+Database server port
+
+.TP
+\fB\-\-prefix\fR PREFIX
+Prefix for table names (default 'planet_osm')
+
+.TP
+\fB\-\-osm\-file\fR FILE
+Get replication information from the given file.
+
+.TP
+\fB\-\-server\fR URL
+Use replication server at the given URL (default: https://planet.openstreetmap.org/replication/minute)
+
+.SH OPTIONS 'osm2pgsql-replication update'
+usage: osm2pgsql-replication update update [options] [-- param [param ...]]
+
+Download newly available data and apply it to the database.
+.br
+
+.br
+The data is downloaded in chunks of '\-\-max\-diff\-size' MB. Each chunk is
+.br
+saved in a temporary file and imported with osm2pgsql from there. The
+.br
+temporary file is normally deleted afterwards unless you state an explicit
+.br
+location with '\-\-diff\-file'. Once the database is up to date with the
+.br
+replication source, the update process exits with 0.
+.br
+
+.br
+Any additional arguments to osm2pgsql need to be given after '\-\-'. Database
+.br
+and the prefix parameter are handed through to osm2pgsql. They do not need
+.br
+to be repeated. '\-\-append' and '\-\-slim' will always be added as well.
+
+.TP
+\fBparam\fR
+Extra parameters to hand in to osm2pgsql.
+
+.TP
+\fB\-\-diff\-file\fR FILE
+File to save changes before they are applied to osm2pgsql.
+
+.TP
+\fB\-\-max\-diff\-size\fR \fI\,MAX_DIFF_SIZE\/\fR
+Maximum data to load in MB (default: 500MB)
+
+.TP
+\fB\-\-osm2pgsql\-cmd\fR \fI\,OSM2PGSQL_CMD\/\fR
+Path to osm2pgsql command (default: osm2pgsql)
+
+.TP
+\fB\-\-once\fR
+Run updates only once, even when more data is available.
+
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+Print only error messages
+
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Increase verboseness of output
+
+.TP
+\fB\-d\fR DB, \fB\-\-database\fR DB
+Name of PostgreSQL database to connect to or conninfo string
+
+.TP
+\fB\-U\fR NAME, \fB\-\-username\fR NAME
+PostgreSQL user name
+
+.TP
+\fB\-H\fR HOST, \fB\-\-host\fR HOST
+Database server host name or socket location
+
+.TP
+\fB\-P\fR PORT, \fB\-\-port\fR PORT
+Database server port
+
+.TP
+\fB\-\-prefix\fR PREFIX
+Prefix for table names (default 'planet_osm')
+
+.SH DISTRIBUTION
+The latest version of osm2pgsql\-replication may be downloaded from
+.UR <<UNSET \-\-url OPTION>>
+.UE
=====================================
docs/osm2pgsql.1
=====================================
@@ -1,4 +1,4 @@
-.TH "OSM2PGSQL" "1" "1.4.1" "" ""
+.TH "OSM2PGSQL" "1" "1.4.2" "" ""
.SH NAME
.PP
osm2pgsql \- Openstreetmap data to PostgreSQL converter
=====================================
scripts/osm2pgsql-replication
=====================================
@@ -0,0 +1,373 @@
+#!/usr/bin/env python3
+
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# This file is part of osm2pgsql (https://osm2pgsql.org/).
+#
+# Copyright (C) 2006-2021 by the osm2pgsql developer community.
+# For a full list of authors see the git log.
+
+"""
+Update an osm2pgsql database with changes from a OSM replication server.
+
+This tool initialises the updating process by looking at the import file
+or the newest object in the database. The state is then saved in a table
+in the database. Subsequent runs download newly available data and apply
+it to the database.
+
+See the help of the 'init' and 'update' command for more information on
+how to use %(prog)s.
+"""
+
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+import datetime as dt
+import json
+import logging
+import sys
+import subprocess
+import tempfile
+from textwrap import dedent
+import traceback
+from pathlib import Path
+import urllib.request as urlrequest
+
+import psycopg2
+
+from osmium.replication.server import ReplicationServer
+from osmium.replication.utils import get_replication_header
+from osmium import WriteHandler
+
+LOG = logging.getLogger()
+
+def connect(args):
+ """ Create a connection from the given command line arguments.
+ """
+ return psycopg2.connect(dbname=args.database, user=args.username,
+ host=args.host, port=args.port)
+
+
+def compute_database_date(conn, prefix):
+ """ Determine the date of the database from the newest object in the
+ database.
+ """
+ # First, find the way with the highest ID in the database
+ # Using nodes would be more reliable but those are not cached by osm2pgsql.
+ with conn.cursor() as cur:
+ cur.execute("SELECT max(id) FROM {}_ways".format(prefix))
+ osmid = cur.fetchone()[0] if cur.rowcount == 1 else None
+
+ if osmid is None:
+ LOG.fatal("No data found in the database.")
+ return None
+
+ LOG.debug("Using way id %d for timestamp lookup", osmid)
+ # Get the way from the API to find the timestamp when it was created.
+ url = 'https://www.openstreetmap.org/api/0.6/way/{}/1'.format(osmid)
+ headers = {"User-Agent" : "osm2pgsql-update",
+ "Accept" : "application/json"}
+ with urlrequest.urlopen(urlrequest.Request(url, headers=headers)) as response:
+ data = json.loads(response.read().decode('utf-8'))
+
+ if not data.get('elements') or not 'timestamp' in data['elements'][0]:
+ LOG.fatal("The way data downloaded from the API does not contain valid data.\n"
+ "URL used: %s", url)
+ return None
+
+ date = data['elements'][0]['timestamp']
+ LOG.debug("Found timestamp %s", date)
+
+ try:
+ date = dt.datetime.fromisoformat(date.replace('Z', '+00:00'))
+ except ValueError:
+ LOG.fatal("Cannot parse timestamp '%s'", date)
+ return None
+
+ return date.replace(tzinfo=dt.timezone.utc)
+
+
+def setup_replication_state(conn, table, base_url, seq, date):
+ """ (Re)create the table for the replication state and fill it with
+ the given state.
+ """
+ with conn.cursor() as cur:
+ cur.execute('DROP TABLE IF EXISTS "{}"'.format(table))
+ cur.execute("""CREATE TABLE "{}"
+ (url TEXT,
+ sequence INTEGER,
+ importdate TIMESTAMP WITH TIME ZONE)
+ """.format(table))
+ cur.execute('INSERT INTO "{}" VALUES(%s, %s, %s)'.format(table),
+ (base_url, seq, date))
+ conn.commit()
+
+
+def update_replication_state(conn, table, seq, date):
+ """ Update sequence and date in the replication state table.
+ The table is assumed to exist.
+ """
+ with conn.cursor() as cur:
+ if date is not None:
+ cur.execute('UPDATE "{}" SET sequence=%s, importdate=%s'.format(table),
+ (seq, date))
+ else:
+ cur.execute('UPDATE "{}" SET sequence=%s'.format(table),
+ (seq,))
+
+ conn.commit()
+
+
+def init(conn, args):
+ """\
+ Initialise the replication process.
+
+ There are two ways to initialise the replication process: if you have imported
+ from a file that contains replication source information, then the
+ initialisation process can use this and set up replication from there.
+ Use the command '%(prog)s --osm-file <filename>' for this.
+
+ If the file has no replication information or you don't have the initial
+ import file anymore then replication can be set up according to
+ the data found in the database. It checks the planet_osm_way table for the
+ newest way in the database and then queries the OSM API when the way was
+ created. The date is used as the start date for replication. In this mode
+ the minutely diffs from the OSM servers are used as a source. You can change
+ this with the '--server' parameter.
+ """
+ if args.osm_file is None:
+ date = compute_database_date(conn, args.prefix)
+ if date is None:
+ return 1
+
+ date = date - dt.timedelta(hours=3)
+ base_url = args.server
+ seq = None
+ else:
+ base_url, seq, date = get_replication_header(args.osm_file)
+ if base_url is None or (seq is None and date is None):
+ LOG.fatal("File '%s' has no usable replication headers. Use '--server' instead.")
+ return 1
+
+ repl = ReplicationServer(base_url)
+ if seq is None:
+ seq = repl.timestamp_to_sequence(date)
+
+ if seq is None:
+ LOG.fatal("Cannot reach the configured replication service '%s'.\n"
+ "Does the URL point to a directory containing OSM update data?",
+ base_url)
+ return 1
+
+ if date is None:
+ state = repl.get_state_info(seq)
+ if state is None:
+ LOG.fatal("Cannot reach the configured replication service '%s'.\n"
+ "Does the URL point to a directory containing OSM update data?",
+ base_url)
+
+ setup_replication_state(conn, args.table, base_url, seq, date)
+
+ LOG.info("Initialised updates for service '%s'.", base_url)
+ LOG.info("Starting at sequence %d (%s).", seq, date)
+
+ return 0
+
+def update(conn, args):
+ """\
+ Download newly available data and apply it to the database.
+
+ The data is downloaded in chunks of '--max-diff-size' MB. Each chunk is
+ saved in a temporary file and imported with osm2pgsql from there. The
+ temporary file is normally deleted afterwards unless you state an explicit
+ location with '--diff-file'. Once the database is up to date with the
+ replication source, the update process exits with 0.
+
+ Any additional arguments to osm2pgsql need to be given after '--'. Database
+ and the prefix parameter are handed through to osm2pgsql. They do not need
+ to be repeated. '--append' and '--slim' will always be added as well.
+
+ Use the '--post-processing' parameter to execute a script after osm2pgsql has
+ run successfully. If the updates consists of multiple runs because the
+ maximum size of downloaded data was reached, then the script is executed
+ each time that osm2pgsql has run. When the post-processing fails, then
+ the entire update run is considered a failure and the replication information
+ is not updated. That means that when 'update' is run the next time it will
+ recommence with downloading the diffs again and reapplying them to the
+ database. This is usually safe. The script receives two parameters:
+ the sequence ID and timestamp of the last successful run. The timestamp
+ may be missing in the rare case that the replication service stops responding
+ after the updates have been downloaded.
+ """
+ with conn.cursor() as cur:
+ cur.execute('SELECT * FROM pg_tables where tablename = %s', (args.table, ))
+ if cur.rowcount < 1:
+ LOG.fatal("Cannot find replication status table. "
+ "Run 'osm2pgsql-replication init' first.")
+ return 1
+
+ cur.execute('SELECT * FROM "{}"'.format(args.table))
+ if cur.rowcount != 1:
+ LOG.fatal("Updates not set up correctly. Run 'osm2pgsql-updates init' first.")
+ return 1
+
+ base_url, seq, ts = cur.fetchone()
+ LOG.info("Using replication service '%s'. Current sequence %d (%s).",
+ base_url, seq, ts)
+
+ repl = ReplicationServer(base_url)
+ current = repl.get_state_info()
+
+ if seq >= current.sequence:
+ LOG.info("Database already up-to-date.")
+ return 0
+
+ if args.diff_file is not None:
+ outfile = Path(args.diff_file)
+ else:
+ tmpdir = tempfile.TemporaryDirectory()
+ outfile = Path(tmpdir.name) / 'osm2pgsql_diff.osc.gz'
+
+ osm2pgsql = [args.osm2pgsql_cmd, '--append', '--slim', '--prefix', args.prefix]
+ osm2pgsql.extend(args.extra_params)
+ if args.database:
+ osm2pgsql.extend(('-d', args.database))
+ if args.username:
+ osm2pgsql.extend(('-U', args.username))
+ if args.host:
+ osm2pgsql.extend(('-H', args.host))
+ if args.port:
+ osm2pgsql.extend(('-P', args.port))
+ osm2pgsql.append(str(outfile))
+ LOG.debug("Calling osm2pgsql with: %s", ' '.join(osm2pgsql))
+
+ while seq < current.sequence:
+ LOG.debug("Importing from sequence %d", seq)
+ if outfile.exists():
+ outfile.unlink()
+ outhandler = WriteHandler(str(outfile))
+ endseq = repl.apply_diffs(outhandler, seq + 1,
+ max_size=args.max_diff_size * 1024)
+ outhandler.close()
+
+ if endseq is None:
+ LOG.debug("No new diffs found.")
+ break
+
+ subprocess.run(osm2pgsql, check=True)
+ seq = endseq
+
+ nextstate = repl.get_state_info(seq)
+ timestamp = nextstate.timestamp if nextstate else None
+
+ if args.post_processing:
+ cmd = [args.post_processing, str(endseq), str(timestamp or '')]
+ LOG.debug('Calling post-processing script: %s', ' '.join(cmd))
+ subprocess.run(cmd, check=True)
+
+ update_replication_state(conn, args.table, seq,
+ nextstate.timestamp if nextstate else None)
+
+ if nextstate is not None:
+ LOG.info("Data imported until %s. Backlog remaining: %s",
+ nextstate.timestamp,
+ dt.datetime.now(dt.timezone.utc) - nextstate.timestamp)
+
+ if args.once:
+ break
+
+ return 0
+
+def get_parser():
+ parser = ArgumentParser(description=__doc__,
+ prog='osm2pgsql-replication',
+ formatter_class=RawDescriptionHelpFormatter)
+ subs = parser.add_subparsers(title='available commands', dest='subcommand')
+
+ default_args = ArgumentParser(add_help=False)
+ group = default_args.add_argument_group('Default arguments')
+ group.add_argument('-h', '--help', action='help',
+ help='Show this help message and exit')
+ group.add_argument('-q', '--quiet', action='store_const', const=0,
+ dest='verbose', default=2,
+ help='Print only error messages')
+ group.add_argument('-v', '--verbose', action='count', default=2,
+ help='Increase verboseness of output')
+ group = default_args.add_argument_group('Database arguments')
+ group.add_argument('-d', '--database', metavar='DB',
+ help='Name of PostgreSQL database to connect to or conninfo string')
+ group.add_argument('-U', '--username', metavar='NAME',
+ help='PostgreSQL user name')
+ group.add_argument('-H', '--host', metavar='HOST',
+ help='Database server host name or socket location')
+ group.add_argument('-P', '--port', metavar='PORT',
+ help='Database server port')
+ group.add_argument('--prefix', metavar='PREFIX', default='planet_osm',
+ help="Prefix for table names (default 'planet_osm')")
+
+ # Arguments for init
+ cmd = subs.add_parser('init', parents=[default_args],
+ help=init.__doc__.split('\n', 1)[0],
+ description=dedent(init.__doc__),
+ formatter_class=RawDescriptionHelpFormatter,
+ add_help=False)
+ grp = cmd.add_argument_group('Replication source arguments')
+ srcgrp = grp.add_mutually_exclusive_group()
+ srcgrp.add_argument('--osm-file', metavar='FILE',
+ help='Get replication information from the given file.')
+ srcgrp.add_argument('--server', metavar='URL',
+ default='https://planet.openstreetmap.org/replication/minute',
+ help='Use replication server at the given URL (default: %(default)s)')
+ cmd.set_defaults(handler=init)
+
+ # Arguments for update
+ cmd = subs.add_parser('update', parents=[default_args],
+ usage='%(prog)s update [options] [-- param [param ...]]',
+ help=update.__doc__.split('\n', 1)[0],
+ description=dedent(update.__doc__),
+ formatter_class=RawDescriptionHelpFormatter,
+ add_help=False)
+ cmd.set_defaults(handler=update)
+ cmd.add_argument('extra_params', nargs='*', metavar='param',
+ help='Extra parameters to hand in to osm2pgsql.')
+ grp = cmd.add_argument_group('Update process arguments')
+ cmd.add_argument('--diff-file', metavar='FILE',
+ help='File to save changes before they are applied to osm2pgsql.')
+ cmd.add_argument('--max-diff-size', type=int, default=500,
+ help='Maximum data to load in MB (default: 500MB)')
+ cmd.add_argument('--osm2pgsql-cmd', default='osm2pgsql',
+ help='Path to osm2pgsql command (default: osm2pgsql)')
+ cmd.add_argument('--once', action='store_true',
+ help='Run updates only once, even when more data is available.')
+ cmd.add_argument('--post-processing', metavar='SCRIPT',
+ help='Post-processing script to run after each execution of osm2pgsql.')
+
+ return parser
+
+def main():
+ parser = get_parser()
+ args = parser.parse_args()
+
+ if args.subcommand is None:
+ parser.print_help()
+ exit(1)
+
+ logging.basicConfig(stream=sys.stderr,
+ format='{asctime} [{levelname}]: {message}',
+ style='{',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ level=max(4 - args.verbose, 1) * 10)
+
+ if '"' in args.prefix:
+ LOG.fatal("Prefix must not contain quotation marks.")
+ return 1
+
+ args.table = '{}_replication_status'.format(args.prefix)
+
+ conn = connect(args)
+ ret = args.handler(conn, args)
+ conn.close()
+
+ return ret
+
+
+if __name__ == '__main__':
+ exit(main())
=====================================
src/expire-tiles.hpp
=====================================
@@ -60,6 +60,8 @@ struct expire_tiles
expire_tiles(uint32_t maxzoom, double maxbbox,
const std::shared_ptr<reprojection> &projection);
+ bool enabled() const noexcept { return maxzoom != 0; }
+
int from_bbox(double min_lon, double min_lat, double max_lon,
double max_lat);
void from_wkb(char const *wkb, osmid_t osm_id);
=====================================
src/middle-pgsql.cpp
=====================================
@@ -48,7 +48,7 @@ static std::string build_sql(options_t const &options, char const *templ)
templ, fmt::arg("prefix", options.prefix),
fmt::arg("schema", options.middle_dbschema.empty()
? ""
- : (options.middle_dbschema + ".")),
+ : ("\"" + options.middle_dbschema + "\".")),
fmt::arg("unlogged", options.droptemp ? "UNLOGGED" : ""),
fmt::arg("using_tablespace", using_tablespace),
fmt::arg("data_tablespace", tablespace_clause(options.tblsslim_data)),
@@ -654,15 +654,17 @@ static table_sql sql_for_nodes(bool create_table) noexcept
sql.name = "{prefix}_nodes";
if (create_table) {
- sql.create_table = "CREATE {unlogged} TABLE {schema}{prefix}_nodes ("
- " id int8 PRIMARY KEY {using_tablespace},"
- " lat int4 NOT NULL,"
- " lon int4 NOT NULL"
- ") {data_tablespace};\n";
+ sql.create_table =
+ "CREATE {unlogged} TABLE {schema}\"{prefix}_nodes\" ("
+ " id int8 PRIMARY KEY {using_tablespace},"
+ " lat int4 NOT NULL,"
+ " lon int4 NOT NULL"
+ ") {data_tablespace};\n";
- sql.prepare_query = "PREPARE get_node_list(int8[]) AS"
- " SELECT id, lon, lat FROM {schema}{prefix}_nodes"
- " WHERE id = ANY($1::int8[]);\n";
+ sql.prepare_query =
+ "PREPARE get_node_list(int8[]) AS"
+ " SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\""
+ " WHERE id = ANY($1::int8[]);\n";
}
return sql;
@@ -675,7 +677,7 @@ static table_sql sql_for_ways(bool has_bucket_index,
sql.name = "{prefix}_ways";
- sql.create_table = "CREATE {unlogged} TABLE {schema}{prefix}_ways ("
+ sql.create_table = "CREATE {unlogged} TABLE {schema}\"{prefix}_ways\" ("
" id int8 PRIMARY KEY {using_tablespace},"
" nodes int8[] NOT NULL,"
" tags text[]"
@@ -683,40 +685,41 @@ static table_sql sql_for_ways(bool has_bucket_index,
sql.prepare_query = "PREPARE get_way(int8) AS"
" SELECT nodes, tags"
- " FROM {schema}{prefix}_ways WHERE id = $1;\n"
+ " FROM {schema}\"{prefix}_ways\" WHERE id = $1;\n"
"PREPARE get_way_list(int8[]) AS"
" SELECT id, nodes, tags"
- " FROM {schema}{prefix}_ways"
+ " FROM {schema}\"{prefix}_ways\""
" WHERE id = ANY($1::int8[]);\n";
if (has_bucket_index) {
sql.prepare_fw_dep_lookups =
"PREPARE mark_ways_by_node(int8) AS"
- " SELECT id FROM {schema}{prefix}_ways w"
+ " SELECT id FROM {schema}\"{prefix}_ways\" w"
" WHERE $1 = ANY(nodes)"
- " AND {schema}{prefix}_index_bucket(w.nodes)"
- " && {schema}{prefix}_index_bucket(ARRAY[$1]);\n";
+ " AND {schema}\"{prefix}_index_bucket\"(w.nodes)"
+ " && {schema}\"{prefix}_index_bucket\"(ARRAY[$1]);\n";
} else {
- sql.prepare_fw_dep_lookups = "PREPARE mark_ways_by_node(int8) AS"
- " SELECT id FROM {schema}{prefix}_ways"
- " WHERE nodes && ARRAY[$1];\n";
+ sql.prepare_fw_dep_lookups =
+ "PREPARE mark_ways_by_node(int8) AS"
+ " SELECT id FROM {schema}\"{prefix}_ways\""
+ " WHERE nodes && ARRAY[$1];\n";
}
if (way_node_index_id_shift == 0) {
sql.create_fw_dep_indexes =
- "CREATE INDEX ON {schema}{prefix}_ways USING GIN (nodes)"
+ "CREATE INDEX ON {schema}\"{prefix}_ways\" USING GIN (nodes)"
" WITH (fastupdate = off) {index_tablespace};\n";
} else {
sql.create_fw_dep_indexes =
"CREATE OR REPLACE FUNCTION"
- " {schema}{prefix}_index_bucket(int8[])"
+ " {schema}\"{prefix}_index_bucket\"(int8[])"
" RETURNS int8[] AS $$\n"
" SELECT ARRAY(SELECT DISTINCT"
" unnest($1) >> {way_node_index_id_shift})\n"
"$$ LANGUAGE SQL IMMUTABLE;\n"
- "CREATE INDEX {schema}{prefix}_ways_nodes_bucket_idx"
- " ON {schema}{prefix}_ways"
- " USING GIN ({schema}{prefix}_index_bucket(nodes))"
+ "CREATE INDEX \"{prefix}_ways_nodes_bucket_idx\""
+ " ON {schema}\"{prefix}_ways\""
+ " USING GIN ({schema}\"{prefix}_index_bucket\"(nodes))"
" WITH (fastupdate = off) {index_tablespace};\n";
}
@@ -729,7 +732,7 @@ static table_sql sql_for_relations() noexcept
sql.name = "{prefix}_rels";
- sql.create_table = "CREATE {unlogged} TABLE {schema}{prefix}_rels ("
+ sql.create_table = "CREATE {unlogged} TABLE {schema}\"{prefix}_rels\" ("
" id int8 PRIMARY KEY {using_tablespace},"
" way_off int2,"
" rel_off int2,"
@@ -740,20 +743,20 @@ static table_sql sql_for_relations() noexcept
sql.prepare_query = "PREPARE get_rel(int8) AS"
" SELECT members, tags"
- " FROM {schema}{prefix}_rels WHERE id = $1;\n";
+ " FROM {schema}\"{prefix}_rels\" WHERE id = $1;\n";
sql.prepare_fw_dep_lookups =
"PREPARE mark_rels_by_node(int8) AS"
- " SELECT id FROM {schema}{prefix}_rels"
+ " SELECT id FROM {schema}\"{prefix}_rels\""
" WHERE parts && ARRAY[$1]"
" AND parts[1:way_off] && ARRAY[$1];\n"
"PREPARE mark_rels_by_way(int8) AS"
- " SELECT id FROM {schema}{prefix}_rels"
+ " SELECT id FROM {schema}\"{prefix}_rels\""
" WHERE parts && ARRAY[$1]"
" AND parts[way_off+1:rel_off] && ARRAY[$1];\n";
sql.create_fw_dep_indexes =
- "CREATE INDEX ON {schema}{prefix}_rels USING GIN (parts)"
+ "CREATE INDEX ON {schema}\"{prefix}_rels\" USING GIN (parts)"
" WITH (fastupdate = off) {index_tablespace};\n";
return sql;
=====================================
src/node-ram-cache.cpp
=====================================
@@ -263,6 +263,10 @@ void node_ram_cache::set_dense(osmid_t id, osmium::Location location)
osmium::Location node_ram_cache::get_sparse(osmid_t id) const
{
+ if (sizeSparseTuples == 0) {
+ return osmium::Location{};
+ }
+
int64_t pivotPos = sizeSparseTuples >> 1;
int64_t minPos = 0;
int64_t maxPos = sizeSparseTuples;
=====================================
src/output-flex.cpp
=====================================
@@ -302,6 +302,11 @@ void write_integer(db_copy_mgr_t<db_deleter_by_type_and_id_t> *copy_mgr,
static void write_double(db_copy_mgr_t<db_deleter_by_type_and_id_t> *copy_mgr,
flex_table_column_t const &column, char const *str)
{
+ if (*str == '\0') {
+ write_null(copy_mgr, column);
+ return;
+ }
+
char *end = nullptr;
double const value = std::strtod(str, &end);
@@ -1289,7 +1294,7 @@ void output_flex_t::delete_from_table(table_connection_t *table_connection,
assert(table_connection);
auto const id = table_connection->table().map_id(type, osm_id);
- if (table_connection->table().has_geom_column()) {
+ if (m_expire.enabled() && table_connection->table().has_geom_column()) {
auto const result = table_connection->get_geom_by_id(type, id);
if (result.num_tuples() == 0) {
@@ -1540,10 +1545,11 @@ void output_flex_t::reprocess_marked()
lua_setfield(lua_state(), -2, "stage");
lua_pop(lua_state(), 1); // osm2pgsql
+ m_stage2_way_ids->sort_unique();
+
log_info(
"There are {} ways to reprocess..."_format(m_stage2_way_ids->size()));
- m_stage2_way_ids->sort_unique();
for (osmid_t const id : *m_stage2_way_ids) {
m_buffer.clear();
if (!m_mid->way_get(id, &m_buffer)) {
=====================================
src/pgsql.cpp
=====================================
@@ -127,6 +127,23 @@ void pg_conn_t::end_copy(std::string const &context) const
}
}
+static std::string concat_params(int num_params,
+ char const *const *param_values)
+{
+ std::string params;
+
+ for (int i = 0; i < num_params; ++i) {
+ params += param_values[i] ? param_values[i] : "<NULL>";
+ params += ',';
+ }
+
+ if (!params.empty()) {
+ params.resize(params.size() - 1);
+ }
+
+ return params;
+}
+
pg_result_t
pg_conn_t::exec_prepared_internal(char const *stmt, int num_params,
char const *const *param_values) const
@@ -134,28 +151,14 @@ pg_conn_t::exec_prepared_internal(char const *stmt, int num_params,
assert(m_conn);
if (get_logger().log_sql()) {
- std::string params;
- for (int i = 0; i < num_params; ++i) {
- params += param_values[i] ? param_values[i] : "<NULL>";
- params += ',';
- }
- if (!params.empty()) {
- params.resize(params.size() - 1);
- }
- log_sql("EXECUTE {}({})", stmt, params);
+ log_sql("EXECUTE {}({})", stmt,
+ concat_params(num_params, param_values));
}
pg_result_t res{PQexecPrepared(m_conn.get(), stmt, num_params, param_values,
nullptr, nullptr, 0)};
if (PQresultStatus(res.get()) != PGRES_TUPLES_OK) {
- std::string params;
- for (int i = 0; i < num_params; ++i) {
- params += param_values[i] ? param_values[i] : "<NULL>";
- params += ',';
- }
- if (!params.empty()) {
- params.resize(params.size() - 1);
- }
- log_error("SQL command failed: EXECUTE {}({})", stmt, params);
+ log_error("SQL command failed: EXECUTE {}({})", stmt,
+ concat_params(num_params, param_values));
throw std::runtime_error{"Database error: {} ({})"_format(
error_msg(), PQresultStatus(res.get()))};
}
=====================================
src/progress-display.cpp
=====================================
@@ -40,7 +40,7 @@ void progress_display_t::print_summary() const
std::time_t const now = std::time(nullptr);
if (m_enabled) {
- fmt::print(stderr, "\r{:78s}\r", "");
+ fmt::print(stderr, "\r{:90s}\r", "");
}
log_info("Reading input files done in {}.",
=====================================
src/table.cpp
=====================================
@@ -225,8 +225,8 @@ void table_t::stop(bool updateable, bool enable_hstore_index,
m_sql_conn->exec(sql);
m_sql_conn->exec("DROP TABLE {}"_format(qual_name));
- m_sql_conn->exec(
- "ALTER TABLE {} RENAME TO {}"_format(qual_tmp_name, m_target->name));
+ m_sql_conn->exec("ALTER TABLE {} RENAME TO \"{}\""_format(
+ qual_tmp_name, m_target->name));
log_info("Creating geometry index on table '{}'...", m_target->name);
=====================================
tests/data/test_output_flex_types.lua
=====================================
@@ -119,6 +119,7 @@ function osm2pgsql.process_node(object)
tint2 = n,
tint4 = n,
tint8 = n,
+ treal = n,
}
end
return
=====================================
tests/test-middle.cpp
=====================================
@@ -44,6 +44,26 @@ struct options_slim_default
}
};
+struct options_slim_with_lc_prefix
+{
+ static options_t options(testing::pg::tempdb_t const &tmpdb)
+ {
+ options_t o = testing::opt_t().slim(tmpdb);
+ o.prefix = "pre";
+ return o;
+ }
+};
+
+struct options_slim_with_uc_prefix
+{
+ static options_t options(testing::pg::tempdb_t const &tmpdb)
+ {
+ options_t o = testing::opt_t().slim(tmpdb);
+ o.prefix = "PRE";
+ return o;
+ }
+};
+
struct options_slim_with_schema
{
static options_t options(testing::pg::tempdb_t const &tmpdb)
@@ -112,6 +132,7 @@ TEST_CASE("elem_cache_t")
}
TEMPLATE_TEST_CASE("middle import", "", options_slim_default,
+ options_slim_with_lc_prefix, options_slim_with_uc_prefix,
options_slim_with_schema, options_slim_dense_cache,
options_ram_optimized, options_ram_flatnode)
{
=====================================
tests/test-output-flex-types.cpp
=====================================
@@ -176,13 +176,13 @@ TEST_CASE("type string (with invalid number)")
CHECK(7 == conn.get_count("nodes"));
// clang-format off
- CHECK(1 == conn.get_count("nodes", "ttext = '' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL"));
- CHECK(1 == conn.get_count("nodes", "ttext = 'abc' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL"));
- CHECK(1 == conn.get_count("nodes", "ttext = '0a' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL"));
- CHECK(1 == conn.get_count("nodes", "ttext = '0xa' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL"));
- CHECK(1 == conn.get_count("nodes", "ttext = '--1' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL"));
- CHECK(1 == conn.get_count("nodes", "ttext = '1foo' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL"));
- CHECK(1 == conn.get_count("nodes", "ttext = '1.2' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL"));
+ CHECK(1 == conn.get_count("nodes", "ttext = '' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL AND treal IS NULL"));
+ CHECK(1 == conn.get_count("nodes", "ttext = 'abc' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL AND treal IS NULL"));
+ CHECK(1 == conn.get_count("nodes", "ttext = '0a' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL AND treal IS NULL"));
+ CHECK(1 == conn.get_count("nodes", "ttext = '0xa' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL AND abs(treal - 10) < 0.0000001"));
+ CHECK(1 == conn.get_count("nodes", "ttext = '--1' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL AND treal IS NULL"));
+ CHECK(1 == conn.get_count("nodes", "ttext = '1foo' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL AND treal IS NULL"));
+ CHECK(1 == conn.get_count("nodes", "ttext = '1.2' AND tint2 IS NULL AND tint4 IS NULL AND tint8 IS NULL AND abs(treal - 1.2) < 0.0000001"));
// clang-format on
}
View it on GitLab: https://salsa.debian.org/debian-gis-team/osm2pgsql/-/commit/42e455ab6773f3205f25e4413261684c37d62434
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/osm2pgsql/-/commit/42e455ab6773f3205f25e4413261684c37d62434
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20210407/719066ed/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list