[Git][debian-gis-team/osm2pgsql][upstream] New upstream version 1.8.1+ds
Bas Couwenberg (@sebastic)
gitlab at salsa.debian.org
Mon Feb 13 13:53:56 GMT 2023
Bas Couwenberg pushed to branch upstream at Debian GIS Project / osm2pgsql
Commits:
fae0d624 by Bas Couwenberg at 2023-02-13T14:38:46+01:00
New upstream version 1.8.1+ds
- - - - -
28 changed files:
- .github/workflows/test-install.yml
- CMakeLists.txt
- docs/osm2pgsql-replication.1
- docs/osm2pgsql.1
- scripts/osm2pgsql-replication
- src/CMakeLists.txt
- src/db-copy-mgr.hpp
- + src/expire-config.hpp
- src/expire-tiles.cpp
- src/expire-tiles.hpp
- + src/flex-lua-table.cpp
- + src/flex-lua-table.hpp
- src/flex-table.cpp
- src/flex-table.hpp
- src/flex-write.cpp
- src/flex-write.hpp
- src/geom-pole-of-inaccessibility.cpp
- src/osmdata.cpp
- src/osmdata.hpp
- src/output-flex.cpp
- src/output-flex.hpp
- src/output-pgsql.cpp
- src/output-pgsql.hpp
- src/output.hpp
- tests/bdd/steps/steps_osm_data.py
- tests/test-expire-from-geometry.cpp
- tests/test-expire-tiles.cpp
- tests/test-parse-osmium.cpp
Changes:
=====================================
.github/workflows/test-install.yml
=====================================
@@ -6,6 +6,17 @@ jobs:
ubuntu-test-install:
runs-on: ubuntu-20.04
+ strategy:
+ matrix:
+ flavour: [public, middle_schema]
+ include:
+ - flavour: public
+ options: ""
+ schema: ""
+ - flavour: middle_schema
+ options: "--middle-schema=myschema"
+ schema: "myschema"
+
env:
LUA_VERSION: 5.3
POSTGRESQL_VERSION: 12
@@ -75,8 +86,16 @@ jobs:
sudo systemctl start postgresql
sudo -u postgres createuser runner
sudo -u postgres createdb -O runner o2ptest
- sudo -u postgres psql o2ptest -c "CREATE EXTENSION postgis;"
- sudo -u postgres psql o2ptest -c "CREATE EXTENSION hstore;"
+ sudo -u postgres psql o2ptest -c "CREATE EXTENSION postgis"
+ sudo -u postgres psql o2ptest -c "CREATE EXTENSION hstore"
+
+ - name: Set up schema
+ run: |
+ sudo -u postgres psql o2ptest -c "CREATE SCHEMA $SCHEMANAME"
+ sudo -u postgres psql o2ptest -c "GRANT ALL ON SCHEMA $SCHEMANAME TO runner"
+ if: ${{ matrix.schema }}
+ env:
+ SCHEMANAME: ${{ matrix.schema }}
- name: Remove repository
# Remove contents of workspace to be sure the install runs independently
@@ -92,15 +111,18 @@ jobs:
run: wget --quiet $OSMURL
working-directory: /tmp
- - name: Test run of osm2pgsql
- run: $PREFIX/bin/osm2pgsql -d o2ptest --slim $OSMFILE
+ - name: Test run of osm2pgsql (no schema)
+ run: $PREFIX/bin/osm2pgsql $EXTRAOPTS -d o2ptest --slim $OSMFILE
working-directory: /tmp
+ env:
+ EXTRAOPTS: ${{ matrix.options }}
- - name: Test run osm2pgsql-replication
+ - name: Test run osm2pgsql-replication (no schema)
run: |
- $PREFIX/bin/osm2pgsql-replication init -v -d o2ptest
- $PREFIX/bin/osm2pgsql-replication status -v -d o2ptest
- $PREFIX/bin/osm2pgsql-replication update -v -d o2ptest --once --max-diff-size=1
- $PREFIX/bin/osm2pgsql-replication status -v -d o2ptest --json
+ $PREFIX/bin/osm2pgsql-replication init $EXTRAOPTS -v -d o2ptest
+ $PREFIX/bin/osm2pgsql-replication status $EXTRAOPTS -v -d o2ptest
+ $PREFIX/bin/osm2pgsql-replication update $EXTRAOPTS -v -d o2ptest --once --max-diff-size=1
+ $PREFIX/bin/osm2pgsql-replication status $EXTRAOPTS -v -d o2ptest --json
working-directory: /tmp
-
+ env:
+ EXTRAOPTS: ${{ matrix.options }}
=====================================
CMakeLists.txt
=====================================
@@ -1,7 +1,7 @@
cmake_minimum_required(VERSION 3.5.0)
-project(osm2pgsql VERSION 1.8.0 LANGUAGES CXX C)
+project(osm2pgsql VERSION 1.8.1 LANGUAGES CXX C)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
=====================================
docs/osm2pgsql-replication.1
=====================================
@@ -1,4 +1,4 @@
-.TH "OSM2PGSQL-REPLICATION" "1" "1.8.0" "" ""
+.TH "OSM2PGSQL-REPLICATION" "1" "1.8.1" "" ""
.SH NAME
osm2pgsql-replication \- osm2pgsql database updater
.SH SYNOPSIS
=====================================
docs/osm2pgsql.1
=====================================
@@ -1,4 +1,4 @@
-.TH "OSM2PGSQL" "1" "1.8.0" "" ""
+.TH "OSM2PGSQL" "1" "1.8.1" "" ""
.SH NAME
.PP
osm2pgsql - Openstreetmap data to PostgreSQL converter
=====================================
scripts/osm2pgsql-replication
=====================================
@@ -95,14 +95,14 @@ def table_exists(conn, table_name, schema_name=None):
return cur.rowcount > 0
-def compute_database_date(conn, prefix):
+def compute_database_date(conn, schema, prefix):
""" Determine the date of the database from the newest object in the
database.
"""
# First, find the way with the highest ID in the database
# Using nodes would be more reliable but those are not cached by osm2pgsql.
with conn.cursor() as cur:
- table = sql.Identifier(f'{prefix}_ways')
+ table = sql.Identifier(schema, f'{prefix}_ways')
cur.execute(sql.SQL("SELECT max(id) FROM {}").format(table))
osmid = cur.fetchone()[0] if cur.rowcount == 1 else None
@@ -290,7 +290,7 @@ def init(conn, args):
this with the `--server` parameter.
"""
if args.osm_file is None:
- date = compute_database_date(conn, args.prefix)
+ date = compute_database_date(conn, args.middle_schema, args.prefix)
if date is None:
return 1
@@ -383,6 +383,8 @@ def update(conn, args):
osm2pgsql = [args.osm2pgsql_cmd, '--append', '--slim', '--prefix', args.prefix]
osm2pgsql.extend(args.extra_params)
+ if args.middle_schema != 'public':
+ osm2pgsql.extend(('--middle-schema', args.middle_schema))
if args.database:
osm2pgsql.extend(('-d', args.database))
if args.username:
=====================================
src/CMakeLists.txt
=====================================
@@ -52,6 +52,7 @@ if (WITH_LUA)
flex-table-column.cpp
flex-lua-geom.cpp
flex-lua-index.cpp
+ flex-lua-table.cpp
flex-write.cpp
geom-transform.cpp
lua-utils.cpp
=====================================
src/db-copy-mgr.hpp
=====================================
@@ -271,6 +271,14 @@ public:
m_current->add_deletable(std::forward<ARGS>(args)...);
}
+ void flush()
+ {
+ // finish any ongoing copy operations
+ if (m_current) {
+ m_processor->add_buffer(std::move(m_current));
+ }
+ }
+
/**
* Synchronize with worker.
*
@@ -278,11 +286,7 @@ public:
*/
void sync()
{
- // finish any ongoing copy operations
- if (m_current) {
- m_processor->add_buffer(std::move(m_current));
- }
-
+ flush();
m_processor->sync_and_wait();
}
=====================================
src/expire-config.hpp
=====================================
@@ -0,0 +1,41 @@
+#ifndef OSM2PGSQL_FLEX_EXPIRE_CONFIG_HPP
+#define OSM2PGSQL_FLEX_EXPIRE_CONFIG_HPP
+
+/**
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * This file is part of osm2pgsql (https://osm2pgsql.org/).
+ *
+ * Copyright (C) 2006-2023 by the osm2pgsql developer community.
+ * For a full list of authors see the git log.
+ */
+
+#include <cstdlib>
+
+enum class expire_mode
+{
+ full_area, // Expire all tiles covered by polygon.
+ boundary_only, // Expire only tiles covered by polygon boundary.
+ hybrid // "full_area" or "boundary_only" mode depending on full_area_limit.
+};
+
+/**
+ * These are the options used for tile expiry calculations.
+ */
+struct expire_config_t
+{
+ /// Buffer around expired feature as fraction of the tile size.
+ double buffer = 0.1;
+
+ /**
+ * Maximum width/heigth of bbox of a (multi)polygon before hybrid mode
+ * expiry switches from full-area to boundary-only expire.
+ */
+ double full_area_limit = 0.0;
+
+ /// Expire mode.
+ expire_mode mode = expire_mode::full_area;
+
+}; // struct expire_config_t
+
+#endif // OSM2PGSQL_FLEX_EXPIRE_CONFIG_HPP
=====================================
src/expire-tiles.cpp
=====================================
@@ -34,13 +34,10 @@
#include "tile.hpp"
#include "wkb.hpp"
-// How many tiles worth of space to leave either side of a changed feature
-static constexpr double const tile_expiry_leeway = 0.1;
-
-expire_tiles::expire_tiles(uint32_t max_zoom, double max_bbox,
+expire_tiles::expire_tiles(uint32_t max_zoom,
std::shared_ptr<reprojection> projection)
-: m_projection(std::move(projection)), m_max_bbox(max_bbox),
- m_maxzoom(max_zoom), m_map_width(1U << m_maxzoom)
+: m_projection(std::move(projection)), m_maxzoom(max_zoom),
+ m_map_width(1U << m_maxzoom)
{}
void expire_tiles::expire_tile(uint32_t x, uint32_t y)
@@ -71,68 +68,94 @@ geom::point_t expire_tiles::coords_to_tile(geom::point_t const &point)
m_map_width * (0.5 - c.y() / tile_t::earth_circumference)};
}
-void expire_tiles::from_point_list(geom::point_list_t const &list)
+void expire_tiles::from_point_list(geom::point_list_t const &list,
+ expire_config_t const &expire_config)
{
for_each_segment(list, [&](geom::point_t const &a, geom::point_t const &b) {
- from_line(a, b);
+ from_line_segment(a, b, expire_config);
});
}
-void expire_tiles::from_geometry(geom::point_t const &geom)
+void expire_tiles::from_geometry(geom::point_t const &geom,
+ expire_config_t const &expire_config)
{
geom::box_t const box = geom::envelope(geom);
- from_bbox(box);
+ from_bbox(box, expire_config);
}
-void expire_tiles::from_geometry(geom::linestring_t const &geom)
+void expire_tiles::from_geometry(geom::linestring_t const &geom,
+ expire_config_t const &expire_config)
{
- from_point_list(geom);
+ from_point_list(geom, expire_config);
}
-void expire_tiles::from_polygon_boundary(geom::polygon_t const &geom)
+void expire_tiles::from_polygon_boundary(geom::polygon_t const &geom,
+ expire_config_t const &expire_config)
{
- from_point_list(geom.outer());
+ from_point_list(geom.outer(), expire_config);
for (auto const &inner : geom.inners()) {
- from_point_list(inner);
+ from_point_list(inner, expire_config);
}
}
-void expire_tiles::from_geometry(geom::polygon_t const &geom)
+void expire_tiles::from_geometry(geom::polygon_t const &geom,
+ expire_config_t const &expire_config)
{
+ if (expire_config.mode == expire_mode::boundary_only) {
+ from_polygon_boundary(geom, expire_config);
+ return;
+ }
+
geom::box_t const box = geom::envelope(geom);
- if (from_bbox(box)) {
+ if (from_bbox(box, expire_config)) {
/* Bounding box too big - just expire tiles on the boundary */
- from_polygon_boundary(geom);
+ from_polygon_boundary(geom, expire_config);
+ }
+}
+
+void expire_tiles::from_polygon_boundary(geom::multipolygon_t const &geom,
+ expire_config_t const &expire_config)
+{
+ for (auto const &sgeom : geom) {
+ from_polygon_boundary(sgeom, expire_config);
}
}
-void expire_tiles::from_geometry(geom::multipolygon_t const &geom)
+void expire_tiles::from_geometry(geom::multipolygon_t const &geom,
+ expire_config_t const &expire_config)
{
+ if (expire_config.mode == expire_mode::boundary_only) {
+ from_polygon_boundary(geom, expire_config);
+ return;
+ }
+
geom::box_t const box = geom::envelope(geom);
- if (from_bbox(box)) {
+ if (from_bbox(box, expire_config)) {
/* Bounding box too big - just expire tiles on the boundary */
- for (auto const &sgeom : geom) {
- from_polygon_boundary(sgeom);
- }
+ from_polygon_boundary(geom, expire_config);
}
}
-void expire_tiles::from_geometry(geom::geometry_t const &geom)
+void expire_tiles::from_geometry(geom::geometry_t const &geom,
+ expire_config_t const &expire_config)
{
- geom.visit([&](auto const &g) { from_geometry(g); });
+ geom.visit([&](auto const &g) { from_geometry(g, expire_config); });
}
-void expire_tiles::from_geometry_if_3857(geom::geometry_t const &geom)
+void expire_tiles::from_geometry_if_3857(geom::geometry_t const &geom,
+ expire_config_t const &expire_config)
{
if (geom.srid() == 3857) {
- from_geometry(geom);
+ from_geometry(geom, expire_config);
}
}
/*
* Expire tiles that a line crosses
*/
-void expire_tiles::from_line(geom::point_t const &a, geom::point_t const &b)
+void expire_tiles::from_line_segment(geom::point_t const &a,
+ geom::point_t const &b,
+ expire_config_t const &expire_config)
{
auto tilec_a = coords_to_tile(a);
auto tilec_b = coords_to_tile(b);
@@ -175,11 +198,11 @@ void expire_tiles::from_line(geom::point_t const &a, geom::point_t const &b)
if (y1 > y2) {
std::swap(y1, y2);
}
- for (int x = x1 - tile_expiry_leeway; x <= x2 + tile_expiry_leeway;
+ for (int x = x1 - expire_config.buffer; x <= x2 + expire_config.buffer;
++x) {
uint32_t const norm_x = normalise_tile_x_coord(x);
- for (int y = y1 - tile_expiry_leeway; y <= y2 + tile_expiry_leeway;
- ++y) {
+ for (int y = y1 - expire_config.buffer;
+ y <= y2 + expire_config.buffer; ++y) {
if (y >= 0) {
expire_tile(norm_x, static_cast<uint32_t>(y));
}
@@ -191,7 +214,8 @@ void expire_tiles::from_line(geom::point_t const &a, geom::point_t const &b)
/*
* Expire tiles within a bounding box
*/
-int expire_tiles::from_bbox(geom::box_t const &box)
+int expire_tiles::from_bbox(geom::box_t const &box,
+ expire_config_t const &expire_config)
{
if (!enabled()) {
return 0;
@@ -203,28 +227,32 @@ int expire_tiles::from_bbox(geom::box_t const &box)
/* Over half the planet's width within the bounding box - assume the
box crosses the international date line and split it into two boxes */
int ret = from_bbox({-tile_t::half_earth_circumference, box.min_y(),
- box.min_x(), box.max_y()});
+ box.min_x(), box.max_y()},
+ expire_config);
ret += from_bbox({box.max_x(), box.min_y(),
- tile_t::half_earth_circumference, box.max_y()});
+ tile_t::half_earth_circumference, box.max_y()},
+ expire_config);
return ret;
}
- if (width > m_max_bbox || height > m_max_bbox) {
+ if (expire_config.mode == expire_mode::hybrid &&
+ (width > expire_config.full_area_limit ||
+ height > expire_config.full_area_limit)) {
return -1;
}
/* Convert the box's Mercator coordinates into tile coordinates */
auto const tmp_min = coords_to_tile({box.min_x(), box.max_y()});
int const min_tile_x =
- std::clamp(int(tmp_min.x() - tile_expiry_leeway), 0, m_map_width);
+ std::clamp(int(tmp_min.x() - expire_config.buffer), 0, m_map_width);
int const min_tile_y =
- std::clamp(int(tmp_min.y() - tile_expiry_leeway), 0, m_map_width);
+ std::clamp(int(tmp_min.y() - expire_config.buffer), 0, m_map_width);
auto const tmp_max = coords_to_tile({box.max_x(), box.min_y()});
int const max_tile_x =
- std::clamp(int(tmp_max.x() + tile_expiry_leeway), 0, m_map_width);
+ std::clamp(int(tmp_max.x() + expire_config.buffer), 0, m_map_width);
int const max_tile_y =
- std::clamp(int(tmp_max.y() + tile_expiry_leeway), 0, m_map_width);
+ std::clamp(int(tmp_max.y() + expire_config.buffer), 0, m_map_width);
for (int iterator_x = min_tile_x; iterator_x <= max_tile_x; ++iterator_x) {
uint32_t const norm_x = normalise_tile_x_coord(iterator_x);
@@ -264,11 +292,11 @@ void expire_tiles::merge_and_destroy(expire_tiles *other)
}
}
-std::size_t output_tiles_to_file(quadkey_list_t const &tiles_maxzoom,
- char const *filename, uint32_t minzoom,
- uint32_t maxzoom)
+std::size_t output_tiles_to_file(quadkey_list_t const &tiles_at_maxzoom,
+ uint32_t minzoom, uint32_t maxzoom,
+ std::string_view filename)
{
- FILE *outfile = std::fopen(filename, "a");
+ FILE *outfile = std::fopen(filename.data(), "a");
if (outfile == nullptr) {
log_warn("Failed to open expired tiles file ({}). Tile expiry "
"list will not be written!",
@@ -276,8 +304,8 @@ std::size_t output_tiles_to_file(quadkey_list_t const &tiles_maxzoom,
return 0;
}
- auto const count =
- for_each_tile(tiles_maxzoom, minzoom, maxzoom, [&](tile_t const &tile) {
+ auto const count = for_each_tile(
+ tiles_at_maxzoom, minzoom, maxzoom, [&](tile_t const &tile) {
fmt::print(outfile, "{}/{}/{}\n", tile.zoom(), tile.x(), tile.y());
});
@@ -286,12 +314,13 @@ std::size_t output_tiles_to_file(quadkey_list_t const &tiles_maxzoom,
return count;
}
-int expire_from_result(expire_tiles *expire, pg_result_t const &result)
+int expire_from_result(expire_tiles *expire, pg_result_t const &result,
+ expire_config_t const &expire_config)
{
auto const num_tuples = result.num_tuples();
for (int i = 0; i < num_tuples; ++i) {
- expire->from_geometry(ewkb_to_geom(result.get(i, 0)));
+ expire->from_geometry(ewkb_to_geom(result.get(i, 0)), expire_config);
}
return num_tuples;
=====================================
src/expire-tiles.hpp
=====================================
@@ -11,10 +11,12 @@
*/
#include <memory>
+#include <string_view>
#include <unordered_set>
#include <utility>
#include <vector>
+#include "expire-config.hpp"
#include "geom.hpp"
#include "geom-box.hpp"
#include "logging.hpp"
@@ -27,32 +29,48 @@ class reprojection;
class expire_tiles
{
public:
- expire_tiles(uint32_t max_zoom, double max_bbox,
- std::shared_ptr<reprojection> projection);
+ expire_tiles(uint32_t max_zoom, std::shared_ptr<reprojection> projection);
bool enabled() const noexcept { return m_maxzoom != 0; }
- void from_polygon_boundary(geom::polygon_t const &geom);
+ void from_polygon_boundary(geom::polygon_t const &geom,
+ expire_config_t const &expire_config);
- void from_geometry(geom::nullgeom_t const & /*geom*/) {}
- void from_geometry(geom::point_t const &geom);
- void from_geometry(geom::linestring_t const &geom);
- void from_geometry(geom::polygon_t const &geom);
- void from_geometry(geom::multipolygon_t const &geom);
+ void from_polygon_boundary(geom::multipolygon_t const &geom,
+ expire_config_t const &expire_config);
+
+ void from_geometry(geom::nullgeom_t const & /*geom*/,
+ expire_config_t const & /*expire_config*/)
+ {}
+
+ void from_geometry(geom::point_t const &geom,
+ expire_config_t const &expire_config);
+
+ void from_geometry(geom::linestring_t const &geom,
+ expire_config_t const &expire_config);
+
+ void from_geometry(geom::polygon_t const &geom,
+ expire_config_t const &expire_config);
+
+ void from_geometry(geom::multipolygon_t const &geom,
+ expire_config_t const &expire_config);
template <typename T>
- void from_geometry(geom::multigeometry_t<T> const &geom)
+ void from_geometry(geom::multigeometry_t<T> const &geom,
+ expire_config_t const &expire_config)
{
for (auto const &sgeom : geom) {
- from_geometry(sgeom);
+ from_geometry(sgeom, expire_config);
}
}
- void from_geometry(geom::geometry_t const &geom);
+ void from_geometry(geom::geometry_t const &geom,
+ expire_config_t const &expire_config);
- void from_geometry_if_3857(geom::geometry_t const &geom);
+ void from_geometry_if_3857(geom::geometry_t const &geom,
+ expire_config_t const &expire_config);
- int from_bbox(geom::box_t const &box);
+ int from_bbox(geom::box_t const &box, expire_config_t const &expire_config);
/**
* Get tiles as a vector of quadkeys and remove them from the expire_tiles
@@ -80,10 +98,14 @@ private:
* \param y y index of the tile to be expired.
*/
void expire_tile(uint32_t x, uint32_t y);
+
uint32_t normalise_tile_x_coord(int x) const;
- void from_line(geom::point_t const &a, geom::point_t const &b);
- void from_point_list(geom::point_list_t const &list);
+ void from_line_segment(geom::point_t const &a, geom::point_t const &b,
+ expire_config_t const &expire_config);
+
+ void from_point_list(geom::point_list_t const &list,
+ expire_config_t const &expire_config);
/// This is where we collect all the expired tiles.
std::unordered_set<quadkey_t> m_dirty_tiles;
@@ -93,7 +115,6 @@ private:
std::shared_ptr<reprojection> m_projection;
- double m_max_bbox;
uint32_t m_maxzoom;
int m_map_width;
@@ -106,9 +127,11 @@ private:
* \param result Result of a database query into some table returning the
* geometries. (This is usually done using the "get_wkb"
* prepared statement.)
+ * \param expire_config Configuration for expiry.
* \return The number of tuples in the result or -1 if expire is disabled.
*/
-int expire_from_result(expire_tiles *expire, pg_result_t const &result);
+int expire_from_result(expire_tiles *expire, pg_result_t const &result,
+ expire_config_t const &expire_config);
/**
* Iterate over tiles and call output function for each tile on all requested
@@ -116,23 +139,23 @@ int expire_from_result(expire_tiles *expire, pg_result_t const &result);
*
* \tparam OUTPUT Class with operator() taking a tile_t argument
*
- * \param tiles The list of tiles at maximum zoom level
+ * \param tiles_at_maxzoom The list of tiles at maximum zoom level
* \param minzoom Minimum zoom level
* \param maxzoom Maximum zoom level
* \param output Output function
*/
template <class OUTPUT>
-std::size_t for_each_tile(quadkey_list_t const &tiles, uint32_t minzoom,
- uint32_t maxzoom, OUTPUT &&output)
+std::size_t for_each_tile(quadkey_list_t const &tiles_at_maxzoom,
+ uint32_t minzoom, uint32_t maxzoom, OUTPUT &&output)
{
assert(minzoom <= maxzoom);
if (minzoom == maxzoom) {
- for (auto const quadkey : tiles) {
+ for (auto const quadkey : tiles_at_maxzoom) {
std::forward<OUTPUT>(output)(
tile_t::from_quadkey(quadkey, maxzoom));
}
- return tiles.size();
+ return tiles_at_maxzoom.size();
}
/**
@@ -141,7 +164,7 @@ std::size_t for_each_tile(quadkey_list_t const &tiles, uint32_t minzoom,
*/
quadkey_t last_quadkey{};
std::size_t count = 0;
- for (auto const quadkey : tiles) {
+ for (auto const quadkey : tiles_at_maxzoom) {
for (uint32_t dz = 0; dz <= maxzoom - minzoom; ++dz) {
auto const qt_current = quadkey.down(dz);
/**
@@ -164,13 +187,13 @@ std::size_t for_each_tile(quadkey_list_t const &tiles, uint32_t minzoom,
/**
* Write the list of tiles to a file.
*
- * \param tiles The list of tiles at maximum zoom level
- * \param filename Name of the file
+ * \param tiles_at_maxzoom The list of tiles at maximum zoom level
* \param minzoom Minimum zoom level
* \param maxzoom Maximum zoom level
+ * \param filename Name of the file
*/
-std::size_t output_tiles_to_file(quadkey_list_t const &tiles,
- char const *filename, uint32_t minzoom,
- uint32_t maxzoom);
+std::size_t output_tiles_to_file(quadkey_list_t const &tiles_at_maxzoom,
+ uint32_t minzoom, uint32_t maxzoom,
+ std::string_view filename);
#endif // OSM2PGSQL_EXPIRE_TILES_HPP
=====================================
src/flex-lua-table.cpp
=====================================
@@ -0,0 +1,293 @@
+/**
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * This file is part of osm2pgsql (https://osm2pgsql.org/).
+ *
+ * Copyright (C) 2006-2023 by the osm2pgsql developer community.
+ * For a full list of authors see the git log.
+ */
+
+#include "flex-lua-index.hpp"
+#include "flex-lua-table.hpp"
+#include "lua-utils.hpp"
+#include "flex-table.hpp"
+#include "pgsql-capabilities.hpp"
+
+#include <lua.hpp>
+
+static void check_tablespace(std::string const &tablespace)
+{
+ if (!has_tablespace(tablespace)) {
+ throw fmt_error(
+ "Tablespace '{0}' not available."
+ " Use 'CREATE TABLESPACE \"{0}\" ...;' to create it.",
+ tablespace);
+ }
+}
+
+static flex_table_t &create_flex_table(lua_State *lua_state,
+ std::vector<flex_table_t> *tables)
+{
+ std::string const table_name =
+ luaX_get_table_string(lua_state, "name", -1, "The table");
+
+ check_identifier(table_name, "table names");
+
+ if (util::find_by_name(*tables, table_name)) {
+ throw fmt_error("Table with name '{}' already exists.", table_name);
+ }
+
+ auto &new_table = tables->emplace_back(table_name);
+
+ lua_pop(lua_state, 1); // "name"
+
+ // optional "schema" field
+ lua_getfield(lua_state, -1, "schema");
+ if (lua_isstring(lua_state, -1)) {
+ std::string const schema = lua_tostring(lua_state, -1);
+ check_identifier(schema, "schema field");
+ if (!has_schema(schema)) {
+ throw fmt_error("Schema '{0}' not available."
+ " Use 'CREATE SCHEMA \"{0}\";' to create it.",
+ schema);
+ }
+ new_table.set_schema(schema);
+ }
+ lua_pop(lua_state, 1);
+
+ // optional "cluster" field
+ lua_getfield(lua_state, -1, "cluster");
+ int const cluster_type = lua_type(lua_state, -1);
+ if (cluster_type == LUA_TSTRING) {
+ std::string const cluster = lua_tostring(lua_state, -1);
+ if (cluster == "auto") {
+ new_table.set_cluster_by_geom(true);
+ } else if (cluster == "no") {
+ new_table.set_cluster_by_geom(false);
+ } else {
+ throw fmt_error("Unknown value '{}' for 'cluster' table option"
+ " (use 'auto' or 'no').",
+ cluster);
+ }
+ } else if (cluster_type == LUA_TNIL) {
+ // ignore
+ } else {
+ throw std::runtime_error{
+ "Unknown value for 'cluster' table option: Must be string."};
+ }
+ lua_pop(lua_state, 1);
+
+ // optional "data_tablespace" field
+ lua_getfield(lua_state, -1, "data_tablespace");
+ if (lua_isstring(lua_state, -1)) {
+ std::string const tablespace = lua_tostring(lua_state, -1);
+ check_identifier(tablespace, "data_tablespace field");
+ check_tablespace(tablespace);
+ new_table.set_data_tablespace(tablespace);
+ }
+ lua_pop(lua_state, 1);
+
+ // optional "index_tablespace" field
+ lua_getfield(lua_state, -1, "index_tablespace");
+ if (lua_isstring(lua_state, -1)) {
+ std::string const tablespace = lua_tostring(lua_state, -1);
+ check_identifier(tablespace, "index_tablespace field");
+ check_tablespace(tablespace);
+ new_table.set_index_tablespace(tablespace);
+ }
+ lua_pop(lua_state, 1);
+
+ return new_table;
+}
+
+static void setup_flex_table_id_columns(lua_State *lua_state,
+ flex_table_t *table)
+{
+ assert(lua_state);
+ assert(table);
+
+ lua_getfield(lua_state, -1, "ids");
+ if (lua_type(lua_state, -1) != LUA_TTABLE) {
+ log_warn("Table '{}' doesn't have an id column. Two-stage"
+ " processing, updates and expire will not work!",
+ table->name());
+ lua_pop(lua_state, 1); // ids
+ return;
+ }
+
+ std::string const type{
+ luaX_get_table_string(lua_state, "type", -1, "The ids field")};
+ lua_pop(lua_state, 1); // "type"
+
+ if (type == "node") {
+ table->set_id_type(osmium::item_type::node);
+ } else if (type == "way") {
+ table->set_id_type(osmium::item_type::way);
+ } else if (type == "relation") {
+ table->set_id_type(osmium::item_type::relation);
+ } else if (type == "area") {
+ table->set_id_type(osmium::item_type::area);
+ } else if (type == "any") {
+ table->set_id_type(osmium::item_type::undefined);
+ lua_getfield(lua_state, -1, "type_column");
+ if (lua_isstring(lua_state, -1)) {
+ std::string const column_name =
+ lua_tolstring(lua_state, -1, nullptr);
+ check_identifier(column_name, "column names");
+ auto &column = table->add_column(column_name, "id_type", "");
+ column.set_not_null();
+ } else if (!lua_isnil(lua_state, -1)) {
+ throw std::runtime_error{"type_column must be a string or nil."};
+ }
+ lua_pop(lua_state, 1); // "type_column"
+ } else {
+ throw fmt_error("Unknown ids type: {}.", type);
+ }
+
+ std::string const name =
+ luaX_get_table_string(lua_state, "id_column", -1, "The ids field");
+ lua_pop(lua_state, 1); // "id_column"
+ check_identifier(name, "column names");
+
+ std::string const create_index = luaX_get_table_string(
+ lua_state, "create_index", -1, "The ids field", "auto");
+ lua_pop(lua_state, 1); // "create_index"
+ if (create_index == "always") {
+ table->set_always_build_id_index();
+ } else if (create_index != "auto") {
+ throw fmt_error("Unknown value '{}' for 'create_index' field of ids",
+ create_index);
+ }
+
+ auto &column = table->add_column(name, "id_num", "");
+ column.set_not_null();
+ lua_pop(lua_state, 1); // "ids"
+}
+
+static void setup_flex_table_columns(lua_State *lua_state, flex_table_t *table)
+{
+ assert(lua_state);
+ assert(table);
+
+ lua_getfield(lua_state, -1, "columns");
+ if (lua_type(lua_state, -1) != LUA_TTABLE) {
+ throw fmt_error("No 'columns' field (or not an array) in table '{}'.",
+ table->name());
+ }
+
+ if (!luaX_is_array(lua_state)) {
+ throw std::runtime_error{"The 'columns' field must contain an array."};
+ }
+ std::size_t num_columns = 0;
+ luaX_for_each(lua_state, [&]() {
+ if (!lua_istable(lua_state, -1)) {
+ throw std::runtime_error{
+ "The entries in the 'columns' array must be tables."};
+ }
+
+ char const *const type = luaX_get_table_string(lua_state, "type", -1,
+ "Column entry", "text");
+ char const *const name =
+ luaX_get_table_string(lua_state, "column", -2, "Column entry");
+ check_identifier(name, "column names");
+ char const *const sql_type = luaX_get_table_string(
+ lua_state, "sql_type", -3, "Column entry", "");
+
+ auto &column = table->add_column(name, type, sql_type);
+ lua_pop(lua_state, 3); // "type", "column", "sql_type"
+
+ column.set_not_null(luaX_get_table_bool(lua_state, "not_null", -1,
+ "Entry 'not_null'", false));
+ lua_pop(lua_state, 1); // "not_null"
+
+ column.set_create_only(luaX_get_table_bool(
+ lua_state, "create_only", -1, "Entry 'create_only'", false));
+ lua_pop(lua_state, 1); // "create_only"
+
+ lua_getfield(lua_state, -1, "projection");
+ if (!lua_isnil(lua_state, -1)) {
+ if (column.is_geometry_column() ||
+ column.type() == table_column_type::area) {
+ column.set_projection(lua_tostring(lua_state, -1));
+ } else {
+ throw std::runtime_error{"Projection can only be set on "
+ "geometry and area columns."};
+ }
+ }
+ lua_pop(lua_state, 1); // "projection"
+
+ ++num_columns;
+ });
+
+ if (num_columns == 0 && !table->has_id_column()) {
+ throw fmt_error("No columns defined for table '{}'.", table->name());
+ }
+
+ lua_pop(lua_state, 1); // "columns"
+}
+
+static void setup_flex_table_indexes(lua_State *lua_state, flex_table_t *table,
+ bool updatable)
+{
+ assert(lua_state);
+ assert(table);
+
+ lua_getfield(lua_state, -1, "indexes");
+ if (lua_type(lua_state, -1) == LUA_TNIL) {
+ if (table->has_geom_column()) {
+ auto &index = table->add_index("gist");
+ index.set_columns(table->geom_column().name());
+
+ if (!updatable) {
+ // If database can not be updated, use fillfactor 100.
+ index.set_fillfactor(100);
+ }
+ index.set_tablespace(table->index_tablespace());
+ }
+ lua_pop(lua_state, 1); // "indexes"
+ return;
+ }
+
+ if (lua_type(lua_state, -1) != LUA_TTABLE) {
+ throw fmt_error("The 'indexes' field in definition of"
+ " table '{}' is not an array.",
+ table->name());
+ }
+
+ if (!luaX_is_array(lua_state)) {
+ throw std::runtime_error{"The 'indexes' field must contain an array."};
+ }
+
+ luaX_for_each(lua_state, [&]() {
+ if (!lua_istable(lua_state, -1)) {
+ throw std::runtime_error{
+ "The entries in the 'indexes' array must be Lua tables."};
+ }
+
+ flex_lua_setup_index(lua_state, table);
+ });
+
+ lua_pop(lua_state, 1); // "indexes"
+}
+
+int setup_flex_table(lua_State *lua_state, std::vector<flex_table_t> *tables,
+ bool updatable)
+{
+ if (lua_type(lua_state, 1) != LUA_TTABLE) {
+ throw std::runtime_error{
+ "Argument #1 to 'define_table' must be a table."};
+ }
+
+ auto &new_table = create_flex_table(lua_state, tables);
+ setup_flex_table_id_columns(lua_state, &new_table);
+ setup_flex_table_columns(lua_state, &new_table);
+ setup_flex_table_indexes(lua_state, &new_table, updatable);
+
+ void *ptr = lua_newuserdata(lua_state, sizeof(std::size_t));
+ auto *num = new (ptr) std::size_t{};
+ *num = tables->size() - 1;
+ luaL_getmetatable(lua_state, osm2pgsql_table_name);
+ lua_setmetatable(lua_state, -2);
+
+ return 1;
+}
=====================================
src/flex-lua-table.hpp
=====================================
@@ -0,0 +1,23 @@
+#ifndef OSM2PGSQL_FLEX_LUA_TABLE_HPP
+#define OSM2PGSQL_FLEX_LUA_TABLE_HPP
+
+/**
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * This file is part of osm2pgsql (https://osm2pgsql.org/).
+ *
+ * Copyright (C) 2006-2023 by the osm2pgsql developer community.
+ * For a full list of authors see the git log.
+ */
+
+#include <vector>
+
+class flex_table_t;
+struct lua_State;
+
+static char const *const osm2pgsql_table_name = "osm2pgsql.Table";
+
+int setup_flex_table(lua_State *lua_state, std::vector<flex_table_t> *tables,
+ bool updatable);
+
+#endif // OSM2PGSQL_FLEX_LUA_TABLE_HPP
=====================================
src/flex-table.cpp
=====================================
@@ -421,4 +421,7 @@ void table_connection_t::task_wait()
auto const run_time = m_task_result.wait();
log_info("All postprocessing on table '{}' done in {}.", table().name(),
util::human_readable_duration(run_time));
+ log_debug("Inserted {} rows into table '{}' ({} not inserted due to"
+ " NOT NULL constraints).",
+ m_count_insert, table().name(), m_count_not_null_error);
}
=====================================
src/flex-table.hpp
=====================================
@@ -247,6 +247,8 @@ public:
pg_result_t get_geom_by_id(osmium::item_type type, osmid_t id) const;
+ void flush() { m_copy_mgr.flush(); }
+
void sync() { m_copy_mgr.sync(); }
void new_line() { m_copy_mgr.new_line(m_target); }
@@ -271,6 +273,13 @@ public:
void task_wait();
+ void increment_insert_counter() noexcept { ++m_count_insert; }
+
+ void increment_not_null_error_counter() noexcept
+ {
+ ++m_count_not_null_error;
+ }
+
private:
std::shared_ptr<reprojection> m_proj;
@@ -289,6 +298,9 @@ private:
task_result_t m_task_result;
+ std::size_t m_count_insert = 0;
+ std::size_t m_count_not_null_error = 0;
+
/// Has the Id index already been created?
bool m_id_index_created = false;
=====================================
src/flex-write.cpp
=====================================
@@ -253,7 +253,8 @@ static bool is_compatible(geom::geometry_t const &geom,
void flex_write_column(lua_State *lua_state,
db_copy_mgr_t<db_deleter_by_type_and_id_t> *copy_mgr,
- flex_table_column_t const &column, expire_tiles *expire)
+ flex_table_column_t const &column, expire_tiles *expire,
+ expire_config_t const &expire_config)
{
// If there is nothing on the Lua stack, then the Lua function add_row()
// was called without a table parameter. In that case this column will
@@ -429,12 +430,12 @@ void flex_write_column(lua_State *lua_state,
type == table_column_type::multilinestring ||
type == table_column_type::multipolygon);
if (geom->srid() == column.srid()) {
- expire->from_geometry_if_3857(*geom);
+ expire->from_geometry_if_3857(*geom, expire_config);
copy_mgr->add_hex_geom(geom_to_ewkb(*geom, wrap_multi));
} else {
auto const &proj = get_projection(column.srid());
auto const tgeom = geom::transform(*geom, proj);
- expire->from_geometry_if_3857(tgeom);
+ expire->from_geometry_if_3857(tgeom, expire_config);
copy_mgr->add_hex_geom(geom_to_ewkb(tgeom, wrap_multi));
}
} else {
@@ -461,7 +462,7 @@ void flex_write_column(lua_State *lua_state,
void flex_write_row(lua_State *lua_state, table_connection_t *table_connection,
osmium::item_type id_type, osmid_t id,
geom::geometry_t const &geom, int srid,
- expire_tiles *expire)
+ expire_tiles *expire, expire_config_t const &expire_config)
{
assert(table_connection);
table_connection->new_line();
@@ -506,7 +507,8 @@ void flex_write_row(lua_State *lua_state, table_connection_t *table_connection,
copy_mgr->add_column(area);
}
} else {
- flex_write_column(lua_state, copy_mgr, column, expire);
+ flex_write_column(lua_state, copy_mgr, column, expire,
+ expire_config);
}
}
=====================================
src/flex-write.hpp
=====================================
@@ -32,11 +32,12 @@ private:
void flex_write_column(lua_State *lua_state,
db_copy_mgr_t<db_deleter_by_type_and_id_t> *copy_mgr,
- flex_table_column_t const &column, expire_tiles *expire);
+ flex_table_column_t const &column, expire_tiles *expire,
+ expire_config_t const &expire_config);
void flex_write_row(lua_State *lua_state, table_connection_t *table_connection,
osmium::item_type id_type, osmid_t id,
geom::geometry_t const &geom, int srid,
- expire_tiles *expire);
+ expire_tiles *expire, expire_config_t const &expire_config);
#endif // OSM2PGSQL_FLEX_WRITE_HPP
=====================================
src/geom-pole-of-inaccessibility.cpp
=====================================
@@ -210,7 +210,6 @@ point_t pole_of_inaccessibility(const polygon_t &polygon, double precision,
best_cell = bbox_cell;
}
- auto num_probes = cell_queue.size();
while (!cell_queue.empty()) {
// pick the most promising cell from the queue
auto cell = cell_queue.top();
@@ -219,8 +218,6 @@ point_t pole_of_inaccessibility(const polygon_t &polygon, double precision,
// update the best cell if we found a better one
if (cell.dist > best_cell.dist) {
best_cell = cell;
- log_debug("polyline: found best {} after {} probes",
- ::round(1e4 * cell.dist) / 1e4, num_probes);
}
// do not drill down further if there's no chance of a better solution
@@ -241,13 +238,8 @@ point_t pole_of_inaccessibility(const polygon_t &polygon, double precision,
}
}
}
-
- num_probes += 4;
}
- log_debug("polyline: num probes: {}", num_probes);
- log_debug("polyline: best distance: {}", best_cell.dist);
-
return {best_cell.center.x(), best_cell.center.y() / stretch};
}
=====================================
src/osmdata.cpp
=====================================
@@ -57,34 +57,56 @@ void osmdata_t::node(osmium::Node const &node)
m_mid->node(node);
if (node.deleted()) {
- node_delete(node.id());
- } else {
- if (m_append) {
- node_modify(node);
+ m_output->node_delete(node.id());
+ return;
+ }
+
+ bool const has_tags_or_attrs = m_with_extra_attrs || !node.tags().empty();
+ if (m_append) {
+ if (has_tags_or_attrs) {
+ m_output->node_modify(node);
} else {
- node_add(node);
+ m_output->node_delete(node.id());
}
+ m_dependency_manager->node_changed(node.id());
+ } else if (has_tags_or_attrs) {
+ m_output->node_add(node);
}
}
-void osmdata_t::after_nodes() { m_mid->after_nodes(); }
+void osmdata_t::after_nodes()
+{
+ m_mid->after_nodes();
+ m_output->after_nodes();
+}
void osmdata_t::way(osmium::Way &way)
{
m_mid->way(way);
if (way.deleted()) {
- way_delete(way.id());
- } else {
- if (m_append) {
- way_modify(&way);
+ m_output->way_delete(way.id());
+ return;
+ }
+
+ bool const has_tags_or_attrs = m_with_extra_attrs || !way.tags().empty();
+ if (m_append) {
+ if (has_tags_or_attrs) {
+ m_output->way_modify(&way);
} else {
- way_add(&way);
+ m_output->way_delete(way.id());
}
+ m_dependency_manager->way_changed(way.id());
+ } else if (has_tags_or_attrs) {
+ m_output->way_add(&way);
}
}
-void osmdata_t::after_ways() { m_mid->after_ways(); }
+void osmdata_t::after_ways()
+{
+ m_mid->after_ways();
+ m_output->after_ways();
+}
void osmdata_t::relation(osmium::Relation const &rel)
{
@@ -102,70 +124,23 @@ void osmdata_t::relation(osmium::Relation const &rel)
m_mid->relation(rel);
if (rel.deleted()) {
- relation_delete(rel.id());
- } else {
- if (m_append) {
- relation_modify(rel);
- } else {
- relation_add(rel);
- }
- }
-}
-
-void osmdata_t::after_relations() { m_mid->after_relations(); }
-
-void osmdata_t::node_add(osmium::Node const &node) const
-{
- if (m_with_extra_attrs || !node.tags().empty()) {
- m_output->node_add(node);
- }
-}
-
-void osmdata_t::way_add(osmium::Way *way) const
-{
- if (m_with_extra_attrs || !way->tags().empty()) {
- m_output->way_add(way);
+ m_output->relation_delete(rel.id());
+ return;
}
-}
-void osmdata_t::relation_add(osmium::Relation const &rel) const
-{
- if (m_with_extra_attrs || !rel.tags().empty()) {
+ bool const has_tags_or_attrs = m_with_extra_attrs || !rel.tags().empty();
+ if (m_append) {
+ if (has_tags_or_attrs) {
+ m_output->relation_modify(rel);
+ } else {
+ m_output->relation_delete(rel.id());
+ }
+ } else if (has_tags_or_attrs) {
m_output->relation_add(rel);
}
}
-void osmdata_t::node_modify(osmium::Node const &node) const
-{
- m_output->node_modify(node);
- m_dependency_manager->node_changed(node.id());
-}
-
-void osmdata_t::way_modify(osmium::Way *way) const
-{
- m_output->way_modify(way);
- m_dependency_manager->way_changed(way->id());
-}
-
-void osmdata_t::relation_modify(osmium::Relation const &rel) const
-{
- m_output->relation_modify(rel);
-}
-
-void osmdata_t::node_delete(osmid_t id) const
-{
- m_output->node_delete(id);
-}
-
-void osmdata_t::way_delete(osmid_t id) const
-{
- m_output->way_delete(id);
-}
-
-void osmdata_t::relation_delete(osmid_t id) const
-{
- m_output->relation_delete(id);
-}
+void osmdata_t::after_relations() { m_mid->after_relations(); }
void osmdata_t::start() const
{
=====================================
src/osmdata.hpp
=====================================
@@ -61,18 +61,6 @@ public:
void stop() const;
private:
- void node_add(osmium::Node const &node) const;
- void way_add(osmium::Way *way) const;
- void relation_add(osmium::Relation const &rel) const;
-
- void node_modify(osmium::Node const &node) const;
- void way_modify(osmium::Way *way) const;
- void relation_modify(osmium::Relation const &rel) const;
-
- void node_delete(osmid_t id) const;
- void way_delete(osmid_t id) const;
- void relation_delete(osmid_t id) const;
-
/**
* Run stage 1b and stage 1c processing: Process dependent objects in
* append mode.
=====================================
src/output-flex.cpp
=====================================
@@ -12,6 +12,7 @@
#include "flex-index.hpp"
#include "flex-lua-geom.hpp"
#include "flex-lua-index.hpp"
+#include "flex-lua-table.hpp"
#include "flex-write.hpp"
#include "format.hpp"
#include "geom-from-osm.hpp"
@@ -92,7 +93,6 @@ TRAMPOLINE(table_insert, insert)
TRAMPOLINE(table_columns, columns)
TRAMPOLINE(table_tostring, __tostring)
-static char const *const osm2pgsql_table_name = "osm2pgsql.Table";
static char const *const osm2pgsql_object_metatable =
"osm2pgsql.object_metatable";
@@ -435,253 +435,6 @@ int output_flex_t::app_as_geometrycollection()
return 1;
}
-static void check_tablespace(std::string const &tablespace)
-{
- if (!has_tablespace(tablespace)) {
- throw fmt_error(
- "Tablespace '{0}' not available."
- " Use 'CREATE TABLESPACE \"{0}\" ...;' to create it.",
- tablespace);
- }
-}
-
-flex_table_t &output_flex_t::create_flex_table()
-{
- std::string const table_name =
- luaX_get_table_string(lua_state(), "name", -1, "The table");
-
- check_identifier(table_name, "table names");
-
- if (util::find_by_name(*m_tables, table_name)) {
- throw fmt_error("Table with name '{}' already exists.", table_name);
- }
-
- auto &new_table = m_tables->emplace_back(table_name);
-
- lua_pop(lua_state(), 1); // "name"
-
- // optional "schema" field
- lua_getfield(lua_state(), -1, "schema");
- if (lua_isstring(lua_state(), -1)) {
- std::string const schema = lua_tostring(lua_state(), -1);
- check_identifier(schema, "schema field");
- if (!has_schema(schema)) {
- throw fmt_error("Schema '{0}' not available."
- " Use 'CREATE SCHEMA \"{0}\";' to create it.",
- schema);
- }
- new_table.set_schema(schema);
- }
- lua_pop(lua_state(), 1);
-
- // optional "cluster" field
- lua_getfield(lua_state(), -1, "cluster");
- int const cluster_type = lua_type(lua_state(), -1);
- if (cluster_type == LUA_TSTRING) {
- std::string const cluster = lua_tostring(lua_state(), -1);
- if (cluster == "auto") {
- new_table.set_cluster_by_geom(true);
- } else if (cluster == "no") {
- new_table.set_cluster_by_geom(false);
- } else {
- throw fmt_error("Unknown value '{}' for 'cluster' table option"
- " (use 'auto' or 'no').",
- cluster);
- }
- } else if (cluster_type == LUA_TNIL) {
- // ignore
- } else {
- throw std::runtime_error{
- "Unknown value for 'cluster' table option: Must be string."};
- }
- lua_pop(lua_state(), 1);
-
- // optional "data_tablespace" field
- lua_getfield(lua_state(), -1, "data_tablespace");
- if (lua_isstring(lua_state(), -1)) {
- std::string const tablespace = lua_tostring(lua_state(), -1);
- check_identifier(tablespace, "data_tablespace field");
- check_tablespace(tablespace);
- new_table.set_data_tablespace(tablespace);
- }
- lua_pop(lua_state(), 1);
-
- // optional "index_tablespace" field
- lua_getfield(lua_state(), -1, "index_tablespace");
- if (lua_isstring(lua_state(), -1)) {
- std::string const tablespace = lua_tostring(lua_state(), -1);
- check_identifier(tablespace, "index_tablespace field");
- check_tablespace(tablespace);
- new_table.set_index_tablespace(tablespace);
- }
- lua_pop(lua_state(), 1);
-
- return new_table;
-}
-
-void output_flex_t::setup_id_columns(flex_table_t *table)
-{
- assert(table);
- lua_getfield(lua_state(), -1, "ids");
- if (lua_type(lua_state(), -1) != LUA_TTABLE) {
- log_warn("Table '{}' doesn't have an id column. Two-stage"
- " processing, updates and expire will not work!",
- table->name());
- lua_pop(lua_state(), 1); // ids
- return;
- }
-
- std::string const type{
- luaX_get_table_string(lua_state(), "type", -1, "The ids field")};
- lua_pop(lua_state(), 1); // "type"
-
- if (type == "node") {
- table->set_id_type(osmium::item_type::node);
- } else if (type == "way") {
- table->set_id_type(osmium::item_type::way);
- } else if (type == "relation") {
- table->set_id_type(osmium::item_type::relation);
- } else if (type == "area") {
- table->set_id_type(osmium::item_type::area);
- } else if (type == "any") {
- table->set_id_type(osmium::item_type::undefined);
- lua_getfield(lua_state(), -1, "type_column");
- if (lua_isstring(lua_state(), -1)) {
- std::string const column_name =
- lua_tolstring(lua_state(), -1, nullptr);
- check_identifier(column_name, "column names");
- auto &column = table->add_column(column_name, "id_type", "");
- column.set_not_null();
- } else if (!lua_isnil(lua_state(), -1)) {
- throw std::runtime_error{"type_column must be a string or nil."};
- }
- lua_pop(lua_state(), 1); // "type_column"
- } else {
- throw fmt_error("Unknown ids type: {}.", type);
- }
-
- std::string const name =
- luaX_get_table_string(lua_state(), "id_column", -1, "The ids field");
- lua_pop(lua_state(), 1); // "id_column"
- check_identifier(name, "column names");
-
- std::string const create_index = luaX_get_table_string(
- lua_state(), "create_index", -1, "The ids field", "auto");
- lua_pop(lua_state(), 1); // "create_index"
- if (create_index == "always") {
- table->set_always_build_id_index();
- } else if (create_index != "auto") {
- throw fmt_error("Unknown value '{}' for 'create_index' field of ids",
- create_index);
- }
-
- auto &column = table->add_column(name, "id_num", "");
- column.set_not_null();
- lua_pop(lua_state(), 1); // "ids"
-}
-
-void output_flex_t::setup_flex_table_columns(flex_table_t *table)
-{
- assert(table);
- lua_getfield(lua_state(), -1, "columns");
- if (lua_type(lua_state(), -1) != LUA_TTABLE) {
- throw fmt_error("No 'columns' field (or not an array) in table '{}'.",
- table->name());
- }
-
- if (!luaX_is_array(lua_state())) {
- throw std::runtime_error{"The 'columns' field must contain an array."};
- }
- std::size_t num_columns = 0;
- luaX_for_each(lua_state(), [&]() {
- if (!lua_istable(lua_state(), -1)) {
- throw std::runtime_error{
- "The entries in the 'columns' array must be tables."};
- }
-
- char const *const type = luaX_get_table_string(lua_state(), "type", -1,
- "Column entry", "text");
- char const *const name =
- luaX_get_table_string(lua_state(), "column", -2, "Column entry");
- check_identifier(name, "column names");
- char const *const sql_type = luaX_get_table_string(
- lua_state(), "sql_type", -3, "Column entry", "");
-
- auto &column = table->add_column(name, type, sql_type);
- lua_pop(lua_state(), 3); // "type", "column", "sql_type"
-
- column.set_not_null(luaX_get_table_bool(lua_state(), "not_null", -1,
- "Entry 'not_null'", false));
- lua_pop(lua_state(), 1); // "not_null"
-
- column.set_create_only(luaX_get_table_bool(
- lua_state(), "create_only", -1, "Entry 'create_only'", false));
- lua_pop(lua_state(), 1); // "create_only"
-
- lua_getfield(lua_state(), -1, "projection");
- if (!lua_isnil(lua_state(), -1)) {
- if (column.is_geometry_column() ||
- column.type() == table_column_type::area) {
- column.set_projection(lua_tostring(lua_state(), -1));
- } else {
- throw std::runtime_error{"Projection can only be set on "
- "geometry and area columns."};
- }
- }
- lua_pop(lua_state(), 1); // "projection"
-
- ++num_columns;
- });
-
- if (num_columns == 0 && !table->has_id_column()) {
- throw fmt_error("No columns defined for table '{}'.", table->name());
- }
-
- lua_pop(lua_state(), 1); // "columns"
-}
-
-void output_flex_t::setup_indexes(flex_table_t *table)
-{
- assert(table);
-
- lua_getfield(lua_state(), -1, "indexes");
- if (lua_type(lua_state(), -1) == LUA_TNIL) {
- if (table->has_geom_column()) {
- auto &index = table->add_index("gist");
- index.set_columns(table->geom_column().name());
-
- if (!get_options()->slim || get_options()->droptemp) {
- // If database can not be updated, use fillfactor 100.
- index.set_fillfactor(100);
- }
- index.set_tablespace(table->index_tablespace());
- }
- lua_pop(lua_state(), 1); // "indexes"
- return;
- }
-
- if (lua_type(lua_state(), -1) != LUA_TTABLE) {
- throw fmt_error("The 'indexes' field in definition of"
- " table '{}' is not an array.",
- table->name());
- }
-
- if (!luaX_is_array(lua_state())) {
- throw std::runtime_error{"The 'indexes' field must contain an array."};
- }
-
- luaX_for_each(lua_state(), [&]() {
- if (!lua_istable(lua_state(), -1)) {
- throw std::runtime_error{
- "The entries in the 'indexes' array must be Lua tables."};
- }
-
- flex_lua_setup_index(lua_state(), table);
- });
-
- lua_pop(lua_state(), 1); // "indexes"
-}
-
int output_flex_t::app_define_table()
{
if (m_calling_context != calling_context::main) {
@@ -690,23 +443,8 @@ int output_flex_t::app_define_table()
" main Lua code, not in any of the callbacks."};
}
- if (lua_type(lua_state(), 1) != LUA_TTABLE) {
- throw std::runtime_error{
- "Argument #1 to 'define_table' must be a table."};
- }
-
- auto &new_table = create_flex_table();
- setup_id_columns(&new_table);
- setup_flex_table_columns(&new_table);
- setup_indexes(&new_table);
-
- void *ptr = lua_newuserdata(lua_state(), sizeof(std::size_t));
- auto *num = new (ptr) std::size_t{};
- *num = m_tables->size() - 1;
- luaL_getmetatable(lua_state(), osm2pgsql_table_name);
- lua_setmetatable(lua_state(), -2);
-
- return 1;
+ return setup_flex_table(lua_state(), m_tables.get(),
+ get_options()->slim && !get_options()->droptemp);
}
// Check that the first element on the Lua stack is an osm2pgsql.Table
@@ -952,9 +690,11 @@ int output_flex_t::table_insert()
} else if (column.type() == table_column_type::id_num) {
copy_mgr->add_column(id);
} else {
- flex_write_column(lua_state(), copy_mgr, column, &m_expire);
+ flex_write_column(lua_state(), copy_mgr, column, &m_expire,
+ m_expire_config);
}
}
+ table_connection.increment_insert_counter();
} catch (not_null_exception const &e) {
copy_mgr->rollback_line();
lua_pushboolean(lua_state(), false);
@@ -962,6 +702,7 @@ int output_flex_t::table_insert()
lua_pushstring(lua_state(), e.column().name().c_str());
push_osm_object_to_lua_stack(lua_state(), object,
get_options()->extra_attributes);
+ table_connection.increment_not_null_error_counter();
return 4;
}
@@ -1065,7 +806,7 @@ void output_flex_t::add_row(table_connection_t *table_connection,
if (!table.has_geom_column()) {
flex_write_row(lua_state(), table_connection, object.type(), id, {}, 0,
- &m_expire);
+ &m_expire, m_expire_config);
return;
}
@@ -1103,9 +844,9 @@ void output_flex_t::add_row(table_connection_t *table_connection,
auto const geoms = geom::split_multi(std::move(geom), split_multi);
for (auto const &sgeom : geoms) {
- m_expire.from_geometry_if_3857(sgeom);
+ m_expire.from_geometry_if_3857(sgeom, m_expire_config);
flex_write_row(lua_state(), table_connection, object.type(), id, sgeom,
- table.geom_column().srid(), &m_expire);
+ table.geom_column().srid(), &m_expire, m_expire_config);
}
}
@@ -1263,6 +1004,20 @@ void output_flex_t::sync()
}
}
+void output_flex_t::after_nodes()
+{
+ for (auto &table : m_table_connections) {
+ table.flush();
+ }
+}
+
+void output_flex_t::after_ways()
+{
+ for (auto &table : m_table_connections) {
+ table.flush();
+ }
+}
+
void output_flex_t::stop()
{
for (auto &table : m_table_connections) {
@@ -1274,9 +1029,9 @@ void output_flex_t::stop()
if (get_options()->expire_tiles_zoom_min > 0) {
auto const count = output_tiles_to_file(
- m_expire.get_tiles(), get_options()->expire_tiles_filename.c_str(),
- get_options()->expire_tiles_zoom_min,
- get_options()->expire_tiles_zoom);
+ m_expire.get_tiles(), get_options()->expire_tiles_zoom_min,
+ get_options()->expire_tiles_zoom,
+ get_options()->expire_tiles_filename);
log_info("Wrote {} entries to expired tiles list", count);
}
}
@@ -1332,7 +1087,7 @@ void output_flex_t::delete_from_table(table_connection_t *table_connection,
if (m_expire.enabled() && table.has_geom_column() &&
table.geom_column().srid() == 3857) {
auto const result = table_connection->get_geom_by_id(type, id);
- expire_from_result(&m_expire, result);
+ expire_from_result(&m_expire, result, m_expire_config);
}
table_connection->delete_rows_with(type, id);
@@ -1398,8 +1153,8 @@ output_flex_t::output_flex_t(output_flex_t const *other,
: output_t(other, std::move(mid)), m_tables(other->m_tables),
m_stage2_way_ids(other->m_stage2_way_ids),
m_copy_thread(std::move(copy_thread)), m_lua_state(other->m_lua_state),
+ m_expire_config(other->m_expire_config),
m_expire(other->get_options()->expire_tiles_zoom,
- other->get_options()->expire_tiles_max_bbox,
other->get_options()->projection),
m_process_node(other->m_process_node), m_process_way(other->m_process_way),
m_process_relation(other->m_process_relation),
@@ -1424,9 +1179,13 @@ output_flex_t::output_flex_t(std::shared_ptr<middle_query_t> const &mid,
options_t const &options)
: output_t(mid, std::move(thread_pool), options),
m_copy_thread(std::make_shared<db_copy_thread_t>(options.conninfo)),
- m_expire(options.expire_tiles_zoom, options.expire_tiles_max_bbox,
- options.projection)
+ m_expire(options.expire_tiles_zoom, options.projection)
{
+ m_expire_config.full_area_limit = get_options()->expire_tiles_max_bbox;
+ if (get_options()->expire_tiles_max_bbox > 0.0) {
+ m_expire_config.mode = expire_mode::hybrid;
+ }
+
init_lua(options.style);
// If the osm2pgsql.select_relation_members() Lua function is defined
=====================================
src/output-flex.hpp
=====================================
@@ -10,6 +10,7 @@
* For a full list of authors see the git log.
*/
+#include "expire-config.hpp"
#include "expire-tiles.hpp"
#include "flex-table-column.hpp"
#include "flex-table.hpp"
@@ -124,6 +125,9 @@ public:
void stop() override;
void sync() override;
+ void after_nodes() override;
+ void after_ways() override;
+
void wait() override;
idset_t const &get_marked_way_ids() override;
@@ -185,11 +189,6 @@ private:
void init_lua(std::string const &filename);
- flex_table_t &create_flex_table();
- void setup_id_columns(flex_table_t *table);
- void setup_flex_table_columns(flex_table_t *table);
- void setup_indexes(flex_table_t *table);
-
// Get the flex table that is as first parameter on the Lua stack.
flex_table_t const &get_table_from_param();
@@ -286,6 +285,7 @@ private:
// accessed while protected using the lua_mutex.
std::shared_ptr<lua_State> m_lua_state;
+ expire_config_t m_expire_config;
expire_tiles m_expire;
way_cache_t m_way_cache;
=====================================
src/output-pgsql.cpp
=====================================
@@ -67,7 +67,7 @@ void output_pgsql_t::pgsql_out_way(osmium::Way const &way, taglist_t *tags,
auto const wkb = geom_to_ewkb(projected_geom);
if (!wkb.empty()) {
- m_expire.from_geometry_if_3857(projected_geom);
+ m_expire.from_geometry_if_3857(projected_geom, m_expire_config);
if (m_enable_way_area) {
double const area = calculate_area(
get_options()->reproject_area, geom, projected_geom);
@@ -82,7 +82,7 @@ void output_pgsql_t::pgsql_out_way(osmium::Way const &way, taglist_t *tags,
auto const geoms = geom::split_multi(geom::segmentize(
geom::transform(geom::create_linestring(way), *m_proj), split_at));
for (auto const &sgeom : geoms) {
- m_expire.from_geometry_if_3857(sgeom);
+ m_expire.from_geometry_if_3857(sgeom, m_expire_config);
auto const wkb = geom_to_ewkb(sgeom);
m_tables[t_line]->write_row(way.id(), *tags, wkb);
if (roads) {
@@ -147,9 +147,9 @@ void output_pgsql_t::stop()
if (get_options()->expire_tiles_zoom_min > 0) {
auto const count = output_tiles_to_file(
- m_expire.get_tiles(), get_options()->expire_tiles_filename.c_str(),
- get_options()->expire_tiles_zoom_min,
- get_options()->expire_tiles_zoom);
+ m_expire.get_tiles(), get_options()->expire_tiles_zoom_min,
+ get_options()->expire_tiles_zoom,
+ get_options()->expire_tiles_filename);
log_info("Wrote {} entries to expired tiles list", count);
}
}
@@ -169,7 +169,7 @@ void output_pgsql_t::node_add(osmium::Node const &node)
}
auto const geom = geom::transform(geom::create_point(node), *m_proj);
- m_expire.from_geometry_if_3857(geom);
+ m_expire.from_geometry_if_3857(geom, m_expire_config);
auto const wkb = geom_to_ewkb(geom);
m_tables[t_point]->write_row(node.id(), outtags, wkb);
}
@@ -272,7 +272,7 @@ void output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel)
}
auto const geoms = geom::split_multi(std::move(projected_geom));
for (auto const &sgeom : geoms) {
- m_expire.from_geometry_if_3857(sgeom);
+ m_expire.from_geometry_if_3857(sgeom, m_expire_config);
auto const wkb = geom_to_ewkb(sgeom);
m_tables[t_line]->write_row(-rel.id(), outtags, wkb);
if (roads) {
@@ -288,7 +288,7 @@ void output_pgsql_t::pgsql_process_relation(osmium::Relation const &rel)
!get_options()->enable_multi);
for (auto const &sgeom : geoms) {
auto const projected_geom = geom::transform(sgeom, *m_proj);
- m_expire.from_geometry_if_3857(projected_geom);
+ m_expire.from_geometry_if_3857(projected_geom, m_expire_config);
auto const wkb = geom_to_ewkb(projected_geom);
if (m_enable_way_area) {
double const area = calculate_area(
@@ -327,7 +327,7 @@ void output_pgsql_t::node_delete(osmid_t osm_id)
{
if (m_expire.enabled()) {
auto const results = m_tables[t_point]->get_wkb(osm_id);
- if (expire_from_result(&m_expire, results) != 0) {
+ if (expire_from_result(&m_expire, results, m_expire_config) != 0) {
m_tables[t_point]->delete_row(osm_id);
}
} else {
@@ -342,7 +342,7 @@ void output_pgsql_t::delete_from_output_and_expire(osmid_t id)
for (auto table : {t_line, t_poly}) {
if (m_expire.enabled()) {
auto const results = m_tables[table]->get_wkb(id);
- if (expire_from_result(&m_expire, results) != 0) {
+ if (expire_from_result(&m_expire, results, m_expire_config) != 0) {
m_tables[table]->delete_row(id);
}
} else {
@@ -422,11 +422,15 @@ output_pgsql_t::output_pgsql_t(std::shared_ptr<middle_query_t> const &mid,
std::shared_ptr<thread_pool_t> thread_pool,
options_t const &options)
: output_t(mid, std::move(thread_pool), options), m_proj(options.projection),
- m_expire(options.expire_tiles_zoom, options.expire_tiles_max_bbox,
- options.projection),
+ m_expire(options.expire_tiles_zoom, options.projection),
m_buffer(32768, osmium::memory::Buffer::auto_grow::yes),
m_rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes)
{
+ m_expire_config.full_area_limit = get_options()->expire_tiles_max_bbox;
+ if (get_options()->expire_tiles_max_bbox > 0.0) {
+ m_expire_config.mode = expire_mode::hybrid;
+ }
+
log_debug("Using projection SRS {} ({})", options.projection->target_srs(),
options.projection->target_desc());
@@ -482,9 +486,8 @@ output_pgsql_t::output_pgsql_t(
std::shared_ptr<db_copy_thread_t> const ©_thread)
: output_t(other, mid), m_tagtransform(other->m_tagtransform->clone()),
m_enable_way_area(other->m_enable_way_area),
- m_proj(get_options()->projection),
- m_expire(get_options()->expire_tiles_zoom,
- get_options()->expire_tiles_max_bbox, get_options()->projection),
+ m_proj(get_options()->projection), m_expire_config(other->m_expire_config),
+ m_expire(get_options()->expire_tiles_zoom, get_options()->projection),
m_buffer(1024, osmium::memory::Buffer::auto_grow::yes),
m_rels_buffer(1024, osmium::memory::Buffer::auto_grow::yes)
{
=====================================
src/output-pgsql.hpp
=====================================
@@ -104,6 +104,7 @@ private:
std::array<std::unique_ptr<table_t>, t_MAX> m_tables;
std::shared_ptr<reprojection> m_proj;
+ expire_config_t m_expire_config;
expire_tiles m_expire;
osmium::memory::Buffer m_buffer;
=====================================
src/output.hpp
=====================================
@@ -62,6 +62,9 @@ public:
virtual void stop() = 0;
virtual void sync() = 0;
+ virtual void after_nodes() {}
+ virtual void after_ways() {}
+
virtual void wait() {}
virtual osmium::index::IdSetSmall<osmid_t> const &get_marked_way_ids()
=====================================
tests/bdd/steps/steps_osm_data.py
=====================================
@@ -31,6 +31,7 @@ def osm_define_node_grid(context, step, origin_x, origin_y):
@given("the (?P<formatted>python-formatted )?OSM data")
def osm_define_data(context, formatted):
+ context.import_file = None
data = context.text
if formatted:
data = eval('f"""' + data + '"""')
=====================================
tests/test-expire-from-geometry.cpp
=====================================
@@ -26,19 +26,20 @@ static constexpr uint32_t const zoom = 12;
TEST_CASE("expire null geometry does nothing", "[NoDB]")
{
- expire_tiles et{zoom, 20000, defproj};
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
SECTION("geom")
{
geom::geometry_t const geom{};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
{
geom::geometry_t geom{};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
REQUIRE(et.get_tiles().empty());
@@ -46,23 +47,24 @@ TEST_CASE("expire null geometry does nothing", "[NoDB]")
TEST_CASE("expire point at tile boundary", "[NoDB]")
{
- expire_tiles et{zoom, 20000, defproj};
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
geom::point_t const pt{0.0, 0.0};
- SECTION("point") { et.from_geometry(pt); }
+ SECTION("point") { et.from_geometry(pt, expire_config); }
SECTION("geom")
{
geom::geometry_t const geom{pt};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
{
geom::geometry_t geom{pt};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -75,23 +77,24 @@ TEST_CASE("expire point at tile boundary", "[NoDB]")
TEST_CASE("expire point away from tile boundary", "[NoDB]")
{
- expire_tiles et{zoom, 20000, defproj};
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
geom::point_t const pt{5000.0, 5000.0};
- SECTION("point") { et.from_geometry(pt); }
+ SECTION("point") { et.from_geometry(pt, expire_config); }
SECTION("geom")
{
geom::geometry_t const geom{pt};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
{
geom::geometry_t geom{pt};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -101,19 +104,20 @@ TEST_CASE("expire point away from tile boundary", "[NoDB]")
TEST_CASE("expire linestring away from tile boundary", "[NoDB]")
{
- expire_tiles et{zoom, 20000, defproj};
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
SECTION("line")
{
geom::linestring_t const line{{5000.0, 4000.0}, {5100.0, 4200.0}};
- et.from_geometry(line);
+ et.from_geometry(line, expire_config);
}
SECTION("geom")
{
geom::linestring_t line{{5000.0, 4000.0}, {5100.0, 4200.0}};
geom::geometry_t const geom{std::move(line)};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
@@ -121,7 +125,7 @@ TEST_CASE("expire linestring away from tile boundary", "[NoDB]")
geom::linestring_t line{{5000.0, 4000.0}, {5100.0, 4200.0}};
geom::geometry_t geom{std::move(line)};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -131,19 +135,20 @@ TEST_CASE("expire linestring away from tile boundary", "[NoDB]")
TEST_CASE("expire linestring crossing tile boundary", "[NoDB]")
{
- expire_tiles et{zoom, 20000, defproj};
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
SECTION("line")
{
geom::linestring_t const line{{5000.0, 5000.0}, {5000.0, 15000.0}};
- et.from_geometry(line);
+ et.from_geometry(line, expire_config);
}
SECTION("geom")
{
geom::linestring_t line{{5000.0, 5000.0}, {5000.0, 15000.0}};
geom::geometry_t const geom{std::move(line)};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
@@ -151,7 +156,7 @@ TEST_CASE("expire linestring crossing tile boundary", "[NoDB]")
geom::linestring_t line{{5000.0, 5000.0}, {5000.0, 15000.0}};
geom::geometry_t geom{std::move(line)};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -162,7 +167,8 @@ TEST_CASE("expire linestring crossing tile boundary", "[NoDB]")
TEST_CASE("expire small polygon", "[NoDB]")
{
- expire_tiles et{zoom, 20000, defproj};
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
SECTION("polygon")
{
@@ -171,7 +177,7 @@ TEST_CASE("expire small polygon", "[NoDB]")
{5100.0, 5100.0},
{5000.0, 5100.0},
{5000.0, 5000.0}}};
- et.from_geometry(poly);
+ et.from_geometry(poly, expire_config);
}
SECTION("geom")
@@ -182,7 +188,7 @@ TEST_CASE("expire small polygon", "[NoDB]")
{5000.0, 5100.0},
{5000.0, 5000.0}}};
geom::geometry_t const geom{std::move(poly)};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
@@ -194,7 +200,7 @@ TEST_CASE("expire small polygon", "[NoDB]")
{5000.0, 5000.0}}};
geom::geometry_t geom{std::move(poly)};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -204,7 +210,10 @@ TEST_CASE("expire small polygon", "[NoDB]")
TEST_CASE("expire large polygon as bbox", "[NoDB]")
{
- expire_tiles et{zoom, 40000, defproj};
+ expire_config_t expire_config;
+ expire_config.mode = expire_mode::hybrid;
+ expire_config.full_area_limit = 40000;
+ expire_tiles et{zoom, defproj};
SECTION("polygon")
{
@@ -213,7 +222,7 @@ TEST_CASE("expire large polygon as bbox", "[NoDB]")
{25000.0, 25000.0},
{5000.0, 25000.0},
{5000.0, 5000.0}}};
- et.from_geometry(poly);
+ et.from_geometry(poly, expire_config);
}
SECTION("geom")
@@ -224,7 +233,7 @@ TEST_CASE("expire large polygon as bbox", "[NoDB]")
{5000.0, 25000.0},
{5000.0, 5000.0}}};
geom::geometry_t const geom{std::move(poly)};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
@@ -236,7 +245,7 @@ TEST_CASE("expire large polygon as bbox", "[NoDB]")
{5000.0, 5000.0}}};
geom::geometry_t geom{std::move(poly)};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -256,7 +265,10 @@ TEST_CASE("expire large polygon as bbox", "[NoDB]")
TEST_CASE("expire large polygon as boundary", "[NoDB]")
{
- expire_tiles et{zoom, 10000, defproj};
+ expire_config_t expire_config;
+ expire_config.mode = expire_mode::hybrid;
+ expire_config.full_area_limit = 10000;
+ expire_tiles et{zoom, defproj};
SECTION("polygon")
{
@@ -265,7 +277,7 @@ TEST_CASE("expire large polygon as boundary", "[NoDB]")
{25000.0, 25000.0},
{5000.0, 25000.0},
{5000.0, 5000.0}}};
- et.from_geometry(poly);
+ et.from_geometry(poly, expire_config);
}
SECTION("polygon boundary")
@@ -275,7 +287,7 @@ TEST_CASE("expire large polygon as boundary", "[NoDB]")
{25000.0, 25000.0},
{5000.0, 25000.0},
{5000.0, 5000.0}}};
- et.from_polygon_boundary(poly);
+ et.from_polygon_boundary(poly, expire_config);
}
SECTION("geom")
@@ -286,7 +298,7 @@ TEST_CASE("expire large polygon as boundary", "[NoDB]")
{5000.0, 25000.0},
{5000.0, 5000.0}}};
geom::geometry_t const geom{std::move(poly)};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
@@ -298,7 +310,7 @@ TEST_CASE("expire large polygon as boundary", "[NoDB]")
{5000.0, 5000.0}}};
geom::geometry_t geom{std::move(poly)};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -317,7 +329,8 @@ TEST_CASE("expire large polygon as boundary", "[NoDB]")
TEST_CASE("expire multipoint geometry", "[NoDB]")
{
- expire_tiles et{zoom, 20000, defproj};
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
geom::point_t p1{0.0, 0.0};
geom::point_t p2{15000.0, 15000.0};
@@ -327,7 +340,7 @@ TEST_CASE("expire multipoint geometry", "[NoDB]")
geom::multipoint_t mpt;
mpt.add_geometry(std::move(p1));
mpt.add_geometry(std::move(p2));
- et.from_geometry(mpt);
+ et.from_geometry(mpt, expire_config);
}
SECTION("geom")
@@ -336,7 +349,7 @@ TEST_CASE("expire multipoint geometry", "[NoDB]")
mpt.add_geometry(std::move(p1));
mpt.add_geometry(std::move(p2));
geom::geometry_t const geom{std::move(mpt)};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
@@ -346,7 +359,7 @@ TEST_CASE("expire multipoint geometry", "[NoDB]")
mpt.add_geometry(std::move(p2));
geom::geometry_t geom{std::move(mpt)};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -360,7 +373,8 @@ TEST_CASE("expire multipoint geometry", "[NoDB]")
TEST_CASE("expire multilinestring geometry", "[NoDB]")
{
- expire_tiles et{zoom, 20000, defproj};
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
geom::linestring_t l1{{2000.0, 2000.0}, {3000.0, 3000.0}};
geom::linestring_t l2{{15000.0, 15000.0}, {25000.0, 15000.0}};
@@ -368,19 +382,19 @@ TEST_CASE("expire multilinestring geometry", "[NoDB]")
ml.add_geometry(std::move(l1));
ml.add_geometry(std::move(l2));
- SECTION("multilinestring") { et.from_geometry(ml); }
+ SECTION("multilinestring") { et.from_geometry(ml, expire_config); }
SECTION("geom")
{
geom::geometry_t const geom{std::move(ml)};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
{
geom::geometry_t geom{std::move(ml)};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -392,7 +406,10 @@ TEST_CASE("expire multilinestring geometry", "[NoDB]")
TEST_CASE("expire multipolygon geometry", "[NoDB]")
{
- expire_tiles et{zoom, 10000, defproj};
+ expire_config_t expire_config;
+ expire_config.mode = expire_mode::hybrid;
+ expire_config.full_area_limit = 10000;
+ expire_tiles et{zoom, defproj};
geom::polygon_t p1{{{2000.0, 2000.0},
{2000.0, 3000.0},
@@ -415,19 +432,19 @@ TEST_CASE("expire multipolygon geometry", "[NoDB]")
mp.add_geometry(std::move(p1));
mp.add_geometry(std::move(p2));
- SECTION("multilinestring") { et.from_geometry(mp); }
+ SECTION("multilinestring") { et.from_geometry(mp, expire_config); }
SECTION("geom")
{
geom::geometry_t const geom{std::move(mp)};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
{
geom::geometry_t geom{std::move(mp)};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -451,7 +468,8 @@ TEST_CASE("expire multipolygon geometry", "[NoDB]")
TEST_CASE("expire geometry collection", "[NoDB]")
{
- expire_tiles et{zoom, 20000, defproj};
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
geom::collection_t collection;
collection.add_geometry(geom::geometry_t{geom::point_t{0.0, 0.0}});
@@ -461,14 +479,14 @@ TEST_CASE("expire geometry collection", "[NoDB]")
SECTION("geom")
{
geom::geometry_t const geom{std::move(collection)};
- et.from_geometry(geom);
+ et.from_geometry(geom, expire_config);
}
SECTION("geom with check")
{
geom::geometry_t geom{std::move(collection)};
geom.set_srid(3857);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
}
auto const tiles = et.get_tiles();
@@ -483,11 +501,12 @@ TEST_CASE("expire geometry collection", "[NoDB]")
TEST_CASE("expire doesn't do anything if not in 3857", "[NoDB]")
{
- expire_tiles et{zoom, 20000, defproj};
+ expire_config_t const expire_config;
+ expire_tiles et{zoom, defproj};
geom::geometry_t geom{geom::point_t{0.0, 0.0}};
geom.set_srid(1234);
- et.from_geometry_if_3857(geom);
+ et.from_geometry_if_3857(geom, expire_config);
auto const tiles = et.get_tiles();
REQUIRE(tiles.empty());
=====================================
tests/test-expire-tiles.cpp
=====================================
@@ -43,7 +43,7 @@ static void expire_centroids(expire_tiles *et, std::set<tile_t> const &tiles)
{
for (auto const &t : tiles) {
auto const p = t.center();
- et->from_bbox({p.x(), p.y(), p.x(), p.y()});
+ et->from_bbox({p.x(), p.y(), p.x(), p.y()}, expire_config_t{});
}
}
@@ -88,11 +88,11 @@ TEST_CASE("simple expire z1", "[NoDB]")
{
uint32_t const minzoom = 1;
uint32_t const maxzoom = 1;
- expire_tiles et{minzoom, 20000, defproj};
+ expire_tiles et{minzoom, defproj};
// as big a bbox as possible at the origin to dirty all four
// quadrants of the world.
- et.from_bbox({-10000, -10000, 10000, 10000});
+ et.from_bbox({-10000, -10000, 10000, 10000}, expire_config_t{});
auto const tiles = get_tiles_ordered(&et, minzoom, maxzoom);
CHECK(tiles.size() == 4);
@@ -108,11 +108,11 @@ TEST_CASE("simple expire z3", "[NoDB]")
{
uint32_t const minzoom = 3;
uint32_t const maxzoom = 3;
- expire_tiles et{minzoom, 20000, defproj};
+ expire_tiles et{minzoom, defproj};
// as big a bbox as possible at the origin to dirty all four
// quadrants of the world.
- et.from_bbox({-10000, -10000, 10000, 10000});
+ et.from_bbox({-10000, -10000, 10000, 10000}, expire_config_t{});
auto const tiles = get_tiles_ordered(&et, minzoom, maxzoom);
CHECK(tiles.size() == 4);
@@ -128,11 +128,11 @@ TEST_CASE("simple expire z18", "[NoDB]")
{
uint32_t const minzoom = 18;
uint32_t const maxzoom = 18;
- expire_tiles et{minzoom, 20000, defproj};
+ expire_tiles et{minzoom, defproj};
// dirty a smaller bbox this time, as at z18 the scale is
// pretty small.
- et.from_bbox({-1, -1, 1, 1});
+ et.from_bbox({-1, -1, 1, 1}, expire_config_t{});
auto const tiles = get_tiles_ordered(&et, minzoom, maxzoom);
CHECK(tiles.size() == 4);
@@ -147,10 +147,11 @@ TEST_CASE("simple expire z18", "[NoDB]")
TEST_CASE("expire a simple line", "[NoDB]")
{
uint32_t const zoom = 18;
- expire_tiles et{zoom, 20000, defproj};
+ expire_tiles et{zoom, defproj};
et.from_geometry(
- geom::linestring_t{{1398725.0, 7493354.0}, {1399030.0, 7493354.0}});
+ geom::linestring_t{{1398725.0, 7493354.0}, {1399030.0, 7493354.0}},
+ expire_config_t{});
auto const tiles = get_tiles_ordered(&et, zoom, zoom);
CHECK(tiles.size() == 3);
@@ -164,10 +165,11 @@ TEST_CASE("expire a simple line", "[NoDB]")
TEST_CASE("expire a line near the tile border", "[NoDB]")
{
uint32_t const zoom = 18;
- expire_tiles et{zoom, 20000, defproj};
+ expire_tiles et{zoom, defproj};
et.from_geometry(
- geom::linestring_t{{1398945.0, 7493267.0}, {1398960.0, 7493282.0}});
+ geom::linestring_t{{1398945.0, 7493267.0}, {1398960.0, 7493282.0}},
+ expire_config_t{});
auto const tiles = get_tiles_ordered(&et, zoom, zoom);
REQUIRE(tiles.size() == 4);
@@ -182,12 +184,13 @@ TEST_CASE("expire a line near the tile border", "[NoDB]")
TEST_CASE("expire a u-shaped linestring", "[NoDB]")
{
uint32_t const zoom = 18;
- expire_tiles et{zoom, 20000, defproj};
+ expire_tiles et{zoom, defproj};
et.from_geometry(geom::linestring_t{{1398586.0, 7493485.0},
{1398575.0, 7493347.0},
{1399020.0, 7493344.0},
- {1399012.0, 7493470.0}});
+ {1399012.0, 7493470.0}},
+ expire_config_t{});
auto const tiles = get_tiles_unordered(&et, zoom);
REQUIRE(tiles.size() == 6);
@@ -203,10 +206,11 @@ TEST_CASE("expire a u-shaped linestring", "[NoDB]")
TEST_CASE("expire longer horizontal line", "[NoDB]")
{
uint32_t const zoom = 18;
- expire_tiles et{zoom, 20000, defproj};
+ expire_tiles et{zoom, defproj};
et.from_geometry(
- geom::linestring_t{{1397815.0, 7493800.0}, {1399316.0, 7493780.0}});
+ geom::linestring_t{{1397815.0, 7493800.0}, {1399316.0, 7493780.0}},
+ expire_config_t{});
auto const tiles = get_tiles_unordered(&et, zoom);
REQUIRE(tiles.size() == 11);
@@ -219,10 +223,11 @@ TEST_CASE("expire longer horizontal line", "[NoDB]")
TEST_CASE("expire longer diagonal line", "[NoDB]")
{
uint32_t const zoom = 18;
- expire_tiles et{zoom, 20000, defproj};
+ expire_tiles et{zoom, defproj};
et.from_geometry(
- geom::linestring_t{{1398427.0, 7494118.0}, {1398869.0, 7493189.0}});
+ geom::linestring_t{{1398427.0, 7494118.0}, {1398869.0, 7493189.0}},
+ expire_config_t{});
auto const tiles = get_tiles_unordered(&et, zoom);
REQUIRE(tiles.size() == 14);
@@ -250,11 +255,11 @@ TEST_CASE("simple expire z17 and z18", "[NoDB]")
{
uint32_t const minzoom = 17;
uint32_t const maxzoom = 18;
- expire_tiles et{maxzoom, 20000, defproj};
+ expire_tiles et{maxzoom, defproj};
// dirty a smaller bbox this time, as at z18 the scale is
// pretty small.
- et.from_bbox({-1, -1, 1, 1});
+ et.from_bbox({-1, -1, 1, 1}, expire_config_t{});
auto const tiles = get_tiles_ordered(&et, minzoom, maxzoom);
CHECK(tiles.size() == 8);
@@ -278,9 +283,9 @@ TEST_CASE("simple expire z17 and z18 in one superior tile", "[NoDB]")
{
uint32_t const minzoom = 17;
uint32_t const maxzoom = 18;
- expire_tiles et{maxzoom, 20000, defproj};
+ expire_tiles et{maxzoom, defproj};
- et.from_bbox({-163, 140, -140, 164});
+ et.from_bbox({-163, 140, -140, 164}, expire_config_t{});
auto const tiles = get_tiles_ordered(&et, minzoom, maxzoom);
CHECK(tiles.size() == 5);
@@ -300,7 +305,7 @@ TEST_CASE("expire centroids", "[NoDB]")
uint32_t const zoom = 18;
for (int i = 0; i < 100; ++i) {
- expire_tiles et{zoom, 20000, defproj};
+ expire_tiles et{zoom, defproj};
auto check_set = generate_random(zoom, 100);
expire_centroids(&et, check_set);
@@ -320,9 +325,9 @@ TEST_CASE("merge expire sets", "[NoDB]")
uint32_t const zoom = 18;
for (int i = 0; i < 100; ++i) {
- expire_tiles et{zoom, 20000, defproj};
- expire_tiles et1{zoom, 20000, defproj};
- expire_tiles et2{zoom, 20000, defproj};
+ expire_tiles et{zoom, defproj};
+ expire_tiles et1{zoom, defproj};
+ expire_tiles et2{zoom, defproj};
auto check_set1 = generate_random(zoom, 100);
expire_centroids(&et1, check_set1);
@@ -351,9 +356,9 @@ TEST_CASE("merge identical expire sets", "[NoDB]")
uint32_t const zoom = 18;
for (int i = 0; i < 100; ++i) {
- expire_tiles et{zoom, 20000, defproj};
- expire_tiles et1{zoom, 20000, defproj};
- expire_tiles et2{zoom, 20000, defproj};
+ expire_tiles et{zoom, defproj};
+ expire_tiles et1{zoom, defproj};
+ expire_tiles et2{zoom, defproj};
auto const check_set = generate_random(zoom, 100);
expire_centroids(&et1, check_set);
@@ -376,9 +381,9 @@ TEST_CASE("merge overlapping expire sets", "[NoDB]")
uint32_t const zoom = 18;
for (int i = 0; i < 100; ++i) {
- expire_tiles et{zoom, 20000, defproj};
- expire_tiles et1{zoom, 20000, defproj};
- expire_tiles et2{zoom, 20000, defproj};
+ expire_tiles et{zoom, defproj};
+ expire_tiles et1{zoom, defproj};
+ expire_tiles et2{zoom, defproj};
auto check_set1 = generate_random(zoom, 100);
expire_centroids(&et1, check_set1);
@@ -410,15 +415,15 @@ TEST_CASE("merge with complete flag", "[NoDB]")
{
uint32_t const zoom = 18;
- expire_tiles et{zoom, 20000, defproj};
- expire_tiles et0{zoom, 20000, defproj};
- expire_tiles et1{zoom, 20000, defproj};
- expire_tiles et2{zoom, 20000, defproj};
+ expire_tiles et{zoom, defproj};
+ expire_tiles et0{zoom, defproj};
+ expire_tiles et1{zoom, defproj};
+ expire_tiles et2{zoom, defproj};
// et1&2 are two halves of et0's box
- et0.from_bbox({-10000, -10000, 10000, 10000});
- et1.from_bbox({-10000, -10000, 0, 10000});
- et2.from_bbox({0, -10000, 10000, 10000});
+ et0.from_bbox({-10000, -10000, 10000, 10000}, expire_config_t{});
+ et1.from_bbox({-10000, -10000, 0, 10000}, expire_config_t{});
+ et2.from_bbox({0, -10000, 10000, 10000}, expire_config_t{});
et.merge_and_destroy(&et1);
et.merge_and_destroy(&et2);
=====================================
tests/test-parse-osmium.cpp
=====================================
@@ -205,8 +205,8 @@ TEST_CASE("parse diff file")
output, "008-ch.osc.gz", false);
REQUIRE(output->node.added == 0);
- REQUIRE(output->node.modified == 1176);
- REQUIRE(output->node.deleted == 16773);
+ REQUIRE(output->node.modified == 153);
+ REQUIRE(output->node.deleted == 17796);
REQUIRE(output->way.added == 0);
REQUIRE(output->way.modified == 161);
REQUIRE(output->way.deleted == 4);
View it on GitLab: https://salsa.debian.org/debian-gis-team/osm2pgsql/-/commit/fae0d624c77f52d9ef86a4aea73e028835bf9cc4
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/osm2pgsql/-/commit/fae0d624c77f52d9ef86a4aea73e028835bf9cc4
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20230213/b3d32550/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list