[Git][debian-gis-team/osmosis][buster-backports] 6 commits: New upstream version 0.48.1
Bas Couwenberg
gitlab at salsa.debian.org
Mon Jul 6 04:56:44 BST 2020
Bas Couwenberg pushed to branch buster-backports at Debian GIS Project / osmosis
Commits:
4744b392 by Bas Couwenberg at 2020-06-30T19:07:52+02:00
New upstream version 0.48.1
- - - - -
45cdac39 by Bas Couwenberg at 2020-06-30T19:07:56+02:00
Update upstream source from tag 'upstream/0.48.1'
Update to upstream version '0.48.1'
with Debian dir b3922f0e6b88059cb87987fe1aa8ca320bf3c348
- - - - -
7b9f389a by Bas Couwenberg at 2020-06-30T19:08:13+02:00
New upstream release.
- - - - -
5d2c93e4 by Bas Couwenberg at 2020-06-30T19:09:38+02:00
Set distribution to unstable.
- - - - -
ccd028d5 by Bas Couwenberg at 2020-07-06T05:48:49+02:00
Merge tag 'debian/0.48.1-1' into buster-backports
releasing package osmosis version 0.48.1-1
- - - - -
5eb0be28 by Bas Couwenberg at 2020-07-06T05:48:56+02:00
Rebuild for buster-backports.
- - - - -
30 changed files:
- README.md
- build-support/docker/db/docker-start.sh
- debian/changelog
- + doc/detailed-usage.adoc
- + doc/development.md
- osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/common/DatabaseContext2.java
- osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbChangeReader.java
- osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbChangeWriter.java
- osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbCurrentReader.java
- osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbFileReplicator.java
- osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbReader.java
- osmosis-core/build.gradle
- + osmosis-core/src/main/java/org/openstreetmap/osmosis/core/database/DatabaseLocker.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/common/DatabaseContext.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlChangeWriter.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlChangeWriterFactory.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriter.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriterFactory.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDatasetReader.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDatasetReaderFactory.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlTruncator.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/ChangeWriter.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/EntityDao.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/NodeDao.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/PostgreSqlDatasetContext.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/RelationDao.java
- osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/WayDao.java
- + osmosis-pgsnapshot/src/test/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/DatabaseLockerTest.java
- package/changes.txt
- + package/script/pgsnapshot_schema_0.6_changes.sql
Changes:
=====================================
README.md
=====================================
@@ -13,9 +13,9 @@ file, components for deriving and applying change sets to data sources,
components for sorting data, etc. It has been written so that it is easy to add
new features without re-writing common tasks such as file or database handling.
-Some brief build, running and installation notes are provided below, however
-most documentation may be found on
-[the project wiki page](http://wiki.openstreetmap.org/wiki/Osmosis).
+The main point of entry for documentation is
+[the project wiki page](http://wiki.openstreetmap.org/wiki/Osmosis), although
+some information is included below.
## Status
@@ -25,6 +25,12 @@ and transitioned to periodic acceptance of pull requests with tests and minor ve
Keep an eye on [osmosis-dev list](https://lists.openstreetmap.org/listinfo/osmosis-dev)
for any updates.
+## Usage
+
+* [The project wiki page](http://wiki.openstreetmap.org/wiki/Osmosis) is the best
+place to begin for new users.
+* [Detailed Usage](./doc/detailed-usage.adoc) is the main reference for experienced users.
+
## Installation
It is recommended to use a pre-built distribution archive rather than compile
@@ -37,53 +43,7 @@ already on the `PATH`.
## Development
-The easiest way to perform a full Osmosis build is to use the docker-based
-development environment. If you have docker and docker-compose installed,
-simply run the following command to build and launch a shell with everything
-required to run the full build and test suite.
-
- ./docker.sh
-
-Osmosis is built using the [Gradle build tool](http://gradle.org). Gradle itself
-does not need to be installed because the `gradlew` script will install Gradle on
-first usage. The only requirements are a 1.7 JDK, and an Internet connection.
-Note that in the docker environment all downloads will still occur and be cached
-in your home directory.
-
-Below are several commands useful to build the software. All commands must be
-run from the root of the source tree.
-
-Perform a complete build including unit tests:
-
- ./docker.sh ./gradlew build
-
-Build the software without running unit tests:
-
- ./docker.sh ./gradlew assemble
-
-Clean the build tree:
-
- ./docker.sh ./gradlew clean
-
-Generate project files to allow the project to be imported into IntelliJ.
-
- ./docker.sh ./gradlew idea
-
-Generate project files to allow the project to be imported into Eclipse.
-
- ./docker.sh ./gradlew eclipse
-
-Verify checkstyle compliance:
-
- ./docker.sh ./gradlew checkstyleMain checkstyleTest
-
-After completing the build process, a working Osmosis installation is contained
-in the `package` sub-directory. The Osmosis launcher scripts reside in the `bin`
-sub-directory of package. On a UNIX-like environment use the "osmosis" script,
-on a Windows environment use the "osmosis.bat" script.
-
-Distribution archives in zip and tar gzipped formats are contained in the
-`package/build/distribution` directory.
+See [Development](./doc/development.md) for details.
## Issue Tracking
=====================================
build-support/docker/db/docker-start.sh
=====================================
@@ -53,6 +53,7 @@ if [ ! "$(ls -A $DATADIR)" ]; then
\i /install/script/pgsnapshot_schema_0.6_action.sql
\i /install/script/pgsnapshot_schema_0.6_bbox.sql
\i /install/script/pgsnapshot_schema_0.6_linestring.sql
+ \i /install/script/pgsnapshot_schema_0.6_changes.sql
EOSQL
# Configure the pgosmsnap06_test_with_schema database as the OSM user.
@@ -65,6 +66,7 @@ if [ ! "$(ls -A $DATADIR)" ]; then
\i /install/script/pgsnapshot_schema_0.6_action.sql
\i /install/script/pgsnapshot_schema_0.6_bbox.sql
\i /install/script/pgsnapshot_schema_0.6_linestring.sql
+ \i /install/script/pgsnapshot_schema_0.6_changes.sql
EOSQL
# Configure the api06_test database as the OSM user.
=====================================
debian/changelog
=====================================
@@ -1,3 +1,15 @@
+osmosis (0.48.1-1~bpo10+1) buster-backports; urgency=medium
+
+ * Rebuild for buster-backports.
+
+ -- Bas Couwenberg <sebastic at debian.org> Mon, 06 Jul 2020 05:48:52 +0200
+
+osmosis (0.48.1-1) unstable; urgency=medium
+
+ * New upstream release.
+
+ -- Bas Couwenberg <sebastic at debian.org> Tue, 30 Jun 2020 19:09:26 +0200
+
osmosis (0.48.0-1~bpo10+1) buster-backports; urgency=medium
* Rebuild for buster-backports.
=====================================
doc/detailed-usage.adoc
=====================================
The diff for this file was not included because it is too large.
=====================================
doc/development.md
=====================================
@@ -0,0 +1,49 @@
+# Development
+
+The easiest way to perform a full Osmosis build is to use the docker-based
+development environment. If you have docker and docker-compose installed,
+simply run the following command to build and launch a shell with everything
+required to run the full build and test suite.
+
+ ./docker.sh
+
+Osmosis is built using the [Gradle build tool](http://gradle.org). Gradle itself
+does not need to be installed because the `gradlew` script will install Gradle on
+first usage. The only requirements are a 1.7 JDK, and an Internet connection.
+Note that in the docker environment all downloads will still occur and be cached
+in your home directory.
+
+Below are several commands useful to build the software. All commands must be
+run from the root of the source tree.
+
+Perform a complete build including unit tests:
+
+ ./docker.sh ./gradlew build
+
+Build the software without running unit tests:
+
+ ./docker.sh ./gradlew assemble
+
+Clean the build tree:
+
+ ./docker.sh ./gradlew clean
+
+Generate project files to allow the project to be imported into IntelliJ.
+
+ ./docker.sh ./gradlew idea
+
+Generate project files to allow the project to be imported into Eclipse.
+
+ ./docker.sh ./gradlew eclipse
+
+Verify checkstyle compliance:
+
+ ./docker.sh ./gradlew checkstyleMain checkstyleTest
+
+After completing the build process, a working Osmosis installation is contained
+in the `package` sub-directory. The Osmosis launcher scripts reside in the `bin`
+sub-directory of package. On a UNIX-like environment use the "osmosis" script,
+on a Windows environment use the "osmosis.bat" script.
+
+Distribution archives in zip and tar gzipped formats are contained in the
+`package/build/distribution` directory.
=====================================
osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/common/DatabaseContext2.java
=====================================
@@ -8,6 +8,8 @@ import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
+import javax.sql.DataSource;
+
import org.apache.commons.dbcp.BasicDataSource;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
@@ -64,6 +66,14 @@ public class DatabaseContext2 implements AutoCloseable {
}
}
+ /**
+ * Retrieves the data source associated with this database context.
+ *
+ * @return {@link DataSource}
+ */
+ public DataSource getDataSource() {
+ return this.dataSource;
+ }
private OsmosisRuntimeException createUnknownDbTypeException() {
return new OsmosisRuntimeException("Unknown database type " + dbType + ".");
=====================================
osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbChangeReader.java
=====================================
@@ -9,6 +9,7 @@ import org.openstreetmap.osmosis.apidb.v0_6.impl.AllEntityDao;
import org.openstreetmap.osmosis.apidb.v0_6.impl.DeltaToDiffReader;
import org.openstreetmap.osmosis.apidb.v0_6.impl.SchemaVersionValidator;
import org.openstreetmap.osmosis.core.container.v0_6.ChangeContainer;
+import org.openstreetmap.osmosis.core.database.DatabaseLocker;
import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
import org.openstreetmap.osmosis.core.database.DatabasePreferences;
import org.openstreetmap.osmosis.core.lifecycle.ReleasableIterator;
@@ -109,13 +110,17 @@ public class ApidbChangeReader implements RunnableChangeSource {
* Reads all data from the database and send it to the sink.
*/
public void run() {
- try (DatabaseContext2 dbCtx = new DatabaseContext2(loginCredentials)) {
+ try (DatabaseContext2 dbCtx = new DatabaseContext2(loginCredentials);
+ DatabaseLocker locker = new DatabaseLocker(dbCtx.getDataSource(), false)) {
dbCtx.executeWithinTransaction(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus arg0) {
+ locker.lockDatabase(this.getClass().getSimpleName());
runImpl(dbCtx);
} });
- }
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
=====================================
osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbChangeWriter.java
=====================================
@@ -4,11 +4,13 @@ package org.openstreetmap.osmosis.apidb.v0_6;
import java.util.HashMap;
import java.util.Map;
+import org.openstreetmap.osmosis.apidb.common.DatabaseContext2;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.apidb.v0_6.impl.ActionChangeWriter;
import org.openstreetmap.osmosis.apidb.v0_6.impl.ChangeWriter;
import org.openstreetmap.osmosis.apidb.v0_6.impl.SchemaVersionValidator;
import org.openstreetmap.osmosis.core.container.v0_6.ChangeContainer;
+import org.openstreetmap.osmosis.core.database.DatabaseLocker;
import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
import org.openstreetmap.osmosis.core.database.DatabasePreferences;
import org.openstreetmap.osmosis.core.task.common.ChangeAction;
@@ -27,6 +29,8 @@ public class ApidbChangeWriter implements ChangeSink {
private final Map<ChangeAction, ActionChangeWriter> actionWriterMap;
private final SchemaVersionValidator schemaVersionValidator;
+ private final DatabaseContext2 dbCtx;
+ private final DatabaseLocker locker;
/**
* Creates a new instance.
@@ -39,12 +43,14 @@ public class ApidbChangeWriter implements ChangeSink {
public ApidbChangeWriter(DatabaseLoginCredentials loginCredentials, DatabasePreferences preferences,
boolean populateCurrentTables) {
changeWriter = new ChangeWriter(loginCredentials, populateCurrentTables);
- actionWriterMap = new HashMap<ChangeAction, ActionChangeWriter>();
+ actionWriterMap = new HashMap<>();
actionWriterMap.put(ChangeAction.Create, new ActionChangeWriter(changeWriter, ChangeAction.Create));
actionWriterMap.put(ChangeAction.Modify, new ActionChangeWriter(changeWriter, ChangeAction.Modify));
actionWriterMap.put(ChangeAction.Delete, new ActionChangeWriter(changeWriter, ChangeAction.Delete));
schemaVersionValidator = new SchemaVersionValidator(loginCredentials, preferences);
+ dbCtx = new DatabaseContext2(loginCredentials);
+ locker = new DatabaseLocker(dbCtx.getDataSource(), true);
}
/**
@@ -59,6 +65,7 @@ public class ApidbChangeWriter implements ChangeSink {
*/
public void process(ChangeContainer change) {
ChangeAction action;
+ this.locker.lockDatabase(this.getClass().getSimpleName());
// Verify that the schema version is supported.
schemaVersionValidator.validateVersion(ApidbVersionConstants.SCHEMA_MIGRATIONS);
@@ -85,6 +92,8 @@ public class ApidbChangeWriter implements ChangeSink {
* {@inheritDoc}
*/
public void close() {
+ this.locker.unlockDatabase();
changeWriter.close();
+ this.dbCtx.close();
}
}
=====================================
osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbCurrentReader.java
=====================================
@@ -9,6 +9,7 @@ import org.openstreetmap.osmosis.apidb.v0_6.impl.AllEntityDao;
import org.openstreetmap.osmosis.apidb.v0_6.impl.SchemaVersionValidator;
import org.openstreetmap.osmosis.core.container.v0_6.BoundContainer;
import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer;
+import org.openstreetmap.osmosis.core.database.DatabaseLocker;
import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
import org.openstreetmap.osmosis.core.database.DatabasePreferences;
import org.openstreetmap.osmosis.core.domain.v0_6.Bound;
@@ -90,13 +91,16 @@ public class ApidbCurrentReader implements RunnableSource {
* Reads all data from the database and send it to the sink.
*/
public void run() {
- try (DatabaseContext2 dbCtx = new DatabaseContext2(loginCredentials)) {
+ try (DatabaseContext2 dbCtx = new DatabaseContext2(loginCredentials);
+ DatabaseLocker locker = new DatabaseLocker(dbCtx.getDataSource(), false)) {
dbCtx.executeWithinTransaction(new TransactionCallbackWithoutResult() {
-
@Override
protected void doInTransactionWithoutResult(TransactionStatus arg0) {
+ locker.lockDatabase(this.getClass().getSimpleName());
runImpl(dbCtx);
} });
- }
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
=====================================
osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbFileReplicator.java
=====================================
@@ -10,6 +10,7 @@ import org.openstreetmap.osmosis.apidb.v0_6.impl.SystemTimeLoader;
import org.openstreetmap.osmosis.apidb.v0_6.impl.TimeDao;
import org.openstreetmap.osmosis.apidb.v0_6.impl.TransactionDao;
import org.openstreetmap.osmosis.apidb.v0_6.impl.TransactionManager;
+import org.openstreetmap.osmosis.core.database.DatabaseLocker;
import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
import org.openstreetmap.osmosis.core.database.DatabasePreferences;
import org.openstreetmap.osmosis.core.task.v0_6.ChangeSink;
@@ -95,8 +96,12 @@ public class ApidbFileReplicator implements RunnableChangeSource {
*/
@Override
public void run() {
- try (DatabaseContext2 dbCtx = new DatabaseContext2(loginCredentials)) {
+ try (DatabaseContext2 dbCtx = new DatabaseContext2(loginCredentials);
+ DatabaseLocker locker = new DatabaseLocker(dbCtx.getDataSource(), false)) {
+ locker.lockDatabase(this.getClass().getSimpleName());
runImpl(dbCtx);
- }
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
=====================================
osmosis-apidb/src/main/java/org/openstreetmap/osmosis/apidb/v0_6/ApidbReader.java
=====================================
@@ -11,6 +11,7 @@ import org.openstreetmap.osmosis.apidb.v0_6.impl.EntitySnapshotReader;
import org.openstreetmap.osmosis.apidb.v0_6.impl.SchemaVersionValidator;
import org.openstreetmap.osmosis.core.container.v0_6.BoundContainer;
import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer;
+import org.openstreetmap.osmosis.core.database.DatabaseLocker;
import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
import org.openstreetmap.osmosis.core.database.DatabasePreferences;
import org.openstreetmap.osmosis.core.domain.v0_6.Bound;
@@ -95,13 +96,17 @@ public class ApidbReader implements RunnableSource {
* Reads all data from the database and send it to the sink.
*/
public void run() {
- try (DatabaseContext2 dbCtx = new DatabaseContext2(loginCredentials)) {
+ try (DatabaseContext2 dbCtx = new DatabaseContext2(loginCredentials);
+ DatabaseLocker locker = new DatabaseLocker(dbCtx.getDataSource(), false)) {
dbCtx.executeWithinTransaction(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus arg0) {
+ locker.lockDatabase(this.getClass().getSimpleName());
runImpl(dbCtx);
}
});
- }
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
=====================================
osmosis-core/build.gradle
=====================================
@@ -8,6 +8,8 @@ dependencies {
compile group: 'com.fasterxml.woodstox', name: 'woodstox-core', version: dependencyVersionWoodstoxCore
compile group: 'org.codehaus.woodstox', name: 'stax2-api', version: dependencyVersionWoodstoxStax2
compile group: 'org.apache.commons', name: 'commons-compress', version: dependencyVersionCommonsCompress
+ compile group: 'xerces', name: 'xercesImpl', version: dependencyVersionXerces
+ compile group: 'org.springframework', name: 'spring-jdbc', version: dependencyVersionSpring
}
/*
=====================================
osmosis-core/src/main/java/org/openstreetmap/osmosis/core/database/DatabaseLocker.java
=====================================
@@ -0,0 +1,142 @@
+// This software is released into the Public Domain. See copying.txt for details.
+package org.openstreetmap.osmosis.core.database;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.logging.Logger;
+
+import javax.sql.DataSource;
+
+/**
+ * A check against the database to see if it is locked.
+ *
+ * @author mcuthbert
+ */
+public class DatabaseLocker implements AutoCloseable {
+
+ private static Logger logger = Logger.getLogger(DatabaseLocker.class.getSimpleName());
+ private final DataSource source;
+ private final boolean writeLock;
+ private boolean enabled = true;
+ private int lockedIdentifier = -1;
+
+ /**
+ * Static function to fully unlock the database.
+ *
+ * @param source {@link DataSource} to execute the query
+ */
+ public static void fullUnlockDatabase(final DataSource source) {
+ unlockDatabase(-1, source);
+ }
+
+ private static void unlockDatabase(final int identifier, final DataSource source) {
+ try (Connection connection = source.getConnection();
+ PreparedStatement statement = connection.prepareStatement("SELECT unlock_database(?)")) {
+ statement.setInt(1, identifier);
+ final ResultSet result = statement.executeQuery();
+ if (result.next()) {
+ final boolean unlocked = result.getBoolean(1);
+ if (unlocked) {
+ logger.info(String.format("Unlocked database using identifier %d.", identifier));
+ return;
+ }
+ }
+ throw new RuntimeException("Failed to unlock the database");
+ } catch (final SQLException e) {
+ throw new RuntimeException("Failed to unlock the database", e);
+ }
+ }
+
+ /**
+ * Default Constructor.
+ *
+ * @param source The DataSource for the connection
+ * @param writeLock Whether the lock that is being requested is a write lock
+ */
+ public DatabaseLocker(final DataSource source, final boolean writeLock) {
+ this.source = source;
+ this.writeLock = writeLock;
+ // check to see if the function exists in the database and if it doesn't then throw a
+ // warning on the log and don't try any locking
+ try (Connection connection = source.getConnection();
+ Statement statement = connection.createStatement()) {
+ statement.executeQuery("SELECT 'lock_database'::regproc, 'unlock_database'::regproc");
+ } catch (final Exception e) {
+ logger.warning("Locking functions do not exist in database. Disabling locking.");
+ this.enabled = false;
+ }
+ }
+
+ /**
+ * Helper function for primary lock database function that simply sets the source to empty string.
+ *
+ * @param process The process that is locking the database
+ */
+ public void lockDatabase(final String process) {
+ this.lockDatabase(process, "");
+ }
+
+ /**
+ * Will attempt to lock the database.
+ *
+ * @param process
+ * The process that will lock the database. This would be something like "Extracts" or "Replication".
+ * @param description
+ * The source of the process. This is basically a description for the process, like
+ * "Pipeline Extraction for Build X".
+ */
+ public void lockDatabase(final String process, final String description) {
+ if (this.enabled) {
+ String lockName = "read";
+ if (this.writeLock) {
+ lockName = "write";
+ }
+ try (Connection connection = source.getConnection();
+ PreparedStatement statement = connection.prepareStatement("SELECT lock_database(?, ?, ?, ?)")) {
+ statement.setString(1, process);
+ statement.setString(2, description);
+ statement.setString(3, InetAddress.getLocalHost().getHostName());
+ statement.setBoolean(4, this.writeLock);
+ final ResultSet result = statement.executeQuery();
+ if (result.next()) {
+ this.lockedIdentifier = result.getInt(1);
+ logger.info(String.format("Obtained %s lock to database for process: %s "
+ + "from source: '%s', with lockedID: %d",
+ lockName, process, description, this.lockedIdentifier));
+ } else {
+ throw new RuntimeException("Failed to retrieve lock for the database.");
+ }
+ } catch (final SQLException | UnknownHostException e) {
+ throw new RuntimeException("Failed to lock the database.", e);
+ }
+ }
+ }
+
+ /**
+ * Will attempt to unlock the database with the given locked identifier.
+ */
+ public void unlockDatabase() {
+ if (this.enabled && this.lockedIdentifier > 0) {
+ unlockDatabase(this.lockedIdentifier, this.source);
+ }
+ }
+
+ /**
+ * Does a full unlock of the database. So no matter what is locking it, this will unlock it.
+ */
+ public void fullUnlockDatabase() {
+ if (this.enabled) {
+ fullUnlockDatabase(this.source);
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.unlockDatabase();
+ }
+}
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/common/DatabaseContext.java
=====================================
@@ -66,6 +66,14 @@ public class DatabaseContext implements AutoCloseable {
}
}
+ /**
+ * Returns the DataSource associated with this database context.
+ *
+ * @return {@link DataSource}
+ */
+ public DataSource getDataSource() {
+ return this.dataSource;
+ }
/**
* Begins a new database transaction. This is not required if
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlChangeWriter.java
=====================================
@@ -1,13 +1,21 @@
// This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.pgsnapshot.v0_6;
+import java.text.SimpleDateFormat;
+import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
import org.openstreetmap.osmosis.core.OsmosisRuntimeException;
import org.openstreetmap.osmosis.core.container.v0_6.ChangeContainer;
+import org.openstreetmap.osmosis.core.database.DatabaseLocker;
import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
import org.openstreetmap.osmosis.core.database.DatabasePreferences;
+import org.openstreetmap.osmosis.core.domain.v0_6.Entity;
+import org.openstreetmap.osmosis.core.domain.v0_6.EntityType;
import org.openstreetmap.osmosis.core.task.common.ChangeAction;
import org.openstreetmap.osmosis.core.task.v0_6.ChangeSink;
import org.openstreetmap.osmosis.pgsnapshot.common.DatabaseContext;
@@ -24,13 +32,19 @@ import org.openstreetmap.osmosis.pgsnapshot.v0_6.impl.ChangeWriter;
* @author Brett Henderson
*/
public class PostgreSqlChangeWriter implements ChangeSink {
+
+ private static final SimpleDateFormat FORMATTER = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private ChangeWriter changeWriter;
private Map<ChangeAction, ActionChangeWriter> actionWriterMap;
private DatabaseContext dbCtx;
private SchemaVersionValidator schemaVersionValidator;
private boolean initialized;
-
+ private final Set<Long> appliedChangeSets;
+ private long earliestTimestamp = 9999999999999L;
+ private long latestTimestamp = 0L;
+ private final Map<String, Integer> modifications;
+ private final DatabaseLocker locker;
/**
* Creates a new instance.
@@ -43,12 +57,15 @@ public class PostgreSqlChangeWriter implements ChangeSink {
* If true, zero and single node ways are kept. Otherwise they are
* silently dropped to avoid putting invalid geometries into the
* database which can cause problems with postgis functions.
+ * @param logging
+ * If true, will log all sql queries to the database that was executed
+ * from the change log
*/
public PostgreSqlChangeWriter(DatabaseLoginCredentials loginCredentials,
- DatabasePreferences preferences, boolean keepInvalidWays) {
+ DatabasePreferences preferences, boolean keepInvalidWays, boolean logging) {
dbCtx = new DatabaseContext(loginCredentials);
- changeWriter = new ChangeWriter(dbCtx);
- actionWriterMap = new HashMap<ChangeAction, ActionChangeWriter>();
+ changeWriter = new ChangeWriter(dbCtx, logging);
+ actionWriterMap = new HashMap<>();
actionWriterMap.put(ChangeAction.Create,
new ActionChangeWriter(changeWriter, ChangeAction.Create, keepInvalidWays));
actionWriterMap.put(ChangeAction.Modify,
@@ -57,15 +74,17 @@ public class PostgreSqlChangeWriter implements ChangeSink {
new ActionChangeWriter(changeWriter, ChangeAction.Delete, keepInvalidWays));
schemaVersionValidator = new SchemaVersionValidator(dbCtx.getJdbcTemplate(), preferences);
-
+ appliedChangeSets = new HashSet<>();
+ modifications = new HashMap<>();
initialized = false;
+ locker = new DatabaseLocker(dbCtx.getDataSource(), true);
}
private void initialize() {
if (!initialized) {
+ this.locker.lockDatabase(this.getClass().getSimpleName());
dbCtx.beginTransaction();
-
initialized = true;
}
}
@@ -95,6 +114,15 @@ public class PostgreSqlChangeWriter implements ChangeSink {
if (!actionWriterMap.containsKey(action)) {
throw new OsmosisRuntimeException("The action " + action + " is unrecognized.");
}
+
+ final Entity entity = change.getEntityContainer().getEntity();
+ this.appliedChangeSets.add(entity.getChangesetId());
+ this.earliestTimestamp = Math.min(this.earliestTimestamp, entity.getTimestamp().getTime());
+ this.latestTimestamp = Math.max(this.latestTimestamp, entity.getTimestamp().getTime());
+
+ final String name = entity.getType().name() + "-" + action.name();
+ final int count = modifications.getOrDefault(name, 0) + 1;
+ modifications.put(name, count);
// Process the entity using the action writer appropriate for the change
// action.
@@ -109,6 +137,26 @@ public class PostgreSqlChangeWriter implements ChangeSink {
initialize();
changeWriter.complete();
+
+ // on complete write to the changes table
+ dbCtx.getJdbcTemplate().execute(String.format("INSERT INTO replication_changes "
+ + "(nodes_added, nodes_modified, nodes_deleted, "
+ + "ways_added, ways_modified, ways_deleted, "
+ + "relations_added, relations_modified, relations_deleted, "
+ + "changesets_applied, earliest_timestamp, latest_timestamp) "
+ + "VALUES "
+ + "(%s, %s, %s, %s, %s, %s, %s, %s, %s, ARRAY[%s]::BIGINT[], '%s', '%s')",
+ modifications.getOrDefault(EntityType.Node.name() + "-" + ChangeAction.Create, 0),
+ modifications.getOrDefault(EntityType.Node.name() + "-" + ChangeAction.Modify, 0),
+ modifications.getOrDefault(EntityType.Node.name() + "-" + ChangeAction.Delete, 0),
+ modifications.getOrDefault(EntityType.Way.name() + "-" + ChangeAction.Create, 0),
+ modifications.getOrDefault(EntityType.Way.name() + "-" + ChangeAction.Modify, 0),
+ modifications.getOrDefault(EntityType.Way.name() + "-" + ChangeAction.Delete, 0),
+ modifications.getOrDefault(EntityType.Relation.name() + "-" + ChangeAction.Create, 0),
+ modifications.getOrDefault(EntityType.Relation.name() + "-" + ChangeAction.Modify, 0),
+ modifications.getOrDefault(EntityType.Relation.name() + "-" + ChangeAction.Delete, 0),
+ appliedChangeSets.stream().map(id -> id + "").collect(Collectors.joining(",")),
+ FORMATTER.format(new Date(earliestTimestamp)), FORMATTER.format(new Date(latestTimestamp))));
dbCtx.commitTransaction();
}
@@ -118,6 +166,9 @@ public class PostgreSqlChangeWriter implements ChangeSink {
* {@inheritDoc}
*/
public void close() {
+ // now that we are finished, unlock the database
+ this.locker.unlockDatabase();
+
changeWriter.release();
dbCtx.close();
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlChangeWriterFactory.java
=====================================
@@ -1,8 +1,6 @@
// This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.pgsnapshot.v0_6;
-import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
-import org.openstreetmap.osmosis.core.database.DatabasePreferences;
import org.openstreetmap.osmosis.core.database.DatabaseTaskManagerFactory;
import org.openstreetmap.osmosis.core.pipeline.common.TaskConfiguration;
import org.openstreetmap.osmosis.core.pipeline.common.TaskManager;
@@ -18,27 +16,24 @@ public class PostgreSqlChangeWriterFactory extends DatabaseTaskManagerFactory {
private static final String ARG_KEEP_INVALID_WAYS = "keepInvalidWays";
private static final boolean DEFAULT_KEEP_INVALID_WAYS = true;
+ private static final String ARG_LOGGING = "logging";
+ private static final boolean DEFAULT_LOGGING = false;
/**
* {@inheritDoc}
*/
@Override
protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
- DatabaseLoginCredentials loginCredentials;
- DatabasePreferences preferences;
-
- // Get the task arguments.
- loginCredentials = getDatabaseLoginCredentials(taskConfig);
- preferences = getDatabasePreferences(taskConfig);
-
boolean keepInvalidWays = getBooleanArgument(taskConfig, ARG_KEEP_INVALID_WAYS, DEFAULT_KEEP_INVALID_WAYS);
+ boolean logging = getBooleanArgument(taskConfig, ARG_LOGGING, DEFAULT_LOGGING);
return new ChangeSinkManager(
taskConfig.getId(),
new PostgreSqlChangeWriter(
- loginCredentials,
- preferences,
- keepInvalidWays
+ getDatabaseLoginCredentials(taskConfig),
+ getDatabasePreferences(taskConfig),
+ keepInvalidWays,
+ logging
),
taskConfig.getPipeArgs()
);
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriter.java
=====================================
@@ -5,6 +5,7 @@ import java.util.Map;
import java.util.logging.Logger;
import org.openstreetmap.osmosis.core.container.v0_6.EntityContainer;
+import org.openstreetmap.osmosis.core.database.DatabaseLocker;
import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
import org.openstreetmap.osmosis.core.database.DatabasePreferences;
import org.openstreetmap.osmosis.core.task.v0_6.Sink;
@@ -36,7 +37,8 @@ public class PostgreSqlCopyWriter implements Sink {
private boolean populateLinestring;
private boolean keepInvalidWays;
private boolean initialized;
-
+ private DatabaseContext dbCtx;
+ private DatabaseLocker locker;
/**
* Creates a new instance.
@@ -59,7 +61,8 @@ public class PostgreSqlCopyWriter implements Sink {
this.preferences = preferences;
this.storeType = storeType;
this.keepInvalidWays = keepInvalidWays;
-
+ this.dbCtx = new DatabaseContext(loginCredentials);
+ this.locker = new DatabaseLocker(dbCtx.getDataSource(), true);
copyFileset = new TempCopyFileset();
}
@@ -68,12 +71,9 @@ public class PostgreSqlCopyWriter implements Sink {
if (!initialized) {
LOG.fine("Initializing the database and temporary processing files.");
- try (DatabaseContext dbCtx = new DatabaseContext(loginCredentials)) {
- DatabaseCapabilityChecker capabilityChecker = new DatabaseCapabilityChecker(dbCtx);
-
- populateBbox = capabilityChecker.isWayBboxSupported();
- populateLinestring = capabilityChecker.isWayLinestringSupported();
- }
+ DatabaseCapabilityChecker capabilityChecker = new DatabaseCapabilityChecker(this.dbCtx);
+ populateBbox = capabilityChecker.isWayBboxSupported();
+ populateLinestring = capabilityChecker.isWayLinestringSupported();
copyFilesetBuilder =
new CopyFilesetBuilder(copyFileset, populateBbox, populateLinestring, storeType, keepInvalidWays);
@@ -81,7 +81,8 @@ public class PostgreSqlCopyWriter implements Sink {
copyFilesetLoader = new CopyFilesetLoader(loginCredentials, preferences, copyFileset);
LOG.fine("Processing input data, building geometries and creating database load files.");
-
+
+ this.locker.lockDatabase(this.getClass().getSimpleName());
initialized = true;
}
}
@@ -129,7 +130,8 @@ public class PostgreSqlCopyWriter implements Sink {
copyFilesetBuilder = null;
}
copyFileset.close();
-
+ this.locker.unlockDatabase();
+ this.dbCtx.close();
initialized = false;
}
}
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriterFactory.java
=====================================
@@ -1,8 +1,6 @@
// This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.pgsnapshot.v0_6;
-import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
-import org.openstreetmap.osmosis.core.database.DatabasePreferences;
import org.openstreetmap.osmosis.core.database.DatabaseTaskManagerFactory;
import org.openstreetmap.osmosis.pgsnapshot.common.NodeLocationStoreType;
import org.openstreetmap.osmosis.core.pipeline.common.TaskConfiguration;
@@ -26,14 +24,9 @@ public class PostgreSqlCopyWriterFactory extends DatabaseTaskManagerFactory {
*/
@Override
protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
- DatabaseLoginCredentials loginCredentials;
- DatabasePreferences preferences;
NodeLocationStoreType storeType;
boolean keepInvalidWays;
-
- // Get the task arguments.
- loginCredentials = getDatabaseLoginCredentials(taskConfig);
- preferences = getDatabasePreferences(taskConfig);
+
storeType = Enum.valueOf(
NodeLocationStoreType.class,
getStringArgument(taskConfig, ARG_NODE_LOCATION_STORE_TYPE, DEFAULT_NODE_LOCATION_STORE_TYPE));
@@ -41,7 +34,11 @@ public class PostgreSqlCopyWriterFactory extends DatabaseTaskManagerFactory {
return new SinkManager(
taskConfig.getId(),
- new PostgreSqlCopyWriter(loginCredentials, preferences, storeType, keepInvalidWays),
+ new PostgreSqlCopyWriter(
+ getDatabaseLoginCredentials(taskConfig),
+ getDatabasePreferences(taskConfig),
+ storeType,
+ keepInvalidWays),
taskConfig.getPipeArgs()
);
}
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDatasetReader.java
=====================================
@@ -3,8 +3,10 @@ package org.openstreetmap.osmosis.pgsnapshot.v0_6;
import org.openstreetmap.osmosis.core.container.v0_6.Dataset;
import org.openstreetmap.osmosis.core.container.v0_6.DatasetContext;
+import org.openstreetmap.osmosis.core.database.DatabaseLocker;
import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
import org.openstreetmap.osmosis.core.database.DatabasePreferences;
+import org.openstreetmap.osmosis.pgsnapshot.common.DatabaseContext;
import org.openstreetmap.osmosis.pgsnapshot.v0_6.impl.PostgreSqlDatasetContext;
import org.openstreetmap.osmosis.core.task.v0_6.DatasetSink;
import org.openstreetmap.osmosis.core.task.v0_6.RunnableDatasetSource;
@@ -49,9 +51,12 @@ public class PostgreSqlDatasetReader implements RunnableDatasetSource, Dataset {
*/
@Override
public void run() {
- try {
+ try (DatabaseContext context = new DatabaseContext(this.loginCredentials);
+ DatabaseLocker locker = new DatabaseLocker(context.getDataSource(), false)) {
+ locker.lockDatabase(this.getClass().getSimpleName());
datasetSink.process(this);
-
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
} finally {
datasetSink.close();
}
@@ -63,6 +68,6 @@ public class PostgreSqlDatasetReader implements RunnableDatasetSource, Dataset {
*/
@Override
public DatasetContext createReader() {
- return new PostgreSqlDatasetContext(loginCredentials, preferences);
+ return new PostgreSqlDatasetContext(loginCredentials, preferences, false);
}
}
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDatasetReaderFactory.java
=====================================
@@ -1,8 +1,6 @@
// This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.pgsnapshot.v0_6;
-import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
-import org.openstreetmap.osmosis.core.database.DatabasePreferences;
import org.openstreetmap.osmosis.core.database.DatabaseTaskManagerFactory;
import org.openstreetmap.osmosis.core.pipeline.common.TaskConfiguration;
import org.openstreetmap.osmosis.core.pipeline.common.TaskManager;
@@ -21,16 +19,11 @@ public class PostgreSqlDatasetReaderFactory extends DatabaseTaskManagerFactory {
*/
@Override
protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) {
- DatabaseLoginCredentials loginCredentials;
- DatabasePreferences preferences;
-
- // Get the task arguments.
- loginCredentials = getDatabaseLoginCredentials(taskConfig);
- preferences = getDatabasePreferences(taskConfig);
-
return new RunnableDatasetSourceManager(
taskConfig.getId(),
- new PostgreSqlDatasetReader(loginCredentials, preferences),
+ new PostgreSqlDatasetReader(
+ getDatabaseLoginCredentials(taskConfig),
+ getDatabasePreferences(taskConfig)),
taskConfig.getPipeArgs()
);
}
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlTruncator.java
=====================================
@@ -3,6 +3,7 @@ package org.openstreetmap.osmosis.pgsnapshot.v0_6;
import java.util.logging.Logger;
+import org.openstreetmap.osmosis.core.database.DatabaseLocker;
import org.openstreetmap.osmosis.core.database.DatabaseLoginCredentials;
import org.openstreetmap.osmosis.core.database.DatabasePreferences;
import org.openstreetmap.osmosis.core.task.common.RunnableTask;
@@ -20,7 +21,6 @@ public class PostgreSqlTruncator implements RunnableTask {
private static final Logger LOG = Logger.getLogger(PostgreSqlTruncator.class.getName());
-
// These tables will be truncated.
private static final String[] SQL_TABLE_NAMES = {
"actions",
@@ -30,10 +30,9 @@ public class PostgreSqlTruncator implements RunnableTask {
"relations", "relation_tags", "relation_members"
};
-
private DatabaseContext dbCtx;
private SchemaVersionValidator schemaVersionValidator;
-
+ private DatabaseLocker locker;
/**
* Creates a new instance.
@@ -45,11 +44,10 @@ public class PostgreSqlTruncator implements RunnableTask {
*/
public PostgreSqlTruncator(DatabaseLoginCredentials loginCredentials, DatabasePreferences preferences) {
dbCtx = new DatabaseContext(loginCredentials);
-
+ this.locker = new DatabaseLocker(new DatabaseContext(loginCredentials).getDataSource(), true);
schemaVersionValidator = new SchemaVersionValidator(dbCtx.getJdbcTemplate(), preferences);
}
-
/**
* Truncates all data from the database.
*/
@@ -58,7 +56,7 @@ public class PostgreSqlTruncator implements RunnableTask {
schemaVersionValidator.validateVersion(PostgreSqlVersionConstants.SCHEMA_VERSION);
dbCtx.beginTransaction();
-
+ this.locker.lockDatabase(this.getClass().getSimpleName());
LOG.fine("Truncating tables.");
for (int i = 0; i < SQL_TABLE_NAMES.length; i++) {
if (dbCtx.doesTableExist(SQL_TABLE_NAMES[i])) {
@@ -77,6 +75,7 @@ public class PostgreSqlTruncator implements RunnableTask {
LOG.fine("Complete.");
} finally {
+ this.locker.unlockDatabase();
dbCtx.close();
}
}
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/ChangeWriter.java
=====================================
@@ -42,15 +42,17 @@ public class ChangeWriter {
*
* @param dbCtx
* The database context to use for accessing the database.
+ * @param logging
+ * Verbose logging directly to the database
*/
- public ChangeWriter(DatabaseContext dbCtx) {
+ public ChangeWriter(DatabaseContext dbCtx, boolean logging) {
this.dbCtx = dbCtx;
actionDao = new ActionDao(dbCtx);
userDao = new UserDao(dbCtx, actionDao);
- nodeDao = new NodeDao(dbCtx, actionDao);
- wayDao = new WayDao(dbCtx, actionDao);
- relationDao = new RelationDao(dbCtx, actionDao);
+ nodeDao = new NodeDao(dbCtx, actionDao, logging);
+ wayDao = new WayDao(dbCtx, actionDao, logging);
+ relationDao = new RelationDao(dbCtx, actionDao, logging);
userSet = new HashSet<Integer>();
}
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/EntityDao.java
=====================================
@@ -1,9 +1,11 @@
// This software is released into the Public Domain. See copying.txt for details.
package org.openstreetmap.osmosis.pgsnapshot.v0_6.impl;
+import java.sql.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.openstreetmap.osmosis.core.database.FeaturePopulator;
import org.openstreetmap.osmosis.core.database.SortingStoreRowMapperListener;
@@ -35,6 +37,7 @@ public abstract class EntityDao<T extends Entity> {
private NamedParameterJdbcTemplate namedParameterJdbcTemplate;
private ActionDao actionDao;
private EntityMapper<T> entityMapper;
+ private boolean logging = false;
/**
@@ -46,12 +49,16 @@ public abstract class EntityDao<T extends Entity> {
* Provides entity type specific JDBC support.
* @param actionDao
* The dao to use for adding action records to the database.
+ * @param logging
+ * Verbose logging directly to the database
*/
- protected EntityDao(JdbcTemplate jdbcTemplate, EntityMapper<T> entityMapper, ActionDao actionDao) {
+ protected EntityDao(JdbcTemplate jdbcTemplate, EntityMapper<T> entityMapper,
+ ActionDao actionDao, boolean logging) {
this.jdbcTemplate = jdbcTemplate;
this.namedParameterJdbcTemplate = new NamedParameterJdbcTemplate(jdbcTemplate);
this.entityMapper = entityMapper;
this.actionDao = actionDao;
+ this.logging = logging;
}
@@ -111,7 +118,9 @@ public abstract class EntityDao<T extends Entity> {
args = new HashMap<String, Object>();
entityMapper.populateEntityParameters(args, entity);
- namedParameterJdbcTemplate.update(entityMapper.getSqlInsert(1), args);
+ final String query = entityMapper.getSqlInsert(1);
+ namedParameterJdbcTemplate.update(query, args);
+ this.updateLoggingTable(query, args, 0);
actionDao.addAction(entityMapper.getEntityType(), ChangesetAction.CREATE, entity.getId());
}
@@ -129,7 +138,9 @@ public abstract class EntityDao<T extends Entity> {
args = new HashMap<String, Object>();
entityMapper.populateEntityParameters(args, entity);
- namedParameterJdbcTemplate.update(entityMapper.getSqlUpdate(true), args);
+ final String query = entityMapper.getSqlUpdate(true);
+ namedParameterJdbcTemplate.update(query, args);
+ this.updateLoggingTable(query, args, 1);
actionDao.addAction(entityMapper.getEntityType(), ChangesetAction.MODIFY, entity.getId());
}
@@ -147,10 +158,29 @@ public abstract class EntityDao<T extends Entity> {
args = new HashMap<String, Object>();
args.put("id", entityId);
- namedParameterJdbcTemplate.update(entityMapper.getSqlDelete(true), args);
+ final String query = entityMapper.getSqlDelete(true);
+ namedParameterJdbcTemplate.update(query, args);
+ this.updateLoggingTable(query, args, 2);
actionDao.addAction(entityMapper.getEntityType(), ChangesetAction.DELETE, entityId);
}
+
+ private void updateLoggingTable(final String query, final Map<String, Object> args, final int action) {
+ if (this.logging) {
+ final Map<String, Object> loggingMap = new HashMap<>(2);
+ loggingMap.put("id", args.getOrDefault("id", -1));
+ loggingMap.put("cid", args.getOrDefault("changesetId", -1));
+ loggingMap.put("cid_time", args.getOrDefault("timestamp", new Date(0)));
+ loggingMap.put("action", action);
+ loggingMap.put("query", query);
+ loggingMap.put("type", entityMapper.getEntityType().getDatabaseValue());
+ loggingMap.put("args", args.entrySet().stream().
+ map(Object::toString).collect(Collectors.joining(",")));
+ namedParameterJdbcTemplate.update(
+ "INSERT INTO sql_changes (entity_id, type, changeset_id, change_time, action, query, arguments) "
+ + "VALUES (:id, :type, :cid, :cid_time, :action, :query, :args)", loggingMap);
+ }
+ }
private ReleasableIterator<T> getFeaturelessEntity(String tablePrefix) {
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/NodeDao.java
=====================================
@@ -48,9 +48,11 @@ public class NodeDao extends EntityDao<Node> {
* The database context to use for accessing the database.
* @param actionDao
* The dao to use for adding action records to the database.
+ * @param logging
+ * Verbose logging directly to the database
*/
- public NodeDao(DatabaseContext dbCtx, ActionDao actionDao) {
- super(dbCtx.getJdbcTemplate(), new NodeMapper(), actionDao);
+ public NodeDao(DatabaseContext dbCtx, ActionDao actionDao, boolean logging) {
+ super(dbCtx.getJdbcTemplate(), new NodeMapper(), actionDao, logging);
jdbcTemplate = dbCtx.getJdbcTemplate();
capabilityChecker = new DatabaseCapabilityChecker(dbCtx);
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/PostgreSqlDatasetContext.java
=====================================
@@ -64,7 +64,7 @@ public class PostgreSqlDatasetContext implements DatasetContext {
private PostgreSqlEntityManager<Way> wayManager;
private PostgreSqlEntityManager<Relation> relationManager;
private PolygonBuilder polygonBuilder;
-
+ private boolean logging;
/**
* Creates a new instance.
@@ -73,14 +73,19 @@ public class PostgreSqlDatasetContext implements DatasetContext {
* Contains all information required to connect to the database.
* @param preferences
* Contains preferences configuring database behaviour.
+ * @param logging
+ * Verbose logging directly to the database
*/
- public PostgreSqlDatasetContext(DatabaseLoginCredentials loginCredentials, DatabasePreferences preferences) {
+ public PostgreSqlDatasetContext(DatabaseLoginCredentials loginCredentials,
+ DatabasePreferences preferences, boolean logging) {
this.loginCredentials = loginCredentials;
this.preferences = preferences;
polygonBuilder = new PolygonBuilder();
initialized = false;
+
+ this.logging = logging;
}
@@ -103,13 +108,13 @@ public class PostgreSqlDatasetContext implements DatasetContext {
actionDao = new ActionDao(dbCtx);
userDao = new UserDao(dbCtx, actionDao);
- nodeDao = new NodeDao(dbCtx, actionDao);
- wayDao = new WayDao(dbCtx, actionDao);
- relationDao = new RelationDao(dbCtx, actionDao);
+ nodeDao = new NodeDao(dbCtx, actionDao, this.logging);
+ wayDao = new WayDao(dbCtx, actionDao, this.logging);
+ relationDao = new RelationDao(dbCtx, actionDao, this.logging);
- nodeManager = new PostgreSqlEntityManager<Node>(nodeDao, userDao);
- wayManager = new PostgreSqlEntityManager<Way>(wayDao, userDao);
- relationManager = new PostgreSqlEntityManager<Relation>(relationDao, userDao);
+ nodeManager = new PostgreSqlEntityManager<>(nodeDao, userDao);
+ wayManager = new PostgreSqlEntityManager<>(wayDao, userDao);
+ relationManager = new PostgreSqlEntityManager<>(relationDao, userDao);
}
initialized = true;
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/RelationDao.java
=====================================
@@ -40,9 +40,11 @@ public class RelationDao extends EntityDao<Relation> {
* The database context to use for accessing the database.
* @param actionDao
* The dao to use for adding action records to the database.
+ * @param logging
+ * Verbose logging directly to the database
*/
- public RelationDao(DatabaseContext dbCtx, ActionDao actionDao) {
- super(dbCtx.getJdbcTemplate(), new RelationMapper(), actionDao);
+ public RelationDao(DatabaseContext dbCtx, ActionDao actionDao, boolean logging) {
+ super(dbCtx.getJdbcTemplate(), new RelationMapper(), actionDao, logging);
jdbcTemplate = dbCtx.getJdbcTemplate();
relationMemberMapper = new RelationMemberMapper();
=====================================
osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/WayDao.java
=====================================
@@ -49,9 +49,11 @@ public class WayDao extends EntityDao<Way> {
* The database context to use for accessing the database.
* @param actionDao
* The dao to use for adding action records to the database.
+ * @param logging
+ * Verbose logging directly to the database
*/
- public WayDao(DatabaseContext dbCtx, ActionDao actionDao) {
- super(dbCtx.getJdbcTemplate(), new WayMapper(), actionDao);
+ public WayDao(DatabaseContext dbCtx, ActionDao actionDao, boolean logging) {
+ super(dbCtx.getJdbcTemplate(), new WayMapper(), actionDao, logging);
jdbcTemplate = dbCtx.getJdbcTemplate();
capabilityChecker = new DatabaseCapabilityChecker(dbCtx);
=====================================
osmosis-pgsnapshot/src/test/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/DatabaseLockerTest.java
=====================================
@@ -0,0 +1,145 @@
+// This software is released into the Public Domain. See copying.txt for details.
+package org.openstreetmap.osmosis.pgsnapshot.v0_6.impl;
+
+import static org.junit.Assert.fail;
+
+import javax.sql.DataSource;
+
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+import org.openstreetmap.osmosis.core.database.DatabaseLocker;
+import org.springframework.jdbc.datasource.DriverManagerDataSource;
+
+/**
+ * Tests the basic functionality around the {@link DatabaseLocker}.
+ *
+ * @author mcuthbert
+ */
+ at FixMethodOrder(MethodSorters.JVM)
+public class DatabaseLockerTest {
+
+ /**
+ * Simple test to make sure that we can obtain a write lock and then release it.
+ */
+ @Test
+ public void writeLockTest() {
+ try (DatabaseLocker locker = new DatabaseLocker(this.dataSource(), true)) {
+ locker.lockDatabase("WriteLock", "WRITE_TEST_LOCK");
+ } catch (final Exception e) {
+ fail("Write lock should not have thrown an exception");
+ }
+ }
+
+ /**
+ * Simple test to make sure that we can obtain a read lock and then release it.
+ */
+ @Test
+ public void readLockTest() {
+ try (DatabaseLocker locker = new DatabaseLocker(this.dataSource(), false)) {
+ locker.lockDatabase("ReadLock", "READ_LOCK_TEST");
+ } catch (final Exception e) {
+ fail("Read lock should not have thrown an exception");
+ }
+ }
+
+ /**
+ * Test to show that if we try to obtain a write lock when there is a read lock that we will
+ * get an exception.
+ *
+ * @throws Exception if the datasource cannot be established
+ */
+ @Test
+ public void writeWithReadLockTest() throws Exception {
+ final DatabaseLocker writeLocker = new DatabaseLocker(this.dataSource(), true);
+ final DatabaseLocker readLocker = new DatabaseLocker(this.dataSource(), false);
+ readLocker.lockDatabase("ReadLock", "WRITE_WITH_READ_LOCK_TEST_1");
+ try {
+ writeLocker.lockDatabase("WriteLock", "WRITE_WITH_READ_LOCK_TEST_2");
+ fail("Write lock should have thrown an exception");
+ } catch (final Exception e) {
+ // we should expect an exception to be thrown
+ }
+ readLocker.unlockDatabase();
+ writeLocker.lockDatabase("WriteLock", "WRITE_WITH_READ_LOCK_TEST_3");
+ writeLocker.unlockDatabase();
+ }
+
+ /**
+ * Test to show that if we try to obtain a write lock when there is already a write lock that we
+ * will get an exception.
+ *
+ * @throws Exception if the datasource cannot be established
+ */
+ @Test
+ public void writeWithWriteLockTest() throws Exception {
+ final DatabaseLocker writeLocker1 = new DatabaseLocker(this.dataSource(), true);
+ final DatabaseLocker writeLocker2 = new DatabaseLocker(this.dataSource(), true);
+ writeLocker1.lockDatabase("WriteLock1", "WRITE_WITH_WRITE_LOCK_TEST_1");
+ try {
+ writeLocker2.lockDatabase("WriteLock2", "WRITE_WITH_WRITE_LOCK_TEST_2");
+ fail("Write lock should have thrown an exception");
+ } catch (final Exception e) {
+ // we should expect an exception to be thrown
+ }
+ writeLocker1.unlockDatabase();
+ writeLocker2.lockDatabase("WriteLock2", "WRITE_WITH_WRITE_LOCK_TEST_3");
+ writeLocker2.unlockDatabase();
+ }
+
+ /**
+ * Test to show that we can obtain multiple read locks.
+ *
+ * @throws Exception if the datasource cannot be established
+ */
+ @Test
+ public void readWithReadLockTest() throws Exception {
+ final DatabaseLocker[] lockers = new DatabaseLocker[10];
+ try {
+ for (int i = 0; i < 10; i++) {
+ lockers[i] = new DatabaseLocker(this.dataSource(), false);
+ lockers[i].lockDatabase("ReadLock" + i, "READ_WITH_READ_LOCK_TEST_" + i);
+ }
+ } finally {
+ // unlock all the databases
+ for (int i = 0; i < 10; i++) {
+ if (lockers[i] != null) {
+ lockers[i].unlockDatabase();
+ }
+ }
+ }
+ }
+
+ /**
+ * Test to show that if we try to obtain a read lock when there is already a write lock that we
+ * will get an exception.
+ *
+ * @throws Exception if the datasource cannot be established
+ */
+ @Test
+ public void readWithWriteLockTest() throws Exception {
+ final DatabaseLocker readLocker = new DatabaseLocker(this.dataSource(), false);
+ final DatabaseLocker writeLocker = new DatabaseLocker(this.dataSource(), true);
+ writeLocker.lockDatabase("WriteLock", "READ_WITH_WRITE_LOCK_TEST_1");
+ try {
+ readLocker.lockDatabase("ReadLock", "READ_WITH_WRITE_LOCK_TEST_2");
+ fail("Read lock should have thrown an exception");
+ } catch (final Exception e) {
+ // we should expect an exception to be thrown
+ }
+ writeLocker.unlockDatabase();
+ readLocker.lockDatabase("ReadLock", "READ_WITH_WRITE_LOCK_TEST_3");
+ readLocker.unlockDatabase();
+ }
+
+ private DataSource dataSource() throws Exception {
+ Class.forName("org.postgresql.Driver");
+ final DriverManagerDataSource dataSource = new DriverManagerDataSource();
+ dataSource.setDriverClassName("org.postgresql.Driver");
+ dataSource.setUrl("jdbc:postgresql://db:5432/pgosmsnap06_test");
+ dataSource.setUsername("osm");
+ dataSource.setPassword("password");
+ dataSource.setSchema("public");
+ return dataSource;
+ }
+}
=====================================
package/changes.txt
=====================================
@@ -1,3 +1,8 @@
+0.48.1
+Added advanced database logging for PgSnapshot module (#67)
+Updated usage docs (#68, #69)
+Added database locking functionality (#70)
+
0.48.0
Improve replication state handling with single-file steps and custom state (#63)
Add HTTP to osmosis-pbf2 module (#62)
=====================================
package/script/pgsnapshot_schema_0.6_changes.sql
=====================================
@@ -0,0 +1,95 @@
+DROP TABLE IF EXISTS replication_changes;
+
+-- Create a table for replication changes that are applied to the database.
+CREATE TABLE replication_changes (
+ id SERIAL,
+ tstamp TIMESTAMP without time zone NOT NULL DEFAULT(NOW()),
+ nodes_modified INT NOT NULL DEFAULT (0),
+ nodes_added INT NOT NULL DEFAULT (0),
+ nodes_deleted INT NOT NULL DEFAULT (0),
+ ways_modified INT NOT NULL DEFAULT (0),
+ ways_added INT NOT NULL DEFAULT (0),
+ ways_deleted INT NOT NULL DEFAULT (0),
+ relations_modified INT NOT NULL DEFAULT (0),
+ relations_added INT NOT NULL DEFAULT (0),
+ relations_deleted INT NOT NULL DEFAULT (0),
+ changesets_applied BIGINT [] NOT NULL,
+ earliest_timestamp TIMESTAMP without time zone NOT NULL,
+ latest_timestamp TIMESTAMP without time zone NOT NULL
+);
+
+ DROP TABLE IF EXISTS sql_changes;
+
+ CREATE TABLE sql_changes (
+ id SERIAL,
+ tstamp TIMESTAMP without time zone NOT NULL DEFAULT(NOW()),
+ entity_id BIGINT NOT NULL,
+ type TEXT NOT NULL,
+ changeset_id BIGINT NOT NULL,
+ change_time TIMESTAMP NOT NULL,
+ action INT NOT NULL,
+ query text NOT NULL,
+ arguments text
+);
+
+DROP TABLE IF EXISTS state;
+
+ CREATE TABLE state (
+ id SERIAL,
+ tstamp TIMESTAMP without time zone NOT NULL DEFAULT(NOW()),
+ sequence_number BIGINT NOT NULL,
+ state_timestamp TIMESTAMP WITHOUT time zone NOT NULL,
+ disabled BOOLEAN NOT NULL DEFAULT(false)
+);
+
+ DROP TABLE IF EXISTS locked;
+
+ CREATE TABLE locked (
+ id SERIAL,
+ started TIMESTAMP WITHOUT time zone NOT NULL DEFAULT(NOW()),
+ process TEXT NOT NULL,
+ source TEXT NOT NULL,
+ location TEXT NOT NULL,
+ write_lock BOOLEAN NOT NULL DEFAULT(false)
+);
+
+ DROP FUNCTION IF EXIST lock_database(TEXT, TEXT, TEXT);
+CREATE OR REPLACE FUNCTION lock_database(new_process TEXT, new_source TEXT, new_location TEXT, request_write_lock BOOLEAN) RETURNS INT AS $$
+ DECLARE locked_id INT;
+ DECLARE current_id INT;
+ DECLARE current_process TEXT;
+ DECLARE current_source TEXT;
+ DECLARE current_location TEXT;
+ DECLARE current_write_lock BOOLEAN;
+BEGIN
+
+ SELECT id, process, source, location, write_lock
+ INTO current_id, current_process, current_source, current_location, current_write_lock
+ FROM locked ORDER BY write_lock DESC NULLS LAST LIMIT 1;
+ IF (current_process IS NULL OR CHAR_LENGTH(current_process) = 0 OR (request_write_lock = FALSE AND current_write_lock = FALSE)) THEN
+ INSERT INTO locked (process, source, location, write_lock) VALUES (new_process, new_source, new_location, request_write_lock) RETURNING id INTO locked_id;
+ RETURN locked_id;
+ ELSE
+ RAISE EXCEPTION 'Database is locked by another id {%}, process {%}, source {%}, location {%}', current_id, current_process, current_source, current_location;
+ END IF;
+END;
+$$ LANGUAGE plpgsql VOLATILE;
+
+ CREATE OR REPLACE FUNCTION unlock_database(locked_id INT) RETURNS BOOLEAN AS $$
+ DECLARE response BOOLEAN;
+ DECLARE exist_count INT;
+BEGIN
+ IF (locked_id = -1) THEN
+ DELETE FROM locked;
+ RETURN true;
+ ELSE
+ SELECT COUNT(*) INTO exist_count FROM locked WHERE id = locked_id;
+ IF (exist_count = 1) THEN
+ DELETE FROM locked WHERE id = locked_id;
+ RETURN true;
+ ELSE
+ RETURN false;
+ END IF;
+ END IF;
+END;
+$$ LANGUAGE plpgsql VOLATILE;
View it on GitLab: https://salsa.debian.org/debian-gis-team/osmosis/-/compare/27b4561aa772daebb4f92654f4cef973b71cac0a...5eb0be28862ddae896273781e81d89a3544f6614
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/osmosis/-/compare/27b4561aa772daebb4f92654f4cef973b71cac0a...5eb0be28862ddae896273781e81d89a3544f6614
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20200706/8db1ccf5/attachment-0001.html>
More information about the Pkg-grass-devel
mailing list