[Git][debian-gis-team/osmpbf][upstream] New upstream version 1.8.0
Bas Couwenberg (@sebastic)
gitlab at salsa.debian.org
Thu Sep 17 22:27:40 BST 2026
Bas Couwenberg pushed to branch upstream at Debian GIS Project / osmpbf
Commits:
f87ad9db by Bas Couwenberg at 2026-09-17T14:07:15-07:00
New upstream version 1.8.0
- - - - -
8 changed files:
- .github/actions/build/action.yml
- .github/workflows/c.yml
- .github/workflows/java.yml
- CHANGELOG.md
- pom.xml
- + src.java/crosby/binary/file/ParallelBlockInputStream.java
- + test.java/crosby/binary/ParallelBlockInputStreamTest.java
- test.java/crosby/binary/ReadFileTest.java
Changes:
=====================================
.github/actions/build/action.yml
=====================================
@@ -19,6 +19,6 @@ runs:
working-directory: build
- name: Run outline
- run: build/tools/osmpbf-outline resources/sample.pbf
+ run: build/demo/osmpbf-outline resources/sample.pbf
shell: bash
=====================================
.github/workflows/c.yml
=====================================
@@ -11,7 +11,7 @@ jobs:
CXXFLAGS: -Werror -Wall -pedantic
BUILD_TYPE: Debug
steps:
- - uses: actions/checkout at v4
+ - uses: actions/checkout at v7
- uses: ./.github/actions/install-ubuntu
- uses: ./.github/actions/build
@@ -23,7 +23,19 @@ jobs:
CXXFLAGS: -Werror -Wall -pedantic
BUILD_TYPE: Debug
steps:
- - uses: actions/checkout at v4
+ - uses: actions/checkout at v7
+ - uses: ./.github/actions/install-ubuntu
+ - uses: ./.github/actions/build
+
+ ubuntu26:
+ runs-on: ubuntu-26.04
+ env:
+ CC: gcc
+ CXX: g++
+ CXXFLAGS: -Werror -Wall -pedantic
+ BUILD_TYPE: Debug
+ steps:
+ - uses: actions/checkout at v7
- uses: ./.github/actions/install-ubuntu
- uses: ./.github/actions/build
@@ -35,7 +47,7 @@ jobs:
CXXFLAGS: -Werror -Wall -pedantic
BUILD_TYPE: Debug
steps:
- - uses: actions/checkout at v4
+ - uses: actions/checkout at v7
- uses: ./.github/actions/install-ubuntu
- uses: ./.github/actions/build
@@ -48,7 +60,7 @@ jobs:
CXXFLAGS: -Werror -Wall -pedantic -Wno-array-bounds
BUILD_TYPE: Release
steps:
- - uses: actions/checkout at v4
+ - uses: actions/checkout at v7
- uses: ./.github/actions/install-ubuntu
- uses: ./.github/actions/build
@@ -60,14 +72,14 @@ jobs:
CXXFLAGS: -Werror -Wall -pedantic -Wno-nullability-extension
BUILD_TYPE: Debug
steps:
- - uses: actions/checkout at v4
+ - uses: actions/checkout at v7
- uses: ./.github/actions/install-macos
- uses: ./.github/actions/build
windows:
runs-on: windows-latest
steps:
- - uses: actions/checkout at v4
+ - uses: actions/checkout at v7
- uses: ./.github/actions/install-windows
- uses: ./.github/actions/build-windows
=====================================
.github/workflows/java.yml
=====================================
@@ -6,10 +6,10 @@ jobs:
maven-ubuntu22:
runs-on: ubuntu-22.04
steps:
- - uses: actions/checkout at v4
+ - uses: actions/checkout at v7
- run: sudo apt-get install protobuf-compiler
- name: Set up JDK 8
- uses: actions/setup-java at v4
+ uses: actions/setup-java at v5
with:
distribution: zulu
java-version: 8
@@ -19,10 +19,23 @@ jobs:
maven-ubuntu24:
runs-on: ubuntu-24.04
steps:
- - uses: actions/checkout at v4
+ - uses: actions/checkout at v7
- run: sudo apt-get install protobuf-compiler
- name: Set up JDK 8
- uses: actions/setup-java at v4
+ uses: actions/setup-java at v5
+ with:
+ distribution: zulu
+ java-version: 8
+ - name: Build with Maven
+ run: mvn -B package --file pom.xml
+
+ maven-ubuntu26:
+ runs-on: ubuntu-26.04
+ steps:
+ - uses: actions/checkout at v7
+ - run: sudo apt-get install protobuf-compiler
+ - name: Set up JDK 8
+ uses: actions/setup-java at v5
with:
distribution: zulu
java-version: 8
@@ -32,10 +45,10 @@ jobs:
ant-ubuntu22:
runs-on: ubuntu-22.04
steps:
- - uses: actions/checkout at v4
+ - uses: actions/checkout at v7
- run: sudo apt-get install protobuf-compiler libprotobuf-java
- name: Set up JDK 8
- uses: actions/setup-java at v4
+ uses: actions/setup-java at v5
with:
distribution: zulu
java-version: 8
@@ -45,10 +58,23 @@ jobs:
ant-ubuntu24:
runs-on: ubuntu-24.04
steps:
- - uses: actions/checkout at v4
+ - uses: actions/checkout at v7
+ - run: sudo apt-get install protobuf-compiler libprotobuf-java
+ - name: Set up JDK 8
+ uses: actions/setup-java at v5
+ with:
+ distribution: zulu
+ java-version: 8
+ - name: Build with Ant
+ run: ant
+
+ ant-ubuntu26:
+ runs-on: ubuntu-26.04
+ steps:
+ - uses: actions/checkout at v7
- run: sudo apt-get install protobuf-compiler libprotobuf-java
- name: Set up JDK 8
- uses: actions/setup-java at v4
+ uses: actions/setup-java at v5
with:
distribution: zulu
java-version: 8
=====================================
CHANGELOG.md
=====================================
@@ -1,5 +1,10 @@
## Unreleased
+## Release notes for 1.8.0 (2026-09-14)
+
+- Java: Upgrade protobuf version [#97](https://github.com/openstreetmap/OSM-binary/pull/97)
+- Java: Experimental parallel block decoding [#96](https://github.com/openstreetmap/OSM-binary/pull/96)
+
## Release notes for 1.7.0 (2026-07-26)
- C++: Fix buffer overflow in osmpbf-outline tool [#95](https://github.com/openstreetmap/OSM-binary/pull/95)
=====================================
pom.xml
=====================================
@@ -6,7 +6,7 @@
<groupId>org.openstreetmap.pbf</groupId>
<artifactId>osmpbf</artifactId>
<packaging>jar</packaging>
- <version>1.7.0</version>
+ <version>1.8.0</version>
<name>OSM-Binary</name>
<description>Library for the OpenStreetMap PBF format</description>
<url>https://github.com/openstreetmap/OSM-binary</url>
@@ -31,7 +31,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <protobuf.version>4.33.2</protobuf.version>
+ <protobuf.version>4.36.1</protobuf.version>
</properties>
<build>
=====================================
src.java/crosby/binary/file/ParallelBlockInputStream.java
=====================================
@@ -0,0 +1,177 @@
+/** Copyright (c) 2026 Leonard Ehrenfried. <mail at leonard.io>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as
+ published by the Free Software Foundation, either version 3 of the
+ License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+package crosby.binary.file;
+
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayDeque;
+import java.util.Queue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * A drop-in, parallel replacement for {@link BlockInputStream}.
+ *
+ * Reading a fileblock's header and raw bytes must stay strictly sequential,
+ * since it consumes an {@link InputStream}, but decompressing and parsing a
+ * block's contents (the CPU-heavy step, dominated by zlib inflate for large
+ * files) is independent per block. This class overlaps that work across a
+ * thread pool while still delivering {@code handleBlock} calls to the
+ * adaptor one at a time, in file order, matching what
+ * {@link BlockReaderAdapter#skipBlock} documents -- so an existing
+ * {@link BlockReaderAdapter} (such as a {@code crosby.binary.BinaryParser}
+ * subclass) needs no changes to benefit.
+ *
+ * Note: a block's header must be read -- and {@code skipBlock} called on it
+ * -- before its body can be read or skipped and the stream advanced past it,
+ * so {@code skipBlock} may be invoked for up to {@code pipelineDepth}
+ * non-skipped blocks ahead of the block whose {@code handleBlock} result was
+ * most recently delivered; a run of skipped blocks doesn't count against
+ * that limit (nothing is queued for them), so {@code skipBlock} can in fact
+ * run further ahead than {@code pipelineDepth} when the file contains long
+ * stretches of skipped blocks. An adaptor whose skip decision depends on
+ * mutable state updated inside {@code handleBlock} (rather than just the
+ * block's own type or metadata) will see stale state under this read-ahead;
+ * such an adaptor should use {@code pipelineDepth} 1, which restores fully
+ * sequential delivery at the cost of the parallelism benefit -- pass it via
+ * the 4-argument constructor, since the convenience constructor that takes a
+ * thread count derives {@code pipelineDepth} as {@code numThreads * 2} and
+ * cannot be used to request a depth of 1.
+ *
+ * Memory use is bounded by keeping at most {@code pipelineDepth} decompressed
+ * blocks in flight at once (queued or in progress); tune it down for memory
+ * constrained environments, or up to keep more threads fed on files with many
+ * small blocks.
+ */
+public class ParallelBlockInputStream implements Closeable {
+ public ParallelBlockInputStream(InputStream input, BlockReaderAdapter adaptor) {
+ this(input, adaptor, Runtime.getRuntime().availableProcessors());
+ }
+
+ public ParallelBlockInputStream(InputStream input, BlockReaderAdapter adaptor, int numThreads) {
+ this(input, adaptor, Executors.newFixedThreadPool(numThreads), true, numThreads * 2);
+ }
+
+ /**
+ * Use a caller-supplied executor, e.g. to share a thread pool across
+ * several files. The executor is not shut down by {@link #close()}.
+ */
+ public ParallelBlockInputStream(InputStream input, BlockReaderAdapter adaptor,
+ ExecutorService executor, int pipelineDepth) {
+ this(input, adaptor, executor, false, pipelineDepth);
+ }
+
+ private ParallelBlockInputStream(InputStream input, BlockReaderAdapter adaptor,
+ ExecutorService executor, boolean ownsExecutor, int pipelineDepth) {
+ this.input = input;
+ this.adaptor = adaptor;
+ this.executor = executor;
+ this.ownsExecutor = ownsExecutor;
+ this.pipelineDepth = Math.max(1, pipelineDepth);
+ }
+
+ public void process() throws IOException {
+ Queue<Future<FileBlock>> inflight = new ArrayDeque<>();
+ try {
+ while (true) {
+ try {
+ FileBlockHead head = FileBlockHead.readHead(input);
+ if (adaptor.skipBlock(head)) {
+ head.skipContents(input);
+ continue;
+ }
+ byte[] buf = new byte[head.getDatasize()];
+ new DataInputStream(input).readFully(buf);
+ inflight.add(submit(head, buf));
+ } catch (EOFException e) {
+ // Matches BlockInputStream: a clean end of stream, or a truncated
+ // header/body, both just end the read -- the trailing partial
+ // block (if any) is dropped, and anything already in flight is
+ // still delivered below.
+ break;
+ }
+
+ if (inflight.size() >= pipelineDepth) {
+ adaptor.handleBlock(take(inflight));
+ }
+ }
+ while (!inflight.isEmpty()) {
+ adaptor.handleBlock(take(inflight));
+ }
+ adaptor.complete();
+ } finally {
+ for (Future<FileBlock> future : inflight) {
+ future.cancel(true);
+ }
+ if (ownsExecutor) {
+ executor.shutdownNow();
+ }
+ }
+ }
+
+ private Future<FileBlock> submit(final FileBlockHead head, final byte[] buf) {
+ return executor.submit(new Callable<FileBlock>() {
+ @Override
+ public FileBlock call() throws IOException {
+ return head.parseData(buf);
+ }
+ });
+ }
+
+ private static FileBlock take(Queue<Future<FileBlock>> inflight) throws IOException {
+ Future<FileBlock> future = inflight.poll();
+ try {
+ return future.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof IOException) {
+ throw (IOException) cause;
+ }
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ }
+ throw new IOException(cause);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ input.close();
+ } finally {
+ if (ownsExecutor) {
+ executor.shutdownNow();
+ }
+ }
+ }
+
+ private final InputStream input;
+ private final BlockReaderAdapter adaptor;
+ private final ExecutorService executor;
+ private final boolean ownsExecutor;
+ private final int pipelineDepth;
+}
=====================================
test.java/crosby/binary/ParallelBlockInputStreamTest.java
=====================================
@@ -0,0 +1,111 @@
+package crosby.binary;
+
+import crosby.binary.file.BlockInputStream;
+import crosby.binary.file.ParallelBlockInputStream;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Verifies that ParallelBlockInputStream still delivers blocks to the adaptor
+ * one at a time, in file order, producing output identical to the sequential
+ * BlockInputStream reader (see ReadFileTest).
+ */
+public class ParallelBlockInputStreamTest {
+
+ @Test
+ public void testParallel() throws Exception {
+ try (InputStream input = ReadFileTest.class.getResourceAsStream("/sample.pbf");
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter printWriter = new PrintWriter(stringWriter);
+ ParallelBlockInputStream blockInput = new ParallelBlockInputStream(
+ input, new ReadFileTest.TestBinaryParser(printWriter), 4)) {
+ blockInput.process();
+ Assert.assertEquals(ReadFileTest.EXPECTED, stringWriter.toString());
+ }
+ }
+
+ /**
+ * sample.pbf only has 4 blocks, so the default pipelineDepth (2 *
+ * numThreads) never fills up mid-stream: every block ends up delivered
+ * from the final drain loop after EOF, never from the backpressure
+ * branch inside the read loop. Force a small pipelineDepth so that
+ * branch -- and the resulting handleBlock/skipBlock interleaving -- is
+ * actually exercised.
+ */
+ @Test
+ public void testParallelWithBackpressure() throws Exception {
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try (InputStream input = ReadFileTest.class.getResourceAsStream("/sample.pbf");
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter printWriter = new PrintWriter(stringWriter);
+ ParallelBlockInputStream blockInput = new ParallelBlockInputStream(
+ input, new ReadFileTest.TestBinaryParser(printWriter), executor, 2)) {
+ blockInput.process();
+ Assert.assertEquals(ReadFileTest.EXPECTED, stringWriter.toString());
+ } finally {
+ executor.shutdownNow();
+ executor.awaitTermination(5, TimeUnit.SECONDS);
+ }
+ }
+
+ /**
+ * A block body truncated mid-stream must be treated as a clean end of
+ * input -- same as BlockInputStream -- rather than aborting process()
+ * before complete() is called and dropping whatever was already
+ * in flight. Compares against BlockInputStream on the exact same bytes
+ * to pin the expected behavior instead of hard-coding it.
+ */
+ @Test
+ public void testTruncatedInputMatchesSequentialReader() throws Exception {
+ byte[] full = readAll(ReadFileTest.class.getResourceAsStream("/sample.pbf"));
+ byte[] truncated = new byte[full.length / 2];
+ System.arraycopy(full, 0, truncated, 0, truncated.length);
+
+ String sequential = runSequential(truncated);
+ String parallel = runParallel(truncated);
+
+ Assert.assertTrue("sequential reader should still complete", sequential.endsWith("Complete!" + System.lineSeparator()));
+ Assert.assertEquals(sequential, parallel);
+ }
+
+ private static String runSequential(byte[] bytes) throws IOException {
+ try (InputStream input = new ByteArrayInputStream(bytes);
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter printWriter = new PrintWriter(stringWriter)) {
+ new BlockInputStream(input, new ReadFileTest.TestBinaryParser(printWriter)).process();
+ return stringWriter.toString();
+ }
+ }
+
+ private static String runParallel(byte[] bytes) throws IOException {
+ try (InputStream input = new ByteArrayInputStream(bytes);
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter printWriter = new PrintWriter(stringWriter);
+ ParallelBlockInputStream blockInput = new ParallelBlockInputStream(
+ input, new ReadFileTest.TestBinaryParser(printWriter), 2)) {
+ blockInput.process();
+ return stringWriter.toString();
+ }
+ }
+
+ private static byte[] readAll(InputStream in) throws IOException {
+ try (InputStream input = in; ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+ byte[] buf = new byte[4096];
+ int n;
+ while ((n = input.read(buf)) >= 0) {
+ out.write(buf, 0, n);
+ }
+ return out.toByteArray();
+ }
+ }
+}
=====================================
test.java/crosby/binary/ReadFileTest.java
=====================================
@@ -19,9 +19,7 @@ import java.util.List;
*/
public class ReadFileTest {
- @Test
- public void test() throws Exception {
- String expected = ("" +
+ static final String EXPECTED = ("" +
"Got header block.\n" +
"Dense node, ID 653970877 @ 51.763603,-0.228757\n" +
"Dense node, ID 647105170 @ 51.763591,-0.234465\n" +
@@ -447,20 +445,23 @@ public class ReadFileTest {
" Key=value pairs: cycleway=track highway=cycleway name=Alban Way ncn_ref=61 \n" +
"Got some relations to parse.\n" +
"Complete!\n").replace("\n", System.lineSeparator());
+
+ @Test
+ public void test() throws Exception {
try (InputStream input = ReadFileTest.class.getResourceAsStream("/sample.pbf");
StringWriter stringWriter = new StringWriter();
PrintWriter printWriter = new PrintWriter(stringWriter)) {
BlockReaderAdapter brad = new TestBinaryParser(printWriter);
new BlockInputStream(input, brad).process();
- Assert.assertEquals(expected, stringWriter.toString());
+ Assert.assertEquals(EXPECTED, stringWriter.toString());
}
}
- private static class TestBinaryParser extends BinaryParser {
+ static class TestBinaryParser extends BinaryParser {
private final PrintWriter writer;
- private TestBinaryParser(PrintWriter writer) {
+ TestBinaryParser(PrintWriter writer) {
this.writer = writer;
}
View it on GitLab: https://salsa.debian.org/debian-gis-team/osmpbf/-/commit/f87ad9db394d1a8fbbf63355d1bd305ac4ff0532
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/osmpbf/-/commit/f87ad9db394d1a8fbbf63355d1bd305ac4ff0532
You're receiving this email because of your account on salsa.debian.org. Manage all notifications: https://salsa.debian.org/-/profile/notifications | Help: https://salsa.debian.org/help
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20260917/8f1dcacb/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list