[mkgmap-splitter] 01/05: Imported Upstream version 0.0.0+svn548
Bas Couwenberg
sebastic at debian.org
Sat Dec 31 23:50:30 UTC 2016
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository mkgmap-splitter.
commit 382eb5bf23ec256d6062d1a4a3a3f986cb2563d5
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Sun Jan 1 00:23:08 2017 +0100
Imported Upstream version 0.0.0+svn548
---
resources/splitter-version.properties | 4 +-
.../me/parabola/splitter/AbstractMapProcessor.java | 47 +-
src/uk/me/parabola/splitter/AreaDictionary.java | 186 +++++
src/uk/me/parabola/splitter/AreaDictionaryInt.java | 96 ---
.../me/parabola/splitter/AreaDictionaryShort.java | 212 ------
src/uk/me/parabola/splitter/AreaGrid.java | 60 +-
src/uk/me/parabola/splitter/AreaGridResult.java | 4 +-
src/uk/me/parabola/splitter/AreaList.java | 52 ++
src/uk/me/parabola/splitter/AreaSet.java | 199 +++++
src/uk/me/parabola/splitter/AreasCalculator.java | 422 -----------
src/uk/me/parabola/splitter/BinaryMapWriter.java | 536 --------------
src/uk/me/parabola/splitter/DataStorer.java | 360 +++++-----
src/uk/me/parabola/splitter/Element.java | 4 +-
src/uk/me/parabola/splitter/Main.java | 797 ++++++++------------
src/uk/me/parabola/splitter/MapProcessor.java | 15 +
src/uk/me/parabola/splitter/MapReader.java | 26 -
.../me/parabola/splitter/MultiTileProcessor.java | 161 ++---
src/uk/me/parabola/splitter/OSMFileHandler.java | 226 +++---
src/uk/me/parabola/splitter/OSMMessage.java | 46 ++
.../me/parabola/splitter/ProblemListProcessor.java | 128 ++--
src/uk/me/parabola/splitter/ProblemLists.java | 784 ++++++++++++++------
src/uk/me/parabola/splitter/QueueProcessor.java | 133 ++++
src/uk/me/parabola/splitter/Relation.java | 4 +-
src/uk/me/parabola/splitter/RoundingUtils.java | 2 +-
src/uk/me/parabola/splitter/SparseBitSet.java | 107 ---
.../me/parabola/splitter/SparseLong2ShortMap.java | 28 -
.../splitter/SparseLong2ShortMapFunction.java | 31 -
.../parabola/splitter/SparseLong2ShortMapHuge.java | 560 ---------------
.../splitter/SparseLong2ShortMapInline.java | 562 ---------------
.../me/parabola/splitter/SplitFailedException.java | 11 +-
src/uk/me/parabola/splitter/SplitProcessor.java | 247 +++----
src/uk/me/parabola/splitter/Utils.java | 4 +-
.../me/parabola/splitter/args/SplitterParams.java | 8 +-
.../me/parabola/splitter/{ => kml}/KmlParser.java | 8 +-
.../me/parabola/splitter/{ => kml}/KmlWriter.java | 7 +-
.../splitter/{ => parser}/BinaryMapParser.java | 133 ++--
.../splitter/{ => parser}/ElementCounter.java | 13 +-
.../splitter/{ => parser}/O5mMapParser.java | 511 +++++++------
.../{OSMParser.java => parser/OSMXMLParser.java} | 15 +-
.../parabola/splitter/solver/AreasCalculator.java | 237 ++++++
.../parabola/splitter/{ => solver}/DensityMap.java | 8 +-
.../splitter/{ => solver}/DensityMapCollector.java | 33 +-
.../splitter/{ => solver}/EnhancedDensityMap.java | 5 +-
.../splitter/{ => solver}/PolygonDesc.java | 22 +-
.../{ => solver}/PolygonDescProcessor.java | 57 +-
.../splitter/{ => solver}/PrecompSeaReader.java | 261 +++----
.../parabola/splitter/{ => solver}/Solution.java | 8 +-
.../{ => solver}/SplittableDensityArea.java | 141 ++--
src/uk/me/parabola/splitter/{ => solver}/Tile.java | 53 +-
.../splitter/{ => solver}/TileMetaInfo.java | 2 +-
.../splitter/{ => tools}/Long2IntClosedMap.java | 6 +-
.../{ => tools}/Long2IntClosedMapFunction.java | 4 +-
.../splitter/{ => tools}/OSMId2ObjectMap.java | 6 +-
.../me/parabola/splitter/tools/SparseBitSet.java | 105 +++
.../parabola/splitter/tools/SparseLong2IntMap.java | 798 +++++++++++++++++++++
.../splitter/{ => writer}/AbstractOSMWriter.java | 26 +-
.../parabola/splitter/writer/BinaryMapWriter.java | 519 ++++++++++++++
.../splitter/{ => writer}/O5mMapWriter.java | 11 +-
.../parabola/splitter/{ => writer}/OSMWriter.java | 10 +-
.../splitter/{ => writer}/OSMXMLWriter.java | 84 ++-
.../splitter/{ => writer}/PseudoOSMWriter.java | 7 +-
.../{ => xml/parser}/AbstractXppParser.java | 6 +-
test/uk/me/parabola/splitter/TestAreaSet.java | 85 +++
.../parabola/splitter/TestCustomCollections.java | 203 ------
test/uk/me/parabola/splitter/TestSparseBitSet.java | 56 --
.../splitter/tools/TestCustomCollections.java | 322 +++++++++
.../parabola/splitter/tools/TestSparseBitSet.java | 60 ++
67 files changed, 5036 insertions(+), 4848 deletions(-)
diff --git a/resources/splitter-version.properties b/resources/splitter-version.properties
index b55d073..036a607 100644
--- a/resources/splitter-version.properties
+++ b/resources/splitter-version.properties
@@ -1,2 +1,2 @@
-svn.version: 468
-build.timestamp: 2016-11-30T15:40:42+0000
+svn.version: 548
+build.timestamp: 2016-12-29T10:11:46+0000
diff --git a/src/uk/me/parabola/splitter/AbstractMapProcessor.java b/src/uk/me/parabola/splitter/AbstractMapProcessor.java
index 4f63800..5f6d449 100644
--- a/src/uk/me/parabola/splitter/AbstractMapProcessor.java
+++ b/src/uk/me/parabola/splitter/AbstractMapProcessor.java
@@ -13,12 +13,11 @@
package uk.me.parabola.splitter;
+import java.util.concurrent.BlockingQueue;
+
public abstract class AbstractMapProcessor implements MapProcessor {
- public static final short UNASSIGNED = Short.MIN_VALUE;
+ public static final int UNASSIGNED = Short.MIN_VALUE;
- public boolean isStartNodeOnly(){
- return false;
- }
public boolean skipTags(){
return false;
}
@@ -46,4 +45,44 @@ public abstract class AbstractMapProcessor implements MapProcessor {
public int getPhase() {
return 1;
}
+
+ public void startFile() {};
+
+ /**
+ * Simple method that allows all processors to use the producer/consumer pattern
+ */
+ public final boolean consume(BlockingQueue<OSMMessage> queue) {
+ while (true) {
+ try {
+ OSMMessage msg = queue.take();
+ switch (msg.type) {
+ case ELEMENTS:
+ for (Element el : msg.elements) {
+ if (el instanceof Node)
+ processNode((Node) el);
+ else if (el instanceof Way)
+ processWay((Way) el);
+ else if (el instanceof Relation)
+ processRelation((Relation) el);
+ }
+ break;
+ case BOUNDS:
+ boundTag(msg.bounds);
+ break;
+ case END_MAP:
+ return endMap();
+ case START_FILE:
+ startFile();
+ break;
+ case EXIT:
+ return true;
+ default:
+ break;
+ }
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
}
diff --git a/src/uk/me/parabola/splitter/AreaDictionary.java b/src/uk/me/parabola/splitter/AreaDictionary.java
new file mode 100644
index 0000000..b8609ce
--- /dev/null
+++ b/src/uk/me/parabola/splitter/AreaDictionary.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright (c) 2011,2012, Gerd Petermann
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+package uk.me.parabola.splitter;
+
+import java.awt.Rectangle;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+
+/**
+ * Maps a set containing the used areas to an int value.
+ * An OSM element is written to one or more areas. Every used
+ * combination of areas is translated to an int.
+ * @author Gerd Petermann
+ *
+ */
+public class AreaDictionary {
+ private static final int DICT_START = Short.MAX_VALUE;
+ private final Area[] areas;
+ private final ArrayList<AreaSet> sets;
+ private final int numOfAreas;
+ private final HashMap<AreaSet, Integer> index;
+ private final HashSet<AreaSet> simpleNeighbours = new HashSet<>();
+ private final int overlapAmount;
+
+ /**
+ * Create a dictionary for a given array of areas.
+ * @param overlapAmount
+ * @param areas the array of areas
+ */
+ AreaDictionary(List<Area> areas, int overlapAmount){
+ this.areas = areas.toArray(new Area[areas.size()]);
+ this.overlapAmount = overlapAmount;
+ this.numOfAreas = areas.size();
+ sets = new ArrayList<>();
+ index = new HashMap<>(areas.size() * 4, 0.5f);
+ init();
+ }
+
+ /**
+ * Initialize the dictionary with sets containing a single area.
+ */
+ private void init() {
+ ArrayList<Rectangle> rectangles = new ArrayList<>(numOfAreas);
+ ArrayList<AreaSet> areaSets = new ArrayList<>(numOfAreas);
+ for (int i = 0; i < numOfAreas; i++) {
+ AreaSet b = new AreaSet(i);
+ translate(b);
+ rectangles.add(Utils.area2Rectangle(areas[i], 0));
+ areaSets.add(b);
+ }
+ findSimpleNeigbours(rectangles, areaSets);
+ System.out.println("cached " + simpleNeighbours.size() + " combinations of areas that form rectangles.");
+ return;
+ }
+
+ /**
+ * Calculate the int value for a given AreaSet. The AreaSet must not
+ * contain values higher than numOfAreas.
+ * @param areaSet the AreaSet
+ * @return an Integer value that identifies this AreaSet, never null
+ */
+ public Integer translate(final AreaSet areaSet) {
+ Integer combiIndex = index.get(areaSet);
+ if (combiIndex == null) {
+ combiIndex = (sets.size() - DICT_START);
+ if (combiIndex == Integer.MAX_VALUE) {
+ throw new SplitFailedException("areaDictionary is full. Try to decrease number of areas.");
+ }
+ AreaSet set = new AreaSet(areaSet);
+ set.lock();
+ sets.add(set);
+ index.put(set, combiIndex);
+ if (sets.size() % 1000 == 0)
+ System.out.println("dictionary contains now " + Utils.format(sets.size()) + " entries");
+ }
+ return combiIndex;
+ }
+
+ /**
+ * Find those areas that build rectangles when they are
+ * added together. A way or relation that lies exactly within
+ * such a combination cannot cross other areas.
+ * @param rectangles
+ * @param areaSets
+ */
+ private void findSimpleNeigbours(ArrayList<Rectangle> rectangles, ArrayList<AreaSet> areaSets){
+ ArrayList<Rectangle> newRectangles = new ArrayList<>();
+ ArrayList<AreaSet> newAreaSets = new ArrayList<>();
+
+ for (int i = 0; i < rectangles.size(); i++) {
+ Rectangle r1 = rectangles.get(i);
+ for (int j = i + 1; j < rectangles.size(); j++) {
+ Rectangle r2 = rectangles.get(j);
+ boolean isSimple = false;
+ if (r1.y == r2.y && r1.height == r2.height && (r1.x == r2.getMaxX() || r2.x == r1.getMaxX()))
+ isSimple = true;
+ else if (r1.x == r2.x && r1.width == r2.width && (r1.y == r2.getMaxY() || r2.y == r1.getMaxY()))
+ isSimple = true;
+ if (isSimple) {
+ AreaSet simpleNeighbour = new AreaSet(areaSets.get(i));
+ simpleNeighbour.or(areaSets.get(j));
+ if (simpleNeighbour.cardinality() <= 10 && !simpleNeighbours.contains(simpleNeighbour)) {
+ simpleNeighbours.add(simpleNeighbour);
+ // System.out.println("simple neighbor: " +
+ // getMapIds(simpleNeighbour));
+ Rectangle pair = new Rectangle(r1);
+ pair.add(r2);
+ newRectangles.add(pair);
+ newAreaSets.add(simpleNeighbour);
+ }
+ }
+ }
+ }
+ if (!newRectangles.isEmpty()) {
+ rectangles.addAll(newRectangles);
+ areaSets.addAll(newAreaSets);
+ newRectangles = null;
+ newAreaSets = null;
+ if (simpleNeighbours.size() < 1000)
+ findSimpleNeigbours(rectangles, areaSets);
+ }
+ }
+
+ /**
+ * Return the AreaSet that is related to the int value.
+ * The caller must make sure that the index is valid.
+ * @param idx a value that was returned by the translate() method.
+ * @return the AreaSet
+ */
+ public AreaSet getSet(final int idx) {
+ return sets.get(DICT_START + idx);
+ }
+
+
+ /**
+ * return the number of sets in this dictionary
+ * @return the number of sets in this dictionary
+ */
+ public int size() {
+ return sets.size();
+ }
+
+ public int getNumOfAreas() {
+ return numOfAreas;
+ }
+
+ public boolean mayCross(AreaSet areaSet) {
+ return simpleNeighbours.contains(areaSet) == false;
+ }
+
+ public Area getArea(int idx) {
+ return areas[idx];
+ }
+
+ public Area getExtendedArea(int idx) {
+ Area bounds = areas[idx];
+ if (overlapAmount == 0)
+ return bounds;
+ return new Area(bounds.getMinLat() - overlapAmount,
+ bounds.getMinLong() - overlapAmount,
+ bounds.getMaxLat() + overlapAmount,
+ bounds.getMaxLong() + overlapAmount);
+ }
+
+ public List<Area> getAreas() {
+ return Collections.unmodifiableList(Arrays.asList(areas));
+ }
+
+ public static int translate(int singleWriterId) {
+ return (singleWriterId - DICT_START);
+ }
+}
diff --git a/src/uk/me/parabola/splitter/AreaDictionaryInt.java b/src/uk/me/parabola/splitter/AreaDictionaryInt.java
deleted file mode 100644
index ef0d350..0000000
--- a/src/uk/me/parabola/splitter/AreaDictionaryInt.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright (C) 2012, Gerd Petermann
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 or
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
- package uk.me.parabola.splitter;
-
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.HashMap;
-
-/**
- * Maps a BitSet containing the used areas to an integer value.
- * An OSM element is written to one or more areas. Every used
- * combination of areas is translated to an integer.
- * Use this dictionary if you expect many different area combinations,
- * e.g. for relations and their members.
- * @author Gerd Petermann
- *
- */
-public class AreaDictionaryInt{
- public final static int UNASSIGNED = -1;
- private final ArrayList<BitSet> sets;
- private final int numOfAreas;
- private final HashMap<BitSet, Integer> index;
-
- /**
- * Create a dictionary for a given array of areas
- * @param num the number of areas that are used
- */
- AreaDictionaryInt (int num){
- this.numOfAreas = num;
- sets = new ArrayList<>();
- index = new HashMap<>();
- init();
- }
-
- /**
- * initialize the dictionary with sets containing a single area.
- */
- private void init(){
- ArrayList<BitSet> areaSets = new ArrayList<>(numOfAreas);
- for (int i = 0; i < numOfAreas; i++) {
- BitSet b = new BitSet();
- b.set(i);
- translate(b);
- areaSets.add(b);
- }
- }
-
-
- /**
- * Calculate the integer value for a given BitSet. The BitSet must not
- * contain values higher than numOfAreas.
- * @param areaSet the BitSet
- * @return an int value that identifies this BitSet
- */
- public Integer translate(final BitSet areaSet){
- Integer combiIndex = index.get(areaSet);
- if (combiIndex == null){
- BitSet bnew = new BitSet();
-
- bnew.or(areaSet);
- combiIndex = sets.size();
- sets.add(bnew);
- index.put(bnew, combiIndex);
- }
- return combiIndex;
- }
-
- /**
- * Return the BitSet that is related to the int value.
- * The caller must make sure that the idx is valid.
- * @param idx an int value that was returned by the translate()
- * method.
- * @return the BitSet
- */
- public BitSet getBitSet (final int idx){
- return sets.get(idx);
- }
-
- /**
- * return the number of sets in this dictionary
- * @return the number of sets in this dictionary
- */
- public int size(){
- return sets.size();
- }
-}
diff --git a/src/uk/me/parabola/splitter/AreaDictionaryShort.java b/src/uk/me/parabola/splitter/AreaDictionaryShort.java
deleted file mode 100644
index c1de7cc..0000000
--- a/src/uk/me/parabola/splitter/AreaDictionaryShort.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Copyright (c) 2011,2012, Gerd Petermann
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-package uk.me.parabola.splitter;
-
-import it.unimi.dsi.fastutil.shorts.ShortArrayList;
-
-import java.awt.Rectangle;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-
-/**
- * Maps a BitSet containing the used areas to a short value.
- * An OSM element is written to one or more areas. Every used
- * combination of areas is translated to a short.
- * @author GerdP
- *
- */
-public class AreaDictionaryShort{
- private final static int DICT_START = Short.MAX_VALUE;
- private final Area[] areas;
- private final ArrayList<BitSet> sets;
- private final ArrayList<ShortArrayList> arrays;
- private final int numOfAreas;
- private final HashMap<BitSet, Short> index;
- private final HashSet<Short> simpleNeighbours = new HashSet<>();
- private final int overlapAmount;
-
- /**
- * Create a dictionary for a given array of areas
- * @param overlapAmount
- * @param areas the array of areas
- */
- AreaDictionaryShort (List<Area> areas, int overlapAmount){
- this.areas = areas.toArray(new Area[areas.size()]);
- this.overlapAmount = overlapAmount;
- this.numOfAreas = areas.size();
- sets = new ArrayList<>();
- arrays = new ArrayList<>();
- index = new HashMap<>();
- init();
- }
-
- /**
- * initialize the dictionary with sets containing a single area.
- */
- private void init(){
- ArrayList<Rectangle> rectangles = new ArrayList<>(numOfAreas);
- ArrayList<BitSet> areaSets = new ArrayList<>(numOfAreas);
- for (int i=0; i < numOfAreas; i++){
- BitSet b = new BitSet();
- b.set(i);
- translate(b);
- rectangles.add(Utils.area2Rectangle(areas[i], 0));
- areaSets.add(b);
- }
- findSimpleNeigbours(rectangles, areaSets);
- System.out.println("cached " + simpleNeighbours.size() + " combinations of areas that form rectangles.");
- return;
- }
-
- /**
- * Calculate the short value for a given BitSet. The BitSet must not
- * contain values higher than numOfAreas.
- * @param areaSet the BitSet
- * @return a short value that identifies this BitSet
- */
- public Short translate(final BitSet areaSet){
- Short combiIndex = index.get(areaSet);
- if (combiIndex == null){
- BitSet bnew = new BitSet();
-
- bnew.or(areaSet);
- ShortArrayList a = new ShortArrayList();
- for (int i = areaSet.nextSetBit(0); i >= 0; i = areaSet.nextSetBit(i + 1)) {
- a.add((short) i);
- }
- combiIndex = (short) (sets.size() - DICT_START);
- if (combiIndex == Short.MAX_VALUE){
- throw new SplitFailedException("areaDictionary is full. Decrease --max-areas value");
- }
- sets.add(bnew);
- arrays.add(a);
- index.put(bnew, combiIndex);
- }
- return combiIndex;
- }
-
- /**
- * find those areas that build rectangles when they are
- * added together. A way or relation that lies exactly within
- * such a combination cannot cross other areas.
- */
- private void findSimpleNeigbours(ArrayList<Rectangle> rectangles, ArrayList<BitSet> areaSets){
- ArrayList<Rectangle> newRectangles = new ArrayList<>();
- ArrayList<BitSet> newAreaSets = new ArrayList<>();
-
- for (int i = 0; i < rectangles.size(); i++){
- Rectangle r1 = rectangles.get(i);
- for (int j = i+1; j < rectangles.size(); j++){
- Rectangle r2 = rectangles.get(j);
- boolean isSimple = false;
- if (r1.y == r2.y && r1.height == r2.height
- && (r1.x == r2.getMaxX() || r2.x == r1.getMaxX()))
- isSimple = true;
- else if (r1.x == r2.x && r1.width == r2.width
- && (r1.y == r2.getMaxY() || r2.y == r1.getMaxY()))
- isSimple = true;
- if (isSimple){
- BitSet simpleNeighbour = new BitSet();
- simpleNeighbour.or(areaSets.get(i));
- simpleNeighbour.or(areaSets.get(j));
- if (simpleNeighbour.cardinality() <= 10){
- Short idx = translate(simpleNeighbour);
- if (simpleNeighbours.contains(idx) == false){
- simpleNeighbours.add(idx);
- //System.out.println("simple neighbor: " + getMapIds(simpleNeighbour));
- Rectangle pair = new Rectangle(r1);
- pair.add(r2);
- newRectangles.add(pair);
- newAreaSets.add(simpleNeighbour);
- }
- }
- }
- }
- }
- if (newRectangles.isEmpty() == false){
- rectangles.addAll(newRectangles);
- areaSets.addAll(newAreaSets);
- newRectangles = null;
- newAreaSets = null;
- if (simpleNeighbours.size() < 1000)
- findSimpleNeigbours(rectangles,areaSets);
- }
- }
- /**
- * Return the BitSet that is related to the short value.
- * The caller must make sure that the short is valid.
- * @param idx a short value that was returned by the translate()
- * method.
- * @return the BitSet
- */
- public BitSet getBitSet (final short idx){
- return sets.get(idx + DICT_START);
- }
-
- /**
- * Return a list containing the area ids for the given
- * short value.
- * @param idx a short value that was returned by the translate() method
- * @return a list containing the area ids
- */
- public ShortArrayList getList (final short idx){
- return arrays.get(DICT_START + idx);
- }
-
- /**
- * return the number of sets in this dictionary
- * @return the number of sets in this dictionary
- */
- public int size(){
- return sets.size();
- }
-
- public int getNumOfAreas(){
- return numOfAreas;
- }
-
- public boolean mayCross(short areaIdx){
- if (areaIdx + DICT_START < numOfAreas)
- return false;
- if (simpleNeighbours.contains(areaIdx))
- return false;
- return true;
- }
-
- public Area getArea(int idx) {
- return areas[idx];
- }
-
- public Area getExtendedArea(int idx) {
- Area bounds = areas[idx];
- if (overlapAmount == 0)
- return bounds;
- return new Area(bounds.getMinLat() - overlapAmount,
- bounds.getMinLong() - overlapAmount,
- bounds.getMaxLat() + overlapAmount,
- bounds.getMaxLong() + overlapAmount);
- }
-
- public List<Area> getAreas() {
- return Collections.unmodifiableList(Arrays.asList(areas));
- }
-
- public static short translate(short lastUsedWriter) {
- return (short) (lastUsedWriter - DICT_START);
- }
-}
diff --git a/src/uk/me/parabola/splitter/AreaGrid.java b/src/uk/me/parabola/splitter/AreaGrid.java
index 9a65aec..849f3cb 100644
--- a/src/uk/me/parabola/splitter/AreaGrid.java
+++ b/src/uk/me/parabola/splitter/AreaGrid.java
@@ -24,29 +24,22 @@ import java.util.BitSet;
*
*/
public class AreaGrid implements AreaIndex{
- private final Area bounds;
private final Grid grid;
protected final AreaGridResult r;
- protected final AreaDictionaryShort areaDictionary;
+ protected final AreaDictionary areaDictionary;
/**
* Create a grid to speed up the search of area candidates.
* @param areaDictionary
- * @param withOuter
*/
- AreaGrid(AreaDictionaryShort areaDictionary){
- this.areaDictionary = areaDictionary;
+ AreaGrid(AreaDictionary areaDictionary) {
+ this.areaDictionary = areaDictionary;
r = new AreaGridResult();
- long start = System.currentTimeMillis();
-
grid = new Grid(null, null);
- bounds = grid.getBounds();
-
- System.out.println("Grid(s) created in " + (System.currentTimeMillis() - start) + " ms");
}
public Area getBounds(){
- return bounds;
+ return grid.getBounds();
}
public AreaGridResult get (final Node n){
@@ -69,15 +62,15 @@ public class AreaGrid implements AreaIndex{
private int gridMinLat, gridMinLon;
// bounds of the complete grid
private Area bounds = null;
- private short [][] grid;
- private boolean [][] testGrid;
+ private int[][] indexGrid;
+ private BitSet[] testGrid;
private Grid[][] subGrid = null;
private final int maxCompares;
private int usedSubGridElems = 0;
private final int gridDimLon;
private final int gridDimLat;
- public Grid(BitSet usedAreas, Area bounds) {
+ public Grid(AreaSet usedAreas, Area bounds) {
// each element contains an index to the areaDictionary or unassigned
if (usedAreas == null){
gridDimLon = TOP_GRID_DIM_LON;
@@ -87,23 +80,26 @@ public class AreaGrid implements AreaIndex{
gridDimLon = SUB_GRID_DIM_LON;
gridDimLat = SUB_GRID_DIM_LAT;
}
- grid = new short[gridDimLon + 1][gridDimLat + 1];
+ indexGrid = new int[gridDimLon + 1][gridDimLat + 1];
// is true for an element if the list of areas needs to be tested
- testGrid = new boolean[gridDimLon + 1][gridDimLat + 1];
+ testGrid = new BitSet[gridDimLon + 1];
+ for (int lon = 0; lon < testGrid.length; lon++) {
+ testGrid[lon] = new BitSet(gridDimLat + 1);
+ }
this.bounds = bounds;
maxCompares = fillGrid(usedAreas);
}
+
public Area getBounds() {
return bounds;
}
+
/**
* Create the grid and fill each element
* @param usedAreas
- * @param testGrid
- * @param grid
- * @return
+ * @return maximum number of area tests needed for any returned GridResult
*/
- private int fillGrid(BitSet usedAreas) {
+ private int fillGrid(AreaSet usedAreas) {
int gridStepLon, gridStepLat;
if (bounds == null){
// calculate grid area
@@ -113,6 +109,8 @@ public class AreaGrid implements AreaIndex{
if (usedAreas == null || usedAreas.get(i))
tmpBounds = (tmpBounds ==null) ? extBounds : tmpBounds.add(extBounds);
}
+ if (tmpBounds == null)
+ return 0;
// create new Area to make sure that we don't update the existing area
bounds = new Area(tmpBounds.getMinLat() , tmpBounds.getMinLong(), tmpBounds.getMaxLat(), tmpBounds.getMaxLong());
}
@@ -130,8 +128,7 @@ public class AreaGrid implements AreaIndex{
assert gridStepLat * gridDimLat >= gridHeight : "gridStepLat is too small";
int maxAreaSearch = 0;
- BitSet areaSet = new BitSet();
- BitSet[][] gridAreas = new BitSet[gridDimLon+1][gridDimLat+1];
+ AreaSet[][] gridAreas = new AreaSet[gridDimLon+1][gridDimLat+1];
for (int j = 0; j < areaDictionary.getNumOfAreas(); j++) {
Area extBounds = areaDictionary.getExtendedArea(j);
@@ -151,24 +148,25 @@ public class AreaGrid implements AreaIndex{
for (int lat = startLat; lat <= endLat; lat++) {
int testMinLat = gridMinLat + gridStepLat * lat;
if (gridAreas[lon][lat]== null)
- gridAreas[lon][lat] = new BitSet();
+ gridAreas[lon][lat] = new AreaSet();
// add this area
gridAreas[lon][lat].set(j);
if (!extBounds.contains(testMinLat, testMinLon)
|| !extBounds.contains(testMinLat+ gridStepLat, testMinLon+ gridStepLon)){
// grid area is not completely within area
- testGrid[lon][lat] = true;
+ testGrid[lon].set(lat);
}
}
}
}
for (int lon = 0; lon <= gridDimLon; lon++) {
for (int lat = 0; lat <= gridDimLat; lat++) {
- areaSet = (gridAreas[lon][lat]);
+ AreaSet areaSet = (gridAreas[lon][lat]);
if (areaSet == null)
- grid[lon][lat] = AbstractMapProcessor.UNASSIGNED;
+ indexGrid[lon][lat] = AbstractMapProcessor.UNASSIGNED;
else {
- if (testGrid[lon][lat]){
+ areaSet.lock();
+ if (testGrid[lon].get(lat)){
int numTests = areaSet.cardinality();
if (numTests > MAX_TESTS){
if (gridStepLat > MIN_GRID_LAT && gridStepLon > MIN_GRID_LON){
@@ -188,7 +186,7 @@ public class AreaGrid implements AreaIndex{
}
maxAreaSearch = Math.max(maxAreaSearch, numTests);
}
- grid[lon][lat] = areaDictionary.translate(areaSet);
+ indexGrid[lon][lat] = areaDictionary.translate(areaSet);
}
}
}
@@ -226,11 +224,11 @@ public class AreaGrid implements AreaIndex{
}
}
// get list of area candidates from grid
- short idx = grid[gridLonIdx][gridLatIdx];
+ int idx = indexGrid[gridLonIdx][gridLatIdx];
if (idx == AbstractMapProcessor.UNASSIGNED)
return null;
- r.testNeeded = testGrid[gridLonIdx][gridLatIdx];
- r.l = areaDictionary.getList(idx);
+ r.testNeeded = testGrid[gridLonIdx].get(gridLatIdx);
+ r.set = areaDictionary.getSet(idx);
return r;
}
}
diff --git a/src/uk/me/parabola/splitter/AreaGridResult.java b/src/uk/me/parabola/splitter/AreaGridResult.java
index d4b396b..3f2c129 100644
--- a/src/uk/me/parabola/splitter/AreaGridResult.java
+++ b/src/uk/me/parabola/splitter/AreaGridResult.java
@@ -13,15 +13,13 @@
package uk.me.parabola.splitter;
-import it.unimi.dsi.fastutil.shorts.ShortArrayList;
-
/**
* A helper class to combine the results of the {@link AreaGrid}
* @author GerdP
*
*/
public class AreaGridResult{
- ShortArrayList l; // list of indexes to the area dictionary
+ AreaSet set; // set of indexes to the area dictionary
boolean testNeeded; // true: the list must be checked with the Area.contains() method
}
diff --git a/src/uk/me/parabola/splitter/AreaList.java b/src/uk/me/parabola/splitter/AreaList.java
index 64260c5..1874d30 100644
--- a/src/uk/me/parabola/splitter/AreaList.java
+++ b/src/uk/me/parabola/splitter/AreaList.java
@@ -14,6 +14,7 @@ package uk.me.parabola.splitter;
import java.awt.Point;
import java.io.BufferedReader;
+import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
@@ -35,6 +36,9 @@ import uk.me.parabola.splitter.geo.City;
import uk.me.parabola.splitter.geo.CityFinder;
import uk.me.parabola.splitter.geo.CityLoader;
import uk.me.parabola.splitter.geo.DefaultCityFinder;
+import uk.me.parabola.splitter.kml.KmlParser;
+import uk.me.parabola.splitter.kml.KmlWriter;
+import uk.me.parabola.splitter.solver.PolygonDesc;
/**
* A list of areas. It can be read and written to a file.
@@ -156,6 +160,18 @@ public class AreaList {
System.out.println(area.getMapId() + " " + area.toString());
}
}
+
+ public void dumpHex() {
+ System.out.println(areas.size() + " areas:");
+ for (Area area : areas) {
+ System.out.format("Area %08d: %d,%d to %d,%d covers %s", area.getMapId(), area.getMinLat(),
+ area.getMinLong(), area.getMaxLat(), area.getMaxLong(), area.toHexString());
+
+ if (area.getName() != null)
+ System.out.print(' ' + area.getName());
+ System.out.println();
+ }
+ }
/**
* Write out a poly file containing the bounding polygon for the areas
@@ -295,4 +311,40 @@ public class AreaList {
areas.clear();
areas.addAll(calculateAreas);
}
+
+ /**
+ *
+ * @param fileOutputDir
+ * @param polygons
+ * @param kmlOutputFile
+ * @param outputType
+ * @throws IOException
+ */
+ public void writeListFiles(File fileOutputDir, List<PolygonDesc> polygons,
+ String kmlOutputFile, String outputType) throws IOException {
+ for (PolygonDesc pd : polygons){
+ List<uk.me.parabola.splitter.Area> areasPart = new ArrayList<>();
+ for (uk.me.parabola.splitter.Area a : areas){
+ if (pd.getArea().intersects(a.getRect()))
+ areasPart.add(a);
+ }
+ if (kmlOutputFile != null){
+ File out = new File(kmlOutputFile);
+ String kmlOutputFilePart = pd.getName() + "-" + out.getName();
+ if (out.getParent() != null)
+ out = new File(out.getParent(), kmlOutputFilePart);
+ else
+ out = new File(kmlOutputFilePart);
+ if (out.getParent() == null)
+ out = new File(fileOutputDir, kmlOutputFilePart);
+ KmlWriter.writeKml(out.getPath(), areasPart);
+ }
+ AreaList al = new AreaList(areasPart, null);
+ al.setGeoNamesFile(geoNamesFile);
+ al.writePoly(new File(fileOutputDir, pd.getName() + "-" + "areas.poly").getPath());
+ al.writeArgsFile(new File(fileOutputDir, pd.getName() + "-" + "template.args").getPath(), outputType, pd.getMapId());
+ }
+ }
+
+
}
diff --git a/src/uk/me/parabola/splitter/AreaSet.java b/src/uk/me/parabola/splitter/AreaSet.java
new file mode 100644
index 0000000..c47b5e9
--- /dev/null
+++ b/src/uk/me/parabola/splitter/AreaSet.java
@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) 2016
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter;
+
+import java.util.Arrays;
+import java.util.Iterator;
+
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+
+/**
+ * A partly set implementation. Used as a replacement for BitSet which is slow when
+ * values are rather high, e.g. > 50000.
+ *
+ * @author Gerd Petermann
+ *
+ */
+public final class AreaSet implements Iterable<Integer> {
+ private static final int BIN_SEARCH_LIMIT = 10;
+ private final IntArrayList list;
+ private boolean locked;
+
+ /** Create empty set. */
+ public AreaSet() {
+ list = new IntArrayList();
+ }
+
+ /** Copy constructor creates set with the same entries.
+ * @param other set to clone
+ */
+ public AreaSet(final AreaSet other) {
+ if (!other.isEmpty()) {
+ list = new IntArrayList(other.list);
+ } else
+ list = new IntArrayList();
+ }
+
+ /**
+ * Create new set with one element.
+ * @param index the index of the element
+ */
+ AreaSet(final int index) {
+ list = new IntArrayList();
+ list.add(index);
+ }
+
+ /**
+ * Lock this set. A locked set cannot be changed.
+ */
+ public void lock() {
+ this.list.trim();
+ this.locked = true;
+ }
+
+ /**
+ * Returns true if the element with the index
+ * {@code bitIndex} is currently in this set; false otherwise.
+ *
+ * @param index the bit index
+ * @return the value of the bit with the specified index
+ */
+ public boolean get(final int index) {
+ if (list.size() < BIN_SEARCH_LIMIT) {
+ return list.contains(index);
+ }
+ return Arrays.binarySearch(list.elements(), 0, list.size(), index) >= 0;
+ }
+
+ /**
+ * Add the element to the set. No effect if index is already in the set.
+ * @param index the element
+ */
+ public void set(final int index) {
+ if (locked)
+ throw new IllegalAccessError("AreaSet is locked");
+ if (list.isEmpty()) {
+ list.add(index);
+ } else {
+ int p = Arrays.binarySearch(list.elements(), 0, list.size(), index);
+ if (p < 0) {
+ list.add(-p - 1, index);
+ }
+ }
+ }
+
+ /**
+ * Remove the element from the set.
+ * @param index the element
+ */
+ public void clear(final int index) {
+ if (locked)
+ throw new IllegalAccessError("AreaSet is locked");
+ int pos;
+ if (list.size() < BIN_SEARCH_LIMIT) {
+ list.rem(index);
+ } else {
+ pos = Arrays.binarySearch(list.elements(), 0, list.size(), index);
+ if (pos >= 0) {
+ list.removeInt(pos);
+ }
+ }
+ }
+
+ /**
+ * Merge with other set. Result contains elements of both sets.
+ * @param other the other set
+ */
+ void or(final AreaSet other) {
+ if (locked)
+ throw new IllegalAccessError("AreaSet is locked");
+ if (other.isEmpty())
+ return;
+ if (list.isEmpty()) {
+ list.addAll(other.list);
+ } else {
+ for (int i : other.list) {
+ set(i);
+ }
+ }
+ }
+
+ /**
+ * Remove elements in this set which are contained in the other set.
+ * @param other the other set
+ */
+ public void subtract(final AreaSet other) {
+ if (locked)
+ throw new IllegalAccessError("AreaSet is locked");
+ for (int i : other.list) {
+ clear(i);
+ }
+ }
+
+ /**
+ * @return number of elements in this set
+ */
+ public int cardinality() {
+ return list.size();
+ }
+
+ /**
+ * @return true if this set contains no elements.
+ */
+ public boolean isEmpty() {
+ return cardinality() == 0;
+ }
+
+ /**
+ * remove all elements from the set. Doesn't free storage.
+ */
+ public void clear() {
+ if (locked)
+ throw new IllegalAccessError("AreaSet is locked");
+ list.clear();
+ }
+
+
+ /**
+ * @return an iterator over this set.
+ */
+ @Override
+ public Iterator<Integer> iterator() {
+ return list.iterator();
+
+ }
+
+ @Override
+ public int hashCode() {
+ return list.hashCode();
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (!(obj instanceof AreaSet))
+ return false;
+ if (this == obj)
+ return true;
+ AreaSet other = (AreaSet) obj;
+ if (isEmpty() && other.isEmpty())
+ return true;
+ return list.equals(other.list);
+ }
+
+ @Override
+ public String toString() {
+ return list.toString();
+ }
+}
+
diff --git a/src/uk/me/parabola/splitter/AreasCalculator.java b/src/uk/me/parabola/splitter/AreasCalculator.java
deleted file mode 100644
index 1c011b6..0000000
--- a/src/uk/me/parabola/splitter/AreasCalculator.java
+++ /dev/null
@@ -1,422 +0,0 @@
-package uk.me.parabola.splitter;
-
-import java.awt.Point;
-import java.awt.Rectangle;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-import org.openstreetmap.osmosis.core.filter.common.PolygonFileReader;
-
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-
-public class AreasCalculator {
- private final List<PolygonDesc> polygons = new ArrayList<>();
- private int resolution = 13;
- private PolygonDescProcessor polygonDescProcessor;
-
- public AreasCalculator() {
- }
-
-
- public void setResolution(int resolution) {
- this.resolution = resolution;
- }
-
-
- /**
- * Check if the bounding polygons are usable.
- * @param polygon
- * @return
- */
- public boolean checkPolygons() {
- for (PolygonDesc pd : polygons){
- if (checkPolygon(pd.area) == false)
- return false;
- }
- return true;
- }
-
-
- /**
- * Check if the bounding polygon is usable.
- * @param polygon
- * @return
- */
- private boolean checkPolygon(java.awt.geom.Area mapPolygonArea) {
- List<List<Point>> shapes = Utils.areaToShapes(mapPolygonArea);
- int shift = 24 - resolution;
- long rectangleWidth = 1L << shift;
- for (List<Point> shape: shapes){
- int estimatedPoints = 0;
- Point p1 = shape.get(0);
- for (int i = 1; i < shape.size(); i++){
- Point p2 = shape.get(i);
- if (p1.x != p2.x && p1.y != p2.y){
- // diagonal line
- int width = Math.abs(p1.x-p2.x);
- int height = Math.abs(p1.y-p2.y);
- estimatedPoints += (Math.min(width, height) / rectangleWidth) * 2;
- }
-
- if (estimatedPoints > SplittableDensityArea.MAX_SINGLE_POLYGON_VERTICES)
- return false; // too complex
-
- p1 = p2;
- }
- }
- return true;
- }
-
- public void readPolygonFile(String polygonFile, int mapId) {
- polygons.clear();
- File f = new File(polygonFile);
-
- if (!f.exists()){
- throw new IllegalArgumentException("polygon file doesn't exist: " + polygonFile);
- }
- PolygonFileReader polyReader = new PolygonFileReader(f);
- java.awt.geom.Area polygonInDegrees = polyReader.loadPolygon();
- PolygonDesc pd = new PolygonDesc(polyReader.getPolygonName(),
- Utils.AreaDegreesToMapUnit(polygonInDegrees),
- mapId);
- polygons.add(pd);
- }
-
-
- public void readPolygonDescFile(String polygonDescFile) {
- polygons.clear();
- File f = new File(polygonDescFile);
-
- if (!f.exists()){
- System.out.println("Error: polygon desc file doesn't exist: " + polygonDescFile);
- System.exit(-1);
- }
- polygonDescProcessor = new PolygonDescProcessor(resolution);
- OSMFileHandler polyDescHandler = new OSMFileHandler();
- polyDescHandler.setFileNames(Arrays.asList(polygonDescFile));
- polyDescHandler.setMixed(false);
- polyDescHandler.process(polygonDescProcessor);
- polygons.addAll(polygonDescProcessor.getPolygons());
- }
-
-
- public void writeListFiles(File outputDir, List<Area> areas, String kmlOutputFile,
- String outputType) throws IOException {
- if (polygonDescProcessor != null)
- polygonDescProcessor.writeListFiles(outputDir, areas, kmlOutputFile, outputType);
- }
-
-
- public List<PolygonDesc> getPolygons() {
- return Collections.unmodifiableList(polygons);
- }
- /**
- * Make sure that our areas cover the planet. This is done by adding
- * pseudo-areas if needed.
- * @param realAreas list of areas (read from split-file or calculated)
- * @return new list of areas containing the real areas and additional areas
- */
- public static List<Area> addPseudoAreas(List<Area> realAreas){
- ArrayList<Area> areas = new ArrayList<>(realAreas);
- Rectangle planetBounds = new Rectangle(Utils.toMapUnit(-180.0), Utils.toMapUnit(-90.0), 2* Utils.toMapUnit(180.0), 2 * Utils.toMapUnit(90.0));
-
- while (!checkIfCovered(planetBounds, areas)){
- boolean changed = addPseudoArea(areas);
-
- if (!changed){
- throw new SplitFailedException("Failed to fill planet with pseudo-areas");
- }
- }
- return areas;
- }
- /**
- * Work around for possible rounding errors in area.subtract processing
- * @param area an area that is considered to be empty or a rectangle
- * @return
- */
- private static java.awt.geom.Area simplifyArea(java.awt.geom.Area area) {
- if (area.isEmpty() || area.isRectangular())
- return area;
- // area.isRectugular() may returns false although the shape is a
- // perfect rectangle :-( If we subtract the area from its bounding
- // box we get better results.
- java.awt.geom.Area bbox = new java.awt.geom.Area (area.getBounds2D());
- bbox.subtract(area);
- if (bbox.isEmpty()) // bbox equals area: is a rectangle
- return new java.awt.geom.Area (area.getBounds2D());
- return area;
- }
-
- private static boolean checkIfCovered(Rectangle bounds, ArrayList<Area> areas){
- java.awt.geom.Area bbox = new java.awt.geom.Area(bounds);
- long sumTiles = 0;
-
- for (Area area: areas){
- sumTiles += (long)area.getHeight() * (long)area.getWidth();
- bbox.subtract(area.getJavaArea());
- }
- long areaBox = (long) bounds.height*(long)bounds.width;
-
- if (sumTiles != areaBox)
- return false;
-
- return bbox.isEmpty();
- }
-
- /**
- * Create a list of areas that do not overlap. If areas in the original
- * list are overlapping, they can be replaced by up to 5 disjoint areas.
- * This is done if parameter makeDisjoint is true
- * @param realAreas the list of areas
- * @return the new list
- */
- public static ArrayList<Area> getNonOverlappingAreas(final List<Area> realAreas){
- java.awt.geom.Area covered = new java.awt.geom.Area();
- ArrayList<Area> splitList = new ArrayList<>();
- int artificialId = -99999999;
- boolean foundOverlap = false;
- for (Area area1 : realAreas) {
- Rectangle r1 = area1.getRect();
- if (covered.intersects(r1) == false){
- splitList.add(area1);
- }
- else {
- if (foundOverlap == false){
- foundOverlap = true;
- System.out.println("Removing overlaps from tiles...");
- }
- //String msg = "splitting " + area1.getMapId() + " " + (i+1) + "/" + realAreas.size() + " overlapping ";
- // find intersecting areas in the already covered part
- ArrayList<Area> splitAreas = new ArrayList<>();
-
- for (int j = 0; j < splitList.size(); j++){
- Area area2 = splitList.get(j);
- if (area2 == null)
- continue;
- Rectangle r2 = area2.getRect();
- if (r1.intersects(r2)){
- java.awt.geom.Area overlap = new java.awt.geom.Area(area1.getRect());
- overlap.intersect(area2.getJavaArea());
- Rectangle ro = overlap.getBounds();
- if (ro.height == 0 || ro.width == 0)
- continue;
- //msg += area2.getMapId() + " ";
- Area aNew = new Area(ro.y, ro.x, (int)ro.getMaxY(),(int)ro.getMaxX());
- aNew.setMapId(artificialId++);
- aNew.setName("" + area1.getMapId());
- aNew.setJoinable(false);
- covered.subtract(area2.getJavaArea());
- covered.add(overlap);
- splitList.set(j, aNew);
-
- java.awt.geom.Area coveredByPair = new java.awt.geom.Area(r1);
- coveredByPair.add(new java.awt.geom.Area(r2));
-
- java.awt.geom.Area originalPair = new java.awt.geom.Area(coveredByPair);
-
- int minX = coveredByPair.getBounds().x;
- int minY = coveredByPair.getBounds().y;
- int maxX = (int) coveredByPair.getBounds().getMaxX();
- int maxY = (int) coveredByPair.getBounds().getMaxY();
- coveredByPair.subtract(overlap);
- if (coveredByPair.isEmpty())
- continue; // two equal areas a
-
- coveredByPair.subtract(covered);
- java.awt.geom.Area testSplit = new java.awt.geom.Area(overlap);
-
- Rectangle[] rectPair = {r1,r2};
- Area[] areaPair = {area1,area2};
- int lx = minX;
- int lw = ro.x-minX;
- int rx = (int)ro.getMaxX();
- int rw = maxX - rx;
- int uy = (int)ro.getMaxY();
- int uh = maxY - uy;
- int by = minY;
- int bh = ro.y - by;
- Rectangle[] clippers = {
- new Rectangle(lx, minY, lw, bh), // lower left
- new Rectangle(ro.x, minY, ro.width, bh), // lower middle
- new Rectangle(rx, minY, rw, bh), // lower right
- new Rectangle(lx, ro.y, lw, ro.height), // left
- new Rectangle(rx, ro.y, rw, ro.height), // right
- new Rectangle(lx, uy, lw, uh), // upper left
- new Rectangle(ro.x, uy, ro.width, uh), // upper middle
- new Rectangle(rx, uy, rw, uh) // upper right
- };
-
- for (Rectangle clipper: clippers){
- for (int k = 0; k <= 1; k++){
- Rectangle test = clipper.intersection(rectPair[k]);
- if (!test.isEmpty()){
- testSplit.add(new java.awt.geom.Area(test));
- if (k==1 || covered.intersects(test) == false){
- aNew = new Area(test.y,test.x,(int)test.getMaxY(),(int)test.getMaxX());
- aNew.setMapId(areaPair[k].getMapId());
- splitAreas.add(aNew);
- covered.add(aNew.getJavaArea());
- }
- }
- }
- }
- assert testSplit.equals(originalPair);
- }
- }
-
- // recombine parts that form a rectangle
- for (Area splitArea: splitAreas){
- if (splitArea.isJoinable()){
- for (int j = 0; j < splitList.size(); j++){
- Area area = splitList.get(j);
- if (area == null || area.isJoinable() == false || area.getMapId() != splitArea.getMapId() )
- continue;
- boolean doJoin = false;
- if (splitArea.getMaxLat() == area.getMaxLat()
- && splitArea.getMinLat() == area.getMinLat()
- && (splitArea.getMinLong() == area.getMaxLong() || splitArea.getMaxLong() == area.getMinLong()))
- doJoin = true;
- else if (splitArea.getMinLong() == area.getMinLong()
- && splitArea.getMaxLong()== area.getMaxLong()
- && (splitArea.getMinLat() == area.getMaxLat() || splitArea.getMaxLat() == area.getMinLat()))
- doJoin = true;
- if (doJoin){
- splitArea = area.add(splitArea);
- splitArea.setMapId(area.getMapId());
- splitList.set(j, splitArea);
- splitArea = null; // don't add later
- break;
- }
- }
- }
- if (splitArea != null){
- splitList.add(splitArea);
- }
- }
- /*
- if (msg.isEmpty() == false)
- System.out.println(msg);
- */
- }
- covered.add(new java.awt.geom.Area(r1));
- }
- covered.reset();
- Iterator <Area> iter = splitList.iterator();
- while (iter.hasNext()){
- Area a = iter.next();
- if (a == null)
- iter.remove();
- else {
- Rectangle r1 = a.getRect();
- if (covered.intersects(r1) == true){
- throw new SplitFailedException("Failed to create list of distinct areas");
- }
- covered.add(a.getJavaArea());
- }
- }
- return splitList;
- }
-
- /**
- * Fill uncovered parts of the planet with pseudo-areas.
- * TODO: check if better algorithm reduces run time in ProblemListProcessor
- * We want a small number of pseudo areas because many of them will
- * require more memory or more passes, esp. when processing whole planet.
- * Also, the total length of all edges should be small.
- * @param areas list of areas (either real or pseudo)
- * @return true if pseudo-areas were added
- */
- private static boolean addPseudoArea(ArrayList<Area> areas) {
- int oldSize = areas.size();
- Rectangle planetBounds = new Rectangle(Utils.toMapUnit(-180.0), Utils.toMapUnit(-90.0), 2* Utils.toMapUnit(180.0), 2 * Utils.toMapUnit(90.0));
- java.awt.geom.Area uncovered = new java.awt.geom.Area(planetBounds);
- java.awt.geom.Area covered = new java.awt.geom.Area();
- for (Area area: areas){
- uncovered.subtract(area.getJavaArea());
- covered.add(area.getJavaArea());
- }
- Rectangle rCov = covered.getBounds();
- Rectangle[] topAndBottom = {
- new Rectangle(planetBounds.x,(int)rCov.getMaxY(),planetBounds.width, (int)(planetBounds.getMaxY()-rCov.getMaxY())), // top
- new Rectangle(planetBounds.x,planetBounds.y,planetBounds.width,rCov.y-planetBounds.y)}; // bottom
- for (Rectangle border: topAndBottom){
- if (!border.isEmpty()){
- uncovered.subtract(new java.awt.geom.Area(border));
- covered.add(new java.awt.geom.Area(border));
- Area pseudo = new Area(border.y,border.x,(int)border.getMaxY(),(int)border.getMaxX());
- pseudo.setMapId(-1 * (areas.size()+1));
- pseudo.setPseudoArea(true);
- areas.add(pseudo);
- }
- }
- while (uncovered.isEmpty() == false){
- boolean changed = false;
- List<List<Point>> shapes = Utils.areaToShapes(uncovered);
- // we divide planet into stripes for all vertices of the uncovered area
- int minX = uncovered.getBounds().x;
- int nextX = Integer.MAX_VALUE;
- for (int i = 0; i < shapes.size(); i++){
- List<Point> shape = shapes.get(i);
- for (Point point: shape){
- int lon = point.x;
- if (lon < nextX && lon > minX)
- nextX = lon;
- }
- }
- java.awt.geom.Area stripeLon = new java.awt.geom.Area(new Rectangle(minX, planetBounds.y, nextX - minX, planetBounds.height));
- // cut out already covered area
- stripeLon.subtract(covered);
- assert stripeLon.isEmpty() == false;
- // the remaining area must be a set of zero or more disjoint rectangles
- List<List<Point>> stripeShapes = Utils.areaToShapes(stripeLon);
- for (int j = 0; j < stripeShapes .size(); j++){
- List<Point> rectShape = stripeShapes .get(j);
- java.awt.geom.Area test = Utils.shapeToArea(rectShape);
- test = simplifyArea(test);
- assert test.isRectangular();
- Rectangle pseudoRect = test.getBounds();
- if (uncovered.contains(pseudoRect)){
- assert test.getBounds().width == stripeLon.getBounds().width;
- boolean wasMerged = false;
- // check if new area can be merged with last rectangles
- for (int k=areas.size()-1; k >= oldSize; k--){
- Area prev = areas.get(k);
- if (prev.getMaxLong() < pseudoRect.x || prev.isPseudoArea() == false)
- continue;
- if (prev.getHeight() == pseudoRect.height && prev.getMaxLong() == pseudoRect.x && prev.getMinLat() == pseudoRect.y){
- // merge
- Area pseudo = prev.add(new Area(pseudoRect.y,pseudoRect.x,(int)pseudoRect.getMaxY(),(int)pseudoRect.getMaxX()));
- pseudo.setMapId(prev.getMapId());
- pseudo.setPseudoArea(true);
- areas.set(k, pseudo);
- //System.out.println("Enlarged pseudo area " + pseudo.getMapId() + " " + pseudo);
- wasMerged = true;
- break;
- }
- }
-
- if (!wasMerged){
- Area pseudo = new Area(pseudoRect.y, pseudoRect.x, (int)pseudoRect.getMaxY(), (int)pseudoRect.getMaxX());
- pseudo.setMapId(-1 * (areas.size()+1));
- pseudo.setPseudoArea(true);
- //System.out.println("Adding pseudo area " + pseudo.getMapId() + " " + pseudo);
- areas.add(pseudo);
- }
- uncovered.subtract(test);
- covered.add(test);
- changed = true;
- }
- }
- if (!changed)
- break;
- }
- return oldSize != areas.size();
- }
-
-
-}
diff --git a/src/uk/me/parabola/splitter/BinaryMapWriter.java b/src/uk/me/parabola/splitter/BinaryMapWriter.java
deleted file mode 100644
index b87390b..0000000
--- a/src/uk/me/parabola/splitter/BinaryMapWriter.java
+++ /dev/null
@@ -1,536 +0,0 @@
-/*
- * Copyright (c) 2009, Francisco Moraes
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-
-package uk.me.parabola.splitter;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-
-import uk.me.parabola.splitter.Relation.Member;
-import crosby.binary.BinarySerializer;
-import crosby.binary.Osmformat;
-import crosby.binary.StringTable;
-import crosby.binary.Osmformat.DenseInfo;
-import crosby.binary.Osmformat.Relation.MemberType;
-import crosby.binary.file.BlockOutputStream;
-import crosby.binary.file.FileBlock;
-
-public class BinaryMapWriter extends AbstractOSMWriter {
-
- protected PBFSerializer serializer;
-
- private BlockOutputStream output;
-
- protected boolean useDense = true;
-
- protected boolean headerWritten = false;
-
- private class PBFSerializer extends BinarySerializer {
-
- public PBFSerializer(BlockOutputStream output)
- {
- super(output);
- configBatchLimit(1000);
-// omit_metadata = true;
- }
-
- /** Base class containing common code needed for serializing each type of primitives. */
- private abstract class Prim<T extends Element> {
- /** Queue that tracks the list of all primitives. */
- ArrayList<T> contents = new ArrayList<>();
-
- /** Add to the queue.
- * @param item The entity to add */
- public void add(T item)
- {
- contents.add(item);
- }
-
- /** Add all of the tags of all entities in the queue to the string table. */
- public void addStringsToStringtable()
- {
- StringTable stable = getStringTable();
- for(T i : contents) {
- Iterator<Element.Tag> tags = i.tagsIterator();
- while(tags.hasNext()) {
- Element.Tag tag = tags.next();
- stable.incr(tag.getKey());
- stable.incr(tag.getValue());
- }
- if(!omit_metadata) {
- // stable.incr(i.getUser().getName());
- }
- }
- }
-
- // private static final int MAXWARN = 100;
-
- public void serializeMetadataDense(DenseInfo.Builder b,
- List<? extends Element> entities)
- {
- if(omit_metadata) {
- return;
- }
-
- // long lasttimestamp = 0, lastchangeset = 0;
- // int lastuserSid = 0, lastuid = 0;
- // StringTable stable = serializer.getStringTable();
- // for(Element e : entities) {
- //
- // if(e.getUser() == OsmUser.NONE && warncount < MAXWARN) {
- // LOG
- // .warning("Attention: Data being output lacks metadata. Please use omitmetadata=true");
- // warncount++;
- // }
- // int uid = e.getUser().getId();
- // int userSid = stable.getIndex(e.getUser().getName());
- // int timestamp = (int)(e.getTimestamp().getTime() / date_granularity);
- // int version = e.getVersion();
- // long changeset = e.getChangesetId();
- //
- // b.addVersion(version);
- // b.addTimestamp(timestamp - lasttimestamp);
- // lasttimestamp = timestamp;
- // b.addChangeset(changeset - lastchangeset);
- // lastchangeset = changeset;
- // b.addUid(uid - lastuid);
- // lastuid = uid;
- // b.addUserSid(userSid - lastuserSid);
- // lastuserSid = userSid;
- // }
-
- for(Element e : entities) {
- int version = getWriteVersion(e);
- if (versionMethod != KEEP_VERSION || version == 0)
- version = 1; // JOSM requires a fake version
- b.addVersion(version);
- b.addTimestamp(0);
- b.addChangeset(0);
- b.addUid(0);
- b.addUserSid(0);
- }
- }
-
- public Osmformat.Info.Builder serializeMetadata(Element e)
- {
- // StringTable stable = serializer.getStringTable();
- Osmformat.Info.Builder b = Osmformat.Info.newBuilder();
-// if(!omit_metadata) {
- // if(e.getUser() == OsmUser.NONE && warncount < MAXWARN) {
- // LOG
- // .warning("Attention: Data being output lacks metadata. Please use omitmetadata=true");
- // warncount++;
- // }
- // if(e.getUser() != OsmUser.NONE) {
- // b.setUid(e.getUser().getId());
- // b.setUserSid(stable.getIndex(e.getUser().getName()));
- // }
- // b.setTimestamp((int)(e.getTimestamp().getTime() / date_granularity));
- // b.setVersion(e.getVersion());
- // b.setChangeset(e.getChangesetId());
-// }
- if (versionMethod != REMOVE_VERSION){
- int version = getWriteVersion(e);
- b.setVersion(version);
- b.setTimestamp(0);
- b.setChangeset(0);
- b.setUid(0);
- b.setUserSid(0);
- }
- return b;
- }
- }
-
- private class NodeGroup extends Prim<Node> implements
- PrimGroupWriterInterface {
-
- public Osmformat.PrimitiveGroup serialize()
- {
- if(useDense)
- return serializeDense();
- return serializeNonDense();
- }
-
- /**
- * Serialize all nodes in the 'dense' format.
- */
- public Osmformat.PrimitiveGroup serializeDense()
- {
- if(contents.size() == 0) {
- return null;
- }
- // System.out.format("%d Dense ",nodes.size());
- Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup
- .newBuilder();
- StringTable stable = serializer.getStringTable();
-
- long lastlat = 0, lastlon = 0, lastid = 0;
- Osmformat.DenseNodes.Builder bi = Osmformat.DenseNodes.newBuilder();
- boolean doesBlockHaveTags = false;
- // Does anything in this block have tags?
- for(Node i : contents) {
- doesBlockHaveTags = doesBlockHaveTags || (i.tagsIterator().hasNext());
- }
- if(!omit_metadata) {
- Osmformat.DenseInfo.Builder bdi = Osmformat.DenseInfo.newBuilder();
- serializeMetadataDense(bdi, contents);
- bi.setDenseinfo(bdi);
- }
-
- for(Node i : contents) {
- long id = i.getId();
- int lat = mapDegrees(i.getLat());
- int lon = mapDegrees(i.getLon());
- bi.addId(id - lastid);
- lastid = id;
- bi.addLon(lon - lastlon);
- lastlon = lon;
- bi.addLat(lat - lastlat);
- lastlat = lat;
-
- // Then we must include tag information.
- if(doesBlockHaveTags) {
- Iterator<Element.Tag> tags = i.tagsIterator();
- while(tags.hasNext()) {
- Element.Tag t = tags.next();
- bi.addKeysVals(stable.getIndex(t.getKey()));
- bi.addKeysVals(stable.getIndex(t.getValue()));
- }
- bi.addKeysVals(0); // Add delimiter.
- }
- }
- builder.setDense(bi);
- return builder.build();
- }
-
- /**
- * Serialize all nodes in the non-dense format.
- *
- * @param parentbuilder Add to this PrimitiveBlock.
- */
- public Osmformat.PrimitiveGroup serializeNonDense()
- {
- if(contents.size() == 0) {
- return null;
- }
- // System.out.format("%d Nodes ",nodes.size());
- StringTable stable = serializer.getStringTable();
- Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup
- .newBuilder();
- for(Node i : contents) {
- long id = i.getId();
- int lat = mapDegrees(i.getLat());
- int lon = mapDegrees(i.getLon());
- Osmformat.Node.Builder bi = Osmformat.Node.newBuilder();
- bi.setId(id);
- bi.setLon(lon);
- bi.setLat(lat);
- Iterator<Element.Tag> tags = i.tagsIterator();
- while(tags.hasNext()) {
- Element.Tag t = tags.next();
- bi.addKeys(stable.getIndex(t.getKey()));
- bi.addVals(stable.getIndex(t.getValue()));
- }
- if(!omit_metadata) {
- bi.setInfo(serializeMetadata(i));
- }
- builder.addNodes(bi);
- }
- return builder.build();
- }
-
- }
-
- private class WayGroup extends Prim<Way> implements
- PrimGroupWriterInterface {
- public Osmformat.PrimitiveGroup serialize()
- {
- if(contents.size() == 0) {
- return null;
- }
-
- // System.out.format("%d Ways ",contents.size());
- StringTable stable = serializer.getStringTable();
- Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup
- .newBuilder();
- for(Way i : contents) {
- Osmformat.Way.Builder bi = Osmformat.Way.newBuilder();
- bi.setId(i.getId());
- long lastid = 0;
- for(long j : i.getRefs()) {
- long id = j;
- bi.addRefs(id - lastid);
- lastid = id;
- }
- Iterator<Element.Tag> tags = i.tagsIterator();
- while(tags.hasNext()) {
- Element.Tag t = tags.next();
- bi.addKeys(stable.getIndex(t.getKey()));
- bi.addVals(stable.getIndex(t.getValue()));
- }
- if(!omit_metadata) {
- bi.setInfo(serializeMetadata(i));
- }
- builder.addWays(bi);
- }
- return builder.build();
- }
- }
-
- private class RelationGroup extends Prim<Relation> implements
- PrimGroupWriterInterface {
- public void addStringsToStringtable()
- {
- StringTable stable = serializer.getStringTable();
- super.addStringsToStringtable();
- for(Relation i : contents) {
- for(Member j : i.getMembers()) {
- stable.incr(j.getRole());
- }
- }
- }
-
- public Osmformat.PrimitiveGroup serialize()
- {
- if(contents.size() == 0) {
- return null;
- }
-
- // System.out.format("%d Relations ",contents.size());
- StringTable stable = serializer.getStringTable();
- Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup
- .newBuilder();
- for(Relation i : contents) {
- Osmformat.Relation.Builder bi = Osmformat.Relation.newBuilder();
- bi.setId(i.getId());
- Member[] arr = new Member[i.getMembers().size()];
- i.getMembers().toArray(arr);
- long lastid = 0;
- for(Member j : i.getMembers()) {
- long id = j.getRef();
- bi.addMemids(id - lastid);
- lastid = id;
- if(j.getType().equals("node")) {
- bi.addTypes(MemberType.NODE);
- } else if(j.getType().equals("way")) {
- bi.addTypes(MemberType.WAY);
- } else if(j.getType().equals("relation")) {
- bi.addTypes(MemberType.RELATION);
- } else {
- assert (false); // Software bug: Unknown entity.
- }
- bi.addRolesSid(stable.getIndex(j.getRole()));
- }
-
- Iterator<Element.Tag> tags = i.tagsIterator();
- while(tags.hasNext()) {
- Element.Tag t = tags.next();
- bi.addKeys(stable.getIndex(t.getKey()));
- bi.addVals(stable.getIndex(t.getValue()));
- }
- if(!omit_metadata) {
- bi.setInfo(serializeMetadata(i));
- }
- builder.addRelations(bi);
- }
- return builder.build();
- }
- }
-
- /* One list for each type */
- protected WayGroup ways;
-
- protected NodeGroup nodes;
-
- protected RelationGroup relations;
-
- protected Processor processor = new Processor();
-
- /**
- * Buffer up events into groups that are all of the same type, or all of the
- * same length, then process each buffer.
- */
- public class Processor {
-
- /**
- * Check if we've reached the batch size limit and process the batch if
- * we have.
- */
- public void checkLimit()
- {
- total_entities++;
- if(++batch_size < batch_limit) {
- return;
- }
- switchTypes();
- processBatch();
- }
-
- public void process(Node node)
- {
- if(nodes == null) {
- writeEmptyHeaderIfNeeded();
- // Need to switch types.
- switchTypes();
- nodes = new NodeGroup();
- }
- nodes.add(node);
- checkLimit();
- }
-
- public void process(Way way)
- {
- if(ways == null) {
- writeEmptyHeaderIfNeeded();
- switchTypes();
- ways = new WayGroup();
- }
- ways.add(way);
- checkLimit();
- }
-
- public void process(Relation relation)
- {
- if(relations == null) {
- writeEmptyHeaderIfNeeded();
- switchTypes();
- relations = new RelationGroup();
- }
- relations.add(relation);
- checkLimit();
- }
- }
-
- /**
- * At the end of this function, all of the lists of unprocessed 'things'
- * must be null
- */
- protected void switchTypes()
- {
- if(nodes != null) {
- groups.add(nodes);
- nodes = null;
- } else if(ways != null) {
- groups.add(ways);
- ways = null;
- } else if(relations != null) {
- groups.add(relations);
- relations = null;
- } else {
- return; // No data. Is this an empty file?
- }
- }
-
- /** Write empty header block when there's no bounds entity. */
- public void writeEmptyHeaderIfNeeded()
- {
- if(headerWritten) {
- return;
- }
- Osmformat.HeaderBlock.Builder headerblock = Osmformat.HeaderBlock
- .newBuilder();
- finishHeader(headerblock);
- }
- }
-
- public BinaryMapWriter(Area bounds, File outputDir, int mapId, int extra) {
- super(bounds, outputDir, mapId, extra);
- }
-
- public void initForWrite()
- {
- String filename = String.format(Locale.ROOT, "%08d.osm.pbf", mapId);
- try {
- output = new BlockOutputStream(new FileOutputStream(new File(outputDir,
- filename)));
- serializer = new PBFSerializer(output);
- writeHeader();
- }
- catch(IOException e) {
- System.out.println("Could not open or write file header. Reason: "
- + e.getMessage());
- throw new RuntimeException(e);
- }
- }
-
- private void writeHeader()
- {
- Osmformat.HeaderBlock.Builder headerblock = Osmformat.HeaderBlock
- .newBuilder();
-
- Osmformat.HeaderBBox.Builder pbfBbox = Osmformat.HeaderBBox.newBuilder();
- pbfBbox.setLeft(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMinLong())));
- pbfBbox.setBottom(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMinLat())));
- pbfBbox.setRight(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMaxLong())));
- pbfBbox.setTop(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMaxLat())));
- headerblock.setBbox(pbfBbox);
-
- // headerblock.setSource("splitter"); //TODO: entity.getOrigin());
- finishHeader(headerblock);
- }
-
- /** Write the header fields that are always needed.
- *
- * @param headerblock Incomplete builder to complete and write.
- * */
- public void finishHeader(Osmformat.HeaderBlock.Builder headerblock)
- {
- headerblock.setWritingprogram("splitter-r" + Version.VERSION);
- headerblock.addRequiredFeatures("OsmSchema-V0.6");
- if(useDense) {
- headerblock.addRequiredFeatures("DenseNodes");
- }
- Osmformat.HeaderBlock message = headerblock.build();
- try {
- output.write(FileBlock.newInstance("OSMHeader", message.toByteString(),
- null));
- }
- catch(IOException e) {
- throw new RuntimeException("Unable to write OSM header.", e);
- }
- headerWritten = true;
- }
-
- public void finishWrite()
- {
- try {
- serializer.switchTypes();
- serializer.processBatch();
- serializer.close();
- serializer = null;
- }
- catch(IOException e) {
- System.out.println("Could not write end of file: " + e);
- }
- }
-
- public void write(Node node)
- {
- serializer.processor.process(node);
- }
-
- public void write(Way way)
- {
- serializer.processor.process(way);
- }
-
- public void write(Relation relation)
- {
- serializer.processor.process(relation);
- }
-}
diff --git a/src/uk/me/parabola/splitter/DataStorer.java b/src/uk/me/parabola/splitter/DataStorer.java
index 0329769..9e6d231 100644
--- a/src/uk/me/parabola/splitter/DataStorer.java
+++ b/src/uk/me/parabola/splitter/DataStorer.java
@@ -14,12 +14,15 @@ package uk.me.parabola.splitter;
import java.io.File;
import java.io.IOException;
-import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import uk.me.parabola.splitter.tools.Long2IntClosedMapFunction;
+import uk.me.parabola.splitter.tools.OSMId2ObjectMap;
+import uk.me.parabola.splitter.tools.SparseLong2IntMap;
+import uk.me.parabola.splitter.writer.OSMWriter;
/**
* Stores data that is needed in different passes of the program.
@@ -28,181 +31,182 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
*
*/
public class DataStorer {
- public static final int NODE_TYPE = 0;
- public static final int WAY_TYPE = 1;
- public static final int REL_TYPE = 2;
-
- private final int numOfAreas;
-
- private final Long2IntClosedMapFunction[] maps = new Long2IntClosedMapFunction[3];
-
- private final AreaDictionaryShort areaDictionary;
- private final AreaDictionaryInt multiTileDictionary;
- private final AreaIndex areaIndex;
- private SparseLong2ShortMapFunction usedWays = null;
- private final OSMId2ObjectMap<Integer> usedRels = new OSMId2ObjectMap<>();
- private boolean idsAreNotSorted;
- private OSMWriter[] writers;
- /** map with relations that should be complete and are written to only one tile */
- private final Long2ObjectOpenHashMap<Integer> oneDistinctAreaOnlyRels = new Long2ObjectOpenHashMap<>();
- private final OSMId2ObjectMap<Integer> oneTileOnlyRels = new OSMId2ObjectMap<>();
-
- /**
- * Create a dictionary for a given number of writers
- *
- * @param overlapAmount
- * @param numOfWriters
- * the number of writers that are used
- */
- DataStorer(List<Area> areas, int overlapAmount) {
- this.numOfAreas = areas.size();
- this.areaDictionary = new AreaDictionaryShort(areas, overlapAmount);
- this.multiTileDictionary = new AreaDictionaryInt(numOfAreas);
- this.areaIndex = new AreaGrid(areaDictionary);
- return;
- }
-
- public int getNumOfAreas() {
- return numOfAreas;
- }
-
- public AreaDictionaryShort getAreaDictionary() {
- return areaDictionary;
- }
-
- public Area getArea(int idx) {
- return areaDictionary.getArea(idx);
- }
-
- public Area getExtendedArea(int idx) {
- return areaDictionary.getExtendedArea(idx);
- }
-
- public void setWriters(OSMWriter[] writers) {
- this.writers = writers;
- }
-
- public void setWriterMap(int type, Long2IntClosedMapFunction nodeWriterMap) {
- maps[type] = nodeWriterMap;
- }
-
- public Long2IntClosedMapFunction getWriterMap(int type) {
- return maps[type];
- }
-
- public AreaIndex getGrid() {
- return areaIndex;
- }
-
- public AreaDictionaryInt getMultiTileDictionary() {
- return multiTileDictionary;
- }
-
- public SparseLong2ShortMapFunction getUsedWays() {
- return usedWays;
- }
-
- public OSMId2ObjectMap<Integer> getUsedRels() {
- return usedRels;
- }
-
- public void setUsedWays(SparseLong2ShortMapFunction ways) {
- usedWays = ways;
- }
-
- public boolean isIdsAreNotSorted() {
- return idsAreNotSorted;
- }
-
- public void setIdsAreNotSorted(boolean idsAreNotSorted) {
- this.idsAreNotSorted = idsAreNotSorted;
- }
-
- public void restartWriterMaps() {
- for (Long2IntClosedMapFunction map : maps) {
- if (map != null) {
- try {
- map.close();
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- }
-
- }
-
- public void switchToSeqAccess(File fileOutputDir) throws IOException {
- boolean msgWritten = false;
- long start = System.currentTimeMillis();
- for (Long2IntClosedMapFunction map : maps) {
- if (map != null) {
- if (!msgWritten) {
- System.out.println("Writing results of MultiTileAnalyser to temp files ...");
- msgWritten = true;
- }
- map.switchToSeqAccess(fileOutputDir);
- }
- }
- System.out
- .println("Writing temp files took " + (System.currentTimeMillis() - start) + " ms");
- }
-
- public void finish() {
- for (Long2IntClosedMapFunction map : maps) {
- if (map != null)
- map.finish();
- }
- }
-
- public void stats(final String prefix) {
- for (Long2IntClosedMapFunction map : maps) {
- if (map != null)
- map.stats(prefix);
- }
- }
-
- public OSMWriter[] getWriters() {
- return writers;
- }
-
- public void storeRelationArea(long id, Integer areaIdx) {
- oneDistinctAreaOnlyRels.put(id, areaIdx);
- }
-
- public Integer getOneTileOnlyRels(long id) {
- return oneTileOnlyRels.get(id);
- }
-
- /**
- * If the BitSet ids in oneTileOnlyRels were produced with a different set of
- * areas we have to translate the values
- * @param distinctAreas list of distinct (non-overlapping) areas
- * @param distinctDataStorer
- */
- public void translateDistinctToRealAreas(DataStorer distinctDataStorer) {
- List<Area> distinctAreas = distinctDataStorer.getAreaDictionary().getAreas();
- Map<Area, Integer> map = new HashMap<>();
- for (Area distinctArea : distinctAreas) {
- if (distinctArea.getMapId() < 0 && !distinctArea.isPseudoArea()) {
- BitSet w = new BitSet();
- for (int i = 0; i < getNumOfAreas(); i++) {
- if (this.areaDictionary.getArea(i).contains(distinctArea)) {
- w.set(i);
- }
- }
- map.put(distinctArea, this.multiTileDictionary.translate(w));
- }
- }
-
- for ( Entry<Long, Integer> e: distinctDataStorer.oneDistinctAreaOnlyRels.entrySet()) {
- if (e.getValue() >= 0 && !distinctAreas.get(e.getValue()).isPseudoArea()) {
- Integer areaIdx = map.get(distinctAreas.get(e.getValue()));
- oneTileOnlyRels.put(e.getKey(), areaIdx != null ? areaIdx: e.getValue());
- } else {
- oneTileOnlyRels.put(e.getKey(), AreaDictionaryInt.UNASSIGNED);
- }
-
- }
- }
+ public static final int NODE_TYPE = 0;
+ public static final int WAY_TYPE = 1;
+ public static final int REL_TYPE = 2;
+
+ private final int numOfAreas;
+
+ private final Long2IntClosedMapFunction[] maps = new Long2IntClosedMapFunction[3];
+
+ private final AreaDictionary areaDictionary;
+ private final AreaIndex areaIndex;
+ private SparseLong2IntMap usedWays = null;
+ private final OSMId2ObjectMap<Integer> usedRels = new OSMId2ObjectMap<>();
+ private boolean idsAreNotSorted;
+ private OSMWriter[] writers;
+ /**
+ * map with relations that should be complete and are written to only one
+ * tile
+ */
+ private final Long2ObjectOpenHashMap<Integer> oneDistinctAreaOnlyRels = new Long2ObjectOpenHashMap<>();
+ private final OSMId2ObjectMap<Integer> oneTileOnlyRels = new OSMId2ObjectMap<>();
+
+ /**
+ * Create a dictionary for a given number of writers
+ *
+ * @param overlapAmount
+ * @param numOfWriters
+ * the number of writers that are used
+ */
+ DataStorer(List<Area> areas, int overlapAmount) {
+ this.numOfAreas = areas.size();
+ this.areaDictionary = new AreaDictionary(areas, overlapAmount);
+ this.areaIndex = new AreaGrid(areaDictionary);
+ return;
+ }
+
+ public int getNumOfAreas() {
+ return numOfAreas;
+ }
+
+ public AreaDictionary getAreaDictionary() {
+ return areaDictionary;
+ }
+
+ public Area getArea(int idx) {
+ return areaDictionary.getArea(idx);
+ }
+
+ public Area getExtendedArea(int idx) {
+ return areaDictionary.getExtendedArea(idx);
+ }
+
+ public void setWriters(OSMWriter[] writers) {
+ this.writers = writers;
+ }
+
+ public void setWriterMap(int type, Long2IntClosedMapFunction nodeWriterMap) {
+ maps[type] = nodeWriterMap;
+ }
+
+ public Long2IntClosedMapFunction getWriterMap(int type) {
+ return maps[type];
+ }
+
+ public AreaIndex getGrid() {
+ return areaIndex;
+ }
+
+ public SparseLong2IntMap getUsedWays() {
+ return usedWays;
+ }
+
+ public OSMId2ObjectMap<Integer> getUsedRels() {
+ return usedRels;
+ }
+
+ public void setUsedWays(SparseLong2IntMap ways) {
+ usedWays = ways;
+ }
+
+ public boolean isIdsAreNotSorted() {
+ return idsAreNotSorted;
+ }
+
+ public void setIdsAreNotSorted(boolean idsAreNotSorted) {
+ this.idsAreNotSorted = idsAreNotSorted;
+ }
+
+ public void restartWriterMaps() {
+ for (Long2IntClosedMapFunction map : maps) {
+ if (map != null) {
+ try {
+ map.close();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ }
+
+ }
+
+ public void switchToSeqAccess(File fileOutputDir) throws IOException {
+ boolean msgWritten = false;
+ long start = System.currentTimeMillis();
+ for (Long2IntClosedMapFunction map : maps) {
+ if (map != null) {
+ if (!msgWritten) {
+ System.out.println("Writing results of MultiTileAnalyser to temp files ...");
+ msgWritten = true;
+ }
+ map.switchToSeqAccess(fileOutputDir);
+ }
+ }
+ System.out.println("Writing temp files took " + (System.currentTimeMillis() - start) + " ms");
+ }
+
+ public void finish() {
+ for (Long2IntClosedMapFunction map : maps) {
+ if (map != null)
+ map.finish();
+ }
+ }
+
+ public void stats(final String prefix) {
+ for (Long2IntClosedMapFunction map : maps) {
+ if (map != null)
+ map.stats(prefix);
+ }
+ }
+
+ public OSMWriter[] getWriters() {
+ return writers;
+ }
+
+ public void storeRelationAreas(long id, AreaSet areaSet) {
+ oneDistinctAreaOnlyRels.put(id, areaDictionary.translate(areaSet));
+ }
+
+ public Integer getOneTileOnlyRels(long id) {
+ return oneTileOnlyRels.get(id);
+ }
+
+ /**
+ * If the ids in oneTileOnlyRels were produced with a different set
+ * of areas we have to translate the values
+ *
+ * @param distinctAreas
+ * list of distinct (non-overlapping) areas
+ * @param distinctDataStorer
+ */
+ public void translateDistinctToRealAreas(DataStorer distinctDataStorer) {
+ List<Area> distinctAreas = distinctDataStorer.getAreaDictionary().getAreas();
+ Map<Area, Integer> map = new HashMap<>();
+ for (Area distinctArea : distinctAreas) {
+ if (distinctArea.getMapId() < 0 && !distinctArea.isPseudoArea()) {
+ AreaSet w = new AreaSet();
+ for (int i = 0; i < getNumOfAreas(); i++) {
+ if (this.areaDictionary.getArea(i).contains(distinctArea)) {
+ w.set(i);
+ }
+ }
+ map.put(distinctArea, this.areaDictionary.translate(w));
+ }
+ }
+
+ for (Entry<Long, Integer> e : distinctDataStorer.oneDistinctAreaOnlyRels.entrySet()) {
+ AreaSet singleArea = distinctDataStorer.getAreaDictionary().getSet(e.getValue());
+ assert singleArea.cardinality() == 1;
+ int pos = singleArea.iterator().next();
+ if (!distinctAreas.get(pos).isPseudoArea()) {
+ Integer areaIdx = map.get(distinctAreas.get(pos));
+ oneTileOnlyRels.put(e.getKey(), areaIdx != null ? areaIdx : e.getValue());
+ } else {
+ oneTileOnlyRels.put(e.getKey(), AbstractMapProcessor.UNASSIGNED);
+ }
+
+ }
+ }
}
diff --git a/src/uk/me/parabola/splitter/Element.java b/src/uk/me/parabola/splitter/Element.java
index dd8fccb..a018a3d 100644
--- a/src/uk/me/parabola/splitter/Element.java
+++ b/src/uk/me/parabola/splitter/Element.java
@@ -40,7 +40,7 @@ public abstract class Element {
this.version = version;
}
- class Tag {
+ public static class Tag {
public Tag(String key,String value) {
this.key = key;
this.value = value;
@@ -62,7 +62,7 @@ public abstract class Element {
return;
// Most elements are nodes. Most nodes have no tags. Create the tag table lazily
if (tags == null)
- tags = new ArrayList<Tag>(4);
+ tags = new ArrayList<>(4);
tags.add(new Tag(key, value));
}
diff --git a/src/uk/me/parabola/splitter/Main.java b/src/uk/me/parabola/splitter/Main.java
index 45ff360..44a8976 100644
--- a/src/uk/me/parabola/splitter/Main.java
+++ b/src/uk/me/parabola/splitter/Main.java
@@ -13,118 +13,60 @@
package uk.me.parabola.splitter;
-import org.xmlpull.v1.XmlPullParserException;
-
-import uk.me.parabola.splitter.args.ParamParser;
-import uk.me.parabola.splitter.args.SplitterParams;
-import java.awt.Rectangle;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
-import java.util.regex.Pattern;
+
+import uk.me.parabola.splitter.args.ParamParser;
+import uk.me.parabola.splitter.args.SplitterParams;
+import uk.me.parabola.splitter.kml.KmlWriter;
+import uk.me.parabola.splitter.solver.AreasCalculator;
+import uk.me.parabola.splitter.writer.AbstractOSMWriter;
+import uk.me.parabola.splitter.writer.BinaryMapWriter;
+import uk.me.parabola.splitter.writer.O5mMapWriter;
+import uk.me.parabola.splitter.writer.OSMWriter;
+import uk.me.parabola.splitter.writer.OSMXMLWriter;
+import uk.me.parabola.splitter.writer.PseudoOSMWriter;
/**
* Splitter for OSM files with the purpose of providing input files for mkgmap.
* <p/>
- * The input file is split so that no piece has more than a given number of nodes in it.
+ * The input file is split so that no piece has more than a given number of
+ * nodes in it.
*
* @author Steve Ratcliffe
*/
public class Main {
-
- private static final String DEFAULT_DIR = ".";
- // We store area IDs and all used combinations of area IDs in a dictionary. The index to this
- // dictionary is saved in short values. If Short.MaxValue() is reached, the user might limit
- // the number of areas that is processed in one pass.
- private int maxAreasPerPass;
+ private static final String DEFAULT_DIR = ".";
- // A list of the OSM files to parse.
+ /** A list of the OSM files to parse. */
private List<String> fileNameList;
- // The description to write into the template.args file.
- private String description;
-
- // The starting map ID.
- private int mapId;
-
- // The amount in map units that tiles overlap. The default is overwritten depending on user settings.
+ /** The amount in map units that tiles overlap. The default is overwritten depending on user settings. */
private int overlapAmount = -1;
- // A threshold value that is used when no split-file is given. Splitting is done so that
- // no tile has more than maxNodes nodes inside the bounding box of the tile.
- // Nodes added by overlap or keep-complete are not taken into account.
- private long maxNodes;
-
+ /** The number of tiles to be written. The default is overwritten depending on user settings. */
private int numTiles = -1;
-
- // This is a value in the range 0-24.
- // Higher numbers mean higher detail. The resolution determines how the tiles must
- // be aligned. Eg a resolution of 13 means the tiles need to have their edges aligned to
- // multiples of 2 ^ (24 - 13) = 2048 map units.
- private int resolution;
-
- // Whether or not to trim tiles of any empty space around their edges.
- private boolean trim;
- // Set if there is a previous area file given on the command line.
- private AreaList areaList;
- // Whether or not the source OSM file(s) contain strictly nodes first, then ways, then rels,
- // or they're all mixed up. Running with mixed enabled takes longer.
- private boolean mixed;
- // A polygon file in osmosis polygon format
- private String polygonFile;
-
- // The path where the results are written out to.
- private File fileOutputDir;
- // A GeoNames file to use for naming the tiles.
- private String geoNamesFile;
- // How often (in seconds) to provide JVM status information. Zero = no information.
- private int statusFreq;
-
- private String kmlOutputFile;
- // The maximum number of threads the splitter should use.
- private int maxThreads;
- // The output type
- private String outputType;
- // a list of way or relation ids that should be handled specially
- private String problemFile;
- // Whether or not splitter should keep
- private boolean keepComplete;
-
- private String problemReport;
-
- // option for fine tuning the keep-complete processing
- private int wantedAdminLevel;
-
- private String[] boundaryTags;
-
- private String stopAfter;
-
- private String precompSeaDir;
- private String polygonDescFile;
-
- private int searchLimit;
-
- private String handleElementVersion;
-
- private boolean ignoreBoundsTags;
+ /** The path where the results are written out to. */
+ private File fileOutputDir;
private final OSMFileHandler osmFileHandler = new OSMFileHandler();
- private final AreasCalculator areasCalculator = new AreasCalculator();
private final ProblemLists problemList = new ProblemLists();
-
+
+ private SplitterParams mainOptions;
+
public static void main(String[] args) {
Main m = new Main();
- try{
+ try {
int rc = m.start(args);
if (rc != 0)
System.exit(1);
- } catch (StopNoErrorException e){
+ } catch (StopNoErrorException e) {
if (e.getMessage() != null)
System.out.println(e.getMessage());
}
@@ -133,49 +75,61 @@ public class Main {
private int start(String[] args) {
int rc = 0;
JVMHealthMonitor healthMonitor = null;
-
- try{
- readArgs(args);
+
+ try {
+ mainOptions = readArgs(args);
} catch (IllegalArgumentException e) {
if (e.getMessage() != null)
System.out.println("Error: " + e.getMessage());
return 1;
}
- if (statusFreq > 0) {
- healthMonitor = new JVMHealthMonitor(statusFreq);
+ if (mainOptions.getStatusFreq() > 0) {
+ healthMonitor = new JVMHealthMonitor(mainOptions.getStatusFreq());
healthMonitor.start();
}
-
- checkJREVersion();
-
+
long start = System.currentTimeMillis();
System.out.println("Time started: " + new Date());
try {
- List<Area> areas = split();
- DataStorer dataStorer;
- if (keepComplete) {
- dataStorer = calcProblemLists(areas);
- useProblemLists(dataStorer);
- } else {
- dataStorer = new DataStorer(areas, overlapAmount);
- }
- writeTiles(dataStorer);
- dataStorer.finish();
+ // configure the input file handler
+ osmFileHandler.setFileNames(fileNameList);
+ osmFileHandler.setMixed(mainOptions.isMixed());
+ osmFileHandler.setMaxThreads(mainOptions.getMaxThreads().getCount());
+
+ if (mainOptions.isKeepComplete() && mainOptions.getProblemFile() != null) {
+ // read the user list now so that possible problems are reported early
+ if (!problemList.readProblemIds(mainOptions.getProblemFile()))
+ throw new IllegalArgumentException();
+ }
+
+ // first step: either read or calculate the list of areas
+ List<Area> areas = split();
+ DataStorer dataStorer;
+ if (mainOptions.isKeepComplete()) {
+ // optional step a: calculate list of ways and relations which are contained in multiple areas
+ dataStorer = calcProblemLists(areas);
+ // optional step b: calculate the writers for the list of "problem" ways and relations
+ useProblemLists(dataStorer);
+ } else {
+ dataStorer = new DataStorer(areas, overlapAmount);
+ }
+ // final step: write the OSM output files
+ writeTiles(dataStorer);
+ dataStorer.finish();
} catch (IOException e) {
System.err.println("Error opening or reading file " + e);
e.printStackTrace();
return 1;
- } catch (XmlPullParserException e) {
- System.err.println("Error parsing xml from file " + e);
- e.printStackTrace();
- return 1;
} catch (SplitFailedException e) {
if (e.getMessage() != null && e.getMessage().length() > 0)
e.printStackTrace();
return 1;
- } catch (StopNoErrorException e){
- if (e.getMessage() != null)
- System.out.println(e.getMessage());
+ } catch (StopNoErrorException e) {
+ if (e.getMessage() != null) {
+ String msg = "Stopped after " + e.getMessage();
+ System.err.println(msg);
+ System.out.println(msg);
+ }
// nothing to do
} catch (RuntimeException e) {
e.printStackTrace();
@@ -187,123 +141,130 @@ public class Main {
}
/**
- * Check if a JRE 1.7.x or higher is installed.
+ * Fill the list of areas. The list might be read from an existing file or it might
+ * be freshly calculated by scanning the input files.
+ * @return List of areas which might overlap each other if they were read from an existing file.
+ * @throws IOException
*/
- private static void checkJREVersion() {
- /*
- String version = System.getProperty("java.version");
- if (version != null) {
- String[] versionParts =version.split(Pattern.quote("."));
- if (versionParts.length >= 2) {
- int major = Integer.valueOf(versionParts[1]);
- if (major < 7) {
- System.out.println("===========================================================");
- System.out.println("You are using an old Java runtime environment "+ version);
- System.out.println("It is no longer supported.");
- System.out.println("Please update Java to the latest release.");
- System.out.println("===========================================================");
- System.exit(1);
- }
+ private List<Area> split() throws IOException {
+ final File outputDir = fileOutputDir;
+ if (!outputDir.exists()) {
+ System.out.println("Output directory not found. Creating directory '" + fileOutputDir + "'");
+ if (!outputDir.mkdirs()) {
+ System.err.println("Unable to create output directory! Using default directory instead");
+ fileOutputDir = new File(DEFAULT_DIR);
+ }
+ } else if (!outputDir.isDirectory()) {
+ System.err.println(
+ "The --output-dir parameter must specify a directory. The --output-dir parameter is being ignored, writing to default directory instead.");
+ fileOutputDir = new File(DEFAULT_DIR);
+ }
+
+ final String splitFile = mainOptions.getSplitFile();
+ // A polygon file in osmosis polygon format
+ final String polygonFile = mainOptions.getPolygonFile();
+ final String polygonDescFile = mainOptions.getPolygonDescFile();
+ final AreaList areaList = new AreaList(mainOptions.getDescription());
+ boolean writeAreas = false;
+
+ if (splitFile != null) {
+ try {
+ areaList.read(splitFile);
+ areaList.dump();
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Could not read area list file " + splitFile);
+ }
+ if (polygonFile != null) {
+ System.out.println("Warning: parameter polygon-file is ignored because split-file is used.");
+ }
+ if (polygonDescFile != null) {
+ System.out.println("Warning: parameter polygon-desc-file is ignored because split-file is used.");
+ }
+ } else {
+ writeAreas = true;
+ }
+ areaList.setGeoNamesFile(mainOptions.getGeonamesFile());
+
+ AreasCalculator areasCalculator = new AreasCalculator(mainOptions, numTiles);
+ if (areaList.getAreas().isEmpty()) {
+ int resolution = mainOptions.getResolution();
+ writeAreas = true;
+ int alignment = 1 << (24 - resolution);
+ System.out.println("Map is being split for resolution " + resolution + ':');
+ System.out.println(" - area boundaries are aligned to 0x" + Integer.toHexString(alignment) + " map units ("
+ + Utils.toDegrees(alignment) + " degrees)");
+ System.out.println(
+ " - areas are multiples of 0x" + Integer.toHexString(alignment) + " map units wide and high");
+ areasCalculator.fillDensityMap(osmFileHandler, fileOutputDir);
+ areaList.setAreas(areasCalculator.calcAreas());
+ if (areaList.getAreas().isEmpty()) {
+ System.err.println("Failed to calculate areas. See stdout messages for details.");
+ System.out.println("Failed to calculate areas.");
+ System.out.println("Sorry. Cannot split the file without creating huge, almost empty, tiles.");
+ System.out.println("Please specify a bounding polygon with the --polygon-file parameter.");
+ throw new SplitFailedException("");
+ }
+ int mapId = mainOptions.getMapid();
+ if (mapId + areaList.getAreas().size() > 99999999) {
+ throw new SplitFailedException("Too many areas for initial mapid " + mapId);
+ }
+ areaList.setMapIds(mapId);
+ }
+ areaList.setAreaNames();
+ if (writeAreas) {
+ areaList.write(new File(fileOutputDir, "areas.list").getPath());
+ areaList.writePoly(new File(fileOutputDir, "areas.poly").getPath());
+ }
+
+ List<Area> areas = areaList.getAreas();
+
+ String kmlOutputFile = mainOptions.getWriteKml();
+ if (kmlOutputFile != null) {
+ File out = new File(kmlOutputFile);
+ if (!out.isAbsolute())
+ out = new File(fileOutputDir, kmlOutputFile);
+ KmlWriter.writeKml(out.getPath(), areas);
+ }
+ String outputType = mainOptions.getOutput();
+
+ if (!areasCalculator.getPolygons().isEmpty()) {
+ areaList.writeListFiles(outputDir, areasCalculator.getPolygons(), kmlOutputFile, outputType);
+ }
+ areaList.writeArgsFile(new File(fileOutputDir, "template.args").getPath(), outputType, -1);
+ areaList.dumpHex();
+
+ if ("split".equals(mainOptions.getStopAfter())) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
}
+ throw new StopNoErrorException(mainOptions.getStopAfter());
}
- */
+
+ return areaList.getAreas();
+ }
+
+ private DataStorer calcProblemLists(List<Area> areas) {
+ DataStorer dataStorer = problemList.calcProblemLists(osmFileHandler, areas, overlapAmount, mainOptions);
+ String problemReport = mainOptions.getProblemReport();
+ if (problemReport != null) {
+ problemList.writeProblemList(fileOutputDir, problemReport);
+ }
+
+ if ("gen-problem-list".equals(mainOptions.getStopAfter())) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+ throw new StopNoErrorException(mainOptions.getStopAfter());
+ }
+ return dataStorer;
}
-
- private List<Area> split() throws IOException, XmlPullParserException {
-
- File outputDir = fileOutputDir;
- if (!outputDir.exists()) {
- System.out.println("Output directory not found. Creating directory '" + fileOutputDir + "'");
- if (!outputDir.mkdirs()) {
- System.err.println("Unable to create output directory! Using default directory instead");
- fileOutputDir = new File(DEFAULT_DIR);
- }
- } else if (!outputDir.isDirectory()) {
- System.err.println("The --output-dir parameter must specify a directory. The --output-dir parameter is being ignored, writing to default directory instead.");
- fileOutputDir = new File(DEFAULT_DIR);
- }
-
- if (fileNameList.isEmpty()) {
- throw new IllegalArgumentException("No input files were supplied");
- }
-
- boolean writeAreas = false;
- if (areaList.getAreas().isEmpty()) {
- writeAreas = true;
- int alignment = 1 << (24 - resolution);
- System.out.println("Map is being split for resolution " + resolution + ':');
- System.out.println(" - area boundaries are aligned to 0x" + Integer.toHexString(alignment) + " map units (" + Utils.toDegrees(alignment) + " degrees)");
- System.out.println(" - areas are multiples of 0x" + Integer.toHexString(alignment) + " map units wide and high");
- areaList.setAreas(calculateAreas());
- if (areaList == null || areaList.getAreas().isEmpty()){
- System.err.println("Failed to calculate areas. See stdout messages for details.");
- System.out.println("Failed to calculate areas.");
- System.out.println("Sorry. Cannot split the file without creating huge, almost empty, tiles.");
- System.out.println("Please specify a bounding polygon with the --polygon-file parameter.");
- throw new SplitFailedException("");
- }
- if (mapId + areaList.getAreas().size() > 99999999){
- System.err.println("Too many areas for initial mapid " + mapId + ", resetting to 63240001");
- mapId = 63240001;
- }
- areaList.setMapIds(mapId);
- }
- areaList.setAreaNames();
- if (writeAreas) {
- areaList.write(new File(fileOutputDir, "areas.list").getPath());
- areaList.writePoly(new File(fileOutputDir, "areas.poly").getPath());
- }
-
- List<Area> areas = areaList.getAreas();
-
- if (kmlOutputFile != null) {
- File out = new File(kmlOutputFile);
- if (!out.isAbsolute())
- out = new File(fileOutputDir, kmlOutputFile);
- KmlWriter.writeKml(out.getPath(), areas);
- }
- areasCalculator.writeListFiles(outputDir, areas, kmlOutputFile, outputType);
- areaList.writeArgsFile(new File(fileOutputDir, "template.args").getPath(), outputType, -1);
-
- if ("split".equals(stopAfter)){
- try {Thread.sleep(1000);}catch (InterruptedException e) {}
- System.err.println("stopped after " + stopAfter);
- throw new StopNoErrorException("stopped after " + stopAfter);
- }
-
- System.out.println(areas.size() + " areas:");
- for (Area area : areas) {
- System.out.format("Area %08d: %d,%d to %d,%d covers %s",
- area.getMapId(),
- area.getMinLat(), area.getMinLong(),
- area.getMaxLat(), area.getMaxLong(),
- area.toHexString());
-
- if (area.getName() != null)
- System.out.print(' ' + area.getName());
- System.out.println();
- }
- return areas;
- }
-
- private DataStorer calcProblemLists(List<Area> areas) {
- DataStorer dataStorer = problemList.calcProblemLists(osmFileHandler, areas, wantedAdminLevel, boundaryTags, maxAreasPerPass, overlapAmount);
- if (problemReport != null){
- problemList.writeProblemList(fileOutputDir, problemReport);
- }
-
- if ("gen-problem-list".equals(stopAfter)){
- try {Thread.sleep(1000);}catch (InterruptedException e) {}
- System.err.println("stopped after " + stopAfter);
- throw new StopNoErrorException("stopped after " + stopAfter);
- }
- return dataStorer;
- }
/**
* Deal with the command line arguments.
*/
- private void readArgs(String[] args) {
+ private SplitterParams readArgs(String[] args) {
ParamParser parser = new ParamParser();
SplitterParams params = parser.parse(SplitterParams.class, args);
@@ -319,313 +280,186 @@ public class Main {
}
System.out.println("Splitter version " + Version.VERSION + " compiled " + Version.TIMESTAMP);
-
+
for (Map.Entry<String, Object> entry : parser.getConvertedParams().entrySet()) {
String name = entry.getKey();
Object value = entry.getValue();
System.out.println(name + '=' + (value == null ? "" : value));
}
fileNameList = parser.getAdditionalParams();
- if (fileNameList.isEmpty()){
+ if (fileNameList.isEmpty()) {
throw new IllegalArgumentException("No file name(s) given");
}
- boolean filesOK = true;
- for (String fileName: fileNameList){
- if (testAndReportFname(fileName, "input file") == false){
- filesOK = false;
- }
- }
- if (!filesOK){
- System.out.println("Make sure that option parameters start with -- " );
+
+ boolean filesOK = fileNameList.stream().allMatch(fname -> testAndReportFname(fname, "input file"));
+ if (!filesOK) {
+ System.out.println("Make sure that option parameters start with -- ");
throw new IllegalArgumentException();
}
- osmFileHandler.setFileNames(fileNameList);
- mapId = params.getMapid();
- if (mapId > 99999999) {
- mapId = 63240001;
- System.err.println("The --mapid parameter must have less than 9 digits. Resetting to " + mapId + ".");
- }
- maxNodes = params.getMaxNodes();
- if (maxNodes < 10000){
- System.err.println("Error: Invalid number "+ params.getMaxNodes() +
- ". The --max-nodes parameter must be an integer value of 10000 or higher.");
+ int mapId = params.getMapid();
+ if (mapId < 0 || mapId > 99999999 ) {
+ System.err.println("The --mapid parameter must be a value between 0 and 99999999.");
+ throw new IllegalArgumentException();
+ }
+ if (params.getMaxNodes() < 10000) {
+ System.err.println("Error: Invalid number " + params.getMaxNodes()
+ + ". The --max-nodes parameter must be an integer value of 10000 or higher.");
throw new IllegalArgumentException();
}
String numTilesParm = params.getNumTiles();
- if (numTilesParm != null){
- try{
+ if (numTilesParm != null) {
+ try {
numTiles = Integer.parseInt(numTilesParm);
- if (numTiles >= 0 && numTiles < 2 ){
- System.err.println("Error: The --num-tiles parameter must be 2 or higher. Resetting to 2.");
- numTiles = 2;
+ if (numTiles < 2) {
+ System.err.println("Error: The --num-tiles parameter must be 2 or higher.");
+ throw new IllegalArgumentException();
}
- } catch(NumberFormatException e){
- System.err.println("Error: Invalid number "+ numTilesParm +
- ". The --num-tiles parameter must be an integer value of 2 or higher.");
- throw new IllegalArgumentException();
- }
- }
- description = params.getDescription();
- areaList = new AreaList(description);
- geoNamesFile = params.getGeonamesFile();
- if (geoNamesFile != null){
- if (testAndReportFname(geoNamesFile, "geonames-file") == false){
+ } catch (NumberFormatException e) {
+ System.err.println("Error: Invalid number " + numTilesParm
+ + ". The --num-tiles parameter must be an integer value of 2 or higher.");
throw new IllegalArgumentException();
}
- areaList.setGeoNamesFile (geoNamesFile);
}
- resolution = params.getResolution();
- trim = !params.isNoTrim();
- outputType = params.getOutput();
- if("xml pbf o5m simulate".contains(outputType) == false) {
+ // The description to write into the template.args file.
+ String geoNamesFile = params.getGeonamesFile();
+ checkOptionalFileOption(geoNamesFile, "geonames-file");
+ String outputType = params.getOutput();
+ if ("xml pbf o5m simulate".contains(outputType) == false) {
System.err.println("The --output parameter must be either xml, pbf, o5m, or simulate. Resetting to xml.");
- outputType = "xml";
+ throw new IllegalArgumentException();
}
-
+
+ int resolution = params.getResolution();
if (resolution < 1 || resolution > 24) {
- System.err.println("The --resolution parameter must be a value between 1 and 24. Resetting to 13.");
- resolution = 13;
+ System.err.println("The --resolution parameter must be a value between 1 and 24. Reasonable values are close to 13.");
+ throw new IllegalArgumentException();
}
- mixed = params.isMixed();
- osmFileHandler.setMixed(mixed);
- statusFreq = params.getStatusFreq();
-
- String outputDir = params.getOutputDir();
- fileOutputDir = new File(outputDir == null? DEFAULT_DIR: outputDir);
- maxAreasPerPass = params.getMaxAreas();
- if (maxAreasPerPass < 1 || maxAreasPerPass > 4096) {
- System.err.println("The --max-areas parameter must be a value between 1 and 4096. Resetting to 4096.");
- maxAreasPerPass = 4096;
- }
- kmlOutputFile = params.getWriteKml();
+ String outputDir = params.getOutputDir();
+ fileOutputDir = new File(outputDir == null ? DEFAULT_DIR : outputDir);
- maxThreads = params.getMaxThreads().getCount();
-
- problemFile = params.getProblemFile();
- if (problemFile != null){
- if (!problemList.readProblemIds(problemFile))
- throw new IllegalArgumentException();
+ int maxAreasPerPass = params.getMaxAreas();
+ if (maxAreasPerPass < 1 || maxAreasPerPass > 9999) {
+ System.err.println("The --max-areas parameter must be a value between 1 and 9999.");
+ throw new IllegalArgumentException();
}
- String splitFile = params.getSplitFile();
- if (splitFile != null) {
- if (testAndReportFname(splitFile, "split-file") == false){
- throw new IllegalArgumentException();
+ String problemFile = params.getProblemFile();
+ checkOptionalFileOption(params.getProblemFile(), "problem-file");
+ checkOptionalFileOption(params.getSplitFile(), "split-file");
+ checkOptionalFileOption(params.getPolygonFile(), "polygon-file");
+ checkOptionalFileOption(params.getPolygonDescFile(), "polygon-desc-file");
+ if (params.getPolygonDescFile() != null && params.getPolygonFile() != null) {
+ throw new IllegalArgumentException("--polygon-desc-file and --polygon-file are mutually exclusive");
+ }
+ String precompSeaDir = params.getPrecompSea();
+ if (precompSeaDir != null) {
+ File dir = new File(precompSeaDir);
+ if (dir.exists() == false || dir.canRead() == false) {
+ throw new IllegalArgumentException(
+ "precomp-sea directory doesn't exist or is not readable: " + precompSeaDir);
}
}
-
- keepComplete = params.isKeepComplete();
- if (mixed && (keepComplete || problemFile != null)){
- System.err.println("--mixed=true is not supported in combination with --keep-complete=true or --problem-file.");
+
+ boolean keepComplete = params.isKeepComplete();
+ if (params.isMixed() && (keepComplete || problemFile != null)) {
+ System.err.println(
+ "--mixed=true is not supported in combination with --keep-complete=true or --problem-file.");
System.err.println("Please use e.g. osomosis to sort the data in the input file(s)");
throw new IllegalArgumentException();
}
-
+
String overlap = params.getOverlap();
- if ("auto".equals(overlap) == false){
- try{
+ if ("auto".equals(overlap) == false) {
+ try {
overlapAmount = Integer.valueOf(overlap);
- }
- catch (NumberFormatException e){
+ if (overlapAmount < 0)
+ throw new IllegalArgumentException("--overlap=" + overlap + " is not is not a valid option.");
+ } catch (NumberFormatException e) {
throw new IllegalArgumentException("--overlap=" + overlap + " is not is not a valid option.");
}
}
- problemReport = params.getProblemReport();
String boundaryTagsParm = params.getBoundaryTags();
- if ("use-exclude-list".equals(boundaryTagsParm) == false){
- boundaryTags = boundaryTagsParm.split(Pattern.quote(","));
+
+ int wantedAdminLevelString = params.getWantedAdminLevel();
+ if (wantedAdminLevelString < 0 || wantedAdminLevelString > 12) {
+ throw new IllegalArgumentException("The --wanted-admin-level parameter must be between 0 and 12.");
}
-
- if (keepComplete){
- String wantedAdminLevelString = params.getWantedAdminLevel();
- try {
- wantedAdminLevel = Integer.valueOf(wantedAdminLevelString);
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("--admin-level=" + wantedAdminLevelString + " is not is not a valid option.");
- }
+ final List<String> validVersionHandling = Arrays.asList("remove", "fake", "keep");
+ if (!validVersionHandling.contains(params.getHandleElementVersion())) {
+ throw new IllegalArgumentException(
+ "the --handle-element-version parameter must be one of " + validVersionHandling + ".");
}
-
- // plausibility checks and default handling
- if (keepComplete){
- if (fileNameList.size() > 1){
+ final List<String> validStopAfter = Arrays.asList("split", "gen-problem-list", "handle-problem-list", "dist");
+ if (!validStopAfter.contains(params.getStopAfter())) {
+ throw new IllegalArgumentException(
+ "the --stop-after parameter must be one of " + validStopAfter + ".");
+ }
+ int searchLimit = params.getSearchLimit();
+ if (searchLimit < 1000) {
+ throw new IllegalArgumentException("The --search-limit parameter must be 1000 or higher.");
+ }
+
+
+ // plausibility checks and default handling
+ if (keepComplete) {
+ if (fileNameList.size() > 1) {
System.err.println("Warning: --keep-complete is only used for the first input file.");
}
- if (overlapAmount > 0){
+ if (overlapAmount > 0) {
System.err.println("Warning: --overlap is used in combination with --keep-complete=true ");
- System.err.println(" The option keep-complete should be used with overlap=0 because it is very unlikely that ");
- System.err.println(" the overlap will add any important data. It will just cause a lot of additional output which ");
+ System.err.println(
+ " The option keep-complete should be used with overlap=0 because it is very unlikely that ");
+ System.err.println(
+ " the overlap will add any important data. It will just cause a lot of additional output which ");
System.err.println(" has to be thrown away again in mkgmap.");
} else
overlapAmount = 0;
- }
- else {
- if (overlapAmount < 0){
+ } else {
+ if (overlapAmount < 0) {
overlapAmount = 2000;
System.out.println("Setting default overlap=2000 because keep-complete=false is in use.");
}
- if (problemReport != null){
- System.out.println("Parameter --problem-report is ignored, because parameter --keep-complete=false is used");
- }
- if (boundaryTagsParm != null){
- System.out.println("Parameter --boundaryTags is ignored, because parameter --keep-complete=false is used");
- }
- }
- if (splitFile != null) {
- try {
- areaList = new AreaList(description);
- areaList.read(splitFile);
- areaList.dump();
- } catch (IOException e) {
- areaList = null;
- System.err.println("Could not read area list file");
- e.printStackTrace();
+ if (mainOptions.getProblemReport() != null) {
+ System.out.println(
+ "Parameter --problem-report is ignored, because parameter --keep-complete=false is used");
}
- }
-
- polygonFile = params.getPolygonFile();
- if (polygonFile != null) {
- if (splitFile != null){
- System.out.println("Warning: parameter polygon-file is ignored because split-file is used.");
- } else {
- areasCalculator.readPolygonFile(polygonFile, mapId);
+ if (boundaryTagsParm != null) {
+ System.out.println(
+ "Parameter --boundaryTags is ignored, because parameter --keep-complete=false is used");
}
}
- polygonDescFile = params.getPolygonDescFile();
- if (polygonDescFile != null) {
- if (splitFile != null){
- System.out.println("Warning: parameter polygon-desc-file is ignored because split-file is used.");
- } else {
- areasCalculator.readPolygonDescFile(polygonDescFile);
+ return params;
+ }
+
+ private static void checkOptionalFileOption(String fname, String option) {
+ if (fname != null) {
+ if (testAndReportFname(fname, option) == false) {
+ throw new IllegalArgumentException();
}
}
- areasCalculator.setResolution(resolution);
- if (!areasCalculator.checkPolygons()) {
- System.out.println(
- "Warning: Bounding polygon is complex. Splitter might not be able to fit all tiles into the polygon!");
- }
- stopAfter = params.getStopAfter();
- if (Arrays.asList("split", "gen-problem-list" , "handle-problem-list", "dist").contains(stopAfter) == false){
- throw new IllegalArgumentException("the --stop-after parameter must be either split, gen-problem-list, handle-problem-list, or dist.");
- }
- precompSeaDir = params.getPrecompSea();
- if (precompSeaDir != null){
- File dir = new File (precompSeaDir);
- if (dir.exists() == false || dir.canRead() == false){
- throw new IllegalArgumentException("precomp-sea directory doesn't exist or is not readable: " + precompSeaDir);
- }
- }
- int numPolygons = areasCalculator.getPolygons().size();
- if (numPolygons > 0 && numTiles > 0){
- if (numPolygons == 1){
- System.out.println("Warning: Parameter polygon-file is only used to calculate the bounds because --num-tiles is used");
- } else {
- System.out.println("Warning: parameter polygon-file is ignored because --num-tiles is used");
- }
- }
- searchLimit = params.getSearchLimit();
- if (searchLimit < 1000){
- searchLimit = 1000;
- System.err.println("The --search-limit parameter must be 1000 or higher. Resetting to 1000.");
- }
- handleElementVersion = params.getHandleElementVersion();
- if (Arrays.asList("remove", "fake" , "keep").contains(handleElementVersion) == false){
- throw new IllegalArgumentException("the --handle-element-version parameter must be either remove, fake, or keep.");
- }
- ignoreBoundsTags = params.getIgnoreOsmBounds();
}
- /**
- * Calculate the areas that we are going to split into by getting the total area and
- * then subdividing down until each area has at most max-nodes nodes in it.
- */
- private List<Area> calculateAreas() throws XmlPullParserException {
-
- DensityMapCollector pass1Collector = new DensityMapCollector(resolution, ignoreBoundsTags);
- MapProcessor processor = pass1Collector;
-
- File densityData = new File("densities.txt");
- File densityOutData = null;
- if (densityData.exists() && densityData.isFile()){
- System.err.println("reading density data from " + densityData.getAbsolutePath());
- pass1Collector.readMap(densityData.getAbsolutePath());
- }
- else {
- densityOutData = new File(fileOutputDir,"densities-out.txt");
- osmFileHandler.process(processor);
- }
- System.out.println("in " + fileNameList.size() + (fileNameList.size() == 1 ? " file" : " files"));
- System.out.println("Time: " + new Date());
- if (densityOutData != null )
- pass1Collector.saveMap(densityOutData.getAbsolutePath());
-
- Area exactArea = pass1Collector.getExactArea();
- System.out.println("Exact map coverage read from input file(s) is " + exactArea);
- if (areasCalculator.getPolygons().size() == 1){
- Rectangle polgonsBoundingBox = areasCalculator.getPolygons().get(0).area.getBounds();
- exactArea = Area.calcArea(exactArea, polgonsBoundingBox);
- if (exactArea != null)
- System.out.println("Exact map coverage after applying bounding box of polygon-file is " + exactArea);
- else {
- System.out.println("Exact map coverage after applying bounding box of polygon-file is an empty area" );
- return Collections.emptyList();
- }
- }
-
- if (precompSeaDir != null){
- System.out.println("Counting nodes of precompiled sea data ...");
- long startSea = System.currentTimeMillis();
- DensityMapCollector seaCollector = new DensityMapCollector(resolution, true);
- PrecompSeaReader precompSeaReader = new PrecompSeaReader(exactArea, new File(precompSeaDir));
- precompSeaReader.processMap(seaCollector);
- pass1Collector.mergeSeaData(seaCollector, trim, resolution);
- System.out.println("Precompiled sea data pass took " + (System.currentTimeMillis()-startSea) + " ms");
- }
- Area roundedBounds = RoundingUtils.round(exactArea, resolution);
- SplittableDensityArea splittableArea = pass1Collector.getSplitArea(searchLimit, roundedBounds);
- if (splittableArea.hasData() == false){
- System.out.println("input file(s) have no data inside calculated bounding box");
- return Collections.emptyList();
- }
- System.out.println("Rounded map coverage is " + splittableArea.getBounds());
-
- splittableArea.setTrim(trim);
- splittableArea.setMapId(mapId);
- long startSplit = System.currentTimeMillis();
- List<Area> areas ;
- if (numTiles >= 2){
- System.out.println("Splitting nodes into " + numTiles + " areas");
- areas = splittableArea.split(numTiles);
- }
- else {
- System.out.println("Splitting nodes into areas containing a maximum of " + Utils.format(maxNodes) + " nodes each...");
- splittableArea.setMaxNodes(maxNodes);
- areas = splittableArea.split(areasCalculator.getPolygons());
- }
- if (areas != null && areas.isEmpty() == false)
- System.out.println("Creating the initial areas took " + (System.currentTimeMillis()- startSplit) + " ms");
- return areas;
- }
-
private OSMWriter[] createWriters(List<Area> areas) {
OSMWriter[] allWriters = new OSMWriter[areas.size()];
for (int j = 0; j < allWriters.length; j++) {
Area area = areas.get(j);
AbstractOSMWriter w;
- if ("pbf".equals(outputType))
+ String outputType = mainOptions.getOutput();
+ if ("pbf".equals(outputType))
w = new BinaryMapWriter(area, fileOutputDir, area.getMapId(), overlapAmount);
else if ("o5m".equals(outputType))
w = new O5mMapWriter(area, fileOutputDir, area.getMapId(), overlapAmount);
else if ("simulate".equals(outputType))
w = new PseudoOSMWriter(area);
- else
+ else
w = new OSMXMLWriter(area, fileOutputDir, area.getMapId(), overlapAmount);
- switch (handleElementVersion) {
- case "keep":
+ switch (mainOptions.getHandleElementVersion()) {
+ case "keep":
w.setVersionMethod(AbstractOSMWriter.KEEP_VERSION);
break;
- case "remove":
+ case "remove":
w.setVersionMethod(AbstractOSMWriter.REMOVE_VERSION);
break;
default:
@@ -635,34 +469,39 @@ public class Main {
}
return allWriters;
}
-
+
private void useProblemLists(DataStorer dataStorer) {
- problemList.calcMultiTileElements(dataStorer, osmFileHandler);
- if ("handle-problem-list".equals(stopAfter)){
- try {Thread.sleep(1000);}catch (InterruptedException e) {}
- System.err.println("stopped after " + stopAfter);
- throw new StopNoErrorException("stopped after " + stopAfter);
- }
+ problemList.calcMultiTileElements(dataStorer, osmFileHandler);
+ if ("handle-problem-list".equals(mainOptions.getStopAfter())) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+ throw new StopNoErrorException(mainOptions.getStopAfter());
+ }
}
-
+
/**
- * Final pass(es), we have the areas so parse the file(s) again.
- * @param dataStorer collects data used in different program passes
+ * Final pass(es), we have the areas so parse the file(s) again.
+ *
+ * @param dataStorer
+ * collects data used in different program passes
*/
private void writeTiles(DataStorer dataStorer) throws IOException {
- List<Area> areas = dataStorer.getAreaDictionary().getAreas();
- // the final split passes,
+ List<Area> areas = dataStorer.getAreaDictionary().getAreas();
+ // the final split passes,
dataStorer.switchToSeqAccess(fileOutputDir);
dataStorer.setWriters(createWriters(areas));
System.out.println("Distributing data " + new Date());
-
- int numPasses = (int) Math.ceil((double) areas.size() / maxAreasPerPass);
- int areasPerPass = (int) Math.ceil((double) areas.size() / numPasses);
+
+ int numPasses = (int) Math.ceil((double) areas.size() / mainOptions.getMaxAreas());
+ int areasPerPass = (int) Math.ceil((double) areas.size() / numPasses);
long startDistPass = System.currentTimeMillis();
if (numPasses > 1) {
- System.out.println("Processing " + areas.size() + " areas in " + numPasses + " passes, " + areasPerPass + " areas at a time");
+ System.out.println("Processing " + areas.size() + " areas in " + numPasses + " passes, " + areasPerPass
+ + " areas at a time");
} else {
System.out.println("Processing " + areas.size() + " areas in a single pass");
}
@@ -670,20 +509,19 @@ public class Main {
int areaOffset = i * areasPerPass;
int numAreasThisPass = Math.min(areasPerPass, areas.size() - i * areasPerPass);
dataStorer.restartWriterMaps();
- SplitProcessor processor = new SplitProcessor(dataStorer, areaOffset, numAreasThisPass, maxThreads);
-
- System.out.println("Starting distribution pass " + (i + 1) + " of " + numPasses + ", processing " + numAreasThisPass +
- " areas (" + areas.get(i * areasPerPass).getMapId() + " to " +
- areas.get(i * areasPerPass + numAreasThisPass - 1).getMapId() + ')');
+ SplitProcessor processor = new SplitProcessor(dataStorer, areaOffset, numAreasThisPass, mainOptions);
- osmFileHandler.process(processor);
+ System.out.println("Starting distribution pass " + (i + 1) + " of " + numPasses + ", processing "
+ + numAreasThisPass + " areas (" + areas.get(i * areasPerPass).getMapId() + " to "
+ + areas.get(i * areasPerPass + numAreasThisPass - 1).getMapId() + ')');
+ osmFileHandler.execute(processor);
}
System.out.println("Distribution pass(es) took " + (System.currentTimeMillis() - startDistPass) + " ms");
}
-
- static boolean testAndReportFname(String fileName, String type){
+
+ static boolean testAndReportFname(String fileName, String type) {
File f = new File(fileName);
- if (f.exists() == false || f.isFile() == false || f.canRead() == false){
+ if (f.exists() == false || f.isFile() == false || f.canRead() == false) {
String msg = "Error: " + type + " doesn't exist or is not a readable file: " + fileName;
System.out.println(msg);
System.err.println(msg);
@@ -691,6 +529,5 @@ public class Main {
}
return true;
}
-
}
diff --git a/src/uk/me/parabola/splitter/MapProcessor.java b/src/uk/me/parabola/splitter/MapProcessor.java
index 8d16063..4dd32cc 100644
--- a/src/uk/me/parabola/splitter/MapProcessor.java
+++ b/src/uk/me/parabola/splitter/MapProcessor.java
@@ -13,6 +13,8 @@
package uk.me.parabola.splitter;
+import java.util.concurrent.BlockingQueue;
+
public interface MapProcessor {
/**
@@ -75,4 +77,17 @@ public interface MapProcessor {
* should be called again with a new reader for all these input files.
*/
boolean endMap();
+
+ /**
+ * For use with the producer/consumer pattern
+ * @param queue
+ * @return
+ */
+ boolean consume(BlockingQueue<OSMMessage> queue);
+
+ /**
+ * Called for each single input file
+ */
+ void startFile();
+
}
diff --git a/src/uk/me/parabola/splitter/MapReader.java b/src/uk/me/parabola/splitter/MapReader.java
deleted file mode 100644
index 6c3de3d..0000000
--- a/src/uk/me/parabola/splitter/MapReader.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright (c) 2009, Chris Miller
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-
-package uk.me.parabola.splitter;
-
-public interface MapReader {
- //long getNodeCount();
-
- //long getWayCount();
-
- //long getRelationCount();
-
- //int getMinNodeId();
-
- //int getMaxNodeId();
-}
diff --git a/src/uk/me/parabola/splitter/MultiTileProcessor.java b/src/uk/me/parabola/splitter/MultiTileProcessor.java
index 02dee4c..888a46c 100644
--- a/src/uk/me/parabola/splitter/MultiTileProcessor.java
+++ b/src/uk/me/parabola/splitter/MultiTileProcessor.java
@@ -13,16 +13,16 @@
package uk.me.parabola.splitter;
import uk.me.parabola.splitter.Relation.Member;
-
+import uk.me.parabola.splitter.tools.Long2IntClosedMap;
+import uk.me.parabola.splitter.tools.Long2IntClosedMapFunction;
+import uk.me.parabola.splitter.tools.OSMId2ObjectMap;
+import uk.me.parabola.splitter.tools.SparseBitSet;
import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectMap.Entry;
import it.unimi.dsi.fastutil.longs.LongArrayList;
-import it.unimi.dsi.fastutil.objects.ObjectIterator;
-
import java.awt.Point;
import java.awt.Rectangle;
import java.util.ArrayList;
-import java.util.BitSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -48,7 +48,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
private int phase = PHASE1_RELS_ONLY;
private final DataStorer dataStorer;
- private final AreaDictionaryInt multiTileDictionary;
+ private final AreaDictionary areaDictionary;
private Long2ObjectLinkedOpenHashMap<MTRelation> relMap = new Long2ObjectLinkedOpenHashMap<>();
private Long2IntClosedMapFunction nodeWriterMap;
private Long2IntClosedMapFunction wayWriterMap;
@@ -58,12 +58,13 @@ class MultiTileProcessor extends AbstractMapProcessor {
private SparseBitSet problemRels = new SparseBitSet();
private SparseBitSet neededWays = new SparseBitSet();
private SparseBitSet neededNodes = new SparseBitSet();
- private OSMId2ObjectMap<Rectangle> wayBboxMap;
+ private OSMId2ObjectMap<Rectangle> wayBboxMap = new OSMId2ObjectMap<>();
private SparseBitSet mpWays = new SparseBitSet();
- private OSMId2ObjectMap<JoinedWay> mpWayEndNodesMap;
+ private OSMId2ObjectMap<JoinedWay> mpWayEndNodesMap = new OSMId2ObjectMap<>();
/** each bit represents one area/tile */
- private final BitSet workWriterSet;
+ private final AreaSet workWriterSet = new AreaSet();
private long lastCoordId = Long.MIN_VALUE;
+
private int foundWays;
private int neededNodesCount;
private int neededWaysCount;
@@ -73,7 +74,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
MultiTileProcessor(DataStorer dataStorer, LongArrayList problemWayList, LongArrayList problemRelList) {
this.dataStorer = dataStorer;
- multiTileDictionary = dataStorer.getMultiTileDictionary();
+ this.areaDictionary = dataStorer.getAreaDictionary();
for (long id: problemWayList){
neededWays.set(id);
}
@@ -81,7 +82,10 @@ class MultiTileProcessor extends AbstractMapProcessor {
problemRels.set(id);
}
// we allocate this once to avoid massive resizing with large number of tiles
- workWriterSet = new BitSet();
+ neededMpWaysCount = mpWays.cardinality();
+ if (problemRelList.isEmpty()) {
+ phase = PHASE2_WAYS_ONLY;
+ }
return;
}
@@ -106,7 +110,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
}
@Override
public boolean skipRels() {
- if (phase == PHASE1_RELS_ONLY)
+ if (phase == PHASE1_RELS_ONLY && problemRels.cardinality() > 0)
return false;
return true;
}
@@ -166,9 +170,9 @@ class MultiTileProcessor extends AbstractMapProcessor {
}
int wayWriterIdx;
if (workWriterSet.isEmpty())
- wayWriterIdx = AreaDictionaryInt.UNASSIGNED;
+ wayWriterIdx = UNASSIGNED;
else
- wayWriterIdx = multiTileDictionary.translate(workWriterSet);
+ wayWriterIdx = areaDictionary.translate(workWriterSet);
try{
wayWriterMap.add(way.getId(), wayWriterIdx);
@@ -183,8 +187,8 @@ class MultiTileProcessor extends AbstractMapProcessor {
if (!neededWays.get(way.getId()))
return;
int wayWriterIdx = wayWriterMap.getRandom(way.getId());
- if (wayWriterIdx != AreaDictionaryInt.UNASSIGNED){
- BitSet wayWriterSet = multiTileDictionary.getBitSet(wayWriterIdx);
+ if (wayWriterIdx != UNASSIGNED){
+ AreaSet wayWriterSet = areaDictionary.getSet(wayWriterIdx);
for (long id : way.getRefs()) {
addOrMergeWriters(nodeWriterMap, wayWriterSet, wayWriterIdx, id);
}
@@ -217,18 +221,11 @@ class MultiTileProcessor extends AbstractMapProcessor {
markParentRels();
}
// free memory for rels that are not causing any trouble
- ObjectIterator<Entry<MTRelation>> it = relMap.long2ObjectEntrySet().fastIterator();
- while (it.hasNext()) {
- Entry<MTRelation> pairs = it.next();
- if (!problemRels.get(pairs.getLongKey())){
- it.remove();
- }
- }
+ relMap.long2ObjectEntrySet().removeIf(e -> !problemRels.get(e.getLongKey()));
problemRels = null;
// reallocate to the needed size
relMap = new Long2ObjectLinkedOpenHashMap<>(relMap);
- mpWayEndNodesMap = new OSMId2ObjectMap<>();
//System.out.println("Finished adding parents and members of problem relations to problem lists.");
System.out.println("Finished adding members of problem relations to problem lists.");
stats("starting to collect ids of needed way nodes ...");
@@ -240,9 +237,8 @@ class MultiTileProcessor extends AbstractMapProcessor {
stats("Finished collecting problem ways.");
neededNodesCount = neededNodes.cardinality();
// critical part: we have to allocate possibly large arrays here
- nodeWriterMap = new Long2IntClosedMap("node", neededNodesCount, AreaDictionaryInt.UNASSIGNED);
- wayWriterMap = new Long2IntClosedMap("way", foundWays, AreaDictionaryInt.UNASSIGNED);
- wayBboxMap = new OSMId2ObjectMap<>();
+ nodeWriterMap = new Long2IntClosedMap("node", neededNodesCount, UNASSIGNED);
+ wayWriterMap = new Long2IntClosedMap("way", foundWays, UNASSIGNED);
dataStorer.setWriterMap(DataStorer.NODE_TYPE, nodeWriterMap);
dataStorer.setWriterMap(DataStorer.WAY_TYPE, wayWriterMap);
nodeLons = new int[neededNodesCount];
@@ -269,12 +265,13 @@ class MultiTileProcessor extends AbstractMapProcessor {
mergeRelMemWriters();
propagateWritersOfRelsToMembers();
+ mpWayEndNodesMap.clear();
wayBboxMap = null;
- relWriterMap = new Long2IntClosedMap("rel", relMap.size(), AreaDictionaryInt.UNASSIGNED);
+ relWriterMap = new Long2IntClosedMap("rel", relMap.size(), UNASSIGNED);
for (Entry<MTRelation> entry : relMap.long2ObjectEntrySet()){
int val = entry.getValue().getMultiTileWriterIndex();
- if (val != AreaDictionaryInt.UNASSIGNED){
+ if (val != UNASSIGNED){
try{
relWriterMap.add(entry.getLongKey(), val);
}catch (IllegalArgumentException e){
@@ -319,9 +316,9 @@ class MultiTileProcessor extends AbstractMapProcessor {
* @return
*/
private void MarkNeededMembers(MTRelation rel, int depth, ArrayList<MTRelation> visited){
- if (rel.getVisitId() == visitId)
+ if (rel.getLastVisitId() == visitId)
return;
- rel.setVisitId(visitId);
+ rel.setLastVisitId(visitId);
if (depth > 15){
System.out.println("MarkNeededMembers reached max. depth: " + rel.getId() + " " + depth);
return ;
@@ -340,7 +337,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
MTRelation subRel = relMap.get(memId);
if (subRel == null)
continue;
- if (subRel.getVisitId() == visitId)
+ if (subRel.getLastVisitId() == visitId)
loopAction(rel, subRel, visited);
else {
problemRels.set(memId);
@@ -388,7 +385,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
if (false == (rel.hasWayMembers() || rel.hasNodeMembers()) )
continue;
- BitSet writerSet = new BitSet();
+ AreaSet writerSet = new AreaSet();
for (int i = 0; i < rel.numMembers; i++){
long memId = rel.memRefs[i];
boolean memFound = false;
@@ -401,8 +398,8 @@ class MultiTileProcessor extends AbstractMapProcessor {
}
else if (rel.memTypes[i] == MEM_WAY_TYPE){
int idx = wayWriterMap.getRandom(memId);
- if (idx != AreaDictionaryInt.UNASSIGNED){
- writerSet.or(multiTileDictionary.getBitSet(idx));
+ if (idx != UNASSIGNED){
+ writerSet.or(areaDictionary.getSet(idx));
memFound = true;
}
if (wayBboxMap.get(memId) != null)
@@ -416,7 +413,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
}
}
if (!writerSet.isEmpty()){
- int idx = multiTileDictionary.translate(writerSet);
+ int idx = areaDictionary.translate(writerSet);
rel.setMultiTileWriterIndex(idx);
}
}
@@ -431,7 +428,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
ArrayList<MTRelation> visited = new ArrayList<>();
for (MTRelation rel: relMap.values()){
- BitSet relWriters = new BitSet();
+ AreaSet relWriters = new AreaSet();
if (rel.isMultiPolygon()){
if (rel.hasRelMembers()){
incVisitID();
@@ -440,7 +437,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
}
checkSpecialMP(relWriters, rel);
if (!relWriters.isEmpty()){
- int writerIdx = multiTileDictionary.translate(relWriters);
+ int writerIdx = areaDictionary.translate(relWriters);
rel.setMultiTileWriterIndex(writerIdx);
int touchedTiles = relWriters.cardinality();
if (touchedTiles > dataStorer.getNumOfAreas() / 2 && dataStorer.getNumOfAreas() > 10){
@@ -473,9 +470,9 @@ class MultiTileProcessor extends AbstractMapProcessor {
if (rel.wasAddedAsParent())
continue;
int relWriterIdx = rel.getMultiTileWriterIndex();
- if (relWriterIdx == AreaDictionaryInt.UNASSIGNED)
+ if (relWriterIdx == UNASSIGNED)
continue;
- BitSet relWriters = multiTileDictionary.getBitSet(relWriterIdx);
+ AreaSet relWriters = areaDictionary.getSet(relWriterIdx);
for (int i = 0; i < rel.numMembers; i++){
long memId = rel.memRefs[i];
switch (rel.memTypes[i]){
@@ -505,7 +502,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
}
int nodePos = -1;
try{
- nodePos = nodeWriterMap.add(id, AreaDictionaryInt.UNASSIGNED);
+ nodePos = nodeWriterMap.add(id, UNASSIGNED);
}catch (IllegalArgumentException e){
System.err.println(e.getMessage());
throw new SplitFailedException(NOT_SORTED_MSG);
@@ -525,17 +522,17 @@ class MultiTileProcessor extends AbstractMapProcessor {
* @return
*/
private void orSubRelWriters(MTRelation rel, int depth, ArrayList<MTRelation> visited ){
- if (rel.getVisitId() == visitId)
+ if (rel.getLastVisitId() == visitId)
return;
- rel.setVisitId(visitId);
+ rel.setLastVisitId(visitId);
if (depth > 15){
System.out.println("orSubRelWriters reached max. depth: " + rel.getId() + " " + depth);
return ;
}
- BitSet relWriters = new BitSet();
+ AreaSet relWriters = new AreaSet();
int relWriterIdx = rel.getMultiTileWriterIndex();
- if (relWriterIdx != AreaDictionaryInt.UNASSIGNED)
- relWriters.or(multiTileDictionary.getBitSet(relWriterIdx));
+ if (relWriterIdx != UNASSIGNED)
+ relWriters.or(areaDictionary.getSet(relWriterIdx));
boolean changed = false;
for (int i = 0; i < rel.numMembers; i++){
@@ -544,20 +541,19 @@ class MultiTileProcessor extends AbstractMapProcessor {
MTRelation subRel = relMap.get(memId);
if (subRel == null)
continue;
- if (subRel.getVisitId() == visitId)
+ if (subRel.getLastVisitId() == visitId)
loopAction(rel, subRel, visited);
else {
visited.add(rel);
orSubRelWriters(subRel, depth+1, visited);
visited.remove(visited.size()-1);
int memWriterIdx = subRel.getMultiTileWriterIndex();
- if (memWriterIdx == AreaDictionaryInt.UNASSIGNED || memWriterIdx == relWriterIdx){
+ if (memWriterIdx == UNASSIGNED || memWriterIdx == relWriterIdx){
continue;
}
- BitSet memWriters = multiTileDictionary.getBitSet(memWriterIdx);
- BitSet test = new BitSet();
- test.or(memWriters);
- test.andNot(relWriters);
+ AreaSet memWriters = areaDictionary.getSet(memWriterIdx);
+ AreaSet test = new AreaSet(memWriters);
+ test.subtract(relWriters);
if (test.isEmpty() == false){
relWriters.or(memWriters);
changed = true;
@@ -566,12 +562,10 @@ class MultiTileProcessor extends AbstractMapProcessor {
}
}
if (changed){
- relWriterIdx = multiTileDictionary.translate(relWriters);
- rel.setMultiTileWriterIndex(relWriterIdx);
+ rel.setMultiTileWriterIndex(areaDictionary.translate(relWriters));
}
}
-
/**
* Report some numbers regarding memory usage
* @param msg
@@ -588,7 +582,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
System.out.println(" " + neededNodes.getClass().getSimpleName() + " neededNodes contains now " + Utils.format(neededNodes.cardinality())+ " Ids.");
if (relMap != null)
System.out.println(" Number of stored relations: " + Utils.format(relMap.size()));
- System.out.println(" Number of stored tile combinations in multiTileDictionary: " + Utils.format(multiTileDictionary.size()));
+ System.out.println(" Number of stored tile combinations in multiTileDictionary: " + Utils.format(areaDictionary.size()));
if (phase == PHASE4_WAYS_ONLY)
dataStorer.stats(" ");
System.out.println("Status: " + msg);
@@ -597,11 +591,11 @@ class MultiTileProcessor extends AbstractMapProcessor {
/**
* Find all writer areas that intersect with a given bounding box.
- * @param writerSet an already allocate BitSet which may be modified
+ * @param writerSet an already allocate AreaSet which may be modified
* @param polygonBbox the bounding box
* @return true if any writer bbox intersects the polygon bbox
*/
- private boolean checkBoundingBox(BitSet writerSet, Rectangle polygonBbox){
+ private boolean checkBoundingBox(AreaSet writerSet, Rectangle polygonBbox){
boolean foundIntersection = false;
if (polygonBbox != null){
for (int i = 0; i < dataStorer.getNumOfAreas(); i++) {
@@ -623,21 +617,20 @@ class MultiTileProcessor extends AbstractMapProcessor {
* @param parentWriterIdx
* @param childId
*/
- private void addOrMergeWriters(Long2IntClosedMapFunction map, BitSet parentWriters, int parentWriterIdx, long childId) {
+ private void addOrMergeWriters(Long2IntClosedMapFunction map, AreaSet parentWriters, int parentWriterIdx, long childId) {
int pos = map.getKeyPos(childId);
if (pos < 0)
return;
int childWriterIdx = map.getRandom(childId);
- if (childWriterIdx != AreaDictionaryInt.UNASSIGNED){
+ if (childWriterIdx != UNASSIGNED){
// we have already calculated writers for this child
if (parentWriterIdx == childWriterIdx)
return;
// we have to merge (without changing the stored BitSets!)
- BitSet childWriters = multiTileDictionary.getBitSet(childWriterIdx);
- BitSet mergedWriters = new BitSet();
+ AreaSet childWriters = areaDictionary.getSet(childWriterIdx);
+ AreaSet mergedWriters = new AreaSet(parentWriters);
mergedWriters.or(childWriters);
- mergedWriters.or(parentWriters);
- childWriterIdx = multiTileDictionary.translate(mergedWriters);
+ childWriterIdx = areaDictionary.translate(mergedWriters);
}
else
childWriterIdx = parentWriterIdx;
@@ -646,20 +639,19 @@ class MultiTileProcessor extends AbstractMapProcessor {
/**
* Calculate the writers for a given point specified by coordinates.
- * Set the corresponding bit in the BitSet.
- * @param writerSet an already allocate BitSet which may be modified
+ * Set the corresponding bit in the AreaSet.
+ * @param writerSet an already allocate AreaSet which may be modified
* @param mapLat latitude value
* @param mapLon longitude value
* @return true if a writer was found
*/
- private boolean addWritersOfPoint(BitSet writerSet, int mapLat, int mapLon){
+ private boolean addWritersOfPoint(AreaSet writerSet, int mapLat, int mapLon){
AreaGridResult writerCandidates = dataStorer.getGrid().get(mapLat,mapLon);
if (writerCandidates == null)
return false;
boolean foundWriter = false;
- for (int i = 0; i < writerCandidates.l.size(); i++) {
- int n = writerCandidates.l.getShort(i);
+ for (int n : writerCandidates.set) {
Area extbbox = dataStorer.getExtendedArea(n);
boolean found = (writerCandidates.testNeeded) ? extbbox.contains(mapLat, mapLon) : true;
foundWriter |= found;
@@ -671,13 +663,13 @@ class MultiTileProcessor extends AbstractMapProcessor {
/**
* Find tiles that are crossed by a line specified by two points.
- * @param writerSet an already allocate BitSet which may be modified
- * @param possibleWriters a BitSet that contains the writers to be checked
+ * @param writerSet an already allocate AreaSet which may be modified
+ * @param possibleWriters a AreaSet that contains the writers to be checked
* @param p1 first point of line
* @param p2 second point of line
*/
- private void addWritersOfCrossedTiles(BitSet writerSet, final BitSet possibleWriters, final Point p1,final Point p2){
- for (int i = possibleWriters.nextSetBit(0); i >= 0; i = possibleWriters.nextSetBit(i+1)){
+ private void addWritersOfCrossedTiles(AreaSet writerSet, final AreaSet possibleWriters, final Point p1,final Point p2){
+ for (int i : possibleWriters) {
Rectangle writerBbox = Utils.area2Rectangle(dataStorer.getArea(i), 1);
if (writerBbox.intersectsLine(p1.x,p1.y,p2.x,p2.y))
writerSet.set(i);
@@ -686,12 +678,12 @@ class MultiTileProcessor extends AbstractMapProcessor {
/**
* Calculate all writer areas that are crossed or directly "touched" by a way.
- * @param writerSet an already allocate BitSet which may be modified
+ * @param writerSet an already allocate AreaSet which may be modified
* @param wayBbox
* @param wayId the id that identifies the way
* @param wayRefs list with the node references
*/
- private void addWritersOfWay (BitSet writerSet, Rectangle wayBbox, long wayId, LongArrayList wayRefs){
+ private void addWritersOfWay (AreaSet writerSet, Rectangle wayBbox, long wayId, LongArrayList wayRefs){
int numRefs = wayRefs.size();
int foundNodes = 0;
boolean needsCrossTileCheck = false;
@@ -715,13 +707,12 @@ class MultiTileProcessor extends AbstractMapProcessor {
if (numWriters == 0)
needsCrossTileCheck = true;
else if (numWriters > 1){
- short idx = dataStorer.getAreaDictionary().translate(writerSet);
- if (dataStorer.getAreaDictionary().mayCross(idx))
+ if (dataStorer.getAreaDictionary().mayCross(writerSet))
needsCrossTileCheck = true;
}
}
if (needsCrossTileCheck){
- BitSet possibleWriters = new BitSet();
+ AreaSet possibleWriters = new AreaSet();
checkBoundingBox(possibleWriters ,wayBbox);
// the way did cross a border tile
for (int i = 0; i<numRefs; i++) {
@@ -781,7 +772,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
// unlikely
visitId = 0;
for (Entry<MTRelation> entry : relMap.long2ObjectEntrySet()){
- entry.getValue().setVisitId(visitId);
+ entry.getValue().setLastVisitId(visitId);
}
}
visitId++;
@@ -818,7 +809,7 @@ class MultiTileProcessor extends AbstractMapProcessor {
* @param relWriters
* @param rel
*/
- private void checkSpecialMP(BitSet relWriters, MTRelation rel) {
+ private void checkSpecialMP(AreaSet relWriters, MTRelation rel) {
long[] joinedWays = null;
List<Long> wayMembers = new LinkedList<>();
LongArrayList polygonWays = new LongArrayList();
@@ -954,8 +945,8 @@ class MultiTileProcessor extends AbstractMapProcessor {
protected final int numMembers;
private final String name;
- private int multiTileWriterIndex = -1;
- private int visitId;
+ private int multiTileWriterIndex = UNASSIGNED;
+ private int lastVisitId;
private short flags; // flags for the MultiTileProcessor
public MTRelation(Relation rel){
@@ -1070,12 +1061,12 @@ class MultiTileProcessor extends AbstractMapProcessor {
this.flags |= IS_MP;
}
- public int getVisitId() {
- return visitId;
+ public int getLastVisitId() {
+ return lastVisitId;
}
- public void setVisitId(int visitId) {
- this.visitId = visitId;
+ public void setLastVisitId(int visitId) {
+ this.lastVisitId = visitId;
}
public String getName(){
diff --git a/src/uk/me/parabola/splitter/OSMFileHandler.java b/src/uk/me/parabola/splitter/OSMFileHandler.java
index 9acd86f..215ba15 100644
--- a/src/uk/me/parabola/splitter/OSMFileHandler.java
+++ b/src/uk/me/parabola/splitter/OSMFileHandler.java
@@ -1,3 +1,16 @@
+/*
+ * Copyright (c) 2016, Gerd Petermann
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
package uk.me.parabola.splitter;
import java.io.File;
@@ -5,109 +18,148 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.io.RandomAccessFile;
import java.io.Reader;
+import java.nio.channels.FileChannel;
import java.util.HashMap;
import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
import org.xmlpull.v1.XmlPullParserException;
import crosby.binary.file.BlockInputStream;
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
+import uk.me.parabola.splitter.parser.BinaryMapParser;
+import uk.me.parabola.splitter.parser.O5mMapParser;
+import uk.me.parabola.splitter.parser.OSMXMLParser;
/**
* A class which stores parameters needed to process input (OSM) files
- *
+ *
* @author Gerd Petermann
*
*/
public class OSMFileHandler {
- /** list of OSM input files to process */
- private List<String> filenames;
- // for faster access on blocks in pbf files
- private final HashMap<String, ShortArrayList> blockTypeMap = new HashMap<>();
- // for faster access on blocks in o5m files
- private final HashMap<String, long[]> skipArrayMap = new HashMap<>();
-
- // Whether or not the source OSM file(s) contain strictly nodes first, then ways, then rels,
- // or they're all mixed up. Running with mixed enabled takes longer.
- private boolean mixed;
+ /** list of OSM input files to process */
+ private List<String> filenames;
+ // for faster access on blocks in pbf files
+ private final HashMap<String, ShortArrayList> blockTypeMap = new HashMap<>();
+ // for faster access on blocks in o5m files
+ private final HashMap<String, long[]> skipArrayMap = new HashMap<>();
+
+ // Whether or not the source OSM file(s) contain strictly nodes first, then
+ // ways, then rels,
+ // or they're all mixed up. Running with mixed enabled takes longer.
+ private boolean mixed;
+
+ private int maxThreads = 1;
+
+ /** if this is true we may not want to use producer/consumer pattern */
+ private MapProcessor realProcessor;
+
+ public void setFileNames(List<String> filenames) {
+ this.filenames = filenames;
+ }
+
+ public void setMixed(boolean f) {
+ mixed = f;
+ }
- private int maxThreads = 1;
+ public void setMaxThreads(int maxThreads) {
+ this.maxThreads = maxThreads;
+ }
- public void setFileNames (List<String> filenames) {
- this.filenames = filenames;
- }
-
- public void setMixed(boolean f) {
- mixed = f;
- }
-
- public void setMaxThreads (int maxThreads) {
- this.maxThreads = maxThreads;
- }
-
- public boolean process(MapProcessor processor) {
- // Create both an XML reader and a binary reader, Dispatch each input to the
- // Appropriate parser.
-
- for (int i = 0; i < filenames.size(); i++){
- String filename = filenames.get(i);
- System.out.println("Processing " + filename);
- if (i == 1 && processor instanceof DensityMapCollector){
- ((DensityMapCollector) processor).checkBounds();
- }
-
- try {
- if (filename.endsWith(".o5m")) {
- File file = new File(filename);
- try(InputStream stream = new FileInputStream(file)){
- long[] skipArray = skipArrayMap.get(filename);
- O5mMapParser o5mParser = new O5mMapParser(processor, stream, skipArray);
- o5mParser.parse();
- if (skipArray == null){
- skipArray = o5mParser.getSkipArray();
- skipArrayMap.put(filename, skipArray);
- }
- }
- }
- else if (filename.endsWith(".pbf")) {
- // Is it a binary file?
- File file = new File(filename);
- ShortArrayList blockTypes = blockTypeMap.get(filename);
- BinaryMapParser binParser = new BinaryMapParser(processor, blockTypes, 1);
- try(InputStream stream = new FileInputStream(file)){
- BlockInputStream blockinput = (new BlockInputStream(stream, binParser));
- blockinput.process();
- if (blockTypes == null){
- // remember this file
- blockTypes = binParser.getBlockList();
- blockTypeMap.put(filename, blockTypes);
- }
- }
- } else {
- // No, try XML.
- try (Reader reader = Utils.openFile(filename, maxThreads > 1)){
- OSMParser parser = new OSMParser(processor, mixed);
- parser.setReader(reader);
- parser.parse();
- }
- }
- } catch (FileNotFoundException e) {
- System.out.println(e);
- throw new SplitFailedException("ERROR: file " + filename + " was not found");
- } catch (XmlPullParserException e) {
- e.printStackTrace();
- throw new SplitFailedException("ERROR: file " + filename + " is not a valid OSM XML file");
- } catch (IllegalArgumentException e) {
- e.printStackTrace();
- throw new SplitFailedException("ERROR: file " + filename + " contains unexpected data");
- } catch (IOException e) {
- e.printStackTrace();
- throw new SplitFailedException("ERROR: file " + filename + " caused I/O exception");
- }
- }
- boolean done = processor.endMap();
- return done;
- }
+ public boolean process(MapProcessor processor) {
+ // create appropriate parser for each input file
+ for (String filename : filenames) {
+ System.out.println("Processing " + filename);
+ processor.startFile();
+ try {
+ if (filename.endsWith(".o5m")) {
+ File file = new File(filename);
+ try (RandomAccessFile raf = new RandomAccessFile(file, "r");
+ FileChannel fileChannel = raf.getChannel()) {
+ long[] skipArray = skipArrayMap.get(filename);
+ O5mMapParser o5mParser = new O5mMapParser(processor, fileChannel, skipArray);
+ o5mParser.parse();
+ if (skipArray == null) {
+ skipArray = o5mParser.getSkipArray();
+ skipArrayMap.put(filename, skipArray);
+ }
+ }
+ } else if (filename.endsWith(".pbf")) {
+ // Is it a binary file?
+ File file = new File(filename);
+ ShortArrayList blockTypes = blockTypeMap.get(filename);
+ BinaryMapParser binParser = new BinaryMapParser(processor, blockTypes, 1);
+ try (InputStream stream = new FileInputStream(file)) {
+ BlockInputStream blockinput = (new BlockInputStream(stream, binParser));
+ blockinput.process();
+ if (blockTypes == null) {
+ // remember this file
+ blockTypes = binParser.getBlockList();
+ blockTypeMap.put(filename, blockTypes);
+ }
+ }
+ } else {
+ // No, try XML.
+ try (Reader reader = Utils.openFile(filename, maxThreads > 1)) {
+ OSMXMLParser parser = new OSMXMLParser(processor, mixed);
+ parser.setReader(reader);
+ parser.parse();
+ }
+ }
+ } catch (FileNotFoundException e) {
+ System.out.println(e);
+ throw new SplitFailedException("ERROR: file " + filename + " was not found");
+ } catch (XmlPullParserException e) {
+ e.printStackTrace();
+ throw new SplitFailedException("ERROR: file " + filename + " is not a valid OSM XML file");
+ } catch (IllegalArgumentException e) {
+ e.printStackTrace();
+ throw new SplitFailedException("ERROR: file " + filename + " contains unexpected data");
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new SplitFailedException("ERROR: file " + filename + " caused I/O exception");
+ } catch (RuntimeException e) {
+ e.printStackTrace();
+ throw new SplitFailedException("ERROR: file " + filename + " caused exception");
+ }
+ }
+ return processor.endMap();
+ }
+
+
+ RuntimeException exception = null;
+ public boolean execute(MapProcessor processor) {
+ realProcessor = processor;
+ if (maxThreads == 1)
+ return process(processor);
+
+ // use two threads
+ BlockingQueue<OSMMessage> queue = new ArrayBlockingQueue<>(10);
+ QueueProcessor queueProcessor = new QueueProcessor(queue, realProcessor);
+
+ // start producer thread
+ new Thread("producer for " + realProcessor.getClass().getSimpleName()){
+ public void run(){
+ try {
+ process(queueProcessor);
+ } catch (SplitFailedException e) {
+ try {
+ queue.put(new OSMMessage(OSMMessage.Type.EXIT));
+ exception = e;
+ } catch (InterruptedException e1) {
+ e1.printStackTrace();
+ }
+ }
+ }
+ }.start();
+ boolean done = realProcessor.consume(queue);
+ if (exception != null)
+ throw exception;
+ return done;
+ }
}
diff --git a/src/uk/me/parabola/splitter/OSMMessage.java b/src/uk/me/parabola/splitter/OSMMessage.java
new file mode 100644
index 0000000..0ce3344
--- /dev/null
+++ b/src/uk/me/parabola/splitter/OSMMessage.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2016, Gerd Petermann
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter;
+
+import java.util.List;
+
+/**
+ * For OSM data which is passed between parsers and processors
+ * @author Gerd Petermann
+ *
+ */
+public class OSMMessage {
+ public enum Type {START_FILE, ELEMENTS, BOUNDS, END_MAP, EXIT};
+
+ // either el or bounds must be null
+ List<Element> elements;
+ Area bounds;
+ Type type;
+
+ public OSMMessage(List<Element> elements) {
+ this.elements = elements;
+ type = Type.ELEMENTS;
+ }
+
+ public OSMMessage(Area bounds) {
+ this.bounds = bounds;
+ type = Type.BOUNDS;
+ }
+
+ public OSMMessage(Type t) {
+ assert !t.equals(Type.BOUNDS);
+ assert !t.equals(Type.ELEMENTS);
+ type = t;
+ }
+}
diff --git a/src/uk/me/parabola/splitter/ProblemListProcessor.java b/src/uk/me/parabola/splitter/ProblemListProcessor.java
index 6ea1604..6209bbd 100644
--- a/src/uk/me/parabola/splitter/ProblemListProcessor.java
+++ b/src/uk/me/parabola/splitter/ProblemListProcessor.java
@@ -13,17 +13,18 @@
package uk.me.parabola.splitter;
import uk.me.parabola.splitter.Relation.Member;
+import uk.me.parabola.splitter.args.SplitterParams;
+import uk.me.parabola.splitter.tools.SparseLong2IntMap;
import it.unimi.dsi.fastutil.longs.LongArrayList;
-
import java.util.Arrays;
-import java.util.BitSet;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.regex.Pattern;
/**
* Find ways and relations that will be incomplete.
* Strategy:
- * - calculate the areas of each node, calculate and store a short that represents the combination of areas
+ * - calculate the areas of each node, calculate and store an integer that represents the combination of areas
* (this is done by the AreaDictionary)
* - a way is a problem way if its nodes are found in different combinations of areas
* - a relation is a problem relation if its members are found in different combinations of areas
@@ -33,21 +34,18 @@ class ProblemListProcessor extends AbstractMapProcessor {
private final static int PHASE1_NODES_AND_WAYS = 1;
private final static int PHASE2_RELS_ONLY = 2;
- private final SparseLong2ShortMapFunction coords;
- private final SparseLong2ShortMapFunction ways;
+ private final SparseLong2IntMap coords;
+ private final SparseLong2IntMap ways;
- private final AreaDictionaryShort areaDictionary;
+ private final AreaDictionary areaDictionary;
private final DataStorer dataStorer;
private final LongArrayList problemWays = new LongArrayList();
private final LongArrayList problemRels = new LongArrayList();
/** each bit represents one distinct area */
- private final BitSet areaSet = new BitSet();
+ private final AreaSet areaSet = new AreaSet();
private int phase = PHASE1_NODES_AND_WAYS;
- // for statistics
- //private long countQuickTest = 0;
- //private long countFullTest = 0;
private long countCoords = 0;
private final int areaOffset;
private final int lastAreaOffset;
@@ -59,11 +57,11 @@ class ProblemListProcessor extends AbstractMapProcessor {
private final HashSet<String> wantedBoundaryTagValues;
ProblemListProcessor(DataStorer dataStorer, int areaOffset,
- int numAreasThisPass, String[] boundaryTagList) {
+ int numAreasThisPass, SplitterParams mainOptions) {
this.dataStorer = dataStorer;
this.areaDictionary = dataStorer.getAreaDictionary();
if (dataStorer.getUsedWays() == null){
- ways = SparseLong2ShortMap.createMap("way");
+ ways = new SparseLong2IntMap("way");
ways.defaultReturnValue(UNASSIGNED);
dataStorer.setUsedWays(ways);
}
@@ -71,17 +69,20 @@ class ProblemListProcessor extends AbstractMapProcessor {
ways = dataStorer.getUsedWays();
this.areaIndex = dataStorer.getGrid();
- this.coords = SparseLong2ShortMap.createMap("coord");
+ this.coords = new SparseLong2IntMap("coord");
this.coords.defaultReturnValue(UNASSIGNED);
this.isFirstPass = (areaOffset == 0);
this.areaOffset = areaOffset;
this.lastAreaOffset = areaOffset + numAreasThisPass - 1;
this.isLastPass = (areaOffset + numAreasThisPass == dataStorer.getNumOfAreas());
- if (boundaryTagList != null && boundaryTagList.length > 0)
- wantedBoundaryTagValues = new HashSet<>(Arrays.asList(boundaryTagList));
- else
+ String boundaryTagsParm = mainOptions.getBoundaryTags();
+ if ("use-exclude-list".equals(boundaryTagsParm))
wantedBoundaryTagValues = null;
- setWantedAdminLevel(5);
+ else {
+ String[] boundaryTags = boundaryTagsParm.split(Pattern.quote(","));
+ wantedBoundaryTagValues = new HashSet<>(Arrays.asList(boundaryTags));
+ }
+ setWantedAdminLevel(mainOptions.getWantedAdminLevel());
}
public void setWantedAdminLevel(int adminLevel) {
@@ -129,44 +130,33 @@ class ProblemListProcessor extends AbstractMapProcessor {
if (phase == PHASE2_RELS_ONLY)
return;
int countAreas = 0;
- short lastUsedArea = UNASSIGNED;
- short areaIdx = UNASSIGNED;
+ int lastUsedArea = UNASSIGNED;
+ int areaIdx = UNASSIGNED;
AreaGridResult areaCandidates = areaIndex.get(node);
if (areaCandidates == null)
return;
- if (areaCandidates.l.size() > 1)
- areaSet.clear();
- for (int i = 0; i < areaCandidates.l.size(); i++) {
- int n = areaCandidates.l.getShort(i);
+ areaSet.clear();
+
+ for (int n : areaCandidates.set) {
if (n < areaOffset || n > lastAreaOffset)
continue;
- boolean found;
- if (areaCandidates.testNeeded){
- found = dataStorer.getArea(n).contains(node);
- //++countFullTest;
- }
- else{
- found = true;
- //++countQuickTest;
- }
- if (found) {
+ if (areaCandidates.testNeeded ? areaDictionary.getArea(n).contains(node) : true) {
areaSet.set(n);
++countAreas;
- lastUsedArea = (short) n;
+ lastUsedArea = n;
}
}
if (countAreas > 0){
if (countAreas > 1)
areaIdx = areaDictionary.translate(areaSet);
else
- areaIdx = AreaDictionaryShort.translate(lastUsedArea); // no need to do lookup in the dictionary
+ areaIdx = AreaDictionary.translate(lastUsedArea); // no need to do lookup in the dictionary
coords.put(node.getId(), areaIdx);
++countCoords;
- if (countCoords % 10000000 == 0){
- System.out.println("coord MAP occupancy: " + Utils.format(countCoords) + ", number of area dictionary entries: " + areaDictionary.size() + " of " + ((1<<16) - 1));
- coords.stats(0);
+ if (countCoords % 10_000_000 == 0){
+ System.out.println("coord MAP occupancy: " + Utils.format(countCoords) + ", number of area dictionary entries: " + areaDictionary.size());
}
}
}
@@ -177,29 +167,20 @@ class ProblemListProcessor extends AbstractMapProcessor {
return;
boolean maybeChanged = false;
int oldclIndex = UNASSIGNED;
- short wayAreaIdx;
areaSet.clear();
- //for (long id: way.getRefs()){
- int refs = way.getRefs().size();
- for (int i = 0; i < refs; i++){
- long id = way.getRefs().getLong(i);
+ for (long id : way.getRefs()){
// Get the list of areas that the way is in.
- short clIdx = coords.get(id);
- if (clIdx == UNASSIGNED){
- continue;
- }
- if (oldclIndex != clIdx){
- BitSet cl = areaDictionary.getBitSet(clIdx);
- areaSet.or(cl);
+ int clIdx = coords.get(id);
+ if (clIdx != UNASSIGNED && oldclIndex != clIdx){
+ areaSet.or(areaDictionary.getSet(clIdx));
oldclIndex = clIdx;
maybeChanged = true;
}
}
-
if (!isFirstPass && maybeChanged || isLastPass){
- wayAreaIdx = ways.get(way.getId());
+ int wayAreaIdx = ways.get(way.getId());
if (wayAreaIdx != UNASSIGNED)
- areaSet.or(areaDictionary.getBitSet(wayAreaIdx));
+ areaSet.or(areaDictionary.getSet(wayAreaIdx));
}
if (isLastPass){
@@ -207,11 +188,11 @@ class ProblemListProcessor extends AbstractMapProcessor {
problemWays.add(way.getId());
}
}
- if (maybeChanged && areaSet.isEmpty() == false){
- wayAreaIdx = areaDictionary.translate(areaSet);
- ways.put(way.getId(), wayAreaIdx);
+ if (maybeChanged && !areaSet.isEmpty()){
+ ways.put(way.getId(), areaDictionary.translate(areaSet));
}
}
+
// default exclude list for boundary tag
private final static HashSet<String> unwantedBoundaryTagValues = new HashSet<>(
Arrays.asList("administrative", "postal_code", "political"));
@@ -251,7 +232,7 @@ class ProblemListProcessor extends AbstractMapProcessor {
if (useThis)
break;
}
- if (isMPRelType && (isWantedBoundary || hasBoundaryTag == false))
+ if (isMPRelType && (isWantedBoundary || !hasBoundaryTag))
useThis = true;
else if (isMPRelType && hasBoundaryTag && admin_level != null){
if (wantedBoundaryAdminLevels.contains(admin_level))
@@ -265,32 +246,30 @@ class ProblemListProcessor extends AbstractMapProcessor {
if (!isFirstPass){
relAreaIdx = dataStorer.getUsedRels().get(rel.getId());
if (relAreaIdx != null)
- areaSet.or(dataStorer.getMultiTileDictionary().getBitSet(relAreaIdx));
+ areaSet.or(areaDictionary.getSet(relAreaIdx));
}
- short oldclIndex = UNASSIGNED;
- short oldwlIndex = UNASSIGNED;
+ int oldclIndex = UNASSIGNED;
+ int oldwlIndex = UNASSIGNED;
//System.out.println("r" + rel.getId() + " " + rel.getMembers().size());
for (Member mem : rel.getMembers()) {
long id = mem.getRef();
if (mem.getType().equals("node")) {
- short clIdx = coords.get(id);
+ int clIdx = coords.get(id);
if (clIdx != UNASSIGNED){
if (oldclIndex != clIdx){
- BitSet wl = areaDictionary.getBitSet(clIdx);
- areaSet.or(wl);
+ areaSet.or(areaDictionary.getSet(clIdx));
}
oldclIndex = clIdx;
}
} else if (mem.getType().equals("way")) {
- short wlIdx = ways.get(id);
+ int wlIdx = ways.get(id);
if (wlIdx != UNASSIGNED){
if (oldwlIndex != wlIdx){
- BitSet wl = areaDictionary.getBitSet(wlIdx);
- areaSet.or(wl);
+ areaSet.or(areaDictionary.getSet(wlIdx));
}
oldwlIndex = wlIdx;
}
@@ -305,18 +284,16 @@ class ProblemListProcessor extends AbstractMapProcessor {
} else {
// the relation is only in one distinct area
- relAreaIdx = dataStorer.getMultiTileDictionary().translate(areaSet);
- // store the info that the rel is only in one distinct area (-1 means pseudo-area)
- dataStorer.storeRelationArea(rel.getId(), relAreaIdx);
+ // store the info that the rel is only in one distinct area
+ dataStorer.storeRelationAreas(rel.getId(), areaSet);
}
return;
}
- relAreaIdx = dataStorer.getMultiTileDictionary().translate(areaSet);
+ relAreaIdx = areaDictionary.translate(areaSet);
dataStorer.getUsedRels().put(rel.getId(), relAreaIdx);
}
-
@Override
public boolean endMap() {
if (phase == PHASE1_NODES_AND_WAYS){
@@ -327,9 +304,10 @@ class ProblemListProcessor extends AbstractMapProcessor {
ways.stats(0);
if (isLastPass){
System.out.println("");
- System.out.println(" Number of stored shorts for ways: " + Utils.format(dataStorer.getUsedWays().size()));
- System.out.println(" Number of stored integers for rels: " + Utils.format(dataStorer.getUsedRels().size()));
- System.out.println(" Number of stored combis in big dictionary: " + Utils.format(dataStorer.getMultiTileDictionary().size()));
+ System.out.println(" Number of stored area combis for nodes: " + Utils.format(coords.size()));
+ System.out.println(" Number of stored area combis for ways: " + Utils.format(dataStorer.getUsedWays().size()));
+ System.out.println(" Number of stored Integers for rels: " + Utils.format(dataStorer.getUsedRels().size()));
+ System.out.println(" Number of stored combis in dictionary: " + Utils.format(areaDictionary.size()));
System.out.println(" Number of detected problem ways: " + Utils.format(problemWays.size()));
System.out.println(" Number of detected problem rels: " + Utils.format(problemRels.size()));
Utils.printMem();
@@ -344,7 +322,7 @@ class ProblemListProcessor extends AbstractMapProcessor {
* @param areaCombis
* @return true if the combination of distinct areas can contain a problem polygon
*/
- static boolean checkIfMultipleAreas(BitSet areaCombis){
+ static boolean checkIfMultipleAreas(AreaSet areaCombis){
// this returns a few false positives for those cases
// where a way or rel crosses two pseudo-areas at a
// place that is far away from the real areas
diff --git a/src/uk/me/parabola/splitter/ProblemLists.java b/src/uk/me/parabola/splitter/ProblemLists.java
index 4b28850..7c2a413 100644
--- a/src/uk/me/parabola/splitter/ProblemLists.java
+++ b/src/uk/me/parabola/splitter/ProblemLists.java
@@ -1,5 +1,20 @@
+/*
+ * Copyright (c) 2016, Gerd Petermann
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
package uk.me.parabola.splitter;
+import java.awt.Point;
+import java.awt.Rectangle;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
@@ -9,232 +24,565 @@ import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.PrintWriter;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;
import it.unimi.dsi.fastutil.longs.LongArrayList;
+import uk.me.parabola.splitter.args.SplitterParams;
public class ProblemLists {
- private final LongArrayList problemWays = new LongArrayList();
- private final LongArrayList problemRels = new LongArrayList();
- private final TreeSet<Long> calculatedProblemWays = new TreeSet<>();
- private final TreeSet<Long> calculatedProblemRels = new TreeSet<>();
-
- /**
- * Calculate lists of ways and relations that appear in multiple areas for a given list
- * of areas.
- * @param osmFileHandler
- * @param realAreas
- * @param wantedAdminLevel
- * @param boundaryTags
- * @param maxAreasPerPass
- * @param overlapAmount
- * @return
- */
- public DataStorer calcProblemLists(OSMFileHandler osmFileHandler, List<Area> realAreas, int wantedAdminLevel,
- String[] boundaryTags, int maxAreasPerPass, int overlapAmount) {
- long startProblemListGenerator = System.currentTimeMillis();
- ArrayList<Area> distinctAreas = AreasCalculator.getNonOverlappingAreas(realAreas);
- if (distinctAreas.size() > realAreas.size()) {
- System.err.println("Waring: The areas given in --split-file are overlapping.");
- Set<Integer> overlappingTiles = new TreeSet<>();
- for (int i = 0; i < realAreas.size(); i++) {
- Area a1 = realAreas.get(i);
- for (int j = i+1; j < realAreas.size(); j++) {
- Area a2 = realAreas.get(j);
- if (a1.intersects(a2)) {
- overlappingTiles.add(a1.getMapId());
- overlappingTiles.add(a2.getMapId());
- }
- }
- }
- if (!overlappingTiles.isEmpty()) {
- System.out.println("Overlaping tiles: " + overlappingTiles.toString());
- }
- }
- System.out.println("Generating problem list for " + distinctAreas.size() + " distinct areas");
- List<Area> workAreas = AreasCalculator.addPseudoAreas(distinctAreas);
-
- int numPasses = (int) Math.ceil((double) workAreas.size() / maxAreasPerPass);
- int areasPerPass = (int) Math.ceil((double) workAreas.size() / numPasses);
- if (numPasses > 1) {
- System.out.println("Processing " + distinctAreas.size() + " areas in " + numPasses + " passes, " + areasPerPass + " areas at a time");
- } else {
- System.out.println("Processing " + distinctAreas.size() + " areas in a single pass");
- }
-
- ArrayList<Area> allAreas = new ArrayList<>();
-
- System.out.println("Pseudo areas:");
- for (int j = 0;j < workAreas.size(); j++){
- Area area = workAreas.get(j);
- allAreas.add(area);
- if (area.isPseudoArea())
- System.out.println("Pseudo area " + area.getMapId() + " covers " + area);
- }
-
- DataStorer distinctDataStorer = new DataStorer(workAreas, overlapAmount);
- System.out.println("Starting problem-list-generator pass(es)");
-
- for (int pass = 0; pass < numPasses; pass++) {
- System.out.println("-----------------------------------");
- System.out.println("Starting problem-list-generator pass " + (pass+1) + " of " + numPasses);
- long startThisPass = System.currentTimeMillis();
- int areaOffset = pass * areasPerPass;
- int numAreasThisPass = Math.min(areasPerPass, workAreas.size() - pass * areasPerPass);
- ProblemListProcessor processor = new ProblemListProcessor(distinctDataStorer, areaOffset,
- numAreasThisPass, boundaryTags);
- processor.setWantedAdminLevel(wantedAdminLevel);
-
- boolean done = false;
- while (!done){
- done = osmFileHandler.process(processor);
- calculatedProblemWays.addAll(processor.getProblemWays());
- calculatedProblemRels.addAll(processor.getProblemRels());
- }
- System.out.println("Problem-list-generator pass " + (pass+1) + " took " + (System.currentTimeMillis() - startThisPass) + " ms");
- }
- System.out.println("Problem-list-generator pass(es) took " + (System.currentTimeMillis() - startProblemListGenerator) + " ms");
- DataStorer dataStorer = new DataStorer(realAreas, overlapAmount);
- dataStorer.translateDistinctToRealAreas(distinctDataStorer);
- return dataStorer;
- }
-
- /** Read user defined problematic relations and ways */
- public boolean readProblemIds(String problemFileName) {
- File fProblem = new File(problemFileName);
- boolean ok = true;
-
- if (!fProblem.exists()) {
- System.out.println("Error: problem file doesn't exist: " + fProblem);
- return false;
- }
- try (InputStream fileStream = new FileInputStream(fProblem);
- LineNumberReader problemReader = new LineNumberReader(
- new InputStreamReader(fileStream));) {
- Pattern csvSplitter = Pattern.compile(Pattern.quote(":"));
- Pattern commentSplitter = Pattern.compile(Pattern.quote("#"));
- String problemLine;
- String[] items;
- while ((problemLine = problemReader.readLine()) != null) {
- items = commentSplitter.split(problemLine);
- if (items.length == 0 || items[0].trim().isEmpty()){
- // comment or empty line
- continue;
- }
- items = csvSplitter.split(items[0].trim());
- if (items.length != 2) {
- System.out.println("Error: Invalid format in problem file, line number " + problemReader.getLineNumber() + ": "
- + problemLine);
- ok = false;
- continue;
- }
- long id = 0;
- try{
- id = Long.parseLong(items[1]);
- }
- catch(NumberFormatException exp){
- System.out.println("Error: Invalid number format in problem file, line number " + + problemReader.getLineNumber() + ": "
- + problemLine + exp);
- ok = false;
- }
- if ("way".equals(items[0]))
- problemWays.add(id);
- else if ("rel".equals(items[0]))
- problemRels.add(id);
- else {
- System.out.println("Error in problem file: Type not way or relation, line number " + + problemReader.getLineNumber() + ": "
- + problemLine);
- ok = false;
- }
- }
- } catch (IOException exp) {
- System.out.println("Error: Cannot read problem file " + fProblem +
- exp);
- return false;
- }
- return ok;
- }
-
- /**
- * Write a file that can be given to mkgmap that contains the correct arguments
- * for the split file pieces. You are encouraged to edit the file and so it
- * contains a template of all the arguments that you might want to use.
- * @param problemRelsThisPass
- * @param problemWaysThisPass
- */
- public void writeProblemList(File fileOutputDir, String fname) {
- try (PrintWriter w = new PrintWriter(new FileWriter(new File(fileOutputDir, fname)));) {
-
- w.println("#");
- w.println("# This file can be given to splitter using the --problem-file option");
- w.println("#");
- w.println("# List of relations and ways that are known to cause problems");
- w.println("# in splitter or mkgmap");
- w.println("# Objects listed here are specially treated by splitter to assure");
- w.println("# that complete data is written to all related tiles");
- w.println("# Format:");
- w.println("# way:<id>");
- w.println("# rel:<id>");
- w.println("# ways");
- for (long id: calculatedProblemWays){
- w.println("way: " + id + " #");
- }
- w.println("# rels");
- for (long id: calculatedProblemRels){
- w.println("rel: " + id + " #");
- }
-
- w.println();
- } catch (IOException e) {
- System.err.println("Warning: Could not write problem-list file " + fname + ", processing continues");
- }
- }
-
- /**
- * Calculate writers for elements which cross areas.
- * @param dataStorer stores data that is needed in different passes of the program.
- * @param osmFileHandler used to access OSM input files
- */
- public void calcMultiTileElements(DataStorer dataStorer, OSMFileHandler osmFileHandler) {
- // merge the calculated problem ids and the user given problem ids
- problemWays.addAll(calculatedProblemWays);
- problemRels.addAll(calculatedProblemRels);
- calculatedProblemRels.clear();
- calculatedProblemWays.clear();
-
- if (problemWays.isEmpty() && problemRels.isEmpty())
- return;
-
- // calculate which ways and relations are written to multiple areas.
- MultiTileProcessor multiProcessor = new MultiTileProcessor(dataStorer, problemWays, problemRels);
- // multiTileProcessor stores the problem relations in its own structures return memory to GC
- problemRels.clear();
- problemWays.clear();
- problemRels.trim();
- problemWays.trim();
-
- boolean done = false;
- long startThisPhase = System.currentTimeMillis();
- int prevPhase = -1;
- while(!done){
- int phase = multiProcessor.getPhase();
- if (prevPhase != phase){
- startThisPhase = System.currentTimeMillis();
- System.out.println("-----------------------------------");
- System.out.println("Executing multi-tile analyses phase " + phase);
- }
- done = osmFileHandler.process(multiProcessor);
- prevPhase = phase;
- if (done || (phase != multiProcessor.getPhase())){
- System.out.println("Multi-tile analyses phase " + phase + " took " + (System.currentTimeMillis() - startThisPhase) + " ms");
- }
- }
-
- System.out.println("-----------------------------------");
- }
+ private final LongArrayList problemWays = new LongArrayList();
+ private final LongArrayList problemRels = new LongArrayList();
+ private final TreeSet<Long> calculatedProblemWays = new TreeSet<>();
+ private final TreeSet<Long> calculatedProblemRels = new TreeSet<>();
+
+
+ /**
+ * Calculate lists of ways and relations that appear in multiple areas for a
+ * given list of areas.
+ * @param osmFileHandler
+ * @param realAreas list of areas, possibly overlapping if read from split-file
+ * @param overlapAmount
+ * @param mainOptions main options
+ * @return
+ */
+ public DataStorer calcProblemLists(OSMFileHandler osmFileHandler, List<Area> realAreas, int overlapAmount, SplitterParams mainOptions) {
+ long startProblemListGenerator = System.currentTimeMillis();
+ ArrayList<Area> distinctAreas = getNonOverlappingAreas(realAreas);
+ if (distinctAreas.size() > realAreas.size()) {
+ System.err.println("Waring: The areas given in --split-file are overlapping.");
+ Set<Integer> overlappingTiles = new TreeSet<>();
+ for (int i = 0; i < realAreas.size(); i++) {
+ Area a1 = realAreas.get(i);
+ for (int j = i + 1; j < realAreas.size(); j++) {
+ Area a2 = realAreas.get(j);
+ if (a1.intersects(a2)) {
+ overlappingTiles.add(a1.getMapId());
+ overlappingTiles.add(a2.getMapId());
+ }
+ }
+ }
+ if (!overlappingTiles.isEmpty()) {
+ System.out.println("Overlaping tiles: " + overlappingTiles.toString());
+ }
+ }
+ System.out.println("Generating problem list for " + distinctAreas.size() + " distinct areas");
+ List<Area> workAreas = addPseudoAreas(distinctAreas);
+
+ int numPasses = (int) Math.ceil((double) workAreas.size() / mainOptions.getMaxAreas());
+ int areasPerPass = (int) Math.ceil((double) workAreas.size() / numPasses);
+ if (numPasses > 1) {
+ System.out.println("Processing " + distinctAreas.size() + " areas in " + numPasses + " passes, "
+ + areasPerPass + " areas at a time");
+ } else {
+ System.out.println("Processing " + distinctAreas.size() + " areas in a single pass");
+ }
+
+ ArrayList<Area> allAreas = new ArrayList<>();
+
+ System.out.println("Pseudo areas:");
+ for (int j = 0; j < workAreas.size(); j++) {
+ Area area = workAreas.get(j);
+ allAreas.add(area);
+ if (area.isPseudoArea())
+ System.out.println("Pseudo area " + area.getMapId() + " covers " + area);
+ }
+
+ DataStorer distinctDataStorer = new DataStorer(workAreas, overlapAmount);
+ System.out.println("Starting problem-list-generator pass(es)");
+
+ for (int pass = 0; pass < numPasses; pass++) {
+ System.out.println("-----------------------------------");
+ System.out.println("Starting problem-list-generator pass " + (pass + 1) + " of " + numPasses);
+ long startThisPass = System.currentTimeMillis();
+ int areaOffset = pass * areasPerPass;
+ int numAreasThisPass = Math.min(areasPerPass, workAreas.size() - pass * areasPerPass);
+ ProblemListProcessor processor = new ProblemListProcessor(distinctDataStorer, areaOffset, numAreasThisPass,
+ mainOptions);
+
+ boolean done = false;
+ while (!done) {
+ done = osmFileHandler.execute(processor);
+
+ calculatedProblemWays.addAll(processor.getProblemWays());
+ calculatedProblemRels.addAll(processor.getProblemRels());
+ }
+ System.out.println("Problem-list-generator pass " + (pass + 1) + " took "
+ + (System.currentTimeMillis() - startThisPass) + " ms");
+ }
+ System.out.println("Problem-list-generator pass(es) took "
+ + (System.currentTimeMillis() - startProblemListGenerator) + " ms");
+ DataStorer dataStorer = new DataStorer(realAreas, overlapAmount);
+ dataStorer.translateDistinctToRealAreas(distinctDataStorer);
+ return dataStorer;
+ }
+
+ /** Read user defined problematic relations and ways */
+ public boolean readProblemIds(String problemFileName) {
+ File fProblem = new File(problemFileName);
+ boolean ok = true;
+
+ if (!fProblem.exists()) {
+ System.out.println("Error: problem file doesn't exist: " + fProblem);
+ return false;
+ }
+ try (InputStream fileStream = new FileInputStream(fProblem);
+ LineNumberReader problemReader = new LineNumberReader(new InputStreamReader(fileStream));) {
+ Pattern csvSplitter = Pattern.compile(Pattern.quote(":"));
+ Pattern commentSplitter = Pattern.compile(Pattern.quote("#"));
+ String problemLine;
+ String[] items;
+ while ((problemLine = problemReader.readLine()) != null) {
+ items = commentSplitter.split(problemLine);
+ if (items.length == 0 || items[0].trim().isEmpty()) {
+ // comment or empty line
+ continue;
+ }
+ items = csvSplitter.split(items[0].trim());
+ if (items.length != 2) {
+ System.out.println("Error: Invalid format in problem file, line number "
+ + problemReader.getLineNumber() + ": " + problemLine);
+ ok = false;
+ continue;
+ }
+ long id = 0;
+ try {
+ id = Long.parseLong(items[1]);
+ } catch (NumberFormatException exp) {
+ System.out.println("Error: Invalid number format in problem file, line number "
+ + +problemReader.getLineNumber() + ": " + problemLine + exp);
+ ok = false;
+ }
+ if ("way".equals(items[0]))
+ problemWays.add(id);
+ else if ("rel".equals(items[0]))
+ problemRels.add(id);
+ else {
+ System.out.println("Error in problem file: Type not way or relation, line number "
+ + +problemReader.getLineNumber() + ": " + problemLine);
+ ok = false;
+ }
+ }
+ } catch (IOException exp) {
+ System.out.println("Error: Cannot read problem file " + fProblem + exp);
+ return false;
+ }
+ return ok;
+ }
+
+ /**
+ * Write a file that can be given to mkgmap that contains the correct
+ * arguments for the split file pieces. You are encouraged to edit the file
+ * and so it contains a template of all the arguments that you might want to
+ * use.
+ *
+ * @param problemRelsThisPass
+ * @param problemWaysThisPass
+ */
+ public void writeProblemList(File fileOutputDir, String fname) {
+ try (PrintWriter w = new PrintWriter(new FileWriter(new File(fileOutputDir, fname)));) {
+
+ w.println("#");
+ w.println("# This file can be given to splitter using the --problem-file option");
+ w.println("#");
+ w.println("# List of relations and ways that are known to cause problems");
+ w.println("# in splitter or mkgmap");
+ w.println("# Objects listed here are specially treated by splitter to assure");
+ w.println("# that complete data is written to all related tiles");
+ w.println("# Format:");
+ w.println("# way:<id>");
+ w.println("# rel:<id>");
+ w.println("# ways");
+ for (long id : calculatedProblemWays) {
+ w.println("way: " + id + " #");
+ }
+ w.println("# rels");
+ for (long id : calculatedProblemRels) {
+ w.println("rel: " + id + " #");
+ }
+
+ w.println();
+ } catch (IOException e) {
+ System.err.println("Warning: Could not write problem-list file " + fname + ", processing continues");
+ }
+ }
+
+ /**
+ * Calculate writers for elements which cross areas.
+ *
+ * @param dataStorer
+ * stores data that is needed in different passes of the program.
+ * @param osmFileHandler
+ * used to access OSM input files
+ */
+ public void calcMultiTileElements(DataStorer dataStorer, OSMFileHandler osmFileHandler) {
+ // merge the calculated problem ids and the user given problem ids
+ problemWays.addAll(calculatedProblemWays);
+ problemRels.addAll(calculatedProblemRels);
+ calculatedProblemRels.clear();
+ calculatedProblemWays.clear();
+
+ if (problemWays.isEmpty() && problemRels.isEmpty())
+ return;
+
+ // calculate which ways and relations are written to multiple areas.
+ MultiTileProcessor multiProcessor = new MultiTileProcessor(dataStorer, problemWays, problemRels);
+ // multiTileProcessor stores the problem relations in its own structures
+ // return memory to GC
+ problemRels.clear();
+ problemWays.clear();
+ problemRels.trim();
+ problemWays.trim();
+
+ boolean done = false;
+ long startThisPhase = System.currentTimeMillis();
+ int prevPhase = -1;
+ while (!done) {
+ int phase = multiProcessor.getPhase();
+ if (prevPhase != phase) {
+ startThisPhase = System.currentTimeMillis();
+ System.out.println("-----------------------------------");
+ System.out.println("Executing multi-tile analyses phase " + phase);
+ }
+ done = osmFileHandler.execute(multiProcessor);
+
+ prevPhase = phase;
+ if (done || (phase != multiProcessor.getPhase())) {
+ System.out.println("Multi-tile analyses phase " + phase + " took "
+ + (System.currentTimeMillis() - startThisPhase) + " ms");
+ }
+ }
+
+ System.out.println("-----------------------------------");
+ }
+
+ /**
+ * Make sure that our areas cover the planet. This is done by adding
+ * pseudo-areas if needed.
+ *
+ * @param realAreas
+ * list of areas (read from split-file or calculated)
+ * @return new list of areas containing the real areas and additional areas
+ */
+ public static List<Area> addPseudoAreas(List<Area> realAreas) {
+ ArrayList<Area> areas = new ArrayList<>(realAreas);
+ Rectangle planetBounds = new Rectangle(Utils.toMapUnit(-180.0), Utils.toMapUnit(-90.0),
+ 2 * Utils.toMapUnit(180.0), 2 * Utils.toMapUnit(90.0));
+
+ while (!checkIfCovered(planetBounds, areas)) {
+ boolean changed = addPseudoArea(areas);
+
+ if (!changed) {
+ throw new SplitFailedException("Failed to fill planet with pseudo-areas");
+ }
+ }
+ return areas;
+ }
+
+ /**
+ * Work around for possible rounding errors in area.subtract processing
+ *
+ * @param area
+ * an area that is considered to be empty or a rectangle
+ * @return
+ */
+ private static java.awt.geom.Area simplifyArea(java.awt.geom.Area area) {
+ if (area.isEmpty() || area.isRectangular())
+ return area;
+ // area.isRectugular() may returns false although the shape is a
+ // perfect rectangle :-( If we subtract the area from its bounding
+ // box we get better results.
+ java.awt.geom.Area bbox = new java.awt.geom.Area(area.getBounds2D());
+ bbox.subtract(area);
+ if (bbox.isEmpty()) // bbox equals area: is a rectangle
+ return new java.awt.geom.Area(area.getBounds2D());
+ return area;
+ }
+
+ private static boolean checkIfCovered(Rectangle bounds, ArrayList<Area> areas) {
+ java.awt.geom.Area bbox = new java.awt.geom.Area(bounds);
+ long sumTiles = 0;
+
+ for (Area area : areas) {
+ sumTiles += (long) area.getHeight() * (long) area.getWidth();
+ bbox.subtract(area.getJavaArea());
+ }
+ long areaBox = (long) bounds.height * (long) bounds.width;
+
+ if (sumTiles != areaBox)
+ return false;
+
+ return bbox.isEmpty();
+ }
+
+ /**
+ * Create a list of areas that do not overlap. If areas in the original list
+ * are overlapping, they can be replaced by up to 5 disjoint areas. This is
+ * done if parameter makeDisjoint is true
+ *
+ * @param realAreas
+ * the list of areas
+ * @return the new list
+ */
+ public static ArrayList<Area> getNonOverlappingAreas(final List<Area> realAreas) {
+ java.awt.geom.Area covered = new java.awt.geom.Area();
+ ArrayList<Area> splitList = new ArrayList<>();
+ int artificialId = -99999999;
+ boolean foundOverlap = false;
+ for (Area area1 : realAreas) {
+ Rectangle r1 = area1.getRect();
+ if (covered.intersects(r1) == false) {
+ splitList.add(area1);
+ } else {
+ if (foundOverlap == false) {
+ foundOverlap = true;
+ System.out.println("Removing overlaps from tiles...");
+ }
+ // String msg = "splitting " + area1.getMapId() + " " + (i+1) +
+ // "/" + realAreas.size() + " overlapping ";
+ // find intersecting areas in the already covered part
+ ArrayList<Area> splitAreas = new ArrayList<>();
+
+ for (int j = 0; j < splitList.size(); j++) {
+ Area area2 = splitList.get(j);
+ if (area2 == null)
+ continue;
+ Rectangle r2 = area2.getRect();
+ if (r1.intersects(r2)) {
+ java.awt.geom.Area overlap = new java.awt.geom.Area(area1.getRect());
+ overlap.intersect(area2.getJavaArea());
+ Rectangle ro = overlap.getBounds();
+ if (ro.height == 0 || ro.width == 0)
+ continue;
+ // msg += area2.getMapId() + " ";
+ Area aNew = new Area(ro.y, ro.x, (int) ro.getMaxY(), (int) ro.getMaxX());
+ aNew.setMapId(artificialId++);
+ aNew.setName("" + area1.getMapId());
+ aNew.setJoinable(false);
+ covered.subtract(area2.getJavaArea());
+ covered.add(overlap);
+ splitList.set(j, aNew);
+
+ java.awt.geom.Area coveredByPair = new java.awt.geom.Area(r1);
+ coveredByPair.add(new java.awt.geom.Area(r2));
+
+ java.awt.geom.Area originalPair = new java.awt.geom.Area(coveredByPair);
+
+ int minX = coveredByPair.getBounds().x;
+ int minY = coveredByPair.getBounds().y;
+ int maxX = (int) coveredByPair.getBounds().getMaxX();
+ int maxY = (int) coveredByPair.getBounds().getMaxY();
+ coveredByPair.subtract(overlap);
+ if (coveredByPair.isEmpty())
+ continue; // two equal areas a
+
+ coveredByPair.subtract(covered);
+ java.awt.geom.Area testSplit = new java.awt.geom.Area(overlap);
+
+ Rectangle[] rectPair = { r1, r2 };
+ Area[] areaPair = { area1, area2 };
+ int lx = minX;
+ int lw = ro.x - minX;
+ int rx = (int) ro.getMaxX();
+ int rw = maxX - rx;
+ int uy = (int) ro.getMaxY();
+ int uh = maxY - uy;
+ int by = minY;
+ int bh = ro.y - by;
+ Rectangle[] clippers = { new Rectangle(lx, minY, lw, bh), // lower
+ // left
+ new Rectangle(ro.x, minY, ro.width, bh), // lower
+ // middle
+ new Rectangle(rx, minY, rw, bh), // lower right
+ new Rectangle(lx, ro.y, lw, ro.height), // left
+ new Rectangle(rx, ro.y, rw, ro.height), // right
+ new Rectangle(lx, uy, lw, uh), // upper left
+ new Rectangle(ro.x, uy, ro.width, uh), // upper
+ // middle
+ new Rectangle(rx, uy, rw, uh) // upper right
+ };
+
+ for (Rectangle clipper : clippers) {
+ for (int k = 0; k <= 1; k++) {
+ Rectangle test = clipper.intersection(rectPair[k]);
+ if (!test.isEmpty()) {
+ testSplit.add(new java.awt.geom.Area(test));
+ if (k == 1 || covered.intersects(test) == false) {
+ aNew = new Area(test.y, test.x, (int) test.getMaxY(), (int) test.getMaxX());
+ aNew.setMapId(areaPair[k].getMapId());
+ splitAreas.add(aNew);
+ covered.add(aNew.getJavaArea());
+ }
+ }
+ }
+ }
+ assert testSplit.equals(originalPair);
+ }
+ }
+
+ // recombine parts that form a rectangle
+ for (Area splitArea : splitAreas) {
+ if (splitArea.isJoinable()) {
+ for (int j = 0; j < splitList.size(); j++) {
+ Area area = splitList.get(j);
+ if (area == null || area.isJoinable() == false || area.getMapId() != splitArea.getMapId())
+ continue;
+ boolean doJoin = false;
+ if (splitArea.getMaxLat() == area.getMaxLat() && splitArea.getMinLat() == area.getMinLat()
+ && (splitArea.getMinLong() == area.getMaxLong()
+ || splitArea.getMaxLong() == area.getMinLong()))
+ doJoin = true;
+ else if (splitArea.getMinLong() == area.getMinLong()
+ && splitArea.getMaxLong() == area.getMaxLong()
+ && (splitArea.getMinLat() == area.getMaxLat()
+ || splitArea.getMaxLat() == area.getMinLat()))
+ doJoin = true;
+ if (doJoin) {
+ splitArea = area.add(splitArea);
+ splitArea.setMapId(area.getMapId());
+ splitList.set(j, splitArea);
+ splitArea = null; // don't add later
+ break;
+ }
+ }
+ }
+ if (splitArea != null) {
+ splitList.add(splitArea);
+ }
+ }
+ /*
+ * if (msg.isEmpty() == false) System.out.println(msg);
+ */
+ }
+ covered.add(new java.awt.geom.Area(r1));
+ }
+ covered.reset();
+ Iterator<Area> iter = splitList.iterator();
+ while (iter.hasNext()) {
+ Area a = iter.next();
+ if (a == null)
+ iter.remove();
+ else {
+ Rectangle r1 = a.getRect();
+ if (covered.intersects(r1) == true) {
+ throw new SplitFailedException("Failed to create list of distinct areas");
+ }
+ covered.add(a.getJavaArea());
+ }
+ }
+ return splitList;
+ }
+
+ /**
+ * Fill uncovered parts of the planet with pseudo-areas. TODO: check if
+ * better algorithm reduces run time in ProblemListProcessor We want a small
+ * number of pseudo areas because many of them will require more memory or
+ * more passes, esp. when processing whole planet. Also, the total length of
+ * all edges should be small.
+ *
+ * @param areas
+ * list of areas (either real or pseudo)
+ * @return true if pseudo-areas were added
+ */
+ private static boolean addPseudoArea(ArrayList<Area> areas) {
+ int oldSize = areas.size();
+ Rectangle planetBounds = new Rectangle(Utils.toMapUnit(-180.0), Utils.toMapUnit(-90.0),
+ 2 * Utils.toMapUnit(180.0), 2 * Utils.toMapUnit(90.0));
+ java.awt.geom.Area uncovered = new java.awt.geom.Area(planetBounds);
+ java.awt.geom.Area covered = new java.awt.geom.Area();
+ for (Area area : areas) {
+ uncovered.subtract(area.getJavaArea());
+ covered.add(area.getJavaArea());
+}
+ Rectangle rCov = covered.getBounds();
+ Rectangle[] topAndBottom = {
+ new Rectangle(planetBounds.x, (int) rCov.getMaxY(), planetBounds.width,
+ (int) (planetBounds.getMaxY() - rCov.getMaxY())), // top
+ new Rectangle(planetBounds.x, planetBounds.y, planetBounds.width, rCov.y - planetBounds.y) }; // bottom
+ for (Rectangle border : topAndBottom) {
+ if (!border.isEmpty()) {
+ uncovered.subtract(new java.awt.geom.Area(border));
+ covered.add(new java.awt.geom.Area(border));
+ Area pseudo = new Area(border.y, border.x, (int) border.getMaxY(), (int) border.getMaxX());
+ pseudo.setMapId(-1 * (areas.size() + 1));
+ pseudo.setPseudoArea(true);
+ areas.add(pseudo);
+ }
+ }
+ while (uncovered.isEmpty() == false) {
+ boolean changed = false;
+ List<List<Point>> shapes = Utils.areaToShapes(uncovered);
+ // we divide planet into stripes for all vertices of the uncovered
+ // area
+ int minX = uncovered.getBounds().x;
+ int nextX = Integer.MAX_VALUE;
+ for (int i = 0; i < shapes.size(); i++) {
+ List<Point> shape = shapes.get(i);
+ for (Point point : shape) {
+ int lon = point.x;
+ if (lon < nextX && lon > minX)
+ nextX = lon;
+ }
+ }
+ java.awt.geom.Area stripeLon = new java.awt.geom.Area(
+ new Rectangle(minX, planetBounds.y, nextX - minX, planetBounds.height));
+ // cut out already covered area
+ stripeLon.subtract(covered);
+ assert stripeLon.isEmpty() == false;
+ // the remaining area must be a set of zero or more disjoint
+ // rectangles
+ List<List<Point>> stripeShapes = Utils.areaToShapes(stripeLon);
+ for (int j = 0; j < stripeShapes.size(); j++) {
+ List<Point> rectShape = stripeShapes.get(j);
+ java.awt.geom.Area test = Utils.shapeToArea(rectShape);
+ test = simplifyArea(test);
+ assert test.isRectangular();
+ Rectangle pseudoRect = test.getBounds();
+ if (uncovered.contains(pseudoRect)) {
+ assert test.getBounds().width == stripeLon.getBounds().width;
+ boolean wasMerged = false;
+ // check if new area can be merged with last rectangles
+ for (int k = areas.size() - 1; k >= oldSize; k--) {
+ Area prev = areas.get(k);
+ if (prev.getMaxLong() < pseudoRect.x || prev.isPseudoArea() == false)
+ continue;
+ if (prev.getHeight() == pseudoRect.height && prev.getMaxLong() == pseudoRect.x
+ && prev.getMinLat() == pseudoRect.y) {
+ // merge
+ Area pseudo = prev.add(new Area(pseudoRect.y, pseudoRect.x, (int) pseudoRect.getMaxY(),
+ (int) pseudoRect.getMaxX()));
+ pseudo.setMapId(prev.getMapId());
+ pseudo.setPseudoArea(true);
+ areas.set(k, pseudo);
+ // System.out.println("Enlarged pseudo area " +
+ // pseudo.getMapId() + " " + pseudo);
+ wasMerged = true;
+ break;
+ }
+ }
+
+ if (!wasMerged) {
+ Area pseudo = new Area(pseudoRect.y, pseudoRect.x, (int) pseudoRect.getMaxY(),
+ (int) pseudoRect.getMaxX());
+ pseudo.setMapId(-1 * (areas.size() + 1));
+ pseudo.setPseudoArea(true);
+ // System.out.println("Adding pseudo area " +
+ // pseudo.getMapId() + " " + pseudo);
+ areas.add(pseudo);
+ }
+ uncovered.subtract(test);
+ covered.add(test);
+ changed = true;
+ }
+ }
+ if (!changed)
+ break;
+ }
+ return oldSize != areas.size();
+ }
}
diff --git a/src/uk/me/parabola/splitter/QueueProcessor.java b/src/uk/me/parabola/splitter/QueueProcessor.java
new file mode 100644
index 0000000..d630a7f
--- /dev/null
+++ b/src/uk/me/parabola/splitter/QueueProcessor.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2016, Gerd Petermann
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+import uk.me.parabola.splitter.OSMMessage.Type;
+
+/**
+ * Simple helper to allow all existing processors to use the producer/consumer
+ * pattern. For each call of a supplier (one of the OSM parsers) it either
+ * passes the call to the original processor or adds messages to the queue..
+ *
+ * @author Gerd Petermann
+ *
+ */
+public class QueueProcessor extends AbstractMapProcessor {
+ private final BlockingQueue<OSMMessage> queue;
+ private final MapProcessor realProcessor;
+
+ public QueueProcessor(BlockingQueue<OSMMessage> queue, MapProcessor realProcessor) {
+ this.queue = queue;
+ this.realProcessor = realProcessor;
+ }
+
+ @Override
+ public boolean skipTags() {
+ return realProcessor.skipTags();
+ }
+
+ @Override
+ public boolean skipNodes() {
+ return realProcessor.skipNodes();
+ }
+
+ @Override
+ public boolean skipWays() {
+ return realProcessor.skipWays();
+ }
+
+ @Override
+ public boolean skipRels() {
+ return realProcessor.skipRels();
+ }
+
+ @Override
+ public void boundTag(Area bounds) {
+ addToQueue(bounds);
+ }
+
+ @Override
+ public void processNode(Node n) {
+ addToQueue(n);
+ }
+
+ @Override
+ public void processWay(Way w) {
+ addToQueue(w);
+ }
+
+ @Override
+ public void processRelation(Relation r) {
+ addToQueue(r);
+ }
+
+ @Override
+ public void startFile() {
+ try {
+ flush();
+ queue.put(new OSMMessage(Type.START_FILE));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean endMap() {
+ try {
+ flush();
+ queue.put(new OSMMessage(Type.END_MAP));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ return true;
+ }
+
+ public int getPhase() {
+ throw new UnsupportedOperationException("call getPhase() of real processor");
+ }
+
+ /** number of OSM elements to collect before adding them to the queue */
+ private static final int NUM_STAGING = 1000;
+ private List<Element> staging = new ArrayList<>(NUM_STAGING);
+
+ private void addToQueue(Element el) {
+ try {
+ staging.add(el);
+ if (staging.size() >= NUM_STAGING)
+ flush();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void addToQueue(Area bounds) {
+ try {
+ flush();
+ queue.put(new OSMMessage(bounds));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void flush() throws InterruptedException {
+ if (staging == null || staging.isEmpty())
+ return;
+ queue.put(new OSMMessage(staging));
+ staging = new ArrayList<>(NUM_STAGING);
+ }
+}
diff --git a/src/uk/me/parabola/splitter/Relation.java b/src/uk/me/parabola/splitter/Relation.java
index 071b47c..c937381 100644
--- a/src/uk/me/parabola/splitter/Relation.java
+++ b/src/uk/me/parabola/splitter/Relation.java
@@ -19,7 +19,7 @@ import java.util.List;
* @author Steve Ratcliffe
*/
public class Relation extends Element {
- private final List<Member> members = new ArrayList<Member>();
+ private final List<Member> members = new ArrayList<>();
public void addMember(String type, long ref, String role) {
Member mem = new Member(type, ref, role);
@@ -30,7 +30,7 @@ public class Relation extends Element {
return members;
}
- static class Member {
+ public static class Member {
private String type;
private long ref;
private String role;
diff --git a/src/uk/me/parabola/splitter/RoundingUtils.java b/src/uk/me/parabola/splitter/RoundingUtils.java
index bb2bf08..26e70b8 100644
--- a/src/uk/me/parabola/splitter/RoundingUtils.java
+++ b/src/uk/me/parabola/splitter/RoundingUtils.java
@@ -60,7 +60,7 @@ public class RoundingUtils {
* @param resolution the map resolution to align the borders at
* @return the rounded area
*/
- static Area round(Area b, int resolution) {
+ public static Area round(Area b, int resolution) {
int shift = 24 - resolution;
int alignment = 1 << shift;
diff --git a/src/uk/me/parabola/splitter/SparseBitSet.java b/src/uk/me/parabola/splitter/SparseBitSet.java
deleted file mode 100644
index 55b3505..0000000
--- a/src/uk/me/parabola/splitter/SparseBitSet.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (C) 2012, Gerd Petermann
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 or
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-package uk.me.parabola.splitter;
-
-import it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap;
-import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
-
-/** A partly BitSet implementation optimized for memory
- * when used to store very large values with a high likelihood
- * that the stored values build groups like e.g. the OSM node IDs.
- * The keys are divided into 3 parts.
- * The 1st part is stored in a small hash map.
- * The 2nd part is stored in larger hash maps addressing long values.
- * The 3rd part (6 bits) is stored in the long value addressed by the upper maps.
- * author GerdP */
-public class SparseBitSet{
- private static final long MID_ID_MASK = 0x7ffffff;
- private static final long TOP_ID_MASK = ~MID_ID_MASK;
- private static final int LOW_MASK = 63;
- private static final int TOP_ID_SHIFT = Long.numberOfTrailingZeros(TOP_ID_MASK);
- private static final int MID_ID_SHIFT = Integer.numberOfTrailingZeros(~LOW_MASK);
-
- private Long2ObjectOpenHashMap<Int2LongOpenHashMap> topMap = new Long2ObjectOpenHashMap<>();
- private long bitCount;
-
- public void set(long key){
- long topId = key >> TOP_ID_SHIFT;
- Int2LongOpenHashMap midMap = topMap.get(topId);
- if (midMap == null){
- midMap = new Int2LongOpenHashMap();
- topMap.put(topId, midMap);
- }
- int midId = (int)((key & MID_ID_MASK) >> MID_ID_SHIFT);
- long chunk = midMap.get(midId);
- int bitPos =(int)(key & LOW_MASK);
- long val = 1L << (bitPos-1);
- if (chunk != 0){
- if ((chunk & val) != 0)
- return;
- val |= chunk;
- }
- midMap.put(midId, val);
- ++bitCount;
- }
-
- public void clear(long key){
- long topId = key >> TOP_ID_SHIFT;
- Int2LongOpenHashMap midMap = topMap.get(topId);
- if (midMap == null)
- return;
- int midId = (int)((key & MID_ID_MASK) >> MID_ID_SHIFT);
- long chunk = midMap.get(midId);
- if (chunk == 0)
- return;
- int bitPos =(int)(key & LOW_MASK);
- long val = 1L << (bitPos-1);
- if ((chunk & val) == 0)
- return;
- chunk &= ~val;
- if (chunk == 0){
- midMap.remove(midId);
- if (midMap.isEmpty()){
- topMap.remove(topId);
- }
- }
- else
- midMap.put(midId, chunk);
- --bitCount;
- }
-
- public boolean get(long key){
- long topId = key >> TOP_ID_SHIFT;
- Int2LongOpenHashMap midMap = topMap.get(topId);
- if (midMap == null)
- return false;
- int midId = (int)((key & MID_ID_MASK) >> MID_ID_SHIFT);
- long chunk = midMap.get(midId);
- if (chunk == 0)
- return false;
- int bitPos =(int)(key & LOW_MASK);
- long val = 1L << (bitPos-1);
- return ((chunk & val) != 0);
- }
-
- public void clear(){
- topMap.clear();
- bitCount = 0;
- }
-
- public int cardinality(){
- if (bitCount > Integer.MAX_VALUE)
- throw new SplitFailedException("cardinality too high for int " + bitCount);
- return (int) bitCount;
- }
-}
-
-
diff --git a/src/uk/me/parabola/splitter/SparseLong2ShortMap.java b/src/uk/me/parabola/splitter/SparseLong2ShortMap.java
deleted file mode 100644
index 66019c4..0000000
--- a/src/uk/me/parabola/splitter/SparseLong2ShortMap.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (C) 2013, Gerd Petermann
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 or
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-package uk.me.parabola.splitter;
-
-/**
- * Helper class to create appropriate instance of a Long/Short map.
- * @author Gerd
- *
- */
-public class SparseLong2ShortMap {
- public static SparseLong2ShortMapFunction createMap(String dataDesc){
- long maxMem = Runtime.getRuntime().maxMemory() / 1024 / 1024;
- // prefer implementation with lower memory footprint when free heap is less than 2 GB
- if (maxMem < 2048)
- return new SparseLong2ShortMapInline(dataDesc);
- return new SparseLong2ShortMapHuge(dataDesc);
- }
-}
diff --git a/src/uk/me/parabola/splitter/SparseLong2ShortMapFunction.java b/src/uk/me/parabola/splitter/SparseLong2ShortMapFunction.java
deleted file mode 100644
index 87523be..0000000
--- a/src/uk/me/parabola/splitter/SparseLong2ShortMapFunction.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright (C) 2012, Gerd Petermann
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 or
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
- package uk.me.parabola.splitter;
-
-//import it.unimi.dsi.fastutil.longs.Long2ShortFunction;
-
-/**
- * Stores long/short pairs.
- *
- */
-interface SparseLong2ShortMapFunction {
- final short UNASSIGNED = Short.MIN_VALUE;
- public short put(long key, short val);
- public void clear();
- public boolean containsKey(long key);
- public short get(long key);
- public void stats(int msgLevel);
- public long size();
- public short defaultReturnValue();
- public void defaultReturnValue(short arg0);
-}
diff --git a/src/uk/me/parabola/splitter/SparseLong2ShortMapHuge.java b/src/uk/me/parabola/splitter/SparseLong2ShortMapHuge.java
deleted file mode 100644
index 84578c0..0000000
--- a/src/uk/me/parabola/splitter/SparseLong2ShortMapHuge.java
+++ /dev/null
@@ -1,560 +0,0 @@
-/*
- * Copyright (c) 2013, Gerd Petermann
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-
-package uk.me.parabola.splitter;
-
-import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
-import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
-import it.unimi.dsi.fastutil.longs.LongArrayList;
-
-import java.util.Arrays;
-
-
-
-/**
- * {@link SparseLong2ShortMapHuge} implements {@link SparseLong2ShortMapFunction}
- * optimized for low memory requirements and inserts in sequential order.
- * Don't use this for a rather small number of pairs.
- *
- * Inspired by SparseInt2ShortMapInline.
- *
- * A HashMap is used to address large vectors which address chunks. The HashMap
- * is the only part that stores long values, and it will be very small as long
- * as long as input is normal OSM data and not something with random numbers.
- * A chunk stores up to CHUNK_SIZE values. A separately stored bit-mask is used
- * to separate used and unused entries in the chunk. Thus, the chunk length
- * depends on the number of used entries, not on the highest used entry.
- * A typical (uncompressed) chunk looks like this:
- * v1,v1,v1,v1,v1,v1,v2,v2,v2,v2,v1,v1,v1,v1,v1,u,?,?,...}
- * v1,v2: values stored in the chunk
- * u: "unassigned" value
- * ?: anything
- *
- * After applying Run Length Encryption on this the chunk looks like this:
- * {u,6,v1,4,v2,5,v1,?,?,?}
- * The unassigned value on index 0 signals a compressed chunk.
- *
- * An (uncompressed) ONE_VALUE_CHUNK may look like this:
- * {v1,v1,v1,v1,v1,v1,v1,v1,v1,v1,v1,u,?,?,...}
- * This is stored without run length info in the shortest possible trunk:
- * {v1}
- *
- * Fortunately, OSM data is distributed in a way that most(!) chunks contain
- * just one distinct value, so most chunks can be stored in 24 or 32 bytes
- * instead of 152 bytes for the worst case (counting also the padding bytes).
-
- * Since we have keys with 64 bits, we have to divide the key into 3 parts:
- * 37 bits for the value that is stored in the HashMap.
- * 21 bits for the chunkId (this gives the required length of a large vector)
- * 6 bits for the position in the chunk
- *
- * The chunkId identifies the position of a 64-bit value (stored in the large vector).
- * A chunk is stored in a chunkStore which is a 3-dimensional array.
- * We group chunks of equally length together in stores of 64 entries.
- * To find the right position of a new chunk, we need three values: x,y, and z.
- * x is the length of the chunk (the number of required shorts) (1-64, we store the value decremented by 1 to have 0-63)
- * y is the position of the store (0-1073741823)
- * z is the position of the chunk within the store. (0-63)
- */
-
-public class SparseLong2ShortMapHuge implements SparseLong2ShortMapFunction{
- private static final long CHUNK_ID_MASK = 0x7ffffffL; // the part of the key that is not saved in the top HashMap
- private static final long TOP_ID_MASK = ~CHUNK_ID_MASK; // the part of the key that is saved in the top HashMap
- private static final int TOP_ID_SHIFT = Long.numberOfTrailingZeros(TOP_ID_MASK);
-
-
- private static final int CHUNK_STORE_BITS_FOR_Z = 6; // may be increased
- private static final int CHUNK_STORE_BITS_FOR_Y = 30;
- private static final int CHUNK_STORE_BITS_FOR_X = 6;
- private static final int CHUNK_STORE_ELEMS = 1 << CHUNK_STORE_BITS_FOR_X;
- private static final long CHUNK_STORE_X_MASK = (1L << CHUNK_STORE_BITS_FOR_X) - 1;
- private static final long CHUNK_STORE_Y_MASK = (1L << CHUNK_STORE_BITS_FOR_Y) - 1;
- private static final long CHUNK_STORE_Z_MASK = (1L << CHUNK_STORE_BITS_FOR_Z) - 1;
- private static final long CHUNK_STORE_USED_FLAG_MASK = 1L<<63;
- private static final int CHUNK_STORE_Y_SHIFT = CHUNK_STORE_BITS_FOR_X;
- private static final int CHUNK_STORE_Z_SHIFT = CHUNK_STORE_BITS_FOR_X + CHUNK_STORE_BITS_FOR_Y;
-
- private static final int CHUNK_SIZE = 64; // 64 = 1<< 6 (last 6 bits of the key)
- private static final long CHUNK_OFFSET_MASK = CHUNK_SIZE-1; // the part of the key that contains the offset in the chunk
- private static final long OLD_CHUNK_ID_MASK = ~CHUNK_OFFSET_MASK; // first 58 bits of a long. If this part of the key changes, a different chunk is needed
-
- private static final long INVALID_CHUNK_ID = 1L; // must NOT be divisible by CHUNK_SIZE
- private static final int LARGE_VECTOR_SIZE = (int)(CHUNK_ID_MASK/ CHUNK_SIZE + 1); // number of entries addressed by one topMap entry
-
- private static final int ONE_VALUE_CHUNK_SIZE = 1;
-
- /** What to return on unassigned indices */
- private short unassigned = UNASSIGNED;
- private long size;
-
-
- private long currentChunkId = INVALID_CHUNK_ID;
- private short [] currentChunk = new short[CHUNK_SIZE]; // stores the values in the real position
- private short [] tmpWork = new short[CHUNK_SIZE]; // a chunk after applying the "mask encoding"
- private short [] RLEWork = new short[CHUNK_SIZE]; // for the RLE-compressed chunk
-
-
- // for statistics
- private final String dataDesc;
- private long [] countChunkLen;
- private long expanded = 0;
- private long uncompressedLen = 0;
- private long compressedLen = 0;
- private int storedLengthOfCurrentChunk = 0;
- private long currentChunkIdInStore = 0;
-
- private Long2ObjectOpenHashMap<long[]> topMap;
- private short[][][] chunkStore;
- private long[][][] maskStore;
- private int[] freePosInSore;
- // maps chunks that can be reused
- private Int2ObjectOpenHashMap<LongArrayList> reusableChunks;
-
- /**
- * A map that stores pairs of (OSM) IDs and short values identifying the
- * areas in which the object (node,way) with the ID occurs.
- * @param dataDesc
- */
- SparseLong2ShortMapHuge(String dataDesc) {
- this.dataDesc = dataDesc;
- clear();
- }
-
- /**
- * Count how many of the lowest X bits in mask are set
- *
- * @return
- */
- private static int countUnder(long mask, int lowest) {
- return Long.bitCount(mask & ((1L << lowest) - 1));
- }
-
- /**
- * Try to use Run Length Encoding to compress the chunk stored in tmpWork. In most
- * cases this works very well because chunks often have only one
- * or two distinct values.
- * @param maxlen: number of elements in the chunk.
- * @return -1 if compression doesn't save space, else the number of elements in the
- * compressed chunk stored in buffer RLEWork.
- */
- private int chunkCompressRLE (int maxlen){
- int opos = 1;
- for (int i = 0; i < maxlen; i++) {
- short runLength = 1;
- while (i+1 < maxlen && tmpWork[i] == tmpWork[i+1]) {
- runLength++;
- i++;
- }
- if (opos+2 >= tmpWork.length)
- return -1; // compressed record is not shorter
- RLEWork[opos++] = runLength;
- RLEWork[opos++] = tmpWork[i];
- }
- if (opos == 3){
- // special case: the chunk contains only one distinct value
- // we can store this in a length-1 chunk because we don't need
- // the length counter nor the compression flag
- RLEWork[0] = RLEWork[2];
- return ONE_VALUE_CHUNK_SIZE;
- }
-
- if (opos < maxlen){
- RLEWork[0] = unassigned; // signal a normal compressed record
- return opos;
- }
- return -1;
- }
-
- /**
- * Try to compress the data in currentChunk and store the result in the chunkStore.
- */
- private void saveCurrentChunk(){
- long mask = 0;
- int RLELen = -1;
- int opos = 0;
- long elementMask = 1L;
- short [] chunkToSave;
- // move used entries to the beginning
- for (int j=0; j < CHUNK_SIZE; j++){
- if (currentChunk[j] != unassigned) {
- mask |= elementMask;
- tmpWork[opos++] = currentChunk[j];
- }
- elementMask <<= 1;
- }
- uncompressedLen += opos;
- if (opos > ONE_VALUE_CHUNK_SIZE)
- RLELen = chunkCompressRLE(opos);
- if (RLELen > 0){
- chunkToSave = RLEWork;
- opos = RLELen;
- }
- else
- chunkToSave = tmpWork;
- compressedLen += opos;
- putChunk(currentChunkId, chunkToSave, opos, mask);
- }
-
- @Override
- public boolean containsKey(long key) {
- return get(key) != unassigned;
- }
-
-
- @Override
- public short put(long key, short val) {
- long chunkId = key & OLD_CHUNK_ID_MASK;
- if (val == unassigned) {
- throw new IllegalArgumentException("Cannot store the value that is reserved as being unassigned. val=" + val);
- }
- int chunkoffset = (int) (key & CHUNK_OFFSET_MASK);
- short out;
- if (currentChunkId == chunkId){
- out = currentChunk[chunkoffset];
- currentChunk[chunkoffset] = val;
- if (out == unassigned)
- size++;
- return out;
- }
-
- if (currentChunkId != INVALID_CHUNK_ID){
- // we need a different chunk
- saveCurrentChunk();
- }
-
- fillCurrentChunk(key);
- out = currentChunk[chunkoffset];
- currentChunkId = chunkId;
- currentChunk[chunkoffset] = val;
- if (out == unassigned)
- size++;
-
- return out;
- }
-
-
- /**
- * Check if we already have a chunk for the given key. If no,
- * fill currentChunk with default value, else with the saved
- * chunk.
- * @param key
- */
- private void fillCurrentChunk(long key) {
- Arrays.fill(currentChunk, unassigned);
- storedLengthOfCurrentChunk = 0;
- currentChunkIdInStore = 0;
- long topID = key >> TOP_ID_SHIFT;
- long[] largeVector = topMap.get(topID);
- if (largeVector == null)
- return;
- int chunkid = (int) (key & CHUNK_ID_MASK) / CHUNK_SIZE;
-
- long idx = largeVector[chunkid];
- if (idx == 0)
- return;
- currentChunkIdInStore = idx;
- int x = (int) (idx & CHUNK_STORE_X_MASK);
- int y = (int) ((idx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK);
- int chunkLen = x + 1;
- short[] store = chunkStore[x][y];
- int z = (int) ((idx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK);
-
- long chunkMask = maskStore[x][y][z];
- long elementmask = 0;
-
- ++expanded;
- storedLengthOfCurrentChunk = x;
- int startPos = z * chunkLen + 1;
- boolean isCompressed = (chunkLen == ONE_VALUE_CHUNK_SIZE || store[startPos] == unassigned);
- if (isCompressed){
- int opos = 0;
- if (chunkLen == ONE_VALUE_CHUNK_SIZE) {
- // decode one-value-chunk
- short val = store[startPos];
- elementmask = 1;
- for (opos = 0; opos<CHUNK_SIZE; opos++){
- if ((chunkMask & elementmask) != 0)
- currentChunk[opos] = val;
- elementmask <<= 1;
- }
- }
- else {
- // decode RLE-compressed chunk with multiple values
- int ipos = startPos + 1;
- int len = store[ipos++];
- short val = store[ipos++];
- while (len > 0){
- while (len > 0 && opos < currentChunk.length){
- if ((chunkMask & 1L << opos) != 0){
- currentChunk[opos] = val;
- --len;
- }
- ++opos;
- }
- if (ipos+1 < startPos + chunkLen){
- len = store[ipos++];
- val = store[ipos++];
- }
- else len = -1;
- }
- }
- }
- else {
- // decode uncompressed chunk
- int ipos = startPos;
- elementmask = 1;
- for (int opos=0; opos < CHUNK_SIZE; opos++) {
- if ((chunkMask & elementmask) != 0)
- currentChunk[opos] = store[ipos++];
- elementmask <<= 1;
- }
- }
- }
-
- @Override
- public short get(long key){
- long chunkId = key & OLD_CHUNK_ID_MASK;
- int chunkoffset = (int) (key & CHUNK_OFFSET_MASK);
-
- if (currentChunkId == chunkId)
- return currentChunk[chunkoffset];
-
- long topID = key >> TOP_ID_SHIFT;
- long[] largeVector = topMap.get(topID);
- if (largeVector == null)
- return unassigned;
- int chunkid = (int) (key & CHUNK_ID_MASK) / CHUNK_SIZE;
-
- long idx = largeVector[chunkid];
- if (idx == 0)
- return unassigned;
- int x = (int) (idx & CHUNK_STORE_X_MASK);
- int y = (int) ((idx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK);
- int chunkLen = x + 1;
- short[] store = chunkStore[x][y];
- int z = (int) ((idx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK);
-
- long chunkMask = maskStore[x][y][z];
-
- long elementmask = 1L << chunkoffset;
- if ((chunkMask & elementmask) == 0)
- return unassigned; // not in chunk
- int startOfChunk = z * chunkLen + 1;
- // the map contains the key, extract the value
- short firstAfterMask = store[startOfChunk];
- if (chunkLen == ONE_VALUE_CHUNK_SIZE)
- return firstAfterMask;
- int index = countUnder(chunkMask, chunkoffset);
- if (firstAfterMask == unassigned){
- // extract from compressed chunk
- short len;
- for (int j=1; j < chunkLen; j+=2){
- len = store[j+startOfChunk];
- index -= len;
- if (index < 0) return store[j+startOfChunk+1];
- }
- return unassigned; // should not happen
- }
- // extract from uncompressed chunk
- return store[index + startOfChunk];
- }
-
- @Override
- public void clear() {
- System.out.println(this.getClass().getSimpleName() + ": Allocating three-tier structure to save area info (HashMap->vector->chunkvector)");
- topMap = new Long2ObjectOpenHashMap<>();
- chunkStore = new short[CHUNK_SIZE+1][][];
- maskStore = new long[CHUNK_SIZE+1][][];
- freePosInSore = new int[CHUNK_SIZE+1];
- countChunkLen = new long[CHUNK_SIZE + 1 ]; // used for statistics
- reusableChunks = new Int2ObjectOpenHashMap<>();
- size = 0;
- uncompressedLen = 0;
- compressedLen = 0;
- expanded = 0;
- //test();
- }
-
- @Override
- public long size() {
- return size;
- }
-
- @Override
- public short defaultReturnValue() {
- return unassigned;
- }
-
- @Override
- public void defaultReturnValue(short arg0) {
- unassigned = arg0;
- }
-
- /**
- * Find the place were a chunk has to be stored and copy the content
- * to this place.
- * @param key the (OSM) id
- * @param chunk the chunk
- * @param len the number of used bytes in the chunk
- */
- private void putChunk (long key, short[] chunk, int len, long mask) {
- long topID = key >> TOP_ID_SHIFT;
- long[] largeVector = topMap.get(topID);
- if (largeVector == null){
- largeVector = new long[LARGE_VECTOR_SIZE];
- topMap.put(topID, largeVector);
- }
-
- int chunkid = (int) (key & CHUNK_ID_MASK) / CHUNK_SIZE;
- int x = len - 1;
- if (storedLengthOfCurrentChunk > 0){
- // this is a rewrite, add the previously used chunk to the reusable list
- LongArrayList reusableChunk = reusableChunks.get(storedLengthOfCurrentChunk);
- if (reusableChunk == null){
- reusableChunk = new LongArrayList(8);
- reusableChunks.put(storedLengthOfCurrentChunk, reusableChunk);
- }
- reusableChunk.add(currentChunkIdInStore);
- }
- if (chunkStore[x] == null){
- chunkStore[x] = new short[2][];
- maskStore[x] = new long[2][];
- }
- LongArrayList reusableChunk = reusableChunks.get(x);
- int y,z;
- short []store;
- if (reusableChunk != null && reusableChunk.isEmpty() == false){
- long reusedIdx = reusableChunk.removeLong(reusableChunk.size()-1);
- y = (int) ((reusedIdx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK);
- z = (int) ((reusedIdx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK);
- store = chunkStore[x][y];
- } else {
- y = ++freePosInSore[x] / CHUNK_STORE_ELEMS;
- if (y >= chunkStore[x].length){
- // resize
- int newElems = Math.min(y*2,1<<CHUNK_STORE_BITS_FOR_Y);
- if (y >= newElems){
- stats(1);
- System.err.println(this.getClass().getSimpleName()
- + ": reached limit of possible length-" + len
- + " chunks: "
- + Utils.format(newElems * CHUNK_STORE_ELEMS));
- throw new SplitFailedException("Try to reduce the max-area value so that one more pass is used.");
- }
- short[][] tmp = new short[newElems][];
- System.arraycopy(chunkStore[x], 0, tmp, 0, y);
- chunkStore[x] = tmp;
- long[][] tmpMask = new long[newElems][];
- System.arraycopy(maskStore[x], 0, tmpMask, 0, y);
- maskStore[x] = tmpMask;
- }
- if (chunkStore[x][y] == null){
- chunkStore[x][y] = new short[len * (CHUNK_STORE_ELEMS)+2];
- maskStore[x][y] = new long[CHUNK_STORE_ELEMS];
- }
- store = chunkStore[x][y];
- z = store[0]++;
- ++countChunkLen[len];
- }
- maskStore[x][y][z] = mask;
-
- if (len > 1)
- System.arraycopy(chunk, 0, store, z*len+1, len);
- else
- store[z*len+1] = chunk[0];
- assert x < 1<<CHUNK_STORE_BITS_FOR_X;
- assert y < 1<<CHUNK_STORE_BITS_FOR_Y;
- assert z < 1<<CHUNK_STORE_BITS_FOR_Z;
- long idx = CHUNK_STORE_USED_FLAG_MASK
- | (z & CHUNK_STORE_Z_MASK)<<CHUNK_STORE_Z_SHIFT
- | (y & CHUNK_STORE_Y_MASK)<< CHUNK_STORE_Y_SHIFT
- | (x & CHUNK_STORE_X_MASK);
-
- assert idx != 0;
- largeVector[chunkid] = idx;
- }
-
- @Override
- /**
- * calculate and print performance values regarding memory
- */
- public void stats(int msgLevel) {
- long totalOverhead = 0;
- long totalBytes = 0;
- long totalChunks = 0;
- int i;
-
- if (size() == 0){
- System.out.println(dataDesc + " Map is empty");
- return;
- }
- for (i=1; i <=CHUNK_SIZE; i++) {
- long bytes = countChunkLen[i] * (i*2+8) ; // 2 bytes for the shorts + 8 bytes for the mask
- totalChunks += countChunkLen[i];
- int freePos = freePosInSore[i-1];
- long overhead = (freePos == 0) ? 0: (64 - freePos % 64) * i * 2 +
- chunkStore[i-1].length * 4 + maskStore[i-1].length * 8;
- if (msgLevel > 0) {
- System.out.println("Length-" + i + " chunks: " + Utils.format(countChunkLen[i]) + ", used Bytes including overhead: " + Utils.format(bytes+overhead));
- //System.out.println("Length-" + i + " stores: " + Utils.format(chunkStore[i-1].length) + " pos " + freePosInSore[i-1]);
- }
- totalBytes += bytes;
- totalOverhead += overhead;
- }
- totalOverhead += topMap.size() * (long)LARGE_VECTOR_SIZE * 8;
-
- float bytesPerKey = (size()==0) ? 0: (float)((totalBytes + totalOverhead)*100 / size()) / 100;
- System.out.println();
- System.out.println(dataDesc + " Map: Number of stored ids: " + Utils.format(size()) + " require ca. " +
- bytesPerKey + " bytes per pair. " +
- totalChunks + " chunks are used, the avg. number of values in one "+CHUNK_SIZE+"-chunk is " +
- ((totalChunks==0) ? 0 :(size() / totalChunks)) + ".");
- System.out.println(dataDesc + " Map details: bytes/overhead " + Utils.format(totalBytes) + " / " + Utils.format(totalOverhead) + ", overhead includes " +
- topMap.size() + " arrays with " + LARGE_VECTOR_SIZE * 8/1024/1024 + " MB");
- if (msgLevel > 0 & uncompressedLen > 0){
- System.out.print(dataDesc + " RLE compression info: compressed / uncompressed size / ratio: " +
- Utils.format(compressedLen) + " / "+
- Utils.format(uncompressedLen) + " / "+
- Utils.format(Math.round(100-(float) (compressedLen*100/uncompressedLen))) + "%");
- if (expanded > 0 )
- System.out.print(", times fully expanded: " + Utils.format(expanded));
- System.out.println();
- }
-
- }
- /*
- void test(){
- for (int z = 0; z < 64; z++){
- for (int y = 0; y < 10; y++){
- for (int x=0; x < 64; x++){
- long idx = CHUNK_STORE_USED_FLAG_MASK
- | (z & CHUNK_STORE_Z_MASK)<<CHUNK_STORE_Z_SHIFT
- | (y & CHUNK_STORE_Y_MASK)<< CHUNK_STORE_Y_SHIFT
- | (x & CHUNK_STORE_X_MASK);
- // extract
- int x2 = (int) (idx & CHUNK_STORE_X_MASK);
- int y2 = (int) ((idx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK);
- int z2 = (int) ((idx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK);
- assert x == x2;
- assert y == y2;
- assert z == z2;
- }
- }
- }
- }
- */
-}
-
-
-
-
diff --git a/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java b/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java
deleted file mode 100644
index 9e4571e..0000000
--- a/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java
+++ /dev/null
@@ -1,562 +0,0 @@
-/*
- * Copyright (c) 2011, Gerd Petermann
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-
-package uk.me.parabola.splitter;
-
-import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
-import it.unimi.dsi.fastutil.ints.IntArrayList;
-import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
-
-import java.util.Arrays;
-
-
-
-/**
- * {@link SparseLong2ShortInline} implements {@link SparseLong2ShortMapFunction}
- * optimized for low memory requirements and inserts in sequential order.
- * Don't use this for a rather small number of pairs.
- *
- * Inspired by SparseInt2ShortMapInline.
- *
- * A HashMap is used to address large vectors which address chunks. The HashMap
- * is the only part that stores long values, and it will be very small as long
- * as long as input is normal OSM data and not something with random numbers.
- * A chunk stores up to CHUNK_SIZE values. A separately stored bit-mask is used
- * to separate used and unused entries in the chunk. Thus, the chunk length
- * depends on the number of used entries, not on the highest used entry.
- * A typical (uncompressed) chunk looks like this:
- * v1,v1,v1,v1,v1,v1,v2,v2,v2,v2,v1,v1,v1,v1,v1,u,?,?,...}
- * v1,v2: values stored in the chunk
- * u: "unassigned" value
- * ?: anything
- *
- * After applying Run Length Encryption on this the chunk looks like this:
- * {u,6,v1,4,v2,5,v1,?,?,?}
- * The unassigned value on index 0 signals a compressed chunk.
- *
- * An (uncompressed) ONE_VALUE_CHUNK may look like this:
- * {v1,v1,v1,v1,v1,v1,v1,v1,v1,v1,v1,u,?,?,...}
- * This is stored without run length info in the shortest possible trunk:
- * {v1}
- *
- * Fortunately, OSM data is distributed in a way that most(!) chunks contain
- * just one distinct value, so most chunks can be stored in 24 or 32 bytes
- * instead of 152 bytes for the worst case (counting also the padding bytes).
-
- * Since we have keys with 64 bits, we have to divide the key into 3 parts:
- * 37 bits for the value that is stored in the HashMap.
- * 21 bits for the chunkId (this gives the required length of a large vector)
- * 6 bits for the position in the chunk
- *
- * The chunkId identifies the position of a 32-bit value (stored in the large vector).
- * A chunk is stored in a chunkStore which is a 3-dimensional array.
- * We group chunks of equally length together in stores of 64 entries.
- * To find the right position of a new chunk, we need three values: x,y, and z.
- * x is the length of the chunk (the number of required shorts) (1-64, we store the value decremented by 1 to have 0-63)
- * y is the position of the store (0-524287)
- * z is the position of the chunk within the store. (0-63)
- * The maximum values for these three values are chosen so that we can place them
- * together into one int (32 bits).
- */
-
-public class SparseLong2ShortMapInline implements SparseLong2ShortMapFunction{
- private static final long CHUNK_ID_MASK = 0x7ffffffL; // the part of the key that is not saved in the top HashMap
- private static final long TOP_ID_MASK = ~CHUNK_ID_MASK; // the part of the key that is saved in the top HashMap
- private static final int TOP_ID_SHIFT = Long.numberOfTrailingZeros(TOP_ID_MASK);
-
- private static final int CHUNK_STORE_BITS_FOR_Z = 6;
- private static final int CHUNK_STORE_BITS_FOR_Y = 19;
- private static final int CHUNK_STORE_BITS_FOR_X = 6;
- private static final int CHUNK_STORE_ELEMS = 1 << CHUNK_STORE_BITS_FOR_X;
- private static final int CHUNK_STORE_X_MASK = (1 << CHUNK_STORE_BITS_FOR_X) - 1;
- private static final int CHUNK_STORE_Y_MASK = (1 << CHUNK_STORE_BITS_FOR_Y) - 1;
- private static final int CHUNK_STORE_Z_MASK = (1 << CHUNK_STORE_BITS_FOR_Z) - 1;
- private static final int CHUNK_STORE_USED_FLAG_MASK = 1<<31;
- private static final int CHUNK_STORE_Y_SHIFT = CHUNK_STORE_BITS_FOR_X;
- private static final int CHUNK_STORE_Z_SHIFT = CHUNK_STORE_BITS_FOR_X + CHUNK_STORE_BITS_FOR_Y;
-
- private static final int CHUNK_SIZE = 64; // 64 = 1<< 6 (last 6 bits of the key)
- private static final long CHUNK_OFFSET_MASK = CHUNK_SIZE-1; // the part of the key that contains the offset in the chunk
- private static final long OLD_CHUNK_ID_MASK = ~CHUNK_OFFSET_MASK; // first 58 bits of a long. If this part of the key changes, a different chunk is needed
-
- private static final long INVALID_CHUNK_ID = 1L; // must NOT be divisible by CHUNK_SIZE
- private static final int LARGE_VECTOR_SIZE = (int)(CHUNK_ID_MASK/ CHUNK_SIZE + 1); // number of entries addressed by one topMap entry
-
- private static final int ONE_VALUE_CHUNK_SIZE = 1;
-
- /** What to return on unassigned indices */
- private short unassigned = UNASSIGNED;
- private long size;
-
-
- private long currentChunkId = INVALID_CHUNK_ID;
- private short [] currentChunk = new short[CHUNK_SIZE]; // stores the values in the real position
- private short [] tmpWork = new short[CHUNK_SIZE]; // a chunk after applying the "mask encoding"
- private short [] RLEWork = new short[CHUNK_SIZE]; // for the RLE-compressed chunk
-
-
- // for statistics
- private final String dataDesc;
- private long [] countChunkLen;
- private long expanded = 0;
- private long uncompressedLen = 0;
- private long compressedLen = 0;
- private int storedLengthOfCurrentChunk = 0;
- private int currentChunkIdInStore = 0;
-
- private Long2ObjectOpenHashMap<int[]> topMap;
- private short[][][] chunkStore;
- private long[][][] maskStore;
- private int[] freePosInSore;
- // maps chunks that can be reused
- private Int2ObjectOpenHashMap<IntArrayList> reusableChunks;
-
- /**
- * A map that stores pairs of (OSM) IDs and short values identifying the
- * areas in which the object (node,way) with the ID occurs.
- * @param dataDesc
- */
- SparseLong2ShortMapInline(String dataDesc) {
- this.dataDesc = dataDesc;
- clear();
- }
-
- /**
- * Count how many of the lowest X bits in mask are set
- *
- * @return
- */
- private static int countUnder(long mask, int lowest) {
- return Long.bitCount(mask & ((1L << lowest) - 1));
- }
-
- /**
- * Try to use Run Length Encoding to compress the chunk stored in tmpWork. In most
- * cases this works very well because chunks often have only one
- * or two distinct values.
- * @param maxlen: number of elements in the chunk.
- * @return -1 if compression doesn't save space, else the number of elements in the
- * compressed chunk stored in buffer RLEWork.
- */
- private int chunkCompressRLE (int maxlen){
- int opos = 1;
- for (int i = 0; i < maxlen; i++) {
- short runLength = 1;
- while (i+1 < maxlen && tmpWork[i] == tmpWork[i+1]) {
- runLength++;
- i++;
- }
- if (opos+2 >= tmpWork.length)
- return -1; // compressed record is not shorter
- RLEWork[opos++] = runLength;
- RLEWork[opos++] = tmpWork[i];
- }
- if (opos == 3){
- // special case: the chunk contains only one distinct value
- // we can store this in a length-1 chunk because we don't need
- // the length counter nor the compression flag
- RLEWork[0] = RLEWork[2];
- return ONE_VALUE_CHUNK_SIZE;
- }
-
- if (opos < maxlen){
- RLEWork[0] = unassigned; // signal a normal compressed record
- return opos;
- }
- return -1;
- }
-
- /**
- * Try to compress the data in currentChunk and store the result in the chunkStore.
- */
- private void saveCurrentChunk(){
- long mask = 0;
- int RLELen = -1;
- int opos = 0;
- long elementMask = 1L;
- short [] chunkToSave;
- // move used entries to the beginning
- for (int j=0; j < CHUNK_SIZE; j++){
- if (currentChunk[j] != unassigned) {
- mask |= elementMask;
- tmpWork[opos++] = currentChunk[j];
- }
- elementMask <<= 1;
- }
- uncompressedLen += opos;
- if (opos > ONE_VALUE_CHUNK_SIZE)
- RLELen = chunkCompressRLE(opos);
- if (RLELen > 0){
- chunkToSave = RLEWork;
- opos = RLELen;
- }
- else
- chunkToSave = tmpWork;
- compressedLen += opos;
- putChunk(currentChunkId, chunkToSave, opos, mask);
- }
-
- @Override
- public boolean containsKey(long key) {
- return get(key) != unassigned;
- }
-
-
- @Override
- public short put(long key, short val) {
- long chunkId = key & OLD_CHUNK_ID_MASK;
- if (val == unassigned) {
- throw new IllegalArgumentException("Cannot store the value that is reserved as being unassigned. val=" + val);
- }
- int chunkoffset = (int) (key & CHUNK_OFFSET_MASK);
- short out;
- if (currentChunkId == chunkId){
- out = currentChunk[chunkoffset];
- currentChunk[chunkoffset] = val;
- if (out == unassigned)
- size++;
- return out;
- }
-
- if (currentChunkId != INVALID_CHUNK_ID){
- // we need a different chunk
- saveCurrentChunk();
- }
-
- fillCurrentChunk(key);
- out = currentChunk[chunkoffset];
- currentChunkId = chunkId;
- currentChunk[chunkoffset] = val;
- if (out == unassigned)
- size++;
-
- return out;
- }
-
-
- /**
- * Check if we already have a chunk for the given key. If no,
- * fill currentChunk with default value, else with the saved
- * chunk.
- * @param key
- */
- private void fillCurrentChunk(long key) {
- Arrays.fill(currentChunk, unassigned);
- storedLengthOfCurrentChunk = 0;
- currentChunkIdInStore = 0;
- long topID = key >> TOP_ID_SHIFT;
- int[] largeVector = topMap.get(topID);
- if (largeVector == null)
- return;
- int chunkid = (int) (key & CHUNK_ID_MASK) / CHUNK_SIZE;
-
- int idx = largeVector[chunkid];
- if (idx == 0)
- return;
- currentChunkIdInStore = idx;
- int x = idx & CHUNK_STORE_X_MASK;
- int y = (idx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK;
- int chunkLen = x + 1;
- short [] store = chunkStore[x][y];
- int z = (idx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK;
-
- long chunkMask = maskStore[x][y][z];
- long elementmask = 0;
-
- ++expanded;
- storedLengthOfCurrentChunk = x;
- int startPos = z * chunkLen + 1;
- boolean isCompressed = (chunkLen == ONE_VALUE_CHUNK_SIZE || store[startPos] == unassigned);
- if (isCompressed){
- int opos = 0;
- if (chunkLen == ONE_VALUE_CHUNK_SIZE) {
- // decode one-value-chunk
- short val = store[startPos];
- elementmask = 1;
- for (opos = 0; opos<CHUNK_SIZE; opos++){
- if ((chunkMask & elementmask) != 0)
- currentChunk[opos] = val;
- elementmask <<= 1;
- }
- }
- else {
- // decode RLE-compressed chunk with multiple values
- int ipos = startPos + 1;
- int len = store[ipos++];
- short val = store[ipos++];
- while (len > 0){
- while (len > 0 && opos < currentChunk.length){
- if ((chunkMask & 1L << opos) != 0){
- currentChunk[opos] = val;
- --len;
- }
- ++opos;
- }
- if (ipos+1 < startPos + chunkLen){
- len = store[ipos++];
- val = store[ipos++];
- }
- else len = -1;
- }
- }
- }
- else {
- // decode uncompressed chunk
- int ipos = startPos;
- elementmask = 1;
- for (int opos=0; opos < CHUNK_SIZE; opos++) {
- if ((chunkMask & elementmask) != 0)
- currentChunk[opos] = store[ipos++];
- elementmask <<= 1;
- }
- }
- }
-
- @Override
- public short get(long key){
- long chunkId = key & OLD_CHUNK_ID_MASK;
- int chunkoffset = (int) (key & CHUNK_OFFSET_MASK);
-
- if (currentChunkId == chunkId)
- return currentChunk[chunkoffset];
-
- long topID = key >> TOP_ID_SHIFT;
- int[] largeVector = topMap.get(topID);
- if (largeVector == null)
- return unassigned;
- int chunkid = (int) (key & CHUNK_ID_MASK) / CHUNK_SIZE;
-
- int idx = largeVector[chunkid];
- if (idx == 0)
- return unassigned;
- int x = idx & CHUNK_STORE_X_MASK;
- int y = (idx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK;
- int chunkLen = x + 1;
- short [] store = chunkStore[x][y];
- int z = (idx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK;
-
- long chunkMask = maskStore[x][y][z];
-
- long elementmask = 1L << chunkoffset;
- if ((chunkMask & elementmask) == 0)
- return unassigned; // not in chunk
- int startOfChunk = z * chunkLen + 1;
- // the map contains the key, extract the value
- short firstAfterMask = store[startOfChunk];
- if (chunkLen == ONE_VALUE_CHUNK_SIZE)
- return firstAfterMask;
- int index = countUnder(chunkMask, chunkoffset);
- if (firstAfterMask == unassigned){
- // extract from compressed chunk
- short len;
- for (int j=1; j < chunkLen; j+=2){
- len = store[j+startOfChunk];
- index -= len;
- if (index < 0) return store[j+startOfChunk+1];
- }
- return unassigned; // should not happen
- }
- // extract from uncompressed chunk
- return store[index + startOfChunk];
- }
-
- @Override
- public void clear() {
- System.out.println(this.getClass().getSimpleName() + ": Allocating three-tier structure to save area info (HashMap->vector->chunkvector)");
- topMap = new Long2ObjectOpenHashMap<>();
- chunkStore = new short[CHUNK_SIZE+1][][];
- maskStore = new long[CHUNK_SIZE+1][][];
- freePosInSore = new int[CHUNK_SIZE+1];
- countChunkLen = new long[CHUNK_SIZE + 1 ]; // used for statistics
- reusableChunks = new Int2ObjectOpenHashMap<>();
- size = 0;
- uncompressedLen = 0;
- compressedLen = 0;
- expanded = 0;
- //test();
- }
-
- @Override
- public long size() {
- return size;
- }
-
- @Override
- public short defaultReturnValue() {
- return unassigned;
- }
-
- @Override
- public void defaultReturnValue(short arg0) {
- unassigned = arg0;
- }
-
- /**
- * Find the place were a chunk has to be stored and copy the content
- * to this place.
- * @param key the (OSM) id
- * @param chunk the chunk
- * @param len the number of used bytes in the chunk
- */
- private void putChunk (long key, short[] chunk, int len, long mask) {
- long topID = key >> TOP_ID_SHIFT;
- int[] largeVector = topMap.get(topID);
- if (largeVector == null){
- largeVector = new int[LARGE_VECTOR_SIZE];
- topMap.put(topID, largeVector);
- }
-
- int chunkid = (int) (key & CHUNK_ID_MASK) / CHUNK_SIZE;
- int x = len - 1;
- if (storedLengthOfCurrentChunk > 0){
- // this is a rewrite, add the previously used chunk to the reusable list
- IntArrayList reusableChunk = reusableChunks.get(storedLengthOfCurrentChunk);
- if (reusableChunk == null){
- reusableChunk = new IntArrayList(8);
- reusableChunks.put(storedLengthOfCurrentChunk, reusableChunk);
- }
- reusableChunk.add(currentChunkIdInStore);
- }
- if (chunkStore[x] == null){
- chunkStore[x] = new short[2][];
- maskStore[x] = new long[2][];
- }
- IntArrayList reusableChunk = reusableChunks.get(x);
- int y,z;
- short []store;
- if (reusableChunk != null && reusableChunk.isEmpty() == false){
- int reusedIdx = reusableChunk.removeInt(reusableChunk.size()-1);
- y = (reusedIdx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK;
- z = (reusedIdx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK;
- store = chunkStore[x][y];
- } else {
- y = ++freePosInSore[x] / CHUNK_STORE_ELEMS;
- if (y >= chunkStore[x].length){
- // resize
- int newElems = Math.min(y*2,1<<CHUNK_STORE_BITS_FOR_Y);
- if (y >= newElems){
- stats(1);
- System.err.println(this.getClass().getSimpleName()
- + ": reached limit of possible length-" + len
- + " chunks: "
- + Utils.format(newElems * CHUNK_STORE_ELEMS));
- Utils.printMem();
- throw new SplitFailedException("Try to increase max heap to a value > 2GB or reduce the max-area value so that one more pass is used.");
- }
- short[][] tmp = new short[newElems][];
- System.arraycopy(chunkStore[x], 0, tmp, 0, y);
- chunkStore[x] = tmp;
- long[][] tmpMask = new long[newElems][];
- System.arraycopy(maskStore[x], 0, tmpMask, 0, y);
- maskStore[x] = tmpMask;
- }
- if (chunkStore[x][y] == null){
- chunkStore[x][y] = new short[len * (CHUNK_STORE_ELEMS)+2];
- maskStore[x][y] = new long[CHUNK_STORE_ELEMS];
- }
- store = chunkStore[x][y];
- z = store[0]++;
- ++countChunkLen[len];
- }
- maskStore[x][y][z] = mask;
-
- if (len > 1)
- System.arraycopy(chunk, 0, store, z*len+1, len);
- else
- store[z*len+1] = chunk[0];
- assert x < 1<<CHUNK_STORE_BITS_FOR_X;
- assert y < 1<<CHUNK_STORE_BITS_FOR_Y;
- assert z < 1<<CHUNK_STORE_BITS_FOR_Z;
- int idx = CHUNK_STORE_USED_FLAG_MASK
- | (z & CHUNK_STORE_Z_MASK)<<CHUNK_STORE_Z_SHIFT
- | (y & CHUNK_STORE_Y_MASK)<< CHUNK_STORE_Y_SHIFT
- | (x & CHUNK_STORE_X_MASK);
-
- assert idx != 0;
- largeVector[chunkid] = idx;
- }
-
- @Override
- /**
- * calculate and print performance values regarding memory
- */
- public void stats(int msgLevel) {
- long totalOverhead = 0;
- long totalBytes = 0;
- long totalChunks = 0;
- int i;
-
- if (size() == 0){
- System.out.println(dataDesc + " Map is empty");
- return;
- }
- for (i=1; i <=CHUNK_SIZE; i++) {
- long bytes = countChunkLen[i] * (i*2+8) ; // 2 bytes for the shorts + 8 bytes for the mask
- totalChunks += countChunkLen[i];
- int freePos = freePosInSore[i-1];
- long overhead = (freePos == 0) ? 0: (64 - freePos % 64) * i * 2 +
- chunkStore[i-1].length * 4 + maskStore[i-1].length * 8;
- if (msgLevel > 0) {
- System.out.println("Length-" + i + " chunks: " + Utils.format(countChunkLen[i]) + ", used Bytes including overhead: " + Utils.format(bytes+overhead));
- //System.out.println("Length-" + i + " stores: " + Utils.format(chunkStore[i-1].length) + " pos " + freePosInSore[i-1]);
- }
- totalBytes += bytes;
- totalOverhead += overhead;
- }
- totalOverhead += topMap.size() * (long)LARGE_VECTOR_SIZE * 4;
-
- float bytesPerKey = (size()==0) ? 0: (float)((totalBytes + totalOverhead)*100 / size()) / 100;
- System.out.println();
- System.out.println(dataDesc + " Map: Number of stored ids: " + Utils.format(size()) + " require ca. " +
- bytesPerKey + " bytes per pair. " +
- totalChunks + " chunks are used, the avg. number of values in one "+CHUNK_SIZE+"-chunk is " +
- ((totalChunks==0) ? 0 :(size() / totalChunks)) + ".");
- System.out.println(dataDesc + " Map details: bytes/overhead " + Utils.format(totalBytes) + " / " + Utils.format(totalOverhead) + ", overhead includes " +
- topMap.size() + " arrays with " + LARGE_VECTOR_SIZE * 4/1024/1024 + " MB");
- if (msgLevel > 0 & uncompressedLen > 0){
- System.out.print("RLE compression info: compressed / uncompressed size / ratio: " +
- Utils.format(compressedLen) + " / "+
- Utils.format(uncompressedLen) + " / "+
- Utils.format(Math.round(100-(float) (compressedLen*100/uncompressedLen))) + "%");
- if (expanded > 0 )
- System.out.print(", times fully expanded: " + Utils.format(expanded));
- System.out.println();
- }
-
- }
- /*
- void test(){
- for (int z = 0; z < 64; z++){
- for (int y = 0; y < 10; y++){
- for (int x=0; x < 64; x++){
- int idx = CHUNK_STORE_USED_FLAG_MASK
- | (z & CHUNK_STORE_Z_MASK)<<CHUNK_STORE_Z_SHIFT
- | (y & CHUNK_STORE_Y_MASK)<< CHUNK_STORE_Y_SHIFT
- | (x & CHUNK_STORE_X_MASK);
- // extract
- int x2 = idx & CHUNK_STORE_X_MASK;
- int y2 = (idx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK;
- int z2 = (idx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK;
- assert x == x2;
- assert y == y2;
- assert z == z2;
- }
- }
- }
- }
- */
-}
-
-
-
-
diff --git a/src/uk/me/parabola/splitter/SplitFailedException.java b/src/uk/me/parabola/splitter/SplitFailedException.java
index 0f72644..d9bcaf6 100644
--- a/src/uk/me/parabola/splitter/SplitFailedException.java
+++ b/src/uk/me/parabola/splitter/SplitFailedException.java
@@ -9,7 +9,7 @@
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
- */
+ */
package uk.me.parabola.splitter;
/**
@@ -21,8 +21,9 @@ public class SplitFailedException extends RuntimeException {
public SplitFailedException(String s) {
super(s);
}
- public SplitFailedException(String message, Throwable cause) {
- super(message, cause);
- }
-
+
+ public SplitFailedException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
}
diff --git a/src/uk/me/parabola/splitter/SplitProcessor.java b/src/uk/me/parabola/splitter/SplitProcessor.java
index b6dafa5..e84772a 100644
--- a/src/uk/me/parabola/splitter/SplitProcessor.java
+++ b/src/uk/me/parabola/splitter/SplitProcessor.java
@@ -13,10 +13,13 @@
package uk.me.parabola.splitter;
import uk.me.parabola.splitter.Relation.Member;
+import uk.me.parabola.splitter.args.SplitterParams;
+import uk.me.parabola.splitter.tools.Long2IntClosedMapFunction;
+import uk.me.parabola.splitter.tools.SparseLong2IntMap;
+import uk.me.parabola.splitter.writer.OSMWriter;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.BitSet;
import java.util.Date;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
@@ -27,50 +30,51 @@ import java.util.concurrent.BlockingQueue;
class SplitProcessor extends AbstractMapProcessor {
private final OSMWriter[] writers;
- private SparseLong2ShortMapFunction coords;
- private SparseLong2ShortMapFunction ways;
- private final AreaDictionaryShort writerDictionary;
+ private SparseLong2IntMap coords;
+ private SparseLong2IntMap ways;
+ private final AreaDictionary writerDictionary;
private final DataStorer dataStorer;
private final Long2IntClosedMapFunction nodeWriterMap;
private final Long2IntClosedMapFunction wayWriterMap;
private final Long2IntClosedMapFunction relWriterMap;
// for statistics
- private long countQuickTest = 0;
- private long countFullTest = 0;
- private long countCoords = 0;
- private long countWays = 0;
+ private long countQuickTest;
+ private long countFullTest;
+ private long countCoords;
+ private long countWays;
private final int writerOffset;
private final int lastWriter;
private final AreaIndex writerIndex;
private final int maxThreads;
- private final short unassigned = Short.MIN_VALUE;
private final InputQueueInfo[] writerInputQueues;
protected final BlockingQueue<InputQueueInfo> toProcess;
private final ArrayList<Thread> workerThreads;
protected final InputQueueInfo STOP_MSG = new InputQueueInfo(null);
-
- // private int currentNodeAreaSet;
- private BitSet currentWayAreaSet;
- private BitSet currentRelAreaSet;
- private BitSet usedWriters;
-
+ private AreaSet usedWriters;
- SplitProcessor(DataStorer dataStorer, int writerOffset, int numWritersThisPass, int maxThreads){
+ /**
+ * Distribute the OSM data to separate OSM files.
+ * @param dataStorer
+ * @param writerOffset first writer to be used
+ * @param numWritersThisPass number of writers to used
+ * @param mainOptions main program options
+ */
+ SplitProcessor(DataStorer dataStorer, int writerOffset, int numWritersThisPass, SplitterParams mainOptions){
this.dataStorer = dataStorer;
this.writerDictionary = dataStorer.getAreaDictionary();
this.writers = dataStorer.getWriters();
- this.coords = SparseLong2ShortMap.createMap("coord");
- this.ways = SparseLong2ShortMap.createMap("way");
- this.coords.defaultReturnValue(unassigned);
- this.ways.defaultReturnValue(unassigned);
+ this.coords = new SparseLong2IntMap("coord");
+ this.ways = new SparseLong2IntMap("way");
+ this.coords.defaultReturnValue(UNASSIGNED);
+ this.ways.defaultReturnValue(UNASSIGNED);
this.writerIndex = dataStorer.getGrid();
this.countWays = ways.size();
this.writerOffset = writerOffset;
this.lastWriter = writerOffset + numWritersThisPass-1;
- this.maxThreads = maxThreads;
+ this.maxThreads = mainOptions.getMaxThreads().getCount();
this.toProcess = new ArrayBlockingQueue<>(numWritersThisPass);
this.writerInputQueues = new InputQueueInfo[numWritersThisPass];
for (int i = 0; i < writerInputQueues.length; i++) {
@@ -80,9 +84,7 @@ class SplitProcessor extends AbstractMapProcessor {
nodeWriterMap = dataStorer.getWriterMap(DataStorer.NODE_TYPE);
wayWriterMap = dataStorer.getWriterMap(DataStorer.WAY_TYPE);
relWriterMap = dataStorer.getWriterMap(DataStorer.REL_TYPE);
- currentWayAreaSet = new BitSet(writers.length);
- currentRelAreaSet = new BitSet(writers.length);
- usedWriters = new BitSet();
+ usedWriters = new AreaSet();
int noOfWorkerThreads = Math.min(this.maxThreads - 1, numWritersThisPass);
workerThreads = new ArrayList<>(noOfWorkerThreads);
@@ -95,6 +97,22 @@ class SplitProcessor extends AbstractMapProcessor {
}
+ /**
+ * Get the active writers associated to the index
+ * @param multiTileWriterIdx
+ */
+ private void setUsedWriters(int multiTileWriterIdx) {
+ if (multiTileWriterIdx != UNASSIGNED) {
+ AreaSet cl = writerDictionary.getSet(multiTileWriterIdx);
+ // set only active writer bits
+ for (int i : cl) {
+ if (i >= writerOffset && i <= lastWriter)
+ usedWriters.set(i);
+ }
+ }
+ }
+
+
@Override
public void processNode(Node n) {
try {
@@ -106,28 +124,19 @@ class SplitProcessor extends AbstractMapProcessor {
@Override
public void processWay(Way w) {
- currentWayAreaSet.clear();
- int multiTileWriterIdx = (wayWriterMap != null) ? wayWriterMap.getSeq(w.getId()): AreaDictionaryInt.UNASSIGNED;
- if (multiTileWriterIdx != AreaDictionaryInt.UNASSIGNED){
- BitSet cl = dataStorer.getMultiTileDictionary().getBitSet(multiTileWriterIdx);
- // set only active writer bits
- for(int i=cl.nextSetBit(writerOffset); i>=0 && i <= lastWriter; i=cl.nextSetBit(i+1)){
- currentWayAreaSet.set(i);
- }
- //System.out.println("added or completed way: " + w.getId());
+ usedWriters.clear();
+ int multiTileWriterIdx = (wayWriterMap != null) ? wayWriterMap.getSeq(w.getId()): UNASSIGNED;
+ if (multiTileWriterIdx != UNASSIGNED){
+ setUsedWriters(multiTileWriterIdx);
}
else{
- short oldclIndex = unassigned;
- //for (long id : w.getRefs()) {
- int refs = w.getRefs().size();
- for (int i = 0; i < refs; i++){
- long id = w.getRefs().getLong(i);
+ int oldclIndex = UNASSIGNED;
+ for (long id : w.getRefs()) {
// Get the list of areas that the way is in.
- short clIdx = coords.get(id);
- if (clIdx != unassigned){
+ int clIdx = coords.get(id);
+ if (clIdx != UNASSIGNED){
if (oldclIndex != clIdx){
- BitSet cl = writerDictionary.getBitSet(clIdx);
- currentWayAreaSet.or(cl);
+ usedWriters.or(writerDictionary.getSet(clIdx));
if (wayWriterMap != null){
// we can stop here because all other nodes
// will be in the same tile
@@ -138,14 +147,12 @@ class SplitProcessor extends AbstractMapProcessor {
}
}
}
- if (!currentWayAreaSet.isEmpty()){
+ if (!usedWriters.isEmpty()){
// store these areas in ways map
- short idx = writerDictionary.translate(currentWayAreaSet);
- ways.put(w.getId(), idx);
+ ways.put(w.getId(), writerDictionary.translate(usedWriters));
++countWays;
- if (countWays % 1000000 == 0){
- System.out.println("way MAP occupancy: " + Utils.format(countWays) + ", number of area dictionary entries: " + writerDictionary.size() + " of " + ((1<<16) - 1));
- ways.stats(0);
+ if (countWays % 10_000_000 == 0){
+ System.out.println(" Number of stored tile combinations in multiTileDictionary: " + Utils.format(writerDictionary.size()));
}
try {
writeWay(w);
@@ -157,60 +164,46 @@ class SplitProcessor extends AbstractMapProcessor {
@Override
public void processRelation(Relation rel) {
- currentRelAreaSet.clear();
+ usedWriters.clear();
Integer singleTileWriterIdx = dataStorer.getOneTileOnlyRels(rel.getId());
if (singleTileWriterIdx != null){
- if (singleTileWriterIdx == AreaDictionaryInt.UNASSIGNED) {
+ if (singleTileWriterIdx == UNASSIGNED) {
// we know that the relation is outside of all real areas
return;
}
// relation is within an area that is overlapped by the writer areas
- BitSet wl = dataStorer.getMultiTileDictionary().getBitSet(singleTileWriterIdx);
- // set only active writer bits
- for (int i = wl.nextSetBit(writerOffset); i >= 0 && i <= lastWriter; i = wl.nextSetBit(i + 1)) {
- currentRelAreaSet.set(i);
- }
+ setUsedWriters(singleTileWriterIdx);
} else {
- int multiTileWriterIdx = (relWriterMap != null) ? relWriterMap.getSeq(rel.getId()): AreaDictionaryInt.UNASSIGNED;
- if (multiTileWriterIdx != AreaDictionaryInt.UNASSIGNED){
-
- BitSet cl = dataStorer.getMultiTileDictionary().getBitSet(multiTileWriterIdx);
- // set only active writer bits
- for (int i = cl.nextSetBit(writerOffset); i >= 0 && i <= lastWriter; i = cl.nextSetBit(i + 1)) {
- currentRelAreaSet.set(i);
- }
- }
- else{
- short oldclIndex = unassigned;
- short oldwlIndex = unassigned;
+ int multiTileWriterIdx = (relWriterMap != null) ? relWriterMap.getSeq(rel.getId())
+ : UNASSIGNED;
+ if (multiTileWriterIdx != UNASSIGNED) {
+ setUsedWriters(multiTileWriterIdx);
+ } else{
+ int oldclIndex = UNASSIGNED;
+ int oldwlIndex = UNASSIGNED;
for (Member mem : rel.getMembers()) {
// String role = mem.getRole();
long id = mem.getRef();
if (mem.getType().equals("node")) {
- short clIdx = coords.get(id);
+ int clIdx = coords.get(id);
- if (clIdx != unassigned){
+ if (clIdx != UNASSIGNED){
if (oldclIndex != clIdx){
- BitSet wl = writerDictionary.getBitSet(clIdx);
- currentRelAreaSet.or(wl);
+ usedWriters.or(writerDictionary.getSet(clIdx));
}
oldclIndex = clIdx;
}
} else if (mem.getType().equals("way")) {
- short wlIdx = ways.get(id);
+ int wlIdx = ways.get(id);
- if (wlIdx != unassigned){
+ if (wlIdx != UNASSIGNED){
if (oldwlIndex != wlIdx){
- BitSet wl = writerDictionary.getBitSet(wlIdx);
- currentRelAreaSet.or(wl);
+ usedWriters.or(writerDictionary.getSet(wlIdx));
}
oldwlIndex = wlIdx;
}
}
}
-// if (currentRelAreaSet.cardinality() > 1 && relWriterMap != null){
-// System.out.println("relation " + rel.getId() + " " + rel.tags + " might be incomplete in some tiles");
-// }
}
}
try {
@@ -262,19 +255,17 @@ class SplitProcessor extends AbstractMapProcessor {
private void writeNode(Node currentNode) throws IOException {
int countWriters = 0;
- short lastUsedWriter = unassigned;
+ int lastUsedWriter = UNASSIGNED;
AreaGridResult writerCandidates = writerIndex.get(currentNode);
- int multiTileWriterIdx = (nodeWriterMap != null) ? nodeWriterMap.getSeq(currentNode.getId()): AreaDictionaryInt.UNASSIGNED;
+ int multiTileWriterIdx = (nodeWriterMap != null) ? nodeWriterMap.getSeq(currentNode.getId()): UNASSIGNED;
- boolean isSpecialNode = (multiTileWriterIdx != AreaDictionaryInt.UNASSIGNED);
+ boolean isSpecialNode = (multiTileWriterIdx != UNASSIGNED);
if (writerCandidates == null && !isSpecialNode) {
return;
}
- if (isSpecialNode || writerCandidates != null && writerCandidates.l.size() > 1)
- usedWriters.clear();
+ usedWriters.clear();
if (writerCandidates != null){
- for (int i = 0; i < writerCandidates.l.size(); i++) {
- int n = writerCandidates.l.getShort(i);
+ for (int n : writerCandidates.set) {
if (n < writerOffset || n > lastWriter)
continue;
OSMWriter writer = writers[n];
@@ -290,7 +281,7 @@ class SplitProcessor extends AbstractMapProcessor {
if (found) {
usedWriters.set(n);
++countWriters;
- lastUsedWriter = (short) n;
+ lastUsedWriter = n;
if (maxThreads > 1) {
addToWorkingQueue(n, currentNode);
} else {
@@ -301,8 +292,11 @@ class SplitProcessor extends AbstractMapProcessor {
}
if (isSpecialNode){
// this node is part of a multi-tile-polygon, add it to all tiles covered by the parent
- BitSet nodeWriters = dataStorer.getMultiTileDictionary().getBitSet(multiTileWriterIdx);
- for(int i=nodeWriters.nextSetBit(writerOffset); i>=0 && i <= lastWriter; i=nodeWriters.nextSetBit(i+1)){
+ AreaSet nodeWriters = writerDictionary.getSet(multiTileWriterIdx);
+ for (int i : nodeWriters) {
+ if (i < writerOffset || i > lastWriter)
+ continue;
+
if (usedWriters.get(i) )
continue;
if (maxThreads > 1) {
@@ -314,16 +308,15 @@ class SplitProcessor extends AbstractMapProcessor {
}
if (countWriters > 0){
- short writersID;
+ int writersID;
if (countWriters > 1)
writersID = writerDictionary.translate(usedWriters);
else
- writersID = AreaDictionaryShort.translate(lastUsedWriter); // no need to do lookup in the dictionary
+ writersID = AreaDictionary.translate(lastUsedWriter); // no need to do lookup in the dictionary
coords.put(currentNode.getId(), writersID);
++countCoords;
- if (countCoords % 10000000 == 0){
- System.out.println("MAP occupancy: " + Utils.format(countCoords) + ", number of area dictionary entries: " + writerDictionary.size() + " of " + ((1<<16) - 1));
- coords.stats(0);
+ if (countCoords % 100_000_000 == 0){
+ System.out.println("coord MAP occupancy: " + Utils.format(countCoords) + ", number of area dictionary entries: " + writerDictionary.size());
}
}
}
@@ -335,16 +328,7 @@ class SplitProcessor extends AbstractMapProcessor {
seenWay = true;
System.out.println("Writing ways " + new Date());
}
-
- if (!currentWayAreaSet.isEmpty()) {
- for (int n = currentWayAreaSet.nextSetBit(0); n >= 0; n = currentWayAreaSet.nextSetBit(n + 1)) {
- if (maxThreads > 1) {
- addToWorkingQueue(n, currentWay);
- } else {
- writers[n].write(currentWay);
- }
- }
- }
+ writeElement(currentWay, usedWriters);
}
private boolean seenRel;
@@ -354,17 +338,23 @@ class SplitProcessor extends AbstractMapProcessor {
seenRel = true;
System.out.println("Writing relations " + new Date());
}
-
- for (int n = currentRelAreaSet.nextSetBit(0); n >= 0; n = currentRelAreaSet.nextSetBit(n + 1)) {
- // if n is out of bounds, then something has gone wrong
- if (maxThreads > 1) {
- addToWorkingQueue(n, currentRelation);
- } else {
- writers[n].write(currentRelation);
+ writeElement(currentRelation, usedWriters);
+ }
+
+ private void writeElement (Element el, AreaSet writersToUse) throws IOException {
+ if (!writersToUse.isEmpty()) {
+ for (int n : writersToUse) {
+ if (n < writerOffset || n > lastWriter)
+ continue;
+ if (maxThreads > 1) {
+ addToWorkingQueue(n, el);
+ } else {
+ writers[n].write(el);
+ }
}
}
}
-
+
private void addToWorkingQueue(int writerNumber, Element element) {
try {
writerInputQueues[writerNumber-writerOffset].put(element);
@@ -386,9 +376,8 @@ class SplitProcessor extends AbstractMapProcessor {
void put(Element e) throws InterruptedException {
staging.add(e);
- if (staging.size() < STAGING_SIZE)
- return;
- flush();
+ if (staging.size() >= STAGING_SIZE)
+ flush();
}
void flush() throws InterruptedException {
@@ -411,17 +400,6 @@ class SplitProcessor extends AbstractMapProcessor {
public OSMWriterWorker() {
}
- public void processElement(Element element, OSMWriter writer)
- throws IOException {
- if (element instanceof Node) {
- writer.write((Node) element);
- } else if (element instanceof Way) {
- writer.write((Way) element);
- } else if (element instanceof Relation) {
- writer.write((Relation) element);
- }
- }
-
@Override
public void run() {
boolean finished = false;
@@ -436,29 +414,28 @@ class SplitProcessor extends AbstractMapProcessor {
if (workPackage == STOP_MSG) {
try {
toProcess.put(STOP_MSG); // Re-inject it so that other
- // threads know that we're
- // exiting.
+ // threads know that we're
+ // exiting.
} catch (InterruptedException e) {
e.printStackTrace();
}
finished = true;
} else {
synchronized (workPackage) {
- while (!workPackage.inputQueue.isEmpty()) {
+ while (!workPackage.inputQueue.isEmpty()) {
ArrayList<Element> elements = null;
- try {
- elements = workPackage.inputQueue.poll();
+ try {
+ elements = workPackage.inputQueue.poll();
for (Element element : elements) {
- processElement(element, workPackage.writer);
- }
-
- } catch (IOException e) {
+ workPackage.writer.write(element);
+ }
+ } catch (IOException e) {
throw new SplitFailedException("Thread "
+ Thread.currentThread().getName()
+ " failed to write element ", e);
+ }
}
}
- }
}
}
diff --git a/src/uk/me/parabola/splitter/Utils.java b/src/uk/me/parabola/splitter/Utils.java
index a117b19..5a0cb7b 100644
--- a/src/uk/me/parabola/splitter/Utils.java
+++ b/src/uk/me/parabola/splitter/Utils.java
@@ -184,7 +184,7 @@ public class Utils {
break;
default:
System.out.println("Unsupported path iterator type " + type
- + ". This is an mkgmap error.");
+ + ". This is an internal splitter error.");
}
pit.next();
@@ -238,7 +238,7 @@ public class Utils {
break;
default:
System.out.println("Unsupported path iterator type " + type
- + ". This is an mkgmap error.");
+ + ". This is an internal splitter error.");
}
pit.next();
diff --git a/src/uk/me/parabola/splitter/args/SplitterParams.java b/src/uk/me/parabola/splitter/args/SplitterParams.java
index 987ca31..0a8a3ca 100644
--- a/src/uk/me/parabola/splitter/args/SplitterParams.java
+++ b/src/uk/me/parabola/splitter/args/SplitterParams.java
@@ -28,7 +28,8 @@ public interface SplitterParams {
@Option(description = "A default description to give to each area.")
String getDescription();
- @Option(defaultValue = "512", description = "The maximum number of areas to process in a single pass. More areas require more memory, but less time. Values: 1-4096.")
+ @Option(defaultValue = "2048", description = "The maximum number of areas to process in a single pass. " +
+ "More areas require more memory, but less time. Values: 1-9999.")
int getMaxAreas();
@Option(defaultValue = "auto", description = "Deprecated. Nodes/ways/rels that fall outside an area will still "
@@ -44,7 +45,8 @@ public interface SplitterParams {
+ "the given number of tiles is produced. The max-nodes value is ignored if this option is given.")
String getNumTiles();
- @Option(defaultValue = "13", description = "The resolution of the overview map to be produced by mkgmap.")
+ @Option(defaultValue = "13", description = "The resolution determines how the tiles must be aligned." +
+ "Eg a resolution of 13 means the tiles need to have their edges aligned to multiples of 2 ^ (24 - 13) = 2048 map units.")
int getResolution();
@Option(description = "Specify this if the input osm file has nodes, ways and relations intermingled.")
@@ -108,7 +110,7 @@ public interface SplitterParams {
@Option(defaultValue="5", description = "The lowest admin_level value that should be kept complete. Reasonable values are 2 .. 11."
+ "Used to filter boundary relations for problem-list processing. Ignored when keep-complete is false.")
- String getWantedAdminLevel();
+ int getWantedAdminLevel();
diff --git a/src/uk/me/parabola/splitter/KmlParser.java b/src/uk/me/parabola/splitter/kml/KmlParser.java
similarity index 94%
rename from src/uk/me/parabola/splitter/KmlParser.java
rename to src/uk/me/parabola/splitter/kml/KmlParser.java
index 7f5761d..a4bedf0 100644
--- a/src/uk/me/parabola/splitter/KmlParser.java
+++ b/src/uk/me/parabola/splitter/kml/KmlParser.java
@@ -11,13 +11,17 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.kml;
import java.util.ArrayList;
import java.util.List;
import org.xmlpull.v1.XmlPullParserException;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.xml.parser.AbstractXppParser;
+
/**
* Parses a KML area file.
*
@@ -30,7 +34,7 @@ public class KmlParser extends AbstractXppParser {
private State state = State.None;
private int currentId;
private int[] currentCoords = new int[10];
- private List<Area> areas = new ArrayList<Area>();
+ private List<Area> areas = new ArrayList<>();
public KmlParser() throws XmlPullParserException {
}
diff --git a/src/uk/me/parabola/splitter/KmlWriter.java b/src/uk/me/parabola/splitter/kml/KmlWriter.java
similarity index 96%
rename from src/uk/me/parabola/splitter/KmlWriter.java
rename to src/uk/me/parabola/splitter/kml/KmlWriter.java
index 456aa94..6549bf2 100644
--- a/src/uk/me/parabola/splitter/KmlWriter.java
+++ b/src/uk/me/parabola/splitter/kml/KmlWriter.java
@@ -10,13 +10,16 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.kml;
import java.awt.geom.PathIterator;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;
import java.util.Locale;
+
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Utils;
/**
* A class to create kml files from java areas (polygons) or rectangular areas.
* @author GerdP
@@ -114,7 +117,7 @@ public class KmlWriter {
default:
// should not happen
System.err.println("Unsupported path iterator type " + type
- + ". This is an mkgmap error.");
+ + ". This is an internal splitter error.");
throw new IOException();
}
pit.next();
diff --git a/src/uk/me/parabola/splitter/BinaryMapParser.java b/src/uk/me/parabola/splitter/parser/BinaryMapParser.java
similarity index 67%
rename from src/uk/me/parabola/splitter/BinaryMapParser.java
rename to src/uk/me/parabola/splitter/parser/BinaryMapParser.java
index 9b65461..9b780d0 100644
--- a/src/uk/me/parabola/splitter/BinaryMapParser.java
+++ b/src/uk/me/parabola/splitter/parser/BinaryMapParser.java
@@ -11,28 +11,34 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.parser;
import crosby.binary.BinaryParser;
import crosby.binary.Osmformat;
import crosby.binary.file.FileBlockPosition;
-
import it.unimi.dsi.fastutil.shorts.ShortArrayList;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.MapProcessor;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.Relation;
+import uk.me.parabola.splitter.UnknownFeatureException;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.Way;
import java.util.List;
-public class BinaryMapParser extends BinaryParser implements MapReader {
- private static final short TYPE_DENSE = 0x1;
- private static final short TYPE_NODES = 0x2;
- private static final short TYPE_WAYS = 0x4;
- private static final short TYPE_RELS = 0x8;
+public class BinaryMapParser extends BinaryParser {
+ private static final short TYPE_DENSE = 0x1;
+ private static final short TYPE_NODES = 0x2;
+ private static final short TYPE_WAYS = 0x4;
+ private static final short TYPE_RELS = 0x8;
private final ShortArrayList blockTypes = new ShortArrayList();
private final ShortArrayList knownBlockTypes;
// for status messages
private final ElementCounter elemCounter = new ElementCounter();
-
+
private short blockType = -1;
private int blockCount = -1;
private boolean skipTags;
@@ -40,9 +46,9 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
private boolean skipWays;
private boolean skipRels;
private short wantedTypeMask = 0;
- private int msgLevel;
-
- BinaryMapParser(MapProcessor processor, ShortArrayList knownBlockTypes, int msgLevel) {
+ private int msgLevel;
+
+ public BinaryMapParser(MapProcessor processor, ShortArrayList knownBlockTypes, int msgLevel) {
this.processor = processor;
this.knownBlockTypes = knownBlockTypes;
this.skipTags = processor.skipTags();
@@ -50,8 +56,8 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
this.skipWays = processor.skipWays();
this.skipRels = processor.skipRels();
this.msgLevel = msgLevel;
-
- if (skipNodes == false){
+
+ if (skipNodes == false) {
wantedTypeMask |= TYPE_DENSE;
wantedTypeMask |= TYPE_NODES;
}
@@ -60,34 +66,34 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
if (skipRels == false)
wantedTypeMask |= TYPE_RELS;
}
+
MapProcessor processor;
- public ShortArrayList getBlockList(){
+ public ShortArrayList getBlockList() {
return blockTypes;
}
-
+
@Override
- public boolean skipBlock(FileBlockPosition block) {
+ public boolean skipBlock(FileBlockPosition block) {
blockCount++;
- if (knownBlockTypes != null){
+ if (knownBlockTypes != null) {
blockType = knownBlockTypes.getShort(blockCount);
if (blockType != 0 && (blockType & wantedTypeMask) == 0)
return true;
- }
- else if (blockType != -1){
- //System.out.println("previous block contained " + blockType );
+ } else if (blockType != -1) {
+ // System.out.println("previous block contained " + blockType );
blockTypes.add(blockType);
}
blockType = 0;
- // System.out.println("Seeing block of type: "+block.getType());
- if (block.getType().equals("OSMData"))
- return false;
- if (block.getType().equals("OSMHeader"))
- return false;
- System.out.println("Skipped block of type: " + block.getType());
- return true;
- }
-
+ // System.out.println("Seeing block of type: "+block.getType());
+ if (block.getType().equals("OSMData"))
+ return false;
+ if (block.getType().equals("OSMHeader"))
+ return false;
+ System.out.println("Skipped block of type: " + block.getType());
+ return true;
+ }
+
@Override
public void complete() {
blockTypes.add(blockType);
@@ -105,10 +111,13 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
int j = 0;
int maxi = nodes.getIdCount();
Node tmp = new Node();
- for (int i=0 ; i < maxi; i++) {
- long lat = nodes.getLat(i)+last_lat; last_lat = lat;
- long lon = nodes.getLon(i)+last_lon; last_lon = lon;
- long id = nodes.getId(i)+last_id; last_id = id;
+ for (int i = 0; i < maxi; i++) {
+ long lat = nodes.getLat(i) + last_lat;
+ last_lat = lat;
+ long lon = nodes.getLon(i) + last_lon;
+ last_lon = lon;
+ long id = nodes.getId(i) + last_id;
+ last_id = id;
double latf = parseLat(lat), lonf = parseLon(lon);
tmp = new Node();
@@ -121,7 +130,7 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
while (nodes.getKeysVals(j) != 0) {
int keyid = nodes.getKeysVals(j++);
int valid = nodes.getKeysVals(j++);
- tmp.addTag(getStringById(keyid),getStringById(valid));
+ tmp.addTag(getStringById(keyid), getStringById(valid));
}
j++; // Skip over the '0' delimiter.
@@ -134,15 +143,15 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
@Override
protected void parseNodes(List<Osmformat.Node> nodes) {
- if (nodes.size() == 0)
+ if (nodes.size() == 0)
return;
blockType |= TYPE_NODES;
- if (skipNodes)
+ if (skipNodes)
return;
for (Osmformat.Node i : nodes) {
Node tmp = new Node();
- for (int j=0 ; j < i.getKeysCount(); j++)
- tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j)));
+ for (int j = 0; j < i.getKeysCount(); j++)
+ tmp.addTag(getStringById(i.getKeys(j)), getStringById(i.getVals(j)));
long id = i.getId();
double latf = parseLat(i.getLat()), lonf = parseLon(i.getLon());
@@ -155,25 +164,24 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
}
}
-
@Override
protected void parseWays(List<Osmformat.Way> ways) {
long numways = ways.size();
- if (numways == 0)
+ if (numways == 0)
return;
blockType |= TYPE_WAYS;
- if (skipWays)
+ if (skipWays)
return;
for (Osmformat.Way i : ways) {
Way tmp = new Way();
- if (skipTags == false){
- for (int j=0 ; j < i.getKeysCount(); j++)
- tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j)));
+ if (skipTags == false) {
+ for (int j = 0; j < i.getKeysCount(); j++)
+ tmp.addTag(getStringById(i.getKeys(j)), getStringById(i.getVals(j)));
}
- long last_id=0;
+ long last_id = 0;
for (long j : i.getRefsList()) {
- tmp.addRef(j+last_id);
- last_id = j+last_id;
+ tmp.addRef(j + last_id);
+ last_id = j + last_id;
}
long id = i.getId();
@@ -186,31 +194,29 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
}
}
-
-
@Override
protected void parseRelations(List<Osmformat.Relation> rels) {
- if (rels.size() == 0)
+ if (rels.size() == 0)
return;
blockType |= TYPE_RELS;
if (skipRels)
return;
for (Osmformat.Relation i : rels) {
Relation tmp = new Relation();
- if (skipTags == false){
- for (int j=0 ; j < i.getKeysCount(); j++)
- tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j)));
+ if (skipTags == false) {
+ for (int j = 0; j < i.getKeysCount(); j++)
+ tmp.addTag(getStringById(i.getKeys(j)), getStringById(i.getVals(j)));
}
long id = i.getId();
tmp.setId(id);
tmp.setVersion(i.getInfo().getVersion());
- long last_mid=0;
- for (int j =0; j < i.getMemidsCount() ; j++) {
+ long last_mid = 0;
+ for (int j = 0; j < i.getMemidsCount(); j++) {
long mid = last_mid + i.getMemids(j);
last_mid = mid;
String role = getStringById(i.getRolesSid(j));
- String etype=null;
+ String etype = null;
if (i.getTypes(j) == Osmformat.Relation.MemberType.NODE)
etype = "node";
@@ -221,7 +227,7 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
else
assert false; // TODO; Illegal file?
- tmp.addMember(etype,mid,role);
+ tmp.addMember(etype, mid, role);
}
processor.processRelation(tmp);
elemCounter.countRelation(tmp.getId());
@@ -232,8 +238,10 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
public void parse(Osmformat.HeaderBlock block) {
for (String s : block.getRequiredFeaturesList()) {
- if (s.equals("OsmSchema-V0.6")) continue; // OK.
- if (s.equals("DenseNodes")) continue; // OK.
+ if (s.equals("OsmSchema-V0.6"))
+ continue; // OK.
+ if (s.equals("DenseNodes"))
+ continue; // OK.
throw new UnknownFeatureException(s);
}
@@ -245,12 +253,9 @@ public class BinaryMapParser extends BinaryParser implements MapReader {
double bottomf = block.getBbox().getBottom() * multiplier;
if (msgLevel > 0)
- System.out.println("Bounding box "+leftf+" "+bottomf+" "+rightf+" "+topf);
+ System.out.println("Bounding box " + leftf + " " + bottomf + " " + rightf + " " + topf);
- Area area = new Area(
- Utils.toMapUnit(bottomf),
- Utils.toMapUnit(leftf),
- Utils.toMapUnit(topf),
+ Area area = new Area(Utils.toMapUnit(bottomf), Utils.toMapUnit(leftf), Utils.toMapUnit(topf),
Utils.toMapUnit(rightf));
if (!area.verify())
throw new IllegalArgumentException("invalid bbox area in pbf file: " + area);
diff --git a/src/uk/me/parabola/splitter/ElementCounter.java b/src/uk/me/parabola/splitter/parser/ElementCounter.java
similarity index 84%
rename from src/uk/me/parabola/splitter/ElementCounter.java
rename to src/uk/me/parabola/splitter/parser/ElementCounter.java
index 9ccbf33..ac31ddd 100644
--- a/src/uk/me/parabola/splitter/ElementCounter.java
+++ b/src/uk/me/parabola/splitter/parser/ElementCounter.java
@@ -11,10 +11,12 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.parser;
+
+import uk.me.parabola.splitter.Utils;
/**
- * Common OSM reader method for status messages
+ * Common OSM parseder method for status messages
* @author GerdP
*
*/
@@ -36,9 +38,8 @@ public class ElementCounter {
protected void countNode(long id) {
nodeCount++;
if (nodeCount % NODE_STATUS_UPDATE_THRESHOLD == 0) {
- System.out.println(Utils.format(nodeCount) + " nodes processed... id=" + id);
+ System.out.println(Utils.format(nodeCount) + " nodes parsed... id=" + id);
}
-
}
/**
@@ -48,7 +49,7 @@ public class ElementCounter {
protected void countWay(long id) {
wayCount++;
if (wayCount % WAY_STATUS_UPDATE_THRESHOLD == 0) {
- System.out.println(Utils.format(wayCount) + " ways processed... id=" + id);
+ System.out.println(Utils.format(wayCount) + " ways parsed... id=" + id);
}
}
@@ -59,7 +60,7 @@ public class ElementCounter {
protected void countRelation(long id) {
relationCount++;
if (relationCount % RELATION_STATUS_UPDATE_THRESHOLD == 0) {
- System.out.println(Utils.format(relationCount) + " relations processed... id=" + id);
+ System.out.println(Utils.format(relationCount) + " relations parsed... id=" + id);
}
}
diff --git a/src/uk/me/parabola/splitter/O5mMapParser.java b/src/uk/me/parabola/splitter/parser/O5mMapParser.java
similarity index 55%
rename from src/uk/me/parabola/splitter/O5mMapParser.java
rename to src/uk/me/parabola/splitter/parser/O5mMapParser.java
index 6dedf04..ed8ded2 100644
--- a/src/uk/me/parabola/splitter/O5mMapParser.java
+++ b/src/uk/me/parabola/splitter/parser/O5mMapParser.java
@@ -9,15 +9,23 @@
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
- */
-package uk.me.parabola.splitter;
+ */
+
+package uk.me.parabola.splitter.parser;
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
-import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.util.Arrays;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Element;
+import uk.me.parabola.splitter.MapProcessor;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.Relation;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.Way;
+
/**
* Parser for the o5m format described here: http://wiki.openstreetmap.org/wiki/O5m
* The routines to are based on the osmconvert.c source from Markus Weber who allows
@@ -25,7 +33,7 @@ import java.util.Arrays;
* @author GerdP
*
*/
-public class O5mMapParser implements MapReader{
+public class O5mMapParser {
// O5M data set constants
private static final int NODE_DATASET = 0x10;
private static final int WAY_DATASET = 0x11;
@@ -36,155 +44,143 @@ public class O5mMapParser implements MapReader{
private static final int EOD_FLAG = 0xfe;
private static final int RESET_FLAG = 0xff;
- private static final int EOF_FLAG = -1;
-
// o5m constants
private static final int STRING_TABLE_SIZE = 15000;
private static final int MAX_STRING_PAIR_SIZE = 250 + 2;
private static final String[] REL_REF_TYPES = {"node", "way", "relation", "?"};
- private static final double FACTOR = 1d/1000000000; // used with 100*<Val>*FACTOR
+ private static final double FACTOR = 1d / 1000000000; // used with 100*<Val>*FACTOR
// for status messages
private final ElementCounter elemCounter = new ElementCounter();
// flags set by the processor to signal what information is not needed
- private boolean skipTags;
- private boolean skipNodes;
- private boolean skipWays;
- private boolean skipRels;
+ private final boolean skipTags;
+ private final boolean skipNodes;
+ private final boolean skipWays;
+ private final boolean skipRels;
+
+ private final FileChannel fileChannel;
+ // Buffer size, must be a power of 2
+ private static final int BUF_SIZE = 0x1000;
+
+ private final ByteBuffer fileBuffer = ByteBuffer.allocate(BUF_SIZE);
+ private long filePos;
+ private long bufStart;
+ private int bufSize = -1;
+
+ private long nextFilePos;
- private final BufferedInputStream fis;
- private InputStream is;
- private ByteArrayInputStream bis;
- private MapProcessor processor;
+
+ private final MapProcessor processor;
// buffer for byte -> String conversions
- private byte[] cnvBuffer;
+ private final byte[] cnvBuffer;
- private byte[] ioBuf;
- private int ioPos;
// the o5m string table
private String[][] stringTable;
- private String[] stringPair;
+ private final String[] stringPair;
private int currStringTablePos;
- // a counter that must be maintained by all routines that read data from the stream
- private int bytesToRead;
- // total number of bytes read from stream
- long countBytes;
+ // a counter that must be maintained by all routines that read data
// performance: save byte position of first occurrence of a data set type (node, way, relation)
// to allow skipping large parts of the stream
- long[] firstPosInFile;
- long[] skipArray;
+ private long[] firstPosInFile;
+ private long[] skipArray;
// for delta calculations
private long lastNodeId;
private long lastWayId;
private long lastRelId;
- private long lastRef[];
+ private long[] lastRef;
private long lastTs;
private long lastChangeSet;
- private int lastLon,lastLat;
+ private int lastLon, lastLat;
/**
- * A parser for the o5m format
+ * A parser for the o5m format.
* @param processor A mapProcessor instance
- * @param stream The InputStream that contains the OSM data in o5m format
+ * @param fc the file channel for the input file
* @param skipArray An Array of longs that is used to hold information of file position of the first occurrence of
* each known 05m data type (esp. nodes, ways, and relations).
+ * @throws IOException
*/
- O5mMapParser(MapProcessor processor, InputStream stream, long[] skipArray) {
+ public O5mMapParser(MapProcessor processor, FileChannel fc, long[] skipArray) throws IOException{
+ this.fileChannel = fc;
this.processor = processor;
- this.fis = new BufferedInputStream(stream, 4*1024*1024);
- is = fis;
this.skipArray = skipArray;
this.skipTags = processor.skipTags();
this.skipNodes = processor.skipNodes();
this.skipWays = processor.skipWays();
this.skipRels = processor.skipRels();
this.cnvBuffer = new byte[4000]; // OSM data should not contain string pairs with length > 512
- this.ioBuf = new byte[8192];
- this.ioPos = 0;
this.stringPair = new String[2];
this.lastRef = new long[3];
- if (skipArray == null){
+ if (skipArray == null) {
firstPosInFile = new long[256];
Arrays.fill(firstPosInFile, -1);
}
reset();
}
+
/**
- * parse the input stream
+ * parse the input stream.
* @throws IOException
*/
- public void parse() throws IOException{
- int start = is.read();
- ++countBytes;
+ public void parse() throws IOException {
+ int start = get() & 0xff;
if (start != RESET_FLAG)
throw new IOException("wrong header byte " + start);
- if (skipArray != null){
- if (skipNodes ){
+ if (skipArray != null) {
+ if (skipNodes) {
if (skipWays)
- skip(skipArray[REL_DATASET]-countBytes); // jump to first relation
+ filePos = skipArray[REL_DATASET]; // jump to first relation
else
- skip(skipArray[WAY_DATASET]-countBytes); // jump to first way
+ filePos = skipArray[WAY_DATASET]; // jump to first way
}
}
readFile();
}
- private void readFile() throws IOException{
+ /**
+ * Read the file following the initial byte.
+ * @throws IOException
+ */
+ private void readFile() throws IOException {
boolean done = false;
- while(!done){
- is = fis;
+ while (!done) {
long size = 0;
- int fileType = is.read();
- ++countBytes;
- if (fileType >= 0 && fileType < 0xf0){
- if (skipArray == null){
+ int fileType = get() & 0xff;
+ if (fileType >= 0 && fileType < 0xf0) {
+ if (skipArray == null) {
// save first occurrence of a data set type
- if (firstPosInFile[fileType] == -1){
- firstPosInFile[fileType] = Math.max(0, countBytes-1);
+ if (firstPosInFile[fileType] == -1) {
+ firstPosInFile[fileType] = Math.max(0, filePos- 1);
}
}
- bytesToRead = 0;
- size = readUnsignedNum64FromStream();
- countBytes += size - bytesToRead; // bytesToRead is negative
- bytesToRead = (int)size;
+ size = readUnsignedNum64();
+ nextFilePos = filePos + size;
+
boolean doSkip = false;
if (fileType == NODE_DATASET && skipNodes) doSkip = true;
else if (fileType == WAY_DATASET && skipWays) doSkip = true;
else if (fileType == REL_DATASET && skipRels) doSkip = true;
- switch(fileType){
+ switch(fileType) {
case NODE_DATASET:
case WAY_DATASET:
case REL_DATASET:
case BBOX_DATASET:
case TIMESTAMP_DATASET:
case HEADER_DATASET:
- if (doSkip){
- skip(bytesToRead);
+ if (doSkip) {
+ filePos = nextFilePos;
continue;
}
- if (bytesToRead > ioBuf.length){
- ioBuf = new byte[bytesToRead+100];
- }
- int bytesRead = 0;
- int neededBytes = bytesToRead;
- while (neededBytes > 0){
- bytesRead += is.read(ioBuf, bytesRead, neededBytes);
- neededBytes -= bytesRead;
- }
- ioPos = 0;
- bis = new ByteArrayInputStream(ioBuf,0,bytesToRead);
- is = bis;
break;
default:
}
}
- if (fileType == EOF_FLAG) done = true;
- else if (fileType == NODE_DATASET) readNode();
+ if (fileType == NODE_DATASET) readNode();
else if (fileType == WAY_DATASET) readWay();
else if (fileType == REL_DATASET) readRel();
else if (fileType == BBOX_DATASET) readBBox();
@@ -193,40 +189,31 @@ public class O5mMapParser implements MapReader{
else if (fileType == EOD_FLAG) done = true;
else if (fileType == RESET_FLAG) reset();
else {
- if (fileType < 0xf0 )skip(size); // skip unknown data set
+ if (fileType < 0xf0)
+ filePos = nextFilePos; // skip unknown data set
}
}
}
/**
- * read (and ignore) the file timestamp data set
+ * read (and ignore) the file timestamp data set.
+ * @throws IOException
*/
- private void readFileTimestamp(){
+ private void readFileTimestamp() throws IOException {
/*long fileTimeStamp = */readSignedNum64();
}
/**
- * Skip the given number of bytes
- * @param bytes
- * @throws IOException
- */
- private void skip(long bytes)throws IOException{
- long toSkip = bytes;
- while (toSkip > 0)
- toSkip -= is.skip(toSkip);
- }
-
- /**
- * read the bounding box data set
+ * read the bounding box data set.
* @throws IOException
*/
- private void readBBox() {
- double leftf = 100L*readSignedNum32() * FACTOR;
- double bottomf = 100L*readSignedNum32() * FACTOR;
- double rightf = 100L*readSignedNum32() * FACTOR;
- double topf = 100L*readSignedNum32() * FACTOR;
- assert bytesToRead == 0;
- System.out.println("Bounding box "+leftf+" "+bottomf+" "+rightf+" "+topf);
+ private void readBBox() throws IOException {
+ double leftf = 100L * readSignedNum32() * FACTOR;
+ double bottomf = 100L * readSignedNum32() * FACTOR;
+ double rightf = 100L * readSignedNum32() * FACTOR;
+ double topf = 100L * readSignedNum32() * FACTOR;
+ assert filePos == nextFilePos;
+ System.out.println("Bounding box " + leftf + " " + bottomf + " " + rightf + " " + topf);
Area area = new Area(
Utils.toMapUnit(bottomf),
@@ -240,25 +227,26 @@ public class O5mMapParser implements MapReader{
}
/**
- * read a node data set
+ * read a node data set.
* @throws IOException
*/
private void readNode() throws IOException{
Node node = new Node();
+
lastNodeId += readSignedNum64();
- if (bytesToRead == 0)
+ if (filePos == nextFilePos)
return; // only nodeId: this is a delete action, we ignore it
int version = readVersionTsAuthor();
node.setVersion(version);
- if (bytesToRead == 0)
+ if (filePos == nextFilePos)
return; // only nodeId+version: this is a delete action, we ignore it
int lon = readSignedNum32() + lastLon; lastLon = lon;
int lat = readSignedNum32() + lastLat; lastLat = lat;
- double flon = 100L*lon * FACTOR;
- double flat = 100L*lat * FACTOR;
- assert flat >= -90.0 && flat <= 90.0;
- assert flon >= -180.0 && flon <= 180.0;
+ double flon = 100L * lon * FACTOR;
+ double flat = 100L * lat * FACTOR;
+ assert flat >= -90.0 && flat <= 90.0;
+ assert flon >= -180.0 && flon <= 180.0;
node.set(lastNodeId, flat, flon);
readTags(node);
@@ -267,24 +255,24 @@ public class O5mMapParser implements MapReader{
}
/**
- * read a way data set
+ * read a way data set.
* @throws IOException
*/
private void readWay() throws IOException{
lastWayId += readSignedNum64();
- if (bytesToRead == 0)
+ if (filePos == nextFilePos)
return; // only wayId: this is a delete action, we ignore it
int version = readVersionTsAuthor();
- if (bytesToRead == 0)
+ if (filePos == nextFilePos)
return; // only wayId + version: this is a delete action, we ignore it
Way way = new Way();
way.setId(lastWayId);
way.setVersion(version);
long refSize = readUnsignedNum32();
- long stop = bytesToRead - refSize;
+ long stop = filePos + refSize;
- while(bytesToRead > stop){
+ while (filePos < stop) {
lastRef[0] += readSignedNum64();
way.addRef(lastRef[0]);
}
@@ -296,23 +284,23 @@ public class O5mMapParser implements MapReader{
}
/**
- * read a relation data set
+ * read a relation data set.
* @throws IOException
*/
private void readRel() throws IOException{
lastRelId += readSignedNum64();
- if (bytesToRead == 0)
+ if (filePos == nextFilePos)
return; // only relId: this is a delete action, we ignore it
int version = readVersionTsAuthor();
- if (bytesToRead == 0)
+ if (filePos == nextFilePos)
return; // only relId + version: this is a delete action, we ignore it
Relation rel = new Relation();
rel.setId(lastRelId);
rel.setVersion(version);
long refSize = readUnsignedNum32();
- long stop = bytesToRead - refSize;
- while(bytesToRead > stop){
+ long stop = filePos + refSize;
+ while (filePos < stop) {
long deltaRef = readSignedNum64();
int refType = readRelRef();
lastRef[refType] += deltaRef;
@@ -326,19 +314,24 @@ public class O5mMapParser implements MapReader{
}
private void readTags(Element elem) throws IOException{
- while (bytesToRead > 0){
+ // we cannot skip the tags if we read relations (roles)
+ if (skipTags && skipRels) {
+ filePos = nextFilePos;
+ return;
+ }
+ while (filePos < nextFilePos) {
readStringPair();
- if (skipTags == false){
- elem.addTag(stringPair[0],stringPair[1]);
+ if (!skipTags) {
+ elem.addTag(stringPair[0], stringPair[1]);
}
}
- assert bytesToRead == 0;
+ assert filePos == nextFilePos;
}
/**
- * Store a new string pair (length check must be performed by caller)
+ * Store a new string pair (length check must be performed by caller).
*/
- private void storeStringPair(){
+ private void storeStringPair() {
stringTable[0][currStringTablePos] = stringPair[0];
stringTable[1][currStringTablePos] = stringPair[1];
++currStringTablePos;
@@ -364,16 +357,15 @@ public class O5mMapParser implements MapReader{
/**
* Read version, time stamp and change set and author.
- * We are not interested in the values, but we have to maintain the string table.
+ * @return the version
* @throws IOException
*/
-
private int readVersionTsAuthor() throws IOException {
int version = readUnsignedNum32();
- if (version != 0){
+ if (version != 0) {
// version info
long ts = readSignedNum64() + lastTs; lastTs = ts;
- if (ts != 0){
+ if (ts != 0) {
long changeSet = readSignedNum32() + lastChangeSet; lastChangeSet = changeSet;
readAuthor();
}
@@ -386,72 +378,46 @@ public class O5mMapParser implements MapReader{
*/
private void readAuthor() throws IOException{
int stringRef = readUnsignedNum32();
- if (stringRef == 0){
- long toReadStart = bytesToRead;
+ if (stringRef == 0) {
+ long toReadStart = filePos;
long uidNum = readUnsignedNum64();
if (uidNum == 0)
stringPair[0] = "";
- else{
+ else {
stringPair[0] = Long.toString(uidNum);
- ioPos++; // skip terminating zero from uid
- --bytesToRead;
- }
- int start = 0;
- int buffPos = 0;
- stringPair[1] = null;
- while(stringPair[1] == null){
- final int b = ioBuf[ioPos++];
- --bytesToRead;
- cnvBuffer[buffPos++] = (byte) b;
-
- if (b == 0)
- stringPair[1] = new String(cnvBuffer, start, buffPos-1, "UTF-8");
+ get(); // skip terminating zero from uid
}
- long bytes = toReadStart - bytesToRead;
- if (bytes <= MAX_STRING_PAIR_SIZE)
+ stringPair[1] = readString();
+ if (filePos - toReadStart <= MAX_STRING_PAIR_SIZE)
storeStringPair();
- }
- else
+ } else {
setStringRefPair(stringRef);
+ }
//System.out.println(pair[0]+ "/" + pair[1]);
}
/**
- * read object type ("0".."2") concatenated with role (single string)
+ * read object type ("0".."2") concatenated with role (single string).
* @return 0..3 for type (3 means unknown)
*/
- private int readRelRef () throws IOException{
+ private int readRelRef() throws IOException {
int refType = -1;
- long toReadStart = bytesToRead;
+ long toReadStart = filePos;
int stringRef = readUnsignedNum32();
- if (stringRef == 0){
- refType = ioBuf[ioPos++] - 0x30;
- --bytesToRead;
+ if (stringRef == 0) {
+ refType = get() - '0';
if (refType < 0 || refType > 2)
refType = 3;
stringPair[0] = REL_REF_TYPES[refType];
-
- int start = 0;
- int buffPos = 0;
- stringPair[1] = null;
- while(stringPair[1] == null){
- final int b = ioBuf[ioPos++];
- --bytesToRead;
- cnvBuffer[buffPos++] = (byte)b;
-
- if (b == 0)
- stringPair[1] = new String(cnvBuffer, start, buffPos-1, "UTF-8");
- }
- long bytes = toReadStart - bytesToRead;
- if (bytes <= MAX_STRING_PAIR_SIZE)
+ stringPair[1] = readString();
+ if (filePos - toReadStart <= MAX_STRING_PAIR_SIZE)
storeStringPair();
- }
- else {
+ } else {
setStringRefPair(stringRef);
char c = stringPair[0].charAt(0);
- switch (c){
+ switch (c) {
case 'n': refType = 0; break;
case 'w': refType = 1; break;
case 'r': refType = 2; break;
@@ -462,162 +428,141 @@ public class O5mMapParser implements MapReader{
}
/**
- * read a string pair (see o5m definition)
+ * read a string pair (see o5m definition).
* @throws IOException
*/
private void readStringPair() throws IOException{
int stringRef = readUnsignedNum32();
- if (stringRef == 0){
- long toReadStart = bytesToRead;
+ if (stringRef == 0) {
+ long toReadStart = filePos;
int cnt = 0;
- int buffPos = 0;
- int start = 0;
- while (cnt < 2){
- final int b = ioBuf[ioPos++];
- --bytesToRead;
- cnvBuffer[buffPos++] = (byte)b;
-
- if (b == 0){
- stringPair[cnt] = new String(cnvBuffer, start, buffPos-start-1, "UTF-8");
- ++cnt;
- start = buffPos;
- }
+ while (cnt < 2) {
+ stringPair[cnt++] = readString();
}
- long bytes = toReadStart - bytesToRead;
- if (bytes <= MAX_STRING_PAIR_SIZE)
+ if (filePos - toReadStart <= MAX_STRING_PAIR_SIZE)
storeStringPair();
- }
- else
+ } else {
setStringRefPair(stringRef);
+ }
}
- /** reset the delta values and string table */
- private void reset(){
- lastNodeId = 0; lastWayId = 0; lastRelId = 0;
- lastRef[0] = 0; lastRef[1] = 0;lastRef[2] = 0;
- lastTs = 0; lastChangeSet = 0;
- lastLon = 0; lastLat = 0;
+ /**
+ * Read a zero-terminated string (see o5m definition).
+ * @throws IOException
+ */
+ String readString() throws IOException {
+ int length = 0;
+ while (true) {
+ final int b = get();
+ if (b == 0)
+ return new String(cnvBuffer, 0, length, "UTF-8");
+ cnvBuffer[length++] = (byte) b;
+ }
+
+ }
+ /** reset the delta values and string table. */
+ private void reset() {
+ lastNodeId = 0;
+ lastWayId = 0;
+ lastRelId = 0;
+ lastRef[0] = 0;
+ lastRef[1] = 0;
+ lastRef[2] = 0;
+ lastTs = 0;
+ lastChangeSet = 0;
+ lastLon = 0;
+ lastLat = 0;
stringTable = new String[2][STRING_TABLE_SIZE];
currStringTablePos = 0;
}
/**
- * read and verify o5m header (known values are o5m2 and o5c2)
+ * read and verify o5m header (known values are o5m2 and o5c2).
* @throws IOException
*/
private void readHeader() throws IOException {
- if (ioBuf[0] != 'o' || ioBuf[1] != '5' || (ioBuf[2]!='c'&&ioBuf[2]!='m') ||ioBuf[3] != '2' ){
+ byte[] header = new byte[4];
+ for (int i = 0; i < header.length; i++) {
+ header[i] = get();
+ }
+ if (header[0] != 'o' || header[1] != '5' || (header[2] != 'c' && header[2] != 'm') || header[3] != '2') {
throw new IOException("unsupported header");
}
}
-
+
/**
- * read a varying length signed number (see o5m definition)
+ * read a varying length signed number (see o5m definition).
* @return the number
* @throws IOException
*/
- private int readSignedNum32() {
+ private int readSignedNum32() throws IOException {
int result;
- int b = ioBuf[ioPos++];
- --bytesToRead;
+ int b = get();
result = b;
- if ((b & 0x80) == 0){ // just one byte
+ if ((b & 0x80) == 0) { // just one byte
if ((b & 0x01) == 1)
- return -1-(result>>1);
- return result>>1;
+ return -1 - (result >> 1);
+ return result >> 1;
}
int sign = b & 0x01;
- result = (result & 0x7e)>>1;
+ result = (result & 0x7e) >> 1;
int fac = 0x40;
- while (((b = ioBuf[ioPos++]) & 0x80) != 0){ // more bytes will follow
- --bytesToRead;
- result += fac * (b & 0x7f) ;
- fac <<= 7;
+ while (((b = get()) & 0x80) != 0) { // more bytes will follow
+ result += fac * (b & 0x7f);
+ fac <<= 7;
}
- --bytesToRead;
result += fac * b;
if (sign == 1) // negative
- return -1-result;
+ return -1 - result;
return result;
}
/**
- * read a varying length signed number (see o5m definition)
+ * read a varying length signed number (see o5m definition).
* @return the number
* @throws IOException
*/
- private long readSignedNum64() {
+ private long readSignedNum64() throws IOException {
long result;
- int b = ioBuf[ioPos++];
- --bytesToRead;
+ int b = get();
result = b;
- if ((b & 0x80) == 0){ // just one byte
+ if ((b & 0x80) == 0) { // just one byte
if ((b & 0x01) == 1)
- return -1-(result>>1);
- return result>>1;
+ return -1 - (result >> 1);
+ return result >> 1;
}
int sign = b & 0x01;
- result = (result & 0x7e)>>1;
+ result = (result & 0x7e) >> 1;
long fac = 0x40;
- while (((b = ioBuf[ioPos++]) & 0x80) != 0){ // more bytes will follow
- --bytesToRead;
- result += fac * (b & 0x7f) ;
- fac <<= 7;
+ while (((b = get()) & 0x80) != 0) { // more bytes will follow
+ result += fac * (b & 0x7f);
+ fac <<= 7;
}
- --bytesToRead;
result += fac * b;
if (sign == 1) // negative
- return -1-result;
+ return -1 - result;
return result;
}
-
- /**
- * read a varying length unsigned number (see o5m definition)
- * @return a long
- * @throws IOException
- */
- private long readUnsignedNum64FromStream()throws IOException {
- int b = is.read();
- --bytesToRead;
- long result = b;
- if ((b & 0x80) == 0){ // just one byte
- return result;
- }
- result &= 0x7f;
- long fac = 0x80;
- while (((b = is.read()) & 0x80) != 0){ // more bytes will follow
- --bytesToRead;
- result += fac * (b & 0x7f) ;
- fac <<= 7;
- }
- --bytesToRead;
- result += fac * b;
- return result;
- }
-
/**
- * read a varying length unsigned number (see o5m definition)
+ * read a varying length unsigned number (see o5m definition).
* @return a long
* @throws IOException
*/
- private long readUnsignedNum64(){
- int b = ioBuf[ioPos++];
- --bytesToRead;
+ private long readUnsignedNum64() throws IOException {
+ int b = get();
long result = b;
- if ((b & 0x80) == 0){ // just one byte
+ if ((b & 0x80) == 0) { // just one byte
return result;
}
result &= 0x7f;
long fac = 0x80;
- while (((b = ioBuf[ioPos++]) & 0x80) != 0){ // more bytes will follow
- --bytesToRead;
- result += fac * (b & 0x7f) ;
- fac <<= 7;
+ while (((b = get()) & 0x80) != 0) { // more bytes will follow
+ result += fac * (b & 0x7f);
+ fac <<= 7;
}
- --bytesToRead;
result += fac * b;
return result;
}
@@ -628,21 +573,18 @@ public class O5mMapParser implements MapReader{
* @return an int
* @throws IOException
*/
- private int readUnsignedNum32(){
- int b = ioBuf[ioPos++];
- --bytesToRead;
+ private int readUnsignedNum32() throws IOException {
+ int b = get();
int result = b;
- if ((b & 0x80) == 0){ // just one byte
+ if ((b & 0x80) == 0) { // just one byte
return result;
}
result &= 0x7f;
long fac = 0x80;
- while (((b = ioBuf[ioPos++]) & 0x80) != 0){ // more bytes will follow
- --bytesToRead;
- result += fac * (b & 0x7f) ;
- fac <<= 7;
+ while (((b = get()) & 0x80) != 0) { // more bytes will follow
+ result += fac * (b & 0x7f);
+ fac <<= 7;
}
- --bytesToRead;
result += fac * b;
return result;
}
@@ -651,4 +593,39 @@ public class O5mMapParser implements MapReader{
return firstPosInFile;
}
+ /**
+ * Read in a single byte from the current position.
+ *
+ * @return The byte that was read.
+ * @throws IOException if buffer contains no data
+ */
+ private byte get() throws IOException {
+ fillBuffer();
+
+ int pos = (int) (filePos - bufStart);
+ if (pos >= bufSize)
+ throw new IOException("no data in file buffer");
+ filePos++;
+ return fileBuffer.get(pos);
+
+
+ }
+
+ /**
+ * Check to see if the buffer contains the byte at the current position.
+ * If not then it is re-read so that it does.
+ * @throws IOException in case of I/O error
+ */
+ private void fillBuffer() throws IOException {
+ // If we are no longer inside the buffer, then re-read it.
+ if (filePos >= bufStart + bufSize) {
+
+ // Get channel position on a block boundary.
+ bufStart = filePos & ~(BUF_SIZE - 1);
+ fileChannel.position(bufStart);
+ // Fill buffer
+ fileBuffer.clear();
+ bufSize = fileChannel.read(fileBuffer);
+ }
+ }
}
diff --git a/src/uk/me/parabola/splitter/OSMParser.java b/src/uk/me/parabola/splitter/parser/OSMXMLParser.java
similarity index 93%
rename from src/uk/me/parabola/splitter/OSMParser.java
rename to src/uk/me/parabola/splitter/parser/OSMXMLParser.java
index 230b038..29baa1a 100644
--- a/src/uk/me/parabola/splitter/OSMParser.java
+++ b/src/uk/me/parabola/splitter/parser/OSMXMLParser.java
@@ -10,15 +10,24 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.parser;
import org.xmlpull.v1.XmlPullParserException;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Convert;
+import uk.me.parabola.splitter.MapProcessor;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.Relation;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.Way;
+import uk.me.parabola.splitter.xml.parser.AbstractXppParser;
+
/**
* Parses an OSM file, calling the appropriate methods on a
* {@code MapProcessor} as it progresses.
*/
-class OSMParser extends AbstractXppParser implements MapReader {
+public class OSMXMLParser extends AbstractXppParser {
private enum State {
@@ -42,7 +51,7 @@ class OSMParser extends AbstractXppParser implements MapReader {
private State state = State.None;
- OSMParser(MapProcessor processor, boolean mixed) throws XmlPullParserException {
+ public OSMXMLParser(MapProcessor processor, boolean mixed) throws XmlPullParserException {
this.processor = processor;
this.mixed = mixed;
diff --git a/src/uk/me/parabola/splitter/solver/AreasCalculator.java b/src/uk/me/parabola/splitter/solver/AreasCalculator.java
new file mode 100644
index 0000000..d47e491
--- /dev/null
+++ b/src/uk/me/parabola/splitter/solver/AreasCalculator.java
@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 2016, Gerd Petermann
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter.solver;
+
+import java.awt.Point;
+import java.awt.Rectangle;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.openstreetmap.osmosis.core.filter.common.PolygonFileReader;
+import org.xmlpull.v1.XmlPullParserException;
+
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.OSMFileHandler;
+import uk.me.parabola.splitter.RoundingUtils;
+import uk.me.parabola.splitter.SplitFailedException;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.args.SplitterParams;
+
+/**
+ * Some helper methods around area calculation.
+ * @author Gerd Petermann
+ *
+ */
+public class AreasCalculator {
+ private final List<PolygonDesc> polygons = new ArrayList<>();
+ private final int resolution;
+ private final int numTiles;
+ private final SplitterParams mainOptions;
+ private final DensityMapCollector pass1Collector;
+ private Area exactArea;
+
+ public AreasCalculator(SplitterParams mainOptions, int numTiles) {
+ this.mainOptions = mainOptions;
+ this.resolution = mainOptions.getResolution();
+ this.numTiles = numTiles;
+ pass1Collector = new DensityMapCollector(mainOptions);
+ readPolygonFile(mainOptions.getPolygonFile(), mainOptions.getMapid());
+ readPolygonDescFile(mainOptions.getPolygonDescFile());
+ int numPolygons = polygons.size();
+ if (numPolygons > 0) {
+ if (!checkPolygons()) {
+ System.out.println(
+ "Warning: Bounding polygon is complex. Splitter might not be able to fit all tiles into the polygon!");
+ }
+ if (numTiles > 0) {
+ System.out.println("Warning: bounding polygons are ignored because --num-tiles is used");
+ }
+ }
+ }
+
+ /**
+ * Check if the bounding polygons are usable.
+ * @return false if any
+ */
+ public boolean checkPolygons() {
+ return polygons.stream().allMatch(pd -> checkPolygon(pd.getArea(), resolution));
+ }
+
+ /**
+ * Check if the bounding polygon is usable.
+ * @param mapPolygonArea
+ * @param resolution
+ * @return false if the polygon is too complex
+ */
+ private static boolean checkPolygon(java.awt.geom.Area mapPolygonArea, int resolution) {
+ List<List<Point>> shapes = Utils.areaToShapes(mapPolygonArea);
+ int shift = 24 - resolution;
+ long rectangleWidth = 1L << shift;
+ for (List<Point> shape : shapes) {
+ int estimatedPoints = 0;
+ Point p1 = shape.get(0);
+ for (int i = 1; i < shape.size(); i++) {
+ Point p2 = shape.get(i);
+ if (p1.x != p2.x && p1.y != p2.y) {
+ // diagonal line
+ int width = Math.abs(p1.x - p2.x);
+ int height = Math.abs(p1.y - p2.y);
+ estimatedPoints += (Math.min(width, height) / rectangleWidth) * 2;
+ }
+
+ if (estimatedPoints > SplittableDensityArea.MAX_SINGLE_POLYGON_VERTICES)
+ return false; // too complex
+
+ p1 = p2;
+ }
+ }
+ return true;
+ }
+
+ private void readPolygonFile(String polygonFile, int mapId) {
+ if (polygonFile == null)
+ return;
+ polygons.clear();
+ File f = new File(polygonFile);
+
+ if (!f.exists()) {
+ throw new IllegalArgumentException("polygon file doesn't exist: " + polygonFile);
+ }
+ PolygonFileReader polyReader = new PolygonFileReader(f);
+ java.awt.geom.Area polygonInDegrees = polyReader.loadPolygon();
+ PolygonDesc pd = new PolygonDesc(polyReader.getPolygonName(), Utils.AreaDegreesToMapUnit(polygonInDegrees),
+ mapId);
+ polygons.add(pd);
+ }
+
+ private void readPolygonDescFile(String polygonDescFile) {
+ if (polygonDescFile == null)
+ return;
+ polygons.clear();
+ if (!new File(polygonDescFile).exists()) {
+ throw new IllegalArgumentException("polygon desc file doesn't exist: " + polygonDescFile);
+ }
+ final PolygonDescProcessor polygonDescProcessor = new PolygonDescProcessor(resolution);
+ final OSMFileHandler polyDescHandler = new OSMFileHandler();
+ polyDescHandler.setFileNames(Arrays.asList(polygonDescFile));
+ polyDescHandler.setMixed(false);
+ polyDescHandler.process(polygonDescProcessor);
+ polygons.addAll(polygonDescProcessor.getPolygons());
+ }
+
+ /**
+ * Fill the density map.
+ * @param osmFileHandler
+ * @param fileOutputDir
+ */
+ public void fillDensityMap(OSMFileHandler osmFileHandler, File fileOutputDir) {
+ long start = System.currentTimeMillis();
+
+ // this is typically only used for debugging
+ File densityData = new File("densities.txt");
+ File densityOutData = null;
+ if (densityData.exists() && densityData.isFile()) {
+ System.err.println("reading density data from " + densityData.getAbsolutePath());
+ pass1Collector.readMap(densityData.getAbsolutePath());
+ } else {
+ // fill the map with data from OSM files
+ osmFileHandler.execute(pass1Collector);
+ densityOutData = new File(fileOutputDir, "densities-out.txt");
+ }
+ exactArea = pass1Collector.getExactArea();
+ if (exactArea == null) {
+ throw new SplitFailedException("no usable data in input file(s)");
+ }
+ System.out.println("Fill-densities-map pass took " + (System.currentTimeMillis() - start) + " ms");
+ System.out.println("Exact map coverage read from input file(s) is " + exactArea);
+
+ if (densityOutData != null)
+ pass1Collector.saveMap(densityOutData.getAbsolutePath());
+
+ if (polygons.size() == 1) {
+ // intersect the bounding polygon with the exact area
+ Rectangle polgonsBoundingBox = polygons.get(0).getArea().getBounds();
+ exactArea = Area.calcArea(exactArea, polgonsBoundingBox);
+ if (exactArea != null)
+ System.out.println("Exact map coverage after applying bounding box of polygon-file is " + exactArea);
+ else {
+ System.out.println("Exact map coverage after applying bounding box of polygon-file is an empty area");
+ return;
+ }
+ }
+
+ addPrecompSeaDensityData();
+ }
+
+ private void addPrecompSeaDensityData () {
+ String precompSeaDir = mainOptions.getPrecompSea();
+ if (precompSeaDir != null) {
+ System.out.println("Counting nodes of precompiled sea data ...");
+ long startSea = System.currentTimeMillis();
+ DensityMapCollector seaCollector = new DensityMapCollector(mainOptions);
+ PrecompSeaReader precompSeaReader = new PrecompSeaReader(exactArea, new File(precompSeaDir));
+ try {
+ precompSeaReader.processMap(seaCollector);
+ } catch (XmlPullParserException e) {
+ // very unlikely because we read generated files
+ e.printStackTrace();
+ }
+ pass1Collector.mergeSeaData(seaCollector, !mainOptions.isNoTrim(), mainOptions.getResolution());
+ System.out.println("Precompiled sea data pass took " + (System.currentTimeMillis() - startSea) + " ms");
+ }
+ }
+
+ /**
+ * Calculate the areas that we are going to split into by getting the total
+ * area and then subdividing down until each area has at most max-nodes
+ * nodes in it.
+ * If {@code --num-tiles} option is used, tries to find a max-nodes value which results in the wanted number of areas.
+ *
+ * @return
+ */
+ public List<Area> calcAreas () {
+ Area roundedBounds = RoundingUtils.round(exactArea, mainOptions.getResolution());
+ SplittableDensityArea splittableArea = pass1Collector.getSplitArea(mainOptions.getSearchLimit(), roundedBounds);
+ if (splittableArea.hasData() == false) {
+ System.out.println("input file(s) have no data inside calculated bounding box");
+ return Collections.emptyList();
+ }
+ System.out.println("Rounded map coverage is " + splittableArea.getBounds());
+
+ splittableArea.setTrim(mainOptions.isNoTrim() == false);
+ splittableArea.setMapId(mainOptions.getMapid());
+ long startSplit = System.currentTimeMillis();
+ List<Area> areas;
+ if (numTiles >= 2) {
+ System.out.println("Splitting nodes into " + numTiles + " areas");
+ areas = splittableArea.split(numTiles);
+ } else {
+ System.out.println(
+ "Splitting nodes into areas containing a maximum of " + Utils.format(mainOptions.getMaxNodes()) + " nodes each...");
+ splittableArea.setMaxNodes(mainOptions.getMaxNodes());
+ areas = splittableArea.split(polygons);
+ }
+ if (areas != null && areas.isEmpty() == false)
+ System.out.println("Creating the initial areas took " + (System.currentTimeMillis() - startSplit) + " ms");
+ return areas;
+ }
+
+ public List<PolygonDesc> getPolygons() {
+ return Collections.unmodifiableList(polygons);
+ }
+
+}
diff --git a/src/uk/me/parabola/splitter/DensityMap.java b/src/uk/me/parabola/splitter/solver/DensityMap.java
similarity index 98%
rename from src/uk/me/parabola/splitter/DensityMap.java
rename to src/uk/me/parabola/splitter/solver/DensityMap.java
index e465db7..cf57ae2 100644
--- a/src/uk/me/parabola/splitter/DensityMap.java
+++ b/src/uk/me/parabola/splitter/solver/DensityMap.java
@@ -11,7 +11,7 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.solver;
import java.awt.Rectangle;
import java.io.File;
@@ -23,6 +23,12 @@ import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.util.regex.Pattern;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.MapDetails;
+import uk.me.parabola.splitter.RoundingUtils;
+import uk.me.parabola.splitter.SplitFailedException;
+import uk.me.parabola.splitter.Utils;
+
/**
* Builds up a map of node densities across the total area being split.
* Density information is held at the maximum desired map resolution.
diff --git a/src/uk/me/parabola/splitter/DensityMapCollector.java b/src/uk/me/parabola/splitter/solver/DensityMapCollector.java
similarity index 78%
rename from src/uk/me/parabola/splitter/DensityMapCollector.java
rename to src/uk/me/parabola/splitter/solver/DensityMapCollector.java
index 367a505..9ae860c 100644
--- a/src/uk/me/parabola/splitter/DensityMapCollector.java
+++ b/src/uk/me/parabola/splitter/solver/DensityMapCollector.java
@@ -11,7 +11,14 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.solver;
+
+import uk.me.parabola.splitter.AbstractMapProcessor;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.MapDetails;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.RoundingUtils;
+import uk.me.parabola.splitter.args.SplitterParams;
/**
* Builds up a density map.
@@ -19,25 +26,18 @@ package uk.me.parabola.splitter;
class DensityMapCollector extends AbstractMapProcessor{
private final DensityMap densityMap;
private final MapDetails details = new MapDetails();
- private final boolean ignoreBoundsTags;
private Area bounds;
+ private final boolean ignoreBoundsTags;
+ private int files;
- /**
- * @param resolution gives the granularity of the grid
- * @param ignoreBoundsTags true means ignore bounds found in the input file(s)
- */
- DensityMapCollector(int resolution, boolean ignoreBoundsTags) {
+ public DensityMapCollector(SplitterParams mainOptions) {
Area densityBounds = new Area(-0x400000, -0x800000, 0x400000, 0x800000);
- densityMap = new DensityMap(densityBounds, resolution);
- this.ignoreBoundsTags = ignoreBoundsTags;
+ densityMap = new DensityMap(densityBounds, mainOptions.getResolution());
+ this.ignoreBoundsTags = mainOptions.getIgnoreOsmBounds();
}
@Override
- public boolean isStartNodeOnly() {
- return true;
- }
- @Override
public boolean skipTags() {
return true;
}
@@ -50,6 +50,12 @@ class DensityMapCollector extends AbstractMapProcessor{
return true;
}
+ @Override
+ public void startFile() {
+ if (++files > 1)
+ checkBounds();
+ }
+
@Override
public void boundTag(Area fileBbox) {
if (ignoreBoundsTags)
@@ -69,6 +75,7 @@ class DensityMapCollector extends AbstractMapProcessor{
details.addToBounds(glat, glon);
}
+
/**
* Check if a bounds tag was found. If not,
* use the bbox of the data that was collected so far.
diff --git a/src/uk/me/parabola/splitter/EnhancedDensityMap.java b/src/uk/me/parabola/splitter/solver/EnhancedDensityMap.java
similarity index 97%
rename from src/uk/me/parabola/splitter/EnhancedDensityMap.java
rename to src/uk/me/parabola/splitter/solver/EnhancedDensityMap.java
index 90ef497..85263de 100644
--- a/src/uk/me/parabola/splitter/EnhancedDensityMap.java
+++ b/src/uk/me/parabola/splitter/solver/EnhancedDensityMap.java
@@ -10,11 +10,14 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.solver;
import java.awt.Rectangle;
import java.util.BitSet;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Utils;
+
/**
* Contains info that is needed by the {@link Tile} class. For a given
* DensityMap we calculate some extra info to allow faster access to row sums
diff --git a/src/uk/me/parabola/splitter/PolygonDesc.java b/src/uk/me/parabola/splitter/solver/PolygonDesc.java
similarity index 69%
rename from src/uk/me/parabola/splitter/PolygonDesc.java
rename to src/uk/me/parabola/splitter/solver/PolygonDesc.java
index 1bacaf0..1019f9a 100644
--- a/src/uk/me/parabola/splitter/PolygonDesc.java
+++ b/src/uk/me/parabola/splitter/solver/PolygonDesc.java
@@ -10,7 +10,7 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.solver;
import java.awt.geom.Area;
@@ -19,13 +19,25 @@ import java.awt.geom.Area;
* @author GerdP
*
*/
-class PolygonDesc {
- final java.awt.geom.Area area;
- final String name;
- final int mapId;
+public class PolygonDesc {
+ private final java.awt.geom.Area area;
+ private final String name;
+ private final int mapId;
public PolygonDesc(String name, Area area, int mapId) {
this.name = name;
this.area = area;
this.mapId = mapId;
}
+
+ public java.awt.geom.Area getArea() {
+ return area;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public int getMapId() {
+ return mapId;
+ }
}
diff --git a/src/uk/me/parabola/splitter/PolygonDescProcessor.java b/src/uk/me/parabola/splitter/solver/PolygonDescProcessor.java
similarity index 59%
rename from src/uk/me/parabola/splitter/PolygonDescProcessor.java
rename to src/uk/me/parabola/splitter/solver/PolygonDescProcessor.java
index d7f70d2..f955327 100644
--- a/src/uk/me/parabola/splitter/PolygonDescProcessor.java
+++ b/src/uk/me/parabola/splitter/solver/PolygonDescProcessor.java
@@ -10,14 +10,17 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.solver;
import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import uk.me.parabola.splitter.AbstractMapProcessor;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.RoundingUtils;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.Way;
import java.awt.geom.Path2D;
import java.awt.geom.Area;
-import java.io.File;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -33,17 +36,17 @@ import java.util.List;
class PolygonDescProcessor extends AbstractMapProcessor {
private Long2ObjectOpenHashMap<Node> nodes = new Long2ObjectOpenHashMap<>();
private final List<PolygonDesc> polygonDescriptions = new ArrayList<>();
- final int resolution;
+ private final int shift;
public PolygonDescProcessor(int resolution) {
- this.resolution = resolution;
+ this.shift = 24 - resolution;
}
@Override
public void processNode(Node n){
// round all coordinates to be on the used grid.
- int lat = getRoundedCoord(n.getMapLat());
- int lon = getRoundedCoord(n.getMapLon());
+ int lat = RoundingUtils.round(n.getMapLat(), shift);
+ int lon = RoundingUtils.round(n.getMapLon(), shift);
double roundedLat = Utils.toDegrees(lat);
double roundedLon = Utils.toDegrees(lon);
@@ -94,46 +97,6 @@ class PolygonDescProcessor extends AbstractMapProcessor {
return true;
}
- /**
- * Calculate and write the area lists for each named polygon.
- * @param fileOutputDir
- * @param areas the list of all areas
- * @param kmlOutputFile optional kml file name or null
- * @param outputType file name extension of output files
- * @throws IOException
- */
- public void writeListFiles(File fileOutputDir,
- List<uk.me.parabola.splitter.Area> areas, String kmlOutputFile, String outputType) throws IOException {
- for (PolygonDesc pd : polygonDescriptions){
- List<uk.me.parabola.splitter.Area> areasPart = new ArrayList<>();
- for (uk.me.parabola.splitter.Area a : areas){
- if (pd.area.intersects(a.getRect()))
- areasPart.add(a);
- }
- if (kmlOutputFile != null){
- File out = new File(kmlOutputFile);
- String kmlOutputFilePart = pd.name + "-" + out.getName();
- if (out.getParent() != null)
- out = new File(out.getParent(), kmlOutputFilePart);
- else
- out = new File(kmlOutputFilePart);
- if (out.getParent() == null)
- out = new File(fileOutputDir, kmlOutputFilePart);
- KmlWriter.writeKml(out.getPath(), areasPart);
- }
- AreaList al = new AreaList(areasPart, null);
- al.writePoly(new File(fileOutputDir, pd.name + "-" + "areas.poly").getPath());
- al.writeArgsFile(new File(fileOutputDir, pd.name + "-" + "template.args").getPath(), outputType, pd.mapId);
- }
- }
-
- private int getRoundedCoord(int val){
- int shift = 24 - resolution;
- int half = 1 << (shift - 1); // 0.5 shifted
- int mask = ~((1 << shift) - 1); // to remove fraction bits
- return (val + half) & mask;
- }
-
public List<PolygonDesc> getPolygons() {
return polygonDescriptions;
}
diff --git a/src/uk/me/parabola/splitter/PrecompSeaReader.java b/src/uk/me/parabola/splitter/solver/PrecompSeaReader.java
similarity index 60%
rename from src/uk/me/parabola/splitter/PrecompSeaReader.java
rename to src/uk/me/parabola/splitter/solver/PrecompSeaReader.java
index 2919584..e509af3 100644
--- a/src/uk/me/parabola/splitter/PrecompSeaReader.java
+++ b/src/uk/me/parabola/splitter/solver/PrecompSeaReader.java
@@ -10,7 +10,7 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.solver;
import java.io.File;
import java.io.FileInputStream;
@@ -29,11 +29,17 @@ import java.util.zip.ZipFile;
import org.xmlpull.v1.XmlPullParserException;
import crosby.binary.file.BlockInputStream;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.SplitFailedException;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.parser.BinaryMapParser;
+import uk.me.parabola.splitter.parser.OSMXMLParser;
/**
- * Reader for precompiled sea data.
- * This is mostly a copy of the corresponding code in mkgmap SeaGenerator.
- * @author GerdP
+ * Reader for precompiled sea data. This is mostly a copy of the corresponding
+ * code in mkgmap SeaGenerator.
+ *
+ * @author GerdP
*
*/
public class PrecompSeaReader {
@@ -45,8 +51,8 @@ public class PrecompSeaReader {
private static final byte LAND_TILE = 'l';
private static final byte MIXED_TILE = 'm';
-
- // useful constants defining the min/max map units of the precompiled sea tiles
+ // useful constants defining the min/max map units of the precompiled sea
+ // tiles
private static final int MIN_LAT = Utils.toMapUnit(-90.0);
private static final int MAX_LAT = Utils.toMapUnit(90.0);
private static final int MIN_LON = Utils.toMapUnit(-180.0);
@@ -69,47 +75,46 @@ public class PrecompSeaReader {
/**
* Process all precompiled sea tiles.
- * @param processor The processor that is called
+ *
+ * @param processor
+ * The processor that is called
* @throws XmlPullParserException
*/
public void processMap(DensityMapCollector processor) throws XmlPullParserException {
- for (String tileName: getPrecompKeyNames()){
+ for (String tileName : getPrecompKeyNames()) {
InputStream is = getStream(tileName);
- if (is != null){
- try{
- if (tileName.endsWith(".pbf")){
+ if (is != null) {
+ try {
+ if (tileName.endsWith(".pbf")) {
BinaryMapParser binParser = new BinaryMapParser(processor, null, 0);
BlockInputStream blockinput = (new BlockInputStream(is, binParser));
blockinput.process();
blockinput.close();
} else {
// No, try XML.
- try (Reader reader = new InputStreamReader(is,
- Charset.forName("UTF-8"));) {
- OSMParser parser = new OSMParser(processor, true);
+ try (Reader reader = new InputStreamReader(is, Charset.forName("UTF-8"));) {
+ OSMXMLParser parser = new OSMXMLParser(processor, true);
parser.setReader(reader);
parser.parse();
}
}
} catch (Exception e) {
e.printStackTrace();
- throw new SplitFailedException(e.getMessage());
+ throw new SplitFailedException(e.getMessage());
}
}
}
}
-
-
/**
* Read the index and set corresponding fields.
*/
private void init() {
- if (precompSeaDir.exists()){
- String internalPath = null;
+ if (precompSeaDir.exists()) {
+ String internalPath = null;
String indexFileName = "index.txt.gz";
- try{
- if (precompSeaDir.isDirectory()){
+ try {
+ if (precompSeaDir.isDirectory()) {
File indexFile = new File(precompSeaDir, indexFileName);
if (indexFile.exists() == false) {
// check if the unzipped index file exists
@@ -117,70 +122,71 @@ public class PrecompSeaReader {
indexFile = new File(precompSeaDir, indexFileName);
}
if (indexFile.exists()) {
- try(InputStream indexStream = new FileInputStream(indexFile)){
+ try (InputStream indexStream = new FileInputStream(indexFile)) {
loadIndex(indexStream, indexFileName);
}
- } else
+ } else
throw new IllegalArgumentException("Cannot find required index.txt[.gz] in " + precompSeaDir);
- } else if (precompSeaDir.getName().endsWith(".zip")){
+ } else if (precompSeaDir.getName().endsWith(".zip")) {
zipFile = new ZipFile(precompSeaDir);
internalPath = "sea/";
ZipEntry entry = zipFile.getEntry(internalPath + indexFileName);
- if (entry == null){
+ if (entry == null) {
indexFileName = "index.txt";
entry = zipFile.getEntry(internalPath + indexFileName);
}
- if (entry == null){
+ if (entry == null) {
internalPath = "";
indexFileName = "index.txt.gz";
entry = zipFile.getEntry(internalPath + indexFileName);
}
- if (entry != null){
- try (InputStream indexStream = zipFile.getInputStream(entry)){
+ if (entry != null) {
+ try (InputStream indexStream = zipFile.getInputStream(entry)) {
precompZipFileInternalPath = internalPath;
loadIndex(indexStream, indexFileName);
}
- } else
+ } else
throw new SplitFailedException("Don't know how to read " + precompSeaDir);
- }
- else {
+ } else {
throw new SplitFailedException("Don't know how to read " + precompSeaDir);
}
} catch (IOException exp) {
exp.printStackTrace();
- throw new SplitFailedException("Cannot read index file " + indexFileName);
- }
+ throw new SplitFailedException("Cannot read index file " + indexFileName);
+ }
} else {
- throw new SplitFailedException("Directory or zip file with precompiled sea does not exist: "
- + precompSeaDir.getName());
+ throw new SplitFailedException(
+ "Directory or zip file with precompiled sea does not exist: " + precompSeaDir.getName());
}
}
-
- private void loadIndex(InputStream indexStream, String indexFileName) throws IOException{
+
+ private void loadIndex(InputStream indexStream, String indexFileName) throws IOException {
if (indexFileName.endsWith(".gz")) {
- try(InputStream stream = new GZIPInputStream(indexStream)){
+ try (InputStream stream = new GZIPInputStream(indexStream)) {
loadIndexFromStream(stream);
return;
}
- }
+ }
loadIndexFromStream(indexStream);
}
-
- /**
- * Read the index from stream and populate the index grid.
- * @param fileStream already opened stream
- */
- private void loadIndexFromStream(InputStream fileStream) throws IOException{
- int indexWidth = (PrecompSeaReader.getPrecompTileStart(MAX_LON) - PrecompSeaReader.getPrecompTileStart(MIN_LON)) / PrecompSeaReader.PRECOMP_RASTER;
- int indexHeight = (PrecompSeaReader.getPrecompTileStart(MAX_LAT) - PrecompSeaReader.getPrecompTileStart(MIN_LAT)) / PrecompSeaReader.PRECOMP_RASTER;
- LineNumberReader indexReader = new LineNumberReader(
- new InputStreamReader(fileStream));
- Pattern csvSplitter = Pattern.compile(Pattern
- .quote(";"));
+
+ /**
+ * Read the index from stream and populate the index grid.
+ *
+ * @param fileStream
+ * already opened stream
+ */
+ private void loadIndexFromStream(InputStream fileStream) throws IOException {
+ int indexWidth = (PrecompSeaReader.getPrecompTileStart(MAX_LON) - PrecompSeaReader.getPrecompTileStart(MIN_LON))
+ / PrecompSeaReader.PRECOMP_RASTER;
+ int indexHeight = (PrecompSeaReader.getPrecompTileStart(MAX_LAT)
+ - PrecompSeaReader.getPrecompTileStart(MIN_LAT)) / PrecompSeaReader.PRECOMP_RASTER;
+ LineNumberReader indexReader = new LineNumberReader(new InputStreamReader(fileStream));
+ Pattern csvSplitter = Pattern.compile(Pattern.quote(";"));
String indexLine = null;
- byte[][] indexGrid = new byte[indexWidth+1][indexHeight+1];
- boolean detectExt = true;
+ byte[][] indexGrid = new byte[indexWidth + 1][indexHeight + 1];
+ boolean detectExt = true;
String prefix = null;
String ext = null;
@@ -195,75 +201,79 @@ public class PrecompSeaReader {
}
String precompKey = items[0];
byte type = updatePrecompSeaTileIndex(precompKey, items[1], indexGrid);
- if (type == '?'){
- throw new IllegalArgumentException("Invalid format in index file name: " + indexLine);
+ if (type == '?') {
+ throw new IllegalArgumentException("Invalid format in index file name: " + indexLine);
}
- if (type == MIXED_TILE){
+ if (type == MIXED_TILE) {
// make sure that all file names are using the same name scheme
int prePos = items[1].indexOf(items[0]);
- if (prePos >= 0){
- if (detectExt){
+ if (prePos >= 0) {
+ if (detectExt) {
prefix = items[1].substring(0, prePos);
- ext = items[1].substring(prePos+items[0].length());
+ ext = items[1].substring(prePos + items[0].length());
detectExt = false;
} else {
StringBuilder sb = new StringBuilder(prefix);
sb.append(precompKey);
- sb.append(ext);
- if (items[1].equals(sb.toString()) == false){
- throw new IllegalArgumentException("Unexpected file name in index file: " + indexLine);
+ sb.append(ext);
+ if (items[1].equals(sb.toString()) == false) {
+ throw new IllegalArgumentException("Unexpected file name in index file: " + indexLine);
}
}
}
}
}
- //
+ //
precompIndex = indexGrid;
precompSeaPrefix = prefix;
precompSeaExt = ext;
- }
+ }
- private InputStream getStream(String tileName){
- InputStream is = null;
- try{
- if (zipFile != null){
- ZipEntry entry = zipFile.getEntry(precompZipFileInternalPath + tileName);
- if (entry != null){
- is = zipFile.getInputStream(entry);
- } else {
- throw new IOException("Preompiled sea tile " + tileName + " not found.");
- }
- } else {
- File precompTile = new File(precompSeaDir,tileName);
- is = new FileInputStream(precompTile);
- }
- } catch (Exception exp) {
- exp.printStackTrace();
- throw new SplitFailedException(exp.getMessage());
- }
+ private InputStream getStream(String tileName) {
+ InputStream is = null;
+ try {
+ if (zipFile != null) {
+ ZipEntry entry = zipFile.getEntry(precompZipFileInternalPath + tileName);
+ if (entry != null) {
+ is = zipFile.getInputStream(entry);
+ } else {
+ throw new IOException("Preompiled sea tile " + tileName + " not found.");
+ }
+ } else {
+ File precompTile = new File(precompSeaDir, tileName);
+ is = new FileInputStream(precompTile);
+ }
+ } catch (Exception exp) {
+ exp.printStackTrace();
+ throw new SplitFailedException(exp.getMessage());
+ }
return is;
- }
-
- /**
- * Retrieves the start value of the precompiled tile.
- * @param value the value for which the start value is calculated
- * @return the tile start value
- */
- private static int getPrecompTileStart(int value) {
- int rem = value % PRECOMP_RASTER;
- if (rem == 0) {
- return value;
- } else if (value >= 0) {
- return value - rem;
- } else {
- return value - PRECOMP_RASTER - rem;
- }
- }
+ }
+
+ /**
+ * Retrieves the start value of the precompiled tile.
+ *
+ * @param value
+ * the value for which the start value is calculated
+ * @return the tile start value
+ */
+ private static int getPrecompTileStart(int value) {
+ int rem = value % PRECOMP_RASTER;
+ if (rem == 0) {
+ return value;
+ } else if (value >= 0) {
+ return value - rem;
+ } else {
+ return value - PRECOMP_RASTER - rem;
+ }
+ }
/**
* Retrieves the end value of the precompiled tile.
- * @param value the value for which the end value is calculated
+ *
+ * @param value
+ * the value for which the end value is calculated
* @return the tile end value
*/
private static int getPrecompTileEnd(int value) {
@@ -275,42 +285,47 @@ public class PrecompSeaReader {
} else {
return value - rem;
}
- }
-
+ }
+
/**
- * Calculates the key names of the precompiled sea tiles for the bounding box.
- * The key names are compiled of {@code lat+"_"+lon}.
+ * Calculates the key names of the precompiled sea tiles for the bounding
+ * box. The key names are compiled of {@code lat+"_"+lon}.
+ *
* @return the key names for the bounding box
*/
private List<String> getPrecompKeyNames() {
List<String> precompKeys = new ArrayList<>();
- for (int lat = getPrecompTileStart(bounds.getMinLat()); lat < getPrecompTileEnd(bounds
- .getMaxLat()); lat += PRECOMP_RASTER) {
- for (int lon = getPrecompTileStart(bounds.getMinLong()); lon < getPrecompTileEnd(bounds
- .getMaxLong()); lon += PRECOMP_RASTER) {
- int latIndex = (MAX_LAT-lat) / PRECOMP_RASTER;
- int lonIndex = (MAX_LON-lon) / PRECOMP_RASTER;
- byte type = precompIndex[lonIndex][latIndex];
+ for (int lat = getPrecompTileStart(bounds.getMinLat()); lat < getPrecompTileEnd(
+ bounds.getMaxLat()); lat += PRECOMP_RASTER) {
+ for (int lon = getPrecompTileStart(bounds.getMinLong()); lon < getPrecompTileEnd(
+ bounds.getMaxLong()); lon += PRECOMP_RASTER) {
+ int latIndex = (MAX_LAT - lat) / PRECOMP_RASTER;
+ int lonIndex = (MAX_LON - lon) / PRECOMP_RASTER;
+ byte type = precompIndex[lonIndex][latIndex];
if (type == MIXED_TILE)
precompKeys.add(precompSeaPrefix + lat + "_" + lon + precompSeaExt);
}
}
return precompKeys;
}
-
+
/**
- * Update the index grid for the element identified by precompKey.
- * @param precompKey The key name is compiled of {@code lat+"_"+lon}.
- * @param fileName either "land", "sea", or a file name containing OSM data
- * @param indexGrid the previously allocated index grid
- * @return the byte that was saved in the index grid
+ * Update the index grid for the element identified by precompKey.
+ *
+ * @param precompKey
+ * The key name is compiled of {@code lat+"_"+lon}.
+ * @param fileName
+ * either "land", "sea", or a file name containing OSM data
+ * @param indexGrid
+ * the previously allocated index grid
+ * @return the byte that was saved in the index grid
*/
- private static byte updatePrecompSeaTileIndex (String precompKey, String fileName, byte[][] indexGrid){
+ private static byte updatePrecompSeaTileIndex(String precompKey, String fileName, byte[][] indexGrid) {
String[] tileCoords = keySplitter.split(precompKey);
byte type = '?';
- if (tileCoords.length == 2){
- int lat = Integer.valueOf(tileCoords[0]);
- int lon = Integer.valueOf(tileCoords[1]);
+ if (tileCoords.length == 2) {
+ int lat = Integer.valueOf(tileCoords[0]);
+ int lon = Integer.valueOf(tileCoords[1]);
int latIndex = (MAX_LAT - lat) / PRECOMP_RASTER;
int lonIndex = (MAX_LON - lon) / PRECOMP_RASTER;
@@ -318,12 +333,12 @@ public class PrecompSeaReader {
type = SEA_TILE;
else if ("land".equals(fileName))
type = LAND_TILE;
- else
+ else
type = MIXED_TILE;
indexGrid[lonIndex][latIndex] = type;
}
return type;
}
-
+
}
diff --git a/src/uk/me/parabola/splitter/Solution.java b/src/uk/me/parabola/splitter/solver/Solution.java
similarity index 97%
rename from src/uk/me/parabola/splitter/Solution.java
rename to src/uk/me/parabola/splitter/solver/Solution.java
index 3614b80..8834e1a 100644
--- a/src/uk/me/parabola/splitter/Solution.java
+++ b/src/uk/me/parabola/splitter/solver/Solution.java
@@ -10,7 +10,7 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.solver;
import java.awt.Rectangle;
import java.util.ArrayList;
@@ -22,7 +22,7 @@ import java.util.List;
* @author GerdP
*
*/
-class Solution {
+public class Solution {
/**
*
*/
@@ -51,7 +51,7 @@ class Solution {
if (aspectRatio < 1.0)
aspectRatio = 1.0 / aspectRatio;
worstAspectRatio = Math.max(aspectRatio, worstAspectRatio);
- worstMinNodes = Math.min(tile.count, worstMinNodes);
+ worstMinNodes = Math.min(tile.getCount(), worstMinNodes);
return true;
}
@@ -163,7 +163,7 @@ class Solution {
Tile candidate = null;
boolean trimmed = false;
for (Tile tile : tiles){
- if (tile.count == 0)
+ if (tile.getCount() == 0)
continue;
switch (side){
case LEFT:
diff --git a/src/uk/me/parabola/splitter/SplittableDensityArea.java b/src/uk/me/parabola/splitter/solver/SplittableDensityArea.java
similarity index 91%
rename from src/uk/me/parabola/splitter/SplittableDensityArea.java
rename to src/uk/me/parabola/splitter/solver/SplittableDensityArea.java
index c78bbcf..54be881 100644
--- a/src/uk/me/parabola/splitter/SplittableDensityArea.java
+++ b/src/uk/me/parabola/splitter/solver/SplittableDensityArea.java
@@ -11,9 +11,13 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.solver;
import it.unimi.dsi.fastutil.ints.IntArrayList;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.RoundingUtils;
+import uk.me.parabola.splitter.SplitFailedException;
+import uk.me.parabola.splitter.Utils;
import java.awt.Point;
import java.awt.Rectangle;
@@ -21,16 +25,14 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Map.Entry;
/**
* Splits a density map into multiple areas, none of which
* exceed the desired threshold.
*
-* @author Chris Miller
+* @author Chris Miller, Gerd Petermann
*/
public class SplittableDensityArea {
private static final int MAX_LAT_DEGREES = 85;
@@ -41,6 +43,8 @@ public class SplittableDensityArea {
private static final int AXIS_VERT = 1;
public static final double NICE_MAX_ASPECT_RATIO = 4;
private static final double VERY_NICE_FILL_RATIO = 0.93;
+ private static final long LARGE_MAX_NODES = 10_000_000;
+ private static final int GOOD_SOL_INIT_SIZE = 1_000_000;
private double maxAspectRatio;
private long minNodes;
@@ -58,6 +62,8 @@ public class SplittableDensityArea {
private HashSet<Tile> knownBad;
private LinkedHashMap<Tile, Integer> incomplete;
private long countBad;
+
+ /** if true enables an alternative algorithm */
private boolean searchAll = false;
final int maxTileHeight;
@@ -70,6 +76,7 @@ public class SplittableDensityArea {
private boolean allowEmptyPart = false;
private int currMapId;
private boolean hasEmptyPart;
+ private boolean ignoreSize;
public SplittableDensityArea(DensityMap densities, int startSearchLimit) {
this.shift = densities.getShift();
@@ -203,15 +210,15 @@ public class SplittableDensityArea {
for (int i = 0; i < namedPolygons.size(); i++){
boolean wasDistinct = true;
PolygonDesc namedPart = namedPolygons.get(i);
- java.awt.geom.Area distinctPart = new java.awt.geom.Area (namedPart.area);
+ java.awt.geom.Area distinctPart = new java.awt.geom.Area (namedPart.getArea());
for(int j = 0; j < namedPolygons.size(); j++){
if (j == i)
continue;
- java.awt.geom.Area test = new java.awt.geom.Area(namedPart.area);
- test.intersect(namedPolygons.get(j).area);
+ java.awt.geom.Area test = new java.awt.geom.Area(namedPart.getArea());
+ test.intersect(namedPolygons.get(j).getArea());
if (test.isEmpty() == false){
wasDistinct = false;
- distinctPart.subtract(namedPolygons.get(j).area);
+ distinctPart.subtract(namedPolygons.get(j).getArea());
if (j > i){
ShareInfo si = new ShareInfo();
si.area = test;
@@ -224,9 +231,9 @@ public class SplittableDensityArea {
if (distinctPart.isEmpty() == false && distinctPart.intersects(Utils.area2Rectangle(allDensities.getBounds(),0))){
// KmlWriter.writeKml("e:/ld_sp/distinct_"+namedPart.name, "distinct", distinctPart);
if (wasDistinct == false)
- System.out.println("splitting distinct part of " + namedPart.name);
+ System.out.println("splitting distinct part of " + namedPart.getName());
else
- System.out.println("splitting " + namedPart.name);
+ System.out.println("splitting " + namedPart.getName());
result.addAll(split(distinctPart));
}
}
@@ -238,7 +245,7 @@ public class SplittableDensityArea {
if (si.sharedBy.contains(j))
continue;
java.awt.geom.Area test = new java.awt.geom.Area(si.area);
- test.intersect(namedPolygons.get(j).area);
+ test.intersect(namedPolygons.get(j).getArea());
if (test.isEmpty() == false){
si.area.subtract(test);
if (j > si.sharedBy.getInt(si.sharedBy.size()-1)){
@@ -255,7 +262,7 @@ public class SplittableDensityArea {
if (si.area.isEmpty() == false && si.area.intersects(Utils.area2Rectangle(allDensities.getBounds(),0))){
String desc = "";
for (int pos : si.sharedBy)
- desc += namedPolygons.get(pos).name + " and ";
+ desc += namedPolygons.get(pos).getName() + " and ";
desc = desc.substring(0,desc.lastIndexOf(" and"));
System.out.println("splitting area shared by exactly " + si.sharedBy.size() + " polygons: " + desc);
// KmlWriter.writeKml("e:/ld_sp/shared_"+desc.replace(" " , "_"), desc, si.area);
@@ -286,6 +293,7 @@ public class SplittableDensityArea {
Pair bestBelow = null;
Pair bestAbove = null;
beQuiet = true;
+ ignoreSize = true;
while (true) {
setMaxNodes(currMaxNodes);
System.out.println("Trying a max-nodes value of " + currMaxNodes + " to split " + allDensities.getNodeCount() + " nodes into " + wantedTiles + " areas");
@@ -295,7 +303,7 @@ public class SplittableDensityArea {
res = split();
return res;
}
- goodSolutions = new HashMap<>();
+ goodSolutions = new HashMap<>(GOOD_SOL_INIT_SIZE);
Pair pair = new Pair(currMaxNodes, res.size());
if (res.size() > wantedTiles){
if (bestAbove == null)
@@ -353,11 +361,8 @@ public class SplittableDensityArea {
return;
if (sol.getWorstMinNodes() > (goodRatio * maxNodes)){
Solution good = sol.copy();
- Solution prevSol = goodSolutions.put(tile, good);
- if (prevSol != null){
- if (prevSol.getWorstMinNodes() > good.getWorstMinNodes())
- goodSolutions.put(tile, prevSol);
- }
+ // add new or replace worse solution
+ goodSolutions.compute(tile, (k,v) -> v == null || v.getWorstMinNodes() < good.getWorstMinNodes() ? good : v);
}
}
@@ -370,12 +375,7 @@ public class SplittableDensityArea {
private void filterGoodSolutions(Solution best){
if (best == null || best.isEmpty())
return;
- Iterator<Entry<Tile, Solution>> iter = goodSolutions.entrySet().iterator();
- while (iter.hasNext()){
- Entry<Tile, Solution> entry = iter.next();
- if (entry.getValue().getWorstMinNodes() <= best.getWorstMinNodes())
- iter.remove();
- }
+ goodSolutions.entrySet().removeIf(entry -> entry.getValue().getWorstMinNodes() <= best.getWorstMinNodes());
goodRatio = Math.max(0.5, (double) best.getWorstMinNodes() / maxNodes);
}
@@ -409,7 +409,7 @@ public class SplittableDensityArea {
int firstEmpty = -1;
int countEmpty = 0;
long countLastPart = 0;
- long countRemaining = tile.count;
+ long countRemaining = tile.getCount();
long maxEmpty = Utils.toMapUnit(30) / (1 << shift);
if (splitHoriz){
for (int i = 0; i < tile.width; i++){
@@ -462,7 +462,7 @@ public class SplittableDensityArea {
for (List<Point> shape: shapes){
java.awt.geom.Area part = Utils.shapeToArea(shape);
Tile t = new Tile(extraDensityInfo, part.getBounds());
- if (t.count > 0)
+ if (t.getCount() > 0)
clusters.addAll(checkForEmptyClusters(depth + 1, t.trim(), !splitHoriz ));
}
}
@@ -585,7 +585,7 @@ public class SplittableDensityArea {
*/
private Solution findSolution(int depth, final Tile tile, Tile parent, TileMetaInfo smiParent){
boolean addAndReturn = false;
- if (tile.count == 0){
+ if (tile.getCount() == 0){
if (!allowEmptyPart){
hasEmptyPart = true;
return null;
@@ -593,25 +593,19 @@ public class SplittableDensityArea {
if (tile.width * tile.height <= 4)
return null;
return new Solution(maxNodes); // allow empty part of the world
- } else if (tile.count > maxNodes && tile.width == 1 && tile.height == 1) {
+ } else if (tile.getCount() > maxNodes && tile.width == 1 && tile.height == 1) {
addAndReturn = true; // can't split further
- } else if (tile.count < minNodes && depth == 0) {
+ } else if (tile.getCount() < minNodes && depth == 0) {
addAndReturn = true; // nothing to do
- } else if (tile.count < minNodes) {
+ } else if (tile.getCount() < minNodes) {
return null;
- } else if (tile.count <= maxNodes) {
+ } else if (tile.getCount() <= maxNodes) {
double ratio = tile.getAspectRatio();
if (ratio < 1.0)
ratio = 1.0 / ratio;
if (ratio < maxAspectRatio){
- if (checkSize(tile))
+ if (ignoreSize || maxNodes >= LARGE_MAX_NODES || checkSize(tile))
addAndReturn = true;
- else {
-// System.out.println("too large at depth " + depth + " " + tile);
-// if (depth > 10){
-// long dd = 4;
-// }
- }
}
} else if (tile.width < 2 && tile.height < 2) {
return null;
@@ -628,7 +622,7 @@ public class SplittableDensityArea {
solution.add(tile); // can't split further
return solution;
}
- if (tile.count < minNodes * 2){
+ if (tile.getCount() < minNodes * 2){
return null;
}
Solution cached = searchGoodSolutions(tile);
@@ -675,10 +669,6 @@ public class SplittableDensityArea {
continue;
}
int splitPos = todoList.getInt(usedTestPos++);
-// if (tested.get(splitPos)){
-// continue;
-// }
-// tested.set(splitPos);
// create the two parts of the tile
boolean ok = false;
if (axis == AXIS_HOR){
@@ -694,7 +684,7 @@ public class SplittableDensityArea {
parts[0] = parts[0].trim();
parts[1] = parts[1].trim();
}
- if (parts[0].count > parts[1].count){
+ if (parts[0].getCount() > parts[1].getCount()){
// first try the less populated part
Tile help = parts[0];
parts[0] = parts[1];
@@ -707,7 +697,7 @@ public class SplittableDensityArea {
// long t1 = System.currentTimeMillis();
sols[i] = findSolution(depth + 1, parts[i], tile, smi);
// long dt = System.currentTimeMillis() - t1;
-// if (dt > 100 && tile.count < 8 * maxNodes)
+// if (dt > 100 && tile.getCount() < 8 * maxNodes)
// System.out.println("took " + dt + " ms to find " + sols[i] + " for "+ parts[i]);
if (sols[i] == null){
countBad++;
@@ -743,10 +733,9 @@ public class SplittableDensityArea {
}
private boolean checkSize(Tile tile) {
- if (tile.height > maxTileHeight|| tile.width > maxTileWidth)
- return false;
- return true;
+ return tile.height <= maxTileHeight && tile.width <= maxTileWidth;
}
+
/**
* Get a first solution and search for better ones until
* either a nice solution is found or no improvement was
@@ -757,7 +746,7 @@ public class SplittableDensityArea {
private Solution solveRectangularArea(Tile startTile){
// start values for optimization process: we make little steps towards a good solution
// spread = 7;
- if (startTile.count == 0)
+ if (startTile.getCount() == 0)
return new Solution(maxNodes);
searchLimit = startSearchLimit;
minNodes = Math.max(Math.min((long)(0.05 * maxNodes), extraDensityInfo.getNodeCount()), 1);
@@ -771,10 +760,10 @@ public class SplittableDensityArea {
if (maxAspectRatio < NICE_MAX_ASPECT_RATIO)
maxAspectRatio = NICE_MAX_ASPECT_RATIO;
}
- goodSolutions = new HashMap<>();
+ goodSolutions = new HashMap<>(GOOD_SOL_INIT_SIZE);
goodRatio = 0.5;
TileMetaInfo smiStart = new TileMetaInfo(startTile, null, null);
- if (startTile.count < 300 * maxNodes && (checkSize(startTile) || startTile.count < 10 * maxNodes) ){
+ if (startTile.getCount() < 300 * maxNodes && (checkSize(startTile) || startTile.getCount() < 10 * maxNodes) ){
searchAll = true;
}
@@ -878,10 +867,7 @@ public class SplittableDensityArea {
}
private void resetCaches(){
- knownBad = new HashSet<>();
-
-// incomplete = new LinkedHashMap<>();
-// System.out.println("resetting caches");
+ knownBad = new HashSet<>(50_000);
}
private void printFinishMsg(Solution solution){
@@ -918,7 +904,7 @@ public class SplittableDensityArea {
boolean fits = true;
for (Tile tile : sol.getTiles()) {
- if (tile.count == 0)
+ if (tile.getCount() == 0)
continue;
if (tile.verify() == false)
throw new SplitFailedException("found invalid tile");
@@ -937,13 +923,13 @@ public class SplittableDensityArea {
Area area = new Area(r.y,r.x,(int)r.getMaxY(),(int)r.getMaxX());
if (!beQuiet){
String note;
- if (tile.count > maxNodes)
+ if (tile.getCount() > maxNodes)
note = " but is already at the minimum size so can't be split further";
else
note = "";
- long percentage = 100 * tile.count / maxNodes;
+ long percentage = 100 * tile.getCount() / maxNodes;
System.out.println("Area " + currMapId++ + " covers " + area
- + " and contains " + tile.count + " nodes (" + percentage + " %)" + note);
+ + " and contains " + tile.getCount() + " nodes (" + percentage + " %)" + note);
}
result.add(area);
}
@@ -969,9 +955,7 @@ public class SplittableDensityArea {
}
double ratio = tile.getAspectRatio();
IntArrayList tests = new IntArrayList();
- if (ratio < 1.0/32 || ratio > 32)
- return tests;
- if (ratio < 1.0/16 && axis == AXIS_HOR || ratio > 16 && axis == AXIS_VERT)
+ if (ratio < 1.0 / 32 || ratio > 32 || ratio < 1.0 / 16 && axis == AXIS_HOR || ratio > 16 && axis == AXIS_VERT)
return tests;
int start = (axis == AXIS_HOR) ? tile.findValidStartX(smi) : tile.findValidStartY(smi);
int end = (axis == AXIS_HOR) ? tile.findValidEndX(smi) : tile.findValidEndY(smi);
@@ -985,30 +969,27 @@ public class SplittableDensityArea {
for (int i = 5; i > 1; --i)
tests.add(start + range / i);
}
- else if (tile.count < maxNodes * 4 && range > 256){
+ else if (tile.getCount() < maxNodes * 4 && range > 256){
// large tile with rather few nodes, allow more tests
int step = (range) / 20;
for (int pos = start; pos <= end; pos += step)
tests.add(pos);
}
- else if (tile.count > maxNodes * 4){
+ else if (tile.getCount() > maxNodes * 4){
int step = range / 7; // 7 turned out to be a good value here
if (step < 1)
step = 1;
for (int pos = start; pos <= end; pos += step)
tests.add(pos);
- if (tests.isEmpty() && end > start){
- tests.add((start + end) / 2);
- }
} else {
// this will be one of the last splits
- long nMax = tile.count / minNodes;
- if (nMax * minNodes < tile.count)
+ long nMax = tile.getCount() / minNodes;
+ if (nMax * minNodes < tile.getCount())
nMax++;
- long nMin = tile.count / maxNodes;
- if (nMin * maxNodes < tile.count)
+ long nMin = tile.getCount() / maxNodes;
+ if (nMin * maxNodes < tile.getCount())
nMin++;
- if (nMin > 2 && nMin * maxNodes - minNodes < tile.count && ratio > 0.125 && ratio < 8){
+ if (nMin > 2 && nMin * maxNodes - minNodes < tile.getCount() && ratio > 0.125 && ratio < 8){
// count is near (but below) a multiple of max-nodes, we have to test all candidates
// to make sure that we don't miss a good split
return (axis == AXIS_HOR) ? tile.genXTests(smi) : tile.genYTests(smi);
@@ -1016,16 +997,22 @@ public class SplittableDensityArea {
if (nMax == 2 || nMin == 2){
tests.add((axis == AXIS_HOR) ? tile.findHorizontalMiddle(smi) : tile.findVerticalMiddle(smi));
int pos = (axis == AXIS_HOR) ? tile.findFirstXHigher(smi, minNodes) + 1 : tile.findFirstYHigher(smi, minNodes) + 1;
- if (tests.contains(pos) == false)
+ if (tests.get(0) != pos)
tests.add(pos);
pos = (axis == AXIS_HOR) ? tile.findFirstXHigher(smi, maxNodes) : tile.findFirstYHigher(smi, maxNodes);
if (tests.contains(pos) == false)
tests.add(pos);
} else {
- if (nMax != 3)
- tests.add((axis == AXIS_HOR) ? tile.findHorizontalMiddle(smi) : tile.findVerticalMiddle(smi));
- tests.add(start);
- tests.add(end);
+ if (range == 0) {
+ tests.add(start);
+ } else {
+ if (nMax != 3)
+ tests.add((axis == AXIS_HOR) ? tile.findHorizontalMiddle(smi) : tile.findVerticalMiddle(smi));
+ if (tests.contains(start) == false)
+ tests.add(start);
+ if (tests.contains(end) == false)
+ tests.add(end);
+ }
}
}
return tests;
diff --git a/src/uk/me/parabola/splitter/Tile.java b/src/uk/me/parabola/splitter/solver/Tile.java
similarity index 94%
rename from src/uk/me/parabola/splitter/Tile.java
rename to src/uk/me/parabola/splitter/solver/Tile.java
index 71efebc..8fea186 100644
--- a/src/uk/me/parabola/splitter/Tile.java
+++ b/src/uk/me/parabola/splitter/solver/Tile.java
@@ -11,9 +11,11 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.solver;
import it.unimi.dsi.fastutil.ints.IntArrayList;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Utils;
import java.awt.Rectangle;
@@ -23,8 +25,6 @@ import java.awt.Rectangle;
* It contains the sum of all nodes in this area and has methods to
* help splitting it into smaller parts.
*
- * It extends java.awt.Rectangle because that implements a useful
- * hashCode method.
* We want to keep the memory footprint of this class small as
* many instances are kept in maps.
* @author GerdP
@@ -35,8 +35,7 @@ import java.awt.Rectangle;
*
*/
private final EnhancedDensityMap densityInfo;
- public final long count;
-// int bestSplit;
+ private final long count;
/**
* Create tile for whole density map.
@@ -83,11 +82,15 @@ import java.awt.Rectangle;
// }
}
+ public long getCount() {
+ return count;
+ }
+
/**
* @return true if the saved count value is correct.
*/
public boolean verify(){
- return (count == calcCount());
+ return (getCount() == calcCount());
}
public IntArrayList genXTests(TileMetaInfo smi) {
@@ -191,13 +194,13 @@ import java.awt.Rectangle;
* @return true if the above fields are usable
*/
public int findHorizontalMiddle(TileMetaInfo smi) {
- if (count == 0 || width < 2)
+ if (getCount() == 0 || width < 2)
smi.setHorMidPos(0);
- else {
+ else if (smi.getHorMidPos() < 0) {
int start = (smi.getFirstNonZeroX() > 0) ? smi.getFirstNonZeroX() : 0;
long sum = 0;
long lastSum = 0;
- long target = count/2;
+ long target = getCount()/2;
for (int pos = start; pos <= width; pos++) {
lastSum = sum;
@@ -228,12 +231,12 @@ import java.awt.Rectangle;
* @return true if the above fields are usable
*/
public int findVerticalMiddle(TileMetaInfo smi) {
- if (count == 0 || height < 2)
+ if (getCount() == 0 || height < 2)
smi.setVertMidPos(0);
- else {
+ else if (smi.getVertMidPos() < 0) {
long sum = 0;
long lastSum;
- long target = count/2;
+ long target = getCount()/2;
int start = (smi.getFirstNonZeroY() > 0) ? smi.getFirstNonZeroY() : 0;
for (int pos = start; pos <= height; pos++) {
lastSum = sum;
@@ -277,14 +280,14 @@ import java.awt.Rectangle;
for (int pos = splitX; pos < end; pos++) {
sum += getColSum(pos, smi.getColSums());
}
- sum = count - sum;
+ sum = getCount() - sum;
}
- if (sum < smi.getMinNodes() || count - sum < smi.getMinNodes())
+ if (sum < smi.getMinNodes() || getCount() - sum < smi.getMinNodes())
return false;
assert splitX > 0 && splitX < width;
Tile[] parts = smi.getParts();
parts[0] = new Tile(densityInfo, x, y, splitX, height, sum);
- parts[1] = new Tile(densityInfo, x + splitX, y, width - splitX,height, count - sum);
+ parts[1] = new Tile(densityInfo, x + splitX, y, width - splitX,height, getCount() - sum);
assert smi.getParts()[0].width + smi.getParts()[1].width == this.width;
return true;
}
@@ -309,15 +312,15 @@ import java.awt.Rectangle;
for (int pos = splitY; pos < end; pos++) {
sum += getRowSum(pos, smi.getRowSums());
}
- sum = count - sum;
+ sum = getCount() - sum;
}
- if (sum < smi.getMinNodes() || count - sum < smi.getMinNodes())
+ if (sum < smi.getMinNodes() || getCount() - sum < smi.getMinNodes())
return false;
assert splitY > 0 && splitY < height;
Tile[] parts = smi.getParts();
parts[0] = new Tile(densityInfo, x, y, width, splitY, sum);
- parts[1] = new Tile(densityInfo, x, y + splitY, width, height- splitY, count- sum);
+ parts[1] = new Tile(densityInfo, x, y + splitY, width, height- splitY, getCount()- sum);
assert parts[0].height + parts[1].height == this.height;
return true;
@@ -514,16 +517,16 @@ import java.awt.Rectangle;
assert minY <= maxY;
assert maxX >= 0;
assert maxY >= 0;
- long newCount = count;
+ long newCount = getCount();
int modWidth = maxX - minX + 1;
int modHeight = maxY - minY + 1;
if (densityInfo.getPolygonArea() != null){
if (modWidth != width || modHeight != height){
// tile was trimmed, try hard to avoid a new costly calculation of the count value
if (width == modWidth){
- newCount = count - sumRemovedRowCounts;
+ newCount = getCount() - sumRemovedRowCounts;
} else if (height == modHeight){
- newCount = count - sumRemovedColCounts;
+ newCount = getCount() - sumRemovedColCounts;
} else {
// System.out.printf("ouch: %d %d %d %d (%d) -> %d %d %d %d\n",x,y,width,height,count,minX,minY, maxX - minX + 1, maxY - minY + 1 );
return new Tile (densityInfo, new Rectangle(minX, minY, modWidth, modHeight));
@@ -604,11 +607,17 @@ import java.awt.Rectangle;
int polyXPos = densityInfo.getDensityMap().getBounds().getMinLong() + (x << shift);
return new Rectangle(polyXPos, polyYPos, width<<shift, height<<shift);
}
+
+ @Override
+ public int hashCode() {
+ int hash = x << 24 | y << 16 | width << 8 | height;
+ return hash;
+ }
@Override
public String toString(){
Area area = densityInfo.getDensityMap().getArea(x,y,width,height);
- return (area.toString() + " with " + Utils.format(count) + " nodes");
+ return (area.toString() + " with " + Utils.format(getCount()) + " nodes");
// StringBuilder sb = new StringBuilder();
// sb.append("(");
// sb.append(x);
diff --git a/src/uk/me/parabola/splitter/TileMetaInfo.java b/src/uk/me/parabola/splitter/solver/TileMetaInfo.java
similarity index 99%
rename from src/uk/me/parabola/splitter/TileMetaInfo.java
rename to src/uk/me/parabola/splitter/solver/TileMetaInfo.java
index 9248c69..fa5627f 100644
--- a/src/uk/me/parabola/splitter/TileMetaInfo.java
+++ b/src/uk/me/parabola/splitter/solver/TileMetaInfo.java
@@ -10,7 +10,7 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.solver;
import java.util.Arrays;
diff --git a/src/uk/me/parabola/splitter/Long2IntClosedMap.java b/src/uk/me/parabola/splitter/tools/Long2IntClosedMap.java
similarity index 97%
rename from src/uk/me/parabola/splitter/Long2IntClosedMap.java
rename to src/uk/me/parabola/splitter/tools/Long2IntClosedMap.java
index 067c503..58ed34d 100644
--- a/src/uk/me/parabola/splitter/Long2IntClosedMap.java
+++ b/src/uk/me/parabola/splitter/tools/Long2IntClosedMap.java
@@ -10,10 +10,12 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
- package uk.me.parabola.splitter;
+ package uk.me.parabola.splitter.tools;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
+import uk.me.parabola.splitter.SplitFailedException;
+import uk.me.parabola.splitter.Utils;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
@@ -33,7 +35,7 @@ import java.util.Arrays;
*
* @author GerdP
*/
-class Long2IntClosedMap implements Long2IntClosedMapFunction{
+public class Long2IntClosedMap implements Long2IntClosedMapFunction{
private static final long LOW_ID_MASK = 0x3fffffffL; // 30 bits
private static final long TOP_ID_MASK = ~LOW_ID_MASK;
private static final int TOP_ID_SHIFT = Long.numberOfTrailingZeros(TOP_ID_MASK);
diff --git a/src/uk/me/parabola/splitter/Long2IntClosedMapFunction.java b/src/uk/me/parabola/splitter/tools/Long2IntClosedMapFunction.java
similarity index 96%
rename from src/uk/me/parabola/splitter/Long2IntClosedMapFunction.java
rename to src/uk/me/parabola/splitter/tools/Long2IntClosedMapFunction.java
index d777148..732f937 100644
--- a/src/uk/me/parabola/splitter/Long2IntClosedMapFunction.java
+++ b/src/uk/me/parabola/splitter/tools/Long2IntClosedMapFunction.java
@@ -10,7 +10,7 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
- package uk.me.parabola.splitter;
+ package uk.me.parabola.splitter.tools;
import java.io.File;
import java.io.IOException;
@@ -19,7 +19,7 @@ import java.io.IOException;
* Stores long/int pairs. Only useful with data that is already in key-sorted order.
*
*/
-interface Long2IntClosedMapFunction {
+public interface Long2IntClosedMapFunction {
/**
* Add a new pair. The key must be higher than then any existing key in the map.
* @param key the key value
diff --git a/src/uk/me/parabola/splitter/OSMId2ObjectMap.java b/src/uk/me/parabola/splitter/tools/OSMId2ObjectMap.java
similarity index 94%
rename from src/uk/me/parabola/splitter/OSMId2ObjectMap.java
rename to src/uk/me/parabola/splitter/tools/OSMId2ObjectMap.java
index cc397b5..b454622 100644
--- a/src/uk/me/parabola/splitter/OSMId2ObjectMap.java
+++ b/src/uk/me/parabola/splitter/tools/OSMId2ObjectMap.java
@@ -10,7 +10,7 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.tools;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
@@ -34,13 +34,13 @@ public class OSMId2ObjectMap<V>{
private int size;
public OSMId2ObjectMap() {
- topMap = new Long2ObjectOpenHashMap<Int2ObjectOpenHashMap<V>>();
+ topMap = new Long2ObjectOpenHashMap<>();
}
public V put(long key, V object){
long topId = key >> TOP_ID_SHIFT;
Int2ObjectOpenHashMap<V> midMap = topMap.get(topId);
if (midMap == null){
- midMap = new Int2ObjectOpenHashMap<V>();
+ midMap = new Int2ObjectOpenHashMap<>();
topMap.put(topId, midMap);
}
int midId = (int)(key & LOW_ID_MASK);
diff --git a/src/uk/me/parabola/splitter/tools/SparseBitSet.java b/src/uk/me/parabola/splitter/tools/SparseBitSet.java
new file mode 100644
index 0000000..de32643
--- /dev/null
+++ b/src/uk/me/parabola/splitter/tools/SparseBitSet.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2012, Gerd Petermann
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 or
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+package uk.me.parabola.splitter.tools;
+
+import it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import uk.me.parabola.splitter.SplitFailedException;
+
+/**
+ * A partly BitSet implementation optimized for memory when used to store very
+ * large values with a high likelihood that the stored values build groups like
+ * e.g. the OSM node IDs. The keys are divided into 3 parts. The 1st part is
+ * stored in a small hash map. The 2nd part is stored in larger hash maps
+ * addressing long values. The 3rd part (6 bits) is stored in the long value
+ * addressed by the upper maps. author GerdP
+ */
+public class SparseBitSet {
+ private static final long MID_ID_MASK = 0x7ffffff;
+ private static final long TOP_ID_MASK = ~MID_ID_MASK;
+ private static final int LOW_MASK = 63;
+ private static final int TOP_ID_SHIFT = Long.numberOfTrailingZeros(TOP_ID_MASK);
+ private static final int MID_ID_SHIFT = Integer.numberOfTrailingZeros(~LOW_MASK);
+
+ private Long2ObjectOpenHashMap<Int2LongOpenHashMap> topMap = new Long2ObjectOpenHashMap<>();
+ private long bitCount;
+
+ public void set(long key) {
+ long topId = key >> TOP_ID_SHIFT;
+ Int2LongOpenHashMap midMap = topMap.get(topId);
+ if (midMap == null) {
+ midMap = new Int2LongOpenHashMap();
+ topMap.put(topId, midMap);
+ }
+ int midId = (int) ((key & MID_ID_MASK) >> MID_ID_SHIFT);
+ long chunk = midMap.get(midId);
+ int bitPos = (int) (key & LOW_MASK);
+ long val = 1L << (bitPos - 1);
+ if (chunk != 0) {
+ if ((chunk & val) != 0)
+ return;
+ val |= chunk;
+ }
+ midMap.put(midId, val);
+ ++bitCount;
+ }
+
+ public void clear(long key) {
+ long topId = key >> TOP_ID_SHIFT;
+ Int2LongOpenHashMap midMap = topMap.get(topId);
+ if (midMap == null)
+ return;
+ int midId = (int) ((key & MID_ID_MASK) >> MID_ID_SHIFT);
+ long chunk = midMap.get(midId);
+ if (chunk == 0)
+ return;
+ int bitPos = (int) (key & LOW_MASK);
+ long val = 1L << (bitPos - 1);
+ if ((chunk & val) == 0)
+ return;
+ chunk &= ~val;
+ if (chunk == 0) {
+ midMap.remove(midId);
+ if (midMap.isEmpty()) {
+ topMap.remove(topId);
+ }
+ } else
+ midMap.put(midId, chunk);
+ --bitCount;
+ }
+
+ public boolean get(long key) {
+ long topId = key >> TOP_ID_SHIFT;
+ Int2LongOpenHashMap midMap = topMap.get(topId);
+ if (midMap == null)
+ return false;
+ int midId = (int) ((key & MID_ID_MASK) >> MID_ID_SHIFT);
+ long chunk = midMap.get(midId);
+ if (chunk == 0)
+ return false;
+ int bitPos = (int) (key & LOW_MASK);
+ long val = 1L << (bitPos - 1);
+ return ((chunk & val) != 0);
+ }
+
+ public void clear() {
+ topMap.clear();
+ bitCount = 0;
+ }
+
+ public int cardinality() {
+ if (bitCount > Integer.MAX_VALUE)
+ throw new SplitFailedException("cardinality too high for int " + bitCount);
+ return (int) bitCount;
+ }
+}
diff --git a/src/uk/me/parabola/splitter/tools/SparseLong2IntMap.java b/src/uk/me/parabola/splitter/tools/SparseLong2IntMap.java
new file mode 100644
index 0000000..4d6fd1e
--- /dev/null
+++ b/src/uk/me/parabola/splitter/tools/SparseLong2IntMap.java
@@ -0,0 +1,798 @@
+/*
+ * Copyright (c) 2016, Gerd Petermann
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter.tools;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import it.unimi.dsi.fastutil.Hash;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
+import uk.me.parabola.splitter.Utils;
+
+/**
+ * Intended usage: Store many pairs of OSM id and an int which represents the position.
+ * Optimized for low memory requirements and inserts in sequential order.
+ * Don't use this for a rather small number of pairs.
+ *
+ * Inspired by SparseInt2ShortMapInline.
+ *
+ * A HashMap is used to address large vectors which address chunks. The HashMap
+ * is the only part that stores long values, and it will be very small as long
+ * as long as input is normal OSM data and not something with random numbers.
+ * A chunk stores up to CHUNK_SIZE values. A separately stored bit-mask is used
+ * to separate used and unused entries in the chunk. Thus, the chunk length
+ * depends on the number of used entries, not on the highest used entry.
+ * Such a "masked encoded" entry may look like this
+ * v1,v1,v1,v1,v1,v1,v2,v2,v2,v2,v1,v1,v1,v1,v1,u,?,?,...}
+ * v1,v2: values stored in the chunk
+ * u: "unassigned" value
+ * ?: anything
+ *
+ * After applying Run Length Encryption on this the chunk looks like this:
+ * {v1,6,v2,4,v1,5,?,?,?}
+ *
+ * Fortunately, OSM data is distributed in a way that a lot of chunks contain
+ * just one distinct value.
+ *
+ * Since we have OSM ids with 64 bits, we have to divide the key into 3 parts:
+ * 37 bits for the value that is stored in the HashMap.
+ * 21 bits for the chunkId (this gives the required length of a large vector)
+ * 6 bits for the position in the chunk
+ *
+ * The chunkId identifies the position of a 32-bit value (stored in the large vector).
+ * A chunk is stored in a chunkStore which is a 3-dimensional array.
+ * We group chunks of equally length together in stores of 64 entries.
+ * To find the right position of a new chunk, we need three values: x,y, and z.
+ * x is the length of the chunk (the number of required bytes) (1-256, we store the value decremented by 1 to have 0-255)
+ * y is the position of the store (0-1048575), we store a value incremented by 1 to ensure a non-zero value for used chunks
+ * z is the position of the chunk within the store. (0-15)
+ * The maximum values for these three values are chosen so that we can place them
+ * together into one int (32 bits).
+ */
+
+public final class SparseLong2IntMap {
+ /** the part of the key that is not saved in the top HashMap. */
+ private static final long CHUNK_ID_MASK = 0x7ffffffL;
+ private static final long TOP_ID_MASK = ~CHUNK_ID_MASK; // the part of the key that is saved in the top HashMap
+ private static final int TOP_ID_SHIFT = Long.numberOfTrailingZeros(TOP_ID_MASK);
+
+ private static final int CHUNK_SIZE = 64; // 64 = 1<< 6 (last 6 bits of the key)
+ /** Number of entries addressed by one topMap entry. */
+ private static final int LARGE_VECTOR_SIZE = (int) (CHUNK_ID_MASK / CHUNK_SIZE + 1);
+ private static final int CHUNK_STORE_BITS_FOR_Z = 5; // must fit into byte field
+ private static final int CHUNK_STORE_BITS_FOR_Y = Integer.numberOfTrailingZeros(LARGE_VECTOR_SIZE) - CHUNK_STORE_BITS_FOR_Z + 1;
+ private static final int CHUNK_STORE_BITS_FOR_X = 8; // values 1 .. 256 are stored as 0..255
+
+ private static final int CHUNK_STORE_ELEMS = 1 << CHUNK_STORE_BITS_FOR_Z;
+ private static final int CHUNK_STORE_X_MASK = (1 << CHUNK_STORE_BITS_FOR_X) - 1;
+ private static final int CHUNK_STORE_Y_MASK = (1 << CHUNK_STORE_BITS_FOR_Y) - 1;
+ private static final int CHUNK_STORE_Z_MASK = (1 << CHUNK_STORE_BITS_FOR_Z) - 1;
+ private static final int CHUNK_STORE_Y_SHIFT = CHUNK_STORE_BITS_FOR_X;
+ private static final int CHUNK_STORE_Z_SHIFT = CHUNK_STORE_BITS_FOR_X + CHUNK_STORE_BITS_FOR_Y;
+
+ private static final int MAX_Y_VAL = LARGE_VECTOR_SIZE / CHUNK_STORE_ELEMS + 1;
+ /** The part of the key that contains the offset in the chunk. */
+ private static final long CHUNK_OFFSET_MASK = CHUNK_SIZE - 1;
+ /** First 58 bits of a long. If this part of the key changes, a different chunk is needed. */
+ private static final long OLD_CHUNK_ID_MASK = ~CHUNK_OFFSET_MASK;
+
+ private static final long INVALID_CHUNK_ID = 1L; // must NOT be divisible by CHUNK_SIZE
+
+ /** What to return on unassigned keys. */
+ private int unassigned = Integer.MIN_VALUE;
+ private long size;
+ private long modCount;
+ private long oldModCount;
+
+ private long currentChunkId;
+ private final int [] currentChunk = new int[CHUNK_SIZE]; // stores the values in the real position
+ private final int [] maskedChunk = new int[CHUNK_SIZE]; // a chunk after applying the "mask encoding"
+ private final int[] tmpChunk = new int[CHUNK_SIZE * 2]; // used for tests of compression methods
+ private static final int MAX_BYTES_FOR_RLE_CHUNK = CHUNK_SIZE * (Integer.BYTES + 1);
+ private final ByteBuffer bufEncoded = ByteBuffer.allocate(MAX_BYTES_FOR_RLE_CHUNK); // for the RLE-compressed chunk
+ private static final int MAX_STORED_BYTES_FOR_CHUNK = CHUNK_SIZE * Integer.BYTES;
+
+ // bit masks for the flag byte
+ private static final int FLAG_USED_BYTES_MASK = 0x03; // number of bytes - 1
+ private static final int FLAG_COMP_METHOD_DELTA = 0x10; // rest of vals are delta encoded, bias is first val
+ private static final int FLAG_COMP_METHOD_RLE = 0x80; // values are run length encoded
+
+ // for statistics
+ private final String dataDesc;
+
+ private int storedLengthOfCurrentChunk;
+ private int currentChunkIdInStore;
+
+ private Long2ObjectOpenHashMap<Mem> topMap;
+
+ static final long MAX_MEM = Runtime.getRuntime().maxMemory() / 1024 / 1024;
+ static final int POINTER_SIZE = (MAX_MEM < 32768) ? 4 : 8; // presuming that compressedOOps is enabled
+
+ private Integer bias1; // used for initial delta encoding
+
+ /**
+ * Helper class to manage memory for chunks.
+ * @author Gerd Petermann
+ *
+ */
+ static class Mem {
+ final long topId;
+ long estimatedBytes; // estimate value for the allocated bytes
+ final int[] largeVector;
+ byte[][][] chunkStore;
+ long[][][] maskStore;
+ final int[] freePosInStore;
+ /** maps chunks that can be reused. */
+ Int2ObjectOpenHashMap<IntArrayList> reusableChunks;
+
+ public Mem(long topID) {
+ this.topId = topID;
+ largeVector = new int[LARGE_VECTOR_SIZE];
+ chunkStore = new byte[MAX_STORED_BYTES_FOR_CHUNK][][];
+ maskStore = new long[MAX_STORED_BYTES_FOR_CHUNK][][];
+ freePosInStore = new int[MAX_STORED_BYTES_FOR_CHUNK];
+ reusableChunks = new Int2ObjectOpenHashMap<>(0);
+ estimatedBytes = LARGE_VECTOR_SIZE * Integer.BYTES
+ + (MAX_STORED_BYTES_FOR_CHUNK) * (2 * 8 + 1 * Integer.BYTES) + 4 * (24 + 16) + 190;
+ }
+
+ public void grow(int x) {
+ int oldCapacity = chunkStore[x].length;
+ int newCapacity = oldCapacity < 1024 ? oldCapacity * 2 : oldCapacity + (oldCapacity >> 1);
+ if (newCapacity >= MAX_Y_VAL)
+ newCapacity = MAX_Y_VAL;
+ if (newCapacity <= oldCapacity)
+ return;
+ resize(x, newCapacity);
+ }
+
+ private void resize(int x, int newCapacity) {
+ int oldCapacity = chunkStore[x].length;
+ if (newCapacity < oldCapacity)
+ assert chunkStore[x][newCapacity] == null;
+ chunkStore[x] = Arrays.copyOf(chunkStore[x], newCapacity);
+ maskStore[x] = Arrays.copyOf(maskStore[x], newCapacity);
+ // bytes for pointers seem to depends on the capacity ?
+ estimatedBytes += (newCapacity - oldCapacity) * (2 * 8); // pointer-pointer
+ }
+
+ public void startStore(int x) {
+ chunkStore[x] = new byte[2][];
+ maskStore[x] = new long[2][];
+ estimatedBytes += 2 * (24 + 2 * (8)); // pointer-pointer
+ }
+ }
+
+ /**
+ * Helper class to store the various positions in the multi-tier data structure.
+ * @author Gerd Petermann
+ *
+ */
+ private static class MemPos{
+ final int x,y,z;
+ final Mem mem;
+ final int largeVectorIndex;
+
+ MemPos(Mem mem, int largeVectorIndex, int x, int y, int z) {
+ this.mem = mem;
+ this.largeVectorIndex = largeVectorIndex;
+ this.x = x;
+ this.y = y;
+ this.z = z;
+ }
+
+ public long getMask() {
+ return mem.maskStore[x][y][z];
+ }
+
+ public ByteBuffer getInBuf() {
+ int chunkLen = x + 1;
+ int startPos = z * chunkLen + 1;
+ byte[] store = mem.chunkStore[x][y];
+ return ByteBuffer.wrap(store, startPos, chunkLen);
+ }
+ }
+
+ /**
+ * A map that stores pairs of (OSM) IDs and int values identifying the
+ * areas in which the object with the ID occurs.
+ * @param dataDesc
+ */
+ public SparseLong2IntMap(String dataDesc) {
+ long reserve = (1L << CHUNK_STORE_BITS_FOR_Y - 1) * CHUNK_SIZE - LARGE_VECTOR_SIZE;
+ assert reserve > 0;
+ this.dataDesc = dataDesc;
+ System.out.println(dataDesc + " Map: uses " + this.getClass().getSimpleName());
+ clear();
+ }
+
+ /**
+ * Count how many of the lowest X bits in mask are set.
+ *
+ * @return how many of the lowest X bits in mask are set.
+ */
+ private static int countUnder(final long mask, final int lowest) {
+ return Long.bitCount(mask & ((1L << lowest) - 1));
+ }
+
+ /**
+ * Put an int value into the byte buffer using the given number of bytes.
+ * @param buf the buffer
+ * @param val the int value to store
+ * @param bytesToUse the number of bytes to use
+ */
+ static void putVal(final ByteBuffer buf, final int val, final int bytesToUse) {
+ switch (bytesToUse) {
+ case 1:
+ assert val >= Byte.MIN_VALUE && val <= Byte.MAX_VALUE : val + " of out Byte range";
+ buf.put((byte) val);
+ break;
+ case 2:
+ buf.putShort((short) val);
+ break;
+ case 3: // put3
+ buf.put((byte) (val & 0xff));
+ buf.putShort((short) (val >> 8));
+ break;
+ default:
+ buf.putInt(val);
+ break;
+ }
+ }
+
+ /**
+ * Read an int value from the byte buffer using the given number of bytes.
+ * @param buf the byte buffer
+ * @param bytesToUse number of bytes (1 - 4)
+ * @return the integer value
+ */
+ static int getVal(final ByteBuffer buf, final int bytesToUse) {
+ switch (bytesToUse) {
+ case 1:
+ return buf.get();
+ case 2:
+ return buf.getShort();
+ case 3:
+ byte b1 = buf.get();
+ short s = buf.getShort();
+ return (b1 & 0xff) | (s << 8);
+ default:
+ return buf.getInt();
+ }
+ }
+
+ /**
+ * Try to use Run Length Encoding (RLE) to compress the chunk stored in tmpWork. In most
+ * cases this works very well because chunks often have only one or two distinct values.
+ * The run length value is always between 1 and 64, which requires just one byte.
+ * If the stored values don't fit into a single byte, also try delta encoding (for values 2 .. n).
+ *
+ * @param numVals number of elements in the chunk, content of {@code maskedChunk} after that is undefined.
+ * @param minVal smallest value in maskedChunk
+ * @param maxVal highest value in maskedChunk
+ *
+ */
+ private void chunkCompress(int numVals, long minVal, long maxVal) {
+ assert minVal != maxVal;
+ int start = maskedChunk[0];
+ int bytesFor1st = calcNeededBytes(start, start);
+ int bytesForRest = calcNeededBytes(minVal, maxVal);
+ int flag = 0;
+ int bias2 = 0;
+ int prefixLen = 1;
+
+ if (bytesForRest > 1) {
+ // check if all values are in a small range which allows
+ int test = testBias(minVal, maxVal, start);
+ if (test < bytesForRest) {
+ bytesForRest = test;
+ flag |= FLAG_COMP_METHOD_DELTA;
+ bias2 = start;
+ }
+ }
+ int lenNoCompress = Math.min(MAX_STORED_BYTES_FOR_CHUNK, prefixLen + bytesFor1st + (numVals-1) * bytesForRest);
+ int lenRLE = prefixLen;
+
+ int numCounts = 0;
+ int opos = 0;
+ for (int i = 0; i < numVals; i++) {
+ int runLength = 1;
+ while (i+1 < numVals && maskedChunk[i] == maskedChunk[i+1]) {
+ runLength++;
+ i++;
+ }
+ numCounts++;
+ tmpChunk[opos++] = maskedChunk[i];
+ tmpChunk[opos++] = runLength;
+ lenRLE += (numCounts == 1 ? bytesFor1st : bytesForRest) + 1;
+ if (lenRLE >= lenNoCompress)
+ break;
+ }
+ flag |= (bytesForRest - 1) << 2 | (bytesFor1st - 1);
+
+ boolean storeFlag = true;
+ if (lenRLE < lenNoCompress) {
+ flag |= FLAG_COMP_METHOD_RLE;
+ } else {
+ // check unlikely special case to make sure that encoded len is below 256
+ // don't write flag if all values are stored with 4 bytes
+ storeFlag = (lenNoCompress < MAX_STORED_BYTES_FOR_CHUNK);
+ }
+ if (storeFlag) {
+ bufEncoded.put((byte) flag);
+ }
+ int bytesToUse = bytesFor1st;
+ int bias = 0;
+ if (lenRLE < lenNoCompress) {
+ int pos = 0;
+ while (pos < opos) {
+ putVal(bufEncoded, tmpChunk[pos++] - bias, bytesToUse);
+ bufEncoded.put((byte) tmpChunk[pos++]); // run length
+ if (pos == 2) {
+ bytesToUse = bytesForRest;
+ bias = bias2;
+ }
+ }
+ assert lenRLE == bufEncoded.position();
+ } else {
+ for (int i = 0; i < numVals; i++) {
+ putVal(bufEncoded, maskedChunk[i] - bias, bytesToUse);
+ if (i == 0) {
+ bytesToUse = bytesForRest;
+ bias = bias2;
+ }
+ }
+ assert lenNoCompress == bufEncoded.position();
+ }
+
+ return;
+ }
+
+ /**
+ * Try to compress the data in currentChunk and store the result in the chunkStore.
+ */
+ private void saveCurrentChunk() {
+ if (currentChunkId == INVALID_CHUNK_ID || modCount == oldModCount)
+ return;
+ // step 1: mask encoding
+ long mask = 0;
+ int simpleLen = 0;
+ long elementMask = 1L;
+ if (bias1 == null) {
+ bias1 = findBias1(); // very simple heuristics
+ }
+ int maxVal = Integer.MIN_VALUE;
+ int minVal = Integer.MAX_VALUE;
+ for (int i = 0; i < CHUNK_SIZE; i++) {
+ if (currentChunk[i] != unassigned) {
+ int v = currentChunk[i] - bias1; // apply bias
+ if (minVal > v)
+ minVal = v;
+ if (maxVal < v)
+ maxVal = v;
+ maskedChunk[simpleLen++] = v;
+ mask |= elementMask;
+ }
+ elementMask <<= 1;
+ }
+ bufEncoded.clear();
+ if (minVal == maxVal) {
+ // nice: single value chunk
+ int bytesFor1st = calcNeededBytes(minVal, maxVal);
+ if (bytesFor1st > 2) {
+ bufEncoded.put((byte) (bytesFor1st - 1)); // flag byte
+ }
+ putVal(bufEncoded, maskedChunk[0], bytesFor1st);
+ } else {
+ chunkCompress(simpleLen, minVal, maxVal);
+ }
+ bufEncoded.flip();
+ putChunk(currentChunkId, bufEncoded, mask);
+ }
+
+ /**
+ * Calculate the needed bytes for the range minVal..maxVal if bias is substructed.
+ * @param minVal start of range (including)
+ * @param maxVal end of range (including)
+ * @param bias the bias value to test
+ * @return the number of needed bytes
+ */
+ private static int testBias(long minVal, long maxVal, int bias) {
+ long minVal2 = minVal - bias;
+ long maxVal2 = maxVal - bias;
+ int test = calcNeededBytes(minVal2, maxVal2);
+ return test;
+ }
+
+ /**
+ * Calculate the number of bytes needed to encode values in the given range.
+ * @param minVal smallest value
+ * @param maxVal highest value
+ * @return number of needed bytes
+ */
+ static int calcNeededBytes (long minVal, long maxVal) {
+ if (minVal >= Byte.MIN_VALUE && maxVal <= Byte.MAX_VALUE) {
+ return Byte.BYTES;
+ } else if (minVal >= Short.MIN_VALUE && maxVal <= Short.MAX_VALUE) {
+ return Short.BYTES;
+ } else if (minVal >= -0x00800000 && maxVal <= 0x7fffff) {
+ return 3;
+ }
+ return Integer.BYTES;
+ }
+
+ private int findBias1() {
+ int minVal = Integer.MAX_VALUE;
+ int maxVal = Integer.MIN_VALUE;
+ for (int i = 0; i < CHUNK_SIZE; i++) {
+ if (currentChunk[i] != unassigned) {
+ if (minVal > currentChunk[i])
+ minVal = currentChunk[i];
+ if (maxVal < currentChunk[i])
+ maxVal = currentChunk[i];
+ }
+ }
+ int avg = minVal + (maxVal-minVal) / 2;
+ if (avg < 0 && avg - Integer.MIN_VALUE < Byte.MAX_VALUE)
+ return Integer.MIN_VALUE + Byte.MAX_VALUE;
+ if (avg > 0 && Integer.MAX_VALUE - avg < Byte.MAX_VALUE)
+ return Integer.MAX_VALUE - Byte.MAX_VALUE;
+ return avg;
+ }
+
+ public boolean containsKey(long key) {
+ return get(key) != unassigned;
+ }
+
+ public int put(long key, int val) {
+ if (val == unassigned) {
+ throw new IllegalArgumentException("Cannot store the value that is reserved as being unassigned. val=" + val);
+ }
+ long chunkId = key & OLD_CHUNK_ID_MASK;
+ if (currentChunkId != chunkId){
+ // we need a different chunk
+ replaceCurrentChunk(key);
+ }
+ int chunkoffset = (int) (key & CHUNK_OFFSET_MASK);
+ int out = currentChunk[chunkoffset];
+ currentChunk[chunkoffset] = val;
+ if (out == unassigned)
+ size++;
+ if (out != val)
+ modCount++;
+ return out;
+ }
+
+ /**
+ * Either decode the encoded chunk data into target or extract a single value.
+ * @param mp the MemPos instance with information about the store
+ * @param targetChunk if not null, data will be decoded into this buffer
+ * @param chunkOffset gives the wanted element (targetChunk must be null)
+ * @return
+ */
+ private int decodeStoredChunk (MemPos mp, int[] targetChunk, int chunkOffset) {
+ int chunkLen = mp.x + 1;
+ ByteBuffer inBuf = mp.getInBuf();
+
+ int flag = 0;
+ int bytesToUse = Integer.BYTES; // assume worst case
+ if (chunkLen == MAX_STORED_BYTES_FOR_CHUNK) {
+ // special case: no flag is written if we have the max. size
+ } else if (chunkLen <= 2) {
+ bytesToUse = chunkLen;
+ } else {
+ flag = inBuf.get();
+ bytesToUse = (flag & FLAG_USED_BYTES_MASK) + 1;
+ }
+ int bias = bias1;
+ int start = bias + getVal(inBuf, bytesToUse);
+ boolean singleValueChunk = (chunkLen <= 2 || chunkLen == 1 + bytesToUse);
+
+ if (targetChunk == null && singleValueChunk) {
+ return start;
+ }
+ long chunkMask = mp.getMask();
+ int index = CHUNK_SIZE + 1;
+ if (targetChunk == null) {
+ // we only want to retrieve one value for the index
+ index = countUnder(chunkMask, chunkOffset);
+ if (index == 0 )
+ return start;
+ }
+ int mPos = 0;
+ maskedChunk[mPos++] = start;
+ // rest of values might be encoded with different number of bytes
+ if (chunkLen != MAX_STORED_BYTES_FOR_CHUNK) {
+ bytesToUse = ((flag >> 2) & FLAG_USED_BYTES_MASK) + 1;
+ if ((flag & FLAG_COMP_METHOD_DELTA) != 0) {
+ bias = start;
+ }
+ }
+ int val = start;
+ if (targetChunk == null && (flag & FLAG_COMP_METHOD_RLE) == 0) {
+ // use shortcut, we can calculate the position of the wanted value
+ inBuf.position(inBuf.position() + (index-1) * bytesToUse);
+ return val = bias + getVal(inBuf, bytesToUse);
+ }
+ // loop through the values
+ while (inBuf.hasRemaining()) {
+ if ((flag & FLAG_COMP_METHOD_RLE) != 0) {
+ int runLength = inBuf.get();
+ index -= runLength - 1;
+ if (index <= 0)
+ return val;
+ while (--runLength > 0) {
+ maskedChunk[mPos++] = val;
+ }
+ if (!inBuf.hasRemaining())
+ break;
+ }
+ val = bias + getVal(inBuf, bytesToUse);
+ if (--index <= 0)
+ return val;
+ maskedChunk[mPos++] = val;
+
+ }
+ if (targetChunk != null) {
+ int j = 0;
+ int opos = 0;
+ while (chunkMask != 0) {
+ if ((chunkMask & 1) != 0) {
+ targetChunk[opos] = maskedChunk[j];
+ if (!singleValueChunk) {
+ j++;
+ }
+ }
+ opos++;
+ chunkMask >>>= 1;
+ }
+ }
+ return unassigned;
+ }
+
+ /**
+ * Use the various bit masks to extract the position of the chunk in the store.
+ * @param key the key for which we want the chunk
+ * @return the filled MemPos instance or null if the chunk is not in the store.
+ */
+ private MemPos getMemPos(long key) {
+ long topID = (key >> TOP_ID_SHIFT);
+ Mem mem = topMap.get(topID);
+ if (mem == null)
+ return null;
+ int chunkid = (int) (key & CHUNK_ID_MASK) / CHUNK_SIZE;
+
+ int idx = mem.largeVector[chunkid]; // performance bottleneck: produces many cache misses
+ if (idx == 0)
+ return null;
+ int x = idx & CHUNK_STORE_X_MASK;
+ int y = (idx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK;
+ y--; // we store the y value incremented by 1
+ assert y < LARGE_VECTOR_SIZE;
+ int z = (idx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK;
+ return new MemPos(mem, idx, x, y, z);
+ }
+
+ /**
+ * Check if we already have a chunk for the given key. If no,
+ * fill currentChunk with default value, else with the saved
+ * chunk.
+ * @param key the key for which we need the current chunk
+ */
+ private void replaceCurrentChunk(long key) {
+
+ saveCurrentChunk();
+ Arrays.fill(currentChunk, unassigned);
+ oldModCount = modCount;
+ currentChunkId = key & OLD_CHUNK_ID_MASK;
+ storedLengthOfCurrentChunk = 0;
+ currentChunkIdInStore = 0;
+ MemPos mp = getMemPos(key);
+ if (mp == null)
+ return;
+
+ currentChunkIdInStore = mp.largeVectorIndex;
+ storedLengthOfCurrentChunk = mp.x;
+ decodeStoredChunk(mp, currentChunk, -1);
+ }
+
+ /**
+ * Returns the value to which the given key is mapped or the {@code unassigned} value.
+ * @param key the key
+ * @return the value to which the given key is mapped or the {@code unassigned} value
+ */
+ public int get(long key){
+ long chunkId = key & OLD_CHUNK_ID_MASK;
+ int chunkoffset = (int) (key & CHUNK_OFFSET_MASK);
+
+ if (currentChunkId == chunkId) {
+ return currentChunk[chunkoffset];
+ }
+ MemPos mp = getMemPos(key);
+ if (mp == null)
+ return unassigned;
+
+ long chunkMask = mp.getMask();
+ long elementmask = 1L << chunkoffset;
+ if ((chunkMask & elementmask) == 0)
+ return unassigned; // not in chunk
+ // the map contains the key, decode it
+ return decodeStoredChunk(mp, null, chunkoffset);
+ }
+
+ public void clear() {
+ topMap = new Long2ObjectOpenHashMap<>(Hash.DEFAULT_INITIAL_SIZE, Hash.VERY_FAST_LOAD_FACTOR);
+
+ Arrays.fill(currentChunk, 0);
+ Arrays.fill(maskedChunk, 0);
+ storedLengthOfCurrentChunk = 0;
+ currentChunkIdInStore = 0;
+ currentChunkId = INVALID_CHUNK_ID;
+ bias1 = null;
+ size = 0;
+ // test();
+ }
+
+ public long size() {
+ return size;
+ }
+
+ public int defaultReturnValue() {
+ return unassigned;
+ }
+
+ public void defaultReturnValue(int arg0) {
+ unassigned = arg0;
+ }
+
+ private void putChunk(long key, ByteBuffer bb, long mask) {
+ long topID = key >> TOP_ID_SHIFT;
+ Mem mem = topMap.get(topID);
+ if (mem == null) {
+ mem = new Mem(topID);
+ topMap.put(topID, mem);
+ }
+
+ int chunkid = (int) (key & CHUNK_ID_MASK) / CHUNK_SIZE;
+ int len = bb.limit();
+ int x = len - 1;
+ if (storedLengthOfCurrentChunk > 0) {
+ // this is a rewrite, add the previously used chunk to the reusable list
+ IntArrayList reusableChunk = mem.reusableChunks.get(storedLengthOfCurrentChunk);
+ if (reusableChunk == null) {
+ reusableChunk = new IntArrayList(8);
+ mem.reusableChunks.put(storedLengthOfCurrentChunk, reusableChunk);
+ mem.estimatedBytes += 8 * Integer.BYTES + 24 + Integer.BYTES + POINTER_SIZE + 16; // for the IntArrayList instance
+ mem.estimatedBytes += 20; // estimate for the hash map entry
+ }
+ reusableChunk.add(currentChunkIdInStore);
+ }
+ if (mem.chunkStore[x] == null) {
+ mem.startStore(x);
+ }
+ IntArrayList reusableChunk = mem.reusableChunks.get(x);
+ int y, z;
+ byte[] store;
+ if (reusableChunk != null && !reusableChunk.isEmpty()) {
+ int reusedIdx = reusableChunk.removeInt(reusableChunk.size() - 1);
+ y = (reusedIdx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK;
+ y--; // we store the y value incremented by 1
+ z = (reusedIdx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK;
+ store = mem.chunkStore[x][y];
+ } else {
+ y = ++mem.freePosInStore[x] / CHUNK_STORE_ELEMS;
+ if (y >= mem.chunkStore[x].length)
+ mem.grow(x);
+ if (mem.chunkStore[x][y] == null) {
+ int numElems = len * CHUNK_STORE_ELEMS + 1;
+ mem.chunkStore[x][y] = new byte[numElems];
+ mem.estimatedBytes += 24 + (numElems) * Byte.BYTES;
+ int padding = 8 - (numElems * Byte.BYTES % 8);
+ if (padding < 8)
+ mem.estimatedBytes += padding;
+ mem.maskStore[x][y] = new long[CHUNK_STORE_ELEMS];
+ mem.estimatedBytes += 24 + CHUNK_STORE_ELEMS * Long.BYTES;
+ }
+ store = mem.chunkStore[x][y];
+ z = store[0]++;
+ }
+
+ mem.maskStore[x][y][z] = mask;
+ ByteBuffer storeBuf = ByteBuffer.wrap(store, z * len + 1, len);
+ storeBuf.put(bb);
+
+ // calculate the position in the large vector
+ y++; // we store the y value incremented by 1
+ assert x < 1 << CHUNK_STORE_BITS_FOR_X;
+ assert y < 1 << CHUNK_STORE_BITS_FOR_Y;
+ assert z < 1 << CHUNK_STORE_BITS_FOR_Z;
+ int idx = (z & CHUNK_STORE_Z_MASK) << CHUNK_STORE_Z_SHIFT
+ | (y & CHUNK_STORE_Y_MASK) << CHUNK_STORE_Y_SHIFT
+ | (x & CHUNK_STORE_X_MASK);
+
+ assert idx != 0;
+ mem.largeVector[chunkid] = idx;
+ }
+
+ /**
+ * calculate and print performance values regarding memory.
+ */
+ public void stats(int msgLevel) {
+ long totalBytes = currentChunk.length * Integer.BYTES;
+ long totalChunks = 1; // current chunk
+
+ if (size() == 0){
+ System.out.println(dataDesc + " Map is empty");
+ return;
+ }
+ int[] all = new int[MAX_STORED_BYTES_FOR_CHUNK];
+ int memCount = 1;
+
+ for (Mem mem : topMap.values()) {
+ for (int x = 0; x < mem.freePosInStore.length; x++) {
+ if (mem.freePosInStore[x] == 0)
+ continue;
+ all[x] += mem.freePosInStore[x];
+ if (msgLevel >= 1) {
+ System.out.println("mem store no: " + memCount + " len: " + (x+1) + " " + mem.freePosInStore[x]);
+ }
+ memCount++;
+ totalChunks += mem.freePosInStore[x];
+ }
+ totalBytes += mem.estimatedBytes;
+ }
+// if (msgLevel >= 0) {
+// for (int x = 0; x < all.length; x++) {
+// if (all[x] != 0)
+// System.out.println("len: " + (x+1) + " " + all[x]);
+// }
+// }
+ float bytesPerKey = (size()==0) ? 0: (float)(totalBytes*100 / size()) / 100;
+ System.out.println(dataDesc + " Map: " + Utils.format(size()) + " stored long/int pairs require ca. " +
+ bytesPerKey + " bytes per pair. " +
+ totalChunks + " chunks are used, the avg. number of values in one "+CHUNK_SIZE+"-chunk is " +
+ ((totalChunks==0) ? 0 :(size() / totalChunks)) + ".");
+ System.out.println(dataDesc + " Map details: bytes ~" + Utils.format(totalBytes/1024/1024) + " MB, including " +
+ topMap.size() + " array(s) with " + LARGE_VECTOR_SIZE * Integer.BYTES/1024/1024 + " MB");
+ System.out.println();
+ }
+
+
+ /*
+ void test(){
+ int[] yVals = { 0, 1, 2, MAX_Y_VAL - 2, MAX_Y_VAL - 1, MAX_Y_VAL };
+ for (int z = 0; z < 64; z++){
+ for (int y : yVals){
+ for (int x=0; x < 64; x++){
+ int idx = (z & CHUNK_STORE_Z_MASK)<<CHUNK_STORE_Z_SHIFT
+ | (y & CHUNK_STORE_Y_MASK)<< CHUNK_STORE_Y_SHIFT
+ | (x & CHUNK_STORE_X_MASK);
+ // extract
+ int x2 = idx & CHUNK_STORE_X_MASK;
+ int y2 = (idx >> CHUNK_STORE_Y_SHIFT) & CHUNK_STORE_Y_MASK;
+ int z2 = (idx >> CHUNK_STORE_Z_SHIFT) & CHUNK_STORE_Z_MASK;
+ assert x == x2;
+ assert y == y2;
+ assert z == z2;
+ }
+ }
+ }
+ }
+ */
+}
+
+
+
diff --git a/src/uk/me/parabola/splitter/AbstractOSMWriter.java b/src/uk/me/parabola/splitter/writer/AbstractOSMWriter.java
similarity index 70%
rename from src/uk/me/parabola/splitter/AbstractOSMWriter.java
rename to src/uk/me/parabola/splitter/writer/AbstractOSMWriter.java
index d9a4919..57fa104 100644
--- a/src/uk/me/parabola/splitter/AbstractOSMWriter.java
+++ b/src/uk/me/parabola/splitter/writer/AbstractOSMWriter.java
@@ -11,15 +11,23 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.writer;
import java.awt.Rectangle;
import java.io.File;
+import java.io.IOException;
+
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Element;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.Relation;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.Way;
public abstract class AbstractOSMWriter implements OSMWriter{
- final static int REMOVE_VERSION = 1;
- final static int FAKE_VERSION = 2;
- final static int KEEP_VERSION = 3;
+ public static final int REMOVE_VERSION = 1;
+ public static final int FAKE_VERSION = 2;
+ public static final int KEEP_VERSION = 3;
protected final Area bounds;
protected final Area extendedBounds;
protected final File outputDir;
@@ -67,4 +75,14 @@ public abstract class AbstractOSMWriter implements OSMWriter{
public Rectangle getBBox(){
return bbox;
}
+
+ public void write (Element element) throws IOException {
+ if (element instanceof Node) {
+ write((Node) element);
+ } else if (element instanceof Way) {
+ write((Way) element);
+ } else if (element instanceof Relation) {
+ write((Relation) element);
+ }
+ }
}
diff --git a/src/uk/me/parabola/splitter/writer/BinaryMapWriter.java b/src/uk/me/parabola/splitter/writer/BinaryMapWriter.java
new file mode 100644
index 0000000..54725c5
--- /dev/null
+++ b/src/uk/me/parabola/splitter/writer/BinaryMapWriter.java
@@ -0,0 +1,519 @@
+/*
+ * Copyright (c) 2009, Francisco Moraes
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter.writer;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Element;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.Relation;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.Version;
+import uk.me.parabola.splitter.Way;
+import uk.me.parabola.splitter.Relation.Member;
+import crosby.binary.BinarySerializer;
+import crosby.binary.Osmformat;
+import crosby.binary.StringTable;
+import crosby.binary.Osmformat.DenseInfo;
+import crosby.binary.Osmformat.Relation.MemberType;
+import crosby.binary.file.BlockOutputStream;
+import crosby.binary.file.FileBlock;
+
+public class BinaryMapWriter extends AbstractOSMWriter {
+
+ protected PBFSerializer serializer;
+
+ private BlockOutputStream output;
+
+ protected boolean useDense = true;
+
+ protected boolean headerWritten = false;
+
+ private class PBFSerializer extends BinarySerializer {
+
+ public PBFSerializer(BlockOutputStream output) {
+ super(output);
+ configBatchLimit(1000);
+ // omit_metadata = true;
+ }
+
+ /**
+ * Base class containing common code needed for serializing each type of
+ * primitives.
+ */
+ private abstract class Prim<T extends Element> {
+ /** Queue that tracks the list of all primitives. */
+ ArrayList<T> contents = new ArrayList<>();
+
+ /**
+ * Add to the queue.
+ *
+ * @param item
+ * The entity to add
+ */
+ public void add(T item) {
+ contents.add(item);
+ }
+
+ /**
+ * Add all of the tags of all entities in the queue to the string
+ * table.
+ */
+ public void addStringsToStringtable() {
+ StringTable stable = getStringTable();
+ for (T i : contents) {
+ Iterator<Element.Tag> tags = i.tagsIterator();
+ while (tags.hasNext()) {
+ Element.Tag tag = tags.next();
+ stable.incr(tag.getKey());
+ stable.incr(tag.getValue());
+ }
+ if (!omit_metadata) {
+ // stable.incr(i.getUser().getName());
+ }
+ }
+ }
+
+ // private static final int MAXWARN = 100;
+
+ public void serializeMetadataDense(DenseInfo.Builder b, List<? extends Element> entities) {
+ if (omit_metadata) {
+ return;
+ }
+
+ // long lasttimestamp = 0, lastchangeset = 0;
+ // int lastuserSid = 0, lastuid = 0;
+ // StringTable stable = serializer.getStringTable();
+ // for(Element e : entities) {
+ //
+ // if(e.getUser() == OsmUser.NONE && warncount < MAXWARN) {
+ // LOG
+ // .warning("Attention: Data being output lacks metadata. Please
+ // use omitmetadata=true");
+ // warncount++;
+ // }
+ // int uid = e.getUser().getId();
+ // int userSid = stable.getIndex(e.getUser().getName());
+ // int timestamp = (int)(e.getTimestamp().getTime() /
+ // date_granularity);
+ // int version = e.getVersion();
+ // long changeset = e.getChangesetId();
+ //
+ // b.addVersion(version);
+ // b.addTimestamp(timestamp - lasttimestamp);
+ // lasttimestamp = timestamp;
+ // b.addChangeset(changeset - lastchangeset);
+ // lastchangeset = changeset;
+ // b.addUid(uid - lastuid);
+ // lastuid = uid;
+ // b.addUserSid(userSid - lastuserSid);
+ // lastuserSid = userSid;
+ // }
+
+ for (Element e : entities) {
+ int version = getWriteVersion(e);
+ if (versionMethod != KEEP_VERSION || version == 0)
+ version = 1; // JOSM requires a fake version
+ b.addVersion(version);
+ b.addTimestamp(0);
+ b.addChangeset(0);
+ b.addUid(0);
+ b.addUserSid(0);
+ }
+ }
+
+ public Osmformat.Info.Builder serializeMetadata(Element e) {
+ // StringTable stable = serializer.getStringTable();
+ Osmformat.Info.Builder b = Osmformat.Info.newBuilder();
+ // if(!omit_metadata) {
+ // if(e.getUser() == OsmUser.NONE && warncount < MAXWARN) {
+ // LOG
+ // .warning("Attention: Data being output lacks metadata. Please
+ // use omitmetadata=true");
+ // warncount++;
+ // }
+ // if(e.getUser() != OsmUser.NONE) {
+ // b.setUid(e.getUser().getId());
+ // b.setUserSid(stable.getIndex(e.getUser().getName()));
+ // }
+ // b.setTimestamp((int)(e.getTimestamp().getTime() /
+ // date_granularity));
+ // b.setVersion(e.getVersion());
+ // b.setChangeset(e.getChangesetId());
+ // }
+ if (versionMethod != REMOVE_VERSION) {
+ int version = getWriteVersion(e);
+ b.setVersion(version);
+ b.setTimestamp(0);
+ b.setChangeset(0);
+ b.setUid(0);
+ b.setUserSid(0);
+ }
+ return b;
+ }
+ }
+
+ private class NodeGroup extends Prim<Node> implements PrimGroupWriterInterface {
+
+ public Osmformat.PrimitiveGroup serialize() {
+ if (useDense)
+ return serializeDense();
+ return serializeNonDense();
+ }
+
+ /**
+ * Serialize all nodes in the 'dense' format.
+ */
+ public Osmformat.PrimitiveGroup serializeDense() {
+ if (contents.size() == 0) {
+ return null;
+ }
+ // System.out.format("%d Dense ",nodes.size());
+ Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup.newBuilder();
+ StringTable stable = serializer.getStringTable();
+
+ long lastlat = 0, lastlon = 0, lastid = 0;
+ Osmformat.DenseNodes.Builder bi = Osmformat.DenseNodes.newBuilder();
+ boolean doesBlockHaveTags = false;
+ // Does anything in this block have tags?
+ for (Node i : contents) {
+ doesBlockHaveTags = doesBlockHaveTags || (i.tagsIterator().hasNext());
+ }
+ if (!omit_metadata) {
+ Osmformat.DenseInfo.Builder bdi = Osmformat.DenseInfo.newBuilder();
+ serializeMetadataDense(bdi, contents);
+ bi.setDenseinfo(bdi);
+ }
+
+ for (Node i : contents) {
+ long id = i.getId();
+ int lat = mapDegrees(i.getLat());
+ int lon = mapDegrees(i.getLon());
+ bi.addId(id - lastid);
+ lastid = id;
+ bi.addLon(lon - lastlon);
+ lastlon = lon;
+ bi.addLat(lat - lastlat);
+ lastlat = lat;
+
+ // Then we must include tag information.
+ if (doesBlockHaveTags) {
+ Iterator<Element.Tag> tags = i.tagsIterator();
+ while (tags.hasNext()) {
+ Element.Tag t = tags.next();
+ bi.addKeysVals(stable.getIndex(t.getKey()));
+ bi.addKeysVals(stable.getIndex(t.getValue()));
+ }
+ bi.addKeysVals(0); // Add delimiter.
+ }
+ }
+ builder.setDense(bi);
+ return builder.build();
+ }
+
+ /**
+ * Serialize all nodes in the non-dense format.
+ *
+ * @param parentbuilder
+ * Add to this PrimitiveBlock.
+ */
+ public Osmformat.PrimitiveGroup serializeNonDense() {
+ if (contents.size() == 0) {
+ return null;
+ }
+ // System.out.format("%d Nodes ",nodes.size());
+ StringTable stable = serializer.getStringTable();
+ Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup.newBuilder();
+ for (Node i : contents) {
+ long id = i.getId();
+ int lat = mapDegrees(i.getLat());
+ int lon = mapDegrees(i.getLon());
+ Osmformat.Node.Builder bi = Osmformat.Node.newBuilder();
+ bi.setId(id);
+ bi.setLon(lon);
+ bi.setLat(lat);
+ Iterator<Element.Tag> tags = i.tagsIterator();
+ while (tags.hasNext()) {
+ Element.Tag t = tags.next();
+ bi.addKeys(stable.getIndex(t.getKey()));
+ bi.addVals(stable.getIndex(t.getValue()));
+ }
+ if (!omit_metadata) {
+ bi.setInfo(serializeMetadata(i));
+ }
+ builder.addNodes(bi);
+ }
+ return builder.build();
+ }
+
+ }
+
+ private class WayGroup extends Prim<Way> implements PrimGroupWriterInterface {
+ public Osmformat.PrimitiveGroup serialize() {
+ if (contents.size() == 0) {
+ return null;
+ }
+
+ // System.out.format("%d Ways ",contents.size());
+ StringTable stable = serializer.getStringTable();
+ Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup.newBuilder();
+ for (Way i : contents) {
+ Osmformat.Way.Builder bi = Osmformat.Way.newBuilder();
+ bi.setId(i.getId());
+ long lastid = 0;
+ for (long j : i.getRefs()) {
+ long id = j;
+ bi.addRefs(id - lastid);
+ lastid = id;
+ }
+ Iterator<Element.Tag> tags = i.tagsIterator();
+ while (tags.hasNext()) {
+ Element.Tag t = tags.next();
+ bi.addKeys(stable.getIndex(t.getKey()));
+ bi.addVals(stable.getIndex(t.getValue()));
+ }
+ if (!omit_metadata) {
+ bi.setInfo(serializeMetadata(i));
+ }
+ builder.addWays(bi);
+ }
+ return builder.build();
+ }
+ }
+
+ private class RelationGroup extends Prim<Relation> implements PrimGroupWriterInterface {
+ public void addStringsToStringtable() {
+ StringTable stable = serializer.getStringTable();
+ super.addStringsToStringtable();
+ for (Relation i : contents) {
+ for (Member j : i.getMembers()) {
+ stable.incr(j.getRole());
+ }
+ }
+ }
+
+ public Osmformat.PrimitiveGroup serialize() {
+ if (contents.size() == 0) {
+ return null;
+ }
+
+ // System.out.format("%d Relations ",contents.size());
+ StringTable stable = serializer.getStringTable();
+ Osmformat.PrimitiveGroup.Builder builder = Osmformat.PrimitiveGroup.newBuilder();
+ for (Relation i : contents) {
+ Osmformat.Relation.Builder bi = Osmformat.Relation.newBuilder();
+ bi.setId(i.getId());
+ Member[] arr = new Member[i.getMembers().size()];
+ i.getMembers().toArray(arr);
+ long lastid = 0;
+ for (Member j : i.getMembers()) {
+ long id = j.getRef();
+ bi.addMemids(id - lastid);
+ lastid = id;
+ if (j.getType().equals("node")) {
+ bi.addTypes(MemberType.NODE);
+ } else if (j.getType().equals("way")) {
+ bi.addTypes(MemberType.WAY);
+ } else if (j.getType().equals("relation")) {
+ bi.addTypes(MemberType.RELATION);
+ } else {
+ assert (false); // Software bug: Unknown entity.
+ }
+ bi.addRolesSid(stable.getIndex(j.getRole()));
+ }
+
+ Iterator<Element.Tag> tags = i.tagsIterator();
+ while (tags.hasNext()) {
+ Element.Tag t = tags.next();
+ bi.addKeys(stable.getIndex(t.getKey()));
+ bi.addVals(stable.getIndex(t.getValue()));
+ }
+ if (!omit_metadata) {
+ bi.setInfo(serializeMetadata(i));
+ }
+ builder.addRelations(bi);
+ }
+ return builder.build();
+ }
+ }
+
+ /* One list for each type */
+ protected WayGroup ways;
+
+ protected NodeGroup nodes;
+
+ protected RelationGroup relations;
+
+ protected Processor processor = new Processor();
+
+ /**
+ * Buffer up events into groups that are all of the same type, or all of
+ * the same length, then process each buffer.
+ */
+ public class Processor {
+
+ /**
+ * Check if we've reached the batch size limit and process the batch
+ * if we have.
+ */
+ public void checkLimit() {
+ total_entities++;
+ if (++batch_size < batch_limit) {
+ return;
+ }
+ switchTypes();
+ processBatch();
+ }
+
+ public void process(Node node) {
+ if (nodes == null) {
+ writeEmptyHeaderIfNeeded();
+ // Need to switch types.
+ switchTypes();
+ nodes = new NodeGroup();
+ }
+ nodes.add(node);
+ checkLimit();
+ }
+
+ public void process(Way way) {
+ if (ways == null) {
+ writeEmptyHeaderIfNeeded();
+ switchTypes();
+ ways = new WayGroup();
+ }
+ ways.add(way);
+ checkLimit();
+ }
+
+ public void process(Relation relation) {
+ if (relations == null) {
+ writeEmptyHeaderIfNeeded();
+ switchTypes();
+ relations = new RelationGroup();
+ }
+ relations.add(relation);
+ checkLimit();
+ }
+ }
+
+ /**
+ * At the end of this function, all of the lists of unprocessed 'things'
+ * must be null
+ */
+ protected void switchTypes() {
+ if (nodes != null) {
+ groups.add(nodes);
+ nodes = null;
+ } else if (ways != null) {
+ groups.add(ways);
+ ways = null;
+ } else if (relations != null) {
+ groups.add(relations);
+ relations = null;
+ } else {
+ return; // No data. Is this an empty file?
+ }
+ }
+
+ /** Write empty header block when there's no bounds entity. */
+ public void writeEmptyHeaderIfNeeded() {
+ if (headerWritten) {
+ return;
+ }
+ Osmformat.HeaderBlock.Builder headerblock = Osmformat.HeaderBlock.newBuilder();
+ finishHeader(headerblock);
+ }
+ }
+
+ public BinaryMapWriter(Area bounds, File outputDir, int mapId, int extra) {
+ super(bounds, outputDir, mapId, extra);
+ }
+
+ public void initForWrite() {
+ String filename = String.format(Locale.ROOT, "%08d.osm.pbf", mapId);
+ try {
+ output = new BlockOutputStream(new FileOutputStream(new File(outputDir, filename)));
+ serializer = new PBFSerializer(output);
+ writeHeader();
+ } catch (IOException e) {
+ System.out.println("Could not open or write file header. Reason: " + e.getMessage());
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void writeHeader() {
+ Osmformat.HeaderBlock.Builder headerblock = Osmformat.HeaderBlock.newBuilder();
+
+ Osmformat.HeaderBBox.Builder pbfBbox = Osmformat.HeaderBBox.newBuilder();
+ pbfBbox.setLeft(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMinLong())));
+ pbfBbox.setBottom(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMinLat())));
+ pbfBbox.setRight(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMaxLong())));
+ pbfBbox.setTop(serializer.mapRawDegrees(Utils.toDegrees(bounds.getMaxLat())));
+ headerblock.setBbox(pbfBbox);
+
+ finishHeader(headerblock);
+ }
+
+ /**
+ * Write the header fields that are always needed.
+ *
+ * @param headerblock
+ * Incomplete builder to complete and write.
+ */
+ public void finishHeader(Osmformat.HeaderBlock.Builder headerblock) {
+ headerblock.setWritingprogram("splitter-r" + Version.VERSION);
+ headerblock.addRequiredFeatures("OsmSchema-V0.6");
+ if (useDense) {
+ headerblock.addRequiredFeatures("DenseNodes");
+ }
+ Osmformat.HeaderBlock message = headerblock.build();
+ try {
+ output.write(FileBlock.newInstance("OSMHeader", message.toByteString(), null));
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to write OSM header.", e);
+ }
+ headerWritten = true;
+ }
+
+ public void finishWrite() {
+ try {
+ serializer.switchTypes();
+ serializer.processBatch();
+ serializer.close();
+ serializer = null;
+ } catch (IOException e) {
+ System.out.println("Could not write end of file: " + e);
+ }
+ }
+
+ public void write(Node node) {
+ serializer.processor.process(node);
+ }
+
+ public void write(Way way) {
+ serializer.processor.process(way);
+ }
+
+ public void write(Relation relation) {
+ serializer.processor.process(relation);
+ }
+}
diff --git a/src/uk/me/parabola/splitter/O5mMapWriter.java b/src/uk/me/parabola/splitter/writer/O5mMapWriter.java
similarity index 98%
rename from src/uk/me/parabola/splitter/O5mMapWriter.java
rename to src/uk/me/parabola/splitter/writer/O5mMapWriter.java
index faac821..bf048c6 100644
--- a/src/uk/me/parabola/splitter/O5mMapWriter.java
+++ b/src/uk/me/parabola/splitter/writer/O5mMapWriter.java
@@ -10,7 +10,7 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.writer;
import it.unimi.dsi.fastutil.longs.LongArrayList;
@@ -27,6 +27,12 @@ import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Element;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.Relation;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.Way;
import uk.me.parabola.splitter.Relation.Member;
/**
@@ -225,7 +231,6 @@ public class O5mMapWriter extends AbstractOSMWriter{
long delta = node.getId() - lastNodeId; lastNodeId = node.getId();
writeSignedNum(delta, stream);
writeVersion(node, stream);
- //TODO : write version
int o5Lon = (int)(node.getLon() * FACTOR);
int o5Lat = (int)(node.getLat() * FACTOR);
int deltaLon = o5Lon - lastLon; lastLon = o5Lon;
@@ -509,7 +514,7 @@ public class O5mMapWriter extends AbstractOSMWriter{
}
do {
numberConversionBuf[cntBytes++] = (byte)(part | 0x80);
- u >>= 7;
+ u >>>= 7;
part = (int)(u & 0x7f);
} while(part !=u);
numberConversionBuf[cntBytes++] = (byte)(part);
diff --git a/src/uk/me/parabola/splitter/OSMWriter.java b/src/uk/me/parabola/splitter/writer/OSMWriter.java
similarity index 80%
rename from src/uk/me/parabola/splitter/OSMWriter.java
rename to src/uk/me/parabola/splitter/writer/OSMWriter.java
index 8bf312f..76eda39 100644
--- a/src/uk/me/parabola/splitter/OSMWriter.java
+++ b/src/uk/me/parabola/splitter/writer/OSMWriter.java
@@ -11,11 +11,17 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.writer;
import java.awt.Rectangle;
import java.io.IOException;
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Element;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.Relation;
+import uk.me.parabola.splitter.Way;
+
public interface OSMWriter {
/**
* @return the bounds of the area (excluding the overlap)
@@ -46,4 +52,6 @@ public interface OSMWriter {
public abstract void write(Way way) throws IOException;
public abstract void write(Relation rel) throws IOException;
+
+ public abstract void write(Element el) throws IOException;
}
diff --git a/src/uk/me/parabola/splitter/OSMXMLWriter.java b/src/uk/me/parabola/splitter/writer/OSMXMLWriter.java
similarity index 84%
rename from src/uk/me/parabola/splitter/OSMXMLWriter.java
rename to src/uk/me/parabola/splitter/writer/OSMXMLWriter.java
index 3d7d45c..e2f7ed7 100644
--- a/src/uk/me/parabola/splitter/OSMXMLWriter.java
+++ b/src/uk/me/parabola/splitter/writer/OSMXMLWriter.java
@@ -11,10 +11,15 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.writer;
import it.unimi.dsi.fastutil.longs.LongArrayList;
-
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Element;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.Relation;
+import uk.me.parabola.splitter.Utils;
+import uk.me.parabola.splitter.Way;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -35,7 +40,6 @@ public class OSMXMLWriter extends AbstractOSMWriter{
);
private Writer writer;
-
public OSMXMLWriter(Area bounds, File outputDir, int mapId, int extra) {
super(bounds, outputDir, mapId, extra);
@@ -58,7 +62,7 @@ public class OSMXMLWriter extends AbstractOSMWriter{
private void writeHeader() throws IOException {
writeString("<?xml version='1.0' encoding='UTF-8'?>\n");
String apiVersion = (versionMethod == REMOVE_VERSION) ? "version='0.5'" : "version='0.6'";
-
+
writeString("<osm " + apiVersion + " generator='splitter' upload='false'>\n");
writeString("<bounds minlat='");
@@ -99,7 +103,7 @@ public class OSMXMLWriter extends AbstractOSMWriter{
} else {
writeString("'/>\n");
}
-
+
}
public void write(Way way) throws IOException {
@@ -128,7 +132,8 @@ public class OSMXMLWriter extends AbstractOSMWriter{
List<Relation.Member> memlist = rel.getMembers();
for (Relation.Member m : memlist) {
if (m.getType() == null || m.getRef() == 0) {
- System.err.println("Invalid relation member found in relation " + rel.getId() + ": member type=" + m.getType() + ", ref=" + m.getRef() + ", role=" + m.getRole() + ". Ignoring this member");
+ System.err.println("Invalid relation member found in relation " + rel.getId() + ": member type="
+ + m.getType() + ", ref=" + m.getRef() + ", role=" + m.getRole() + ". Ignoring this member");
continue;
}
writeString("<member type='");
@@ -162,26 +167,26 @@ public class OSMXMLWriter extends AbstractOSMWriter{
for (int i = 0; i < value.length(); i++) {
char c = value.charAt(i);
switch (c) {
- case '\'':
- writeString("'");
- break;
- case '&':
- writeString("&");
- break;
- case '<':
- writeString("<");
- break;
- case '\n':
- writeString("
");
- break;
- case '\r':
- writeString("
");
- break;
- case '\t':
- writeString(" ");
- break;
- default:
- writeChar(c);
+ case '\'':
+ writeString("'");
+ break;
+ case '&':
+ writeString("&");
+ break;
+ case '<':
+ writeString("<");
+ break;
+ case '\n':
+ writeString("
");
+ break;
+ case '\r':
+ writeString("
");
+ break;
+ case '\t':
+ writeString(" ");
+ break;
+ default:
+ writeChar(c);
}
}
}
@@ -217,9 +222,11 @@ public class OSMXMLWriter extends AbstractOSMWriter{
/** Write a double to full precision */
private void writeLongDouble(double value) throws IOException {
checkFlush(22);
- writeString(Double.toString(value));
+ writeString(Double.toString(value));
}
- /** Write a double truncated to OSM's 7 digits of precision
+
+ /**
+ * Write a double truncated to OSM's 7 digits of precision
*/
private void writeDouble(double value) throws IOException {
checkFlush(22);
@@ -227,23 +234,22 @@ public class OSMXMLWriter extends AbstractOSMWriter{
if (value < -200 || value > 200 || (value > -1 && value < 1))
writeString(numberFormat.format(value));
else {
- if (value < 0) {
- charBuf[index++] = '-'; // Write directly.
- value = -value;
- }
-
- int val = (int)Math.round(value*10000000);
- StringBuilder s = new StringBuilder(Integer.toString(val));
- s.insert(s.length()-7, '.');
- writeString(s.toString());
+ if (value < 0) {
+ charBuf[index++] = '-'; // Write directly.
+ value = -value;
+ }
+
+ int val = (int) Math.round(value * 10000000);
+ StringBuilder s = new StringBuilder(Integer.toString(val));
+ s.insert(s.length() - 7, '.');
+ writeString(s.toString());
}
}
-
+
private void writeLong(long value) throws IOException {
checkFlush(20);
writeString(Long.toString(value));
}
-
private void writeChar(char value) throws IOException {
checkFlush(1);
diff --git a/src/uk/me/parabola/splitter/PseudoOSMWriter.java b/src/uk/me/parabola/splitter/writer/PseudoOSMWriter.java
similarity index 83%
rename from src/uk/me/parabola/splitter/PseudoOSMWriter.java
rename to src/uk/me/parabola/splitter/writer/PseudoOSMWriter.java
index 938d2b7..c0021a4 100644
--- a/src/uk/me/parabola/splitter/PseudoOSMWriter.java
+++ b/src/uk/me/parabola/splitter/writer/PseudoOSMWriter.java
@@ -11,7 +11,12 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.writer;
+
+import uk.me.parabola.splitter.Area;
+import uk.me.parabola.splitter.Node;
+import uk.me.parabola.splitter.Relation;
+import uk.me.parabola.splitter.Way;
/**
* A do-nothing writer (used with --output=simulate)
diff --git a/src/uk/me/parabola/splitter/AbstractXppParser.java b/src/uk/me/parabola/splitter/xml/parser/AbstractXppParser.java
similarity index 93%
rename from src/uk/me/parabola/splitter/AbstractXppParser.java
rename to src/uk/me/parabola/splitter/xml/parser/AbstractXppParser.java
index bdecc95..6811daa 100644
--- a/src/uk/me/parabola/splitter/AbstractXppParser.java
+++ b/src/uk/me/parabola/splitter/xml/parser/AbstractXppParser.java
@@ -11,7 +11,7 @@
* General Public License for more details.
*/
-package uk.me.parabola.splitter;
+package uk.me.parabola.splitter.xml.parser;
import java.io.IOException;
import java.io.Reader;
@@ -31,7 +31,7 @@ public abstract class AbstractXppParser {
parser = factory.newPullParser();
}
- protected void setReader(Reader reader) throws XmlPullParserException {
+ public void setReader(Reader reader) throws XmlPullParserException {
parser.setInput(reader);
}
@@ -51,7 +51,7 @@ public abstract class AbstractXppParser {
return parser.getText();
}
- protected void parse() throws IOException, XmlPullParserException {
+ public void parse() throws IOException, XmlPullParserException {
boolean done = false;
int eventType = parser.getEventType();
do {
diff --git a/test/uk/me/parabola/splitter/TestAreaSet.java b/test/uk/me/parabola/splitter/TestAreaSet.java
new file mode 100644
index 0000000..e7babf2
--- /dev/null
+++ b/test/uk/me/parabola/splitter/TestAreaSet.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2012, Gerd Petermann
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 or
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import uk.me.parabola.splitter.AreaSet;
+
+/**
+ * Unit tests for the sparse BitSet implementation
+ */
+public class TestAreaSet {
+ private final int NUM = 10000;
+ private final int[] POS = { 1, 63, 64, 65, 4711, 78231};
+
+ public void allTests() {
+ testAreaSetRandom();
+ testAreaSetSequential();
+ }
+
+ @Test
+ public void testAreaSetSequential() {
+ AreaSet set = new AreaSet();
+ for (int i = 1; i < NUM; i++) {
+ Assert.assertEquals(set.get(i), false, "get(" + i + ")");
+ }
+ for (int i = 1; i < NUM; i++) {
+ set.set(i);
+ Assert.assertEquals(set.get(i), true, "get(" + i + ")");
+ }
+ Assert.assertEquals(set.cardinality(), NUM - 1, "cardinality() returns wrong value");
+ for (int i = 1; i < NUM; i++) {
+ set.clear(i);
+ Assert.assertEquals(set.get(i), false, "get(" + i + ")");
+ Assert.assertEquals(set.cardinality(), NUM - i - 1, "cardinality() returns wrong value");
+ }
+
+ }
+
+ @Test
+ public void testAreaSetRandom() {
+ AreaSet set = new AreaSet();
+ for (int i : POS) {
+ set.set(i);
+ Assert.assertEquals(set.get(i), true, "get(" + i + ")");
+ Assert.assertEquals(set.cardinality(), 1, "cardinality() returns wrong value");
+ set.clear(i);
+ Assert.assertEquals(set.get(i), false, "get(" + i + ")");
+ Assert.assertEquals(set.cardinality(), 0, "cardinality() returns wrong value");
+ }
+
+ }
+
+ @Test
+ public void testErr542() {
+ // crashed with r542
+ AreaSet set = new AreaSet();
+ set.set(1);
+ set.set(4);
+ set.set(7);
+ set.set(8);
+ set.set(9);
+ set.set(10);
+ set.set(11);
+ set.set(12);
+ set.set(13);
+ set.set(14);
+ set.set(15);
+ set.set(29);
+ set.clear(29);
+ set.clear(29);
+ }
+}
diff --git a/test/uk/me/parabola/splitter/TestCustomCollections.java b/test/uk/me/parabola/splitter/TestCustomCollections.java
deleted file mode 100644
index d053958..0000000
--- a/test/uk/me/parabola/splitter/TestCustomCollections.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Copyright (c) 2009, Chris Miller
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-
-package uk.me.parabola.splitter;
-
-import java.io.IOException;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-/**
- *
- */
-public class TestCustomCollections {
-
- //@Test(expectedExceptions = IllegalArgumentException.class)
- //public void testInit() {
- // new IntObjMap<String>(123, 0.5f);
- //}
-
- @Test
- public static void testLongShortMap() {
- testMap(new SparseLong2ShortMapInline("test"), 0L);
- testMap(new SparseLong2ShortMapInline("test"), -10000L);
- testMap(new SparseLong2ShortMapInline("test"), 1L << 35);
- testMap(new SparseLong2ShortMapInline("test"), -1L << 35);
- testMap(new SparseLong2ShortMapHuge("test"), 0L);
- testMap(new SparseLong2ShortMapHuge("test"), -10000L);
- testMap(new SparseLong2ShortMapHuge("test"), 1L << 35);
- testMap(new SparseLong2ShortMapHuge("test"), -1L << 35);
- }
-
- private static void testMap(SparseLong2ShortMapFunction map, long idOffset) {
- map.defaultReturnValue(Short.MIN_VALUE);
-
- for (short i = 1; i < 1000; i++) {
- int j = map.put(idOffset + i, i);
- Assert.assertEquals(j, Short.MIN_VALUE);
- Assert.assertEquals(map.size(), i);
- }
-
- for (short i = 1; i < 1000; i++) {
- boolean b = map.containsKey(idOffset + i);
- Assert.assertEquals(b, true);
- }
-
-
- for (short i = 1; i < 1000; i++) {
- Assert.assertEquals(map.get(idOffset + i), i);
- }
-
- // random read access
- for (short i = 1; i < 1000; i++) {
- short key = (short) Math.max(1, (Math.random() * 1000));
- Assert.assertEquals(map.get(idOffset + key), key);
- }
-
- for (short i = 1000; i < 2000; i++) {
- Assert.assertEquals(map.get(idOffset + i), Short.MIN_VALUE);
- }
- for (short i = 1000; i < 2000; i++) {
- boolean b = map.containsKey(idOffset + i);
- Assert.assertEquals(b, false);
- }
- for (short i = 1000; i < 1200; i++) {
- short j = map.put(idOffset + i, (short) 333);
- Assert.assertEquals(j, Short.MIN_VALUE);
- Assert.assertEquals(map.size(), i);
- }
- // random read access 2
- for (int i = 1; i < 1000; i++) {
- int key = 1000 + (short) (Math.random() * 200);
- Assert.assertEquals(map.get(idOffset + key), 333);
- }
-
-
- for (short i = -2000; i < -1000; i++) {
- Assert.assertEquals(map.get(idOffset + i), Short.MIN_VALUE);
- }
- for (short i = -2000; i < -1000; i++) {
- boolean b = map.containsKey(idOffset + i);
- Assert.assertEquals(b, false);
- }
- long mapSize = map.size();
- // seq. update existing records
- for (int i = 1; i < 1000; i++) {
- short j = map.put(idOffset + i, (short) (i+333));
- Assert.assertEquals(j, i);
- Assert.assertEquals(map.size(), mapSize);
- }
- // random read access 3, update existing entries
- for (int i = 1; i < 1000; i++) {
- short j = map.put(idOffset + i, (short) (i+555));
- Assert.assertEquals(true, j == i+333 | j == i+555);
- Assert.assertEquals(map.size(), mapSize);
- }
-
- Assert.assertEquals(map.get(idOffset + 123456), Short.MIN_VALUE);
- map.put(idOffset + 123456, (short) 999);
- Assert.assertEquals(map.get(idOffset + 123456), 999);
- map.put(idOffset + 123456, (short) 888);
- Assert.assertEquals(map.get(idOffset + 123456), 888);
-
- Assert.assertEquals(map.get(idOffset - 123456), Short.MIN_VALUE);
- map.put(idOffset - 123456, (short) 999);
- Assert.assertEquals(map.get(idOffset - 123456), 999);
- map.put(idOffset - 123456, (short) 888);
- Assert.assertEquals(map.get(idOffset - 123456), 888);
- map.put(idOffset + 3008, (short) 888);
- map.put(idOffset + 3009, (short) 888);
- map.put(idOffset + 3010, (short) 876);
- map.put(idOffset + 3011, (short) 876);
- map.put(idOffset + 3012, (short) 678);
- map.put(idOffset + 3013, (short) 678);
- map.put(idOffset + 3014, (short) 678);
- map.put(idOffset + 3015, (short) 678);
- map.put(idOffset + 3016, (short) 678);
- map.put(idOffset + 3017, (short) 678);
- map.put(idOffset + 4000, (short) 888);
- map.put(idOffset + 4001, (short) 888);
- map.put(idOffset + 4002, (short) 876);
- map.put(idOffset + 4003, (short) 876);
- // update the first one
- map.put(idOffset + 3008, (short) 889);
- // update the 2nd one
- map.put(idOffset + 4000, (short) 889);
- // add a very different key
- map.put(idOffset + 5000, (short) 889);
- map.put(idOffset + 5001, (short) 222);
- Assert.assertEquals(map.get(idOffset + 3008), 889);
- Assert.assertEquals(map.get(idOffset + 3009), 888);
- Assert.assertEquals(map.get(idOffset + 3010), 876);
- Assert.assertEquals(map.get(idOffset + 3011), 876);
- Assert.assertEquals(map.get(idOffset + 3012), 678);
- Assert.assertEquals(map.get(idOffset + 3013), 678);
- Assert.assertEquals(map.get(idOffset + 3014), 678);
- Assert.assertEquals(map.get(idOffset + 4000), 889);
- Assert.assertEquals(map.get(idOffset + 4001), 888);
- Assert.assertEquals(map.get(idOffset + 4002), 876);
- Assert.assertEquals(map.get(idOffset + 4003), 876);
- Assert.assertEquals(map.get(idOffset + 5000), 889);
- Assert.assertEquals(map.get(idOffset + 5001), 222);
- }
-
- @Test
- public static void testLong2IntMap() {
- testMap(new Long2IntClosedMap("test", 10000, -1));
- }
-
- private static void testMap(Long2IntClosedMapFunction map) {
- int val;
- for (int i = 1; i < 1000; i++) {
- int j = map.add((long)i*10, i);
- Assert.assertEquals(j, i-1);
- Assert.assertEquals(map.size(), i);
- }
-
- for (int i = 1; i < 1000; i++) {
- int pos = map.getKeyPos(i*10);
- Assert.assertEquals(i, pos+1);
- }
-
- for (int i = 1; i < 1000; i++) {
- val = map.getRandom(i*10);
- Assert.assertEquals(i, val);
- }
-
- try {
- map.switchToSeqAccess(null);
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- try{
- val = map.getRandom(5);
- } catch (IllegalArgumentException e){
- Assert.assertEquals(e.getMessage(), "random access on sequential-only map requested");
- }
- val = map.getSeq(5);
- Assert.assertEquals(val,-1);
- val = map.getSeq(10);
- Assert.assertEquals(val,1);
- val = map.getSeq(19);
- Assert.assertEquals(val,-1);
- val = map.getSeq(30);
- Assert.assertEquals(val,3);
-
- map.finish();
- }
-
-
-}
diff --git a/test/uk/me/parabola/splitter/TestSparseBitSet.java b/test/uk/me/parabola/splitter/TestSparseBitSet.java
deleted file mode 100644
index 7f3336d..0000000
--- a/test/uk/me/parabola/splitter/TestSparseBitSet.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (C) 2012, Gerd Petermann
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 3 or
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * General Public License for more details.
- */
-
-package uk.me.parabola.splitter;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-/**
- * Unit tests for the sparse BitSet implementaion
- */
-public class TestSparseBitSet{
- private final int NUM = 10000;
- private final long[] POS = {1, 63, 64, 65, 4711, 12345654321L};
- @Test
- public void testSparseBitSetSequential() {
- SparseBitSet sparseSet = new SparseBitSet();
- for (long i = 1; i < NUM; i++){
- Assert.assertEquals(sparseSet.get(i), false, "get("+i+")");
- }
- for (long i = 1; i < NUM; i++){
- sparseSet.set(i);
- Assert.assertEquals(sparseSet.get(i), true, "get("+i+")");
- }
- Assert.assertEquals(sparseSet.cardinality(), NUM-1, "cardinality() returns wrong value");
- for (long i = 1; i < NUM; i++){
- sparseSet.clear(i);
- Assert.assertEquals(sparseSet.get(i), false, "get("+i+")");
- Assert.assertEquals(sparseSet.cardinality(), NUM-i-1, "cardinality() returns wrong value");
- }
-
- }
- @Test
- public void testSparseBitSetRandom() {
- SparseBitSet sparseSet = new SparseBitSet();
- for (long i: POS){
- sparseSet.set(i);
- Assert.assertEquals(sparseSet.get(i), true, "get("+i+")");
- Assert.assertEquals(sparseSet.cardinality(), 1, "cardinality() returns wrong value");
- sparseSet.clear(i);
- Assert.assertEquals(sparseSet.get(i), false, "get("+i+")");
- Assert.assertEquals(sparseSet.cardinality(), 0, "cardinality() returns wrong value");
- }
-
- }
-}
diff --git a/test/uk/me/parabola/splitter/tools/TestCustomCollections.java b/test/uk/me/parabola/splitter/tools/TestCustomCollections.java
new file mode 100644
index 0000000..6827407
--- /dev/null
+++ b/test/uk/me/parabola/splitter/tools/TestCustomCollections.java
@@ -0,0 +1,322 @@
+/*
+ * Copyright (c) 2009, Chris Miller
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter.tools;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import uk.me.parabola.splitter.tools.Long2IntClosedMap;
+import uk.me.parabola.splitter.tools.Long2IntClosedMapFunction;
+import uk.me.parabola.splitter.tools.SparseLong2IntMap;
+
+/**
+ *
+ */
+public class TestCustomCollections {
+
+ //@Test(expectedExceptions = IllegalArgumentException.class)
+ //public void testInit() {
+ // new IntObjMap<String>(123, 0.5f);
+ //}
+
+ @Test
+ public static void testLong2IntMap() {
+ testMap(new Long2IntClosedMap("test", 10000, -1));
+ }
+
+ private static void testMap(Long2IntClosedMapFunction map) {
+ int val;
+ for (int i = 1; i < 1000; i++) {
+ int j = map.add((long)i*10, i);
+ Assert.assertEquals(j, i-1);
+ Assert.assertEquals(map.size(), i);
+ }
+
+ for (int i = 1; i < 1000; i++) {
+ int pos = map.getKeyPos(i*10);
+ Assert.assertEquals(i, pos+1);
+ }
+
+ for (int i = 1; i < 1000; i++) {
+ val = map.getRandom(i*10);
+ Assert.assertEquals(i, val);
+ }
+
+ try {
+ map.switchToSeqAccess(null);
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ try{
+ val = map.getRandom(5);
+ } catch (IllegalArgumentException e){
+ Assert.assertEquals(e.getMessage(), "random access on sequential-only map requested");
+ }
+ val = map.getSeq(5);
+ Assert.assertEquals(val,-1);
+ val = map.getSeq(10);
+ Assert.assertEquals(val,1);
+ val = map.getSeq(19);
+ Assert.assertEquals(val,-1);
+ val = map.getSeq(30);
+ Assert.assertEquals(val,3);
+
+ map.finish();
+ }
+
+ private static void testVals(SparseLong2IntMap map, long idOffset, List<Integer> vals) {
+ map.clear();
+ map.put(1, -12000);
+ long key = 128;
+ for (int val : vals) {
+ map.put(idOffset + key++, val);
+ }
+ map.put(1, 0); // trigger saving of chunk
+ key = 128;
+ for (int val : vals) {
+ Assert.assertEquals(map.get(idOffset + key++), val, "values " + vals.toString());
+ }
+ map.clear();
+ }
+
+ @Test
+ public static void testSparseLong2IntMap() {
+ ByteBuffer buf = ByteBuffer.allocate(4);
+ for (int i = 0; i < 32; i++) {
+ int val = 1 << i;
+ do {
+ for (int j = 1; j <= 4; j++) {
+ int bytesToUse = j;
+ if (bytesToUse == 1 && val >= Byte.MIN_VALUE && val <= Byte.MAX_VALUE
+ || bytesToUse == 2 && val >= Short.MIN_VALUE && val <= Short.MAX_VALUE
+ || bytesToUse == 3 && val >= -0x800000 && val <= 0x7fffff) {
+ buf.clear();
+ SparseLong2IntMap.putVal(buf, val, bytesToUse);
+ buf.flip();
+ Assert.assertEquals(val, SparseLong2IntMap.getVal(buf, bytesToUse));
+ }
+ }
+ val = ~val;
+ } while (val < 0);
+ }
+
+ testMap(0L);
+ testMap(-10000L);
+ testMap(1L << 35);
+ testMap(-1L << 35);
+ }
+
+ private static void testMap(long idOffset) {
+ SparseLong2IntMap map = new SparseLong2IntMap("test");
+ map.defaultReturnValue(Integer.MIN_VALUE);
+
+ // special patterns
+ testVals(map, idOffset, Arrays.asList(1,2,1,1,1,2,1,1,2,1,1,2));
+ testVals(map, idOffset, Arrays.asList(1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2,1,2));
+ testVals(map, idOffset, Arrays.asList(66560, 7936, 7936, 6144));
+ testVals(map, idOffset, Arrays.asList(Integer.MIN_VALUE + 1, 1234));
+ testVals(map, idOffset, Arrays.asList(1)); // single value chunk with 1 byte value
+ testVals(map, idOffset, Arrays.asList(1000)); // single value chunk with 2 byte value
+ testVals(map, idOffset, Arrays.asList(33000)); // single value chunk with 3 byte value
+ testVals(map, idOffset, Arrays.asList(1<<25)); // single value chunk with 4 byte value
+ testVals(map, idOffset, Arrays.asList(856, 856, 844, 844, 646, 646, 646, 646, 646, 646));
+ testVals(map, idOffset, Arrays.asList(260, 31, 31, 24));
+ testVals(map, idOffset, Arrays.asList(137, 110, 114, 128, 309, 114));
+ testVals(map, idOffset, Arrays.asList(254, 12, 12, 12, 12));
+ testVals(map, idOffset, Arrays.asList(254, 254, 12, 12));
+ testVals(map, idOffset, Arrays.asList(254, 12, 13));
+ testVals(map, idOffset, Arrays.asList(1000, 800, 700, 820));
+ testVals(map, idOffset, Arrays.asList(1000, 1000, 700));
+ testVals(map, idOffset, Arrays.asList(-32519, 255, -32519));
+ testVals(map, idOffset, Arrays.asList(-1, 1, 200, 1));
+ testVals(map, idOffset, Arrays.asList(Integer.MIN_VALUE + 1, Integer.MIN_VALUE + 1, 1234));
+ testVals(map, idOffset, Arrays.asList(Integer.MIN_VALUE + 1, 1234, Integer.MIN_VALUE + 1));
+
+ for (int i = 1; i < 1000; i++) {
+ int j = map.put(idOffset + i, i);
+ Assert.assertEquals(j, Integer.MIN_VALUE);
+ Assert.assertEquals(map.size(), i);
+ }
+
+ for (int i = 1; i < 1000; i++) {
+ boolean b = map.containsKey(idOffset + i);
+ Assert.assertEquals(b, true);
+ }
+
+
+ for (int i = 1; i < 1000; i++) {
+ Assert.assertEquals(map.get(idOffset + i), i);
+ }
+
+ // random read access
+ for (int i = 1; i < 1000; i++) {
+ int key = (int) Math.max(1, (Math.random() * 1000));
+ Assert.assertEquals(map.get(idOffset + key), key);
+ }
+
+ for (int i = 1000; i < 2000; i++) {
+ Assert.assertEquals(map.get(idOffset + i), Integer.MIN_VALUE);
+ }
+ for (int i = 1000; i < 2000; i++) {
+ boolean b = map.containsKey(idOffset + i);
+ Assert.assertEquals(b, false);
+ }
+ for (int i = 1000; i < 1200; i++) {
+ int j = map.put(idOffset + i, 333);
+ Assert.assertEquals(j, Integer.MIN_VALUE);
+ Assert.assertEquals(map.size(), i);
+ }
+ // random read access 2
+ Assert.assertEquals(map.get(idOffset + 1010), 333);
+ for (int i = 1; i < 1000; i++) {
+ int key = 1000 + (int) (Math.random() * 200);
+
+ Assert.assertEquals(map.get(idOffset + key), 333);
+ }
+
+
+ for (int i = -2000; i < -1000; i++) {
+ Assert.assertEquals(map.get(idOffset + i), Integer.MIN_VALUE);
+ }
+ for (int i = -2000; i < -1000; i++) {
+ boolean b = map.containsKey(idOffset + i);
+ Assert.assertEquals(b, false);
+ }
+ long mapSize = map.size();
+ // seq. update existing records
+ for (int i = 1; i < 1000; i++) {
+ int j = map.put(idOffset + i, i+333);
+ Assert.assertEquals(j, i);
+ Assert.assertEquals(map.size(), mapSize);
+ }
+ // random read access 3, update existing entries
+ for (int i = 1; i < 1000; i++) {
+ int j = map.put(idOffset + i, i+555);
+ Assert.assertEquals(true, j == i+333 | j == i+555);
+ Assert.assertEquals(map.size(), mapSize);
+ }
+
+ Assert.assertEquals(map.get(idOffset + 123456), Integer.MIN_VALUE);
+ map.put(idOffset + 123456, 999);
+ Assert.assertEquals(map.get(idOffset + 123456), 999);
+ map.put(idOffset + 123456, 888);
+ Assert.assertEquals(map.get(idOffset + 123456), 888);
+
+ Assert.assertEquals(map.get(idOffset - 123456), Integer.MIN_VALUE);
+ map.put(idOffset - 123456, 999);
+ Assert.assertEquals(map.get(idOffset - 123456), 999);
+ map.put(idOffset - 123456, 888);
+ Assert.assertEquals(map.get(idOffset - 123456), 888);
+ map.put(idOffset + 3008, 888);
+ map.put(idOffset + 3009, 888);
+ map.put(idOffset + 3010, 876);
+ map.put(idOffset + 3011, 876);
+ map.put(idOffset + 3012, 678);
+ map.put(idOffset + 3013, 678);
+ map.put(idOffset + 3014, 678);
+ map.put(idOffset + 3015, 678);
+ map.put(idOffset + 3016, 678);
+ map.put(idOffset + 3017, 678);
+ map.put(idOffset + 4000, 888);
+ map.put(idOffset + 4001, 888);
+ map.put(idOffset + 4002, 876);
+ map.put(idOffset + 4003, 876);
+ // update the first one
+ map.put(idOffset + 3008, 889);
+ // update the 2nd one
+ map.put(idOffset + 4000, 889);
+ // add a very different key
+ map.put(idOffset + 5000, 889);
+ map.put(idOffset + 5001, 222);
+ Assert.assertEquals(map.get(idOffset + 3008), 889);
+ Assert.assertEquals(map.get(idOffset + 3009), 888);
+ Assert.assertEquals(map.get(idOffset + 3010), 876);
+ Assert.assertEquals(map.get(idOffset + 3011), 876);
+ Assert.assertEquals(map.get(idOffset + 3012), 678);
+ Assert.assertEquals(map.get(idOffset + 3013), 678);
+ Assert.assertEquals(map.get(idOffset + 3014), 678);
+ Assert.assertEquals(map.get(idOffset + 4000), 889);
+ Assert.assertEquals(map.get(idOffset + 4001), 888);
+ Assert.assertEquals(map.get(idOffset + 4002), 876);
+ Assert.assertEquals(map.get(idOffset + 4003), 876);
+ Assert.assertEquals(map.get(idOffset + 5000), 889);
+ Assert.assertEquals(map.get(idOffset + 5001), 222);
+
+ map.clear();
+ // special pattern 1
+ Assert.assertEquals(map.put(idOffset + 1, 0), Integer.MIN_VALUE);
+ Assert.assertEquals(map.put(idOffset + 65, -1), Integer.MIN_VALUE);
+ Assert.assertEquals(map.get(idOffset + 999), Integer.MIN_VALUE);
+ Assert.assertEquals(map.get(idOffset + 1), 0);
+ Assert.assertEquals(map.get(idOffset + 65), -1);
+
+ map.clear();
+ map.put(idOffset + 1, 22);
+ map.put(idOffset + 5, 22);
+ map.put(idOffset + 100, 44);
+ Assert.assertEquals(map.put(idOffset + 5, 33), 22);
+
+
+ map.clear();
+ // larger values
+ for (int i = 100_000; i < 110_000; i++) {
+ map.put(idOffset + i, i);
+ }
+ for (int i = 100_000; i < 110_000; i++) {
+ Assert.assertEquals(map.get(idOffset + i), i);
+ }
+ Random random = new Random(101);
+ Map<Long,Integer> ref = new HashMap<>();
+ // special cases long chunks (all 64 values used and random
+ for (int i = 0; i < 3; i++) {
+ for (int j = 0; j < 1000; j++) {
+ int val = random.nextInt(Integer.MAX_VALUE);
+ map.put(idOffset + j, val);
+ ref.put(idOffset + j, val);
+ }
+ }
+// map.stats(0);
+ ref.entrySet().forEach(e -> {
+ long id = e.getKey();
+ int val = map.get(id);
+ Assert.assertEquals(val, (int)e.getValue());
+ });
+
+
+ ref.clear();
+ map.clear();
+ for (int i = 0; i < 100_00; i++) {
+ long id = Math.round((1L << 29) * random.nextDouble());
+ int val = (-1 * (1 << 20) + (int) Math.round((1 << 20) * random.nextDouble()));
+ map.put(idOffset + id, val);
+ ref.put(idOffset + id, val);
+ }
+// map.stats(0);
+ ref.entrySet().forEach(e -> {
+ long id = e.getKey();
+ int val = map.get(id);
+ Assert.assertEquals(val, (int)e.getValue());
+ });
+ }
+}
diff --git a/test/uk/me/parabola/splitter/tools/TestSparseBitSet.java b/test/uk/me/parabola/splitter/tools/TestSparseBitSet.java
new file mode 100644
index 0000000..5c7a0e6
--- /dev/null
+++ b/test/uk/me/parabola/splitter/tools/TestSparseBitSet.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2012, Gerd Petermann
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 or
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ */
+
+package uk.me.parabola.splitter.tools;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import uk.me.parabola.splitter.tools.SparseBitSet;
+
+/**
+ * Unit tests for the sparse BitSet implementation
+ */
+public class TestSparseBitSet {
+ private final int NUM = 10000;
+ private final long[] POS = { 1, 63, 64, 65, 4711, 12345654321L };
+
+ @Test
+ public void testSparseBitSetSequential() {
+ SparseBitSet sparseSet = new SparseBitSet();
+ for (long i = 1; i < NUM; i++) {
+ Assert.assertEquals(sparseSet.get(i), false, "get(" + i + ")");
+ }
+ for (long i = 1; i < NUM; i++) {
+ sparseSet.set(i);
+ Assert.assertEquals(sparseSet.get(i), true, "get(" + i + ")");
+ }
+ Assert.assertEquals(sparseSet.cardinality(), NUM - 1, "cardinality() returns wrong value");
+ for (long i = 1; i < NUM; i++) {
+ sparseSet.clear(i);
+ Assert.assertEquals(sparseSet.get(i), false, "get(" + i + ")");
+ Assert.assertEquals(sparseSet.cardinality(), NUM - i - 1, "cardinality() returns wrong value");
+ }
+
+ }
+
+ @Test
+ public void testSparseBitSetRandom() {
+ SparseBitSet sparseSet = new SparseBitSet();
+ for (long i : POS) {
+ sparseSet.set(i);
+ Assert.assertEquals(sparseSet.get(i), true, "get(" + i + ")");
+ Assert.assertEquals(sparseSet.cardinality(), 1, "cardinality() returns wrong value");
+ sparseSet.clear(i);
+ Assert.assertEquals(sparseSet.get(i), false, "get(" + i + ")");
+ Assert.assertEquals(sparseSet.cardinality(), 0, "cardinality() returns wrong value");
+ }
+
+ }
+}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/mkgmap-splitter.git
More information about the Pkg-grass-devel
mailing list