[imposm-parser] 16/29: Imported Upstream version 1.0.5
Sebastiaan Couwenberg
sebastic at moszumanska.debian.org
Fri Mar 13 19:11:21 UTC 2015
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository imposm-parser.
commit dc2c4a49646f923ef2b6b2d1180f6a96b8132cc0
Author: David Paleino <dapal at debian.org>
Date: Mon Nov 4 17:24:25 2013 +0100
Imported Upstream version 1.0.5
---
CHANGES | 12 +++++++
MANIFEST.in | 2 +-
PKG-INFO | 26 ++++++++++-----
README => README.rst | 8 ++---
imposm.parser.egg-info/PKG-INFO | 26 ++++++++++-----
imposm.parser.egg-info/SOURCES.txt | 3 +-
imposm/parser/example.py | 45 --------------------------
imposm/parser/pbf/multiproc.py | 33 +++++++++----------
imposm/parser/pbf/parser.py | 2 +-
imposm/parser/xml/multiproc.py | 65 ++++++++++++++++++++------------------
setup.py | 8 ++---
11 files changed, 110 insertions(+), 120 deletions(-)
diff --git a/CHANGES b/CHANGES
index 579d1e5..0c5b554 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,18 @@
Changelog
---------
+1.0.5 2013-09-13
+~~~~~~~~~~~~~~~~
+
+- support PBF without granularity value
+- improved support for non-pretty-printed XML
+
+1.0.4 2012-12-10
+~~~~~~~~~~~~~~~~
+
+- improved support for non-pretty-printed XML
+- fixed dependency check for multiprocessing
+
1.0.3 2011-07-21
~~~~~~~~~~~~~~~~
diff --git a/MANIFEST.in b/MANIFEST.in
index eba65a9..7c6f728 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,5 @@
include MANIFEST.in
-include README
+include README.rst
include CHANGES
include LICENSE
include setup.py
diff --git a/PKG-INFO b/PKG-INFO
index e89360c..3d16631 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,14 +1,12 @@
Metadata-Version: 1.0
Name: imposm.parser
-Version: 1.0.3
+Version: 1.0.5
Summary: Fast and easy OpenStreetMap XML/PBF parser.
-Home-page: http://dev.omniscale.net/imposm.parser/
+Home-page: http://imposm.org/docs/imposm.parser/latest/
Author: Oliver Tonnhofer
Author-email: olt at omniscale.de
License: Apache Software License 2.0
-Description: .. # -*- restructuredtext -*-
-
- imposm.parser - OpenStreetMap XML/PBF parser for Python
+Description: imposm.parser - OpenStreetMap XML/PBF parser for Python
=======================================================
``imposm.parser`` is a Python library that parses OpenStreetMap data in `XML <http://wiki.openstreetmap.org/wiki/.osm>`_ and `PBF <http://wiki.openstreetmap.org/wiki/PBF_Format>`_ format.
@@ -28,7 +26,7 @@ Description: .. # -*- restructuredtext -*-
# simple class that handles the parsed OSM data.
class HighwayCounter(object):
highways = 0
-
+
def ways(self, ways):
# callback method for ways
for osmid, tags, refs in ways:
@@ -39,7 +37,7 @@ Description: .. # -*- restructuredtext -*-
counter = HighwayCounter()
p = OSMParser(concurrency=4, ways_callback=counter.ways)
p.parse('germany.osm.pbf')
-
+
# done
print counter.highways
@@ -47,11 +45,23 @@ Description: .. # -*- restructuredtext -*-
Source and issue tracker
------------------------
- Source code and issue tracker are available at `<https://bitbucket.org/olt/imposm.parser/src>`_.
+ Source code and issue tracker are available at `<https://github.com/omniscale/imposm-parser>`_.
Changelog
---------
+ 1.0.5 2013-09-13
+ ~~~~~~~~~~~~~~~~
+
+ - support PBF without granularity value
+ - improved support for non-pretty-printed XML
+
+ 1.0.4 2012-12-10
+ ~~~~~~~~~~~~~~~~
+
+ - improved support for non-pretty-printed XML
+ - fixed dependency check for multiprocessing
+
1.0.3 2011-07-21
~~~~~~~~~~~~~~~~
diff --git a/README b/README.rst
similarity index 90%
rename from README
rename to README.rst
index c03ad8c..48322e8 100644
--- a/README
+++ b/README.rst
@@ -1,5 +1,3 @@
-.. # -*- restructuredtext -*-
-
imposm.parser - OpenStreetMap XML/PBF parser for Python
=======================================================
@@ -20,7 +18,7 @@ Here is an example that parses an OSM file and counts all ways that are tagged a
# simple class that handles the parsed OSM data.
class HighwayCounter(object):
highways = 0
-
+
def ways(self, ways):
# callback method for ways
for osmid, tags, refs in ways:
@@ -31,7 +29,7 @@ Here is an example that parses an OSM file and counts all ways that are tagged a
counter = HighwayCounter()
p = OSMParser(concurrency=4, ways_callback=counter.ways)
p.parse('germany.osm.pbf')
-
+
# done
print counter.highways
@@ -39,5 +37,5 @@ Here is an example that parses an OSM file and counts all ways that are tagged a
Source and issue tracker
------------------------
-Source code and issue tracker are available at `<https://bitbucket.org/olt/imposm.parser/src>`_.
+Source code and issue tracker are available at `<https://github.com/omniscale/imposm-parser>`_.
diff --git a/imposm.parser.egg-info/PKG-INFO b/imposm.parser.egg-info/PKG-INFO
index e89360c..3d16631 100644
--- a/imposm.parser.egg-info/PKG-INFO
+++ b/imposm.parser.egg-info/PKG-INFO
@@ -1,14 +1,12 @@
Metadata-Version: 1.0
Name: imposm.parser
-Version: 1.0.3
+Version: 1.0.5
Summary: Fast and easy OpenStreetMap XML/PBF parser.
-Home-page: http://dev.omniscale.net/imposm.parser/
+Home-page: http://imposm.org/docs/imposm.parser/latest/
Author: Oliver Tonnhofer
Author-email: olt at omniscale.de
License: Apache Software License 2.0
-Description: .. # -*- restructuredtext -*-
-
- imposm.parser - OpenStreetMap XML/PBF parser for Python
+Description: imposm.parser - OpenStreetMap XML/PBF parser for Python
=======================================================
``imposm.parser`` is a Python library that parses OpenStreetMap data in `XML <http://wiki.openstreetmap.org/wiki/.osm>`_ and `PBF <http://wiki.openstreetmap.org/wiki/PBF_Format>`_ format.
@@ -28,7 +26,7 @@ Description: .. # -*- restructuredtext -*-
# simple class that handles the parsed OSM data.
class HighwayCounter(object):
highways = 0
-
+
def ways(self, ways):
# callback method for ways
for osmid, tags, refs in ways:
@@ -39,7 +37,7 @@ Description: .. # -*- restructuredtext -*-
counter = HighwayCounter()
p = OSMParser(concurrency=4, ways_callback=counter.ways)
p.parse('germany.osm.pbf')
-
+
# done
print counter.highways
@@ -47,11 +45,23 @@ Description: .. # -*- restructuredtext -*-
Source and issue tracker
------------------------
- Source code and issue tracker are available at `<https://bitbucket.org/olt/imposm.parser/src>`_.
+ Source code and issue tracker are available at `<https://github.com/omniscale/imposm-parser>`_.
Changelog
---------
+ 1.0.5 2013-09-13
+ ~~~~~~~~~~~~~~~~
+
+ - support PBF without granularity value
+ - improved support for non-pretty-printed XML
+
+ 1.0.4 2012-12-10
+ ~~~~~~~~~~~~~~~~
+
+ - improved support for non-pretty-printed XML
+ - fixed dependency check for multiprocessing
+
1.0.3 2011-07-21
~~~~~~~~~~~~~~~~
diff --git a/imposm.parser.egg-info/SOURCES.txt b/imposm.parser.egg-info/SOURCES.txt
index d31e249..76b071f 100644
--- a/imposm.parser.egg-info/SOURCES.txt
+++ b/imposm.parser.egg-info/SOURCES.txt
@@ -1,7 +1,7 @@
CHANGES
LICENSE
MANIFEST.in
-README
+README.rst
osm.proto
setup.py
imposm/__init__.py
@@ -11,7 +11,6 @@ imposm.parser.egg-info/dependency_links.txt
imposm.parser.egg-info/namespace_packages.txt
imposm.parser.egg-info/top_level.txt
imposm/parser/__init__.py
-imposm/parser/example.py
imposm/parser/simple.py
imposm/parser/util.py
imposm/parser/pbf/__init__.py
diff --git a/imposm/parser/example.py b/imposm/parser/example.py
deleted file mode 100644
index 59ae9f0..0000000
--- a/imposm/parser/example.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# Copyright 2011 Omniscale GmbH & Co. KG
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-import multiprocessing
-from imposm.parser import OSMParser
-
-if __name__ == '__main__':
-
- class Counter(object):
- def __init__(self):
- self.coords_counter = 0
- self.nodes_counter = 0
- self.ways_counter = 0
- self.relations_counter = 0
-
- def incr_coords(self, coords):
- self.coords_counter += len(coords)
- def incr_nodes(self, nodes):
- self.nodes_counter += len(nodes)
- def incr_ways(self, ways):
- self.ways_counter += len(ways)
- def incr_relations(self, relations):
- self.relations_counter += len(relations)
-
- counter = Counter()
-
- p = OSMParser(
- nodes_callback=counter.incr_nodes,
- coords_callback=counter.incr_coords,
- ways_callback=counter.incr_ways,
- relations_callback=counter.incr_relations)
- p.parse(sys.argv[1])
- print counter.coords_counter, counter.nodes_counter, counter.ways_counter, counter.relations_counter
\ No newline at end of file
diff --git a/imposm/parser/pbf/multiproc.py b/imposm/parser/pbf/multiproc.py
index c5be9dc..b5a0048 100644
--- a/imposm/parser/pbf/multiproc.py
+++ b/imposm/parser/pbf/multiproc.py
@@ -1,11 +1,11 @@
# Copyright 2011 Omniscale GmbH & Co. KG
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,10 +19,11 @@ from imposm.parser.util import setproctitle
class PBFParserProcess(PBFParser, multiprocessing.Process):
def __init__(self, pos_queue, *args, **kw):
- self.pos_queue = pos_queue
- PBFParser.__init__(self, *args, **kw)
multiprocessing.Process.__init__(self)
-
+ PBFParser.__init__(self, *args, **kw)
+ self.daemon = True
+ self.pos_queue = pos_queue
+
def run(self):
setproctitle('imposm pbf parser')
while True:
@@ -30,7 +31,7 @@ class PBFParserProcess(PBFParser, multiprocessing.Process):
if pos is None:
self.pos_queue.task_done()
break
-
+
self.parse(pos['filename'], offset=pos['blob_pos'],
size=pos['blob_size'])
self.pos_queue.task_done()
@@ -39,7 +40,7 @@ class PBFMultiProcParser(object):
nodes_tag_filter = None
ways_tag_filter = None
relations_tag_filter = None
-
+
def __init__(self, pool_size, nodes_queue=None, ways_queue=None,
relations_queue=None, coords_queue=None, marshal_elem_data=False):
self.pool_size = pool_size
@@ -62,12 +63,12 @@ class PBFMultiProcParser(object):
)
pool.append(proc)
proc.start()
-
+
reader = PBFFile(filename)
-
+
for pos in reader.blob_offsets():
pos_queue.put(pos)
-
+
pos_queue.join()
for proc in pool:
@@ -90,12 +91,12 @@ if __name__ == '__main__':
queue.task_done()
print type, count
return count
-
-
+
+
nodes_queue = multiprocessing.JoinableQueue(128)
ways_queue = multiprocessing.JoinableQueue(128)
relations_queue = multiprocessing.JoinableQueue(128)
-
+
procs = [
multiprocessing.Process(target=count_proc('nodes', nodes_queue)),
multiprocessing.Process(target=count_proc('ways', ways_queue)),
@@ -103,11 +104,11 @@ if __name__ == '__main__':
]
for proc in procs:
proc.start()
-
+
parser = PBFMultiProcParser(2, nodes_queue=nodes_queue,
ways_queue=ways_queue, relations_queue=relations_queue)
parser.parse(sys.argv[1])
-
+
nodes_queue.put(None)
ways_queue.put(None)
relations_queue.put(None)
diff --git a/imposm/parser/pbf/parser.py b/imposm/parser/pbf/parser.py
index b1adf1f..d4f8d01 100644
--- a/imposm/parser/pbf/parser.py
+++ b/imposm/parser/pbf/parser.py
@@ -192,7 +192,7 @@ class PrimitiveBlockParser(object):
for group in self.primitivegroup:
dense = group.dense
if dense:
- granularity = self.primitive_block.granularity
+ granularity = self.primitive_block.granularity or 100
lat_offset = self.primitive_block.lat_offset or 0
lon_offset = self.primitive_block.lon_offset or 0
coord_scale = 0.000000001
diff --git a/imposm/parser/xml/multiproc.py b/imposm/parser/xml/multiproc.py
index e6fb6e8..ab6b92a 100644
--- a/imposm/parser/xml/multiproc.py
+++ b/imposm/parser/xml/multiproc.py
@@ -1,11 +1,11 @@
# Copyright 2011 Omniscale GmbH & Co. KG
-#
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -32,14 +32,14 @@ class MMapReader(object):
self.m = m
self.m.seek(0)
self.size = size
-
+
def read(self, size=None):
if size is None:
size = self.size - self.m.tell()
else:
size = min(self.size - self.m.tell(), size)
return self.m.read(size)
-
+
def readline(self):
cur_pos = self.m.tell()
if cur_pos >= self.size:
@@ -47,17 +47,17 @@ class MMapReader(object):
nl_pos = self.m.find('\n')
self.m.seek(cur_pos)
return self.m.read(nl_pos-cur_pos)
-
+
def seek(self, n):
self.m.seek(n)
class XMLParserProcess(XMLParser, multiprocessing.Process):
def __init__(self, mmap_pool, mmap_queue, *args, **kw):
- self.mmap_pool = mmap_pool
- self.mmap_queue = mmap_queue
- XMLParser.__init__(self, *args, **kw)
multiprocessing.Process.__init__(self)
+ XMLParser.__init__(self, *args, **kw)
self.daemon = True
+ self.mmap_pool = mmap_pool
+ self.mmap_queue = mmap_queue
def run(self):
setproctitle('imposm xml parser')
@@ -77,7 +77,7 @@ class XMLMultiProcParser(object):
nodes_tag_filter = None
ways_tag_filter = None
relations_tag_filter = None
-
+
def __init__(self, pool_size, nodes_queue=None, ways_queue=None,
relations_queue=None, coords_queue=None, marshal_elem_data=False):
self.pool_size = pool_size
@@ -90,10 +90,10 @@ class XMLMultiProcParser(object):
self.mmap_pool = MMapPool(pool_size*8, xml_chunk_size*8)
self.mmap_queue = multiprocessing.JoinableQueue(8)
self.marshal_elem_data = marshal_elem_data
-
+
def parse(self, stream):
assert not self.pool
-
+
for _ in xrange(self.pool_size):
proc = XMLParserProcess(self.mmap_pool, self.mmap_queue, nodes_callback=self.nodes_callback,
coords_callback=self.coords_callback, ways_callback=self.ways_callback,
@@ -105,16 +105,16 @@ class XMLMultiProcParser(object):
)
self.pool.append(proc)
proc.start()
-
+
chunker = XMLChunker(stream, self.mmap_pool, xml_chunk_size=READ_SIZE)
chunker.read(self.mmap_queue, coords_callback=self.coords_callback)
-
+
self.mmap_queue.join()
for proc in self.pool:
self.mmap_queue.put((None, None))
for proc in self.pool:
proc.join()
-
+
class MMapPool(object):
"""
@@ -127,11 +127,11 @@ class MMapPool(object):
self.pool = [mmap.mmap(-1, mmap_size) for _ in range(n)]
self.free_mmaps = set(range(n))
self.free_queue = multiprocessing.JoinableQueue()
-
+
def new(self):
"""
Return a free mmap file.
-
+
:returns: index, mmap file
"""
if not self.free_mmaps:
@@ -146,7 +146,7 @@ class MMapPool(object):
break
mmap_idx = self.free_mmaps.pop()
return mmap_idx, self.pool[mmap_idx]
-
+
def join(self):
while len(self.free_mmaps) < self.n:
self.free_mmaps.add(self.free_queue.get())
@@ -167,10 +167,10 @@ class MMapPool(object):
class XMLChunker(object):
"""
Reads and chunks OSM XML file.
-
+
Reads OSM XML from `stream` and writes chunks of it into mmap files from
the `mmap_pool`.
-
+
:params xml_chunk_size: chunk XML after this many bytes
"""
def __init__(self, stream, mmap_pool, xml_chunk_size):
@@ -200,13 +200,14 @@ class XMLChunker(object):
def read(self, mmaps_queue, coords_callback=None):
"""
- Read and chunk all
+ Read and chunk all
"""
coord_node_match = None
xml_nodes = self._new_xml_outstream()
coords = []
coord_node_re_match = re.compile(r'^\s*<node id="(\d+)" .*lat="([-0-9.]+)" '
'lon="([-0-9.]+)".*/>').match
+ node_re_match = re.compile(r'^\s*<node .*/>').match
xml_nodes.write(self._last_line)
split = False
line = ''
@@ -224,9 +225,9 @@ class XMLChunker(object):
else:
xml_nodes.write(line)
if split:
- if (line.lstrip().startswith('</')
+ if (line.rstrip().endswith(('</way>', '</node>', '</relation>'))
or (coords_callback and coord_node_match)
- or (not coords_callback and coord_node_re_match(line))):
+ or (not coords_callback and node_re_match(line))):
mmaps_queue.put(self._finished_xml_outstream(line, xml_nodes))
xml_nodes = self._new_xml_outstream()
split = False
@@ -234,7 +235,11 @@ class XMLChunker(object):
split = True
if coords_callback:
coords_callback(coords)
-
+
+ # we are at the end of the stream and assume we wrote the end tag
+ # to xml_nodes. we set line to closing tag here to avoid additional
+ # end tag in case the last line(s) is blank
+ line = '</osm'
mmaps_queue.put(self._finished_xml_outstream(line, xml_nodes))
if __name__ == '__main__':
@@ -253,13 +258,13 @@ if __name__ == '__main__':
queue.task_done()
print type, count
return count
-
-
+
+
nodes_queue = multiprocessing.JoinableQueue(128)
ways_queue = multiprocessing.JoinableQueue(128)
relations_queue = multiprocessing.JoinableQueue(128)
-
+
procs = [
multiprocessing.Process(target=count_proc('nodes', nodes_queue)),
multiprocessing.Process(target=count_proc('ways', ways_queue)),
@@ -267,14 +272,14 @@ if __name__ == '__main__':
]
for proc in procs:
proc.start()
-
+
parser = XMLMultiProcParser(open(sys.argv[1]), 2, nodes_queue=nodes_queue,
ways_queue=ways_queue, relations_queue=relations_queue)
parser.start()
-
+
nodes_queue.put(None)
nodes_queue.join()
-
+
ways_queue.put(None)
ways_queue.join()
relations_queue.put(None)
diff --git a/setup.py b/setup.py
index 9160157..1af1a03 100644
--- a/setup.py
+++ b/setup.py
@@ -31,17 +31,17 @@ class build_ext_with_protpbuf(build_ext):
install_requires = []
-if platform.python_version_tuple() < ('2', '6'):
+if tuple(map(str, platform.python_version_tuple())) < ('2', '6'):
install_requires.append('multiprocessing>=2.6')
setup(
name='imposm.parser',
- version="1.0.3",
+ version="1.0.5",
description='Fast and easy OpenStreetMap XML/PBF parser.',
- long_description=open('README').read() + open('CHANGES').read(),
+ long_description=open('README.rst').read() + open('CHANGES').read(),
author='Oliver Tonnhofer',
author_email='olt at omniscale.de',
- url='http://dev.omniscale.net/imposm.parser/',
+ url='http://imposm.org/docs/imposm.parser/latest/',
license='Apache Software License 2.0',
packages=find_packages(),
namespace_packages = ['imposm'],
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/imposm-parser.git
More information about the Pkg-grass-devel
mailing list