[pyosmium] 02/10: Imported Upstream version 2.6.0
Sebastiaan Couwenberg
sebastic at moszumanska.debian.org
Sat Feb 6 20:58:26 UTC 2016
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository pyosmium.
commit fdbfe209a3afd0a55d0cd6340634419aad3a5db9
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Sat Feb 6 20:22:02 2016 +0100
Imported Upstream version 2.6.0
---
CHANGELOG.md | 19 ++-
README.md | 2 +-
doc/conf.py | 6 +-
doc/intro.rst | 75 +++++++++++-
doc/ref_osm.rst | 33 ++++-
doc/ref_osmium.rst | 28 ++++-
examples/convert.py | 37 ++++++
examples/filter_coastlines.py | 56 +++++++++
examples/normalize_boolean.py | 66 ++++++++++
lib/generic_handler.hpp | 4 +-
lib/generic_writer.hpp | 278 ++++++++++++++++++++++++++++++++++++++++++
lib/geom.cc | 2 +-
lib/index.cc | 2 +-
lib/io.cc | 2 +-
lib/osm.cc | 20 +--
lib/osmium.cc | 43 ++++++-
osmium/__init__.py | 2 +-
osmium/geom/__init__.py | 1 -
osmium/index/__init__.py | 1 -
osmium/io/__init__.py | 1 -
osmium/osm/__init__.py | 14 +++
osmium/osm/mutable.py | 75 ++++++++++++
setup.py | 15 ++-
test/test_osm.py | 23 +++-
test/test_writer.py | 142 +++++++++++++++++++++
25 files changed, 900 insertions(+), 47 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b38f524..0c82122 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,20 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Fixed
+## [2.6.0] - 2016-02-04
+
+### Added
+
+- Experimental write support, see documentation
+- Multiple examples for writing data
+
+### Changed
+
+- Use current libosmium
+- Improve timestamp to datetime conversion
+- Simplified package structure that uses the compiled libs directly
+
+
## [2.5.4] - 2015-12-03
### Changed
@@ -52,8 +66,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
- Exception not caught in test.
-[unreleased]: https://github.com/osmcode/pyosmium/compare/v2.5.4...HEAD
-[2.5.4]: https://github.com/osmcode/pyosmium/compare/v2.4.3...v2.5.4
+[unreleased]: https://github.com/osmcode/pyosmium/compare/v2.6.0...HEAD
+[2.6.0]: https://github.com/osmcode/pyosmium/compare/v2.5.4...v2.6.0
+[2.5.4]: https://github.com/osmcode/pyosmium/compare/v2.5.3...v2.5.4
[2.5.3]: https://github.com/osmcode/pyosmium/compare/v2.4.1...v2.5.3
[2.4.1]: https://github.com/osmcode/pyosmium/compare/v2.3.0...v2.4.1
[2.3.0]: https://github.com/osmcode/pyosmium/compare/v2.2.0...v2.3.0
diff --git a/README.md b/README.md
index 9b5187c..c738975 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ manner.
## Dependencies
-Python >= 2.7 is supported (that includes python 3.x).
+Python >= 2.7 is supported but a version >= 3.3 is strongly recommended.
pyosmium uses [Boost.Python](http://www.boost.org/doc/libs/1_56_0/libs/python/doc/index.html)
to create the bindings. On Debian/Ubuntu install `libboost-python-dev`. OS X run `brew install boost-python` or `brew install boost-python --with-python3` depending on which python version you want to use – You can also (re)install both.
diff --git a/doc/conf.py b/doc/conf.py
index 368f03c..f100bbc 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -56,16 +56,16 @@ master_doc = 'index'
# General information about the project.
project = 'Pyosmium'
-copyright = '2015, Sarah Hoffmann'
+copyright = '2015-2016, Sarah Hoffmann'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
-version = '2.5'
+version = '2.6'
# The full version, including alpha/beta/rc tags.
-release = '2.5.4'
+release = '2.6.0'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/doc/intro.rst b/doc/intro.rst
index 1cdb4b8..74cb783 100644
--- a/doc/intro.rst
+++ b/doc/intro.rst
@@ -11,8 +11,11 @@ reader is referred to the `osmium documentation`_.
.. _OSM data model: http://wiki.openstreetmap.org/wiki/Elements
.. _osmium documentation: http://osmcode.org/libosmium/manual/libosmium-manual.html
+Reading OSM Data
+----------------
+
Using Handler Classes
-+++++++++++++++++++++
+^^^^^^^^^^^^^^^^^^^^^
OSM file parsing by osmium is built around the concept of handlers. A handler
is a class with a set of callback functions. Each function processes exactly
@@ -54,7 +57,7 @@ therefore looks like this::
That already finishes our node counting program.
Inspecting the OSM objects
-++++++++++++++++++++++++++
+^^^^^^^^^^^^^^^^^^^^^^^^^^
Counting nodes is actually boring because it completely ignores the
content of the nodes. So let's change the handler to only count hotels
@@ -94,7 +97,7 @@ copy any data that should be kept for later use into their own data
structures. This also includes attributes like tag lists.
Handling Geometries
-+++++++++++++++++++
+^^^^^^^^^^^^^^^^^^^
Because of the way that OSM data is structured, osmium needs to internally
cache node geometries, when the handler wants to process the geometries of
@@ -117,7 +120,7 @@ cache like that::
where `example.nodecache` is the name of the cache file.
Interfacing with Shapely
-++++++++++++++++++++++++
+^^^^^^^^^^^^^^^^^^^^^^^^
Pyosmium is a library for processing OSM files and therefore offers almost
no functionality for processing geometries further. For this other libraries
@@ -149,3 +152,67 @@ example uses the libgeos wrapper `Shapely`_ to compute the total way length::
print("Total length: %f" % h.total)
.. _Shapely: http://toblerity.org/shapely/index.html
+
+
+Writing OSM Data
+----------------
+
+:py:class:`osmium.SimpleWriter` is the main class that takes care of
+writing out OSM data to a file. The file name must be given when the
+writer is constructed. Its suffix determines the format of the data.
+For example::
+
+ writer = osmium.SimpleWriter('nodes.osm.bz2')
+
+opens a new writer for a packed OSM XML file. Objects can be written
+by using one of the writers ``add_*`` functions.
+
+A simple handler, that only writes out all the nodes from the input
+file into out new ``nodes.osm.bz2`` file would look like this::
+
+ import osmium
+
+ class NodeWriter(osmium.SimpleHandler):
+ def __init__(self, writer):
+ osmium.SimpleHandler.__init__(self)
+ self.writer = writer
+
+ def node(self, n):
+ self.writer.add_node(n)
+
+This example shows that an unmodified object can be written out directly
+to the writer. Normally, however, you want to modify some data. The native
+osmium OSM types are immutable and cannot be changed directly. Therefore
+you have create a copy that can be changed. The ``node``, ``way`` and ``relation``
+objects offer a convenient ``replace()`` function to achieve exactly that.
+The function makes a copy and a the same time replaces all attibutes where
+new values are given as parameters to the function.
+
+Let's say you want to
+remove all the user names from your nodes before saving them to the new
+file (maybe to save some space), then the ``node()`` handler callback above
+needs to be changed like that::
+
+ class NodeWriter(osmium.SimpleHandler):
+ ...
+
+ def node(self, n):
+ self.writer.add_node(n.replace(user=""))
+
+``replace()`` creates a new instance of an ``osmium.osm.mutable.`` object. These
+class a real python versions of the native object types in ``osmium.osm``. They
+have exactly the same attributes but they are mutable.
+
+A writer is able to process the mutable datatypes just like the native osmium
+types. In fact, a writer is able to process any python object. It just expects
+suitably named attributes and will simply assume sensible default values for
+attributes that are missing.
+
+.. note::
+
+ It is important to understand that ``replace()`` only makes a shallow copy
+ of the object. Tag, node and member lists are still native osmium objects.
+ Normally this is what you want because the writer is much faster writing
+ these native objects than pythonized copies. However, it means that you
+ cannot use ``replace()`` to create a copy of the object that can be kept
+ after the handler callback has finished.
diff --git a/doc/ref_osm.rst b/doc/ref_osm.rst
index b52df63..7e4b1a9 100644
--- a/doc/ref_osm.rst
+++ b/doc/ref_osm.rst
@@ -4,8 +4,12 @@
The ``osm`` submodule contains definition of the basic data types used
throughout the library.
-OSM Objects
-^^^^^^^^^^^
+Native OSM Objects
+^^^^^^^^^^^^^^^^^^
+
+Native OSM object classes are lightwight wrappers around the osmium OSM
+data classes. They are immutable and generally bound to the life-time of
+the buffer they are saved in.
There are five classes representing the basic OSM entities.
@@ -33,6 +37,31 @@ There are five classes representing the basic OSM entities.
:members:
:undoc-members:
+.. _mutable-objects:
+
+Mutable OSM Objects
+^^^^^^^^^^^^^^^^^^^
+
+The objects in ``osmium.osm.mutable`` are Python versions of the native OSM
+objects that can be modified. You can use these classes as a base class for
+your own objects or to modify objects read from a file.
+
+.. autoclass:: osmium.osm.mutable.OSMObject
+ :members:
+ :undoc-members:
+
+.. autoclass:: osmium.osm.mutable.Node
+ :members:
+ :undoc-members:
+
+.. autoclass:: osmium.osm.mutable.Way
+ :members:
+ :undoc-members:
+
+.. autoclass:: osmium.osm.mutable.Relation
+ :members:
+ :undoc-members:
+
Node Reference Lists
^^^^^^^^^^^^^^^^^^^^
diff --git a/doc/ref_osmium.rst b/doc/ref_osmium.rst
index 8efbdac..1867ddb 100644
--- a/doc/ref_osmium.rst
+++ b/doc/ref_osmium.rst
@@ -9,13 +9,37 @@ can easily be derived.
For more fine grained control of the processing chain, the more basic
functions and processors are exported as well in this module.
-Simple Handlers
-^^^^^^^^^^^^^^^
+Input Handlers
+^^^^^^^^^^^^^^
+
+An input handler implements provides the base class for writing custom
+data processors. They take input data, usually from a file, and forward
+it to handler functions.
.. autoclass:: osmium.SimpleHandler
:members:
:undoc-members:
+SimpleWriter
+^^^^^^^^^^^^
+
+The writer class can be used to create an OSM file. The writer is able to
+handle native ``osmium.osm`` objects as well as any Python object that
+exposes the same attributes. It is not necessary to implement the full
+list of attributes as any missing attributes will be replaced with a
+sensible default value when writing. See :ref:`mutable-objects`
+for a detailed discussion what data formats are understood for each attribute.
+
+.. warning::
+
+ Writers are considerably faster in handling native osmium data types than
+ Python objects. You should therefore avoid converting objects whereever
+ possible. This is not only true for the OSM data types like Node, Way and
+ Relation but also for tag lists, node lists and member lists.
+
+.. autoclass:: osmium.SimpleWriter
+ :members:
+ :undoc-members:
Low-level Functions and Classes
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/examples/convert.py b/examples/convert.py
new file mode 100644
index 0000000..0c97c2d
--- /dev/null
+++ b/examples/convert.py
@@ -0,0 +1,37 @@
+"""
+Converts a file from one format to another.
+
+This example shows how to write objects to a file.
+"""
+
+import osmium as o
+
+import sys
+
+class Convert(o.SimpleHandler):
+
+ def __init__(self, writer):
+ o.SimpleHandler.__init__(self)
+ self.writer = writer
+
+ def node(self, n):
+ self.writer.add_node(n)
+
+ def way(self, w):
+ self.writer.add_way(w)
+
+ def relation(self, r):
+ self.writer.add_relation(r)
+
+if __name__ == '__main__':
+ if len(sys.argv) != 3:
+ print("Usage: python convert.py <infile> <outfile>")
+ sys.exit(-1)
+
+ writer = o.SimpleWriter(sys.argv[2])
+ handler = Convert(writer)
+
+ handler.apply_file(sys.argv[1])
+
+ writer.close()
+
diff --git a/examples/filter_coastlines.py b/examples/filter_coastlines.py
new file mode 100644
index 0000000..ced0ce3
--- /dev/null
+++ b/examples/filter_coastlines.py
@@ -0,0 +1,56 @@
+"""
+Filter all objects with a coastline tag.
+
+This example shows how to write objects to a file.
+
+We need to go twice over the file. First read the ways, filter the ones
+we are interested in and remember the nodes required. Then, in a second
+run all the relevant nodes and ways are written out.
+"""
+
+import osmium as o
+import sys
+
+class WayFilter(o.SimpleHandler):
+
+ def __init__(self):
+ o.SimpleHandler.__init__(self)
+ self.nodes = set()
+
+ def way(self, w):
+ if 'natural' in w.tags and w.tags['natural'] == 'coastline':
+ for n in w.nodes:
+ self.nodes.add(n.ref)
+
+
+class CoastlineWriter(o.SimpleHandler):
+
+ def __init__(self, writer, nodes):
+ o.SimpleHandler.__init__(self)
+ self.writer = writer
+ self.nodes = nodes
+
+ def node(self, n):
+ if n.id in self.nodes:
+ self.writer.add_node(n)
+
+ def way(self, w):
+ if 'natural' in w.tags and w.tags['natural'] == 'coastline':
+ self.writer.add_way(w)
+
+
+if __name__ == '__main__':
+ if len(sys.argv) != 3:
+ print("Usage: python filter_coastlines.py <infile> <outfile>")
+ sys.exit(-1)
+
+
+ # go through the ways to find all relevant nodes
+ ways = WayFilter(writer)
+ ways.apply_file(sys.argv[1])
+
+ # go through the file again and write out the data
+ writer = o.SimpleWriter(sys.argv[2])
+ CoastlineWriter(writer, ways.nodes).apply_file(sys.argv[1])
+
+ writer.close()
diff --git a/examples/normalize_boolean.py b/examples/normalize_boolean.py
new file mode 100644
index 0000000..7639985
--- /dev/null
+++ b/examples/normalize_boolean.py
@@ -0,0 +1,66 @@
+"""
+This example shows how to filter and modify tags and write the rusults back.
+It changes all tag values 'yes/no' to '1/0'.
+"""
+
+import osmium as o
+import sys
+
+class BoolNormalizer(o.SimpleHandler):
+
+ def __init__(self, writer):
+ o.SimpleHandler.__init__(self)
+ self.writer = writer
+
+ def normalize(self, o):
+ # if there are no tags we are done
+ if len(o.tags) == 0:
+ return o
+
+ # new tags should be kept in a list so that the order is preserved
+ newtags = []
+ # pyosmium is much faster writing an original osmium object than
+ # a osmium.mutable.*. Therefore, keep track if the tags list was
+ # actually changed.
+ modified = False
+ for t in o.tags:
+ if t.v == 'yes':
+ # custom tags should be added as a key/value tuple
+ newtags.append((t.k, '1'))
+ modified = True
+ elif t.v == 'no':
+ newtags.append((t.k, '0'))
+ modified = True
+ else:
+ # if the tag is not modified, simply readd it to the list
+ newtags.append(t)
+
+ if modified:
+ # We have changed tags. Create a new object as a copy of the
+ # original one with the tag list replaced.
+ return o.replace(tags=newtags)
+ else:
+ # Nothing changed, so simply return the original object
+ # and discard the tag list we just created.
+ return o
+
+ def node(self, o):
+ self.writer.add_node(self.normalize(o))
+
+ def way(self, o):
+ self.writer.add_way(self.normalize(o))
+
+ def relation(self, o):
+ self.writer.add_relation(self.normalize(o))
+
+
+if __name__ == '__main__':
+ if len(sys.argv) != 3:
+ print("Usage: python normalize_boolean.py <infile> <outfile>")
+ sys.exit(-1)
+
+
+ writer = o.SimpleWriter(sys.argv[2])
+ BoolNormalizer(writer).apply_file(sys.argv[1])
+
+ writer.close()
diff --git a/lib/generic_handler.hpp b/lib/generic_handler.hpp
index 916c965..e7d0597 100644
--- a/lib/generic_handler.hpp
+++ b/lib/generic_handler.hpp
@@ -1,8 +1,6 @@
#ifndef PYOSMIUM_GENERIC_HANDLER_HPP
#define PYOSMIUM_GENERIC_HANDLER_HPP
-#include <boost/python.hpp>
-
#include <osmium/area/assembler.hpp>
#include <osmium/area/multipolygon_collector.hpp>
#include <osmium/handler.hpp>
@@ -11,6 +9,8 @@
#include <osmium/io/any_input.hpp>
#include <osmium/visitor.hpp>
+#include <boost/python.hpp>
+
typedef osmium::index::map::Map<osmium::unsigned_object_id_type, osmium::Location> index_type;
diff --git a/lib/generic_writer.hpp b/lib/generic_writer.hpp
new file mode 100644
index 0000000..7068d2b
--- /dev/null
+++ b/lib/generic_writer.hpp
@@ -0,0 +1,278 @@
+#ifndef PYOSMIUM_GENERIC_WRITER_HPP
+#define PYOSMIUM_GENERIC_WRITER_HPP
+
+
+#include <osmium/osm.hpp>
+#include <osmium/io/any_output.hpp>
+#include <osmium/io/writer.hpp>
+#include <osmium/memory/buffer.hpp>
+#include <osmium/builder/osm_object_builder.hpp>
+#include <boost/python.hpp>
+
+class SimpleWriterWrap {
+
+ enum { BUFFER_WRAP = 4096 };
+
+public:
+ SimpleWriterWrap(const char* filename, size_t bufsz=4096*1024)
+ : writer(filename),
+ buffer(bufsz < 2*BUFFER_WRAP ? 2*BUFFER_WRAP : bufsz, osmium::memory::Buffer::auto_grow::yes)
+ {}
+
+ virtual ~SimpleWriterWrap()
+ {
+ close();
+ }
+
+ void add_osmium_object(const osmium::OSMObject& o) {
+ buffer.add_item(o);
+ flush_buffer();
+ }
+
+ void add_node(boost::python::object o) {
+ boost::python::extract<osmium::Node&> node(o);
+ if (node.check()) {
+ buffer.add_item(node());
+ } else {
+ osmium::builder::NodeBuilder builder(buffer);
+
+ if (hasattr(o, "location")) {
+ osmium::Node& n = builder.object();
+ n.set_location(get_location(o.attr("location")));
+ }
+
+ set_common_attributes(o, builder);
+
+ if (hasattr(o, "tags"))
+ set_taglist(o.attr("tags"), builder);
+ }
+
+ flush_buffer();
+ }
+
+ void add_way(const boost::python::object& o) {
+ boost::python::extract<osmium::Way&> way(o);
+ if (way.check()) {
+ buffer.add_item(way());
+ } else {
+ osmium::builder::WayBuilder builder(buffer);
+
+ set_common_attributes(o, builder);
+
+ if (hasattr(o, "nodes"))
+ set_nodelist(o.attr("nodes"), &builder);
+
+ if (hasattr(o, "tags"))
+ set_taglist(o.attr("tags"), builder);
+ }
+
+ flush_buffer();
+ }
+
+ void add_relation(boost::python::object o) {
+ boost::python::extract<osmium::Relation&> rel(o);
+ if (rel.check()) {
+ buffer.add_item(rel());
+ } else {
+ osmium::builder::RelationBuilder builder(buffer);
+
+ set_common_attributes(o, builder);
+
+ if (hasattr(o, "members"))
+ set_memberlist(o.attr("members"), &builder);
+
+ if (hasattr(o, "tags"))
+ set_taglist(o.attr("tags"), builder);
+ }
+
+ flush_buffer();
+ }
+
+ void close() {
+ if (buffer) {
+ writer(std::move(buffer));
+ writer.close();
+ buffer = osmium::memory::Buffer();
+ }
+ }
+
+private:
+ void set_object_attributes(const boost::python::object& o, osmium::OSMObject& t) {
+ if (hasattr(o, "id"))
+ t.set_id(boost::python::extract<osmium::object_id_type>(o.attr("id")));
+ if (hasattr(o, "visible"))
+ t.set_visible(boost::python::extract<bool>(o.attr("visible")));
+ if (hasattr(o, "version"))
+ t.set_version(boost::python::extract<osmium::object_version_type>(o.attr("version")));
+ if (hasattr(o, "changeset"))
+ t.set_changeset(boost::python::extract<osmium::changeset_id_type>(o.attr("changeset")));
+ if (hasattr(o, "uid"))
+ t.set_uid_from_signed(boost::python::extract<osmium::signed_user_id_type>(o.attr("uid")));
+ if (hasattr(o, "timestamp")) {
+ boost::python::object ts = o.attr("timestamp");
+ boost::python::extract<osmium::Timestamp> ots(ts);
+ if (ots.check()) {
+ t.set_timestamp(ots());
+ } else {
+ if (hasattr(ts, "timestamp")) {
+ double epoch = boost::python::extract<double>(ts.attr("timestamp")());
+ t.set_timestamp(osmium::Timestamp(uint32_t(epoch)));
+ } else
+ {
+ // XXX terribly inefficient because of the double string conversion
+ // but the only painless method for converting a datetime
+ // in python < 3.3.
+ if (hasattr(ts, "strftime"))
+ ts = ts.attr("strftime")("%Y-%m-%dT%H:%M:%SZ");
+
+ t.set_timestamp(osmium::Timestamp(boost::python::extract<const char *>(ts)));
+ }
+ }
+ }
+ }
+
+ template <typename T>
+ void set_common_attributes(const boost::python::object& o, T& builder) {
+ set_object_attributes(o, builder.object());
+
+ if (hasattr(o, "user")) {
+ auto s = boost::python::extract<const char *>(o.attr("user"));
+ builder.add_user(s);
+ } else {
+ builder.add_user("", 0);
+ }
+ }
+
+ template <typename T>
+ void set_taglist(const boost::python::object& o, T& obuilder) {
+
+ // original taglist
+ boost::python::extract<osmium::TagList&> otl(o);
+ if (otl.check()) {
+ if (otl().size() > 0)
+ obuilder.add_item(&otl());
+ return;
+ }
+
+ // dict
+ boost::python::extract<boost::python::dict> tagdict(o);
+ if (tagdict.check()) {
+ auto items = tagdict().items();
+ auto len = boost::python::len(items);
+ if (len == 0)
+ return;
+
+ osmium::builder::TagListBuilder builder(buffer, &obuilder);
+ auto iter = items.attr("__iter__")();
+ for (int i = 0; i < len; ++i) {
+#if PY_VERSION_HEX < 0x03000000
+ auto tag = iter.attr("next")();
+#else
+ auto tag = iter.attr("__next__")();
+#endif
+ builder.add_tag(boost::python::extract<const char *>(tag[0]),
+ boost::python::extract<const char *>(tag[1]));
+ }
+ return;
+ }
+
+ // any other iterable
+ auto l = boost::python::len(o);
+ if (l == 0)
+ return;
+
+ osmium::builder::TagListBuilder builder(buffer, &obuilder);
+ for (int i = 0; i < l; ++i) {
+ auto tag = o[i];
+
+ boost::python::extract<osmium::Tag> ot(tag);
+ if (ot.check()) {
+ builder.add_tag(ot);
+ } else {
+ builder.add_tag(boost::python::extract<const char *>(tag[0]),
+ boost::python::extract<const char *>(tag[1]));
+ }
+ }
+ }
+
+ void set_nodelist(const boost::python::object& o,
+ osmium::builder::WayBuilder *builder) {
+ // original nodelist
+ boost::python::extract<osmium::NodeRefList&> onl(o);
+ if (onl.check()) {
+ if (onl().size() > 0)
+ builder->add_item(&onl());
+ return;
+ }
+
+ auto len = boost::python::len(o);
+ if (len == 0)
+ return;
+
+ osmium::builder::WayNodeListBuilder wnl_builder(buffer, builder);
+
+ for (int i = 0; i < len; ++i) {
+ boost::python::extract<osmium::NodeRef> ref(o[i]);
+ if (ref.check())
+ wnl_builder.add_node_ref(ref());
+ else
+ wnl_builder.add_node_ref(boost::python::extract<osmium::object_id_type>(o[i]));
+ }
+ }
+
+ void set_memberlist(const boost::python::object& o,
+ osmium::builder::RelationBuilder *builder) {
+ // original nodelist
+ boost::python::extract<osmium::RelationMemberList&> oml(o);
+ if (oml.check()) {
+ if (oml().size() > 0)
+ builder->add_item(&oml());
+ return;
+ }
+
+ auto len = boost::python::len(o);
+ if (len == 0)
+ return;
+
+ osmium::builder::RelationMemberListBuilder rml_builder(buffer, builder);
+
+ for (int i = 0; i < len; ++i) {
+ auto member = o[i];
+ auto type = osmium::char_to_item_type(boost::python::extract<const char*>(member[0])()[0]);
+ auto id = boost::python::extract<osmium::object_id_type>(member[1])();
+ auto role = boost::python::extract<const char*>(member[2])();
+ rml_builder.add_member(type, id, role);
+ }
+ }
+
+ osmium::Location get_location(const boost::python::object& o) {
+ boost::python::extract<osmium::Location> ol(o);
+ if (ol.check())
+ return ol;
+
+ // default is a tuple with two floats
+ return osmium::Location(boost::python::extract<float>(o[0]),
+ boost::python::extract<float>(o[1]));
+ }
+
+ bool hasattr(const boost::python::object& obj, char const *attr) {
+ return PyObject_HasAttrString(obj.ptr(), attr)
+ && (obj.attr(attr) != boost::python::object());
+ }
+
+ void flush_buffer() {
+ buffer.commit();
+
+ if (buffer.committed() > buffer.capacity() - BUFFER_WRAP) {
+ osmium::memory::Buffer new_buffer(buffer.capacity(), osmium::memory::Buffer::auto_grow::yes);
+ using std::swap;
+ swap(buffer, new_buffer);
+ writer(std::move(new_buffer));
+ }
+ }
+
+ osmium::io::Writer writer;
+ osmium::memory::Buffer buffer;
+};
+
+#endif // PYOSMIUM_GENERIC_WRITER_HPP
diff --git a/lib/geom.cc b/lib/geom.cc
index 1b17440..626c5e3 100644
--- a/lib/geom.cc
+++ b/lib/geom.cc
@@ -12,7 +12,7 @@ public:
{}
};
-BOOST_PYTHON_MODULE(_geom)
+BOOST_PYTHON_MODULE(geom)
{
using namespace boost::python;
docstring_options doc_options(true, true, false);
diff --git a/lib/index.cc b/lib/index.cc
index 200eea8..981feeb 100644
--- a/lib/index.cc
+++ b/lib/index.cc
@@ -17,7 +17,7 @@ std::vector<std::string> map_types() {
return map_factory.map_types();
}
-BOOST_PYTHON_MODULE(_index)
+BOOST_PYTHON_MODULE(index)
{
docstring_options doc_options(true, true, false);
diff --git a/lib/io.cc b/lib/io.cc
index 6e8c44e..213cefe 100644
--- a/lib/io.cc
+++ b/lib/io.cc
@@ -4,7 +4,7 @@
#include "osm.cc"
-BOOST_PYTHON_MODULE(_io)
+BOOST_PYTHON_MODULE(io)
{
using namespace boost::python;
docstring_options doc_options(true, true, false);
diff --git a/lib/osm.cc b/lib/osm.cc
index c39e08c..a038302 100644
--- a/lib/osm.cc
+++ b/lib/osm.cc
@@ -1,6 +1,7 @@
+
+#include <cassert>
#include <time.h>
#include <boost/python.hpp>
-#include <datetime.h>
#include <osmium/osm.hpp>
#include <osmium/osm/entity_bits.hpp>
@@ -30,21 +31,20 @@ inline const char member_item_type(osmium::RelationMember& obj)
// Converter for osmium::Timestamp -> datetime.datetime
struct Timestamp_to_python {
static PyObject* convert(osmium::Timestamp const& s) {
- struct tm tm;
- time_t sse = s.seconds_since_epoch();
- gmtime_r(&sse, &tm);
-
- return boost::python::incref(
- PyDateTime_FromDateAndTime(tm.tm_year + 1900, tm.tm_mon + 1,
- tm.tm_mday, tm.tm_hour, tm.tm_min,
- tm.tm_sec, 0));
+#if PY_VERSION_HEX >= 0x03000000
+ static auto fconv = boost::python::import("datetime").attr("datetime").attr("fromtimestamp");
+ static boost::python::object utc = boost::python::import("datetime").attr("timezone").attr("utc");
+ return boost::python::incref(fconv(s.seconds_since_epoch(), utc).ptr());
+#else
+ static auto fconv = boost::python::import("datetime").attr("datetime").attr("utcfromtimestamp");
+ return boost::python::incref(fconv(s.seconds_since_epoch()).ptr());
+#endif
}
};
BOOST_PYTHON_MODULE(_osm)
{
- PyDateTime_IMPORT;
using namespace boost::python;
docstring_options doc_options(true, true, false);
diff --git a/lib/osmium.cc b/lib/osmium.cc
index 949f0e5..70d926a 100644
--- a/lib/osmium.cc
+++ b/lib/osmium.cc
@@ -1,11 +1,10 @@
-#include <boost/python.hpp>
-
#include <osmium/visitor.hpp>
#include <osmium/index/map/all.hpp>
#include <osmium/handler/node_locations_for_ways.hpp>
#include <osmium/area/multipolygon_collector.hpp>
#include <osmium/area/assembler.hpp>
+#include "generic_writer.hpp"
#include "generic_handler.hpp"
template <typename T>
@@ -24,11 +23,11 @@ void apply_reader_simple_with_location(osmium::io::Reader &rd,
PyObject *invalidLocationExceptionType = NULL;
PyObject *notFoundExceptionType = NULL;
-void translator1(osmium::invalid_location const& x) {
+void translator1(osmium::invalid_location const&) {
PyErr_SetString(invalidLocationExceptionType, "Invalid location");
}
-void translator2(osmium::not_found const& x) {
+void translator2(osmium::not_found const&) {
PyErr_SetString(notFoundExceptionType, "Element not found in index");
}
@@ -66,7 +65,7 @@ BOOST_PYTHON_MODULE(_osmium)
;
class_<SimpleHandlerWrap, boost::noncopyable>("SimpleHandler",
- "A handler implements custom processing of OSM data. For each data type "
+ "The most generic of OSM data handlers. For each data type "
"a callback can be implemented where the object is processed. Note that "
"all objects that are handed into the handler are only readable and are "
"only valid until the end of the callback is reached. Any data that "
@@ -103,4 +102,38 @@ BOOST_PYTHON_MODULE(_osmium)
"Apply a chain of handlers.");
def("apply", &apply_reader_simple<osmium::handler::NodeLocationsForWays<LocationTable>>);
def("apply", &apply_reader_simple_with_location<LocationTable>);
+
+ class_<SimpleWriterWrap, boost::noncopyable>("SimpleWriter",
+ "The most generic class to write osmium objects into a file. The writer "
+ "takes a file name as its mandatory parameter. The file must not yet "
+ "exists. The file type to output is determined from the file extension. "
+ "The second (optional) parameter is the buffer size. osmium caches the "
+ "output data in an internal memory buffer before writing it on disk. This "
+ "parameter allows to change the default buffer size of 4MB. Larger buffers "
+ "are normally better but you should be aware that there are normally multiple "
+ "buffers in use during the write process.",
+ init<const char*, unsigned long>())
+ .def(init<const char*>())
+ .def("add_node", &SimpleWriterWrap::add_node,
+ (arg("self"), arg("node")),
+ "Add a new node to the file. The node may be an ``osmium.osm.Node`` object, "
+ "an ``osmium.osm.mutable.Node`` object or any other Python object that "
+ "implements the same attributes.")
+ .def("add_way", &SimpleWriterWrap::add_way,
+ (arg("self"), arg("way")),
+ "Add a new way to the file. The way may be an ``osmium.osm.Way`` object, "
+ "an ``osmium.osm.mutable.Way`` object or any other Python object that "
+ "implements the same attributes.")
+ .def("add_relation", &SimpleWriterWrap::add_relation,
+ (arg("self"), arg("relation")),
+ "Add a new relation to the file. The relation may be an "
+ "``osmium.osm.Relation`` object, an ``osmium.osm.mutable.Way`` "
+ "object or any other Python object that implements the same attributes.")
+ .def("close", &SimpleWriterWrap::close,
+ args("self"),
+ "Flush the remaining buffers and close the writer. While it is not "
+ "strictly necessary to call this function explicitly, it is still "
+ "strongly recommended to close the writer as soon as possible, so "
+ "that the buffer memory can be freed.")
+ ;
}
diff --git a/osmium/__init__.py b/osmium/__init__.py
index bac8392..d65b238 100644
--- a/osmium/__init__.py
+++ b/osmium/__init__.py
@@ -1,4 +1,4 @@
-from ._osmium import *
+from osmium._osmium import *
import osmium.io
import osmium.osm
import osmium.index
diff --git a/osmium/geom/__init__.py b/osmium/geom/__init__.py
deleted file mode 100644
index 8cb8e1c..0000000
--- a/osmium/geom/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from ._geom import *
diff --git a/osmium/index/__init__.py b/osmium/index/__init__.py
deleted file mode 100644
index a9af58a..0000000
--- a/osmium/index/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from ._index import *
diff --git a/osmium/io/__init__.py b/osmium/io/__init__.py
deleted file mode 100644
index 262a731..0000000
--- a/osmium/io/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from ._io import *
diff --git a/osmium/osm/__init__.py b/osmium/osm/__init__.py
index 7af4485..8a4908b 100644
--- a/osmium/osm/__init__.py
+++ b/osmium/osm/__init__.py
@@ -1 +1,15 @@
from ._osm import *
+import osmium.osm.mutable
+
+def create_mutable_node(node, **args):
+ return osmium.osm.mutable.Node(base=node, **args)
+
+def create_mutable_way(node, **args):
+ return osmium.osm.mutable.Way(base=node, **args)
+
+def create_mutable_relation(node, **args):
+ return osmium.osm.mutable.Relation(base=node, **args)
+
+Node.replace = create_mutable_node
+Way.replace = create_mutable_way
+Relation.replace = create_mutable_relation
diff --git a/osmium/osm/mutable.py b/osmium/osm/mutable.py
new file mode 100644
index 0000000..edf0421
--- /dev/null
+++ b/osmium/osm/mutable.py
@@ -0,0 +1,75 @@
+class OSMObject(object):
+ """Mutable version of ``osmium.osm.OSMObject``. It exposes the following
+ attributes ``id``, ``version``, ``visible``, ``changeset``, ``timestamp``,
+ ``uid`` and ``tags``. Timestamps may be strings or datetime objects.
+ Tags can be an osmium.osm.TagList, a dict-like object
+ or a list of tuples, where each tuple contains a (key value) string pair.
+
+ If the ``base`` parameter is given in the constructor, then the object
+ will be initialised first from the attributes of this base object.
+ """
+
+ def __init__(self, base=None, id=None, version=None, visible=None, changeset=None,
+ timestamp=None, uid=None, tags=None):
+ if base is None:
+ self.id = id
+ self.version = version
+ self.visible = visible
+ self.changeset = changeset
+ self.timestamp = timestamp
+ self.uid = uid
+ self.tags = tags
+ else:
+ self.id = base.id if id is None else id
+ self.version = base.version if version is None else version
+ self.visible = base.visible if visible is None else visible
+ self.changeset = base.changeset if changeset is None else changeset
+ self.timestamp = base.timestamp if timestamp is None else timestamp
+ self.uid = base.uid if uid is None else uid
+ self.tags = base.tags if tags is None else tags
+
+
+class Node(OSMObject):
+ """The mutable version of ``osmium.osm.Node``. It inherits all attributes
+ from osmium.osm.mutable.OSMObject and adds a `location` attribute. This
+ may either be an `osmium.osm.Location` or a tuple of lon/lat coordinates.
+ """
+
+ def __init__(self, base=None, location=None, **attrs):
+ OSMObject.__init__(self, base=base, **attrs)
+ if base is None:
+ self.location = location
+ else:
+ self.location = loctation if location is not None else base.location
+
+
+class Way(OSMObject):
+ """The mutable version of ``osmium.osm.Way``. It inherits all attributes
+ from osmium.osm.mutable.OSMObject and adds a `nodes` attribute. This may
+ either be and ``osmium.osm.NodeList`` or a list consisting of
+ ``osmium.osm.NodeRef`` or simple node ids.
+ """
+
+ def __init__(self, base=None, nodes=None, **attrs):
+ OSMObject.__init__(self, base=base, **attrs)
+ if base is None:
+ self.nodes = nodes
+ else:
+ self.nodes = nodes if nodes is not None else base.nodes
+
+class Relation(OSMObject):
+ """The mutable version of ``osmium.osm.Relation``. It inherits all attributes
+ from osmium.osm.mutable.OSMObject and adds a `members` attribute. This
+ may either be an ``osmium.osm.RelationMemberList`` or a list consisting
+ of ``osmium.osm.RelationMember`` or tuples of (type, id, role). The
+ member type should be a single character 'n', 'w' or 'r'.
+ """
+
+ def __init__(self, base=None, members=None, **attrs):
+ OSMObject.__init__(self, base=base, **attrs)
+ if base is None:
+ self.members = members
+ else:
+ self.members = members if members is not None else base.members
+
+
diff --git a/setup.py b/setup.py
index 913e495..d97ae28 100644
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,17 @@ extensions.append(Extension('osmium._osmium',
))
packages = ['osmium']
-for ext in ('io', 'osm', 'index', 'geom'):
+for ext in ('io', 'index', 'geom'):
+ extensions.append(Extension('osmium.%s' % ext,
+ sources = ['lib/%s.cc' % ext],
+ include_dirs = includes,
+ libraries = libs,
+ library_dirs = libdirs,
+ language = 'c++',
+ extra_compile_args = extra_compile_args
+ ))
+
+for ext in ('osm', ):
extensions.append(Extension('osmium.%s._%s' % (ext, ext),
sources = ['lib/%s.cc' % ext],
include_dirs = includes,
@@ -59,9 +69,8 @@ for ext in ('io', 'osm', 'index', 'geom'):
))
packages.append('osmium.%s' % ext)
-
setup (name = 'pyosmium',
- version = '2.5.4',
+ version = '2.6.0',
description = 'Provides python bindings for libosmium.',
packages = packages,
ext_modules = extensions)
diff --git a/test/test_osm.py b/test/test_osm.py
index 0dd3fd6..86821db 100644
--- a/test/test_osm.py
+++ b/test/test_osm.py
@@ -1,8 +1,19 @@
from nose.tools import *
import unittest
import os
+import sys
from datetime import datetime
+if sys.version_info[0] == 3:
+ from datetime import timezone
+
+ def mkdate(*args):
+ return datetime(*args, tzinfo=timezone.utc)
+else:
+ def mkdate(*args):
+ return datetime(*args)
+
+
from helpers import create_osm_file, osmobj, HandlerTestBase
import osmium as o
@@ -33,7 +44,7 @@ class TestNodeAttributes(HandlerTestBase, unittest.TestCase):
assert_equals(n.changeset, 58674)
assert_equals(n.uid, 42)
assert_equals(n.user_is_anonymous(), False)
- assert_equals(n.timestamp, datetime(2014, 1, 31, 6, 23, 35))
+ assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35))
assert_equals(n.user, 'anonymous')
assert_equals(n.positive_id(), 1)
@@ -66,7 +77,7 @@ class TestWayAttributes(HandlerTestBase, unittest.TestCase):
assert_equals(n.changeset, 58674)
assert_equals(n.uid, 42)
assert_equals(n.user_is_anonymous(), False)
- assert_equals(n.timestamp, datetime(2014, 1, 31, 6, 23, 35))
+ assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35))
assert_equals(n.user, 'anonymous')
assert_equals(n.positive_id(), 1)
assert_false(n.is_closed())
@@ -87,7 +98,7 @@ class TestRelationAttributes(HandlerTestBase, unittest.TestCase):
assert_equals(n.changeset, 58674)
assert_equals(n.uid, 42)
assert_equals(n.user_is_anonymous(), False)
- assert_equals(n.timestamp, datetime(2014, 1, 31, 6, 23, 35))
+ assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35))
assert_equals(n.user, 'anonymous')
assert_equals(n.positive_id(), 1)
@@ -109,7 +120,7 @@ class TestAreaFromWayAttributes(HandlerTestBase, unittest.TestCase):
assert_equals(n.changeset, 58674)
assert_equals(n.uid, 42)
assert_equals(n.user_is_anonymous(), False)
- assert_equals(n.timestamp, datetime(2014, 1, 31, 6, 23, 35))
+ assert_equals(n.timestamp, mkdate(2014, 1, 31, 6, 23, 35))
assert_equals(n.user, 'anonymous')
assert_equals(n.positive_id(), 46)
assert_equals(n.orig_id(), 23)
@@ -139,8 +150,8 @@ class TestChangesetAttributes(HandlerTestBase, unittest.TestCase):
assert_equals(1, c.uid)
assert_false(c.user_is_anonymous())
assert_equals("Steve", c.user)
- assert_equals(datetime(2005, 4, 9, 19, 54, 13), c.created_at)
- assert_equals(datetime(2005, 4, 9, 20, 54, 39), c.closed_at)
+ assert_equals(mkdate(2005, 4, 9, 19, 54, 13), c.created_at)
+ assert_equals(mkdate(2005, 4, 9, 20, 54, 39), c.closed_at)
assert_false(c.open)
assert_equals(2, c.num_changes)
assert_equals(0, len(c.tags))
diff --git a/test/test_writer.py b/test/test_writer.py
new file mode 100644
index 0000000..04f7062
--- /dev/null
+++ b/test/test_writer.py
@@ -0,0 +1,142 @@
+from nose.tools import *
+import unittest
+import tempfile
+import os
+from contextlib import contextmanager
+from datetime import datetime
+from collections import OrderedDict
+import logging
+import sys
+
+import osmium as o
+
+log = logging.getLogger(__name__)
+
+if sys.version_info[0] == 3:
+ from datetime import timezone
+
+ def mkdate(*args):
+ return datetime(*args, tzinfo=timezone.utc)
+else:
+ def mkdate(*args):
+ return datetime(*args)
+
+ at contextmanager
+def WriteExpect(expected):
+ fname = tempfile.mktemp(dir='/tmp', suffix='.opl')
+ writer = o.SimpleWriter(fname, 1024*1024)
+ try:
+ yield writer
+ finally:
+ writer.close()
+
+ with open(fname, 'r') as fd:
+ line = fd.readline().strip()
+ assert_equals(line, expected)
+ os.remove(fname)
+
+class O(object):
+ def __init__(self, **params):
+ for k,v in params.items():
+ setattr(self, k, v)
+
+class TestWriteSimpleAttributes(unittest.TestCase):
+
+ test_data_simple_attr = (
+ (O(id=None), '0 v0 dV c0 t i0 u T'),
+ (O(visible=None), '0 v0 dV c0 t i0 u T'),
+ (O(version=None), '0 v0 dV c0 t i0 u T'),
+ (O(uid=None), '0 v0 dV c0 t i0 u T'),
+ (O(user=None), '0 v0 dV c0 t i0 u T'),
+ (O(timestamp=None), '0 v0 dV c0 t i0 u T'),
+ (O(id=1), '1 v0 dV c0 t i0 u T'),
+ (O(id=-99), '-99 v0 dV c0 t i0 u T'),
+ (O(visible=True), '0 v0 dV c0 t i0 u T'),
+ (O(visible=False), '0 v0 dD c0 t i0 u T'),
+ (O(version=23), '0 v23 dV c0 t i0 u T'),
+ (O(user="Schmidt"), '0 v0 dV c0 t i0 uSchmidt T'),
+ (O(user=""), '0 v0 dV c0 t i0 u T'),
+ (O(uid=987), '0 v0 dV c0 t i987 u T'),
+ (O(timestamp='2012-04-14T20:58:35Z'), '0 v0 dV c0 t2012-04-14T20:58:35Z i0 u T'),
+ (O(timestamp=mkdate(2009, 4, 14, 20, 58, 35)), '0 v0 dV c0 t2009-04-14T20:58:35Z i0 u T'),
+ (O(timestamp='1970-01-01T00:00:01Z'), '0 v0 dV c0 t1970-01-01T00:00:01Z i0 u T'),
+ )
+
+ def test_node_simple_attr(self):
+ for node, out in self.test_data_simple_attr:
+ with WriteExpect('n' + out + ' x y') as w:
+ w.add_node(node)
+
+ def test_way_simple_attr(self):
+ for way, out in self.test_data_simple_attr:
+ with WriteExpect('w' + out + ' N') as w:
+ w.add_way(way)
+
+ def test_relation_simple_attr(self):
+ for rel, out in self.test_data_simple_attr:
+ with WriteExpect('r' + out + ' M') as w:
+ w.add_relation(rel)
+
+class TestWriteTags(unittest.TestCase):
+
+ test_data_tags = (
+ (None, 'T'),
+ ([], 'T'),
+ ({}, 'T'),
+ ((("foo", "bar"), ), 'Tfoo=bar'),
+ ((("foo", "bar"), ("2", "1")), 'Tfoo=bar,2=1'),
+ ({'test' : 'drive'}, 'Ttest=drive'),
+ (OrderedDict((('a', 'b'), ('c', '3'))), 'Ta=b,c=3'),
+ )
+
+ def test_node_tags(self):
+ for tags, out in self.test_data_tags:
+ with WriteExpect('n0 v0 dV c0 t i0 u ' + out + ' x y') as w:
+ w.add_node(O(tags=tags))
+
+ def test_way_tags(self):
+ for tags, out in self.test_data_tags:
+ with WriteExpect('w0 v0 dV c0 t i0 u ' + out + ' N') as w:
+ w.add_way(O(tags=tags))
+
+ def test_relation_tags(self):
+ for tags, out in self.test_data_tags:
+ with WriteExpect('r0 v0 dV c0 t i0 u ' + out + ' M') as w:
+ w.add_relation(O(tags=tags))
+
+
+class TestWriteNode(unittest.TestCase):
+
+ def test_location_tuple(self):
+ with WriteExpect('n0 v0 dV c0 t i0 u T x1.0000000 y2.0000000') as w:
+ w.add_node(O(location=(1, 2)))
+
+ def test_location_none(self):
+ with WriteExpect('n0 v0 dV c0 t i0 u T x y') as w:
+ w.add_node(O(location=None))
+
+class TestWriteWay(unittest.TestCase):
+
+ def test_node_list(self):
+ with WriteExpect('w0 v0 dV c0 t i0 u T Nn1,n2,n3,n-4') as w:
+ w.add_way(O(nodes=(1, 2, 3, -4)))
+
+ def test_node_list_none(self):
+ with WriteExpect('w0 v0 dV c0 t i0 u T N') as w:
+ w.add_way(O(nodes=None))
+
+class TestWriteRelation(unittest.TestCase):
+
+ def test_relation_members(self):
+ with WriteExpect('r0 v0 dV c0 t i0 u T Mn34 at foo,r200@,w1111 at x') as w:
+ w.add_relation(O(members=(('n', 34, 'foo'),
+ ('r', 200, ''),
+ ('w', 1111, 'x')
+ )))
+
+ def test_relation_members_None(self):
+ with WriteExpect('r0 v0 dV c0 t i0 u T M') as w:
+ w.add_relation(O(members=None))
+
+if __name__ == '__main__':
+ unittest.main()
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/pyosmium.git
More information about the Pkg-grass-devel
mailing list