[Git][debian-gis-team/python-pdal][master] 6 commits: New upstream version 2.2.0+ds
Bas Couwenberg
gitlab at salsa.debian.org
Sat Sep 7 07:23:29 BST 2019
Bas Couwenberg pushed to branch master at Debian GIS Project / python-pdal
Commits:
5a313af5 by Bas Couwenberg at 2019-09-07T05:56:39Z
New upstream version 2.2.0+ds
- - - - -
46ef7a14 by Bas Couwenberg at 2019-09-07T05:56:40Z
Update upstream source from tag 'upstream/2.2.0+ds'
Update to upstream version '2.2.0+ds'
with Debian dir ec93c3c38b898180cce6bc59e00c280b5e4a9b65
- - - - -
0de8c9a2 by Bas Couwenberg at 2019-09-07T05:56:53Z
New upstream release.
- - - - -
6b978bfb by Bas Couwenberg at 2019-09-07T05:59:13Z
Update copyright years for copyright holders.
- - - - -
a2687e19 by Bas Couwenberg at 2019-09-07T06:13:10Z
Refresh patches.
- - - - -
a8f729e0 by Bas Couwenberg at 2019-09-07T06:13:22Z
Set distribution to unstable.
- - - - -
15 changed files:
- PKG-INFO
- README.rst
- VERSION.txt
- debian/changelog
- debian/copyright
- debian/patches/clean-target.patch
- + pdal/PyArray.cpp
- pdal/PyArray.hpp
- pdal/PyPipeline.cpp
- pdal/PyPipeline.hpp
- pdal/__init__.py
- pdal/libpdalpython.cpp
- pdal/libpdalpython.pyx
- setup.py
- test/test_pipeline.py
Changes:
=====================================
PKG-INFO
=====================================
@@ -1,6 +1,6 @@
Metadata-Version: 1.2
Name: PDAL
-Version: 2.1.8
+Version: 2.2.0
Summary: Point cloud data processing
Home-page: http://pdal.io
Author: Howard Butler
@@ -60,6 +60,9 @@ Description: ===================================================================
.. image:: https://travis-ci.org/PDAL/python.svg?branch=master
:target: https://travis-ci.org/PDAL/python
+ .. image:: https://ci.appveyor.com/api/projects/status/of4kecyahpo8892d
+ :target: https://ci.appveyor.com/project/hobu/python/
+
Requirements
================================================================================
@@ -93,5 +96,5 @@ Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Topic :: Scientific/Engineering :: GIS
-Requires: Python (>=2.7)
+Requires: Python (>=3.0)
Requires: Numpy
=====================================
README.rst
=====================================
@@ -50,6 +50,9 @@ sorts it by the ``X`` dimension:
.. image:: https://travis-ci.org/PDAL/python.svg?branch=master
:target: https://travis-ci.org/PDAL/python
+.. image:: https://ci.appveyor.com/api/projects/status/of4kecyahpo8892d
+ :target: https://ci.appveyor.com/project/hobu/python/
+
Requirements
================================================================================
=====================================
VERSION.txt
=====================================
@@ -1 +1 @@
-2.1.8
\ No newline at end of file
+2.2.0
\ No newline at end of file
=====================================
debian/changelog
=====================================
@@ -1,3 +1,11 @@
+python-pdal (2.2.0+ds-1) unstable; urgency=medium
+
+ * New upstream release.
+ * Update copyright years for copyright holders.
+ * Refresh patches.
+
+ -- Bas Couwenberg <sebastic at debian.org> Sat, 07 Sep 2019 08:13:12 +0200
+
python-pdal (2.1.8+ds-3) unstable; urgency=medium
* Add filenamemangle to distinguish it from pdal releases.
=====================================
debian/copyright
=====================================
@@ -7,9 +7,8 @@ Comment: The upstream sources are repacked to excluded the .egg-info
Files-Excluded: PDAL.egg-info/*
Files: *
-Copyright: 2015, Hobu, Inc. <howard at hobu.co>
- 2016, 2018, Howard Butler <howard at hobu.co>
- 2011, Michael P. Gerlek <mpg at flaxen.com>
+Copyright: 2015, 2019, Hobu, Inc. <howard at hobu.co>
+ 2016, 2018, Howard Butler <howard at hobu.co>
License: BSD-3-Clause
Files: setup.py
=====================================
debian/patches/clean-target.patch
=====================================
@@ -1,16 +1,15 @@
-Description: Don't append library in clean target.
+Description: Fix clean target.
Author: Bas Couwenberg <sebastic at debian.org>
-Forwarded: https://github.com/PDAL/python/pull/24
-Applied-Upstream: https://github.com/PDAL/python/commit/b8b192925814828cabdb4e527697b77c30edd943
+Forwarded: https://github.com/PDAL/python/pull/32
--- a/setup.py
+++ b/setup.py
-@@ -157,7 +157,7 @@ if DEBUG:
+@@ -156,7 +156,7 @@ if DEBUG:
+ if os.name != 'nt':
extra_compile_args += ['-g','-O0']
- # readers.numpy doesn't exist until PDAL 1.8
--if PDALVERSION >= Version('1.8'):
-+if PDALVERSION is not None and PDALVERSION >= Version('1.8'):
- libraries.append('pdal_plugin_reader_numpy')
+-if PDALVERSION < Version('2.0.0'):
++if PDALVERSION is not None and PDALVERSION < Version('2.0.0'):
+ raise Exception("PDAL version '%s' is not compatible with PDAL Python library version '%s'"%(PDALVERSION, module_version))
+
- if os.name in ['nt']:
=====================================
pdal/PyArray.cpp
=====================================
@@ -0,0 +1,339 @@
+/******************************************************************************
+* Copyright (c) 2019, Hobu Inc. (info at hobu.co)
+*
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following
+* conditions are met:
+*
+* * Redistributions of source code must retain the above copyright
+* notice, this list of conditions and the following disclaimer.
+* * Redistributions in binary form must reproduce the above copyright
+* notice, this list of conditions and the following disclaimer in
+* the documentation and/or other materials provided
+* with the distribution.
+* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
+* names of its contributors may be used to endorse or promote
+* products derived from this software without specific prior
+* written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+* OF SUCH DAMAGE.
+****************************************************************************/
+
+#include "PyArray.hpp"
+#include <pdal/io/MemoryViewReader.hpp>
+
+#include <numpy/arrayobject.h>
+
+namespace pdal
+{
+namespace python
+{
+
+namespace
+{
+
+Dimension::Type pdalType(int t)
+{
+ using namespace Dimension;
+
+ switch (t)
+ {
+ case NPY_FLOAT32:
+ return Type::Float;
+ case NPY_FLOAT64:
+ return Type::Double;
+ case NPY_INT8:
+ return Type::Signed8;
+ case NPY_INT16:
+ return Type::Signed16;
+ case NPY_INT32:
+ return Type::Signed32;
+ case NPY_INT64:
+ return Type::Signed64;
+ case NPY_UINT8:
+ return Type::Unsigned8;
+ case NPY_UINT16:
+ return Type::Unsigned16;
+ case NPY_UINT32:
+ return Type::Unsigned32;
+ case NPY_UINT64:
+ return Type::Unsigned64;
+ default:
+ return Type::None;
+ }
+ assert(0);
+
+ return Type::None;
+}
+
+std::string toString(PyObject *pname)
+{
+ PyObject* r = PyObject_Str(pname);
+ if (!r)
+ throw pdal_error("couldn't make string representation value");
+ Py_ssize_t size;
+ return std::string(PyUnicode_AsUTF8AndSize(r, &size));
+}
+
+} // unnamed namespace
+
+Array::Array() : m_array(nullptr)
+{
+ if (_import_array() < 0)
+ throw pdal_error("Could not import numpy.core.multiarray.");
+}
+
+Array::Array(PyArrayObject* array) : m_array(array), m_rowMajor(true)
+{
+ if (_import_array() < 0)
+ throw pdal_error("Could not import numpy.core.multiarray.");
+
+ Py_XINCREF(array);
+
+ PyArray_Descr *dtype = PyArray_DTYPE(m_array);
+ npy_intp ndims = PyArray_NDIM(m_array);
+ npy_intp *shape = PyArray_SHAPE(m_array);
+ int numFields = (dtype->fields == Py_None) ?
+ 0 :
+ static_cast<int>(PyDict_Size(dtype->fields));
+
+ int xyz = 0;
+ if (numFields == 0)
+ {
+ if (ndims != 3)
+ throw pdal_error("Array without fields must have 3 dimensions.");
+ m_fields.push_back({"Intensity", pdalType(dtype->type_num), 0});
+ }
+ else
+ {
+ PyObject *names_dict = dtype->fields;
+ PyObject *names = PyDict_Keys(names_dict);
+ PyObject *values = PyDict_Values(names_dict);
+ if (!names || !values)
+ throw pdal_error("Bad field specification in numpy array.");
+
+ for (int i = 0; i < numFields; ++i)
+ {
+ std::string name = toString(PyList_GetItem(names, i));
+ if (name == "X")
+ xyz |= 1;
+ else if (name == "Y")
+ xyz |= 2;
+ else if (name == "Z")
+ xyz |= 4;
+ PyObject *tup = PyList_GetItem(values, i);
+
+ // Get offset.
+ size_t offset = PyLong_AsLong(PySequence_Fast_GET_ITEM(tup, 1));
+
+ // Get type.
+ PyArray_Descr *descriptor =
+ (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0);
+ Dimension::Type type = pdalType(descriptor->type_num);
+ if (type == Dimension::Type::None)
+ throw pdal_error("Incompatible type for field '" + name + "'.");
+
+ m_fields.push_back({name, type, offset});
+ }
+
+ if (xyz != 0 && xyz != 7)
+ throw pdal_error("Array fields must contain all or none "
+ "of X, Y and Z");
+ if (xyz == 0 && ndims != 3)
+ throw pdal_error("Array without named X/Y/Z fields "
+ "must have three dimensions.");
+ }
+ if (xyz == 0)
+ m_shape = { (size_t)shape[0], (size_t)shape[1], (size_t)shape[2] };
+ m_rowMajor = !(PyArray_FLAGS(m_array) & NPY_ARRAY_F_CONTIGUOUS);
+}
+
+Array::~Array()
+{
+ if (m_array)
+ Py_XDECREF((PyObject *)m_array);
+}
+
+
+void Array::update(PointViewPtr view)
+{
+ if (m_array)
+ Py_XDECREF((PyObject *)m_array);
+ m_array = nullptr; // Just in case of an exception.
+
+ Dimension::IdList dims = view->dims();
+ npy_intp size = view->size();
+
+ PyObject *dtype_dict = (PyObject*)buildNumpyDescription(view);
+ if (!dtype_dict)
+ throw pdal_error("Unable to build numpy dtype "
+ "description dictionary");
+
+ PyArray_Descr *dtype = nullptr;
+ if (PyArray_DescrConverter(dtype_dict, &dtype) == NPY_FAIL)
+ throw pdal_error("Unable to build numpy dtype");
+ Py_XDECREF(dtype_dict);
+
+ // This is a 1 x size array.
+ m_array = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype,
+ 1, &size, 0, nullptr, NPY_ARRAY_CARRAY, nullptr);
+
+ // copy the data
+ DimTypeList types = view->dimTypes();
+ for (PointId idx = 0; idx < view->size(); idx++)
+ {
+ char *p = (char *)PyArray_GETPTR1(m_array, idx);
+ view->getPackedPoint(types, idx, p);
+ }
+}
+
+
+//ABELL - Who's responsible for incrementing the ref count?
+PyArrayObject *Array::getPythonArray() const
+{
+ return m_array;
+}
+
+PyObject* Array::buildNumpyDescription(PointViewPtr view) const
+{
+ // Build up a numpy dtype dictionary
+ //
+ // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1',
+ // 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'],
+ // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber',
+ // 'NumberOfReturns', 'ScanDirectionFlag', 'EdgeOfFlightLine',
+ // 'Classification', 'ScanAngleRank', 'UserData',
+ // 'PointSourceId', 'GpsTime', 'Red', 'Green', 'Blue']}
+ //
+
+ Dimension::IdList dims = view->dims();
+
+ PyObject* dict = PyDict_New();
+ PyObject* sizes = PyList_New(dims.size());
+ PyObject* formats = PyList_New(dims.size());
+ PyObject* titles = PyList_New(dims.size());
+
+ for (size_t i = 0; i < dims.size(); ++i)
+ {
+ Dimension::Id id = dims[i];
+ Dimension::Type t = view->dimType(id);
+ npy_intp stride = view->dimSize(id);
+
+ std::string name = view->dimName(id);
+
+ std::string kind("i");
+ Dimension::BaseType b = Dimension::base(t);
+ if (b == Dimension::BaseType::Unsigned)
+ kind = "u";
+ else if (b == Dimension::BaseType::Signed)
+ kind = "i";
+ else if (b == Dimension::BaseType::Floating)
+ kind = "f";
+ else
+ throw pdal_error("Unable to map kind '" + kind +
+ "' to PDAL dimension type");
+
+ std::stringstream oss;
+ oss << kind << stride;
+ PyObject* pySize = PyLong_FromLong(stride);
+ PyObject* pyTitle = PyUnicode_FromString(name.c_str());
+ PyObject* pyFormat = PyUnicode_FromString(oss.str().c_str());
+
+ PyList_SetItem(sizes, i, pySize);
+ PyList_SetItem(titles, i, pyTitle);
+ PyList_SetItem(formats, i, pyFormat);
+ }
+
+ PyDict_SetItemString(dict, "names", titles);
+ PyDict_SetItemString(dict, "formats", formats);
+
+ return dict;
+}
+
+bool Array::rowMajor() const
+{
+ return m_rowMajor;
+}
+
+Array::Shape Array::shape() const
+{
+ return m_shape;
+}
+
+const Array::Fields& Array::fields() const
+{
+ return m_fields;
+}
+
+ArrayIter& Array::iterator()
+{
+ ArrayIter *it = new ArrayIter(*this);
+ m_iterators.push_back(std::unique_ptr<ArrayIter>(it));
+ return *it;
+}
+
+ArrayIter::ArrayIter(Array& array)
+{
+ m_iter = NpyIter_New(array.getPythonArray(),
+ NPY_ITER_EXTERNAL_LOOP | NPY_ITER_READONLY | NPY_ITER_REFS_OK,
+ NPY_KEEPORDER, NPY_NO_CASTING, NULL);
+ if (!m_iter)
+ throw pdal_error("Unable to create numpy iterator.");
+
+ char *itererr;
+ m_iterNext = NpyIter_GetIterNext(m_iter, &itererr);
+ if (!m_iterNext)
+ {
+ NpyIter_Deallocate(m_iter);
+ throw pdal_error(std::string("Unable to create numpy iterator: ") +
+ itererr);
+ }
+ m_data = NpyIter_GetDataPtrArray(m_iter);
+ m_stride = NpyIter_GetInnerStrideArray(m_iter);
+ m_size = NpyIter_GetInnerLoopSizePtr(m_iter);
+ m_done = false;
+}
+
+ArrayIter::~ArrayIter()
+{
+ NpyIter_Deallocate(m_iter);
+}
+
+ArrayIter& ArrayIter::operator++()
+{
+ if (m_done)
+ return *this;
+
+ if (--(*m_size))
+ *m_data += *m_stride;
+ else if (!m_iterNext(m_iter))
+ m_done = true;
+ return *this;
+}
+
+ArrayIter::operator bool () const
+{
+ return !m_done;
+}
+
+char * ArrayIter::operator * () const
+{
+ return *m_data;
+}
+
+} // namespace python
+} // namespace pdal
+
=====================================
pdal/PyArray.hpp
=====================================
@@ -1,5 +1,5 @@
/******************************************************************************
-* Copyright (c) 2011, Michael P. Gerlek (mpg at flaxen.com)
+* Copyright (c) 2019, Hobu Inc. (info at hobu.co)
*
* All rights reserved.
*
@@ -13,7 +13,7 @@
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided
* with the distribution.
-* * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
+* * Neither the name of Hobu, Inc. nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior
* written permission.
@@ -34,204 +34,69 @@
#pragma once
-#include <pdal/PointView.hpp>
-
-#include <algorithm>
-
-#pragma warning(disable: 4127) // conditional expression is constant
-
-
-#include <Python.h>
-#undef toupper
-#undef tolower
-#undef isspace
+#include <numpy/ndarraytypes.h>
-#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
-#include <numpy/arrayobject.h>
-
-// forward declare PyObject so we don't need the python headers everywhere
-// see: http://mail.python.org/pipermail/python-dev/2003-August/037601.html
-#ifndef PyObject_HEAD
-struct _object;
-typedef _object PyObject;
-#endif
+#include <pdal/PointView.hpp>
+#include <pdal/io/MemoryViewReader.hpp>
namespace pdal
{
namespace python
{
+class ArrayIter;
class PDAL_DLL Array
{
public:
+ using Shape = std::array<size_t, 3>;
+ using Fields = std::vector<MemoryViewReader::Field>;
+
+ // Create an array for reading data from PDAL.
+ Array();
+
+ // Create an array for writing data to PDAL.
+ Array(PyArrayObject* array);
- Array() : m_py_array(0), m_own_array(true)
- {
-#undef NUMPY_IMPORT_ARRAY_RETVAL
-#define NUMPY_IMPORT_ARRAY_RETVAL
- import_array();
- }
-
- Array(PyObject* array) : m_py_array(array), m_own_array(false)
- {
-#undef NUMPY_IMPORT_ARRAY_RETVAL
-#define NUMPY_IMPORT_ARRAY_RETVAL
- import_array();
- if (!PyArray_Check(array))
- throw pdal::pdal_error("pdal::python::Array constructor object is not a numpy array");
- Py_XINCREF(array);
-
- }
-
- ~Array()
- {
- cleanup();
- }
-
-
- inline void update(PointViewPtr view)
- {
- typedef std::unique_ptr<std::vector<uint8_t>> DataPtr;
- cleanup();
- int nd = 1;
- Dimension::IdList dims = view->dims();
- npy_intp mydims = view->size();
- npy_intp* ndims = &mydims;
- std::vector<npy_intp> strides(dims.size());
-
- DataPtr pdata( new std::vector<uint8_t>(view->pointSize()* view->size(), 0));
-
- PyArray_Descr *dtype = nullptr;
- PyObject * dtype_dict = (PyObject*)buildNumpyDescription(view);
- if (!dtype_dict)
- throw pdal_error("Unable to build numpy dtype description dictionary");
-
- int did_convert = PyArray_DescrConverter(dtype_dict, &dtype);
- if (did_convert == NPY_FAIL)
- throw pdal_error("Unable to build numpy dtype");
- Py_XDECREF(dtype_dict);
-
-#ifdef NPY_ARRAY_CARRAY
- int flags = NPY_ARRAY_CARRAY;
-#else
- int flags = NPY_CARRAY;
-#endif
- uint8_t* sp = pdata.get()->data();
- PyObject * pyArray = PyArray_NewFromDescr(&PyArray_Type,
- dtype,
- nd,
- ndims,
- 0,
- sp,
- flags,
- NULL);
-
- // copy the data
- uint8_t* p(sp);
- DimTypeList types = view->dimTypes();
- for (PointId idx = 0; idx < view->size(); idx++)
- {
- p = sp + (view->pointSize() * idx);
- view->getPackedPoint(types, idx, (char*)p);
- }
-
- m_py_array = pyArray;
- m_data_array = std::move(pdata);
- }
-
-
- inline PyObject* getPythonArray() const
- {
- return m_py_array;
- }
+ ~Array();
+ void update(PointViewPtr view);
+ PyArrayObject *getPythonArray() const;
+ bool rowMajor() const;
+ Shape shape() const;
+ const Fields& fields() const;
+ ArrayIter& iterator();
private:
+ inline PyObject* buildNumpyDescription(PointViewPtr view) const;
- inline void cleanup()
- {
- PyObject* p = (PyObject*)(m_py_array);
- if (m_own_array)
- {
- m_data_array.reset();
- }
-
- Py_XDECREF(p);
- }
-
- inline PyObject* buildNumpyDescription(PointViewPtr view) const
- {
-
- // Build up a numpy dtype dictionary
- //
- // {'formats': ['f8', 'f8', 'f8', 'u2', 'u1', 'u1', 'u1', 'u1', 'u1', 'f4', 'u1', 'u2', 'f8', 'u2', 'u2', 'u2'],
- // 'names': ['X', 'Y', 'Z', 'Intensity', 'ReturnNumber', 'NumberOfReturns',
- // 'ScanDirectionFlag', 'EdgeOfFlightLine', 'Classification',
- // 'ScanAngleRank', 'UserData', 'PointSourceId', 'GpsTime', 'Red', 'Green',
- // 'Blue']}
- //
-
- std::stringstream oss;
- Dimension::IdList dims = view->dims();
-
- PyObject* dict = PyDict_New();
- PyObject* sizes = PyList_New(dims.size());
- PyObject* formats = PyList_New(dims.size());
- PyObject* titles = PyList_New(dims.size());
-
- for (Dimension::IdList::size_type i=0; i < dims.size(); ++i)
- {
- Dimension::Id id = (dims[i]);
- Dimension::Type t = view->dimType(id);
- npy_intp stride = view->dimSize(id);
-
- std::string name = view->dimName(id);
-
- std::string kind("i");
- Dimension::BaseType b = Dimension::base(t);
- if (b == Dimension::BaseType::Unsigned)
- kind = "u";
- else if (b == Dimension::BaseType::Signed)
- kind = "i";
- else if (b == Dimension::BaseType::Floating)
- kind = "f";
- else
- {
- std::stringstream o;
- oss << "unable to map kind '" << kind <<"' to PDAL dimension type";
- throw pdal::pdal_error(o.str());
- }
-
- oss << kind << stride;
- PyObject* pySize = PyLong_FromLong(stride);
- PyObject* pyTitle = PyUnicode_FromString(name.c_str());
- PyObject* pyFormat = PyUnicode_FromString(oss.str().c_str());
-
- PyList_SetItem(sizes, i, pySize);
- PyList_SetItem(titles, i, pyTitle);
- PyList_SetItem(formats, i, pyFormat);
-
- oss.str("");
- }
-
- PyDict_SetItemString(dict, "names", titles);
- PyDict_SetItemString(dict, "formats", formats);
-
- // PyObject* obj = PyUnicode_AsASCIIString(PyObject_Str(dict));
- // const char* s = PyBytes_AsString(obj);
- // std::string output(s);
- // std::cout << "array: " << output << std::endl;
- return dict;
- }
-
-
-
-
- PyObject* m_py_array;
- std::unique_ptr<std::vector<uint8_t> > m_data_array;
- bool m_own_array;
+ PyArrayObject* m_array;
Array& operator=(Array const& rhs);
+ Fields m_fields;
+ bool m_rowMajor;
+ Shape m_shape;
+ std::vector<std::unique_ptr<ArrayIter>> m_iterators;
+};
+
+class ArrayIter
+{
+public:
+ ArrayIter(const ArrayIter&) = delete;
+
+ ArrayIter(Array& array);
+ ~ArrayIter();
+
+ ArrayIter& operator++();
+ operator bool () const;
+ char *operator * () const;
+
+private:
+ NpyIter *m_iter;
+ NpyIter_IterNextFunc *m_iterNext;
+ char **m_data;
+ npy_intp *m_size;
+ npy_intp *m_stride;
+ bool m_done;
};
} // namespace python
=====================================
pdal/PyPipeline.cpp
=====================================
@@ -33,124 +33,125 @@
****************************************************************************/
#include "PyPipeline.hpp"
-#ifdef PDAL_HAVE_LIBXML2
-#include <pdal/XMLSchema.hpp>
-#endif
#ifndef _WIN32
#include <dlfcn.h>
#endif
#include <Python.h>
-#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#include <numpy/arrayobject.h>
-#include "PyArray.hpp"
#include <pdal/Stage.hpp>
#include <pdal/pdal_features.hpp>
-#include <pdal/PipelineWriter.hpp>
-#include <pdal/io/NumpyReader.hpp>
-
-namespace libpdalpython
-{
-using namespace pdal::python;
+#include "PyArray.hpp"
-Pipeline::Pipeline(std::string const& json, std::vector<Array*> arrays)
+namespace pdal
+{
+namespace python
{
+// Create a pipeline for writing data to PDAL
+Pipeline::Pipeline(std::string const& json, std::vector<Array*> arrays) :
+ m_executor(new PipelineExecutor(json))
+{
#ifndef _WIN32
+ // See comment in alternate constructor below.
::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL);
- ::dlopen("libpdal_plugin_reader_numpy.so", RTLD_NOLOAD | RTLD_GLOBAL);
#endif
-#undef NUMPY_IMPORT_ARRAY_RETVAL
-#define NUMPY_IMPORT_ARRAY_RETVAL
- import_array();
-
- m_executor = std::shared_ptr<pdal::PipelineExecutor>(new pdal::PipelineExecutor(json));
+ if (_import_array() < 0)
+ throw pdal_error("Could not impory numpy.core.multiarray.");
- pdal::PipelineManager& manager = m_executor->getManager();
+ PipelineManager& manager = m_executor->getManager();
std::stringstream strm(json);
manager.readPipeline(strm);
+ std::vector<Stage *> roots = manager.roots();
+ if (roots.size() != 1)
+ throw pdal_error("Filter pipeline must contain a single root stage.");
- pdal::Stage *r = manager.getStage();
- if (!r)
- throw pdal::pdal_error("pipeline had no stages!");
-
-#if PDAL_VERSION_MAJOR > 1 || PDAL_VERSION_MINOR >=8
- int counter = 1;
- for (auto array: arrays)
+ for (auto array : arrays)
{
// Create numpy reader for each array
- pdal::Options options;
- std::stringstream tag;
- tag << "readers_numpy" << counter;
- pdal::StageCreationOptions opts { "", "readers.numpy", nullptr, options, tag.str()};
- pdal::Stage& reader = manager.makeReader(opts);
-
- pdal::NumpyReader* np_reader = dynamic_cast<pdal::NumpyReader*>(&reader);
- if (!np_reader)
- throw pdal::pdal_error("couldn't cast reader!");
-
+ // Options
+
+ Options options;
+ options.add("order", array->rowMajor() ?
+ MemoryViewReader::Order::RowMajor :
+ MemoryViewReader::Order::ColumnMajor);
+ options.add("shape", MemoryViewReader::Shape(array->shape()));
+
+ Stage& s = manager.makeReader("", "readers.memoryview", options);
+ MemoryViewReader& r = dynamic_cast<MemoryViewReader &>(s);
+ for (auto f : array->fields())
+ r.pushField(f);
+
+ ArrayIter& iter = array->iterator();
+ auto incrementer = [&iter](PointId id) -> char *
+ {
+ if (! iter)
+ return nullptr;
+
+ char *c = *iter;
+ ++iter;
+ return c;
+ };
+
+ r.setIncrementer(incrementer);
PyObject* parray = (PyObject*)array->getPythonArray();
if (!parray)
- throw pdal::pdal_error("array was none!");
-
- np_reader->setArray(parray);
-
- r->setInput(reader);
- counter++;
+ throw pdal_error("array was none!");
+ roots[0]->setInput(r);
}
-#endif
manager.validateStageOptions();
}
-Pipeline::Pipeline(std::string const& json)
+// Create a pipeline for reading data from PDAL
+Pipeline::Pipeline(std::string const& json) :
+ m_executor(new PipelineExecutor(json))
{
// Make the symbols in pdal_base global so that they're accessible
// to PDAL plugins. Python dlopen's this extension with RTLD_LOCAL,
// which means that without this, symbols in libpdal_base aren't available
// for resolution of symbols on future runtime linking. This is an issue
- // on Apline and other Linux variants that doesn't use UNIQUE symbols
- // for C++ template statics. only
+ // on Alpine and other Linux variants that don't use UNIQUE symbols
+ // for C++ template statics only. Without this, you end up with multiple
+ // copies of template statics.
#ifndef _WIN32
::dlopen("libpdal_base.so", RTLD_NOLOAD | RTLD_GLOBAL);
#endif
-#undef NUMPY_IMPORT_ARRAY_RETVAL
-#define NUMPY_IMPORT_ARRAY_RETVAL
- import_array();
-
- m_executor = std::shared_ptr<pdal::PipelineExecutor>(new pdal::PipelineExecutor(json));
+ if (_import_array() < 0)
+ throw pdal_error("Could not impory numpy.core.multiarray.");
}
Pipeline::~Pipeline()
-{
-}
+{}
+
void Pipeline::setLogLevel(int level)
{
m_executor->setLogLevel(level);
}
+
int Pipeline::getLogLevel() const
{
return static_cast<int>(m_executor->getLogLevel());
}
+
int64_t Pipeline::execute()
{
-
- int64_t count = m_executor->execute();
- return count;
+ return m_executor->execute();
}
bool Pipeline::validate()
{
- return m_executor->validate();
+ auto res = m_executor->validate();
+ return res;
}
std::vector<Array *> Pipeline::getArrays() const
@@ -160,16 +161,18 @@ std::vector<Array *> Pipeline::getArrays() const
if (!m_executor->executed())
throw python_error("call execute() before fetching arrays");
- const pdal::PointViewSet& pvset = m_executor->getManagerConst().views();
+ const PointViewSet& pvset = m_executor->getManagerConst().views();
for (auto i: pvset)
{
//ABELL - Leak?
- Array *array = new pdal::python::Array;
+ Array *array = new python::Array;
array->update(i);
output.push_back(array);
}
return output;
}
-} //namespace libpdalpython
+
+} // namespace python
+} // namespace pdal
=====================================
pdal/PyPipeline.hpp
=====================================
@@ -43,20 +43,12 @@
#include <sstream>
#include <memory>
-#undef toupper
-#undef tolower
-#undef isspace
-
namespace pdal
{
namespace python
{
- class Array;
-}
-}
-namespace libpdalpython
-{
+class Array;
class python_error : public std::runtime_error
{
@@ -65,10 +57,12 @@ public:
{}
};
-class Pipeline {
+class Pipeline
+{
public:
Pipeline(std::string const& json);
- Pipeline(std::string const& json, std::vector<pdal::python::Array*> arrays);
+ Pipeline(std::string const& json,
+ std::vector<pdal::python::Array*> arrays);
~Pipeline();
int64_t execute();
@@ -98,4 +92,5 @@ private:
std::shared_ptr<pdal::PipelineExecutor> m_executor;
};
-}
+} // namespace python
+} // namespace pdal
=====================================
pdal/__init__.py
=====================================
@@ -1,4 +1,4 @@
-__version__='2.1.8'
+__version__='2.2.0'
from .pipeline import Pipeline
from .array import Array
=====================================
pdal/libpdalpython.cpp
=====================================
The diff for this file was not included because it is too large.
=====================================
pdal/libpdalpython.pyx
=====================================
@@ -23,7 +23,6 @@ cdef extern from "pdal/pdal_config.hpp" namespace "pdal::Config":
def getVersionString():
return versionString()
-
def getVersionMajor():
return versionMajor()
def getVersionMinor():
@@ -39,10 +38,10 @@ def getPluginInstallPath():
cdef extern from "PyArray.hpp" namespace "pdal::python":
cdef cppclass Array:
- Array(object) except +
- void* getPythonArray() except+
+ Array(np.ndarray) except +
+ void *getPythonArray() except+
-cdef extern from "PyPipeline.hpp" namespace "libpdalpython":
+cdef extern from "PyPipeline.hpp" namespace "pdal::python":
cdef cppclass Pipeline:
Pipeline(const char* ) except +
Pipeline(const char*, vector[Array*]& ) except +
@@ -56,11 +55,9 @@ cdef extern from "PyPipeline.hpp" namespace "libpdalpython":
int getLogLevel()
void setLogLevel(int)
-
-
cdef class PyArray:
cdef Array *thisptr
- def __cinit__(self, object array):
+ def __cinit__(self, np.ndarray array):
self.thisptr = new Array(array)
def __dealloc__(self):
del self.thisptr
@@ -109,24 +106,14 @@ cdef class PyPipeline:
cdef Array* a
if arrays is not None:
+ print("Looping arrays\n")
for array in arrays:
a = new Array(array)
c_arrays.push_back(a)
- if PY_MAJOR_VERSION >= 3:
- if arrays:
- self.thisptr = new Pipeline(json.encode('UTF-8'), c_arrays)
- else:
- self.thisptr = new Pipeline(json.encode('UTF-8'))
+ self.thisptr = new Pipeline(json.encode('UTF-8'), c_arrays)
else:
- if arrays:
- self.thisptr = new Pipeline(json, c_arrays)
- else:
- self.thisptr = new Pipeline(json)
-# if arrays:
-# self.thisptr = new Pipeline(json.encode('UTF-8'), c_arrays)
-# else:
-# self.thisptr = new Pipeline(json.encode('UTF-8'))
+ self.thisptr = new Pipeline(json.encode('UTF-8'))
def __dealloc__(self):
del self.thisptr
@@ -158,6 +145,7 @@ cdef class PyPipeline:
return json.loads(j)
property arrays:
+
def __get__(self):
v = self.thisptr.getArrays()
output = []
@@ -171,6 +159,7 @@ cdef class PyPipeline:
inc(it)
return output
+
def execute(self):
if not self.thisptr:
raise Exception("C++ Pipeline object not constructed!")
=====================================
setup.py
=====================================
@@ -156,9 +156,9 @@ if DEBUG:
if os.name != 'nt':
extra_compile_args += ['-g','-O0']
-# readers.numpy doesn't exist until PDAL 1.8
-if PDALVERSION >= Version('1.8'):
- libraries.append('pdal_plugin_reader_numpy')
+if PDALVERSION < Version('2.0.0'):
+ raise Exception("PDAL version '%s' is not compatible with PDAL Python library version '%s'"%(PDALVERSION, module_version))
+
if os.name in ['nt']:
if os.environ.get('OSGEO4W_ROOT'):
@@ -168,8 +168,6 @@ if os.name in ['nt']:
library_dirs = ['%s\Library\lib' % prefix]
libraries = ['pdalcpp','pdal_util','ws2_32']
- if PDALVERSION >= Version('1.8'):
- libraries.append('libpdal_plugin_reader_numpy')
extra_compile_args = ['/DNOMINMAX',]
@@ -182,7 +180,7 @@ if 'linux' in sys.platform or 'linux2' in sys.platform or 'darwin' in sys.platfo
-sources=['pdal/libpdalpython'+ext, "pdal/PyPipeline.cpp" ]
+sources=['pdal/libpdalpython'+ext, "pdal/PyPipeline.cpp", "pdal/PyArray.cpp" ]
extensions = [DistutilsExtension("*",
sources,
include_dirs=include_dirs,
@@ -192,12 +190,12 @@ extensions = [DistutilsExtension("*",
extra_link_args=extra_link_args,)]
if USE_CYTHON and "clean" not in sys.argv:
from Cython.Build import cythonize
- extensions= cythonize(extensions, language="c++")
+ extensions= cythonize(extensions, compiler_directives={'language_level':3})
setup_args = dict(
name = 'PDAL',
version = str(module_version),
- requires = ['Python (>=2.7)', 'Numpy'],
+ requires = ['Python (>=3.0)', 'Numpy'],
description = 'Point cloud data processing',
license = 'BSD',
keywords = 'point cloud spatial',
=====================================
test/test_pipeline.py
=====================================
@@ -31,14 +31,14 @@ class PDALTest(unittest.TestCase):
return output
class TestPipeline(PDALTest):
-#
+
@unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')),
"missing test data")
def test_construction(self):
"""Can we construct a PDAL pipeline"""
json = self.fetch_json('sort.json')
r = pdal.Pipeline(json)
-#
+
@unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')),
"missing test data")
def test_execution(self):
@@ -48,13 +48,13 @@ class TestPipeline(PDALTest):
r.validate()
r.execute()
self.assertGreater(len(r.pipeline), 200)
-#
+
def test_validate(self):
"""Do we complain with bad pipelines"""
r = pdal.Pipeline(bad_json)
with self.assertRaises(RuntimeError):
r.validate()
-#
+
@unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')),
"missing test data")
def test_array(self):
@@ -65,11 +65,11 @@ class TestPipeline(PDALTest):
r.execute()
arrays = r.arrays
self.assertEqual(len(arrays), 1)
-#
+
a = arrays[0]
self.assertAlmostEqual(a[0][0], 635619.85, 7)
self.assertAlmostEqual(a[1064][2], 456.92, 7)
-#
+
@unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')),
"missing test data")
def test_metadata(self):
@@ -82,8 +82,8 @@ class TestPipeline(PDALTest):
import json
j = json.loads(metadata)
self.assertEqual(j["metadata"]["readers.las"][0]["count"], 1065)
-#
-#
+
+
@unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')),
"missing test data")
def test_no_execute(self):
@@ -93,17 +93,17 @@ class TestPipeline(PDALTest):
with self.assertRaises(RuntimeError):
r.arrays
#
- @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'reproject.json')),
- "missing test data")
- def test_logging(self):
- """Can we fetch log output"""
- json = self.fetch_json('reproject.json')
- r = pdal.Pipeline(json)
- r.loglevel = 8
- r.validate()
- count = r.execute()
- self.assertEqual(count, 789)
- self.assertEqual(r.log.split()[0], '(pypipeline')
+# @unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'reproject.json')),
+# "missing test data")
+# def test_logging(self):
+# """Can we fetch log output"""
+# json = self.fetch_json('reproject.json')
+# r = pdal.Pipeline(json)
+# r.loglevel = 8
+# r.validate()
+# count = r.execute()
+# self.assertEqual(count, 789)
+# self.assertEqual(r.log.split()[0], '(pypipeline')
#
@unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'sort.json')),
"missing test data")
@@ -114,7 +114,7 @@ class TestPipeline(PDALTest):
r.validate()
r.execute()
self.assertEqual(r.schema['schema']['dimensions'][0]['name'], 'X')
-#
+
@unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'chip.json')),
"missing test data")
def test_merged_arrays(self):
@@ -125,16 +125,17 @@ class TestPipeline(PDALTest):
r.execute()
arrays = r.arrays
self.assertEqual(len(arrays), 43)
-#
+
+
class TestArrayLoad(PDALTest):
@unittest.skipUnless(os.path.exists(os.path.join(DATADIRECTORY, 'perlin.npy')),
"missing test data")
def test_merged_arrays(self):
- """Can we load data from a a list of arrays to PDAL"""
+ """Can we load data from a list of arrays to PDAL"""
if Version(pdal.info.version) < Version('1.8'):
return True
- data = np.load(os.path.join(DATADIRECTORY, 'perlin.npy'))
+ data = np.load(os.path.join(DATADIRECTORY, 'test3d.npy'))
arrays = [data, data, data]
@@ -143,7 +144,7 @@ class TestArrayLoad(PDALTest):
"pipeline":[
{
"type":"filters.range",
- "limits":"Intensity[0:0.10]"
+ "limits":"Intensity[100:300)"
}
]
}"""
@@ -154,9 +155,9 @@ class TestArrayLoad(PDALTest):
arrays = p.arrays
self.assertEqual(len(arrays), 3)
- data = arrays[0]
- self.assertEqual(len(data), 1836)
- self.assertEqual(sum([len(i) for i in arrays]), 3*1836)
+ for data in arrays:
+ self.assertEqual(len(data), 12)
+ self.assertEqual(data['Intensity'].sum(), 1926)
class TestDimensions(PDALTest):
def test_fetch_dimensions(self):
View it on GitLab: https://salsa.debian.org/debian-gis-team/python-pdal/compare/ab8e6273e083cb527782fda0eecc2dae1ea37df4...a8f729e0e08edef82226ede11044a9bb95daf8c3
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/python-pdal/compare/ab8e6273e083cb527782fda0eecc2dae1ea37df4...a8f729e0e08edef82226ede11044a9bb95daf8c3
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20190907/65c7ae1a/attachment-0001.html>
More information about the Pkg-grass-devel
mailing list