[Git][debian-gis-team/pdal][upstream] New upstream version 2.2.0~rc3+ds
Bas Couwenberg
gitlab at salsa.debian.org
Sat Sep 5 06:18:36 BST 2020
Bas Couwenberg pushed to branch upstream at Debian GIS Project / pdal
Commits:
6c68f077 by Bas Couwenberg at 2020-09-05T06:36:49+02:00
New upstream version 2.2.0~rc3+ds
- - - - -
13 changed files:
- RELEASENOTES.txt
- doc/development/release-notes/2.2.0.md
- plugins/tiledb/io/TileDBReader.cpp
- plugins/tiledb/io/TileDBWriter.cpp
- plugins/tiledb/io/TileDBWriter.hpp
- plugins/tiledb/test/TileDBReaderTest.cpp
- plugins/tiledb/test/TileDBWriterTest.cpp
- − test/data/tiledb/array/__20c2cf6f2bf546adbc7fe83cb44f180b_1554218258459/OffsetTime.tdb
- − test/data/tiledb/array/__20c2cf6f2bf546adbc7fe83cb44f180b_1554218258459/__coords.tdb
- − test/data/tiledb/array/__20c2cf6f2bf546adbc7fe83cb44f180b_1554218258459/__fragment_metadata.tdb
- − test/data/tiledb/array/__array_schema.tdb
- − test/data/tiledb/array/__lock.tdb
- − test/data/tiledb/array/pdal.json
Changes:
=====================================
RELEASENOTES.txt
=====================================
@@ -97,6 +97,8 @@ Enhancements
directory.
- ``filters.assign`` added the ``value`` option to allow standard
expression-based conditional assignment of values.(#1468)
+- ``readers.tiledb`` and ``writers.tiledb`` now support TileDB version
+ 2 and duplicate points when using version 2 of TileDB. (#3030, #3047)
Bug Fixes
=========
=====================================
doc/development/release-notes/2.2.0.md
=====================================
@@ -50,6 +50,7 @@
- `readers.i3s` is now streamable.
- `readers.slpk` no longer expands the source file into a temporary directory.
- `filters.assign` added the `value` option to allow standard expression-based conditional assignment of values.(#1468)
+- `readers.tiledb` and `writers.tiledb` now support TileDB version 2 and duplicate points when using version 2 of TileDB. (#3030, #3047)
# Bug Fixes
=====================================
plugins/tiledb/io/TileDBReader.cpp
=====================================
@@ -127,6 +127,9 @@ void TileDBReader::initialize()
try
{
+ if (m_stats)
+ tiledb::Stats::enable();
+
m_array.reset(new tiledb::Array(*m_ctx, m_filename, TILEDB_READ));
}
catch (const tiledb::TileDBError& err)
@@ -149,8 +152,13 @@ void TileDBReader::addDimensions(PointLayoutPtr layout)
DimInfo di;
di.m_name = dim.name();
+#if TILEDB_VERSION_MAJOR == 1
di.m_offset = i;
di.m_span = dims.size();
+#else
+ di.m_offset = 0;
+ di.m_span = 1;
+#endif
di.m_dimCategory = DimCategory::Dimension;
di.m_tileType = dim.type();
di.m_type = getPdalType(di.m_tileType);
@@ -251,22 +259,27 @@ void TileDBReader::localReady()
DimInfo& di = *it;
Buffer *dimBuf = new Buffer(di.m_tileType, m_chunkSize * numDims);
+
+#if TILEDB_VERSION_MAJOR == 1
m_query->set_coordinates(dimBuf->get<double>(), dimBuf->count());
m_buffers.push_back(std::unique_ptr<Buffer>(dimBuf));
+#endif
for (DimInfo& di : m_dims)
{
// All dimensions use the same buffer.
+#if TILEDB_VERSION_MAJOR == 1
if (di.m_dimCategory == DimCategory::Dimension)
- di.m_buffer = dimBuf;
- else
{
- std::unique_ptr<Buffer> dimBuf(
- new Buffer(di.m_tileType, m_chunkSize));
- di.m_buffer = dimBuf.get();
- m_buffers.push_back(std::move(dimBuf));
- setQueryBuffer(di);
+ di.m_buffer = dimBuf;
+ continue;
}
+#endif
+ std::unique_ptr<Buffer> dimBuf(
+ new Buffer(di.m_tileType, m_chunkSize));
+ di.m_buffer = dimBuf.get();
+ m_buffers.push_back(std::move(dimBuf));
+ setQueryBuffer(di);
}
// Set the extent of the query.
@@ -295,7 +308,7 @@ void TileDBReader::localReady()
// read spatial reference
NL::json meta = nullptr;
-#if TILEDB_VERSION_MAJOR >= 1 && TILEDB_VERSION_MINOR >= 7
+#if TILEDB_VERSION_MAJOR > 1 || TILEDB_VERSION_MINOR >= 7
tiledb_datatype_t v_type = TILEDB_UINT8;
const void* v_r;
uint32_t v_num;
@@ -310,7 +323,7 @@ void TileDBReader::localReady()
tiledb::VFS::filebuf fbuf(vfs);
std::string metaFName = m_filename + pathSeparator + "pdal.json";
- if (vfs.is_dir(m_filename))
+ if (vfs.is_file(metaFName))
{
auto nBytes = vfs.file_size(metaFName);
tiledb::VFS::filebuf fbuf(vfs);
@@ -412,13 +425,13 @@ bool TileDBReader::processPoint(PointRef& point)
else
{
tiledb::Query::Status status;
- if (m_stats)
- tiledb::Stats::enable();
+
m_query->submit();
+
if (m_stats)
{
tiledb::Stats::dump(stdout);
- tiledb::Stats::disable();
+ tiledb::Stats::reset();
}
status = m_query->query_status();
@@ -427,10 +440,13 @@ bool TileDBReader::processPoint(PointRef& point)
// returned by the query for dimensions. So if there are three
// dimensions, the number of points returned is the buffer count
// divided by the number of dimensions.
+#if TILEDB_VERSION_MAJOR == 1
m_resultSize =
(int)m_query->result_buffer_elements()[TILEDB_COORDS].second /
m_array->schema().domain().dimensions().size();
-
+#else
+ m_resultSize = (int)m_query->result_buffer_elements()["X"].second;
+#endif
if (status == tiledb::Query::Status::INCOMPLETE &&
m_resultSize == 0)
throwError("Need to increase chunk_size for reader.");
=====================================
plugins/tiledb/io/TileDBWriter.cpp
=====================================
@@ -328,7 +328,9 @@ void TileDBWriter::initialize()
{
opts = m_args->m_defaults["coords"];
}
-
+#if TILEDB_VERSION_MAJOR > 1
+ m_schema->set_allows_dups(true);
+#endif
m_schema->set_coords_filter_list(
*createFilterList(*m_ctx, opts));
}
@@ -345,6 +347,9 @@ void TileDBWriter::ready(pdal::BasePointTable &table)
auto layout = table.layout();
auto all = layout->dims();
+ if (m_args->m_stats)
+ tiledb::Stats::enable();
+
// get a list of all the dimensions & their types and add to schema
// x,y,z will be tiledb dimensions other pdal dimensions will be
// tiledb attributes
@@ -427,8 +432,6 @@ void TileDBWriter::ready(pdal::BasePointTable &table)
TILEDB_WRITE));
}
- m_query.reset(new tiledb::Query(*m_ctx, *m_array));
- m_query->set_layout(TILEDB_UNORDERED);
m_current_idx = 0;
}
@@ -442,9 +445,9 @@ bool TileDBWriter::processOne(PointRef& point)
for (auto& a : m_attrs)
writeAttributeValue(a, point, m_current_idx);
- m_coords.push_back(x);
- m_coords.push_back(y);
- m_coords.push_back(z);
+ m_xs.push_back(x);
+ m_ys.push_back(y);
+ m_zs.push_back(z);
if (++m_current_idx == m_args->m_cache_size)
{
@@ -524,7 +527,25 @@ void TileDBWriter::done(PointTableRef table)
bool TileDBWriter::flushCache(size_t size)
{
- m_query->set_coordinates(m_coords);
+ tiledb::Query query(*m_ctx, *m_array);
+ query.set_layout(TILEDB_UNORDERED);
+
+#if TILEDB_VERSION_MAJOR == 1
+ // backwards compatibility requires a copy
+ std::vector<double> coords;
+
+ for(unsigned i = 0; i < m_xs.size(); i++)
+ {
+ coords.push_back(m_xs[i]);
+ coords.push_back(m_ys[i]);
+ coords.push_back(m_zs[i]);
+ }
+ query.set_coordinates(coords);
+#else
+ query.set_buffer("X", m_xs);
+ query.set_buffer("Y", m_ys);
+ query.set_buffer("Z", m_zs);
+#endif
// set tiledb buffers
for (const auto& a : m_attrs)
@@ -533,43 +554,43 @@ bool TileDBWriter::flushCache(size_t size)
switch (a.m_type)
{
case Dimension::Type::Double:
- m_query->set_buffer(a.m_name, reinterpret_cast<double *>(buf),
+ query.set_buffer(a.m_name, reinterpret_cast<double *>(buf),
size);
break;
case Dimension::Type::Float:
- m_query->set_buffer(a.m_name, reinterpret_cast<float *>(buf),
+ query.set_buffer(a.m_name, reinterpret_cast<float *>(buf),
size);
break;
case Dimension::Type::Signed8:
- m_query->set_buffer(a.m_name, reinterpret_cast<int8_t *>(buf),
+ query.set_buffer(a.m_name, reinterpret_cast<int8_t *>(buf),
size);
break;
case Dimension::Type::Signed16:
- m_query->set_buffer(a.m_name, reinterpret_cast<int16_t *>(buf),
+ query.set_buffer(a.m_name, reinterpret_cast<int16_t *>(buf),
size);
break;
case Dimension::Type::Signed32:
- m_query->set_buffer(a.m_name, reinterpret_cast<int32_t *>(buf),
+ query.set_buffer(a.m_name, reinterpret_cast<int32_t *>(buf),
size);
break;
case Dimension::Type::Signed64:
- m_query->set_buffer(a.m_name, reinterpret_cast<int64_t *>(buf),
+ query.set_buffer(a.m_name, reinterpret_cast<int64_t *>(buf),
size);
break;
case Dimension::Type::Unsigned8:
- m_query->set_buffer(a.m_name, reinterpret_cast<uint8_t *>(buf),
+ query.set_buffer(a.m_name, reinterpret_cast<uint8_t *>(buf),
size);
break;
case Dimension::Type::Unsigned16:
- m_query->set_buffer(a.m_name, reinterpret_cast<uint16_t *>(buf),
+ query.set_buffer(a.m_name, reinterpret_cast<uint16_t *>(buf),
size);
break;
case Dimension::Type::Unsigned32:
- m_query->set_buffer(a.m_name, reinterpret_cast<uint32_t *>(buf),
+ query.set_buffer(a.m_name, reinterpret_cast<uint32_t *>(buf),
size);
break;
case Dimension::Type::Unsigned64:
- m_query->set_buffer(a.m_name, reinterpret_cast<uint64_t *>(buf),
+ query.set_buffer(a.m_name, reinterpret_cast<uint64_t *>(buf),
size);
break;
case Dimension::Type::None:
@@ -578,19 +599,18 @@ bool TileDBWriter::flushCache(size_t size)
}
}
- if (m_args->m_stats)
- tiledb::Stats::enable();
-
- tiledb::Query::Status status = m_query->submit();
+ tiledb::Query::Status status = query.submit();
if (m_args->m_stats)
{
tiledb::Stats::dump(stdout);
- tiledb::Stats::disable();
+ tiledb::Stats::reset();
}
m_current_idx = 0;
- m_coords.clear();
+ m_xs.clear();
+ m_ys.clear();
+ m_zs.clear();
if (status == tiledb::Query::Status::FAILED)
return false;
=====================================
plugins/tiledb/io/TileDBWriter.hpp
=====================================
@@ -80,9 +80,10 @@ private:
std::unique_ptr<tiledb::Context> m_ctx;
std::unique_ptr<tiledb::ArraySchema> m_schema;
std::unique_ptr<tiledb::Array> m_array;
- std::unique_ptr<tiledb::Query> m_query;
std::vector<DimBuffer> m_attrs;
- std::vector<double> m_coords;
+ std::vector<double> m_xs;
+ std::vector<double> m_ys;
+ std::vector<double> m_zs;
TileDBWriter(const TileDBWriter&) = delete;
TileDBWriter& operator=(const TileDBWriter&) = delete;
=====================================
plugins/tiledb/test/TileDBReaderTest.cpp
=====================================
@@ -58,7 +58,31 @@ class TileDBReaderTest : public ::testing::Test
protected:
virtual void SetUp()
{
+ tiledb::Context ctx;
+ FauxReader rdr;
+ TileDBWriter writer;
+ Options writer_options;
+ Options reader_options;
+
+ if (Utils::fileExists(data_path))
+ tiledb::Object::remove(ctx, data_path);
+
+ writer_options.add("array_name", data_path);
+
+ reader_options.add("mode", "ramp");
+ reader_options.add("count", 100);
+ rdr.setOptions(reader_options);
+
+ writer.setOptions(writer_options);
+ writer.setInput(rdr);
+ writer.setSpatialReference(utm16);
+
+ FixedPointTable table(10);
+ writer.prepare(table);
+ writer.execute(table);
}
+
+ std::string data_path = Support::temppath("tiledb_array");
};
TEST_F(TileDBReaderTest, constructor)
@@ -78,9 +102,8 @@ class TileDBReaderTest : public ::testing::Test
{
tiledb::Context ctx;
tiledb::VFS vfs(ctx);
- std::string pth(Support::datapath("tiledb/array"));
Options options;
- options.add("array_name", pth);
+ options.add("array_name", data_path);
options.add("bbox3d", "([0, 0.5], [0, 0.5], [0, 0.5])");
TileDBReader reader;
@@ -96,9 +119,8 @@ class TileDBReaderTest : public ::testing::Test
{
tiledb::Context ctx;
tiledb::VFS vfs(ctx);
- std::string pth(Support::datapath("tiledb/array"));
Options options;
- options.add("array_name", pth);
+ options.add("array_name", data_path);
options.add("bbox3d", "([1.1, 1.2], [1.1, 1.2], [1.1, 1.2])");
TileDBReader reader;
@@ -161,11 +183,10 @@ class TileDBReaderTest : public ::testing::Test
tiledb::Context ctx;
tiledb::VFS vfs(ctx);
- std::string pth(Support::datapath("tiledb/array"));
Options options;
- options.add("array_name", pth);
+ options.add("array_name", data_path);
- tiledb::Array array(ctx, pth, TILEDB_READ);
+ tiledb::Array array(ctx, data_path, TILEDB_READ);
auto domain = array.non_empty_domain<double>();
std::vector<double> subarray;
@@ -193,11 +214,9 @@ class TileDBReaderTest : public ::testing::Test
c.setInput(reader);
c.prepare(table);
c.execute(table);
- // test using a sidecar file
- EXPECT_EQ(reader.getSpatialReference(), utm16);
+ EXPECT_TRUE(reader.getSpatialReference().equals(utm16));
}
-#if TILEDB_VERSION_MAJOR >= 1 && TILEDB_VERSION_MINOR >= 7
TEST_F(TileDBReaderTest, spatial_reference)
{
tiledb::Context ctx;
@@ -233,8 +252,7 @@ class TileDBReaderTest : public ::testing::Test
FixedPointTable table2(100);
rdr.prepare(table2);
rdr.execute(table2);
- EXPECT_EQ(rdr.getSpatialReference(), utm16);
+ EXPECT_TRUE(rdr.getSpatialReference().equals(utm16));
}
-#endif
}
=====================================
plugins/tiledb/test/TileDBWriterTest.cpp
=====================================
@@ -387,4 +387,60 @@ namespace pdal
tiledb::FilterList flAtts = att.filter_list();
EXPECT_EQ(flAtts.nfilters(), 0U);
}
+
+#if TILEDB_VERSION_MAJOR > 1
+ TEST_F(TileDBWriterTest, dup_points)
+ {
+ Options reader_options;
+ FauxReader reader;
+ BOX3D bounds(1.0, 1.0, 1.0, 2.0, 2.0, 2.0);
+ reader_options.add("bounds", bounds);
+ reader_options.add("mode", "constant");
+ reader_options.add("count", count);
+ reader.setOptions(reader_options);
+
+ tiledb::Context ctx;
+ tiledb::VFS vfs(ctx);
+ std::string pth = Support::temppath("tiledb_test_dups");
+
+ Options writer_options;
+ writer_options.add("array_name", pth);
+
+ if (vfs.is_dir(pth))
+ {
+ vfs.remove_dir(pth);
+ }
+
+ TileDBWriter writer;
+ writer.setOptions(writer_options);
+ writer.setInput(reader);
+
+ FixedPointTable table(count);
+ writer.prepare(table);
+ writer.execute(table);
+
+ tiledb::Array array(ctx, pth, TILEDB_READ);
+ auto domain = array.non_empty_domain<double>();
+ std::vector<double> subarray;
+
+ for (const auto& kv: domain)
+ {
+ subarray.push_back(kv.second.first);
+ subarray.push_back(kv.second.second);
+ }
+
+ tiledb::Query q(ctx, array, TILEDB_READ);
+ q.set_subarray(subarray);
+
+ auto max_el = array.max_buffer_elements(subarray);
+ std::vector<double> coords(max_el[TILEDB_COORDS].second);
+ q.set_coordinates(coords);
+ q.submit();
+ array.close();
+
+ EXPECT_EQ(reader.count() * 3, coords.size());
+ for (const double& v : coords)
+ EXPECT_EQ(v, 1.0);
+ }
+#endif
}
=====================================
test/data/tiledb/array/__20c2cf6f2bf546adbc7fe83cb44f180b_1554218258459/OffsetTime.tdb deleted
=====================================
Binary files a/test/data/tiledb/array/__20c2cf6f2bf546adbc7fe83cb44f180b_1554218258459/OffsetTime.tdb and /dev/null differ
=====================================
test/data/tiledb/array/__20c2cf6f2bf546adbc7fe83cb44f180b_1554218258459/__coords.tdb deleted
=====================================
Binary files a/test/data/tiledb/array/__20c2cf6f2bf546adbc7fe83cb44f180b_1554218258459/__coords.tdb and /dev/null differ
=====================================
test/data/tiledb/array/__20c2cf6f2bf546adbc7fe83cb44f180b_1554218258459/__fragment_metadata.tdb deleted
=====================================
Binary files a/test/data/tiledb/array/__20c2cf6f2bf546adbc7fe83cb44f180b_1554218258459/__fragment_metadata.tdb and /dev/null differ
=====================================
test/data/tiledb/array/__array_schema.tdb deleted
=====================================
Binary files a/test/data/tiledb/array/__array_schema.tdb and /dev/null differ
=====================================
test/data/tiledb/array/__lock.tdb deleted
=====================================
=====================================
test/data/tiledb/array/pdal.json deleted
=====================================
@@ -1,12 +0,0 @@
-{
- "bounds": "([0, 1], [0, 1], [0, 1])",
- "pipeline":
- [
- {
- "type": "readers.tiledb"
- }
- ],
- "writers.tiledb": {
- "spatialreference": "EPSG:26916"
- }
-}
View it on GitLab: https://salsa.debian.org/debian-gis-team/pdal/-/commit/6c68f077470ef807d1c6c2e2785a5d0b74900ee5
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/pdal/-/commit/6c68f077470ef807d1c6c2e2785a5d0b74900ee5
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20200905/9264fc9c/attachment-0001.html>
More information about the Pkg-grass-devel
mailing list