[mapnik] 01/04: Imported Upstream version 3.0.15~rc1+ds

Bas Couwenberg sebastic at debian.org
Thu Jun 15 16:39:42 UTC 2017


This is an automated email from the git hooks/post-receive script.

sebastic pushed a commit to branch master
in repository mapnik.

commit 8d6418fd226b97cd8c024c12964089d7f5e3899b
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date:   Thu Jun 15 17:57:23 2017 +0200

    Imported Upstream version 3.0.15~rc1+ds
---
 CHANGELOG.md                               |  18 +++-
 include/mapnik/version.hpp                 |   2 +-
 plugins/input/gdal/gdal_featureset.cpp     |  13 +--
 plugins/input/raster/raster_featureset.cpp |   7 +-
 plugins/input/raster/raster_featureset.hpp |   1 +
 src/tiff_reader.cpp                        | 130 +++++++++++++++++++----------
 test/unit/imaging/tiff_io.cpp              |  16 ++--
 7 files changed, 130 insertions(+), 57 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b984b33..40609d2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,11 +6,26 @@ Developers: Please commit along with changes.
 
 For a complete change history, see the git log.
 
+## 3.0.15
+
+Released: June 15, 2017
+
+#### Summary
+
+(Packaged from xxxxxxx)
+
+- Restored `filter_factor` logic in `gdal.input` and added to `raster.input` (#3699)
+   (updated tests https://github.com/mapnik/test-data-visual/commit/fd518f1f512b8aea4ac740c2ce12c249616a291c)
+- Fixed bug related to rows swapping implementation in `tiff_reader` ref #3679
+   (updated visual tests to catch this regression in the future
+      https://github.com/mapnik/test-data-visual/commit/be0ba965cd2240576a8edfca84801cbb7a4832d6)
+- TIFF I/O - port memory mapped based I/O from master
+
 ## 3.0.14
 
 Released: June 5, 2017
 
-(Packaged from 7660b9e)
+(Packaged from 2b42e17)
 
 #### Summary
 
@@ -20,6 +35,7 @@ Released: June 5, 2017
 - TIFF I/O - added support for grey scale multiband images + fixed and made generic `read_stripped` and `read_generic`.
 - shapeindex - return error code when no features can read from shapefile (#3198)
 - Upgrade Scons to `2.5.1`
+- Fixed bug (typo) in `raster_featureset.cpp` (#3696)
 
 ## 3.0.13
 
diff --git a/include/mapnik/version.hpp b/include/mapnik/version.hpp
index afa1f3a..9798d98 100644
--- a/include/mapnik/version.hpp
+++ b/include/mapnik/version.hpp
@@ -27,7 +27,7 @@
 
 #define MAPNIK_MAJOR_VERSION 3
 #define MAPNIK_MINOR_VERSION 0
-#define MAPNIK_PATCH_VERSION 14
+#define MAPNIK_PATCH_VERSION 15
 
 #define MAPNIK_VERSION (MAPNIK_MAJOR_VERSION*100000) + (MAPNIK_MINOR_VERSION*100) + (MAPNIK_PATCH_VERSION)
 
diff --git a/plugins/input/gdal/gdal_featureset.cpp b/plugins/input/gdal/gdal_featureset.cpp
index f71535f..8f755fd 100644
--- a/plugins/input/gdal/gdal_featureset.cpp
+++ b/plugins/input/gdal/gdal_featureset.cpp
@@ -149,6 +149,9 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
     box2d<double> intersect = raster_extent_.intersect(q.get_bbox());
     box2d<double> box = t.forward(intersect);
 
+    // get the filter factor if it is set
+    double filter_factor = q.get_filter_factor();
+
     //size of resized output pixel in source image domain
     double margin_x = 1.0 / (std::fabs(dx_) * std::get<0>(q.resolution()));
     double margin_y = 1.0 / (std::fabs(dy_) * std::get<1>(q.resolution()));
@@ -224,7 +227,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
                 {
                     throw datasource_exception(CPLGetLastErrorMsg());
                 }
-                mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, image, 0.0);
+                mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, image, filter_factor);
                 // set nodata value to be used in raster colorizer
                 if (nodata_value_) raster->set_nodata(*nodata_value_);
                 else raster->set_nodata(raster_nodata);
@@ -244,7 +247,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
                 {
                     throw datasource_exception(CPLGetLastErrorMsg());
                 }
-                mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, image, 0.0);
+                mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, image, filter_factor);
                 // set nodata value to be used in raster colorizer
                 if (nodata_value_) raster->set_nodata(*nodata_value_);
                 else raster->set_nodata(raster_nodata);
@@ -263,7 +266,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
                 {
                     throw datasource_exception(CPLGetLastErrorMsg());
                 }
-                mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, image, 0.0);
+                mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, image, filter_factor);
                 // set nodata value to be used in raster colorizer
                 if (nodata_value_) raster->set_nodata(*nodata_value_);
                 else raster->set_nodata(raster_nodata);
@@ -283,7 +286,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
                 {
                     throw datasource_exception(CPLGetLastErrorMsg());
                 }
-                mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, image, 0.0);
+                mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, image, filter_factor);
                 // set nodata value to be used in raster colorizer
                 if (nodata_value_) raster->set_nodata(*nodata_value_);
                 else raster->set_nodata(raster_nodata);
@@ -583,7 +586,7 @@ feature_ptr gdal_featureset::get_feature(mapnik::query const& q)
                     }
                 }
             }
-            mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, image, 0.0);
+            mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, image, filter_factor);
             // set nodata value to be used in raster colorizer
             if (nodata_value_) raster->set_nodata(*nodata_value_);
             else raster->set_nodata(raster_nodata);
diff --git a/plugins/input/raster/raster_featureset.cpp b/plugins/input/raster/raster_featureset.cpp
index edea943..adaf655 100644
--- a/plugins/input/raster/raster_featureset.cpp
+++ b/plugins/input/raster/raster_featureset.cpp
@@ -55,7 +55,8 @@ raster_featureset<LookupPolicy>::raster_featureset(LookupPolicy const& policy,
       extent_(extent),
       bbox_(q.get_bbox()),
       curIter_(policy_.begin()),
-      endIter_(policy_.end())
+      endIter_(policy_.end()),
+      filter_factor_(q.get_filter_factor())
 {
 }
 
@@ -97,7 +98,7 @@ feature_ptr raster_featureset<LookupPolicy>::next()
 
                     // clip to available data
                     if (x_off >= image_width) x_off = image_width - 1;
-                    if (y_off >= image_width) y_off = image_width - 1;
+                    if (y_off >= image_height) y_off = image_height - 1;
                     if (x_off < 0) x_off = 0;
                     if (y_off < 0) y_off = 0;
                     if (end_x > image_width)  end_x = image_width;
@@ -115,7 +116,7 @@ feature_ptr raster_featureset<LookupPolicy>::next()
                                                         rem.maxy() + y_off + height);
                     feature_raster_extent = t.backward(feature_raster_extent);
                     mapnik::image_any data = reader->read(x_off, y_off, width, height);
-                    mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, std::move(data), 1.0);
+                    mapnik::raster_ptr raster = std::make_shared<mapnik::raster>(feature_raster_extent, intersect, std::move(data), filter_factor_);
                     feature->set_raster(raster);
                 }
             }
diff --git a/plugins/input/raster/raster_featureset.hpp b/plugins/input/raster/raster_featureset.hpp
index 0076892..6daa898 100644
--- a/plugins/input/raster/raster_featureset.hpp
+++ b/plugins/input/raster/raster_featureset.hpp
@@ -321,6 +321,7 @@ private:
     mapnik::box2d<double> bbox_;
     iterator_type curIter_;
     iterator_type endIter_;
+    double filter_factor_;
 };
 
 #endif // RASTER_FEATURESET_HPP
diff --git a/src/tiff_reader.cpp b/src/tiff_reader.cpp
index 81ac112..65850e4 100644
--- a/src/tiff_reader.cpp
+++ b/src/tiff_reader.cpp
@@ -29,11 +29,20 @@ extern "C"
 #include <tiffio.h>
 }
 
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
+#pragma GCC diagnostic push
+#include <mapnik/warning_ignore.hpp>
+#include <boost/interprocess/mapped_region.hpp>
+#include <boost/interprocess/streams/bufferstream.hpp>
+#pragma GCC diagnostic pop
+#include <mapnik/mapped_memory_cache.hpp>
+#endif
+
 // stl
 #include <memory>
 #include <fstream>
 
-namespace mapnik { namespace impl {
+namespace mapnik { namespace detail {
 
 static toff_t tiff_seek_proc(thandle_t handle, toff_t off, int whence)
 {
@@ -93,6 +102,19 @@ static int tiff_map_proc(thandle_t, tdata_t* , toff_t*)
     return 0;
 }
 
+template <typename T>
+struct tiff_io_traits
+{
+    using input_stream_type = std::istream;
+};
+
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
+template <>
+struct tiff_io_traits<boost::interprocess::ibufferstream>
+{
+    using input_stream_type = boost::interprocess::ibufferstream;
+};
+#endif
 }
 
 template <typename T>
@@ -100,16 +122,16 @@ class tiff_reader : public image_reader
 {
     using tiff_ptr = std::shared_ptr<TIFF>;
     using source_type = T;
-    using input_stream = std::istream;
+    using input_stream = typename detail::tiff_io_traits<source_type>::input_stream_type;
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
+    mapnik::mapped_region_ptr mapped_region_;
+#endif
 
     struct tiff_closer
     {
         void operator() (TIFF * tif)
         {
-            if (tif != 0)
-            {
-                TIFFClose(tif);
-            }
+            if (tif != 0) TIFFClose(tif);
         }
     };
 
@@ -183,7 +205,11 @@ namespace
 
 image_reader* create_tiff_reader(std::string const& filename)
 {
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
+    return new tiff_reader<boost::interprocess::ibufferstream>(filename);
+#else
     return new tiff_reader<std::filebuf>(filename);
+#endif
 }
 
 image_reader* create_tiff_reader2(char const * data, std::size_t size)
@@ -198,27 +224,49 @@ const bool registered2 = register_image_reader("tiff", create_tiff_reader2);
 
 template <typename T>
 tiff_reader<T>::tiff_reader(std::string const& filename)
-    : source_(),
-      stream_(&source_),
-      tif_(nullptr),
-      read_method_(generic),
-      rows_per_strip_(0),
-      tile_width_(0),
-      tile_height_(0),
-      width_(0),
-      height_(0),
-      bps_(0),
-      sample_format_(SAMPLEFORMAT_UINT),
-      photometric_(0),
-      bands_(1),
-      planar_config_(PLANARCONFIG_CONTIG),
-      compression_(COMPRESSION_NONE),
-      has_alpha_(false),
-      is_tiled_(false)
+    :
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
+    stream_(),
+#else
+    source_(),
+    stream_(&source_),
+#endif
+
+    tif_(nullptr),
+    read_method_(generic),
+    rows_per_strip_(0),
+    tile_width_(0),
+    tile_height_(0),
+    width_(0),
+    height_(0),
+    bps_(0),
+    sample_format_(SAMPLEFORMAT_UINT),
+    photometric_(0),
+    bands_(1),
+    planar_config_(PLANARCONFIG_CONTIG),
+    compression_(COMPRESSION_NONE),
+    has_alpha_(false),
+    is_tiled_(false)
 {
-    source_.open(filename, std::ios_base::in | std::ios_base::binary);
-    if (!stream_) throw image_reader_exception("TIFF reader: cannot open file "+ filename);
-    init();
+
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
+     boost::optional<mapnik::mapped_region_ptr> memory =
+         mapnik::mapped_memory_cache::instance().find(filename,true);
+
+     if (memory)
+     {
+         mapped_region_ = *memory;
+         stream_.buffer(static_cast<char*>(mapped_region_->get_address()),mapped_region_->get_size());
+     }
+     else
+     {
+         throw image_reader_exception("could not create file mapping for " + filename);
+     }
+#else
+     source_.open(filename, std::ios_base::in | std::ios_base::binary);
+#endif
+     if (!stream_) throw image_reader_exception("TIFF reader: cannot open file " + filename);
+     init();
 }
 
 template <typename T>
@@ -303,7 +351,7 @@ void tiff_reader<T>::init()
         if (extrasamples > 0 &&
             sampleinfo[0] == EXTRASAMPLE_UNSPECIFIED)
         {
-            throw std::runtime_error("Unspecified provided for extra samples to tiff reader.");
+            throw image_reader_exception("Unspecified provided for extra samples to tiff reader.");
         }
     }
     // Try extracting bounding box from geoTIFF tags
@@ -398,7 +446,6 @@ image_any tiff_reader<T>::read_any_gray(std::size_t x0, std::size_t y0, std::siz
         read_tiled<image_type>(x0, y0, data);
         return image_any(std::move(data));
     }
-    // TODO: temp disable and default to `scanline` method for stripped images.
     else if (read_method_ == stripped)
     {
         image_type data(width, height);
@@ -558,7 +605,7 @@ image_any tiff_reader<T>::read(unsigned x, unsigned y, unsigned width, unsigned
             }
             default:
             {
-                throw std::runtime_error("tiff_reader: This sample format is not supported for this bits per sample");
+                throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample");
             }
             }
         }
@@ -576,7 +623,7 @@ image_any tiff_reader<T>::read(unsigned x, unsigned y, unsigned width, unsigned
             }
             default:
             {
-                throw std::runtime_error("tiff_reader: This sample format is not supported for this bits per sample");
+                throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample");
             }
             }
         }
@@ -598,7 +645,7 @@ image_any tiff_reader<T>::read(unsigned x, unsigned y, unsigned width, unsigned
             }
             default:
             {
-                throw std::runtime_error("tiff_reader: This sample format is not supported for this bits per sample");
+                throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample");
             }
             }
         }
@@ -620,7 +667,7 @@ image_any tiff_reader<T>::read(unsigned x, unsigned y, unsigned width, unsigned
             }
             default:
             {
-                throw std::runtime_error("tiff_reader: This sample format is not supported for this bits per sample");
+                throw image_reader_exception("tiff_reader: This sample format is not supported for this bits per sample");
             }
             }
         }
@@ -649,7 +696,7 @@ template <typename T>
 template <typename ImageData>
 void tiff_reader<T>::read_generic(std::size_t, std::size_t, ImageData &)
 {
-    throw std::runtime_error("tiff_reader: TODO - tiff is not stripped or tiled");
+    throw image_reader_exception("tiff_reader: TODO - tiff is not stripped or tiled");
 }
 
 template <typename T>
@@ -732,7 +779,6 @@ void tiff_reader<T>::read_stripped(std::size_t x0, std::size_t y0, ImageData & i
         std::size_t start_y = (y0 / rows_per_strip_) * rows_per_strip_;
         std::size_t end_y = std::min(y0 + height, height_);
         std::size_t tx0, tx1, ty0, ty1;
-
         tx0 = x0;
         tx1 = std::min(width + x0, width_);
         std::size_t row = 0;
@@ -758,7 +804,7 @@ void tiff_reader<T>::read_stripped(std::size_t x0, std::size_t y0, ImageData & i
 
             if (detail::tiff_reader_traits<ImageData>::reverse)
             {
-                std::size_t num_rows = std::min(end_y - y, static_cast<std::size_t>(rows_per_strip_));
+                std::size_t num_rows = std::min(height_ - y, static_cast<std::size_t>(rows_per_strip_));
                 for (std::size_t ty = ty0; ty < ty1; ++ty)
                 {
                     // This is in reverse because the TIFFReadRGBAStrip reads are inverted
@@ -783,13 +829,13 @@ TIFF* tiff_reader<T>::open(std::istream & input)
     {
         tif_ = tiff_ptr(TIFFClientOpen("tiff_input_stream", "rcm",
                                        reinterpret_cast<thandle_t>(&input),
-                                       impl::tiff_read_proc,
-                                       impl::tiff_write_proc,
-                                       impl::tiff_seek_proc,
-                                       impl::tiff_close_proc,
-                                       impl::tiff_size_proc,
-                                       impl::tiff_map_proc,
-                                       impl::tiff_unmap_proc), tiff_closer());
+                                       detail::tiff_read_proc,
+                                       detail::tiff_write_proc,
+                                       detail::tiff_seek_proc,
+                                       detail::tiff_close_proc,
+                                       detail::tiff_size_proc,
+                                       detail::tiff_map_proc,
+                                       detail::tiff_unmap_proc), tiff_closer());
     }
     return tif_.get();
 }
diff --git a/test/unit/imaging/tiff_io.cpp b/test/unit/imaging/tiff_io.cpp
index 2c53800..4145285 100644
--- a/test/unit/imaging/tiff_io.cpp
+++ b/test/unit/imaging/tiff_io.cpp
@@ -11,11 +11,17 @@
 #include <mapnik/image_reader.hpp>
 #include <mapnik/util/file_io.hpp>
 #include <mapnik/util/fs.hpp>
-
+#include <boost/interprocess/streams/bufferstream.hpp>
 #include "../../../src/tiff_reader.cpp"
 
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
+using source_type = boost::interprocess::ibufferstream;
+#else
+using source_type = std::filebuf;
+#endif
+
 #define TIFF_ASSERT(filename)                                           \
-    mapnik::tiff_reader<std::filebuf> tiff_reader(filename);            \
+    mapnik::tiff_reader<source_type> tiff_reader(filename);            \
     REQUIRE( tiff_reader.width() == 256 );                              \
     REQUIRE( tiff_reader.height() == 256 );                             \
     REQUIRE( tiff_reader.planar_config() == PLANARCONFIG_CONTIG );      \
@@ -141,7 +147,7 @@ void test_tiff_reader(std::string const& pattern)
         if (boost::iends_with(path,".tif")
             && boost::istarts_with(mapnik::util::basename(path), pattern))
         {
-            mapnik::tiff_reader<std::filebuf> tiff_reader(path);
+            mapnik::tiff_reader<source_type> tiff_reader(path);
             auto width = tiff_reader.width();
             auto height = tiff_reader.height();
             {
@@ -180,7 +186,7 @@ TEST_CASE("tiff io")
     SECTION("scan rgb8 striped")
     {
         std::string filename("./test/data/tiff/scan_512x512_rgb8_striped.tif");
-        mapnik::tiff_reader<std::filebuf> tiff_reader(filename);
+        mapnik::tiff_reader<source_type> tiff_reader(filename);
         REQUIRE( tiff_reader.width() == 512 );
         REQUIRE( tiff_reader.height() == 512 );
         REQUIRE( tiff_reader.planar_config() == PLANARCONFIG_CONTIG );
@@ -210,7 +216,7 @@ TEST_CASE("tiff io")
 
     SECTION("scan rgb8 tiled") {
         std::string filename("./test/data/tiff/scan_512x512_rgb8_tiled.tif");
-        mapnik::tiff_reader<std::filebuf> tiff_reader(filename);
+        mapnik::tiff_reader<source_type> tiff_reader(filename);
         REQUIRE( tiff_reader.width() == 512 );
         REQUIRE( tiff_reader.height() == 512 );
         REQUIRE( tiff_reader.planar_config() == PLANARCONFIG_CONTIG );

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/mapnik.git



More information about the Pkg-grass-devel mailing list