[mapnik] 01/05: Imported Upstream version 3.0.8+ds
Sebastiaan Couwenberg
sebastic at moszumanska.debian.org
Sat Oct 24 12:55:34 UTC 2015
This is an automated email from the git hooks/post-receive script.
sebastic pushed a commit to branch master
in repository mapnik.
commit ecefdf0d646989eb8001086b36f4191a9171868c
Author: Bas Couwenberg <sebastic at xs4all.nl>
Date: Sat Oct 24 12:45:26 2015 +0200
Imported Upstream version 3.0.8+ds
---
.travis.yml | 3 +-
CHANGELOG.md | 17 +
SConstruct | 6 +-
demo/viewer/mainwindow.cpp | 8 +-
demo/viewer/mapwidget.cpp | 2 +-
include/mapnik/attribute_descriptor.hpp | 24 +-
include/mapnik/feature.hpp | 2 -
include/mapnik/feature_layer_desc.hpp | 20 +-
include/mapnik/json/feature_collection_grammar.hpp | 23 +-
.../json/feature_collection_grammar_impl.hpp | 63 +-
include/mapnik/text/harfbuzz_shaper.hpp | 6 +
include/mapnik/version.hpp | 2 +-
localize.sh | 3 +-
plugins/input/csv/csv_datasource.cpp | 41 +-
plugins/input/csv/csv_featureset.cpp | 6 +-
plugins/input/csv/csv_featureset.hpp | 4 +-
plugins/input/csv/csv_index_featureset.cpp | 6 +-
plugins/input/csv/csv_index_featureset.hpp | 4 +-
plugins/input/csv/csv_utils.hpp | 35 +-
plugins/input/geojson/build.py | 4 +-
plugins/input/geojson/geojson_datasource.cpp | 243 +++-
plugins/input/geojson/geojson_datasource.hpp | 2 +
.../geojson_index_featureset.cpp} | 82 +-
.../geojson_index_featureset.hpp} | 43 +-
...set.cpp => geojson_memory_index_featureset.cpp} | 13 +-
...set.hpp => geojson_memory_index_featureset.hpp} | 14 +-
plugins/input/ogr/ogr_index_featureset.cpp | 4 +-
plugins/input/shape/dbfile.cpp | 8 +-
plugins/input/shape/dbfile.hpp | 4 +-
plugins/input/shape/shape_datasource.cpp | 36 +-
plugins/input/shape/shape_featureset.cpp | 38 +-
plugins/input/shape/shape_featureset.hpp | 3 +-
plugins/input/shape/shape_index_featureset.cpp | 15 +-
plugins/input/shape/shape_io.cpp | 13 +-
plugins/input/shape/shape_io.hpp | 8 +-
plugins/input/shape/shapefile.hpp | 17 +-
.../mapnik_json_feature_collection_grammar.cpp | 1 +
src/mapped_memory_cache.cpp | 2 +-
test/standalone/csv_test.cpp | 676 -----------
test/unit/datasource/csv.cpp | 1173 ++++++++++++++++++++
test/unit/datasource/geojson.cpp | 492 ++++++--
utils/mapnik-index/build.py | 1 -
utils/mapnik-index/mapnik-index.cpp | 30 +-
utils/mapnik-index/process_csv_file.cpp | 49 +-
utils/mapnik-index/process_geojson_file.cpp | 26 +-
utils/shapefile/shapefile_reader.py | 23 +-
utils/shapeindex/shapeindex.cpp | 167 ++-
utils/svg2png/svg2png.cpp | 2 +-
48 files changed, 2319 insertions(+), 1145 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index c0e2afb..01ff425 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -72,13 +72,14 @@ script:
./configure;
fi
- make
- - make test
+ - make test || TEST_RESULT=$?
- if [[ ${COVERAGE} == true ]]; then
./mason_packages/.link/bin/cpp-coveralls --build-root . --gcov-options '\-lp' --exclude mason_packages --exclude .sconf_temp --exclude benchmark --exclude deps --exclude scons --exclude test --exclude demo --exclude docs --exclude fonts --exclude utils > /dev/null;
fi
- if [[ ${COVERAGE} != true ]]; then
make bench;
fi
+ - if [[ ${TEST_RESULT} != 0 ]]; then exit $TEST_RESULT ; fi;
- if [[ ${MASON_PUBLISH} == true ]]; then
./mason_latest.sh build;
./mason_latest.sh link;
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bbeea03..486a20c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,23 @@ Developers: Please commit along with changes.
For a complete change history, see the git log.
+## 3.0.8
+
+Released: October 23, 2015
+
+(Packaged from 2d15567)
+
+#### Summary
+
+ - Renamed `SHAPE_MEMORY_MAPPED_FILE` define to `MAPNIK_MEMORY_MAPPED_FILE`. Pass `./configure MEMORY_MAPPED_FILE=True|False` to request
+ support for memory mapped files across Mapnik plugins (currently shape, csv, and geojson).
+ - Unified `mapnik-index` utility supporing GeoJSON and CSV formats
+ - Increased unit test coverage for GeoJSON and CSV plugins
+ - shape.input - refactor to support *.shx and improve handling various bogus shapefiles
+ - geojson.input - make JSON parser stricter + support single Feature/Geometry as well as FeatureCollection
+ - maintain 'FT_LOAD_NO_HINTING' + support >= harfbuzz 1.0.5
+ - geojson.input - implement on-disk-index support
+
## 3.0.7
Released: October 12, 2015
diff --git a/SConstruct b/SConstruct
index 59faf04..642ea1d 100644
--- a/SConstruct
+++ b/SConstruct
@@ -389,7 +389,7 @@ opts.AddVariables(
EnumVariable('PLUGIN_LINKING', "Set plugin linking with libmapnik", 'shared', ['shared','static']),
# Other variables
- BoolVariable('SHAPE_MEMORY_MAPPED_FILE', 'Utilize memory-mapped files in Shapefile Plugin (higher memory usage, better performance)', 'True'),
+ BoolVariable('MEMORY_MAPPED_FILE', 'Utilize memory-mapped files in Shapefile Plugin (higher memory usage, better performance)', 'True'),
('SYSTEM_FONTS','Provide location for python bindings to register fonts (if provided then the bundled DejaVu fonts are not installed)',''),
('LIB_DIR_NAME','Name to use for the subfolder beside libmapnik where fonts and plugins are installed','mapnik'),
PathVariable('PYTHON','Full path to Python executable used to build bindings', sys.executable),
@@ -1209,8 +1209,8 @@ if not preconfigured:
thread_suffix = ''
env.Append(LIBS = 'pthread')
- if env['SHAPE_MEMORY_MAPPED_FILE']:
- env.Append(CPPDEFINES = '-DSHAPE_MEMORY_MAPPED_FILE')
+ if env['MEMORY_MAPPED_FILE']:
+ env.Append(CPPDEFINES = '-DMAPNIK_MEMORY_MAPPED_FILE')
# allow for mac osx /usr/lib/libicucore.dylib compatibility
# requires custom supplied headers since Apple does not include them
diff --git a/demo/viewer/mainwindow.cpp b/demo/viewer/mainwindow.cpp
index bd80bcf..9e9a34d 100644
--- a/demo/viewer/mainwindow.cpp
+++ b/demo/viewer/mainwindow.cpp
@@ -196,10 +196,10 @@ void MainWindow::load_map_file(QString const& filename)
mapnik::auto_cpu_timer t(std::clog, "loading map took: ");
mapnik::load_map(*map,filename.toStdString());
}
- //catch (mapnik::config_error & ex)
- //{
- // std::cout << ex.what() << "\n";
- //}
+ catch (std::exception const& ex)
+ {
+ std::cout << ex.what() << "\n";
+ }
catch (...)
{
std::cerr << "Exception caught in load_map\n";
diff --git a/demo/viewer/mapwidget.cpp b/demo/viewer/mapwidget.cpp
index b317cb4..04ce9c7 100644
--- a/demo/viewer/mapwidget.cpp
+++ b/demo/viewer/mapwidget.cpp
@@ -511,7 +511,7 @@ void render_agg(mapnik::Map const& map, double scaling_factor, QPixmap & pix)
//{
// std::cerr << ex.what() << std::endl;
//}
- catch (const std::exception & ex)
+ catch (std::exception const& ex)
{
std::cerr << "exception: " << ex.what() << std::endl;
}
diff --git a/include/mapnik/attribute_descriptor.hpp b/include/mapnik/attribute_descriptor.hpp
index ff97157..a5245ea 100644
--- a/include/mapnik/attribute_descriptor.hpp
+++ b/include/mapnik/attribute_descriptor.hpp
@@ -48,7 +48,7 @@ public:
type_(type),
size_(size),
precision_(precision),
- primary_key_(primary_key) {}
+ primary_key_(primary_key) {}
attribute_descriptor(attribute_descriptor const& other)
: name_(other.name_),
@@ -57,21 +57,15 @@ public:
precision_(other.precision_),
primary_key_(other.primary_key_) {}
- attribute_descriptor& operator=(attribute_descriptor const& other)
+ attribute_descriptor& operator=(attribute_descriptor rhs)
{
- if (this == &other)
- {
- return *this;
- }
- else
- {
- name_=other.name_;
- type_=other.type_;
- size_=other.size_;
- precision_=other.precision_;
- primary_key_=other.primary_key_;
- return *this;
- }
+ using std::swap;
+ std::swap(name_, rhs.name_);
+ std::swap(type_, rhs.type_);
+ std::swap(size_, rhs.size_);
+ std::swap(precision_, rhs.precision_);
+ std::swap(primary_key_, rhs.primary_key_);
+ return *this;
}
std::string const& get_name() const
diff --git a/include/mapnik/feature.hpp b/include/mapnik/feature.hpp
index fff06a2..769ca05 100644
--- a/include/mapnik/feature.hpp
+++ b/include/mapnik/feature.hpp
@@ -109,9 +109,7 @@ public:
raster_() {}
inline mapnik::value_integer id() const { return id_;}
-
inline void set_id(mapnik::value_integer id) { id_ = id;}
-
template <typename T>
inline void put(context_type::key_type const& key, T const& val)
{
diff --git a/include/mapnik/feature_layer_desc.hpp b/include/mapnik/feature_layer_desc.hpp
index c89d944..9d1186a 100644
--- a/include/mapnik/feature_layer_desc.hpp
+++ b/include/mapnik/feature_layer_desc.hpp
@@ -30,6 +30,7 @@
// stl
#include <iosfwd>
#include <vector>
+#include <algorithm>
namespace mapnik
{
@@ -40,13 +41,13 @@ public:
layer_descriptor(std::string const& name, std::string const& encoding)
: name_(name),
encoding_(encoding),
- desc_ar_(),
+ descriptors_(),
extra_params_() {}
layer_descriptor(layer_descriptor const& other)
: name_(other.name_),
encoding_(other.encoding_),
- desc_ar_(other.desc_ar_),
+ descriptors_(other.descriptors_),
extra_params_(other.extra_params_) {}
void set_name(std::string const& name)
@@ -71,17 +72,17 @@ public:
void add_descriptor(attribute_descriptor const& desc)
{
- desc_ar_.push_back(desc);
+ descriptors_.push_back(desc);
}
std::vector<attribute_descriptor> const& get_descriptors() const
{
- return desc_ar_;
+ return descriptors_;
}
std::vector<attribute_descriptor>& get_descriptors()
{
- return desc_ar_;
+ return descriptors_;
}
parameters const& get_extra_parameters() const
@@ -93,11 +94,16 @@ public:
{
return extra_params_;
}
-
+ bool has_name(std::string const& name) const
+ {
+ auto result = std::find_if(std::begin(descriptors_), std::end(descriptors_),
+ [&name](attribute_descriptor const& desc) { return name == desc.get_name();});
+ return result != std::end(descriptors_);
+ }
private:
std::string name_;
std::string encoding_;
- std::vector<attribute_descriptor> desc_ar_;
+ std::vector<attribute_descriptor> descriptors_;
parameters extra_params_;
};
diff --git a/include/mapnik/json/feature_collection_grammar.hpp b/include/mapnik/json/feature_collection_grammar.hpp
index 5c13f8c..5f62cdf 100644
--- a/include/mapnik/json/feature_collection_grammar.hpp
+++ b/include/mapnik/json/feature_collection_grammar.hpp
@@ -66,19 +66,40 @@ struct feature_collection_grammar :
feature_collection_grammar(mapnik::transcoder const& tr);
// grammars
feature_grammar<Iterator,FeatureType> feature_g;
- geometry_grammar<Iterator> geometry_g;
+ //geometry_grammar<Iterator> geometry_g;
// rules
qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> start; // START
qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> feature_collection;
qi::rule<Iterator, space_type> type;
qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> features;
qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature;
+ //qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature_from_geometry;
+ // phoenix functions
+ //phoenix::function<json::set_geometry_impl> set_geometry;
+ phoenix::function<apply_feature_callback> on_feature;
+};
+
+template <typename Iterator, typename FeatureType, typename FeatureCallback = default_feature_callback>
+struct feature_grammar_callback :
+ qi::grammar<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback &), space_type>
+{
+ feature_grammar_callback(mapnik::transcoder const& tr);
+ // grammars
+ feature_grammar<Iterator, FeatureType> feature_g;
+ geometry_grammar<Iterator> geometry_g;
+ // rules
+ qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> start; // START
+ //qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> feature_collection;
+ //qi::rule<Iterator, space_type> type;
+ //qi::rule<Iterator, void(context_ptr const&, std::size_t&, FeatureCallback&), space_type> features;
+ qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature;
qi::rule<Iterator, qi::locals<feature_ptr,int>, void(context_ptr const& ctx, std::size_t, FeatureCallback&), space_type> feature_from_geometry;
// phoenix functions
phoenix::function<json::set_geometry_impl> set_geometry;
phoenix::function<apply_feature_callback> on_feature;
};
+
}}
#endif // MAPNIK_FEATURE_COLLECTION_GRAMMAR_HPP
diff --git a/include/mapnik/json/feature_collection_grammar_impl.hpp b/include/mapnik/json/feature_collection_grammar_impl.hpp
index 8f73918..d4a07e3 100644
--- a/include/mapnik/json/feature_collection_grammar_impl.hpp
+++ b/include/mapnik/json/feature_collection_grammar_impl.hpp
@@ -38,7 +38,7 @@ feature_collection_grammar<Iterator,FeatureType, FeatureCallback>::feature_colle
{
qi::lit_type lit;
qi::eps_type eps;
- qi::_1_type _1;
+ //qi::_1_type _1;
qi::_2_type _2;
qi::_3_type _3;
qi::_4_type _4;
@@ -50,7 +50,7 @@ feature_collection_grammar<Iterator,FeatureType, FeatureCallback>::feature_colle
using phoenix::new_;
using phoenix::val;
- start = feature_collection(_r1, _r2, _r3) | feature_from_geometry(_r1, _r2, _r3) | feature(_r1, _r2, _r3)
+ start = /*feature_from_geometry(_r1, _r2, _r3) | feature(_r1, _r2, _r3) | */feature_collection(_r1, _r2, _r3)
;
feature_collection = lit('{') >> (type | features(_r1, _r2, _r3) | feature_g.json_.key_value) % lit(',') >> lit('}')
@@ -70,14 +70,67 @@ feature_collection_grammar<Iterator,FeatureType, FeatureCallback>::feature_colle
>> feature_g(*_a)[on_feature(_r3,_a)]
;
+ //feature_from_geometry =
+ // eps[_a = phoenix::construct<mapnik::feature_ptr>(new_<mapnik::feature_impl>(_r1, _r2))]
+ // >> geometry_g[set_geometry(*_a, _1)] [on_feature(_r3, _a)]
+ // ;
+
+ start.name("start");
+ type.name("type");
+ features.name("features");
+ feature.name("feature");
+ //feature_from_geometry.name("feature-from-geometry");
+ feature_g.name("feature-grammar");
+ //geometry_g.name("geometry-grammar");
+
+ qi::on_error<qi::fail>
+ (
+ start
+ , std::clog
+ << phoenix::val("Error parsing GeoJSON ")
+ << _4
+ << phoenix::val(" here: \"")
+ << construct<std::string>(_3, _2)
+ << phoenix::val('\"')
+ << std::endl
+ );
+}
+
+//
+
+
+template <typename Iterator, typename FeatureType, typename FeatureCallback>
+feature_grammar_callback<Iterator,FeatureType, FeatureCallback>::feature_grammar_callback(mapnik::transcoder const& tr)
+ : feature_grammar_callback::base_type(start,"start"),
+ feature_g(tr)
+{
+ qi::lit_type lit;
+ qi::eps_type eps;
+ qi::_1_type _1;
+ qi::_2_type _2;
+ qi::_3_type _3;
+ qi::_4_type _4;
+ qi::_a_type _a;
+ qi::_r1_type _r1;
+ qi::_r2_type _r2;
+ qi::_r3_type _r3;
+ using phoenix::construct;
+ using phoenix::new_;
+ using phoenix::val;
+
+ start = feature_from_geometry(_r1, _r2, _r3) | feature(_r1, _r2, _r3)
+ ;
+
+ feature = eps[_a = phoenix::construct<mapnik::feature_ptr>(new_<mapnik::feature_impl>(_r1, _r2))]
+ >> feature_g(*_a)[on_feature(_r3,_a)]
+ ;
+
feature_from_geometry =
eps[_a = phoenix::construct<mapnik::feature_ptr>(new_<mapnik::feature_impl>(_r1, _r2))]
>> geometry_g[set_geometry(*_a, _1)] [on_feature(_r3, _a)]
;
start.name("start");
- type.name("type");
- features.name("features");
feature.name("feature");
feature_from_geometry.name("feature-from-geometry");
feature_g.name("feature-grammar");
@@ -85,7 +138,7 @@ feature_collection_grammar<Iterator,FeatureType, FeatureCallback>::feature_colle
qi::on_error<qi::fail>
(
- feature_collection
+ start
, std::clog
<< phoenix::val("Error parsing GeoJSON ")
<< _4
diff --git a/include/mapnik/text/harfbuzz_shaper.hpp b/include/mapnik/text/harfbuzz_shaper.hpp
index ac436a1..8ba5d19 100644
--- a/include/mapnik/text/harfbuzz_shaper.hpp
+++ b/include/mapnik/text/harfbuzz_shaper.hpp
@@ -101,6 +101,12 @@ static void shape_text(text_line & line,
hb_buffer_set_direction(buffer.get(), (text_item.dir == UBIDI_RTL)?HB_DIRECTION_RTL:HB_DIRECTION_LTR);
hb_buffer_set_script(buffer.get(), _icu_script_to_script(text_item.script));
hb_font_t *font(hb_ft_font_create(face->get_face(), nullptr));
+ // https://github.com/mapnik/test-data-visual/pull/25
+ #if HB_VERSION_MAJOR > 0
+ #if HB_VERSION_ATLEAST(1, 0 , 5)
+ hb_ft_font_set_load_flags(font,FT_LOAD_DEFAULT | FT_LOAD_NO_HINTING);
+ #endif
+ #endif
hb_shape(font, buffer.get(), ff_settings.get_features(), ff_count);
hb_font_destroy(font);
diff --git a/include/mapnik/version.hpp b/include/mapnik/version.hpp
index 6fcd056..3fa21b8 100644
--- a/include/mapnik/version.hpp
+++ b/include/mapnik/version.hpp
@@ -25,7 +25,7 @@
#define MAPNIK_MAJOR_VERSION 3
#define MAPNIK_MINOR_VERSION 0
-#define MAPNIK_PATCH_VERSION 7
+#define MAPNIK_PATCH_VERSION 8
#define MAPNIK_VERSION (MAPNIK_MAJOR_VERSION*100000) + (MAPNIK_MINOR_VERSION*100) + (MAPNIK_PATCH_VERSION)
diff --git a/localize.sh b/localize.sh
index 31c6a6e..d24bdcb 100755
--- a/localize.sh
+++ b/localize.sh
@@ -7,7 +7,8 @@ else
export LD_LIBRARY_PATH="${CURRENT_DIR}/src/":${LD_LIBRARY_PATH}
fi
-export PATH=$(pwd)/utils/nik2img/:${PATH}
+export PATH=$(pwd)/utils/mapnik-render/:${PATH}
+export PATH=$(pwd)/utils/mapnik-index/:${PATH}
export PATH=$(pwd)/utils/mapnik-config/:${PATH}
# mapnik-settings.env is an optional file to store
diff --git a/plugins/input/csv/csv_datasource.cpp b/plugins/input/csv/csv_datasource.cpp
index 8da4691..9aa4ff6 100644
--- a/plugins/input/csv/csv_datasource.cpp
+++ b/plugins/input/csv/csv_datasource.cpp
@@ -41,7 +41,7 @@
#include <mapnik/util/fs.hpp>
#include <mapnik/util/spatial_index.hpp>
#include <mapnik/geom_util.hpp>
-#ifdef CSV_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
#pragma GCC diagnostic ignored "-Wsign-conversion"
@@ -125,7 +125,7 @@ csv_datasource::csv_datasource(parameters const& params)
}
else
{
-#if defined (CSV_MEMORY_MAPPED_FILE)
+#if defined (MAPNIK_MEMORY_MAPPED_FILE)
using file_source_type = boost::interprocess::ibufferstream;
file_source_type in;
mapnik::mapped_region_ptr mapped_region;
@@ -195,7 +195,7 @@ void csv_datasource::parse_csv(T & stream)
<< "' quote: '" << quote_ << "'";
stream.seekg(0, std::ios::beg);
- int line_number = 1;
+ int line_number = 0;
if (!manual_headers_.empty())
{
std::size_t index = 0;
@@ -230,7 +230,7 @@ void csv_datasource::parse_csv(T & stream)
s << "CSV Plugin: expected a column header at line ";
s << line_number << ", column " << index;
s << " - ensure this row contains valid header fields: '";
- s << csv_line << "'\n";
+ s << csv_line;
throw mapnik::datasource_exception(s.str());
}
else
@@ -261,16 +261,19 @@ void csv_datasource::parse_csv(T & stream)
}
}
- if (locator_.type == detail::geometry_column_locator::UNKNOWN)
+ std::size_t num_headers = headers_.size();
+ if (!detail::valid(locator_, num_headers))
{
- throw mapnik::datasource_exception("CSV Plugin: could not detect column headers with the name of wkt, geojson, x/y, or "
- "latitude/longitude - this is required for reading geometry data");
+ std::string str("CSV Plugin: could not detect column(s) with the name(s) of wkt, geojson, x/y, or ");
+ str += "latitude/longitude in:\n";
+ str += csv_line;
+ str += "\n - this is required for reading geometry data";
+ throw mapnik::datasource_exception(str);
}
mapnik::value_integer feature_count = 0;
bool extent_started = false;
- std::size_t num_headers = headers_.size();
std::for_each(headers_.begin(), headers_.end(),
[ & ](std::string const& header){ ctx_->push(header); });
@@ -294,8 +297,8 @@ void csv_datasource::parse_csv(T & stream)
std::vector<item_type> boxes;
while (is_first_row || csv_utils::getline_csv(stream, csv_line, newline, quote_))
{
-
- if ((row_limit_ > 0) && (line_number++ > row_limit_))
+ ++line_number;
+ if ((row_limit_ > 0) && (line_number > row_limit_))
{
MAPNIK_LOG_DEBUG(csv) << "csv_datasource: row limit hit, exiting at feature: " << feature_count;
break;
@@ -326,7 +329,7 @@ void csv_datasource::parse_csv(T & stream)
std::ostringstream s;
s << "CSV Plugin: # of columns("
<< num_fields << ") > # of headers("
- << num_headers << ") parsed for row " << line_number << "\n";
+ << num_headers << ") parsed for row " << line_number;
throw mapnik::datasource_exception(s.str());
}
@@ -353,18 +356,6 @@ void csv_datasource::parse_csv(T & stream)
for (std::size_t i = 0; i < num_headers; ++i)
{
std::string const& header = headers_.at(i);
- if (beg == end) // there are more headers than column values for this row
- {
- // add an empty string here to represent a missing value
- // not using null type here since nulls are not a csv thing
- if (feature_count == 1)
- {
- desc_.add_descriptor(mapnik::attribute_descriptor(header, mapnik::String));
- }
- // continue here instead of break so that all missing values are
- // encoded consistenly as empty strings
- continue;
- }
std::string value = mapnik::util::trim_copy(*beg++);
int value_length = value.length();
if (locator_.index == i && (locator_.type == detail::geometry_column_locator::WKT
@@ -431,7 +422,7 @@ void csv_datasource::parse_csv(T & stream)
std::ostringstream s;
s << "CSV Plugin: expected geometry column: could not parse row "
<< line_number << " "
- << values[locator_.index] << "'";
+ << values.at(locator_.index) << "'";
throw mapnik::datasource_exception(s.str());
}
}
@@ -440,7 +431,7 @@ void csv_datasource::parse_csv(T & stream)
if (strict_) throw ex;
else
{
- MAPNIK_LOG_ERROR(csv) << ex.what();
+ MAPNIK_LOG_ERROR(csv) << ex.what() << " at line: " << line_number;
}
}
catch (std::exception const& ex)
diff --git a/plugins/input/csv/csv_featureset.cpp b/plugins/input/csv/csv_featureset.cpp
index 8a94875..5017c82 100644
--- a/plugins/input/csv/csv_featureset.cpp
+++ b/plugins/input/csv/csv_featureset.cpp
@@ -34,7 +34,7 @@
csv_featureset::csv_featureset(std::string const& filename, detail::geometry_column_locator const& locator, char separator, char quote,
std::vector<std::string> const& headers, mapnik::context_ptr const& ctx, array_type && index_array)
:
-#if defined(CSV_MEMORY_MAPPED_FILE)
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
//
#elif defined( _WINDOWS)
file_(_wfopen(mapnik::utf8_to_utf16(filename).c_str(), L"rb"), std::fclose),
@@ -51,7 +51,7 @@ csv_featureset::csv_featureset(std::string const& filename, detail::geometry_col
locator_(locator),
tr_("utf8")
{
-#if defined (CSV_MEMORY_MAPPED_FILE)
+#if defined (MAPNIK_MEMORY_MAPPED_FILE)
boost::optional<mapnik::mapped_region_ptr> memory =
mapnik::mapped_memory_cache::instance().find(filename, true);
if (memory)
@@ -90,7 +90,7 @@ mapnik::feature_ptr csv_featureset::next()
csv_datasource::item_type const& item = *index_itr_++;
std::size_t file_offset = item.second.first;
std::size_t size = item.second.second;
-#if defined(CSV_MEMORY_MAPPED_FILE)
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
char const* start = (char const*)mapped_region_->get_address() + file_offset;
char const* end = start + size;
#else
diff --git a/plugins/input/csv/csv_featureset.hpp b/plugins/input/csv/csv_featureset.hpp
index 3f05c08..e09bd4c 100644
--- a/plugins/input/csv/csv_featureset.hpp
+++ b/plugins/input/csv/csv_featureset.hpp
@@ -30,7 +30,7 @@
#include <deque>
#include <cstdio>
-#ifdef CSV_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
#pragma GCC diagnostic ignored "-Wsign-conversion"
@@ -56,7 +56,7 @@ public:
mapnik::feature_ptr next();
private:
mapnik::feature_ptr parse_feature(char const* beg, char const* end);
-#if defined (CSV_MEMORY_MAPPED_FILE)
+#if defined (MAPNIK_MEMORY_MAPPED_FILE)
using file_source_type = boost::interprocess::ibufferstream;
mapnik::mapped_region_ptr mapped_region_;
#else
diff --git a/plugins/input/csv/csv_index_featureset.cpp b/plugins/input/csv/csv_index_featureset.cpp
index 4a13551..c064579 100644
--- a/plugins/input/csv/csv_index_featureset.cpp
+++ b/plugins/input/csv/csv_index_featureset.cpp
@@ -48,7 +48,7 @@ csv_index_featureset::csv_index_featureset(std::string const& filename,
ctx_(ctx),
locator_(locator),
tr_("utf8")
-#if defined(CSV_MEMORY_MAPPED_FILE)
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
//
#elif defined( _WINDOWS)
,file_(_wfopen(mapnik::utf8_to_utf16(filename).c_str(), L"rb"), std::fclose)
@@ -57,7 +57,7 @@ csv_index_featureset::csv_index_featureset(std::string const& filename,
#endif
{
-#if defined (CSV_MEMORY_MAPPED_FILE)
+#if defined (MAPNIK_MEMORY_MAPPED_FILE)
boost::optional<mapnik::mapped_region_ptr> memory =
mapnik::mapped_memory_cache::instance().find(filename, true);
if (memory)
@@ -112,7 +112,7 @@ mapnik::feature_ptr csv_index_featureset::next()
while( itr_ != positions_.end())
{
auto pos = *itr_++;
-#if defined(CSV_MEMORY_MAPPED_FILE)
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
char const* start = (char const*)mapped_region_->get_address() + pos.first;
char const* end = start + pos.second;
#else
diff --git a/plugins/input/csv/csv_index_featureset.hpp b/plugins/input/csv/csv_index_featureset.hpp
index 5980afb..a17da59 100644
--- a/plugins/input/csv/csv_index_featureset.hpp
+++ b/plugins/input/csv/csv_index_featureset.hpp
@@ -29,7 +29,7 @@
#include "csv_utils.hpp"
#include "csv_datasource.hpp"
-#ifdef CSV_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
#pragma GCC diagnostic ignored "-Wsign-conversion"
@@ -63,7 +63,7 @@ private:
mapnik::value_integer feature_id_ = 0;
detail::geometry_column_locator const& locator_;
mapnik::transcoder tr_;
-#if defined (CSV_MEMORY_MAPPED_FILE)
+#if defined (MAPNIK_MEMORY_MAPPED_FILE)
using file_source_type = boost::interprocess::ibufferstream;
mapnik::mapped_region_ptr mapped_region_;
#else
diff --git a/plugins/input/csv/csv_utils.hpp b/plugins/input/csv/csv_utils.hpp
index 2d3cb46..b1894da 100644
--- a/plugins/input/csv/csv_utils.hpp
+++ b/plugins/input/csv/csv_utils.hpp
@@ -32,6 +32,7 @@
#include <mapnik/util/conversions.hpp>
#include <mapnik/csv/csv_grammar.hpp>
#include <mapnik/util/trim.hpp>
+#include <mapnik/datasource.hpp>
// boost
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
@@ -44,10 +45,6 @@
#include <cstdio>
#include <algorithm>
-#ifndef _WINDOWS
-#define CSV_MEMORY_MAPPED_FILE
-#endif
-
namespace csv_utils
{
@@ -61,7 +58,7 @@ static mapnik::csv_line parse_line(Iterator start, Iterator end, char separator,
if (num_columns > 0) values.reserve(num_columns);
if (!boost::spirit::qi::phrase_parse(start, end, (line_g)(separator, quote), skipper, values))
{
- throw std::runtime_error("Failed to parse CSV line:\n" + std::string(start, end));
+ throw mapnik::datasource_exception("Failed to parse CSV line:\n" + std::string(start, end));
}
return values;
}
@@ -251,39 +248,51 @@ static inline void locate_geometry_column(std::string const& header, std::size_t
}
}
+static inline bool valid(geometry_column_locator const& locator, std::size_t max_size)
+{
+ if (locator.type == geometry_column_locator::UNKNOWN) return false;
+ if (locator.index >= max_size) return false;
+ if (locator.type == geometry_column_locator::LON_LAT && locator.index2 >= max_size) return false;
+ return true;
+}
+
static inline mapnik::geometry::geometry<double> extract_geometry(std::vector<std::string> const& row, geometry_column_locator const& locator)
{
mapnik::geometry::geometry<double> geom;
if (locator.type == geometry_column_locator::WKT)
{
- if (mapnik::from_wkt(row[locator.index], geom))
+ auto wkt_value = row.at(locator.index);
+ if (mapnik::from_wkt(wkt_value, geom))
{
// correct orientations ..
mapnik::geometry::correct(geom);
}
else
{
- throw std::runtime_error("Failed to parse WKT:" + row[locator.index]);
+ throw mapnik::datasource_exception("Failed to parse WKT: '" + wkt_value + "'");
}
}
else if (locator.type == geometry_column_locator::GEOJSON)
{
- if (!mapnik::json::from_geojson(row[locator.index], geom))
+ auto json_value = row.at(locator.index);
+ if (!mapnik::json::from_geojson(json_value, geom))
{
- throw std::runtime_error("Failed to parse GeoJSON:" + row[locator.index]);
+ throw mapnik::datasource_exception("Failed to parse GeoJSON: '" + json_value + "'");
}
}
else if (locator.type == geometry_column_locator::LON_LAT)
{
double x, y;
- if (!mapnik::util::string2double(row[locator.index],x))
+ auto long_value = row.at(locator.index);
+ auto lat_value = row.at(locator.index2);
+ if (!mapnik::util::string2double(long_value,x))
{
- throw std::runtime_error("Failed to parse Longitude(Easting):" + row[locator.index]);
+ throw mapnik::datasource_exception("Failed to parse Longitude: '" + long_value + "'");
}
- if (!mapnik::util::string2double(row[locator.index2],y))
+ if (!mapnik::util::string2double(lat_value,y))
{
- throw std::runtime_error("Failed to parse Latitude(Northing):" + row[locator.index2]);
+ throw mapnik::datasource_exception("Failed to parse Latitude: '" + lat_value + "'");
}
geom = mapnik::geometry::point<double>(x,y);
}
diff --git a/plugins/input/geojson/build.py b/plugins/input/geojson/build.py
index 9001360..a5d738f 100644
--- a/plugins/input/geojson/build.py
+++ b/plugins/input/geojson/build.py
@@ -42,7 +42,9 @@ else:
"""
%(PLUGIN_NAME)s_datasource.cpp
%(PLUGIN_NAME)s_featureset.cpp
- large_%(PLUGIN_NAME)s_featureset.cpp
+ %(PLUGIN_NAME)s_index_featureset.cpp
+ %(PLUGIN_NAME)s_memory_index_featureset.cpp
+
""" % locals()
)
diff --git a/plugins/input/geojson/geojson_datasource.cpp b/plugins/input/geojson/geojson_datasource.cpp
index ea3f30b..afeb84d 100644
--- a/plugins/input/geojson/geojson_datasource.cpp
+++ b/plugins/input/geojson/geojson_datasource.cpp
@@ -22,7 +22,8 @@
#include "geojson_datasource.hpp"
#include "geojson_featureset.hpp"
-#include "large_geojson_featureset.hpp"
+#include "geojson_index_featureset.hpp"
+#include "geojson_memory_index_featureset.hpp"
#include <fstream>
#include <algorithm>
@@ -57,8 +58,11 @@
#include <mapnik/geometry_adapters.hpp>
#include <mapnik/json/feature_collection_grammar.hpp>
#include <mapnik/json/extract_bounding_box_grammar_impl.hpp>
+#include <mapnik/util/fs.hpp>
+#include <mapnik/util/spatial_index.hpp>
+#include <mapnik/geom_util.hpp>
-#if defined(SHAPE_MEMORY_MAPPED_FILE)
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
#pragma GCC diagnostic ignored "-Wsign-conversion"
@@ -136,17 +140,23 @@ geojson_datasource::geojson_datasource(parameters const& params)
filename_ = *base + "/" + *file;
else
filename_ = *file;
+ has_disk_index_ = mapnik::util::exists(filename_ + ".index");
}
+
if (!inline_string_.empty())
{
char const* start = inline_string_.c_str();
char const* end = start + inline_string_.size();
parse_geojson(start, end);
}
+ else if (has_disk_index_)
+ {
+ initialise_disk_index(filename_);
+ }
else
{
cache_features_ = *params.get<mapnik::boolean_type>("cache_features", true);
-#if !defined(SHAPE_MEMORY_MAPPED_FILE)
+#if !defined(MAPNIK_MEMORY_MAPPED_FILE)
mapnik::util::file file(filename_);
if (!file.open())
{
@@ -192,10 +202,60 @@ namespace {
using base_iterator_type = char const*;
const mapnik::transcoder geojson_datasource_static_tr("utf8");
const mapnik::json::feature_collection_grammar<base_iterator_type,mapnik::feature_impl> geojson_datasource_static_fc_grammar(geojson_datasource_static_tr);
+const mapnik::json::feature_grammar_callback<base_iterator_type,mapnik::feature_impl> geojson_datasource_static_feature_callback_grammar(geojson_datasource_static_tr);
const mapnik::json::feature_grammar<base_iterator_type, mapnik::feature_impl> geojson_datasource_static_feature_grammar(geojson_datasource_static_tr);
const mapnik::json::extract_bounding_box_grammar<base_iterator_type> geojson_datasource_static_bbox_grammar;
}
+void geojson_datasource::initialise_disk_index(std::string const& filename)
+{
+ // read extent
+ using value_type = std::pair<std::size_t, std::size_t>;
+ std::ifstream index(filename_ + ".index", std::ios::binary);
+ if (!index) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + ".index'");
+ extent_ = mapnik::util::spatial_index<value_type,
+ mapnik::filter_in_box,
+ std::ifstream>::bounding_box(index);
+ mapnik::filter_in_box filter(extent_);
+ std::vector<value_type> positions;
+ mapnik::util::spatial_index<value_type,
+ mapnik::filter_in_box,
+ std::ifstream>::query_first_n(filter, index, positions, 5);
+
+ mapnik::util::file file(filename_);
+ if (!file.open()) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
+
+ for (auto const& pos : positions)
+ {
+ std::fseek(file.get(), pos.first, SEEK_SET);
+ std::vector<char> record;
+ record.resize(pos.second);
+ std::fread(record.data(), pos.second, 1, file.get());
+ auto const* start = record.data();
+ auto const* end = start + record.size();
+ mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
+ mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
+ using namespace boost::spirit;
+ standard::space_type space;
+ if (!boost::spirit::qi::phrase_parse(start, end,
+ (geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space)
+ || start != end)
+ {
+ throw std::runtime_error("Failed to parse geojson feature");
+ }
+ for ( auto const& kv : *feature)
+ {
+ auto const& name = std::get<0>(kv);
+ if (!desc_.has_name(name))
+ {
+ desc_.add_descriptor(mapnik::attribute_descriptor(name,
+ mapnik::util::apply_visitor(attr_value_converter(),
+ std::get<1>(kv))));
+ }
+ }
+ }
+}
+
template <typename Iterator>
void geojson_datasource::initialise_index(Iterator start, Iterator end)
{
@@ -204,39 +264,88 @@ void geojson_datasource::initialise_index(Iterator start, Iterator end)
Iterator itr = start;
if (!boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_bbox_grammar)(boost::phoenix::ref(boxes)) , space))
{
- throw mapnik::datasource_exception("GeoJSON Plugin: could not parse: '" + filename_ + "'");
+ cache_features_ = true; // force caching single feature
+ itr = start; // reset iteraror
+ // try parsing as single Feature or single Geometry JSON
+ mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
+ std::size_t start_id = 1;
+ mapnik::json::default_feature_callback callback(features_);
+ bool result = boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_feature_callback_grammar)
+ (boost::phoenix::ref(ctx),boost::phoenix::ref(start_id), boost::phoenix::ref(callback)),
+ space);
+ if (!result || itr != end)
+ {
+ if (!inline_string_.empty()) throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file from in-memory string");
+ else throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file '" + filename_ + "'");
+ }
+
+ using values_container = std::vector< std::pair<box_type, std::pair<std::size_t, std::size_t>>>;
+ values_container values;
+ values.reserve(features_.size());
+
+ std::size_t geometry_index = 0;
+ for (mapnik::feature_ptr const& f : features_)
+ {
+ mapnik::box2d<double> box = f->envelope();
+ if (box.valid())
+ {
+ if (geometry_index == 0)
+ {
+ extent_ = box;
+ for ( auto const& kv : *f)
+ {
+ desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
+ mapnik::util::apply_visitor(attr_value_converter(),
+ std::get<1>(kv))));
+ }
+ }
+ else
+ {
+ extent_.expand_to_include(box);
+ }
+ values.emplace_back(box, std::make_pair(geometry_index,0));
+ }
+ ++geometry_index;
+ }
+ // packing algorithm
+ tree_ = std::make_unique<spatial_index_type>(values);
}
- // bulk insert initialise r-tree
- tree_ = std::make_unique<spatial_index_type>(boxes);
- // calculate total extent
- for (auto const& item : boxes)
+ else
{
- auto const& box = std::get<0>(item);
- auto const& geometry_index = std::get<1>(item);
- if (!extent_.valid())
+ // bulk insert initialise r-tree
+ tree_ = std::make_unique<spatial_index_type>(boxes);
+ // calculate total extent
+ for (auto const& item : boxes)
{
- extent_ = box;
- // parse first feature to extract attributes schema.
- // NOTE: this doesn't yield correct answer for geoJSON in general, just an indication
- Iterator itr2 = start + geometry_index.first;
- Iterator end2 = itr2 + geometry_index.second;
- mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
- mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
- if (!boost::spirit::qi::phrase_parse(itr2, end2, (geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space))
+ auto const& box = std::get<0>(item);
+ auto const& geometry_index = std::get<1>(item);
+ if (!extent_.valid())
{
- throw std::runtime_error("Failed to parse geojson feature");
+ extent_ = box;
+ // parse first feature to extract attributes schema.
+ // NOTE: this doesn't yield correct answer for geoJSON in general, just an indication
+ Iterator itr2 = start + geometry_index.first;
+ Iterator end2 = itr2 + geometry_index.second;
+ mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
+ mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,-1)); // temp feature
+ if (!boost::spirit::qi::phrase_parse(itr2, end2,
+ (geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space)
+ || itr2 != end2)
+ {
+ throw std::runtime_error("Failed to parse geojson feature");
+ }
+ for ( auto const& kv : *feature)
+ {
+ desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
+ mapnik::util::apply_visitor(attr_value_converter(),
+ std::get<1>(kv))));
+ }
}
- for ( auto const& kv : *feature)
+ else
{
- desc_.add_descriptor(mapnik::attribute_descriptor(std::get<0>(kv),
- mapnik::util::apply_visitor(attr_value_converter(),
- std::get<1>(kv))));
+ extent_.expand_to_include(box);
}
}
- else
- {
- extent_.expand_to_include(box);
- }
}
}
@@ -248,16 +357,30 @@ void geojson_datasource::parse_geojson(Iterator start, Iterator end)
std::size_t start_id = 1;
mapnik::json::default_feature_callback callback(features_);
-
- bool result = boost::spirit::qi::phrase_parse(start, end, (geojson_datasource_static_fc_grammar)
+ Iterator itr = start;
+ bool result = boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_fc_grammar)
(boost::phoenix::ref(ctx),boost::phoenix::ref(start_id), boost::phoenix::ref(callback)),
space);
- if (!result)
+ if (!result || itr != end)
{
if (!inline_string_.empty()) throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file from in-memory string");
else throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file '" + filename_ + "'");
}
+ if (features_.size() == 0)
+ {
+ itr = start;
+ // try parsing as single Feature or single Geometry JSON
+ result = boost::spirit::qi::phrase_parse(itr, end, (geojson_datasource_static_feature_callback_grammar)
+ (boost::phoenix::ref(ctx),boost::phoenix::ref(start_id), boost::phoenix::ref(callback)),
+ space);
+ if (!result || itr != end)
+ {
+ if (!inline_string_.empty()) throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file from in-memory string");
+ else throw mapnik::datasource_exception("geojson_datasource: Failed parse GeoJSON file '" + filename_ + "'");
+ }
+ }
+
using values_container = std::vector< std::pair<box_type, std::pair<std::size_t, std::size_t>>>;
values_container values;
values.reserve(features_.size());
@@ -317,7 +440,53 @@ boost::optional<mapnik::datasource_geometry_t> geojson_datasource::get_geometry_
{
boost::optional<mapnik::datasource_geometry_t> result;
int multi_type = 0;
- if (cache_features_)
+ if (has_disk_index_)
+ {
+ using value_type = std::pair<std::size_t, std::size_t>;
+ std::ifstream index(filename_ + ".index", std::ios::binary);
+ if (!index) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + ".index'");
+ mapnik::filter_in_box filter(extent_);
+ std::vector<value_type> positions;
+ mapnik::util::spatial_index<value_type,
+ mapnik::filter_in_box,
+ std::ifstream>::query_first_n(filter, index, positions, 5);
+
+ mapnik::util::file file(filename_);
+
+ if (!file.open()) throw mapnik::datasource_exception("GeoJSON Plugin: could not open: '" + filename_ + "'");
+
+ for (auto const& pos : positions)
+ {
+ std::fseek(file.get(), pos.first, SEEK_SET);
+ std::vector<char> record;
+ record.resize(pos.second);
+ std::fread(record.data(), pos.second, 1, file.get());
+ auto const* start = record.data();
+ auto const* end = start + record.size();
+ mapnik::context_ptr ctx = std::make_shared<mapnik::context_type>();
+ mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, -1)); // temp feature
+ using namespace boost::spirit;
+ standard::space_type space;
+ if (!boost::spirit::qi::phrase_parse(start, end,
+ (geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space)
+ || start != end)
+ {
+ throw std::runtime_error("Failed to parse geojson feature");
+ }
+ result = mapnik::util::to_ds_type(feature->get_geometry());
+ if (result)
+ {
+ int type = static_cast<int>(*result);
+ if (multi_type > 0 && multi_type != type)
+ {
+ result.reset(mapnik::datasource_geometry_t::Collection);
+ return result;
+ }
+ multi_type = type;
+ }
+ }
+ }
+ else if (cache_features_)
{
unsigned num_features = features_.size();
for (unsigned i = 0; i < num_features && i < 5; ++i)
@@ -362,7 +531,7 @@ boost::optional<mapnik::datasource_geometry_t> geojson_datasource::get_geometry_
using namespace boost::spirit;
standard::space_type space;
- mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx,1));
+ mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx, -1)); // temp feature
if (!qi::phrase_parse(start2, end2, (geojson_datasource_static_feature_grammar)(boost::phoenix::ref(*feature)), space))
{
throw std::runtime_error("Failed to parse geojson feature");
@@ -405,9 +574,15 @@ mapnik::featureset_ptr geojson_datasource::features(mapnik::query const& q) cons
{
return item0.second.first < item1.second.first;
});
- return std::make_shared<large_geojson_featureset>(filename_, std::move(index_array));
+ return std::make_shared<geojson_memory_index_featureset>(filename_, std::move(index_array));
}
}
+ else if (has_disk_index_)
+ {
+ mapnik::filter_in_box filter(q.get_bbox());
+ return std::make_shared<geojson_index_featureset>(filename_, filter);
+ }
+
}
// otherwise return an empty featureset pointer
return mapnik::featureset_ptr();
diff --git a/plugins/input/geojson/geojson_datasource.hpp b/plugins/input/geojson/geojson_datasource.hpp
index 870f649..212796c 100644
--- a/plugins/input/geojson/geojson_datasource.hpp
+++ b/plugins/input/geojson/geojson_datasource.hpp
@@ -98,6 +98,7 @@ public:
void parse_geojson(Iterator start, Iterator end);
template <typename Iterator>
void initialise_index(Iterator start, Iterator end);
+ void initialise_disk_index(std::string const& filename);
private:
mapnik::datasource::datasource_t type_;
mapnik::layer_descriptor desc_;
@@ -107,6 +108,7 @@ private:
std::vector<mapnik::feature_ptr> features_;
std::unique_ptr<spatial_index_type> tree_;
bool cache_features_ = true;
+ bool has_disk_index_ = false;
};
diff --git a/plugins/input/csv/csv_index_featureset.cpp b/plugins/input/geojson/geojson_index_featureset.cpp
similarity index 56%
copy from plugins/input/csv/csv_index_featureset.cpp
copy to plugins/input/geojson/geojson_index_featureset.cpp
index 4a13551..2856916 100644
--- a/plugins/input/csv/csv_index_featureset.cpp
+++ b/plugins/input/geojson/geojson_index_featureset.cpp
@@ -21,43 +21,31 @@
*****************************************************************************/
// mapnik
-#include "csv_index_featureset.hpp"
-#include <mapnik/debug.hpp>
+#include "geojson_index_featureset.hpp"
#include <mapnik/feature.hpp>
#include <mapnik/feature_factory.hpp>
+#include <mapnik/json/geometry_grammar.hpp>
+#include <mapnik/json/feature_grammar.hpp>
#include <mapnik/util/utf_conv_win.hpp>
-#include <mapnik/util/trim.hpp>
#include <mapnik/util/spatial_index.hpp>
-#include <mapnik/geometry.hpp>
// stl
#include <string>
#include <vector>
-#include <deque>
#include <fstream>
-csv_index_featureset::csv_index_featureset(std::string const& filename,
- mapnik::filter_in_box const& filter,
- detail::geometry_column_locator const& locator,
- char separator,
- char quote,
- std::vector<std::string> const& headers,
- mapnik::context_ptr const& ctx)
- : separator_(separator),
- quote_(quote),
- headers_(headers),
- ctx_(ctx),
- locator_(locator),
- tr_("utf8")
-#if defined(CSV_MEMORY_MAPPED_FILE)
- //
-#elif defined( _WINDOWS)
- ,file_(_wfopen(mapnik::utf8_to_utf16(filename).c_str(), L"rb"), std::fclose)
+geojson_index_featureset::geojson_index_featureset(std::string const& filename, mapnik::filter_in_box const& filter)
+ :
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
+ //
+#elif defined _WINDOWS
+ file_(_wfopen(mapnik::utf8_to_utf16(filename).c_str(), L"rb"), std::fclose),
#else
- ,file_(std::fopen(filename.c_str(),"rb"), std::fclose)
+ file_(std::fopen(filename.c_str(),"rb"), std::fclose),
#endif
-
+ ctx_(std::make_shared<mapnik::context_type>())
{
-#if defined (CSV_MEMORY_MAPPED_FILE)
+
+#if defined (MAPNIK_MEMORY_MAPPED_FILE)
boost::optional<mapnik::mapped_region_ptr> memory =
mapnik::mapped_memory_cache::instance().find(filename, true);
if (memory)
@@ -69,12 +57,11 @@ csv_index_featureset::csv_index_featureset(std::string const& filename,
throw std::runtime_error("could not create file mapping for " + filename);
}
#else
- if (!file_) throw mapnik::datasource_exception("CSV Plugin: can't open file " + filename);
+ if (!file_) throw std::runtime_error("Can't open " + filename);
#endif
-
std::string indexname = filename + ".index";
std::ifstream index(indexname.c_str(), std::ios::binary);
- if (!index) throw mapnik::datasource_exception("CSV Plugin: can't open index file " + indexname);
+ if (!index) throw mapnik::datasource_exception("GeoJSON Plugin: can't open index file " + indexname);
mapnik::util::spatial_index<value_type,
mapnik::filter_in_box,
std::ifstream>::query(filter, index, positions_);
@@ -84,35 +71,14 @@ csv_index_featureset::csv_index_featureset(std::string const& filename,
itr_ = positions_.begin();
}
-csv_index_featureset::~csv_index_featureset() {}
-
-mapnik::feature_ptr csv_index_featureset::parse_feature(char const* beg, char const* end)
-{
- auto values = csv_utils::parse_line(beg, end, separator_, quote_, headers_.size());
- auto geom = detail::extract_geometry(values, locator_);
- if (!geom.is<mapnik::geometry::geometry_empty>())
- {
- mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx_, ++feature_id_));
- feature->set_geometry(std::move(geom));
- detail::process_properties(*feature, headers_, values, locator_, tr_);
- return feature;
- }
- return mapnik::feature_ptr();
-}
+geojson_index_featureset::~geojson_index_featureset() {}
-mapnik::feature_ptr csv_index_featureset::next()
+mapnik::feature_ptr geojson_index_featureset::next()
{
- /*
- if (row_limit_ && count_ >= row_limit_)
- {
- return feature_ptr();
- }
- */
-
while( itr_ != positions_.end())
{
auto pos = *itr_++;
-#if defined(CSV_MEMORY_MAPPED_FILE)
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
char const* start = (char const*)mapped_region_->get_address() + pos.first;
char const* end = start + pos.second;
#else
@@ -123,8 +89,16 @@ mapnik::feature_ptr csv_index_featureset::next()
auto const* start = record.data();
auto const* end = start + record.size();
#endif
- auto feature = parse_feature(start, end);
- if (feature) return feature;
+ static const mapnik::transcoder tr("utf8");
+ static const mapnik::json::feature_grammar<char const*, mapnik::feature_impl> grammar(tr);
+ using namespace boost::spirit;
+ standard::space_type space;
+ mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx_, feature_id_++));
+ if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space) || start != end)
+ {
+ throw std::runtime_error("Failed to parse geojson feature");
+ }
+ return feature;
}
return mapnik::feature_ptr();
}
diff --git a/plugins/input/csv/csv_index_featureset.hpp b/plugins/input/geojson/geojson_index_featureset.hpp
similarity index 63%
copy from plugins/input/csv/csv_index_featureset.hpp
copy to plugins/input/geojson/geojson_index_featureset.hpp
index 5980afb..2cd934c 100644
--- a/plugins/input/csv/csv_index_featureset.hpp
+++ b/plugins/input/geojson/geojson_index_featureset.hpp
@@ -20,16 +20,14 @@
*
*****************************************************************************/
-#ifndef CSV_INDEX_FEATURESET_HPP
-#define CSV_INDEX_FEATURESET_HPP
+#ifndef GEOJSON_INDEX_FEATURESET_HPP
+#define GEOJSON_INDEX_FEATURESET_HPP
+#include "geojson_datasource.hpp"
#include <mapnik/feature.hpp>
-#include <mapnik/unicode.hpp>
#include <mapnik/geom_util.hpp>
-#include "csv_utils.hpp"
-#include "csv_datasource.hpp"
-#ifdef CSV_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
#pragma GCC diagnostic ignored "-Wsign-conversion"
@@ -39,40 +37,29 @@
#include <mapnik/mapped_memory_cache.hpp>
#endif
-class csv_index_featureset : public mapnik::Featureset
+#include <deque>
+#include <cstdio>
+
+class geojson_index_featureset : public mapnik::Featureset
{
using value_type = std::pair<std::size_t, std::size_t>;
- using locator_type = detail::geometry_column_locator;
public:
-
- csv_index_featureset(std::string const& filename,
- mapnik::filter_in_box const& filter,
- locator_type const& locator,
- char separator,
- char quote,
- std::vector<std::string> const& headers,
- mapnik::context_ptr const& ctx);
- ~csv_index_featureset();
+ geojson_index_featureset(std::string const& filename, mapnik::filter_in_box const& filter);
+ virtual ~geojson_index_featureset();
mapnik::feature_ptr next();
+
private:
- mapnik::feature_ptr parse_feature(char const* beg, char const* end);
- char separator_;
- char quote_;
- std::vector<std::string> headers_;
- mapnik::context_ptr ctx_;
- mapnik::value_integer feature_id_ = 0;
- detail::geometry_column_locator const& locator_;
- mapnik::transcoder tr_;
-#if defined (CSV_MEMORY_MAPPED_FILE)
+#if defined (MAPNIK_MEMORY_MAPPED_FILE)
using file_source_type = boost::interprocess::ibufferstream;
mapnik::mapped_region_ptr mapped_region_;
#else
using file_ptr = std::unique_ptr<std::FILE, int (*)(std::FILE *)>;
file_ptr file_;
#endif
+ mapnik::value_integer feature_id_ = 1;
+ mapnik::context_ptr ctx_;
std::vector<value_type> positions_;
std::vector<value_type>::iterator itr_;
};
-
-#endif // CSV_INDEX_FEATURESET_HPP
+#endif // GEOJSON_INDEX_FEATURESE_HPP
diff --git a/plugins/input/geojson/large_geojson_featureset.cpp b/plugins/input/geojson/geojson_memory_index_featureset.cpp
similarity index 89%
rename from plugins/input/geojson/large_geojson_featureset.cpp
rename to plugins/input/geojson/geojson_memory_index_featureset.cpp
index 6f61d53..99a79ff 100644
--- a/plugins/input/geojson/large_geojson_featureset.cpp
+++ b/plugins/input/geojson/geojson_memory_index_featureset.cpp
@@ -30,9 +30,9 @@
#include <string>
#include <vector>
-#include "large_geojson_featureset.hpp"
+#include "geojson_memory_index_featureset.hpp"
-large_geojson_featureset::large_geojson_featureset(std::string const& filename,
+geojson_memory_index_featureset::geojson_memory_index_featureset(std::string const& filename,
array_type && index_array)
:
#ifdef _WINDOWS
@@ -48,9 +48,9 @@ large_geojson_featureset::large_geojson_featureset(std::string const& filename,
if (!file_) throw std::runtime_error("Can't open " + filename);
}
-large_geojson_featureset::~large_geojson_featureset() {}
+geojson_memory_index_featureset::~geojson_memory_index_featureset() {}
-mapnik::feature_ptr large_geojson_featureset::next()
+mapnik::feature_ptr geojson_memory_index_featureset::next()
{
if (index_itr_ != index_end_)
{
@@ -65,13 +65,12 @@ mapnik::feature_ptr large_geojson_featureset::next()
using chr_iterator_type = char const*;
chr_iterator_type start = json.data();
chr_iterator_type end = start + json.size();
-
static const mapnik::transcoder tr("utf8");
static const mapnik::json::feature_grammar<chr_iterator_type,mapnik::feature_impl> grammar(tr);
using namespace boost::spirit;
standard::space_type space;
- mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx_,1));
- if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space))
+ mapnik::feature_ptr feature(mapnik::feature_factory::create(ctx_, feature_id_++));
+ if (!qi::phrase_parse(start, end, (grammar)(boost::phoenix::ref(*feature)), space) || start != end)
{
throw std::runtime_error("Failed to parse geojson feature");
}
diff --git a/plugins/input/geojson/large_geojson_featureset.hpp b/plugins/input/geojson/geojson_memory_index_featureset.hpp
similarity index 81%
rename from plugins/input/geojson/large_geojson_featureset.hpp
rename to plugins/input/geojson/geojson_memory_index_featureset.hpp
index 8321ff3..a29751d 100644
--- a/plugins/input/geojson/large_geojson_featureset.hpp
+++ b/plugins/input/geojson/geojson_memory_index_featureset.hpp
@@ -20,8 +20,8 @@
*
*****************************************************************************/
-#ifndef LARGE_GEOJSON_FEATURESET_HPP
-#define LARGE_GEOJSON_FEATURESET_HPP
+#ifndef GEOJSON_MEMORY_INDEX_FEATURESET_HPP
+#define GEOJSON_MEMORY_INDEX_FEATURESET_HPP
#include <mapnik/feature.hpp>
#include "geojson_datasource.hpp"
@@ -29,24 +29,24 @@
#include <deque>
#include <cstdio>
-class large_geojson_featureset : public mapnik::Featureset
+class geojson_memory_index_featureset : public mapnik::Featureset
{
public:
using array_type = std::deque<geojson_datasource::item_type>;
using file_ptr = std::unique_ptr<std::FILE, int (*)(std::FILE *)>;
- large_geojson_featureset(std::string const& filename,
+ geojson_memory_index_featureset(std::string const& filename,
array_type && index_array);
- virtual ~large_geojson_featureset();
+ virtual ~geojson_memory_index_featureset();
mapnik::feature_ptr next();
private:
file_ptr file_;
-
+ mapnik::value_integer feature_id_ = 1;
const array_type index_array_;
array_type::const_iterator index_itr_;
array_type::const_iterator index_end_;
mapnik::context_ptr ctx_;
};
-#endif // LARGE_GEOJSON_FEATURESET_HPP
+#endif // GEOJSON_MEMORY_INDEX_FEATURESET_HPP
diff --git a/plugins/input/ogr/ogr_index_featureset.cpp b/plugins/input/ogr/ogr_index_featureset.cpp
index 9338f95..348c618 100644
--- a/plugins/input/ogr/ogr_index_featureset.cpp
+++ b/plugins/input/ogr/ogr_index_featureset.cpp
@@ -33,7 +33,7 @@
#include <mapnik/geometry_correct.hpp>
// boost
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#include <mapnik/mapped_memory_cache.hpp>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
@@ -72,7 +72,7 @@ ogr_index_featureset<filterT>::ogr_index_featureset(mapnik::context_ptr const &
feature_envelope_()
{
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
boost::optional<mapnik::mapped_region_ptr> memory = mapnik::mapped_memory_cache::instance().find(index_file, true);
if (memory)
{
diff --git a/plugins/input/shape/dbfile.cpp b/plugins/input/shape/dbfile.cpp
index 9211657..c26a117 100644
--- a/plugins/input/shape/dbfile.cpp
+++ b/plugins/input/shape/dbfile.cpp
@@ -36,7 +36,7 @@
#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
#pragma GCC diagnostic ignored "-Wsign-conversion"
#include <boost/spirit/include/qi.hpp>
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#include <boost/interprocess/mapped_region.hpp>
#include <mapnik/mapped_memory_cache.hpp>
#endif
@@ -58,7 +58,7 @@ dbf_file::dbf_file(std::string const& file_name)
:num_records_(0),
num_fields_(0),
record_length_(0),
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
file_(),
#elif defined(_WINDOWS)
file_(mapnik::utf8_to_utf16(file_name), std::ios::in | std::ios::binary),
@@ -68,7 +68,7 @@ dbf_file::dbf_file(std::string const& file_name)
record_(0)
{
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
boost::optional<mapnik::mapped_region_ptr> memory = mapnik::mapped_memory_cache::instance().find(file_name,true);
if (memory)
{
@@ -95,7 +95,7 @@ dbf_file::~dbf_file()
bool dbf_file::is_open()
{
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
return (file_.buffer().second > 0);
#else
return file_.is_open();
diff --git a/plugins/input/shape/dbfile.hpp b/plugins/input/shape/dbfile.hpp
index 358d737..6812e54 100644
--- a/plugins/input/shape/dbfile.hpp
+++ b/plugins/input/shape/dbfile.hpp
@@ -27,7 +27,7 @@
#include <mapnik/feature.hpp>
#include <mapnik/util/noncopyable.hpp>
#include <mapnik/unicode.hpp>
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#include <mapnik/mapped_memory_cache.hpp>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
@@ -59,7 +59,7 @@ private:
int num_fields_;
std::size_t record_length_;
std::vector<field_descriptor> fields_;
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
boost::interprocess::ibufferstream file_;
mapnik::mapped_region_ptr mapped_region_;
#else
diff --git a/plugins/input/shape/shape_datasource.cpp b/plugins/input/shape/shape_datasource.cpp
index cdde8ee..f2a5160 100644
--- a/plugins/input/shape/shape_datasource.cpp
+++ b/plugins/input/shape/shape_datasource.cpp
@@ -104,7 +104,7 @@ shape_datasource::shape_datasource(parameters const& params)
init(*shape_ref);
for (int i=0;i<shape_ref->dbf().num_fields();++i)
{
- field_descriptor const& fd=shape_ref->dbf().descriptor(i);
+ field_descriptor const& fd = shape_ref->dbf().descriptor(i);
std::string fld_name=fd.name_;
switch (fd.type_)
{
@@ -167,36 +167,42 @@ void shape_datasource::init(shape_io& shape)
#endif
//first read header from *.shp
- int file_code=shape.shp().read_xdr_integer();
- if (file_code!=9994)
+ shape_file::record_type header(100);
+ shape.shp().read_record(header);
+
+ int file_code = header.read_xdr_integer();
+ if (file_code != 9994)
{
std::ostringstream s;
s << "Shape Plugin: wrong file code " << file_code;
throw datasource_exception(s.str());
}
+ header.skip(5 * 4);
+ file_length_ = header.read_xdr_integer();
+ int version = header.read_ndr_integer();
- shape.shp().skip(5*4);
- file_length_=shape.shp().read_xdr_integer();
- int version=shape.shp().read_ndr_integer();
-
- if (version!=1000)
+ if (version != 1000)
{
std::ostringstream s;
s << "Shape Plugin: nvalid version number " << version;
throw datasource_exception(s.str());
}
- shape_type_ = static_cast<shape_io::shapeType>(shape.shp().read_ndr_integer());
+ shape_type_ = static_cast<shape_io::shapeType>(header.read_ndr_integer());
if (shape_type_ == shape_io::shape_multipatch)
throw datasource_exception("Shape Plugin: shapefile multipatch type is not supported");
- shape.shp().read_envelope(extent_);
+ const double lox = header.read_double();
+ const double loy = header.read_double();
+ const double hix = header.read_double();
+ const double hiy = header.read_double();
+ extent_.init(lox, loy, hix, hiy);
#ifdef MAPNIK_LOG
- const double zmin = shape.shp().read_double();
- const double zmax = shape.shp().read_double();
- const double mmin = shape.shp().read_double();
- const double mmax = shape.shp().read_double();
+ const double zmin = header.read_double();
+ const double zmax = header.read_double();
+ const double mmin = header.read_double();
+ const double mmax = header.read_double();
MAPNIK_LOG_DEBUG(shape) << "shape_datasource: Z min/max=" << zmin << "," << zmax;
MAPNIK_LOG_DEBUG(shape) << "shape_datasource: M min/max=" << mmin << "," << mmax;
@@ -250,7 +256,6 @@ featureset_ptr shape_datasource::features(query const& q) const
shape_name_,
q.property_names(),
desc_.get_encoding(),
- file_length_,
row_limit_);
}
}
@@ -288,7 +293,6 @@ featureset_ptr shape_datasource::features_at_point(coord2d const& pt, double tol
shape_name_,
names,
desc_.get_encoding(),
- file_length_,
row_limit_);
}
}
diff --git a/plugins/input/shape/shape_featureset.cpp b/plugins/input/shape/shape_featureset.cpp
index 8a2b164..490fa3d 100644
--- a/plugins/input/shape/shape_featureset.cpp
+++ b/plugins/input/shape/shape_featureset.cpp
@@ -22,7 +22,7 @@
// stl
#include <iostream>
-
+#include <cassert>
// mapnik
#include <mapnik/debug.hpp>
#include <mapnik/feature_factory.hpp>
@@ -41,20 +41,26 @@ shape_featureset<filterT>::shape_featureset(filterT const& filter,
std::string const& shape_name,
std::set<std::string> const& attribute_names,
std::string const& encoding,
- long file_length,
int row_limit)
: filter_(filter),
shape_(shape_name, false),
query_ext_(),
feature_bbox_(),
tr_(new transcoder(encoding)),
- file_length_(file_length),
+ shx_file_length_(0),
row_limit_(row_limit),
count_(0),
ctx_(std::make_shared<mapnik::context_type>())
{
- shape_.shp().skip(100);
- setup_attributes(ctx_, attribute_names, shape_name, shape_,attr_ids_);
+ if (!shape_.shx().is_open())
+ {
+ throw mapnik::datasource_exception("Shape Plugin: can't open '" + shape_name + ".shx' file");
+ }
+ shape_file::record_type shx_header(100);
+ shape_.shx().read_record(shx_header);
+ shx_header.skip(6 * 4);
+ shx_file_length_ = shx_header.read_xdr_integer();
+ setup_attributes(ctx_, attribute_names, shape_name, shape_, attr_ids_);
}
template <typename filterT>
@@ -65,18 +71,22 @@ feature_ptr shape_featureset<filterT>::next()
return feature_ptr();
}
-
- while (shape_.shp().pos() < std::streampos(file_length_ * 2))
+ std::streampos position_limit = 2 * shx_file_length_ - 2 * sizeof(int);
+ while (shape_.shx().is_good() && shape_.shx().pos() <= position_limit)
{
- shape_.move_to(shape_.shp().pos());
- shape_file::record_type record(shape_.reclength_ * 2);
+ int offset = shape_.shx().read_xdr_integer();
+ int record_length = shape_.shx().read_xdr_integer();
+ shape_.move_to(2 * offset);
+ mapnik::value_integer feature_id = shape_.id();
+ assert(record_length == shape_.reclength_);
+ shape_file::record_type record(record_length * 2);
shape_.shp().read_record(record);
int type = record.read_ndr_integer();
// skip null shapes
if (type == shape_io::shape_null) continue;
- feature_ptr feature(feature_factory::create(ctx_, shape_.id_));
+ feature_ptr feature(feature_factory::create(ctx_, feature_id));
switch (type)
{
case shape_io::shape_point:
@@ -131,18 +141,14 @@ feature_ptr shape_featureset<filterT>::next()
return feature_ptr();
}
- // FIXME: https://github.com/mapnik/mapnik/issues/1020
- feature->set_id(shape_.id_);
if (attr_ids_.size())
{
shape_.dbf().move_to(shape_.id_);
- std::vector<int>::const_iterator itr = attr_ids_.begin();
- std::vector<int>::const_iterator end = attr_ids_.end();
try
{
- for (; itr != end; ++itr)
+ for (auto id : attr_ids_)
{
- shape_.dbf().add_attribute(*itr, *tr_, *feature); //TODO optimize!!!
+ shape_.dbf().add_attribute(id, *tr_, *feature); //TODO optimize!!!
}
}
catch (...)
diff --git a/plugins/input/shape/shape_featureset.hpp b/plugins/input/shape/shape_featureset.hpp
index a047bae..caa9625 100644
--- a/plugins/input/shape/shape_featureset.hpp
+++ b/plugins/input/shape/shape_featureset.hpp
@@ -50,7 +50,6 @@ public:
std::string const& shape_file,
std::set<std::string> const& attribute_names,
std::string const& encoding,
- long file_length,
int row_limit);
virtual ~shape_featureset();
feature_ptr next();
@@ -61,7 +60,7 @@ private:
box2d<double> query_ext_;
mutable box2d<double> feature_bbox_;
const std::unique_ptr<transcoder> tr_;
- long file_length_;
+ long shx_file_length_;
std::vector<int> attr_ids_;
mapnik::value_integer row_limit_;
mutable int count_;
diff --git a/plugins/input/shape/shape_index_featureset.cpp b/plugins/input/shape/shape_index_featureset.cpp
index 1fa6f0b..e352efd 100644
--- a/plugins/input/shape/shape_index_featureset.cpp
+++ b/plugins/input/shape/shape_index_featureset.cpp
@@ -32,7 +32,7 @@
#pragma GCC diagnostic ignored "-Wunused-parameter"
#pragma GCC diagnostic ignored "-Wunused-local-typedef"
#include <boost/algorithm/string.hpp>
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#include <boost/interprocess/streams/bufferstream.hpp>
#endif
#pragma GCC diagnostic pop
@@ -63,7 +63,7 @@ shape_index_featureset<filterT>::shape_index_featureset(filterT const& filter,
auto index = shape_ptr_->index();
if (index)
{
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
mapnik::util::spatial_index<int, filterT,boost::interprocess::ibufferstream>::query(filter, index->file(), offsets_);
#else
mapnik::util::spatial_index<int, filterT, std::ifstream>::query(filter, index->file(), offsets_);
@@ -85,10 +85,11 @@ feature_ptr shape_index_featureset<filterT>::next()
while ( itr_ != offsets_.end())
{
shape_ptr_->move_to(*itr_++);
+ mapnik::value_integer feature_id = shape_ptr_->id();
shape_file::record_type record(shape_ptr_->reclength_ * 2);
shape_ptr_->shp().read_record(record);
int type = record.read_ndr_integer();
- feature_ptr feature(feature_factory::create(ctx_,shape_ptr_->id_));
+ feature_ptr feature(feature_factory::create(ctx_, feature_id));
switch (type)
{
@@ -141,18 +142,14 @@ feature_ptr shape_index_featureset<filterT>::next()
return feature_ptr();
}
- // FIXME: https://github.com/mapnik/mapnik/issues/1020
- feature->set_id(shape_ptr_->id_);
if (attr_ids_.size())
{
shape_ptr_->dbf().move_to(shape_ptr_->id_);
- std::vector<int>::const_iterator itr = attr_ids_.begin();
- std::vector<int>::const_iterator end = attr_ids_.end();
try
{
- for (; itr!=end; ++itr)
+ for (auto id : attr_ids_)
{
- shape_ptr_->dbf().add_attribute(*itr, *tr_, *feature);
+ shape_ptr_->dbf().add_attribute(id, *tr_, *feature);
}
}
catch (...)
diff --git a/plugins/input/shape/shape_io.cpp b/plugins/input/shape/shape_io.cpp
index 7f14982..bb9d783 100644
--- a/plugins/input/shape/shape_io.cpp
+++ b/plugins/input/shape/shape_io.cpp
@@ -31,18 +31,20 @@
using mapnik::datasource_exception;
const std::string shape_io::SHP = ".shp";
+const std::string shape_io::SHX = ".shx";
const std::string shape_io::DBF = ".dbf";
const std::string shape_io::INDEX = ".index";
shape_io::shape_io(std::string const& shape_name, bool open_index)
: type_(shape_null),
shp_(shape_name + SHP),
+ shx_(shape_name + SHX),
dbf_(shape_name + DBF),
reclength_(0),
id_(0)
{
bool ok = (shp_.is_open() && dbf_.is_open());
- if (! ok)
+ if (!ok)
{
throw datasource_exception("Shape Plugin: cannot read shape file '" + shape_name + "'");
}
@@ -58,6 +60,10 @@ shape_io::shape_io(std::string const& shape_name, bool open_index)
MAPNIK_LOG_WARN(shape) << "shape_io: Could not open index=" << shape_name << INDEX;
}
}
+ if (!index_ && !shx_.is_open())
+ {
+ throw datasource_exception("Shape Plugin: cannot read shape index file '" + shape_name + ".shx'");
+ }
}
shape_io::~shape_io() {}
@@ -74,6 +80,11 @@ shape_file& shape_io::shp()
return shp_;
}
+shape_file& shape_io::shx()
+{
+ return shx_;
+}
+
dbf_file& shape_io::dbf()
{
return dbf_;
diff --git a/plugins/input/shape/shape_io.hpp b/plugins/input/shape/shape_io.hpp
index 497dbda..ab28f99 100644
--- a/plugins/input/shape/shape_io.hpp
+++ b/plugins/input/shape/shape_io.hpp
@@ -61,6 +61,7 @@ public:
~shape_io();
shape_file& shp();
+ shape_file& shx();
dbf_file& dbf();
inline boost::optional<shape_file&> index()
@@ -74,6 +75,7 @@ public:
return (index_ && index_->is_open());
}
+ inline int id() const { return id_;}
void move_to(std::streampos pos);
static void read_bbox(shape_file::record_type & record, mapnik::box2d<double> & bbox);
static mapnik::geometry::geometry<double> read_polyline(shape_file::record_type & record);
@@ -81,13 +83,15 @@ public:
shapeType type_;
shape_file shp_;
+ shape_file shx_;
dbf_file dbf_;
std::unique_ptr<shape_file> index_;
- unsigned reclength_;
- unsigned id_;
+ int reclength_;
+ int id_;
box2d<double> cur_extent_;
static const std::string SHP;
+ static const std::string SHX;
static const std::string DBF;
static const std::string INDEX;
};
diff --git a/plugins/input/shape/shapefile.hpp b/plugins/input/shape/shapefile.hpp
index 3f70e3a..1c61df0 100644
--- a/plugins/input/shape/shapefile.hpp
+++ b/plugins/input/shape/shapefile.hpp
@@ -33,7 +33,7 @@
#include <mapnik/global.hpp>
#include <mapnik/util/utf_conv_win.hpp>
#include <mapnik/box2d.hpp>
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
#pragma GCC diagnostic ignored "-Wsign-conversion"
@@ -141,7 +141,7 @@ class shape_file : mapnik::util::noncopyable
{
public:
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
using file_source_type = boost::interprocess::ibufferstream;
using record_type = shape_record<MappedRecordTag>;
mapnik::mapped_region_ptr mapped_region_;
@@ -155,7 +155,7 @@ public:
shape_file() {}
shape_file(std::string const& file_name) :
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
file_()
#elif defined (_WINDOWS)
file_(mapnik::utf8_to_utf16(file_name), std::ios::in | std::ios::binary)
@@ -163,7 +163,7 @@ public:
file_(file_name.c_str(), std::ios::in | std::ios::binary)
#endif
{
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
boost::optional<mapnik::mapped_region_ptr> memory =
mapnik::mapped_memory_cache::instance().find(file_name,true);
@@ -188,7 +188,7 @@ public:
inline bool is_open() const
{
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
return (file_.buffer().second > 0);
#else
return file_.is_open();
@@ -197,7 +197,7 @@ public:
inline void read_record(record_type& rec)
{
-#ifdef SHAPE_MEMORY_MAPPED_FILE
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
rec.set_data(file_.buffer().first + file_.tellg());
file_.seekg(rec.size, std::ios::cur);
#else
@@ -259,6 +259,11 @@ public:
{
return file_.eof();
}
+
+ inline bool is_good()
+ {
+ return file_.good();
+ }
};
#endif // SHAPEFILE_HPP
diff --git a/src/json/mapnik_json_feature_collection_grammar.cpp b/src/json/mapnik_json_feature_collection_grammar.cpp
index 5f598f5..e262d7c 100644
--- a/src/json/mapnik_json_feature_collection_grammar.cpp
+++ b/src/json/mapnik_json_feature_collection_grammar.cpp
@@ -26,3 +26,4 @@
using iterator_type = char const*;
template struct mapnik::json::feature_collection_grammar<iterator_type,mapnik::feature_impl, mapnik::json::default_feature_callback> ;
+template struct mapnik::json::feature_grammar_callback<iterator_type,mapnik::feature_impl, mapnik::json::default_feature_callback> ;
diff --git a/src/mapped_memory_cache.cpp b/src/mapped_memory_cache.cpp
index abe08d1..00ad963 100644
--- a/src/mapped_memory_cache.cpp
+++ b/src/mapped_memory_cache.cpp
@@ -20,7 +20,7 @@
*
*****************************************************************************/
-#if defined(SHAPE_MEMORY_MAPPED_FILE)
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
// mapnik
#include <mapnik/debug.hpp>
diff --git a/test/standalone/csv_test.cpp b/test/standalone/csv_test.cpp
deleted file mode 100644
index 7d08bb5..0000000
--- a/test/standalone/csv_test.cpp
+++ /dev/null
@@ -1,676 +0,0 @@
-#define CATCH_CONFIG_MAIN
-#include "catch.hpp"
-
-#include <mapnik/map.hpp>
-#include <mapnik/datasource.hpp>
-#include <mapnik/datasource_cache.hpp>
-#include <mapnik/geometry.hpp>
-#include <mapnik/geometry_types.hpp>
-#include <mapnik/geometry_type.hpp>
-#include <mapnik/expression.hpp>
-#include <mapnik/expression_evaluator.hpp>
-#include <mapnik/debug.hpp>
-#include <mapnik/util/fs.hpp>
-#include <boost/filesystem.hpp>
-#include <boost/range/iterator_range_core.hpp>
-#include <boost/format.hpp>
-#include <boost/optional/optional_io.hpp>
-
-#include <iostream>
-
-namespace bfs = boost::filesystem;
-
-namespace {
-void add_csv_files(bfs::path dir, std::vector<bfs::path> &csv_files)
-{
- for (auto const &entry : boost::make_iterator_range(
- bfs::directory_iterator(dir), bfs::directory_iterator()))
- {
- auto path = entry.path();
- if (path.extension().native() == ".csv")
- {
- csv_files.emplace_back(path);
- }
- }
-}
-
-mapnik::datasource_ptr get_csv_ds(std::string const &file_name, bool strict = true)
-{
- mapnik::parameters params;
- params["type"] = std::string("csv");
- params["file"] = file_name;
- params["strict"] = mapnik::value_bool(strict);
- auto ds = mapnik::datasource_cache::instance().create(params);
- // require a non-null pointer returned
- REQUIRE(ds != nullptr);
- return ds;
-}
-
-void require_field_names(std::vector<mapnik::attribute_descriptor> const &fields,
- std::initializer_list<std::string> const &names)
-{
- REQUIRE(fields.size() == names.size());
- auto itr_a = fields.begin();
- auto const end_a = fields.end();
- auto itr_b = names.begin();
- for (; itr_a != end_a; ++itr_a, ++itr_b)
- {
- CHECK(itr_a->get_name() == *itr_b);
- }
-}
-
-void require_field_types(std::vector<mapnik::attribute_descriptor> const &fields,
- std::initializer_list<mapnik::eAttributeType> const &types) {
- REQUIRE(fields.size() == types.size());
- auto itr_a = fields.begin();
- auto const end_a = fields.end();
- auto itr_b = types.begin();
- for (; itr_a != end_a; ++itr_a, ++itr_b) {
- CHECK(itr_a->get_type() == *itr_b);
- }
-}
-
-mapnik::featureset_ptr all_features(mapnik::datasource_ptr ds) {
- auto fields = ds->get_descriptor().get_descriptors();
- mapnik::query query(ds->envelope());
- for (auto const &field : fields) {
- query.add_property_name(field.get_name());
- }
- return ds->features(query);
-}
-
-std::size_t count_features(mapnik::featureset_ptr features) {
- std::size_t count = 0;
- while (features->next()) {
- ++count;
- }
- return count;
-}
-
-using attr = std::tuple<std::string, mapnik::value>;
-void require_attributes(mapnik::feature_ptr feature,
- std::initializer_list<attr> const &attrs) {
- REQUIRE(bool(feature));
- for (auto const &kv : attrs) {
- REQUIRE(feature->has_key(std::get<0>(kv)));
- CHECK(feature->get(std::get<0>(kv)) == std::get<1>(kv));
- }
-}
-
-namespace detail {
-struct feature_count {
- template <typename T>
- std::size_t operator()(T const &geom) const {
- return mapnik::util::apply_visitor(*this, geom);
- }
-
- std::size_t operator()(mapnik::geometry::geometry_empty const &) const {
- return 0;
- }
-
- template <typename T>
- std::size_t operator()(mapnik::geometry::point<T> const &) const {
- return 1;
- }
-
- template <typename T>
- std::size_t operator()(mapnik::geometry::line_string<T> const &) const {
- return 1;
- }
-
- template <typename T>
- std::size_t operator()(mapnik::geometry::polygon<T> const &) const {
- return 1;
- }
-
- template <typename T>
- std::size_t operator()(mapnik::geometry::multi_point<T> const &mp) const {
- return mp.size();
- }
-
- template <typename T>
- std::size_t operator()(mapnik::geometry::multi_line_string<T> const &mls) const {
- return mls.size();
- }
-
- template <typename T>
- std::size_t operator()(mapnik::geometry::multi_polygon<T> const &mp) const {
- return mp.size();
- }
-
- template <typename T>
- std::size_t operator()(mapnik::geometry::geometry_collection<T> const &col) const {
- std::size_t sum = 0;
- for (auto const &geom : col) {
- sum += operator()(geom);
- }
- return sum;
- }
-};
-} // namespace detail
-
-template <typename T>
-std::size_t feature_count(mapnik::geometry::geometry<T> const &g) {
- return detail::feature_count()(g);
-}
-
-void require_geometry(mapnik::feature_ptr feature,
- std::size_t num_parts,
- mapnik::geometry::geometry_types type) {
- REQUIRE(bool(feature));
- CHECK(mapnik::geometry::geometry_type(feature->get_geometry()) == type);
- CHECK(feature_count(feature->get_geometry()) == num_parts);
-}
-} // anonymous namespace
-
-static const std::string csv_plugin("./plugins/input/csv.input");
-
-const bool registered = mapnik::datasource_cache::instance().register_datasources(csv_plugin);
-
-TEST_CASE("csv") {
-
- if (mapnik::util::exists(csv_plugin))
- {
- REQUIRE(registered);
- // make the tests silent since we intentially test error conditions that are noisy
- auto const severity = mapnik::logger::instance().get_severity();
- mapnik::logger::instance().set_severity(mapnik::logger::none);
-
- // check the CSV datasource is loaded
- const std::vector<std::string> plugin_names =
- mapnik::datasource_cache::instance().plugin_names();
- const bool have_csv_plugin =
- std::find(plugin_names.begin(), plugin_names.end(), "csv") != plugin_names.end();
-
- SECTION("broken files") {
- if (have_csv_plugin) {
- std::vector<bfs::path> broken;
- add_csv_files("test/data/csv/fails", broken);
- add_csv_files("test/data/csv/warns", broken);
- broken.emplace_back("test/data/csv/fails/does_not_exist.csv");
-
- for (auto const &path : broken)
- {
- INFO(path);
- REQUIRE_THROWS(get_csv_ds(path.native()));
- }
- }
- } // END SECTION
-
- SECTION("good files") {
- if (have_csv_plugin) {
- std::vector<bfs::path> good;
- add_csv_files("test/data/csv", good);
- add_csv_files("test/data/csv/warns", good);
-
- for (auto const& path : good)
- {
- auto ds = get_csv_ds(path.native(), false);
- // require a non-null pointer returned
- REQUIRE(bool(ds));
- }
- }
- } // END SECTION
-
- SECTION("lon/lat detection")
- {
- for (auto const& lon_name : {std::string("lon"), std::string("lng")})
- {
- auto ds = get_csv_ds((boost::format("test/data/csv/%1%_lat.csv") % lon_name).str());
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {lon_name, "lat"});
- require_field_types(fields, {mapnik::Integer, mapnik::Integer});
-
- CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
-
- mapnik::query query(ds->envelope());
- for (auto const &field : fields)
- {
- query.add_property_name(field.get_name());
- }
- auto features = ds->features(query);
- auto feature = features->next();
-
- require_attributes(feature, {
- attr { lon_name, mapnik::value_integer(0) },
- attr { "lat", mapnik::value_integer(0) }
- });
- }
- } // END SECTION
-
- SECTION("type detection") {
- auto ds = get_csv_ds("test/data/csv/nypd.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"Precinct", "Phone", "Address", "City", "geo_longitude", "geo_latitude", "geo_accuracy"});
- require_field_types(fields, {mapnik::String, mapnik::String, mapnik::String, mapnik::String, mapnik::Double, mapnik::Double, mapnik::String});
-
- CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
- CHECK(count_features(all_features(ds)) == 2);
-
- auto feature = all_features(ds)->next();
- require_attributes(feature, {
- attr { "City", mapnik::value_unicode_string("New York, NY") }
- , attr { "geo_accuracy", mapnik::value_unicode_string("house") }
- , attr { "Phone", mapnik::value_unicode_string("(212) 334-0711") }
- , attr { "Address", mapnik::value_unicode_string("19 Elizabeth Street") }
- , attr { "Precinct", mapnik::value_unicode_string("5th Precinct") }
- , attr { "geo_longitude", mapnik::value_integer(-70) }
- , attr { "geo_latitude", mapnik::value_integer(40) }
- });
- } // END SECTION
-
- SECTION("skipping blank rows") {
- auto ds = get_csv_ds("test/data/csv/blank_rows.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "name"});
- require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
-
- CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
- CHECK(count_features(all_features(ds)) == 2);
- } // END SECTION
-
- SECTION("empty rows") {
- auto ds = get_csv_ds("test/data/csv/empty_rows.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "text", "date", "integer", "boolean", "float", "time", "datetime", "empty_column"});
- require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::String, mapnik::Integer, mapnik::Boolean, mapnik::Double, mapnik::String, mapnik::String, mapnik::String});
-
- CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
- CHECK(count_features(all_features(ds)) == 4);
-
- auto featureset = all_features(ds);
- auto feature = featureset->next();
- require_attributes(feature, {
- attr { "x", mapnik::value_integer(0) }
- , attr { "empty_column", mapnik::value_unicode_string("") }
- , attr { "text", mapnik::value_unicode_string("a b") }
- , attr { "float", mapnik::value_double(1.0) }
- , attr { "datetime", mapnik::value_unicode_string("1971-01-01T04:14:00") }
- , attr { "y", mapnik::value_integer(0) }
- , attr { "boolean", mapnik::value_bool(true) }
- , attr { "time", mapnik::value_unicode_string("04:14:00") }
- , attr { "date", mapnik::value_unicode_string("1971-01-01") }
- , attr { "integer", mapnik::value_integer(40) }
- });
-
- while (bool(feature = featureset->next())) {
- CHECK(feature->size() == 10);
- CHECK(feature->get("empty_column") == mapnik::value_unicode_string(""));
- }
- } // END SECTION
-
- SECTION("slashes") {
- auto ds = get_csv_ds("test/data/csv/has_attributes_with_slashes.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "name"});
- // NOTE: y column is integer, even though a double value is used below in the test?
- require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
-
- auto featureset = all_features(ds);
- require_attributes(featureset->next(), {
- attr{"x", 0}
- , attr{"y", 0}
- , attr{"name", mapnik::value_unicode_string("a/a") } });
- require_attributes(featureset->next(), {
- attr{"x", 1}
- , attr{"y", 4}
- , attr{"name", mapnik::value_unicode_string("b/b") } });
- require_attributes(featureset->next(), {
- attr{"x", 10}
- , attr{"y", 2.5}
- , attr{"name", mapnik::value_unicode_string("c/c") } });
- } // END SECTION
-
- SECTION("wkt field") {
- using mapnik::geometry::geometry_types;
-
- auto ds = get_csv_ds("test/data/csv/wkt.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"type"});
- require_field_types(fields, {mapnik::String});
-
- auto featureset = all_features(ds);
- require_geometry(featureset->next(), 1, geometry_types::Point);
- require_geometry(featureset->next(), 1, geometry_types::LineString);
- require_geometry(featureset->next(), 1, geometry_types::Polygon);
- require_geometry(featureset->next(), 1, geometry_types::Polygon);
- require_geometry(featureset->next(), 4, geometry_types::MultiPoint);
- require_geometry(featureset->next(), 2, geometry_types::MultiLineString);
- require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
- require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
- } // END SECTION
-
- SECTION("handling of missing header") {
- // TODO: does this mean 'missing_header.csv' should be in the warnings
- // subdirectory, since it doesn't work in strict mode?
- auto ds = get_csv_ds("test/data/csv/missing_header.csv", false);
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"one", "two", "x", "y", "_4", "aftermissing"});
- auto feature = all_features(ds)->next();
- REQUIRE(feature);
- REQUIRE(feature->has_key("_4"));
- CHECK(feature->get("_4") == mapnik::value_unicode_string("missing"));
- } // END SECTION
-
- SECTION("handling of headers that are numbers") {
- auto ds = get_csv_ds("test/data/csv/numbers_for_headers.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "1990", "1991", "1992"});
- auto feature = all_features(ds)->next();
- require_attributes(feature, {
- attr{"x", 0}
- , attr{"y", 0}
- , attr{"1990", 1}
- , attr{"1991", 2}
- , attr{"1992", 3}
- });
- auto expression = mapnik::parse_expression("[1991]=2");
- REQUIRE(bool(expression));
- auto value = mapnik::util::apply_visitor(
- mapnik::evaluate<mapnik::feature_impl, mapnik::value_type, mapnik::attributes>(
- *feature, mapnik::attributes()), *expression);
- CHECK(value == true);
- } // END SECTION
-
- SECTION("quoted numbers") {
- using ustring = mapnik::value_unicode_string;
-
- auto ds = get_csv_ds("test/data/csv/quoted_numbers.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "label"});
- auto featureset = all_features(ds);
-
- require_attributes(featureset->next(), {
- attr{"x", 0}, attr{"y", 0}, attr{"label", ustring("0,0") } });
- require_attributes(featureset->next(), {
- attr{"x", 5}, attr{"y", 5}, attr{"label", ustring("5,5") } });
- require_attributes(featureset->next(), {
- attr{"x", 0}, attr{"y", 5}, attr{"label", ustring("0,5") } });
- require_attributes(featureset->next(), {
- attr{"x", 5}, attr{"y", 0}, attr{"label", ustring("5,0") } });
- require_attributes(featureset->next(), {
- attr{"x", 2.5}, attr{"y", 2.5}, attr{"label", ustring("2.5,2.5") } });
-
- } // END SECTION
-
- SECTION("reading newlines") {
- for (auto const &platform : {std::string("windows"), std::string("mac")}) {
- std::string file_name = (boost::format("test/data/csv/%1%_newlines.csv") % platform).str();
- auto ds = get_csv_ds(file_name);
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "z"});
- require_attributes(all_features(ds)->next(), {
- attr{"x", 1}, attr{"y", 10}, attr{"z", 9999.9999} });
- }
- } // END SECTION
-
- SECTION("mixed newlines") {
- using ustring = mapnik::value_unicode_string;
-
- for (auto const &file : {
- std::string("test/data/csv/mac_newlines_with_unix_inline.csv")
- , std::string("test/data/csv/mac_newlines_with_unix_inline_escaped.csv")
- , std::string("test/data/csv/windows_newlines_with_unix_inline.csv")
- , std::string("test/data/csv/windows_newlines_with_unix_inline_escaped.csv")
- }) {
- auto ds = get_csv_ds(file);
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "line"});
- require_attributes(all_features(ds)->next(), {
- attr{"x", 0}, attr{"y", 0}
- , attr{"line", ustring("many\n lines\n of text\n with unix newlines")} });
- }
- } // END SECTION
-
- SECTION("tabs") {
- auto ds = get_csv_ds("test/data/csv/tabs_in_csv.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "z"});
- require_attributes(all_features(ds)->next(), {
- attr{"x", -122}, attr{"y", 48}, attr{"z", 0} });
- } // END SECTION
-
- SECTION("separators") {
- using ustring = mapnik::value_unicode_string;
-
- for (auto const &file : {
- std::string("test/data/csv/pipe_delimiters.csv")
- , std::string("test/data/csv/semicolon_delimiters.csv")
- }) {
- auto ds = get_csv_ds(file);
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "z"});
- require_attributes(all_features(ds)->next(), {
- attr{"x", 0}, attr{"y", 0}, attr{"z", ustring("hello")} });
- }
- } // END SECTION
-
- SECTION("null and bool keywords are empty strings") {
- using ustring = mapnik::value_unicode_string;
-
- auto ds = get_csv_ds("test/data/csv/nulls_and_booleans_as_strings.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "null", "boolean"});
- require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::Boolean});
-
- auto featureset = all_features(ds);
- require_attributes(featureset->next(), {
- attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("null")}, attr{"boolean", true}});
- require_attributes(featureset->next(), {
- attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("")}, attr{"boolean", false}});
- } // END SECTION
-
- SECTION("nonexistent query fields throw") {
- auto ds = get_csv_ds("test/data/csv/lon_lat.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"lon", "lat"});
- require_field_types(fields, {mapnik::Integer, mapnik::Integer});
-
- mapnik::query query(ds->envelope());
- for (auto const &field : fields) {
- query.add_property_name(field.get_name());
- }
- // also add an invalid one, triggering throw
- query.add_property_name("bogus");
-
- REQUIRE_THROWS(ds->features(query));
- } // END SECTION
-
- SECTION("leading zeros mean strings") {
- using ustring = mapnik::value_unicode_string;
-
- auto ds = get_csv_ds("test/data/csv/leading_zeros.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "fips"});
- require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
-
- auto featureset = all_features(ds);
- require_attributes(featureset->next(), {
- attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("001")}});
- require_attributes(featureset->next(), {
- attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("003")}});
- require_attributes(featureset->next(), {
- attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("005")}});
- } // END SECTION
-
- SECTION("advanced geometry detection") {
- using row = std::pair<std::string, mapnik::datasource_geometry_t>;
-
- for (row r : {
- row{"point", mapnik::datasource_geometry_t::Point}
- , row{"poly", mapnik::datasource_geometry_t::Polygon}
- , row{"multi_poly", mapnik::datasource_geometry_t::Polygon}
- , row{"line", mapnik::datasource_geometry_t::LineString}
- }) {
- std::string file_name = (boost::format("test/data/csv/%1%_wkt.csv") % r.first).str();
- auto ds = get_csv_ds(file_name);
- CHECK(ds->get_geometry_type() == r.second);
- }
- } // END SECTION
-
- SECTION("creation of CSV from in-memory strings") {
- using ustring = mapnik::value_unicode_string;
-
- for (auto const &name : {std::string("Winthrop, WA"), std::string(u8"Qu\u00e9bec")}) {
- std::string csv_string =
- (boost::format(
- "wkt,Name\n"
- "\"POINT (120.15 48.47)\",\"%1%\"\n"
- ) % name).str();
-
- mapnik::parameters params;
- params["type"] = std::string("csv");
- params["inline"] = csv_string;
- auto ds = mapnik::datasource_cache::instance().create(params);
- REQUIRE(bool(ds));
-
- auto feature = all_features(ds)->next();
- REQUIRE(bool(feature));
- REQUIRE(feature->has_key("Name"));
- CHECK(feature->get("Name") == ustring(name.c_str()));
- }
- } // END SECTION
-
- SECTION("geojson quoting") {
- using mapnik::geometry::geometry_types;
-
- for (auto const &file : {
- std::string("test/data/csv/geojson_double_quote_escape.csv")
- , std::string("test/data/csv/geojson_single_quote.csv")
- , std::string("test/data/csv/geojson_2x_double_quote_filebakery_style.csv")
- }) {
- auto ds = get_csv_ds(file);
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"type"});
- require_field_types(fields, {mapnik::String});
-
- auto featureset = all_features(ds);
- require_geometry(featureset->next(), 1, geometry_types::Point);
- require_geometry(featureset->next(), 1, geometry_types::LineString);
- require_geometry(featureset->next(), 1, geometry_types::Polygon);
- require_geometry(featureset->next(), 1, geometry_types::Polygon);
- require_geometry(featureset->next(), 4, geometry_types::MultiPoint);
- require_geometry(featureset->next(), 2, geometry_types::MultiLineString);
- require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
- require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
- }
- } // END SECTION
-
- SECTION("fewer headers than rows throws") {
- REQUIRE_THROWS(get_csv_ds("test/data/csv/more_column_values_than_headers.csv"));
- } // END SECTION
-
- SECTION("feature ID only incremented for valid rows") {
- auto ds = get_csv_ds("test/data/csv/warns/feature_id_counting.csv", false);
- auto fs = all_features(ds);
-
- // first
- auto feature = fs->next();
- REQUIRE(bool(feature));
- CHECK(feature->id() == 1);
-
- // second, should have skipped bogus one
- feature = fs->next();
- REQUIRE(bool(feature));
- CHECK(feature->id() == 2);
-
- feature = fs->next();
- CHECK(!feature);
- } // END SECTION
-
- SECTION("dynamically defining headers") {
- using ustring = mapnik::value_unicode_string;
- using row = std::pair<std::string, std::size_t>;
-
- for (auto const &r : {
- row{"test/data/csv/fails/needs_headers_two_lines.csv", 2},
- row{"test/data/csv/fails/needs_headers_one_line.csv", 1},
- row{"test/data/csv/fails/needs_headers_one_line_no_newline.csv", 1}})
- {
- mapnik::parameters params;
- params["type"] = std::string("csv");
- params["file"] = r.first;
- params["headers"] = "x,y,name";
- auto ds = mapnik::datasource_cache::instance().create(params);
- REQUIRE(bool(ds));
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "name"});
- require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
- require_attributes(all_features(ds)->next(), {
- attr{"x", 0}, attr{"y", 0}, attr{"name", ustring("data_name")} });
- REQUIRE(count_features(all_features(ds)) == r.second);
- CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
- }
- } // END SECTION
-
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wlong-long"
- SECTION("64bit int fields work") {
- auto ds = get_csv_ds("test/data/csv/64bit_int.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "bigint"});
- require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Integer});
-
- auto fs = all_features(ds);
- auto feature = fs->next();
- require_attributes(feature, {
- attr{"x", 0}, attr{"y", 0}, attr{"bigint", 2147483648} });
-
- feature = fs->next();
- require_attributes(feature, {
- attr{"x", 0}, attr{"y", 0}, attr{"bigint", 9223372036854775807ll} });
- require_attributes(feature, {
- attr{"x", 0}, attr{"y", 0}, attr{"bigint", 0x7FFFFFFFFFFFFFFFll} });
- } // END SECTION
-#pragma GCC diagnostic pop
-
- SECTION("various number types") {
- auto ds = get_csv_ds("test/data/csv/number_types.csv");
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {"x", "y", "floats"});
- require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Double});
- auto fs = all_features(ds);
- for (double d : { .0, +.0, 1e-06, -1e-06, 0.000001, 1.234e+16, 1.234e+16 }) {
- auto feature = fs->next();
- REQUIRE(bool(feature));
- CHECK(feature->get("floats").get<mapnik::value_double>() == Approx(d));
- }
- } // END SECTION
-
- SECTION("manually supplied extent") {
- std::string csv_string("wkt,Name\n");
- mapnik::parameters params;
- params["type"] = std::string("csv");
- params["inline"] = csv_string;
- params["extent"] = "-180,-90,180,90";
- auto ds = mapnik::datasource_cache::instance().create(params);
- REQUIRE(bool(ds));
- auto box = ds->envelope();
- CHECK(box.minx() == -180);
- CHECK(box.miny() == -90);
- CHECK(box.maxx() == 180);
- CHECK(box.maxy() == 90);
- } // END SECTION
-
- SECTION("inline geojson") {
- std::string csv_string = "geojson\n'{\"coordinates\":[-92.22568,38.59553],\"type\":\"Point\"}'";
- mapnik::parameters params;
- params["type"] = std::string("csv");
- params["inline"] = csv_string;
- params["quote"] = "'";
- auto ds = mapnik::datasource_cache::instance().create(params);
- REQUIRE(bool(ds));
-
- auto fields = ds->get_descriptor().get_descriptors();
- require_field_names(fields, {});
-
- // TODO: this originally had the following comment:
- // - re-enable after https://github.com/mapnik/mapnik/issues/2319 is fixed
- // but that seems to have been merged and tested separately?
- auto fs = all_features(ds);
- auto feat = fs->next();
- CHECK(feature_count(feat->get_geometry()) == 1);
- } // END SECTION
- mapnik::logger::instance().set_severity(severity);
- }
-} // END TEST CASE
diff --git a/test/unit/datasource/csv.cpp b/test/unit/datasource/csv.cpp
new file mode 100644
index 0000000..eee9b91
--- /dev/null
+++ b/test/unit/datasource/csv.cpp
@@ -0,0 +1,1173 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ *****************************************************************************/
+
+#include "catch.hpp"
+
+#include <mapnik/map.hpp>
+#include <mapnik/datasource.hpp>
+#include <mapnik/datasource_cache.hpp>
+#include <mapnik/geometry.hpp>
+#include <mapnik/geometry_types.hpp>
+#include <mapnik/geometry_type.hpp>
+#include <mapnik/expression.hpp>
+#include <mapnik/expression_evaluator.hpp>
+#include <mapnik/debug.hpp>
+#include <mapnik/util/fs.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/range/iterator_range_core.hpp>
+#include <boost/format.hpp>
+#include <boost/optional/optional_io.hpp>
+
+#include <iostream>
+
+namespace bfs = boost::filesystem;
+
+namespace {
+void add_csv_files(bfs::path dir, std::vector<bfs::path> &csv_files)
+{
+ for (auto const &entry : boost::make_iterator_range(
+ bfs::directory_iterator(dir), bfs::directory_iterator()))
+ {
+ auto path = entry.path();
+ if (path.extension().native() == ".csv")
+ {
+ csv_files.emplace_back(path);
+ }
+ }
+}
+
+mapnik::datasource_ptr get_csv_ds(std::string const &file_name, bool strict = true)
+{
+ mapnik::parameters params;
+ params["type"] = std::string("csv");
+ params["file"] = file_name;
+ params["strict"] = mapnik::value_bool(strict);
+ auto ds = mapnik::datasource_cache::instance().create(params);
+ // require a non-null pointer returned
+ REQUIRE(ds != nullptr);
+ return ds;
+}
+
+void require_field_names(std::vector<mapnik::attribute_descriptor> const &fields,
+ std::initializer_list<std::string> const &names)
+{
+ REQUIRE(fields.size() == names.size());
+ auto itr_a = fields.begin();
+ auto const end_a = fields.end();
+ auto itr_b = names.begin();
+ for (; itr_a != end_a; ++itr_a, ++itr_b)
+ {
+ CHECK(itr_a->get_name() == *itr_b);
+ }
+}
+
+void require_field_types(std::vector<mapnik::attribute_descriptor> const &fields,
+ std::initializer_list<mapnik::eAttributeType> const &types) {
+ REQUIRE(fields.size() == types.size());
+ auto itr_a = fields.begin();
+ auto const end_a = fields.end();
+ auto itr_b = types.begin();
+ for (; itr_a != end_a; ++itr_a, ++itr_b) {
+ CHECK(itr_a->get_type() == *itr_b);
+ }
+}
+
+mapnik::featureset_ptr all_features(mapnik::datasource_ptr ds) {
+ auto fields = ds->get_descriptor().get_descriptors();
+ mapnik::query query(ds->envelope());
+ for (auto const &field : fields) {
+ query.add_property_name(field.get_name());
+ }
+ return ds->features(query);
+}
+
+std::size_t count_features(mapnik::featureset_ptr features) {
+ std::size_t count = 0;
+ while (features->next()) {
+ ++count;
+ }
+ return count;
+}
+
+using attr = std::tuple<std::string, mapnik::value>;
+void require_attributes(mapnik::feature_ptr feature,
+ std::initializer_list<attr> const &attrs) {
+ REQUIRE(bool(feature));
+ for (auto const &kv : attrs) {
+ REQUIRE(feature->has_key(std::get<0>(kv)));
+ CHECK(feature->get(std::get<0>(kv)) == std::get<1>(kv));
+ }
+}
+
+namespace detail {
+struct feature_count {
+ template <typename T>
+ std::size_t operator()(T const &geom) const {
+ return mapnik::util::apply_visitor(*this, geom);
+ }
+
+ std::size_t operator()(mapnik::geometry::geometry_empty const &) const {
+ return 0;
+ }
+
+ template <typename T>
+ std::size_t operator()(mapnik::geometry::point<T> const &) const {
+ return 1;
+ }
+
+ template <typename T>
+ std::size_t operator()(mapnik::geometry::line_string<T> const &) const {
+ return 1;
+ }
+
+ template <typename T>
+ std::size_t operator()(mapnik::geometry::polygon<T> const &) const {
+ return 1;
+ }
+
+ template <typename T>
+ std::size_t operator()(mapnik::geometry::multi_point<T> const &mp) const {
+ return mp.size();
+ }
+
+ template <typename T>
+ std::size_t operator()(mapnik::geometry::multi_line_string<T> const &mls) const {
+ return mls.size();
+ }
+
+ template <typename T>
+ std::size_t operator()(mapnik::geometry::multi_polygon<T> const &mp) const {
+ return mp.size();
+ }
+
+ template <typename T>
+ std::size_t operator()(mapnik::geometry::geometry_collection<T> const &col) const {
+ std::size_t sum = 0;
+ for (auto const &geom : col) {
+ sum += operator()(geom);
+ }
+ return sum;
+ }
+};
+} // namespace detail
+
+template <typename T>
+std::size_t feature_count(mapnik::geometry::geometry<T> const &g) {
+ return detail::feature_count()(g);
+}
+
+void require_geometry(mapnik::feature_ptr feature,
+ std::size_t num_parts,
+ mapnik::geometry::geometry_types type) {
+ REQUIRE(bool(feature));
+ CHECK(mapnik::geometry::geometry_type(feature->get_geometry()) == type);
+ CHECK(feature_count(feature->get_geometry()) == num_parts);
+}
+
+int create_disk_index(std::string const& filename, bool silent = true)
+{
+ std::string cmd;
+ if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
+ {
+ cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
+ }
+ cmd += "mapnik-index " + filename;
+ if (silent)
+ {
+#ifndef _WINDOWS
+ cmd += " 2>/dev/null";
+#else
+ cmd += " 2> nul";
+#endif
+ }
+ return std::system(cmd.c_str());
+}
+
+} // anonymous namespace
+
+static const std::string csv_plugin("./plugins/input/csv.input");
+
+const bool registered = mapnik::datasource_cache::instance().register_datasources(csv_plugin);
+
+TEST_CASE("csv") {
+
+ if (mapnik::util::exists(csv_plugin))
+ {
+ REQUIRE(registered);
+ // make the tests silent since we intentially test error conditions that are noisy
+ auto const severity = mapnik::logger::instance().get_severity();
+ mapnik::logger::instance().set_severity(mapnik::logger::none);
+
+ // check the CSV datasource is loaded
+ const std::vector<std::string> plugin_names =
+ mapnik::datasource_cache::instance().plugin_names();
+ const bool have_csv_plugin =
+ std::find(plugin_names.begin(), plugin_names.end(), "csv") != plugin_names.end();
+
+ SECTION("CSV I/O errors")
+ {
+ std::string filename = "does_not_exist.csv";
+ for (auto create_index : { true, false })
+ {
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ // index wont be created
+ CHECK(!mapnik::util::exists(filename + ".index"));
+ }
+ mapnik::parameters params;
+ params["type"] = "csv";
+ params["file"] = filename;
+ REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
+ }
+ }
+
+ SECTION("broken files")
+ {
+ for (auto create_index : { false, true })
+ {
+ if (have_csv_plugin)
+ {
+ std::vector<bfs::path> broken;
+ add_csv_files("test/data/csv/fails", broken);
+ add_csv_files("test/data/csv/warns", broken);
+ broken.emplace_back("test/data/csv/fails/does_not_exist.csv");
+
+ for (auto const& path : broken)
+ {
+ bool require_fail = true;
+ if (create_index)
+ {
+ int ret = create_disk_index(path.native());
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ require_fail = (path.native() == "test/data/csv/warns/feature_id_counting.csv") ? false : true;
+ if (!require_fail)
+ {
+ REQUIRE(mapnik::util::exists(path.native() + ".index"));
+ }
+ }
+ INFO(path);
+ if (require_fail)
+ {
+ REQUIRE_THROWS(get_csv_ds(path.native()));
+ }
+ else
+ {
+ CHECK(bool(get_csv_ds(path.native())));
+ }
+ if (mapnik::util::exists(path.native() + ".index"))
+ {
+ CHECK(mapnik::util::remove(path.native() + ".index"));
+ }
+ }
+ }
+ }
+ } // END SECTION
+
+ SECTION("good files")
+ {
+ if (have_csv_plugin)
+ {
+ std::vector<bfs::path> good;
+ add_csv_files("test/data/csv", good);
+ add_csv_files("test/data/csv/warns", good);
+
+ for (auto const& path : good)
+ {
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(path.native() + ".index"))
+ {
+ boost::filesystem::remove(path.native() + ".index");
+ }
+ for (auto create_index : { false, true })
+ {
+ if (create_index)
+ {
+ int ret = create_disk_index(path.native());
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ if (path.native() != "test/data/csv/more_headers_than_column_values.csv") // mapnik-index won't create *.index for 0 features
+ {
+ CHECK(mapnik::util::exists(path.native() + ".index"));
+ }
+ }
+ auto ds = get_csv_ds(path.native(), false);
+ // require a non-null pointer returned
+ REQUIRE(bool(ds));
+ if (mapnik::util::exists(path.native() + ".index"))
+ {
+ CHECK(mapnik::util::remove(path.native() + ".index"));
+ }
+ }
+ }
+ }
+ } // END SECTION
+
+ SECTION("lon/lat detection")
+ {
+ for (auto create_index : { false, true })
+ {
+ for (auto const& lon_name : {std::string("lon"), std::string("lng")})
+ {
+ std::string filename = (boost::format("test/data/csv/%1%_lat.csv") % lon_name).str();
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {lon_name, "lat"});
+ require_field_types(fields, {mapnik::Integer, mapnik::Integer});
+
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
+
+ mapnik::query query(ds->envelope());
+ for (auto const &field : fields)
+ {
+ query.add_property_name(field.get_name());
+ }
+ auto features = ds->features(query);
+ auto feature = features->next();
+
+ require_attributes(feature, {
+ attr { lon_name, mapnik::value_integer(0) },
+ attr { "lat", mapnik::value_integer(0) }
+ });
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ }
+ } // END SECTION
+
+ SECTION("type detection")
+ {
+ for (auto create_index : { false, true })
+ {
+ std::string filename = "test/data/csv/nypd.csv";
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"Precinct", "Phone", "Address", "City", "geo_longitude", "geo_latitude", "geo_accuracy"});
+ require_field_types(fields, {mapnik::String, mapnik::String, mapnik::String, mapnik::String, mapnik::Double, mapnik::Double, mapnik::String});
+
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
+ CHECK(count_features(all_features(ds)) == 2);
+
+ auto feature = all_features(ds)->next();
+ require_attributes(feature, {
+ attr { "City", mapnik::value_unicode_string("New York, NY") }
+ , attr { "geo_accuracy", mapnik::value_unicode_string("house") }
+ , attr { "Phone", mapnik::value_unicode_string("(212) 334-0711") }
+ , attr { "Address", mapnik::value_unicode_string("19 Elizabeth Street") }
+ , attr { "Precinct", mapnik::value_unicode_string("5th Precinct") }
+ , attr { "geo_longitude", mapnik::value_integer(-70) }
+ , attr { "geo_latitude", mapnik::value_integer(40) }
+ });
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("skipping blank rows")
+ {
+ for (auto create_index : { false, true })
+ {
+ std::string filename = "test/data/csv/blank_rows.csv";
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "name"});
+ require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
+ CHECK(count_features(all_features(ds)) == 2);
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("empty rows")
+ {
+ for (auto create_index : { false, true })
+ {
+ std::string filename = "test/data/csv/empty_rows.csv";
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "text", "date", "integer", "boolean", "float", "time", "datetime", "empty_column"});
+ require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::String,
+ mapnik::Integer, mapnik::Boolean, mapnik::Double, mapnik::String, mapnik::String, mapnik::String});
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
+ CHECK(count_features(all_features(ds)) == 4);
+
+ auto featureset = all_features(ds);
+ auto feature = featureset->next();
+ require_attributes(feature, {
+ attr { "x", mapnik::value_integer(0) }
+ , attr { "empty_column", mapnik::value_unicode_string("") }
+ , attr { "text", mapnik::value_unicode_string("a b") }
+ , attr { "float", mapnik::value_double(1.0) }
+ , attr { "datetime", mapnik::value_unicode_string("1971-01-01T04:14:00") }
+ , attr { "y", mapnik::value_integer(0) }
+ , attr { "boolean", mapnik::value_bool(true) }
+ , attr { "time", mapnik::value_unicode_string("04:14:00") }
+ , attr { "date", mapnik::value_unicode_string("1971-01-01") }
+ , attr { "integer", mapnik::value_integer(40) }
+ });
+
+ while (bool(feature = featureset->next())) {
+ CHECK(feature->size() == 10);
+ CHECK(feature->get("empty_column") == mapnik::value_unicode_string(""));
+ }
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("slashes")
+ {
+ for (auto create_index : { false, true })
+ {
+ std::string filename = "test/data/csv/has_attributes_with_slashes.csv";
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "name"});
+ // NOTE: y column is integer, even though a double value is used below in the test?
+ require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
+
+ auto featureset = all_features(ds);
+ require_attributes(featureset->next(), {
+ attr{"x", 0}
+ , attr{"y", 0}
+ , attr{"name", mapnik::value_unicode_string("a/a") } });
+ require_attributes(featureset->next(), {
+ attr{"x", 1}
+ , attr{"y", 4}
+ , attr{"name", mapnik::value_unicode_string("b/b") } });
+ require_attributes(featureset->next(), {
+ attr{"x", 10}
+ , attr{"y", 2.5}
+ , attr{"name", mapnik::value_unicode_string("c/c") } });
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("wkt field")
+ {
+ for (auto create_index : { false, true })
+ {
+ std::string filename = "test/data/csv/wkt.csv";
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ using mapnik::geometry::geometry_types;
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"type"});
+ require_field_types(fields, {mapnik::String});
+
+ auto featureset = all_features(ds);
+ require_geometry(featureset->next(), 1, geometry_types::Point);
+ require_geometry(featureset->next(), 1, geometry_types::LineString);
+ require_geometry(featureset->next(), 1, geometry_types::Polygon);
+ require_geometry(featureset->next(), 1, geometry_types::Polygon);
+ require_geometry(featureset->next(), 4, geometry_types::MultiPoint);
+ require_geometry(featureset->next(), 2, geometry_types::MultiLineString);
+ require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
+ require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("handling of missing header")
+ {
+ for (auto create_index : { false, true })
+ {
+ std::string filename = "test/data/csv/missing_header.csv";
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ // TODO: does this mean 'missing_header.csv' should be in the warnings
+ // subdirectory, since it doesn't work in strict mode?
+ auto ds = get_csv_ds(filename, false);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"one", "two", "x", "y", "_4", "aftermissing"});
+ auto feature = all_features(ds)->next();
+ REQUIRE(feature);
+ REQUIRE(feature->has_key("_4"));
+ CHECK(feature->get("_4") == mapnik::value_unicode_string("missing"));
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("handling of headers that are numbers")
+ {
+ for (auto create_index : { false, true })
+ {
+ std::string filename = "test/data/csv/numbers_for_headers.csv";
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "1990", "1991", "1992"});
+ auto feature = all_features(ds)->next();
+ require_attributes(feature, {
+ attr{"x", 0}
+ , attr{"y", 0}
+ , attr{"1990", 1}
+ , attr{"1991", 2}
+ , attr{"1992", 3}
+ });
+ auto expression = mapnik::parse_expression("[1991]=2");
+ REQUIRE(bool(expression));
+ auto value = mapnik::util::apply_visitor(
+ mapnik::evaluate<mapnik::feature_impl, mapnik::value_type, mapnik::attributes>(
+ *feature, mapnik::attributes()), *expression);
+ CHECK(value == true);
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("quoted numbers")
+ {
+ using ustring = mapnik::value_unicode_string;
+ for (auto create_index : { false, true })
+ {
+ std::string filename = "test/data/csv/quoted_numbers.csv";
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "label"});
+ auto featureset = all_features(ds);
+
+ require_attributes(featureset->next(), {
+ attr{"x", 0}, attr{"y", 0}, attr{"label", ustring("0,0") } });
+ require_attributes(featureset->next(), {
+ attr{"x", 5}, attr{"y", 5}, attr{"label", ustring("5,5") } });
+ require_attributes(featureset->next(), {
+ attr{"x", 0}, attr{"y", 5}, attr{"label", ustring("0,5") } });
+ require_attributes(featureset->next(), {
+ attr{"x", 5}, attr{"y", 0}, attr{"label", ustring("5,0") } });
+ require_attributes(featureset->next(), {
+ attr{"x", 2.5}, attr{"y", 2.5}, attr{"label", ustring("2.5,2.5") } });
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("reading newlines")
+ {
+ for (auto create_index : { false, true })
+ {
+ for (auto const& platform : {std::string("windows"), std::string("mac")})
+ {
+ std::string filename = (boost::format("test/data/csv/%1%_newlines.csv") % platform).str();
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "z"});
+ require_attributes(all_features(ds)->next(), {
+ attr{"x", 1}, attr{"y", 10}, attr{"z", 9999.9999} });
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ }
+ } // END SECTION
+
+ SECTION("mixed newlines")
+ {
+ using ustring = mapnik::value_unicode_string;
+ for (auto create_index : { false, true })
+ {
+ for (auto const& filename : {
+ std::string("test/data/csv/mac_newlines_with_unix_inline.csv")
+ , std::string("test/data/csv/mac_newlines_with_unix_inline_escaped.csv")
+ , std::string("test/data/csv/windows_newlines_with_unix_inline.csv")
+ , std::string("test/data/csv/windows_newlines_with_unix_inline_escaped.csv")
+ })
+ {
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "line"});
+ require_attributes(all_features(ds)->next(), {
+ attr{"x", 0}, attr{"y", 0}
+ , attr{"line", ustring("many\n lines\n of text\n with unix newlines")} });
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ }
+ } // END SECTION
+
+ SECTION("tabs")
+ {
+ for (auto create_index : { false, true })
+ {
+ std::string filename = "test/data/csv/tabs_in_csv.csv";
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "z"});
+ require_attributes(all_features(ds)->next(), {
+ attr{"x", -122}, attr{"y", 48}, attr{"z", 0} });
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("separators")
+ {
+ using ustring = mapnik::value_unicode_string;
+ for (auto const& filename : {
+ std::string("test/data/csv/pipe_delimiters.csv")
+ , std::string("test/data/csv/semicolon_delimiters.csv")
+ })
+ {
+ for (auto create_index : { false, true })
+ {
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "z"});
+ require_attributes(all_features(ds)->next(), {
+ attr{"x", 0}, attr{"y", 0}, attr{"z", ustring("hello")} });
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ }
+ } // END SECTION
+
+ SECTION("null and bool keywords are empty strings")
+ {
+ using ustring = mapnik::value_unicode_string;
+ std::string filename = "test/data/csv/nulls_and_booleans_as_strings.csv";
+ for (auto create_index : { false, true })
+ {
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "null", "boolean"});
+ require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String, mapnik::Boolean});
+
+ auto featureset = all_features(ds);
+ require_attributes(featureset->next(), {
+ attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("null")}, attr{"boolean", true}});
+ require_attributes(featureset->next(), {
+ attr{"x", 0}, attr{"y", 0}, attr{"null", ustring("")}, attr{"boolean", false}});
+
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("nonexistent query fields throw")
+ {
+ std::string filename = "test/data/csv/lon_lat.csv";
+ for (auto create_index : { false, true })
+ {
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"lon", "lat"});
+ require_field_types(fields, {mapnik::Integer, mapnik::Integer});
+
+ mapnik::query query(ds->envelope());
+ for (auto const &field : fields)
+ {
+ query.add_property_name(field.get_name());
+ }
+ // also add an invalid one, triggering throw
+ query.add_property_name("bogus");
+ REQUIRE_THROWS(ds->features(query));
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("leading zeros mean strings")
+ {
+ using ustring = mapnik::value_unicode_string;
+ std::string filename = "test/data/csv/leading_zeros.csv";
+ for (auto create_index : { false, true })
+ {
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ auto ds = get_csv_ds(filename);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "fips"});
+ require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
+
+ auto featureset = all_features(ds);
+ require_attributes(featureset->next(), {
+ attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("001")}});
+ require_attributes(featureset->next(), {
+ attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("003")}});
+ require_attributes(featureset->next(), {
+ attr{"x", 0}, attr{"y", 0}, attr{"fips", ustring("005")}});
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ } // END SECTION
+
+ SECTION("advanced geometry detection")
+ {
+ using row = std::pair<std::string, mapnik::datasource_geometry_t>;
+ for (row r : {
+ row{"point", mapnik::datasource_geometry_t::Point}
+ , row{"poly", mapnik::datasource_geometry_t::Polygon}
+ , row{"multi_poly", mapnik::datasource_geometry_t::Polygon}
+ , row{"line", mapnik::datasource_geometry_t::LineString}
+ }) {
+ std::string file_name = (boost::format("test/data/csv/%1%_wkt.csv") % r.first).str();
+ auto ds = get_csv_ds(file_name);
+ CHECK(ds->get_geometry_type() == r.second);
+ }
+ } // END SECTION
+
+ SECTION("creation of CSV from in-memory strings")
+ {
+ using ustring = mapnik::value_unicode_string;
+
+ for (auto const &name : {std::string("Winthrop, WA"), std::string(u8"Qu\u00e9bec")}) {
+ std::string csv_string =
+ (boost::format(
+ "wkt,Name\n"
+ "\"POINT (120.15 48.47)\",\"%1%\"\n"
+ ) % name).str();
+
+ mapnik::parameters params;
+ params["type"] = std::string("csv");
+ params["inline"] = csv_string;
+ auto ds = mapnik::datasource_cache::instance().create(params);
+ REQUIRE(bool(ds));
+
+ auto feature = all_features(ds)->next();
+ REQUIRE(bool(feature));
+ REQUIRE(feature->has_key("Name"));
+ CHECK(feature->get("Name") == ustring(name.c_str()));
+ }
+ } // END SECTION
+
+ SECTION("creation of CSV from in-memory strings with bogus headers")
+ {
+ mapnik::parameters params;
+ params["type"] = std::string("csv");
+
+ // should throw
+ params["inline"] = "latitude, longtitude, Name\n" // misspellt (!)
+ "120.15,48.47,Winhrop";
+ REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
+
+ // should throw
+ params["strict"] = true;
+ params["inline"] = "latitude, longitude\n" // -- missing header
+ "120.15,48.47,Winhrop";
+ REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
+
+ // should not throw
+ params["strict"] = false;
+ params["inline"] = "latitude, longitude,Name\n"
+ "0,0,Unknown, extra bogus field\n"
+ "120.15,48.47,Winhrop\n";
+ auto ds = mapnik::datasource_cache::instance().create(params);
+ REQUIRE(bool(ds));
+ REQUIRE(ds->envelope() == mapnik::box2d<double>(48.47,120.15,48.47,120.15));
+ auto feature = all_features(ds)->next();
+ REQUIRE(bool(feature));
+ REQUIRE(feature->has_key("Name"));
+
+ // should throw
+ params["strict"] = false;
+ params["inline"] = "x, Name\n" // -- missing required *geometry* header
+ "120.15,Winhrop";
+ REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
+
+ } // END SECTION
+
+ SECTION("geojson quoting") {
+ using mapnik::geometry::geometry_types;
+
+ for (auto const &file : {
+ std::string("test/data/csv/geojson_double_quote_escape.csv")
+ , std::string("test/data/csv/geojson_single_quote.csv")
+ , std::string("test/data/csv/geojson_2x_double_quote_filebakery_style.csv")
+ }) {
+ auto ds = get_csv_ds(file);
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"type"});
+ require_field_types(fields, {mapnik::String});
+
+ auto featureset = all_features(ds);
+ require_geometry(featureset->next(), 1, geometry_types::Point);
+ require_geometry(featureset->next(), 1, geometry_types::LineString);
+ require_geometry(featureset->next(), 1, geometry_types::Polygon);
+ require_geometry(featureset->next(), 1, geometry_types::Polygon);
+ require_geometry(featureset->next(), 4, geometry_types::MultiPoint);
+ require_geometry(featureset->next(), 2, geometry_types::MultiLineString);
+ require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
+ require_geometry(featureset->next(), 2, geometry_types::MultiPolygon);
+ }
+ } // END SECTION
+
+ SECTION("fewer headers than rows throws") {
+ REQUIRE_THROWS(get_csv_ds("test/data/csv/more_column_values_than_headers.csv"));
+ } // END SECTION
+
+ SECTION("feature ID only incremented for valid rows") {
+ auto ds = get_csv_ds("test/data/csv/warns/feature_id_counting.csv", false);
+ auto fs = all_features(ds);
+
+ // first
+ auto feature = fs->next();
+ REQUIRE(bool(feature));
+ CHECK(feature->id() == 1);
+
+ // second, should have skipped bogus one
+ feature = fs->next();
+ REQUIRE(bool(feature));
+ CHECK(feature->id() == 2);
+
+ feature = fs->next();
+ CHECK(!feature);
+ } // END SECTION
+
+ SECTION("dynamically defining headers") {
+ using ustring = mapnik::value_unicode_string;
+ using row = std::pair<std::string, std::size_t>;
+
+ for (auto const &r : {
+ row{"test/data/csv/fails/needs_headers_two_lines.csv", 2},
+ row{"test/data/csv/fails/needs_headers_one_line.csv", 1},
+ row{"test/data/csv/fails/needs_headers_one_line_no_newline.csv", 1}})
+ {
+ mapnik::parameters params;
+ params["type"] = std::string("csv");
+ params["file"] = r.first;
+ params["headers"] = "x,y,name";
+ auto ds = mapnik::datasource_cache::instance().create(params);
+ REQUIRE(bool(ds));
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "name"});
+ require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::String});
+ require_attributes(all_features(ds)->next(), {
+ attr{"x", 0}, attr{"y", 0}, attr{"name", ustring("data_name")} });
+ REQUIRE(count_features(all_features(ds)) == r.second);
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
+ }
+ } // END SECTION
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wlong-long"
+ SECTION("64bit int fields work") {
+ auto ds = get_csv_ds("test/data/csv/64bit_int.csv");
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "bigint"});
+ require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Integer});
+
+ auto fs = all_features(ds);
+ auto feature = fs->next();
+ require_attributes(feature, {
+ attr{"x", 0}, attr{"y", 0}, attr{"bigint", 2147483648} });
+
+ feature = fs->next();
+ require_attributes(feature, {
+ attr{"x", 0}, attr{"y", 0}, attr{"bigint", 9223372036854775807ll} });
+ require_attributes(feature, {
+ attr{"x", 0}, attr{"y", 0}, attr{"bigint", 0x7FFFFFFFFFFFFFFFll} });
+ } // END SECTION
+#pragma GCC diagnostic pop
+
+ SECTION("various number types") {
+ auto ds = get_csv_ds("test/data/csv/number_types.csv");
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {"x", "y", "floats"});
+ require_field_types(fields, {mapnik::Integer, mapnik::Integer, mapnik::Double});
+ auto fs = all_features(ds);
+ for (double d : { .0, +.0, 1e-06, -1e-06, 0.000001, 1.234e+16, 1.234e+16 }) {
+ auto feature = fs->next();
+ REQUIRE(bool(feature));
+ CHECK(feature->get("floats").get<mapnik::value_double>() == Approx(d));
+ }
+ } // END SECTION
+
+ SECTION("manually supplied extent") {
+ std::string csv_string("wkt,Name\n");
+ mapnik::parameters params;
+ params["type"] = std::string("csv");
+ params["inline"] = csv_string;
+ params["extent"] = "-180,-90,180,90";
+ auto ds = mapnik::datasource_cache::instance().create(params);
+ REQUIRE(bool(ds));
+ auto box = ds->envelope();
+ CHECK(box.minx() == -180);
+ CHECK(box.miny() == -90);
+ CHECK(box.maxx() == 180);
+ CHECK(box.maxy() == 90);
+ } // END SECTION
+
+ SECTION("inline geojson") {
+ std::string csv_string = "geojson\n'{\"coordinates\":[-92.22568,38.59553],\"type\":\"Point\"}'";
+ mapnik::parameters params;
+ params["type"] = std::string("csv");
+ params["inline"] = csv_string;
+ params["quote"] = "'";
+ auto ds = mapnik::datasource_cache::instance().create(params);
+ REQUIRE(bool(ds));
+
+ auto fields = ds->get_descriptor().get_descriptors();
+ require_field_names(fields, {});
+
+ // TODO: this originally had the following comment:
+ // - re-enable after https://github.com/mapnik/mapnik/issues/2319 is fixed
+ // but that seems to have been merged and tested separately?
+ auto fs = all_features(ds);
+ auto feat = fs->next();
+ CHECK(feature_count(feat->get_geometry()) == 1);
+ } // END SECTION
+ mapnik::logger::instance().set_severity(severity);
+ }
+} // END TEST CASE
diff --git a/test/unit/datasource/geojson.cpp b/test/unit/datasource/geojson.cpp
index 9bcacca..d8e8e2a 100644
--- a/test/unit/datasource/geojson.cpp
+++ b/test/unit/datasource/geojson.cpp
@@ -26,97 +26,457 @@
#include <mapnik/datasource.hpp>
#include <mapnik/datasource_cache.hpp>
#include <mapnik/geometry.hpp>
+#include <mapnik/geometry_type.hpp>
#include <mapnik/util/fs.hpp>
+#include <cstdlib>
+
+#include <boost/filesystem/operations.hpp>
+#include <boost/optional/optional_io.hpp>
+
+namespace {
+
+std::pair<mapnik::datasource_ptr,mapnik::feature_ptr> fetch_first_feature(std::string const& filename, bool cache_features)
+{
+ mapnik::parameters params;
+ params["type"] = "geojson";
+ params["file"] = filename;
+ params["cache_features"] = cache_features;
+ auto ds = mapnik::datasource_cache::instance().create(params);
+ CHECK(ds->type() == mapnik::datasource::datasource_t::Vector);
+ auto fields = ds->get_descriptor().get_descriptors();
+ mapnik::query query(ds->envelope());
+ for (auto const& field : fields)
+ {
+ query.add_property_name(field.get_name());
+ }
+ auto features = ds->features(query);
+ auto feature = features->next();
+ return std::make_pair(ds,feature);
+}
+
+int create_disk_index(std::string const& filename, bool silent = true)
+{
+ std::string cmd;
+ if (std::getenv("DYLD_LIBRARY_PATH") != nullptr)
+ {
+ cmd += std::string("export DYLD_LIBRARY_PATH=") + std::getenv("DYLD_LIBRARY_PATH") + " && ";
+ }
+ cmd += "mapnik-index " + filename;
+ if (silent)
+ {
+#ifndef _WINDOWS
+ cmd += " 2>/dev/null";
+#else
+ cmd += " 2> nul";
+#endif
+ }
+ return std::system(cmd.c_str());
+}
+
+}
TEST_CASE("geojson") {
std::string geojson_plugin("./plugins/input/geojson.input");
if (mapnik::util::exists(geojson_plugin))
{
- SECTION("json feature cache-feature=\"true\"")
+ SECTION("GeoJSON I/O errors")
+ {
+ std::string filename = "does_not_exist.geojson";
+ for (auto create_index : { true, false })
+ {
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ // index wont be created
+ CHECK(!mapnik::util::exists(filename + ".index"));
+ }
+
+ for (auto cache_features : {true, false})
+ {
+ mapnik::parameters params;
+ params["type"] = "geojson";
+ params["file"] = filename;
+ params["cache_features"] = cache_features;
+ REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
+ }
+ }
+ }
+
+ SECTION("GeoJSON invalid Point")
+ {
+ for (auto cache_features : {true, false})
+ {
+ mapnik::parameters params;
+ params["type"] = "geojson";
+ params["file"] = "./test/data/json/point-invalid.json";
+ params["cache_features"] = cache_features;
+ REQUIRE_THROWS(mapnik::datasource_cache::instance().create(params));
+ }
+ }
+
+ SECTION("GeoJSON Point ")
+ {
+ for (auto cache_features : {true, false})
+ {
+ auto result = fetch_first_feature("./test/data/json/point.json", cache_features);
+ auto feature = result.second;
+ auto ds = result.first;
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
+ auto const& geometry = feature->get_geometry();
+ REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::Point);
+ auto const& pt = mapnik::util::get<mapnik::geometry::point<double> >(geometry);
+ REQUIRE(pt.x == 100);
+ REQUIRE(pt.y == 0);
+ }
+ }
+
+ SECTION("GeoJSON LineString")
+ {
+ for (auto cache_features : {true, false})
+ {
+ auto result = fetch_first_feature("./test/data/json/linestring.json", cache_features);
+ auto feature = result.second;
+ auto ds = result.first;
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::LineString);
+ auto const& geometry = feature->get_geometry();
+ REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::LineString);
+ auto const& line = mapnik::util::get<mapnik::geometry::line_string<double> >(geometry);
+ REQUIRE(line.size() == 2);
+ REQUIRE(mapnik::geometry::envelope(line) == mapnik::box2d<double>(100,0,101,1));
+
+ }
+ }
+
+ SECTION("GeoJSON Polygon")
+ {
+ for (auto cache_features : {true, false})
+ {
+ auto result = fetch_first_feature("./test/data/json/polygon.json", cache_features);
+ auto feature = result.second;
+ auto ds = result.first;
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Polygon);
+ auto const& geometry = feature->get_geometry();
+ REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::Polygon);
+ auto const& poly = mapnik::util::get<mapnik::geometry::polygon<double> >(geometry);
+ REQUIRE(poly.num_rings() == 2);
+ REQUIRE(poly.exterior_ring.size() == 5);
+ REQUIRE(poly.interior_rings.size() == 1);
+ REQUIRE(poly.interior_rings[0].size() == 5);
+ REQUIRE(mapnik::geometry::envelope(poly) == mapnik::box2d<double>(100,0,101,1));
+
+ }
+ }
+
+ SECTION("GeoJSON MultiPoint")
+ {
+ for (auto cache_features : {true, false})
+ {
+ auto result = fetch_first_feature("./test/data/json/multipoint.json", cache_features);
+ auto feature = result.second;
+ auto ds = result.first;
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
+ auto const& geometry = feature->get_geometry();
+ REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::MultiPoint);
+ auto const& multi_pt = mapnik::util::get<mapnik::geometry::multi_point<double> >(geometry);
+ REQUIRE(multi_pt.size() == 2);
+ REQUIRE(mapnik::geometry::envelope(multi_pt) == mapnik::box2d<double>(100,0,101,1));
+ }
+ }
+
+ SECTION("GeoJSON MultiLineString")
+ {
+ for (auto cache_features : {true, false})
+ {
+ auto result = fetch_first_feature("./test/data/json/multilinestring.json", cache_features);
+ auto feature = result.second;
+ auto ds = result.first;
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::LineString);
+ auto const& geometry = feature->get_geometry();
+ REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::MultiLineString);
+ auto const& multi_line = mapnik::util::get<mapnik::geometry::multi_line_string<double> >(geometry);
+ REQUIRE(multi_line.size() == 2);
+ REQUIRE(multi_line[0].size() == 2);
+ REQUIRE(multi_line[1].size() == 2);
+ REQUIRE(mapnik::geometry::envelope(multi_line) == mapnik::box2d<double>(100,0,103,3));
+
+ }
+ }
+
+ SECTION("GeoJSON MultiPolygon")
+ {
+ for (auto cache_features : {true, false})
+ {
+ auto result = fetch_first_feature("./test/data/json/multipolygon.json", cache_features);
+ auto feature = result.second;
+ auto ds = result.first;
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Polygon);
+ // test
+ auto const& geometry = feature->get_geometry();
+ REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::MultiPolygon);
+ auto const& multi_poly = mapnik::util::get<mapnik::geometry::multi_polygon<double> >(geometry);
+ REQUIRE(multi_poly.size() == 2);
+ REQUIRE(multi_poly[0].num_rings() == 1);
+ REQUIRE(multi_poly[1].num_rings() == 2);
+ REQUIRE(mapnik::geometry::envelope(multi_poly) == mapnik::box2d<double>(100,0,103,3));
+
+ }
+ }
+
+ SECTION("GeoJSON GeometryCollection")
+ {
+ std::string filename("./test/data/json/geometrycollection.json");
+ for (auto create_index : { true, false })
+ {
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ // index will not exist because this is not a featurecollection
+ CHECK(!mapnik::util::exists(filename + ".index"));
+ }
+
+ for (auto cache_features : {true, false})
+ {
+ auto result = fetch_first_feature(filename, cache_features);
+ auto feature = result.second;
+ auto ds = result.first;
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Collection);
+ // test
+ auto const& geometry = feature->get_geometry();
+ REQUIRE(mapnik::geometry::geometry_type(geometry) == mapnik::geometry::GeometryCollection);
+ auto const& collection = mapnik::util::get<mapnik::geometry::geometry_collection<double> >(geometry);
+ REQUIRE(collection.size() == 2);
+ REQUIRE(mapnik::geometry::geometry_type(collection[0]) == mapnik::geometry::Point);
+ REQUIRE(mapnik::geometry::geometry_type(collection[1]) == mapnik::geometry::LineString);
+ REQUIRE(mapnik::geometry::envelope(collection) == mapnik::box2d<double>(100,0,102,1));
+ }
+ }
+ }
+
+ SECTION("GeoJSON Feature")
{
// Create datasource
mapnik::parameters params;
params["type"] = "geojson";
- params["file"] = "./test/data/json/feature.json";
- params["cache-features"] = true;
- auto ds = mapnik::datasource_cache::instance().create(params);
- REQUIRE(bool(ds));
- auto fields = ds->get_descriptor().get_descriptors();
- mapnik::query query(ds->envelope());
- for (auto const &field : fields)
- {
- query.add_property_name(field.get_name());
- }
- auto features = ds->features(query);
- REQUIRE(features != nullptr);
- auto feature = features->next();
- REQUIRE(feature != nullptr);
+ std::string base("./test/data/json/");
+ std::string file("feature.json");
+ params["base"] = base;
+ params["file"] = file;
+ std::string filename = base + file;
+ for (auto create_index : { true, false })
+ {
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ // index will not exist because this is not a featurecollection
+ CHECK(!mapnik::util::exists(filename + ".index"));
+ }
+
+ for (auto cache_features : {true, false})
+ {
+ params["cache_features"] = cache_features;
+ auto ds = mapnik::datasource_cache::instance().create(params);
+ REQUIRE(bool(ds));
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
+ auto fields = ds->get_descriptor().get_descriptors();
+ mapnik::query query(ds->envelope());
+ for (auto const& field : fields)
+ {
+ query.add_property_name(field.get_name());
+ }
+ auto features = ds->features(query);
+ auto features2 = ds->features_at_point(ds->envelope().center(),0);
+ REQUIRE(features != nullptr);
+ REQUIRE(features2 != nullptr);
+ auto feature = features->next();
+ auto feature2 = features2->next();
+ REQUIRE(feature != nullptr);
+ REQUIRE(feature2 != nullptr);
+ CHECK(feature->id() == 1);
+ CHECK(feature2->id() == 1);
+ mapnik::value val = feature->get("name");
+ CHECK(val.to_string() == "Dinagat Islands");
+ mapnik::value val2 = feature2->get("name");
+ CHECK(val2.to_string() == "Dinagat Islands");
+ REQUIRE(features->next() == nullptr);
+ REQUIRE(features2->next() == nullptr);
+ }
+ }
}
- SECTION("json feature cache-feature=\"false\"")
+ SECTION("GeoJSON FeatureCollection")
{
- mapnik::parameters params;
- params["type"] = "geojson";
- params["file"] = "./test/data/json/feature.json";
- params["cache-features"] = false;
- auto ds = mapnik::datasource_cache::instance().create(params);
- REQUIRE(bool(ds));
- auto fields = ds->get_descriptor().get_descriptors();
- mapnik::query query(ds->envelope());
- for (auto const &field : fields)
- {
- query.add_property_name(field.get_name());
- }
- auto features = ds->features(query);
- REQUIRE(features != nullptr);
- auto feature = features->next();
- REQUIRE(feature != nullptr);
+ std::string filename("./test/data/json/featurecollection.json");
+
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+
+ for (auto create_index : { true, false })
+ {
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+
+ mapnik::parameters params;
+ params["type"] = "geojson";
+ params["file"] = filename;
+
+ for (auto cache_features : {true, false})
+ {
+ params["cache_features"] = cache_features;
+
+ auto ds = mapnik::datasource_cache::instance().create(params);
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Collection);
+ auto fields = ds->get_descriptor().get_descriptors();
+ mapnik::query query(ds->envelope());
+ for (auto const& field : fields)
+ {
+ query.add_property_name(field.get_name());
+ }
+ auto features = ds->features(query);
+ auto features2 = ds->features_at_point(ds->envelope().center(),10);
+ auto bounding_box = ds->envelope();
+ mapnik::box2d<double> bbox;
+ mapnik::value_integer count = 0;
+ while (true)
+ {
+ auto feature = features->next();
+ auto feature2 = features2->next();
+ if (!feature || !feature2) break;
+ if (!bbox.valid()) bbox = feature->envelope();
+ else bbox.expand_to_include(feature->envelope());
+ ++count;
+ REQUIRE(feature->id() == count);
+ REQUIRE(feature2->id() == count);
+ }
+ REQUIRE(count == 3);
+ REQUIRE(bounding_box == bbox);
+ }
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ CHECK(mapnik::util::remove(filename + ".index"));
+ }
+ }
}
- SECTION("json extra properties cache-feature=\"true\"")
+ SECTION("GeoJSON extra properties")
{
// Create datasource
mapnik::parameters params;
params["type"] = "geojson";
- params["file"] = "./test/data/json/feature_collection_extra_properties.json";
- params["cache-features"] = true;
- auto ds = mapnik::datasource_cache::instance().create(params);
- REQUIRE(bool(ds));
- auto fields = ds->get_descriptor().get_descriptors();
- mapnik::query query(ds->envelope());
- for (auto const &field : fields)
- {
- query.add_property_name(field.get_name());
- }
- auto features = ds->features(query);
- REQUIRE(features != nullptr);
- auto feature = features->next();
- REQUIRE(feature != nullptr);
- REQUIRE(feature->envelope() == mapnik::box2d<double>(123,456,123,456));
+ std::string filename("./test/data/json/feature_collection_extra_properties.json");
+ params["file"] = filename;
+
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+
+ for (auto create_index : { true, false })
+ {
+ if (create_index)
+ {
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+
+ for (auto cache_features : {true, false})
+ {
+ params["cache_features"] = cache_features;
+ auto ds = mapnik::datasource_cache::instance().create(params);
+ CHECK(ds->get_geometry_type() == mapnik::datasource_geometry_t::Point);
+ REQUIRE(bool(ds));
+ auto fields = ds->get_descriptor().get_descriptors();
+ mapnik::query query(ds->envelope());
+ for (auto const& field : fields)
+ {
+ query.add_property_name(field.get_name());
+ }
+ auto features = ds->features(query);
+ REQUIRE(features != nullptr);
+ auto feature = features->next();
+ REQUIRE(feature != nullptr);
+ REQUIRE(feature->envelope() == mapnik::box2d<double>(123,456,123,456));
+ }
+
+ // cleanup
+ if (create_index && mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+
+ }
}
- SECTION("json extra properties cache-feature=\"false\"")
+ SECTION("GeoJSON ensure input fully consumed and throw exception otherwise")
{
- // Create datasource
mapnik::parameters params;
params["type"] = "geojson";
- params["file"] = "./test/data/json/feature_collection_extra_properties.json";
- params["cache-features"] = false;
- auto ds = mapnik::datasource_cache::instance().create(params);
- REQUIRE(bool(ds));
- auto fields = ds->get_descriptor().get_descriptors();
- mapnik::query query(ds->envelope());
- for (auto const &field : fields)
- {
- query.add_property_name(field.get_name());
- }
- auto features = ds->features(query);
- REQUIRE(features != nullptr);
- auto feature = features->next();
- REQUIRE(feature != nullptr);
- REQUIRE(feature->envelope() == mapnik::box2d<double>(123,456,123,456));
- }
+ std::string filename("./test/data/json/points-malformed.geojson");
+ params["file"] = filename; // mismatched parentheses
+
+ // cleanup in the case of a failed previous run
+ if (mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+
+ for (auto create_index : { true, false })
+ {
+ if (create_index)
+ {
+ CHECK(!mapnik::util::exists(filename + ".index"));
+ int ret = create_disk_index(filename);
+ int ret_posix = (ret >> 8) & 0x000000ff;
+ INFO(ret);
+ INFO(ret_posix);
+ CHECK(mapnik::util::exists(filename + ".index"));
+ }
+ for (auto cache_features : {true, false})
+ {
+ // unfortunately when using an index or not
+ // caching features we use the bbox grammar
+ // which is not strict (and would be a perf hit if it were strict).
+ // So this is one known hole where invalid data may silently parse okay
+ // refs https://github.com/mapnik/mapnik/issues/3125
+ if (!create_index && cache_features == true)
+ {
+ std::stringstream msg;
+ msg << "testcase: create index " << create_index << " cache_features " << cache_features;
+ params["cache_features"] = cache_features;
+ INFO(msg.str());
+ CHECK_THROWS(mapnik::datasource_cache::instance().create(params));
+ }
+ }
+
+ // cleanup
+ if (create_index && mapnik::util::exists(filename + ".index"))
+ {
+ boost::filesystem::remove(filename + ".index");
+ }
+ }
+ }
}
}
diff --git a/utils/mapnik-index/build.py b/utils/mapnik-index/build.py
index 3358744..39bc00e 100644
--- a/utils/mapnik-index/build.py
+++ b/utils/mapnik-index/build.py
@@ -41,7 +41,6 @@ headers = env['CPPPATH']
boost_program_options = 'boost_program_options%s' % env['BOOST_APPEND']
boost_system = 'boost_system%s' % env['BOOST_APPEND']
libraries = [env['MAPNIK_NAME'], boost_program_options, boost_system]
-libraries.append(env['ICU_LIB_NAME'])
libraries.append('mapnik-json')
libraries.append('mapnik-wkt')
diff --git a/utils/mapnik-index/mapnik-index.cpp b/utils/mapnik-index/mapnik-index.cpp
index c3547d3..bceaa64 100644
--- a/utils/mapnik-index/mapnik-index.cpp
+++ b/utils/mapnik-index/mapnik-index.cpp
@@ -133,22 +133,36 @@ int main (int argc, char** argv)
return EXIT_FAILURE;
}
- std::clog << "max tree depth:" << depth << std::endl;
- std::clog << "split ratio:" << ratio << std::endl;
+ std::vector<std::string> files_to_process;
+
+ for (auto const& filename : files)
+ {
+ if (!mapnik::util::exists(filename))
+ {
+ continue;
+ }
- if (files.size() == 0)
+ if (mapnik::detail::is_csv(filename) || mapnik::detail::is_geojson(filename))
+ {
+ files_to_process.push_back(filename);
+ }
+ }
+
+ if (files_to_process.size() == 0)
{
std::clog << "no files to index" << std::endl;
return EXIT_FAILURE;
}
+ std::clog << "max tree depth:" << depth << std::endl;
+ std::clog << "split ratio:" << ratio << std::endl;
+
using box_type = mapnik::box2d<double>;
using item_type = std::pair<box_type, std::pair<std::size_t, std::size_t>>;
- for (auto const& filename : files)
+ for (auto const& filename : files_to_process)
{
- std::clog << "processing " << filename << std::endl;
- if (!mapnik::util::exists (filename))
+ if (!mapnik::util::exists(filename))
{
std::clog << "Error : file " << filename << " does not exist" << std::endl;
continue;
@@ -158,12 +172,14 @@ int main (int argc, char** argv)
mapnik::box2d<double> extent;
if (mapnik::detail::is_csv(filename))
{
+ std::clog << "processing '" << filename << "' as CSV\n";
auto result = mapnik::detail::process_csv_file(boxes, filename, manual_headers, separator, quote);
if (!result.first) continue;
extent = result.second;
}
else if (mapnik::detail::is_geojson(filename))
{
+ std::clog << "processing '" << filename << "' as GeoJSON\n";
auto result = mapnik::detail::process_geojson_file(boxes, filename);
if (!result.first) continue;
extent = result.second;
@@ -189,7 +205,7 @@ int main (int argc, char** argv)
{
tree.trim();
std::clog << "number nodes=" << tree.count() << std::endl;
- //tree.print();
+ std::clog << "number element=" << tree.count_items() << std::endl;
file.exceptions(std::ios::failbit | std::ios::badbit);
tree.write(file);
file.flush();
diff --git a/utils/mapnik-index/process_csv_file.cpp b/utils/mapnik-index/process_csv_file.cpp
index 3b4344a..c7b18a0 100644
--- a/utils/mapnik-index/process_csv_file.cpp
+++ b/utils/mapnik-index/process_csv_file.cpp
@@ -23,7 +23,9 @@
#include "process_csv_file.hpp"
#include "../../plugins/input/csv/csv_utils.hpp"
#include <mapnik/geometry_envelope.hpp>
+#include <mapnik/util/utf_conv_win.hpp>
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
#pragma GCC diagnostic ignored "-Wsign-conversion"
@@ -31,15 +33,19 @@
#include <boost/interprocess/streams/bufferstream.hpp>
#pragma GCC diagnostic pop
#include <mapnik/mapped_memory_cache.hpp>
+#endif
+
+#include <fstream>
namespace mapnik { namespace detail {
template <typename T>
std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& filename, std::string const& manual_headers, char separator, char quote)
{
+ mapnik::box2d<double> extent;
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
using file_source_type = boost::interprocess::ibufferstream;
file_source_type csv_file;
- mapnik::box2d<double> extent;
mapnik::mapped_region_ptr mapped_region;
boost::optional<mapnik::mapped_region_ptr> memory =
mapnik::mapped_memory_cache::instance().find(filename, true);
@@ -53,6 +59,18 @@ std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& fil
std::clog << "Error : cannot mmap " << filename << std::endl;
return std::make_pair(false, extent);
}
+#else
+ #if defined(_WINDOWS)
+ std::ifstream csv_file(mapnik::utf8_to_utf16(filename),std::ios_base::in | std::ios_base::binary);
+ #else
+ std::ifstream csv_file(filename.c_str(),std::ios_base::in | std::ios_base::binary);
+ #endif
+ if (!csv_file.is_open())
+ {
+ std::clog << "Error : cannot open " << filename << std::endl;
+ return std::make_pair(false, extent);
+ }
+#endif
auto file_length = ::detail::file_length(csv_file);
// set back to start
csv_file.seekg(0, std::ios::beg);
@@ -68,7 +86,7 @@ std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& fil
csv_utils::getline_csv(csv_file, csv_line, newline, quote);
if (separator == 0) separator = ::detail::detect_separator(csv_line);
csv_file.seekg(0, std::ios::beg);
- int line_number = 1;
+ int line_number = 0;
::detail::geometry_column_locator locator;
std::vector<std::string> headers;
std::clog << "Parsing CSV using SEPARATOR=" << separator << " QUOTE=" << quote << std::endl;
@@ -96,6 +114,7 @@ std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& fil
std::size_t index = 0;
for (auto & header : headers)
{
+ mapnik::util::trim(header);
if (header.empty())
{
// create a placeholder for the empty header
@@ -123,14 +142,17 @@ std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& fil
}
}
- if (locator.type == ::detail::geometry_column_locator::UNKNOWN)
+ std::size_t num_headers = headers.size();
+ if (!::detail::valid(locator, num_headers))
{
- std::clog << "CSV index: could not detect column headers with the name of wkt, geojson, x/y, or "
- << "latitude/longitude - this is required for reading geometry data" << std::endl;
+ std::clog << "CSV index: could not detect column(s) with the name(s) of wkt, geojson, x/y, or "
+ << "latitude/longitude in:\n"
+ << csv_line
+ << "\n - this is required for reading geometry data"
+ << std::endl;
return std::make_pair(false, extent);
}
- std::size_t num_headers = headers.size();
auto pos = csv_file.tellg();
// handle rare case of a single line of data and user-provided headers
@@ -145,9 +167,9 @@ std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& fil
is_first_row = true;
}
}
-
while (is_first_row || csv_utils::getline_csv(csv_file, csv_line, newline, quote))
{
+ ++line_number;
auto record_offset = pos;
auto record_size = csv_line.length();
pos = csv_file.tellg();
@@ -169,12 +191,12 @@ std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& fil
unsigned num_fields = values.size();
if (num_fields > num_headers || num_fields < num_headers)
{
+ // skip this row
std::ostringstream s;
s << "CSV Index: # of columns("
<< num_fields << ") > # of headers("
- << num_headers << ") parsed for row " << line_number << "\n";
- std::clog << s.str() << std::endl;
- return std::make_pair(false, extent);
+ << num_headers << ") parsed for row " << line_number;
+ throw mapnik::datasource_exception(s.str());
}
auto geom = ::detail::extract_geometry(values, locator);
@@ -191,9 +213,13 @@ std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& fil
s << "CSV Index: expected geometry column: could not parse row "
<< line_number << " "
<< values[locator.index] << "'";
- std::clog << s.str() << std::endl;;
+ throw mapnik::datasource_exception(s.str());
}
}
+ catch (mapnik::datasource_exception const& ex )
+ {
+ std::clog << ex.what() << " at line: " << line_number << std::endl;
+ }
catch (std::exception const& ex)
{
std::ostringstream s;
@@ -201,7 +227,6 @@ std::pair<bool,box2d<double>> process_csv_file(T & boxes, std::string const& fil
<< " - found " << headers.size() << " with values like: " << csv_line << "\n"
<< " and got error like: " << ex.what();
std::clog << s.str() << std::endl;
- return std::make_pair(false, extent);
}
}
return std::make_pair(true, extent);;
diff --git a/utils/mapnik-index/process_geojson_file.cpp b/utils/mapnik-index/process_geojson_file.cpp
index ed9adc3..687edac 100644
--- a/utils/mapnik-index/process_geojson_file.cpp
+++ b/utils/mapnik-index/process_geojson_file.cpp
@@ -24,6 +24,10 @@
#include <mapnik/geometry.hpp>
#include <mapnik/geometry_envelope.hpp>
#include <mapnik/geometry_adapters.hpp>
+#include <mapnik/util/file_io.hpp>
+#include <mapnik/util/utf_conv_win.hpp>
+
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wshadow"
#pragma GCC diagnostic ignored "-Wsign-compare"
@@ -33,6 +37,8 @@
#include <boost/spirit/include/qi.hpp>
#pragma GCC diagnostic pop
#include <mapnik/mapped_memory_cache.hpp>
+#endif
+
#include <mapnik/json/positions_grammar.hpp>
#include <mapnik/json/extract_bounding_box_grammar_impl.hpp>
@@ -47,12 +53,13 @@ template <typename T>
std::pair<bool,box2d<double>> process_geojson_file(T & boxes, std::string const& filename)
{
mapnik::box2d<double> extent;
+#if defined(MAPNIK_MEMORY_MAPPED_FILE)
mapnik::mapped_region_ptr mapped_region;
boost::optional<mapnik::mapped_region_ptr> memory =
mapnik::mapped_memory_cache::instance().find(filename, true);
if (!memory)
{
- std::clog << "Error : cannot mmap " << filename << std::endl;
+ std::clog << "Error : cannot memory map " << filename << std::endl;
return std::make_pair(false, extent);
}
else
@@ -61,12 +68,27 @@ std::pair<bool,box2d<double>> process_geojson_file(T & boxes, std::string const&
}
char const* start = reinterpret_cast<char const*>(mapped_region->get_address());
char const* end = start + mapped_region->get_size();
+#else
+ mapnik::util::file file(filename);
+ if (!file.open())
+ {
+ std::clog << "Error : cannot open " << filename << std::endl;
+ return std::make_pair(false, extent);
+ }
+ std::string file_buffer;
+ file_buffer.resize(file.size());
+ std::fread(&file_buffer[0], file.size(), 1, file.get());
+ char const* start = file_buffer.c_str();
+ char const* end = start + file_buffer.length();
+#endif
+
boost::spirit::standard::space_type space;
try
{
if (!boost::spirit::qi::phrase_parse(start, end, (geojson_datasource_static_bbox_grammar)(boost::phoenix::ref(boxes)) , space))
{
- std::clog << "mapnik-index (GeoJSON) : could not parse: '" << filename << "'";
+ std::clog << "mapnik-index (GeoJSON) : could extract bounding boxes from : '" << filename << "'";
+ std::clog << " expected FeatureCollection" << std::endl;
return std::make_pair(false, extent);
}
}
diff --git a/utils/shapefile/shapefile_reader.py b/utils/shapefile/shapefile_reader.py
index 27f7487..1519c67 100755
--- a/utils/shapefile/shapefile_reader.py
+++ b/utils/shapefile/shapefile_reader.py
@@ -25,6 +25,8 @@ def test_record(_type, record) :
print "NULL shape"
elif _type == 11: #PointZ
test_pointz(record)
+ elif _type == 5:
+ test_polygon(record)
def test_pointz(record):
if len(record) != 36 :
@@ -35,6 +37,17 @@ def test_pointz(record):
print>>sys.stderr,"BAD SHAPE FILE: expected PointZ or NullShape got",_type
sys.exit(1)
+def test_polygon(record):
+ _type, x0, y0, x1, y0, num_parts, num_points = struct.unpack("<iddddii", record[0:44])
+ if _type != 5:
+ print>>sys.stderr, "BAD SHAPE FILE: expected Polygon or NullShape got", _type
+ sys.exit(1)
+ length = len(record)
+ rec_length = 44 + num_parts * 4 + num_points * 16
+ if rec_length <> length:
+ print>>sys.stderr, "BAD SHAPE FILE: expected", rec_length, "got", length
+ sys.exit(1)
+
if __name__ == "__main__" :
if len(sys.argv) !=2:
@@ -52,10 +65,17 @@ if __name__ == "__main__" :
_,_,_,_,_,_,shx_file_length = header[0].unpack_from(shx.read(28))
_,_,lox,loy,hix,hiy,_,_,_,_ = header[1].unpack_from(shx.read(72))
+ shx_bbox = [lox,loy,hix,hiy]
+
# SHP header
_,_,_,_,_,_,shp_file_length = header[0].unpack_from(shp.read(28))
version,_type,lox,loy,hix,hiy,_,_,_,_ = header[1].unpack_from(shp.read(72))
+ shp_bbox = [lox,loy,hix,hiy]
+ if shx_bbox <> shp_bbox :
+ print "BAD SHAPE FILE: bounding box mismatch in *.shp and *.shx", shp_bbox, shx_bbox
+ sys.exit(1)
+
print "SHX FILE_LENGTH=",shx_file_length,"bytes"
print "SHP FILE_LENGTH=",shp_file_length,"bytes"
@@ -66,10 +86,9 @@ if __name__ == "__main__" :
calc_total_size = 50
count = 0
while shx.tell() < shx_file_length * 2 :
- offset,shx_content_length = record.unpack_from(shx.read(8))
+ offset,shx_content_length = record.unpack_from(shx.read(8))
shp.seek(offset*2, os.SEEK_SET)
record_number,content_length = record_header.unpack_from(shp.read(8))
-
if shx_content_length <> content_length:
print "BAD SHAPE FILE: content_lenght mismatch in SHP and SHX",shx_content_length,content_length
sys.exit(1)
diff --git a/utils/shapeindex/shapeindex.cpp b/utils/shapeindex/shapeindex.cpp
index 6b2324f..2e03180 100644
--- a/utils/shapeindex/shapeindex.cpp
+++ b/utils/shapeindex/shapeindex.cpp
@@ -43,15 +43,11 @@ int main (int argc,char** argv)
{
using namespace mapnik;
namespace po = boost::program_options;
- using std::string;
- using std::vector;
- using std::clog;
- using std::endl;
bool verbose=false;
unsigned int depth=DEFAULT_DEPTH;
double ratio=DEFAULT_RATIO;
- vector<string> shape_files;
+ std::vector<std::string> shape_files;
try
{
@@ -62,7 +58,7 @@ int main (int argc,char** argv)
("verbose,v","verbose output")
("depth,d", po::value<unsigned int>(), "max tree depth\n(default 8)")
("ratio,r",po::value<double>(),"split ratio (default 0.55)")
- ("shape_files",po::value<vector<string> >(),"shape files to index: file1 file2 ...fileN")
+ ("shape_files",po::value<std::vector<std::string> >(),"shape files to index: file1 file2 ...fileN")
;
po::positional_options_description p;
@@ -73,13 +69,13 @@ int main (int argc,char** argv)
if (vm.count("version"))
{
- clog << "version 0.3.0" <<std::endl;
+ std::clog << "version 0.3.0" <<std::endl;
return 1;
}
if (vm.count("help"))
{
- clog << desc << endl;
+ std::clog << desc << std::endl;
return 1;
}
if (vm.count("verbose"))
@@ -97,148 +93,125 @@ int main (int argc,char** argv)
if (vm.count("shape_files"))
{
- shape_files=vm["shape_files"].as< vector<string> >();
+ shape_files=vm["shape_files"].as< std::vector<std::string> >();
}
}
catch (std::exception const& ex)
{
- clog << "Error: " << ex.what() << endl;
+ std::clog << "Error: " << ex.what() << std::endl;
return -1;
}
- clog << "max tree depth:" << depth << endl;
- clog << "split ratio:" << ratio << endl;
+ std::clog << "max tree depth:" << depth << std::endl;
+ std::clog << "split ratio:" << ratio << std::endl;
- //vector<string>::const_iterator itr = shape_files.begin();
if (shape_files.size() == 0)
{
- clog << "no shape files to index" << endl;
+ std::clog << "no shape files to index" << std::endl;
return 0;
}
for (auto const& filename : shape_files)
{
- clog << "processing " << filename << endl;
+ std::clog << "processing " << filename << std::endl;
std::string shapename (filename);
boost::algorithm::ireplace_last(shapename,".shp","");
std::string shapename_full (shapename + ".shp");
-
+ std::string shxname(shapename + ".shx");
if (! mapnik::util::exists (shapename_full))
{
- clog << "Error : file " << shapename_full << " does not exist" << endl;
+ std::clog << "Error : file " << shapename_full << " does not exist" << std::endl;
+ continue;
+ }
+ if (! mapnik::util::exists(shxname))
+ {
+ std::clog << "Error : shapefile index file (*.shx) " << shxname << " does not exist" << std::endl;
continue;
}
-
shape_file shp (shapename_full);
- if (! shp.is_open()) {
- clog << "Error : cannot open " << shapename_full << endl;
+ if (! shp.is_open())
+ {
+ std::clog << "Error : cannot open " << shapename_full << std::endl;
+ continue;
+ }
+
+ shape_file shx (shxname);
+ if (!shx.is_open())
+ {
+ std::clog << "Error : cannot open " << shxname << std::endl;
continue;
}
- int code = shp.read_xdr_integer(); //file_code == 9994
- clog << code << endl;
- shp.skip(5*4);
+ int code = shx.read_xdr_integer(); //file_code == 9994
+ std::clog << code << std::endl;
+ shx.skip(5*4);
- int file_length=shp.read_xdr_integer();
- int version=shp.read_ndr_integer();
- int shape_type=shp.read_ndr_integer();
+ int file_length=shx.read_xdr_integer();
+ int version=shx.read_ndr_integer();
+ int shape_type=shx.read_ndr_integer();
box2d<double> extent;
- shp.read_envelope(extent);
+ shx.read_envelope(extent);
- clog << "length=" << file_length << endl;
- clog << "version=" << version << endl;
- clog << "type=" << shape_type << endl;
- clog << "extent:" << extent << endl;
+ std::clog << "length=" << file_length << std::endl;
+ std::clog << "version=" << version << std::endl;
+ std::clog << "type=" << shape_type << std::endl;
+ std::clog << "extent:" << extent << std::endl;
- int pos=50;
- shp.seek(pos*2);
- mapnik::quad_tree<int> tree(extent,depth,ratio);
- int count=0;
- while (true) {
+ int pos = 50;
+ shx.seek(pos * 2);
+ mapnik::quad_tree<int> tree(extent, depth, ratio);
+ int count = 0;
- long offset=shp.pos();
- int record_number=shp.read_xdr_integer();
- int content_length=shp.read_xdr_integer();
- shape_type = shp.read_ndr_integer();
+ while (true)
+ {
+ int offset = shx.read_xdr_integer();
+ int content_length = shx.read_xdr_integer();
+ pos += 4;
box2d<double> item_ext;
- if (shape_type==shape_io::shape_null)
+ shp.seek(offset * 2);
+ int record_number = shp.read_xdr_integer();
+ if (content_length != shp.read_xdr_integer())
{
- if (pos >= file_length)
- {
- break;
- }
- else
- {
- // still need to increment pos, or the pos counter
- // won't indicate EOF until too late.
- pos+=4+content_length;
- continue;
- }
+ std::clog << "Content length mismatch for record number " << record_number << std::endl;
+ continue;
}
- else if (shape_type==shape_io::shape_point)
- {
- double x=shp.read_double();
- double y=shp.read_double();
- item_ext=box2d<double>(x,y,x,y);
- }
- else if (shape_type==shape_io::shape_pointm)
- {
- double x=shp.read_double();
- double y=shp.read_double();
- // skip m
- shp.read_double();
- item_ext=box2d<double>(x,y,x,y);
- }
- else if (shape_type==shape_io::shape_pointz)
+ shape_type = shp.read_ndr_integer();
+
+ if (shape_type==shape_io::shape_point
+ || shape_type==shape_io::shape_pointm
+ || shape_type == shape_io::shape_pointz)
{
double x=shp.read_double();
double y=shp.read_double();
- // skip z
- shp.read_double();
- // According to ESRI shapefile doc
- // A PointZ consists of a triplet of double-precision coordinates in the order X, Y, Z plus a
- // measure.
- // PointZ
- // {
- // Double X // X coordinate
- // Double Y // Y coordinate
- // Double Z // Z coordinate
- // Double M // Measure
- // }
- // But OGR creates shapefiles with M missing so we need to skip M only if present
- // NOTE: content_length is in 16-bit words
- if ( content_length == 18)
- {
- shp.read_double();
- }
item_ext=box2d<double>(x,y,x,y);
}
else
{
shp.read_envelope(item_ext);
- shp.skip(2*content_length-4*8-4);
}
- tree.insert(offset,item_ext);
+
+ tree.insert(offset * 2,item_ext);
+
if (verbose)
{
- clog << "record number " << record_number << " box=" << item_ext << endl;
+ std::clog << "record number " << record_number << " box=" << item_ext << std::endl;
}
-
- pos+=4+content_length;
++count;
-
if (pos >= file_length) break;
}
- clog << " number shapes=" << count << endl;
+ std::clog << " number shapes=" << count << std::endl;
std::fstream file((shapename+".index").c_str(),
std::ios::in | std::ios::out | std::ios::trunc | std::ios::binary);
- if (!file) {
- clog << "cannot open index file for writing file \""
- << (shapename+".index") << "\"" << endl;
- } else {
+ if (!file)
+ {
+ std::clog << "cannot open index file for writing file \""
+ << (shapename+".index") << "\"" << std::endl;
+ }
+ else
+ {
tree.trim();
std::clog << " number nodes=" << tree.count() << std::endl;
file.exceptions(std::ios::failbit | std::ios::badbit);
@@ -248,6 +221,6 @@ int main (int argc,char** argv)
}
}
- clog << "done!" << endl;
+ std::clog << "done!" << std::endl;
return 0;
}
diff --git a/utils/svg2png/svg2png.cpp b/utils/svg2png/svg2png.cpp
index 3e00663..582e5e5 100644
--- a/utils/svg2png/svg2png.cpp
+++ b/utils/svg2png/svg2png.cpp
@@ -108,7 +108,7 @@ struct main_marker_visitor
#else
s << "xdg-open " << png_name;
#endif
- int ret = system(s.str().c_str());
+ int ret = std::system(s.str().c_str());
if (ret != 0)
status = ret;
}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-grass/mapnik.git
More information about the Pkg-grass-devel
mailing list