[Git][debian-gis-team/fiona][master] 6 commits: New upstream version 1.7.10.post1
Bas Couwenberg
gitlab at salsa.debian.org
Sat May 5 20:17:58 BST 2018
Bas Couwenberg pushed to branch master at Debian GIS Project / fiona
Commits:
2a43de82 by Johan Van de Wauw at 2017-11-01T22:00:23+01:00
New upstream version 1.7.10.post1
- - - - -
fc0384b2 by Bas Couwenberg at 2018-05-05T20:54:47+02:00
New upstream version 1.7.11
- - - - -
bd1e7b71 by Bas Couwenberg at 2018-05-05T20:54:48+02:00
Merge tag 'upstream/1.7.11'
Upstream version 1.7.11
- - - - -
bdc46556 by Bas Couwenberg at 2018-05-05T20:55:01+02:00
New upstream release.
- - - - -
87de3386 by Bas Couwenberg at 2018-05-05T21:08:45+02:00
Add upstream metadata.
- - - - -
5da5404b by Bas Couwenberg at 2018-05-05T21:08:45+02:00
Set distribution to unstable.
- - - - -
19 changed files:
- .travis.yml
- CHANGES.txt
- CREDITS.txt
- debian/changelog
- + debian/upstream/metadata
- fiona/__init__.py
- fiona/fio/load.py
- fiona/ogrext1.pyx
- fiona/ogrext2.pxd
- fiona/ogrext2.pyx
- + tests/data/gre.cpg
- + tests/data/gre.dbf
- + tests/data/gre.prj
- + tests/data/gre.shp
- + tests/data/gre.shx
- + tests/test_encoding.py
- tests/test_fio_ls.py
- tests/test_geopackage.py
- tests/test_listing.py
Changes:
=====================================
.travis.yml
=====================================
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,10 +11,9 @@ env:
- GDALINST=$HOME/gdalinstall
- GDALBUILD=$HOME/gdalbuild
matrix:
- - GDALVERSION="1.9.2"
- GDALVERSION="1.11.5"
- GDALVERSION="2.0.3"
- - GDALVERSION="2.1.1"
+ - GDALVERSION="2.1.4"
addons:
apt:
packages:
=====================================
CHANGES.txt
=====================================
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,24 @@ Changes
All issue numbers are relative to https://github.com/Toblerity/Fiona/issues.
+1.7.11 (2017-12-14)
+-------------------
+
+- The ``encoding`` keyword argument for ``fiona.open()``, which is intended
+ to allow a caller to override a data source's own and possibly erroneous
+ encoding, has not been working (#510, #512). The problem is that we weren't
+ always setting GDAL open or config options before opening the data sources.
+ This bug is resolved by a number of commits in the maint-1.7 branch and
+ the fix is demonstrated in tests/test_encoding.py.
+- An ``--encoding`` option has been added to fio-load to enable creation of
+ encoded shapefiles with an accompanying .cpg file (#499, #517).
+
+1.7.10.post1 (2017-10-30)
+-------------------------
+
+- A post-release has been made to fix a problem with macosx wheels uploaded
+ to PyPI.
+
1.7.10 (2017-10-26)
-------------------
=====================================
CREDITS.txt
=====================================
--- a/CREDITS.txt
+++ b/CREDITS.txt
@@ -33,6 +33,7 @@ Fiona is written by:
- dimlev <dimlev at gmail.com>
- wilsaj <wilson.andrew.j+github at gmail.com>
- Jesse Crocker <jesse at gaiagps.com>
+- Arne Schlüter
Fiona would not be possible without the great work of Frank Warmerdam and other
GDAL/OGR developers.
=====================================
debian/changelog
=====================================
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,13 +1,15 @@
-fiona (1.7.10-2) UNRELEASED; urgency=medium
+fiona (1.7.11-1) unstable; urgency=medium
* Team upload.
+ * New upstream release.
* Strip trailing whitespace from changelog, control & rules files.
* Update copyright-format URL to use HTTPS.
* Update Vcs-* URLs for Salsa.
* Bump Standards-Version to 4.1.4, no changes.
* Add module import tests to autopkgtest configuration.
+ * Add upstream metadata.
- -- Bas Couwenberg <sebastic at debian.org> Wed, 15 Nov 2017 21:48:42 +0100
+ -- Bas Couwenberg <sebastic at debian.org> Sat, 05 May 2018 20:58:47 +0200
fiona (1.7.10-1) unstable; urgency=medium
=====================================
debian/upstream/metadata
=====================================
--- /dev/null
+++ b/debian/upstream/metadata
@@ -0,0 +1,6 @@
+---
+Bug-Database: https://github.com/Toblerity/Fiona/issues
+Bug-Submit: https://github.com/Toblerity/Fiona/issues/new
+Name: Fiona
+Repository: https://github.com/Toblerity/Fiona.git
+Repository-Browse: https://github.com/Toblerity/Fiona
=====================================
fiona/__init__.py
=====================================
--- a/fiona/__init__.py
+++ b/fiona/__init__.py
@@ -81,7 +81,7 @@ import uuid
__all__ = ['bounds', 'listlayers', 'open', 'prop_type', 'prop_width']
-__version__ = "1.7.10"
+__version__ = "1.7.11"
__gdal_version__ = get_gdal_release_name().decode('utf-8')
log = logging.getLogger(__name__)
=====================================
fiona/fio/load.py
=====================================
--- a/fiona/fio/load.py
+++ b/fiona/fio/load.py
@@ -30,8 +30,11 @@ FIELD_TYPES_MAP_REV = dict([(v, k) for k, v in fiona.FIELD_TYPES_MAP.items()])
@click.option('--layer', metavar="INDEX|NAME", callback=options.cb_layer,
help="Load features into specified layer. Layers use "
"zero-based numbering when accessed by index.")
+ at click.option('--encoding',
+ help="Output encoding. Defaults to latin1 for "
+ "backward compatibility.")
@click.pass_context
-def load(ctx, output, driver, src_crs, dst_crs, sequence, layer):
+def load(ctx, output, driver, src_crs, dst_crs, sequence, layer, encoding):
"""Load features from JSON to a file in another format.
The input is a GeoJSON feature collection or optionally a sequence of
@@ -98,7 +101,8 @@ def load(ctx, output, driver, src_crs, dst_crs, sequence, layer):
driver=driver,
crs=dst_crs,
schema=schema,
- layer=layer) as dst:
+ layer=layer,
+ encoding=encoding) as dst:
dst.write(first)
dst.writerecords(source)
=====================================
fiona/ogrext1.pyx
=====================================
--- a/fiona/ogrext1.pyx
+++ b/fiona/ogrext1.pyx
@@ -395,7 +395,16 @@ cdef class Session:
# Presume already a UTF-8 encoded string
path_b = path
path_c = path_b
-
+
+ userencoding = collection.encoding
+ if userencoding:
+ self._fileencoding = userencoding.upper()
+ val = self._fileencoding.encode('utf-8')
+ ogrext1.CPLSetThreadLocalConfigOption('SHAPE_ENCODING', val)
+ log.debug("SHAPE_ENCODING set to %r", val)
+ else:
+ ogrext1.CPLSetThreadLocalConfigOption('SHAPE_ENCODING', "")
+
with cpl_errs:
drivers = []
if collection._driver:
@@ -422,7 +431,7 @@ cdef class Session:
"No dataset found at path '%s' using drivers: %s" % (
collection.path,
drivers or '*'))
-
+
if isinstance(collection.name, string_types):
name_b = collection.name.encode('utf-8')
name_c = name_b
@@ -437,14 +446,8 @@ cdef class Session:
if self.cogr_layer == NULL:
raise ValueError("Null layer: " + repr(collection.name))
-
- self.collection = collection
-
- userencoding = self.collection.encoding
- if userencoding:
- ogrext1.CPLSetThreadLocalConfigOption('SHAPE_ENCODING', '')
- self._fileencoding = userencoding.upper()
- else:
+
+ if not userencoding:
self._fileencoding = (
ogrext1.OGR_L_TestCapability(
self.cogr_layer, OLC_STRINGSASUTF8) and
@@ -452,6 +455,8 @@ cdef class Session:
self.get_driver() == "ESRI Shapefile" and
'ISO-8859-1') or locale.getpreferredencoding().upper()
+ self.collection = collection
+
def stop(self):
self.cogr_layer = NULL
if self.cogr_ds != NULL:
@@ -697,19 +702,27 @@ cdef class WritingSession(Session):
cdef object _schema_mapping
def start(self, collection):
- cdef void *cogr_fielddefn
- cdef void *cogr_driver
+ cdef void *cogr_fielddefn = NULL
+ cdef void *cogr_driver = NULL
cdef void *cogr_ds = NULL
cdef void *cogr_layer = NULL
cdef void *cogr_srs = NULL
cdef char **options = NULL
- self.collection = collection
- cdef char *path_c
- cdef char *driver_c
- cdef char *name_c
- cdef char *proj_c
- cdef char *fileencoding_c
+ cdef char *path_c = NULL
+ cdef char *driver_c = NULL
+ cdef char *name_c = NULL
+ cdef char *proj_c = NULL
+ cdef char *fileencoding_c = NULL
+
path = collection.path
+ self.collection = collection
+
+ userencoding = collection.encoding
+ if userencoding:
+ self._fileencoding = userencoding.upper()
+ val = self._fileencoding.encode('utf-8')
+ ogrext1.CPLSetThreadLocalConfigOption('SHAPE_ENCODING', val)
+ log.debug("SHAPE_ENCODING set to %r", val)
if collection.mode == 'a':
if os.path.exists(path):
=====================================
fiona/ogrext2.pxd
=====================================
--- a/fiona/ogrext2.pxd
+++ b/fiona/ogrext2.pxd
@@ -60,11 +60,13 @@ cdef extern from "cpl_conv.h":
void * CPLMalloc (size_t)
void CPLFree (void *ptr)
void CPLSetThreadLocalConfigOption (char *key, char *val)
+ void CPLSetConfigOption (char *key, char *val)
const char *CPLGetConfigOption (char *, char *)
cdef extern from "cpl_string.h":
char ** CSLSetNameValue (char **list, const char *name, const char *value)
+ char ** CSLAddNameValue (char **list, const char *name, const char *value)
void CSLDestroy (char **list)
char ** CSLAddString(char **list, const char *string)
=====================================
fiona/ogrext2.pyx
=====================================
--- a/fiona/ogrext2.pyx
+++ b/fiona/ogrext2.pyx
@@ -378,6 +378,7 @@ cdef class Session:
cdef void *drv = NULL
cdef void *ds = NULL
cdef char **drvs = NULL
+ cdef char **open_opts = NULL
if collection.path == '-':
path = '/vsistdin/'
@@ -390,6 +391,15 @@ cdef class Session:
path_b = path
path_c = path_b
+ userencoding = collection.encoding
+ log.debug("Userencoding: %r", userencoding)
+
+ if userencoding:
+ self._fileencoding = userencoding.upper()
+ val = self._fileencoding.encode('utf-8')
+ open_opts = ogrext2.CSLAddNameValue(open_opts, "ENCODING", val)
+ log.debug("ENCODING open option set to %r", val)
+
# TODO: eliminate this context manager in 2.0 as we have done
# in Rasterio 1.0.
with cpl_errs:
@@ -414,19 +424,22 @@ cdef class Session:
if drv != NULL:
drvs = ogrext2.CSLAddString(drvs, name_c)
+
flags = ogrext2.GDAL_OF_VECTOR | ogrext2.GDAL_OF_READONLY
try:
self.cogr_ds = ogrext2.GDALOpenEx(
- path_c, flags, <const char *const *>drvs, NULL, NULL)
+ path_c, flags, <const char *const *>drvs, open_opts, NULL)
finally:
ogrext2.CSLDestroy(drvs)
+ ogrext2.CSLDestroy(open_opts)
+ open_opts = NULL
if self.cogr_ds == NULL:
raise FionaValueError(
"No dataset found at path '%s' using drivers: %s" % (
collection.path,
drivers or '*'))
-
+
if isinstance(collection.name, string_types):
name_b = collection.name.encode('utf-8')
name_c = name_b
@@ -441,14 +454,8 @@ cdef class Session:
if self.cogr_layer == NULL:
raise ValueError("Null layer: " + repr(collection.name))
-
- self.collection = collection
-
- userencoding = self.collection.encoding
- if userencoding:
- ogrext2.CPLSetThreadLocalConfigOption('SHAPE_ENCODING', '')
- self._fileencoding = userencoding.upper()
- else:
+
+ if not userencoding:
self._fileencoding = (
ogrext2.OGR_L_TestCapability(
self.cogr_layer, OLC_STRINGSASUTF8) and
@@ -456,6 +463,8 @@ cdef class Session:
self.get_driver() == "ESRI Shapefile" and
'ISO-8859-1') or locale.getpreferredencoding().upper()
+ self.collection = collection
+
def stop(self):
self.cogr_layer = NULL
if self.cogr_ds != NULL:
@@ -709,13 +718,24 @@ cdef class WritingSession(Session):
cdef void *cogr_layer = NULL
cdef void *cogr_srs = NULL
cdef char **options = NULL
- self.collection = collection
cdef const char *path_c = NULL
cdef const char *driver_c = NULL
cdef const char *name_c = NULL
cdef const char *proj_c = NULL
cdef const char *fileencoding_c = NULL
+ cdef char **open_opts = NULL
+
path = collection.path
+ self.collection = collection
+
+ userencoding = collection.encoding
+ log.debug("Userencoding: %r", userencoding)
+
+ if userencoding:
+ self._fileencoding = userencoding.upper()
+ val = self._fileencoding.encode('utf-8')
+ open_opts = ogrext2.CSLAddNameValue(open_opts, "ENCODING", val)
+ log.debug("ENCODING open option set to %r", val)
if collection.mode == 'a':
if os.path.exists(path):
@@ -726,10 +746,11 @@ cdef class WritingSession(Session):
path_c = path_b
self.cogr_ds = ogrext2.GDALOpenEx(path_c,
ogrext2.GDAL_OF_VECTOR | ogrext2.GDAL_OF_UPDATE,
- NULL, NULL, NULL)
+ NULL, open_opts, NULL)
cogr_driver = ogrext2.GDALGetDatasetDriver(self.cogr_ds)
if cogr_driver == NULL:
+ ogrext2.CSLDestroy(open_opts)
raise ValueError("Null driver")
if isinstance(collection.name, string_types):
@@ -742,9 +763,11 @@ cdef class WritingSession(Session):
self.cogr_ds, collection.name)
if self.cogr_layer == NULL:
+ ogrext2.CSLDestroy(open_opts)
raise RuntimeError(
"Failed to get layer %s" % collection.name)
else:
+ ogrext2.CSLDestroy(open_opts)
raise OSError("No such file or directory %s" % path)
userencoding = self.collection.encoding
@@ -766,6 +789,7 @@ cdef class WritingSession(Session):
cogr_driver = ogrext2.GDALGetDriverByName(driver_c)
if cogr_driver == NULL:
+ ogrext2.CSLDestroy(open_opts)
raise ValueError("Null driver")
# Our most common use case is the creation of a new data
@@ -790,7 +814,7 @@ cdef class WritingSession(Session):
cogr_ds = ogrext2.GDALOpenEx(path_c,
ogrext2.GDAL_OF_VECTOR | ogrext2.GDAL_OF_UPDATE,
NULL,
- NULL,
+ open_opts,
NULL)
# TODO: use exc_wrap_pointer()
@@ -820,6 +844,10 @@ cdef class WritingSession(Session):
else:
pass
+ if open_opts != NULL:
+ ogrext2.CSLDestroy(open_opts)
+ open_opts = NULL
+
if cogr_ds == NULL:
raise RuntimeError("Failed to open %s" % path)
else:
=====================================
tests/data/gre.cpg
=====================================
--- /dev/null
+++ b/tests/data/gre.cpg
@@ -0,0 +1 @@
+UTF-8
=====================================
tests/data/gre.dbf
=====================================
Binary files /dev/null and b/tests/data/gre.dbf differ
=====================================
tests/data/gre.prj
=====================================
--- /dev/null
+++ b/tests/data/gre.prj
@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]
\ No newline at end of file
=====================================
tests/data/gre.shp
=====================================
Binary files /dev/null and b/tests/data/gre.shp differ
=====================================
tests/data/gre.shx
=====================================
Binary files /dev/null and b/tests/data/gre.shx differ
=====================================
tests/test_encoding.py
=====================================
--- /dev/null
+++ b/tests/test_encoding.py
@@ -0,0 +1,56 @@
+# coding=utf-8
+"""Encoding tests"""
+
+from glob import glob
+import logging
+import os
+import shutil
+import tempfile
+import unittest
+
+import pytest
+
+import fiona
+
+
+logging.basicConfig(level=logging.DEBUG)
+
+
+# Uncomment when merged to master, where we're using pytest.
+# We're still holding our noses in maint-1.7.
+# @pytest.fixture(scope='function')
+# def gre_shp_cp1252():
+# """A tempdir containing copies of gre.* files, .cpg set to cp1252
+#
+# The shapefile attributes are in fact utf-8 encoded.
+# """
+# test_files = glob(os.path.join(os.path.dirname(__file__), 'data/gre.*'))
+# tmpdir = pytest.ensuretemp('tests/data')
+# for filename in test_files:
+# shutil.copy(filename, str(tmpdir))
+# tmpdir.join('gre.cpg').write('cp1252')
+# return tmpdir
+
+class BadCodePointTest(unittest.TestCase):
+
+ def setUp(self):
+ self.tempdir = tempfile.mkdtemp(prefix='badcodepointtest')
+ test_files = glob(os.path.join(os.path.dirname(__file__), 'data/gre.*'))
+ for filename in test_files:
+ shutil.copy(filename, self.tempdir)
+ with open(os.path.join(self.tempdir, 'gre.cpg'), 'w') as cpg:
+ cpg.write('cp1252')
+ self.shapefile = os.path.join(self.tempdir, 'gre.shp')
+
+ def tearDown(self):
+ shutil.rmtree(self.tempdir)
+
+ def test_broken_encoding(self):
+ """Reading as cp1252 mis-encodes a Russian name"""
+ with fiona.open(self.shapefile) as src:
+ self.assertNotEqual(next(iter(src))['properties']['name_ru'], u'Гренада')
+
+ def test_override_encoding(self):
+ """utf-8 override succeeds"""
+ with fiona.open(self.shapefile, encoding='utf-8') as src:
+ self.assertEqual(next(iter(src))['properties']['name_ru'], u'Гренада')
=====================================
tests/test_fio_ls.py
=====================================
--- a/tests/test_fio_ls.py
+++ b/tests/test_fio_ls.py
@@ -23,7 +23,7 @@ def test_fio_ls_single_layer():
'tests/data/'])
assert result.exit_code == 0
assert len(result.output.splitlines()) == 1
- assert json.loads(result.output) == ['coutwildrnp']
+ assert json.loads(result.output) == ['coutwildrnp', 'gre']
@unittest.skipIf(FIXME_WINDOWS,
=====================================
tests/test_geopackage.py
=====================================
--- a/tests/test_geopackage.py
+++ b/tests/test_geopackage.py
@@ -33,7 +33,7 @@ class ReadingTest(unittest.TestCase):
self.assertRaises(DriverError, fiona.open, 'tests/data/coutwildrnp.gpkg', 'r', driver="GPKG")
else:
with fiona.open('tests/data/coutwildrnp.gpkg', 'r', driver="GPKG") as c:
- self.assertEquals(len(c), 48)
+ self.assertEquals(len(c), 67)
class WritingTest(unittest.TestCase):
=====================================
tests/test_listing.py
=====================================
--- a/tests/test_listing.py
+++ b/tests/test_listing.py
@@ -19,12 +19,12 @@ def test_single_file():
assert fiona.listlayers('tests/data/coutwildrnp.shp') == ['coutwildrnp']
def test_directory():
- assert fiona.listlayers('tests/data') == ['coutwildrnp']
+ assert fiona.listlayers('tests/data') == ['coutwildrnp', 'gre']
@unittest.skipIf(FIXME_WINDOWS,
reason="FIXME on Windows. ValueError raised. Please look into why this test isn't working.")
def test_directory_trailing_slash():
- assert fiona.listlayers('tests/data/') == ['coutwildrnp']
+ assert fiona.listlayers('tests/data/') == ['coutwildrnp', 'gre']
def test_zip_path():
assert fiona.listlayers('zip://tests/data/coutwildrnp.zip') == ['coutwildrnp']
View it on GitLab: https://salsa.debian.org/debian-gis-team/fiona/compare/a7dc270920890cbe0202c52b9a24e2851cea9cbb...5da5404b6893b72adf40f0c76db7d40038cf2b70
---
View it on GitLab: https://salsa.debian.org/debian-gis-team/fiona/compare/a7dc270920890cbe0202c52b9a24e2851cea9cbb...5da5404b6893b72adf40f0c76db7d40038cf2b70
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20180505/83def5d7/attachment-0001.html>
More information about the Pkg-grass-devel
mailing list