[Git][debian-gis-team/rasterio][master] 4 commits: New upstream version 1.0.24
Bas Couwenberg
gitlab at salsa.debian.org
Thu Jun 6 05:44:45 BST 2019
Bas Couwenberg pushed to branch master at Debian GIS Project / rasterio
Commits:
83a24ce7 by Bas Couwenberg at 2019-06-06T04:11:23Z
New upstream version 1.0.24
- - - - -
67656e60 by Bas Couwenberg at 2019-06-06T04:11:27Z
Merge tag 'upstream/1.0.24'
Upstream version 1.0.24
- - - - -
f3913fe4 by Bas Couwenberg at 2019-06-06T04:14:53Z
New upstream release.
- - - - -
5925cfc7 by Bas Couwenberg at 2019-06-06T04:16:24Z
Set distribution to unstable.
- - - - -
12 changed files:
- .travis.yml
- CHANGES.txt
- debian/changelog
- rasterio/__init__.py
- rasterio/_base.pyx
- rasterio/_env.pyx
- rasterio/env.py
- rasterio/gdal.pxi
- rasterio/io.py
- tests/conftest.py
- + tests/test_open_overview_level.py
- + tests/test_open_sharing.py
Changes:
=====================================
.travis.yml
=====================================
@@ -16,11 +16,13 @@ env:
- GDALVERSION="2.1.4"
- GDALVERSION="2.2.4"
- GDALVERSION="2.3.3"
- - GDALVERSION="2.4.0"
+ - GDALVERSION="2.4.1"
+ - GDALVERSION="3.0.0"
- GDALVERSION="master"
matrix:
allow_failures:
+ - env: GDALVERSION="3.0.0"
- env: GDALVERSION="master"
addons:
=====================================
CHANGES.txt
=====================================
@@ -1,6 +1,17 @@
Changes
=======
+1.0.24 (2019-06-05)
+-------------------
+
+- By default GDAL reuses existing dataset connections when possible.
+ Multi-threaded programs must avoid this as must programs that re-open
+ datasets in a different context, such as the overview-seeking program shown
+ in issue #1504. The sharing keyword argument of rasterio.open() was supposed
+ to allow users to disable connection reuse on a case-by-case basis, but the
+ implementation was faulty and connection reuse persisted (#1701). This has
+ been fixed in PR #1704.
+
1.0.23 (2019-05-15)
-------------------
=====================================
debian/changelog
=====================================
@@ -1,8 +1,10 @@
-rasterio (1.0.23-2) UNRELEASED; urgency=medium
+rasterio (1.0.24-1) unstable; urgency=medium
+ * Team upload.
+ * New upstream release.
* Update dh_install override to use --list-missing.
- -- Bas Couwenberg <sebastic at debian.org> Mon, 27 May 2019 06:51:52 +0200
+ -- Bas Couwenberg <sebastic at debian.org> Thu, 06 Jun 2019 06:16:12 +0200
rasterio (1.0.23-1) unstable; urgency=medium
=====================================
rasterio/__init__.py
=====================================
@@ -42,7 +42,7 @@ import rasterio.path
__all__ = ['band', 'open', 'pad', 'Env']
-__version__ = "1.0.23"
+__version__ = "1.0.24"
__gdal_version__ = gdal_version()
# Rasterio attaches NullHandler to the 'rasterio' logger and its
@@ -174,7 +174,7 @@ def open(fp, mode='r', driver=None, width=None, height=None, count=None,
@contextmanager
def fp_reader(fp):
memfile = MemoryFile(fp.read())
- dataset = memfile.open()
+ dataset = memfile.open(driver=driver, sharing=sharing)
try:
yield dataset
finally:
@@ -190,7 +190,7 @@ def open(fp, mode='r', driver=None, width=None, height=None, count=None,
memfile = MemoryFile()
dataset = memfile.open(driver=driver, width=width, height=height,
count=count, crs=crs, transform=transform,
- dtype=dtype, nodata=nodata, **kwargs)
+ dtype=dtype, nodata=nodata, sharing=sharing, **kwargs)
try:
yield dataset
finally:
@@ -213,15 +213,16 @@ def open(fp, mode='r', driver=None, width=None, height=None, count=None,
# be taken over by the dataset's context manager if it is not
# None.
if mode == 'r':
- s = DatasetReader(path, driver=driver, **kwargs)
+ s = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)
elif mode == 'r+':
- s = get_writer_for_path(path)(path, mode, driver=driver, **kwargs)
+ s = get_writer_for_path(path)(path, mode, driver=driver, sharing=sharing, **kwargs)
elif mode.startswith("w"):
s = get_writer_for_driver(driver)(path, mode, driver=driver,
width=width, height=height,
count=count, crs=crs,
transform=transform,
dtype=dtype, nodata=nodata,
+ sharing=sharing,
**kwargs)
else:
raise ValueError(
=====================================
rasterio/_base.pyx
=====================================
@@ -197,6 +197,8 @@ cdef class DatasetBase(object):
cdef int flags = 0
cdef int sharing_flag = (0x20 if sharing else 0x0)
+ log.debug("Sharing flag: %r", sharing_flag)
+
self._hds = NULL
if path is not None:
=====================================
rasterio/_env.pyx
=====================================
@@ -21,6 +21,8 @@ from rasterio._base cimport _safe_osr_release
from rasterio._err import CPLE_BaseError
from rasterio._err cimport exc_wrap_ogrerr, exc_wrap_int
+from libc.stdio cimport stderr
+
level_map = {
0: 0,
@@ -400,3 +402,6 @@ cdef class GDALEnv(ConfigEnv):
result[key] = val
return result
+
+ def _dump_open_datasets(self):
+ GDALDumpOpenDatasets(stderr)
=====================================
rasterio/env.py
=====================================
@@ -245,6 +245,13 @@ class Env(object):
"""Return a mapping of registered drivers."""
return local._env.drivers()
+ def _dump_open_datasets(self):
+ """Writes descriptions of open datasets to stderr
+
+ For debugging and testing purposes.
+ """
+ return local._env._dump_open_datasets()
+
def __enter__(self):
log.debug("Entering env context: %r", self)
if local._env is None:
=====================================
rasterio/gdal.pxi
=====================================
@@ -280,6 +280,9 @@ cdef extern from "gdal.h" nogil:
CPLErr GDALSetRasterScale(GDALRasterBandH hBand, double dfNewScale)
CPLErr GDALSetRasterOffset(GDALRasterBandH hBand, double dfNewOffset)
+ int GDALDumpOpenDatasets(FILE *fp)
+
+
cdef extern from "ogr_api.h" nogil:
ctypedef void * OGRLayerH
=====================================
rasterio/io.py
=====================================
@@ -106,7 +106,7 @@ class MemoryFile(MemoryFileBase):
@ensure_env
def open(self, driver=None, width=None, height=None, count=None, crs=None,
- transform=None, dtype=None, nodata=None, **kwargs):
+ transform=None, dtype=None, nodata=None, sharing=False, **kwargs):
"""Open the file and return a Rasterio dataset object.
If data has already been written, the file is opened in 'r'
@@ -127,13 +127,13 @@ class MemoryFile(MemoryFileBase):
raise IOError("I/O operation on closed file.")
if self.exists():
log.debug("VSI path: {}".format(vsi_path.path))
- return DatasetReader(vsi_path, driver=driver, **kwargs)
+ return DatasetReader(vsi_path, driver=driver, sharing=sharing, **kwargs)
else:
writer = get_writer_for_driver(driver)
return writer(vsi_path, 'w+', driver=driver, width=width,
height=height, count=count, crs=crs,
transform=transform, dtype=dtype,
- nodata=nodata, **kwargs)
+ nodata=nodata, sharing=sharing, **kwargs)
def __enter__(self):
self._env = env_ctx_if_needed()
@@ -156,7 +156,7 @@ class ZipMemoryFile(MemoryFile):
super(ZipMemoryFile, self).__init__(file_or_bytes, ext='zip')
@ensure_env
- def open(self, path, driver=None, **kwargs):
+ def open(self, path, driver=None, sharing=False, **kwargs):
"""Open a dataset within the zipped stream.
Parameters
@@ -165,6 +165,9 @@ class ZipMemoryFile(MemoryFile):
Path to a dataset in the zip file, relative to the root of the
archive.
+ Other parameters are optional and have the same semantics as the
+ parameters of `rasterio.open()`.
+
Returns
-------
A Rasterio dataset object
@@ -173,7 +176,7 @@ class ZipMemoryFile(MemoryFile):
if self.closed:
raise IOError("I/O operation on closed file.")
- return DatasetReader(vsi_path, driver=driver, **kwargs)
+ return DatasetReader(vsi_path, driver=driver, sharing=sharing, **kwargs)
def get_writer_for_driver(driver):
=====================================
tests/conftest.py
=====================================
@@ -488,6 +488,11 @@ def path_rgb_msk_byte_tif(data_dir):
return os.path.join(data_dir, 'RGB2.byte.tif')
+ at pytest.fixture(scope='session')
+def path_cogeo_tif(data_dir):
+ return os.path.join(data_dir, 'cogeo.tif')
+
+
@pytest.fixture(scope='function')
def _path_multiband_no_colorinterp(tmpdir):
=====================================
tests/test_open_overview_level.py
=====================================
@@ -0,0 +1,21 @@
+"""Tests for fix of #1504"""
+
+import rasterio
+
+from .conftest import requires_gdal2
+
+
+ at requires_gdal2
+def test_overview_levels(path_cogeo_tif):
+ """With sharing turned off, problem noted in #1504 vanishes"""
+ olevel = 0
+ with rasterio.open(path_cogeo_tif, overview_level=olevel) as src:
+ assert src.shape == (512, 512)
+
+ olevel = 1
+ with rasterio.open(path_cogeo_tif, sharing=False, overview_level=olevel) as src:
+ assert src.shape == (256, 256)
+
+ olevel = 2
+ with rasterio.open(path_cogeo_tif, sharing=False, overview_level=olevel) as src:
+ assert src.shape == (128, 128)
=====================================
tests/test_open_sharing.py
=====================================
@@ -0,0 +1,45 @@
+"""Tests of dataset connection sharing"""
+
+import rasterio
+
+from .conftest import requires_gdal2
+
+
+ at requires_gdal2
+def test_sharing_on(capfd, path_rgb_byte_tif):
+ """Datasets are shared"""
+ with rasterio.Env() as env:
+
+ # Opens a new file.
+ with rasterio.open(path_rgb_byte_tif, sharing=False) as srcx:
+ env._dump_open_datasets()
+ captured = capfd.readouterr()
+ assert "1 N GTiff" in captured.err
+ assert "1 S GTiff" not in captured.err
+
+ # Does not open a new file.
+ with rasterio.open(path_rgb_byte_tif, sharing=True) as srcy:
+ env._dump_open_datasets()
+ captured = capfd.readouterr()
+ assert "1 N GTiff" in captured.err
+ assert "1 S GTiff" in captured.err
+
+
+ at requires_gdal2
+def test_sharing_off(capfd, path_rgb_byte_tif):
+ """Datasets are not shared"""
+ with rasterio.Env() as env:
+
+ # Opens a new file.
+ with rasterio.open(path_rgb_byte_tif, sharing=False) as srcx:
+ env._dump_open_datasets()
+ captured = capfd.readouterr()
+ assert "1 N GTiff" in captured.err
+ assert "1 S GTiff" not in captured.err
+
+ # Does not open a new file.
+ with rasterio.open(path_rgb_byte_tif, sharing=False) as srcy:
+ env._dump_open_datasets()
+ captured = capfd.readouterr()
+ assert captured.err.count("1 N GTiff") == 2
+ assert "1 S GTiff" not in captured.err
View it on GitLab: https://salsa.debian.org/debian-gis-team/rasterio/compare/aad452ca5ae0f6af2849be8a041c9a614ee4e9fd...5925cfc79abe76d7e205fdaa5b6091a5fbe4bb6d
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/rasterio/compare/aad452ca5ae0f6af2849be8a041c9a614ee4e9fd...5925cfc79abe76d7e205fdaa5b6091a5fbe4bb6d
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20190606/c4ad684c/attachment-0001.html>
More information about the Pkg-grass-devel
mailing list