[Git][debian-gis-team/rasterio][upstream] New upstream version 1.2.10
Bas Couwenberg (@sebastic)
gitlab at salsa.debian.org
Tue Oct 12 05:04:09 BST 2021
Bas Couwenberg pushed to branch upstream at Debian GIS Project / rasterio
Commits:
24e8ca53 by Bas Couwenberg at 2021-10-12T05:50:23+02:00
New upstream version 1.2.10
- - - - -
8 changed files:
- .travis.yml
- CHANGES.txt
- rasterio/__init__.py
- rasterio/_err.pyx
- rasterio/_warp.pyx
- rasterio/errors.py
- + tests/rangehttpserver.py
- tests/test_warp.py
Changes:
=====================================
.travis.yml
=====================================
@@ -85,6 +85,7 @@ install:
script:
- python -m pytest -v -m "not wheel" -rxXs --cov rasterio --cov-report term-missing
+ - CPL_DEBUG=YES GDAL_HTTP_MAX_RETRY=2 GDAL_HTTP_RETRY_DELAY=1 GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR python -m pytest tests/test_warp.py -k error_prop --log-level DEBUG -rP
after_success:
- coveralls || echo "!! intermittent coveralls failure"
=====================================
CHANGES.txt
=====================================
@@ -1,6 +1,12 @@
Changes
=======
+1.2.10 (2021-10-11)
+-------------------
+
+- Raise WarpOperationError if ChunkAndWarp* do not succeed instead of silently
+ failing to write to the warp output dataset (#2305).
+
1.2.9 (2021-10-01)
------------------
=====================================
rasterio/__init__.py
=====================================
@@ -40,7 +40,7 @@ import rasterio.enums
import rasterio.path
__all__ = ['band', 'open', 'pad', 'Env']
-__version__ = "1.2.9"
+__version__ = "1.2.10"
__gdal_version__ = gdal_version()
# Rasterio attaches NullHandler to the 'rasterio' logger and its
=====================================
rasterio/_err.pyx
=====================================
@@ -11,6 +11,7 @@ from enum import IntEnum
import logging
import sys
+log = logging.getLogger(__name__)
# Python exceptions expressing the CPL error numbers.
=====================================
rasterio/_warp.pyx
=====================================
@@ -28,7 +28,8 @@ from rasterio.crs import CRS
from rasterio.errors import (
GDALOptionNotImplementedError,
DriverRegistrationError, CRSError, RasterioIOError,
- RasterioDeprecationWarning, WarpOptionsError, WarpedVRTError)
+ RasterioDeprecationWarning, WarpOptionsError, WarpedVRTError,
+ WarpOperationError)
from rasterio.transform import Affine, from_bounds, guard_transform, tastes_like_gdal
cimport numpy as np
@@ -581,24 +582,33 @@ def _reproject(
if num_threads > 1:
with nogil:
- oWarper.ChunkAndWarpMulti(0, 0, cols, rows)
+ err = oWarper.ChunkAndWarpMulti(0, 0, cols, rows)
else:
with nogil:
- oWarper.ChunkAndWarpImage(0, 0, cols, rows)
+ err = oWarper.ChunkAndWarpImage(0, 0, cols, rows)
+
+ try:
+ exc_wrap_int(err)
+ except CPLE_BaseError as base:
+ raise WarpOperationError("Chunk and warp failed") from base
if dtypes.is_ndarray(destination):
exc_wrap_int(io_auto(destination, dst_dataset, 0))
# Clean up transformer, warp options, and dataset handles.
finally:
+
if bUseApproxTransformer:
GDALDestroyApproxTransformer(hTransformArg)
else:
GDALDestroyGenImgProjTransformer(hTransformArg)
+
GDALDestroyWarpOptions(psWOptions)
CPLFree(imgProjOptions)
+
if mem_raster is not None:
mem_raster.close()
+
if src_mem is not None:
src_mem.close()
=====================================
rasterio/errors.py
=====================================
@@ -146,3 +146,7 @@ class WarpedVRTError(RasterioError):
class DatasetIOShapeError(RasterioError):
"""Raised when data buffer shape is a mismatch when reading and writing"""
+
+
+class WarpOperationError(RasterioError):
+ """Raised when a warp operation fails."""
=====================================
tests/rangehttpserver.py
=====================================
@@ -0,0 +1,114 @@
+#!/usr/bin/python
+'''
+Use this in the same way as Python's SimpleHTTPServer:
+
+ python -m RangeHTTPServer [port]
+
+The only difference from SimpleHTTPServer is that RangeHTTPServer supports
+'Range:' headers to load portions of files. This is helpful for doing local web
+development with genomic data files, which tend to be to large to load into the
+browser all at once.
+'''
+
+import os
+import re
+
+try:
+ # Python3
+ from http.server import SimpleHTTPRequestHandler
+
+except ImportError:
+ # Python 2
+ from SimpleHTTPServer import SimpleHTTPRequestHandler
+
+
+def copy_byte_range(infile, outfile, start=None, stop=None, bufsize=16*1024):
+ '''Like shutil.copyfileobj, but only copy a range of the streams.
+
+ Both start and stop are inclusive.
+ '''
+ if start is not None: infile.seek(start)
+ while 1:
+ to_read = min(bufsize, stop + 1 - infile.tell() if stop else bufsize)
+ buf = infile.read(to_read)
+ if not buf:
+ break
+ outfile.write(buf)
+
+
+BYTE_RANGE_RE = re.compile(r'bytes=(\d+)-(\d+)?$')
+def parse_byte_range(byte_range):
+ '''Returns the two numbers in 'bytes=123-456' or throws ValueError.
+
+ The last number or both numbers may be None.
+ '''
+ if byte_range.strip() == '':
+ return None, None
+
+ m = BYTE_RANGE_RE.match(byte_range)
+ if not m:
+ raise ValueError('Invalid byte range %s' % byte_range)
+
+ first, last = [x and int(x) for x in m.groups()]
+ if last and last < first:
+ raise ValueError('Invalid byte range %s' % byte_range)
+ return first, last
+
+
+class RangeRequestHandler(SimpleHTTPRequestHandler):
+ """Adds support for HTTP 'Range' requests to SimpleHTTPRequestHandler
+
+ The approach is to:
+ - Override send_head to look for 'Range' and respond appropriately.
+ - Override copyfile to only transmit a range when requested.
+ """
+ def send_head(self):
+ if 'Range' not in self.headers:
+ self.range = None
+ return SimpleHTTPRequestHandler.send_head(self)
+ try:
+ self.range = parse_byte_range(self.headers['Range'])
+ except ValueError as e:
+ self.send_error(400, 'Invalid byte range')
+ return None
+ first, last = self.range
+
+ # Mirroring SimpleHTTPServer.py here
+ path = self.translate_path(self.path)
+ f = None
+ ctype = self.guess_type(path)
+ try:
+ f = open(path, 'rb')
+ except IOError:
+ self.send_error(404, 'File not found')
+ return None
+
+ fs = os.fstat(f.fileno())
+ file_len = fs[6]
+ if first >= file_len:
+ self.send_error(416, 'Requested Range Not Satisfiable')
+ return None
+
+ self.send_response(206)
+ self.send_header('Content-type', ctype)
+ self.send_header('Accept-Ranges', 'bytes')
+
+ if last is None or last >= file_len:
+ last = file_len - 1
+ response_length = last - first + 1
+
+ self.send_header('Content-Range',
+ 'bytes %s-%s/%s' % (first, last, file_len))
+ self.send_header('Content-Length', str(response_length))
+ self.send_header('Last-Modified', self.date_time_string(fs.st_mtime))
+ self.end_headers()
+ return f
+
+ def copyfile(self, source, outputfile):
+ if not self.range:
+ return SimpleHTTPRequestHandler.copyfile(self, source, outputfile)
+
+ # SimpleHTTPRequestHandler uses shutil.copyfileobj, which doesn't let
+ # you stop the copying before the end of the file.
+ start, stop = self.range # set in send_head()
+ copy_byte_range(source, outputfile, start, stop)
=====================================
tests/test_warp.py
=====================================
@@ -2,6 +2,7 @@
import json
import logging
+import sys
from affine import Affine
import numpy as np
@@ -17,6 +18,7 @@ from rasterio.errors import (
CRSError,
GDALVersionError,
TransformError,
+ WarpOperationError,
)
from rasterio.warp import (
reproject,
@@ -1920,3 +1922,102 @@ def test_reproject_rpcs_approx_transformer(caplog):
)
assert "Created approximate transformer" in caplog.text
+
+
+ at pytest.fixture
+def http_error_server(data):
+ """Serves files from the test data directory, poorly."""
+ import functools
+ import multiprocessing
+ import http.server
+ import os
+
+ from . import rangehttpserver
+
+ class RangeRequestErrorHandler(rangehttpserver.RangeRequestHandler):
+ """Return 500 for a range of bytes to simulate a malfunctioning server.
+
+ The byte range is specific to rasterio's RGB.byte.tif file.
+
+ """
+
+ def send_head(self):
+ if "Range" not in self.headers:
+ self.range = None
+ return super().send_head()
+ try:
+ self.range = rangehttpserver.parse_byte_range(self.headers["Range"])
+ except ValueError as e:
+ self.send_error(400, "Invalid byte range")
+ return None
+ first, last = self.range
+
+ # Our "poison byte" is at position 1609000.
+ if first <= 1609000 <= last:
+ self.send_error(500, "Boom!")
+ return None
+
+ # Mirroring SimpleHTTPServer.py here
+ path = self.translate_path(self.path)
+ f = None
+ ctype = self.guess_type(path)
+ try:
+ f = open(path, "rb")
+ except IOError:
+ self.send_error(404, "File not found")
+ return None
+
+ fs = os.fstat(f.fileno())
+ file_len = fs[6]
+ if first >= file_len:
+ self.send_error(416, "Requested Range Not Satisfiable")
+ return None
+
+ self.send_response(206)
+ self.send_header("Content-type", ctype)
+ self.send_header("Accept-Ranges", "bytes")
+
+ if last is None or last >= file_len:
+ last = file_len - 1
+ response_length = last - first + 1
+
+ self.send_header(
+ "Content-Range", "bytes %s-%s/%s" % (first, last, file_len)
+ )
+ self.send_header("Content-Length", str(response_length))
+ self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
+ self.end_headers()
+ return f
+
+ PORT = 8000
+ Handler = functools.partial(RangeRequestErrorHandler, directory=str(data))
+ httpd = http.server.HTTPServer(("", PORT), Handler)
+ p = multiprocessing.Process(target=httpd.serve_forever)
+ p.start()
+ yield
+ p.terminate()
+ p.join()
+
+
+ at requires_gdal3
+ at pytest.mark.skipif(
+ sys.version_info < (3, 7),
+ reason="Python 3.7 required to serve the data fixture directory",
+)
+def test_reproject_error_propagation(http_error_server, caplog):
+ """Propagate errors up from ChunkAndWarpMulti and check for a retry."""
+
+ with rasterio.open(
+ "/vsicurl?max_retry=1&retry_delay=.1&url=http://localhost:8000/RGB.byte.tif"
+ ) as src:
+ out = np.zeros((src.count, src.height, src.width), dtype="uint8")
+
+ with pytest.raises(WarpOperationError):
+ reproject(
+ rasterio.band(src, (1, 2, 3)),
+ out,
+ dst_crs=src.crs,
+ dst_transform=src.transform,
+ )
+
+ assert len([rec for rec in caplog.records if "Retrying again" in rec.message]) == 2
View it on GitLab: https://salsa.debian.org/debian-gis-team/rasterio/-/commit/24e8ca531060d7d4f9533be9a6cfe66e9bdbfbd7
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/rasterio/-/commit/24e8ca531060d7d4f9533be9a6cfe66e9bdbfbd7
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20211012/a7eab8cc/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list