[Git][debian-gis-team/rasterio][upstream] New upstream version 1.3.4
Bas Couwenberg (@sebastic)
gitlab at salsa.debian.org
Thu Nov 17 06:01:18 GMT 2022
Bas Couwenberg pushed to branch upstream at Debian GIS Project / rasterio
Commits:
843cb230 by Bas Couwenberg at 2022-11-17T05:25:10+01:00
New upstream version 1.3.4
- - - - -
15 changed files:
- + .github/workflows/test_gdal_latest.yaml
- CHANGES.txt
- + ci/gdal-compile.sh
- rasterio/__init__.py
- rasterio/_base.pyx
- rasterio/_err.pyx
- rasterio/_filepath.pyx
- rasterio/gdal.pxi
- rasterio/session.py
- rasterio/transform.py
- rasterio/windows.py
- tests/test__env.py
- tests/test_session.py
- tests/test_warp.py
- tests/test_windows.py
Changes:
=====================================
.github/workflows/test_gdal_latest.yaml
=====================================
@@ -0,0 +1,84 @@
+name: Test GDAL Latest
+
+on:
+ push:
+ branches: [ main, 'maint-*' ]
+ schedule:
+ - cron: '0 0 * * 0'
+ pull_request: # also build on PRs touching this file
+ paths:
+ - ".github/workflows/test_gdal_latest.yaml"
+ - "ci/gdal-compile.sh"
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ test_gdal_latest:
+ name: GDAL Latest
+ runs-on: ubuntu-latest
+ container: osgeo/proj:9.1.0
+ env:
+ GDAL_DIR: ${{ github.workspace }}/gdal_install
+ GDAL_DATA: ${{ github.workspace }}/gdal_install/share/gdal
+ LD_LIBRARY_PATH: "${{ github.workspace }}/gdal_install/lib/:${LD_LIBRARY_PATH}"
+ steps:
+ - uses: actions/checkout at v3
+ - name: Update
+ run: |
+ apt-get update
+ apt-get -y install software-properties-common
+ add-apt-repository -y ppa:deadsnakes/ppa
+ apt-get update
+
+ - name: Set up Python
+ run: |
+ apt-get install -y --no-install-recommends \
+ python3.10 \
+ python3.10-dev \
+ python3.10-venv \
+ python3-pip \
+ g++
+
+ - name: Install GDAL
+ shell: bash
+ run: |
+ apt-get update
+ apt-get install -qq \
+ libcurl4-gnutls-dev \
+ libgeos-dev \
+ libjpeg-dev \
+ libnetcdf-dev \
+ libhdf4-alt-dev \
+ libhdf5-serial-dev \
+ libssl-dev \
+ libsqlite3-dev \
+ libexpat-dev \
+ libxerces-c-dev \
+ libpng-dev \
+ libopenjp2-7-dev \
+ libzstd-dev \
+ libwebp-dev \
+ cmake \
+ curl \
+ git
+ bash ci/gdal-compile.sh git
+
+ - name: Install dependencies
+ run: |
+ export PATH="${GDAL_DIR}/bin/:${PATH}"
+ python3.10 -m venv testenv
+ . testenv/bin/activate
+ python -m pip install --upgrade pip
+ python -m pip wheel -r requirements-dev.txt
+ python -m pip install -r requirements-dev.txt
+ python setup.py clean
+ python -m pip install --no-deps --force-reinstall --no-use-pep517 -e .[test]
+
+ - name: Test
+ shell: bash
+ run: |
+ export PATH="${GDAL_DIR}/bin/:${PATH}"
+ . testenv/bin/activate
+ python -m pytest -v -m "not wheel" -rxXs --cov rasterio --cov-report term-missing
=====================================
CHANGES.txt
=====================================
@@ -1,6 +1,29 @@
Changes
=======
+1.3.4 (2022-11-16)
+------------------
+
+This version is good for use with GDAL 3.6.0 as well as previous versions.
+
+Changes:
+
+- Resampling tests have been updated to account for changes in GDAL 3.6.0
+ (#2653).
+- Variables in TransformerBase.xy have been renamed for better clarity (#2609).
+- Avoid use of a fixed port for the warp test server (#2619).
+
+Bug fixes:
+
+- Use Python long for FilePath VSI plugin's file position (#2652).
+- In the case that a GDAL function returns NULL but doesn't set an error, we
+ now raise SystemError instead of silently failing (#2645).
+- Azure storage access key can be taken from the environment if not explicitly
+ provided to AzureSession (#2637).
+- Skip empty parts of PATH on Windows when adding DLL directories (#2626).
+- get_data_window is fixed for the case where the nodata value is nan (#2629).
+- Ensure that GTiff, not COG, driver is used by default for TIFFs (#2634).
+
1.3.3 (2022-10-19)
------------------
=====================================
ci/gdal-compile.sh
=====================================
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Example usage:
+# GDAL_DIR=$PWD/gdal bash gdal_compile.sh 3.6.0rc2
+set -e
+pushd .
+echo "Building GDAL ($1) from source..."
+BUILD_GDAL_DIR=gdal-${1:0:5}
+# Download PROJ
+if [[ $1 == "git" ]]; then
+ git clone https://github.com/OSGeo/GDAL.git ${BUILD_GDAL_DIR}
+else
+ curl https://download.osgeo.org/gdal/${1:0:5}/gdal-$1.tar.gz > ${BUILD_GDAL_DIR}.tar.gz
+ tar zxf ${BUILD_GDAL_DIR}.tar.gz
+ rm ${BUILD_GDAL_DIR}.tar.gz
+fi
+cd ${BUILD_GDAL_DIR}
+mkdir build
+cd build
+# build using cmake
+cmake .. \
+ -DCMAKE_INSTALL_PREFIX=$GDAL_DIR \
+ -DBUILD_SHARED_LIBS=ON \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DOGR_BUILD_OPTIONAL_DRIVERS=OFF \
+ -DBUILD_CSHARP_BINDINGS=OFF \
+ -DBUILD_PYTHON_BINDINGS=OFF \
+ -DBUILD_JAVA_BINDINGS=OFF
+cmake --build . -j$(nproc)
+cmake --install .
+# cleanup
+cd ../..
+rm -rf ${BUILD_GDAL_DIR}
+popd
=====================================
rasterio/__init__.py
=====================================
@@ -20,7 +20,7 @@ if platform.system() == "Windows":
else:
if "PATH" in os.environ:
for p in os.environ["PATH"].split(os.pathsep):
- if glob.glob(os.path.join(p, "gdal*.dll")):
+ if p and glob.glob(os.path.join(p, "gdal*.dll")):
os.add_dll_directory(p)
@@ -81,7 +81,7 @@ except ImportError:
have_vsi_plugin = False
__all__ = ['band', 'open', 'pad', 'Env', 'CRS']
-__version__ = "1.3.3"
+__version__ = "1.3.4"
__gdal_version__ = gdal_version()
__proj_version__ = ".".join([str(version) for version in get_proj_version()])
__geos_version__ = ".".join([str(version) for version in get_geos_version()])
=====================================
rasterio/_base.pyx
=====================================
@@ -148,6 +148,11 @@ def _raster_driver_extensions():
for extension in extensions.split():
driver_extensions[extension] = drivername
+
+ # ensure default driver for tif to be GTiff instead of COG
+ driver_extensions.update(
+ {'tif': 'GTiff', 'tiff': 'GTiff'}
+ )
return driver_extensions
=====================================
rasterio/_err.pyx
=====================================
@@ -10,6 +10,10 @@ import sys
log = logging.getLogger(__name__)
+_GDAL_DEBUG_DOCS = (
+ "https://rasterio.readthedocs.io/en/latest/topics/errors.html"
+ "#debugging-internal-gdal-functions"
+)
class CPLE_BaseError(Exception):
"""Base CPL error class
@@ -215,6 +219,9 @@ cdef void *exc_wrap_pointer(void *ptr) except NULL:
exc = exc_check()
if exc:
raise exc
+ raise SystemError(
+ f"Unkown GDAL Error. To debug: {_GDAL_DEBUG_DOCS}"
+ )
CPLErrorReset()
return ptr
@@ -229,5 +236,8 @@ cdef VSILFILE *exc_wrap_vsilfile(VSILFILE *vsifile) except NULL:
exc = exc_check()
if exc:
raise exc
+ raise SystemError(
+ f"Unkown GDAL Error. To debug: {_GDAL_DEBUG_DOCS}"
+ )
CPLErrorReset()
return vsifile
=====================================
rasterio/_filepath.pyx
=====================================
@@ -130,7 +130,7 @@ cdef void* filepath_open(void *pUserData, const char *pszFilename, const char *p
cdef vsi_l_offset filepath_tell(void *pFile) with gil:
cdef object file_wrapper = <object>pFile
cdef object file_obj = file_wrapper._file_obj
- cdef int pos = file_obj.tell()
+ cdef long pos = file_obj.tell()
return <vsi_l_offset>pos
=====================================
rasterio/gdal.pxi
=====================================
@@ -65,7 +65,7 @@ cdef extern from "sys/stat.h" nogil:
cdef extern from "cpl_vsi.h" nogil:
- ctypedef int vsi_l_offset
+ ctypedef unsigned long long vsi_l_offset
ctypedef FILE VSILFILE
ctypedef stat VSIStatBufL
ctypedef enum VSIRangeStatus:
=====================================
rasterio/session.py
=====================================
@@ -563,7 +563,8 @@ class AzureSession(Session):
"""
self.unsigned = bool(os.getenv("AZURE_NO_SIGN_REQUEST", azure_unsigned))
- self.storage_account = os.getenv("AZURE_STORAGE_ACCOUNT", azure_storage_account)
+ self.storage_account = azure_storage_account or os.getenv("AZURE_STORAGE_ACCOUNT")
+ self.storage_access_key = azure_storage_access_key or os.getenv("AZURE_STORAGE_ACCESS_KEY")
if azure_storage_connection_string:
self._creds = {
@@ -572,7 +573,7 @@ class AzureSession(Session):
elif not self.unsigned:
self._creds = {
"azure_storage_account": self.storage_account,
- "azure_storage_access_key": azure_storage_access_key
+ "azure_storage_access_key": self.storage_access_key
}
else:
self._creds = {
=====================================
rasterio/transform.py
=====================================
@@ -436,22 +436,22 @@ class TransformerBase():
# shift input coordinates according to offset
T = IDENTITY.translation(coff, roff)
- temp_rows = []
- temp_cols = []
+ offset_rows = []
+ offset_cols = []
try:
- for pt in zip(cols, rows):
- y, x = T * pt
- temp_rows.append(y)
- temp_cols.append(x)
+ for colrow in zip(cols, rows):
+ offset_col, offset_row = T * colrow
+ offset_rows.append(offset_row)
+ offset_cols.append(offset_col)
- new_ys, new_xs = self._transform(
- temp_rows, temp_cols, zs, transform_direction=TransformDirection.forward
+ new_xs, new_ys = self._transform(
+ offset_cols, offset_rows, zs, transform_direction=TransformDirection.forward
)
- if len(new_ys) == 1 and not AS_ARR:
- return (new_ys[0], new_xs[0])
+ if len(new_xs) == 1 and not AS_ARR:
+ return (new_xs[0], new_ys[0])
else:
- return (new_ys, new_xs)
+ return (new_xs, new_ys)
except TypeError:
raise TransformError("Invalid inputs")
=====================================
rasterio/windows.py
=====================================
@@ -160,7 +160,10 @@ def get_data_window(arr, nodata=None):
# Otherwise retrieve mask from array (if it is masked)
# Finally try returning a full window (nodata=None and nothing in arr is masked)
if nodata is not None:
- arr_mask = arr != nodata
+ if np.isnan(nodata):
+ arr_mask = ~np.isnan(arr)
+ else:
+ arr_mask = arr != nodata
elif np.ma.is_masked(arr):
arr_mask = ~np.ma.getmask(arr)
else:
@@ -180,10 +183,10 @@ def get_data_window(arr, nodata=None):
v.append((nz.min(), nz.max() + 1))
else:
v.append((0, 0))
-
+
if arr_mask.ndim == 1:
v.append((0, 0))
-
+
return Window.from_slices(*v)
=====================================
tests/test__env.py
=====================================
@@ -35,15 +35,13 @@ def mock_fhs(tmpdir):
@pytest.fixture
def mock_debian(tmpdir):
"""A fake Debian multi-install system"""
- tmpdir.ensure("share/gdal/2.3/header.dxf")
- tmpdir.ensure("share/gdal/2.4/header.dxf")
- tmpdir.ensure("share/gdal/3.0/header.dxf")
tmpdir.ensure("share/gdal/3.1/header.dxf")
tmpdir.ensure("share/gdal/3.2/header.dxf")
tmpdir.ensure("share/gdal/3.3/header.dxf")
tmpdir.ensure("share/gdal/3.4/header.dxf")
tmpdir.ensure("share/gdal/3.5/header.dxf")
tmpdir.ensure("share/gdal/3.6/header.dxf")
+ tmpdir.ensure(f"share/gdal/{gdal_version.major}.{gdal_version.minor}/header.dxf")
tmpdir.ensure("share/proj/epsg")
return tmpdir
=====================================
tests/test_session.py
=====================================
@@ -282,6 +282,16 @@ def test_session_factory_az_kwargs_connection_string():
assert sesh.get_credential_options()['AZURE_STORAGE_CONNECTION_STRING'] == 'AccountName=myaccount;AccountKey=MY_ACCOUNT_KEY'
+def test_session_factory_az_env(monkeypatch):
+ """Get an AzureSession for az:// paths with environment variables"""
+ monkeypatch.setenv('AZURE_STORAGE_ACCOUNT', 'foo')
+ monkeypatch.setenv('AZURE_STORAGE_ACCESS_KEY', 'bar')
+ sesh = Session.from_path("az://lol/wut")
+ assert isinstance(sesh, AzureSession)
+ assert sesh.get_credential_options()['AZURE_STORAGE_ACCOUNT'] == 'foo'
+ assert sesh.get_credential_options()['AZURE_STORAGE_ACCESS_KEY'] == 'bar'
+
+
def test_azure_no_sign_request(monkeypatch):
"""If AZURE_NO_SIGN_REQUEST is set do not default to azure_unsigned=False"""
monkeypatch.setenv('AZURE_NO_SIGN_REQUEST', 'YES')
=====================================
tests/test_warp.py
=====================================
@@ -1208,7 +1208,7 @@ def test_reproject_resampling(path_rgb_byte_tif, method):
Resampling.med: [437194],
Resampling.q1: [436397],
Resampling.q3: [438948],
- Resampling.sum: [439118],
+ Resampling.sum: [439118, 439142], # 439142 for GDAL 3.6+
Resampling.rms: [439385],
}
@@ -1301,7 +1301,7 @@ def test_reproject_resampling_alpha(method):
Resampling.med: [437194],
Resampling.q1: [436397],
Resampling.q3: [438948],
- Resampling.sum: [439118],
+ Resampling.sum: [439118, 439142], # 439142 for GDAL 3.6+
Resampling.rms: [439385],
}
@@ -1966,12 +1966,11 @@ def http_error_server(data):
import multiprocessing
import http.server
- PORT = 8000
Handler = functools.partial(RangeRequestErrorHandler, directory=str(data))
- httpd = http.server.HTTPServer(("", PORT), Handler)
+ httpd = http.server.HTTPServer(("", 0), Handler)
p = multiprocessing.Process(target=httpd.serve_forever)
p.start()
- yield
+ yield f'{httpd.server_name}:{httpd.server_port}'
p.terminate()
p.join()
@@ -1984,7 +1983,7 @@ def test_reproject_error_propagation(http_error_server, caplog):
"""Propagate errors up from ChunkAndWarpMulti and check for a retry."""
with rasterio.open(
- "/vsicurl?max_retry=1&retry_delay=.1&url=http://localhost:8000/RGB.byte.tif"
+ f"/vsicurl?max_retry=1&retry_delay=.1&url=http://{http_error_server}/RGB.byte.tif"
) as src:
out = np.zeros((src.count, src.height, src.width), dtype="uint8")
=====================================
tests/test_windows.py
=====================================
@@ -508,6 +508,14 @@ def test_data_window_nodata():
assert window == Window.from_slices((1, 3), (0, 3))
+def test_data_window_nodata_nan():
+ """Get window of arr with nodata."""
+ arr = np.ones((3, 3))
+ arr[0, :] = np.nan
+ window = get_data_window(arr, nodata=np.nan)
+ assert window == Window.from_slices((1, 3), (0, 3))
+
+
def test_data_window_novalid():
"""Get window of arr with nodata."""
arr = np.ones((3, 3))
View it on GitLab: https://salsa.debian.org/debian-gis-team/rasterio/-/commit/843cb230c1be4351e18e453ded83d9d374a93a90
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/rasterio/-/commit/843cb230c1be4351e18e453ded83d9d374a93a90
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20221117/a680293d/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list