[Git][debian-gis-team/cdsetool][master] 5 commits: New upstream version 0.2.13
Antonio Valentino (@antonio.valentino)
gitlab at salsa.debian.org
Tue Oct 15 15:05:48 BST 2024
Antonio Valentino pushed to branch master at Debian GIS Project / cdsetool
Commits:
949d5acf by Antonio Valentino at 2024-10-15T06:10:42+00:00
New upstream version 0.2.13
- - - - -
144a50e8 by Antonio Valentino at 2024-10-15T06:10:42+00:00
Update upstream source from tag 'upstream/0.2.13'
Update to upstream version '0.2.13'
with Debian dir c91c21bc3dd110d02fcc05e3736ff0ca9d51cab7
- - - - -
431248d3 by Antonio Valentino at 2024-10-15T06:11:22+00:00
New upstream release
- - - - -
f6218c41 by Antonio Valentino at 2024-10-15T06:18:19+00:00
Remove outdated lintian overrides
- - - - -
7851e8c4 by Antonio Valentino at 2024-10-15T06:18:47+00:00
Set distribution to unstable
- - - - -
12 changed files:
- .github/workflows/lint.yml
- .github/workflows/publish.yml
- .github/workflows/pytest.yml
- .gitignore
- README.md
- debian/changelog
- − debian/source/lintian-overrides
- pyproject.toml
- src/cdsetool/_processing.py
- src/cdsetool/download.py
- src/cdsetool/query.py
- tests/query/query_features_test.py
Changes:
=====================================
.github/workflows/lint.yml
=====================================
@@ -3,17 +3,19 @@ name: Lint
on: [push, pull_request]
jobs:
- black:
+ ruff:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout at v4
- - uses: psf/black at stable
+ - uses: chartboost/ruff-action at v1
+ with:
+ args: 'format --check --diff'
pylint:
runs-on: ubuntu-latest
strategy:
matrix:
- python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout at v4
- name: Set up Python ${{ matrix.python-version }}
@@ -32,7 +34,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout at v4
- name: Set up Python ${{ matrix.python-version }}
=====================================
.github/workflows/publish.yml
=====================================
@@ -33,7 +33,7 @@ jobs:
- name: Build package
run: python -m build
- name: Publish package
- uses: pypa/gh-action-pypi-publish at ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0
+ uses: pypa/gh-action-pypi-publish at f7600683efdcb7656dec5b29656edb7bc586e597
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}
=====================================
.github/workflows/pytest.yml
=====================================
@@ -14,7 +14,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+ python-version: ["3.9", "3.10", "3.11", "3.12"]
permissions:
pull-requests: write
contents: write
=====================================
.gitignore
=====================================
@@ -161,4 +161,5 @@ cython_debug/
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
+.vscode/
=====================================
README.md
=====================================
@@ -71,7 +71,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
Install `cdsetool` using pip:
```bash
-pip install cdsetool==0.2.12
+pip install cdsetool==0.2.13
```
## Usage
=====================================
debian/changelog
=====================================
@@ -1,8 +1,10 @@
-cdsetool (0.2.12-2) UNRELEASED; urgency=medium
+cdsetool (0.2.13-1) unstable; urgency=medium
+ * New upstream release.
* Improve lintian overrides patterns.
+ * Remove outdated lintian overrides.
- -- Antonio Valentino <antonio.valentino at tiscali.it> Sun, 04 Aug 2024 09:26:05 +0000
+ -- Antonio Valentino <antonio.valentino at tiscali.it> Tue, 15 Oct 2024 06:12:36 +0000
cdsetool (0.2.12-1) unstable; urgency=medium
=====================================
debian/source/lintian-overrides deleted
=====================================
@@ -1,2 +0,0 @@
-# False positive: "cryptography" is imported instead.
-uses-deprecated-python-stdlib crypt (deprecated in Python 3.11, removed in Python 3.13) [tests/credentials/credentials_test.py:*]
=====================================
pyproject.toml
=====================================
@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
[project]
name = "cdsetool"
-version = "0.2.12"
+version = "0.2.13"
authors = [
{ name="Jacob Vejby", email="javej at sdfi.dk" },
]
description = "Tools & CLI for interacting with CDSE product APIs"
readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
classifiers = [
"Programming Language :: Python :: 3",
"Operating System :: OS Independent",
@@ -19,16 +19,16 @@ dependencies = [
"typer >= 0.9,< 1",
"rich >= 13.6,< 14",
"requests >= 2.28.1,< 3",
- "pyjwt[crypto] ~= 2.8.0",
+ "pyjwt[crypto] >= 2.8,< 2.10",
"geopandas >= 0.13.2",
]
[project.optional-dependencies]
test = [
- "black==24.4.2",
- "pylint==3.2.6",
- "pytest==8.3.2",
+ "pylint==3.3.1",
+ "pytest==8.3.3",
"pytest-cov==5.0.0",
"requests-mock==1.12.1",
+ "ruff==0.6.6",
"pytest-mock==3.14.0",
]
=====================================
src/cdsetool/_processing.py
=====================================
@@ -52,7 +52,6 @@ def _concurrent_process(
# Continue until no more futures are queued
while futures:
-
# Wait for the first future(s) to complete
done, not_done = wait(futures, return_when=FIRST_COMPLETED)
futures = list(not_done)
=====================================
src/cdsetool/download.py
=====================================
@@ -44,7 +44,7 @@ def download_feature(
log.debug(f"Bad URL ('{url}') or title ('{title}')")
return None
- filename = title.replace(".SAFE", ".zip")
+ filename = title + ".zip"
result_path = os.path.join(path, filename)
if not options.get("overwrite_existing", False) and os.path.exists(result_path):
@@ -67,9 +67,12 @@ def download_feature(
continue
url = _follow_redirect(url, session)
name_dir_prefix = filename.replace(".zip", "____")
- with session.get(url, stream=True) as response, tempfile.TemporaryDirectory(
- prefix=name_dir_prefix, dir=temp_dir_usr
- ) as temp_dir:
+ with (
+ session.get(url, stream=True) as response,
+ tempfile.TemporaryDirectory(
+ prefix=name_dir_prefix, dir=temp_dir_usr
+ ) as temp_dir,
+ ):
if response.status_code != 200:
log.warning(f"Status code {response.status_code}, retrying..")
time.sleep(60 * (1 + (random.random() / 4)))
=====================================
src/cdsetool/query.py
=====================================
@@ -9,8 +9,13 @@ from xml.etree import ElementTree
from datetime import datetime, date
import re
import json
+from random import random
+from time import sleep
import geopandas as gpd
+from requests.exceptions import ChunkedEncodingError
+from urllib3.exceptions import ProtocolError
from cdsetool.credentials import Credentials
+from cdsetool.logger import NoopLogger
class _FeatureIterator:
@@ -48,9 +53,11 @@ class FeatureQuery:
collection: str,
search_terms: Dict[str, Any],
proxies: Union[Dict[str, str], None] = None,
+ options: Union[Dict[str, Any], None] = None,
) -> None:
self.features = []
self.proxies = proxies
+ self.log = (options or {}).get("logger") or NoopLogger()
self.next_url = _query_url(
collection, {**search_terms, "exactCount": "1"}, proxies=proxies
)
@@ -76,16 +83,29 @@ class FeatureQuery:
session = Credentials.make_session(
None, False, Credentials.RETRIES, self.proxies
)
- with session.get(self.next_url) as response:
- response.raise_for_status()
- res = response.json()
- self.features += res.get("features") or []
-
- total_results = res.get("properties", {}).get("totalResults")
- if total_results is not None:
- self.total_results = total_results
-
- self.__set_next_url(res)
+ attempts = 0
+ while attempts < 10:
+ attempts += 1
+ try:
+ with session.get(self.next_url) as response:
+ if response.status_code != 200:
+ self.log.warning(
+ f"Status code {response.status_code}, retrying.."
+ )
+ sleep(60 * (1 + (random() / 4)))
+ continue
+ res = response.json()
+ self.features += res.get("features") or []
+
+ total_results = res.get("properties", {}).get("totalResults")
+ if total_results is not None:
+ self.total_results = total_results
+
+ self.__set_next_url(res)
+ return
+ except (ChunkedEncodingError, ConnectionResetError, ProtocolError) as e:
+ self.log.warning(e)
+ continue
def __set_next_url(self, res) -> None:
links = res.get("properties", {}).get("links") or []
@@ -101,11 +121,14 @@ def query_features(
collection: str,
search_terms: Dict[str, Any],
proxies: Union[Dict[str, str], None] = None,
+ options: Union[Dict[str, Any], None] = None,
) -> FeatureQuery:
"""
Returns an iterator over the features matching the search terms
"""
- return FeatureQuery(collection, {"maxRecords": 2000, **search_terms}, proxies)
+ return FeatureQuery(
+ collection, {"maxRecords": 2000, **search_terms}, proxies, options
+ )
def shape_to_wkt(shape: str) -> str:
=====================================
tests/query/query_features_test.py
=====================================
@@ -1,6 +1,4 @@
from cdsetool.query import query_features, FeatureQuery
-import pytest
-import requests
def _mock_describe(requests_mock):
@@ -40,12 +38,6 @@ def _mock_sentinel_1(requests_mock):
requests_mock.get(url, text=file.read())
-def test_query_features(requests_mock) -> None:
- _mock_describe(requests_mock)
-
- assert type(query_features("Sentinel1", {"maxRecords": 10})) is FeatureQuery
-
-
def test_query_features_length(requests_mock) -> None:
_mock_describe(requests_mock)
_mock_sentinel_1(requests_mock)
View it on GitLab: https://salsa.debian.org/debian-gis-team/cdsetool/-/compare/feeb4c1fe4126d7a3a94c6f43e2970346a8c1105...7851e8c47a97138beec45b945d7222881704f2a9
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/cdsetool/-/compare/feeb4c1fe4126d7a3a94c6f43e2970346a8c1105...7851e8c47a97138beec45b945d7222881704f2a9
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20241015/e850ec44/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list