[Git][debian-gis-team/cdsetool][upstream] New upstream version 0.2.12
Antonio Valentino (@antonio.valentino)
gitlab at salsa.debian.org
Sat Aug 3 11:07:04 BST 2024
Antonio Valentino pushed to branch upstream at Debian GIS Project / cdsetool
Commits:
914397e7 by Antonio Valentino at 2024-08-03T09:34:31+00:00
New upstream version 0.2.12
- - - - -
7 changed files:
- .github/workflows/publish.yml
- README.md
- pyproject.toml
- src/cdsetool/cli.py
- src/cdsetool/credentials.py
- src/cdsetool/download.py
- src/cdsetool/query.py
Changes:
=====================================
.github/workflows/publish.yml
=====================================
@@ -33,7 +33,7 @@ jobs:
- name: Build package
run: python -m build
- name: Publish package
- uses: pypa/gh-action-pypi-publish at 81e9d935c883d0b210363ab89cf05f3894778450
+ uses: pypa/gh-action-pypi-publish at ec4db0b4ddc65acdf4bff5fa45ac92d78b56bdf0
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}
=====================================
README.md
=====================================
@@ -71,7 +71,7 @@ cdsetool download Sentinel2 PATH/TO/DIR --concurrency 4 --search-term startDate=
Install `cdsetool` using pip:
```bash
-pip install cdsetool==0.2.11
+pip install cdsetool==0.2.12
```
## Usage
=====================================
pyproject.toml
=====================================
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "cdsetool"
-version = "0.2.11"
+version = "0.2.12"
authors = [
{ name="Jacob Vejby", email="javej at sdfi.dk" },
]
@@ -25,16 +25,16 @@ dependencies = [
[project.optional-dependencies]
test = [
"black==24.4.2",
- "pylint==3.2.2",
- "pytest==8.2.1",
+ "pylint==3.2.6",
+ "pytest==8.3.2",
"pytest-cov==5.0.0",
"requests-mock==1.12.1",
"pytest-mock==3.14.0",
]
[project.urls]
-"Homepage" = "https://github.com/SDFIdk/CDSETool"
-"Bug Tracker" = "https://github.com/SDFIdk/CDSETool/issues"
+"Homepage" = "https://github.com/CDSETool/CDSETool"
+"Bug Tracker" = "https://github.com/CDSETool/CDSETool/issues"
[project.scripts]
cdsetool = "cdsetool:cli.main"
=====================================
src/cdsetool/cli.py
=====================================
@@ -12,9 +12,9 @@ from cdsetool.query import describe_collection, query_features
from cdsetool.monitor import StatusMonitor
from cdsetool.download import download_features
-app = typer.Typer()
+app = typer.Typer(no_args_is_help=True)
-query_app = typer.Typer()
+query_app = typer.Typer(no_args_is_help=True)
app.add_typer(query_app, name="query")
@@ -120,3 +120,7 @@ def _to_dict(term_list: List[str]) -> Dict[str, str]:
key, value = item.split("=")
search_terms[key] = value
return search_terms
+
+
+if __name__ == "__main__":
+ main()
=====================================
src/cdsetool/credentials.py
=====================================
@@ -147,7 +147,7 @@ class Credentials: # pylint: disable=too-few-public-methods disable=too-many-in
caller=self,
authorization=False,
max_retries=Retry(
- total=2,
+ total=15,
backoff_factor=0.5,
allowed_methods=None,
raise_on_status=False,
=====================================
src/cdsetool/download.py
=====================================
@@ -38,6 +38,7 @@ def download_feature(
log = _get_logger(options)
url = _get_feature_url(feature)
title = feature.get("properties").get("title")
+ temp_dir_usr = _get_temp_dir(options)
if not url or not title:
log.debug(f"Bad URL ('{url}') or title ('{title}')")
@@ -58,19 +59,25 @@ def download_feature(
# Always get a new session, credentials might have expired.
try:
session = _get_credentials(options).get_session()
- except (TokenClientConnectionError, TokenExpiredSignatureError) as e:
- log.warning(e)
+ except TokenClientConnectionError:
+ log.warning("Token client connection failed, retrying..")
+ continue
+ except TokenExpiredSignatureError:
+ log.warning("Token signature expired, retrying..")
continue
url = _follow_redirect(url, session)
- with session.get(url, stream=True) as response:
+ name_dir_prefix = filename.replace(".zip", "____")
+ with session.get(url, stream=True) as response, tempfile.TemporaryDirectory(
+ prefix=name_dir_prefix, dir=temp_dir_usr
+ ) as temp_dir:
if response.status_code != 200:
log.warning(f"Status code {response.status_code}, retrying..")
time.sleep(60 * (1 + (random.random() / 4)))
continue
status.set_filesize(int(response.headers["Content-Length"]))
-
- with tempfile.NamedTemporaryFile() as file:
+ tmp_file = os.path.join(temp_dir, "download.zip")
+ with open(tmp_file, "wb") as file:
# Server might not send all bytes specified by the
# Content-Length header before closing connection.
# Log as a warning and try again.
@@ -85,8 +92,8 @@ def download_feature(
) as e:
log.warning(e)
continue
- shutil.copy(file.name, result_path)
-
+ # Close file before copy so all buffers are flushed.
+ shutil.copy(tmp_file, result_path)
return filename
log.error(f"Failed to download {filename}")
return None
@@ -143,3 +150,7 @@ def _get_credentials(options: Dict) -> Credentials:
return options.get("credentials") or Credentials(
proxies=options.get("proxies", None)
)
+
+
+def _get_temp_dir(options: Dict) -> Union[str, None]:
+ return options.get("tmpdir") or None
=====================================
src/cdsetool/query.py
=====================================
@@ -156,7 +156,7 @@ def describe_collection(
)
parameters = {}
- if not parameter_node_parent:
+ if parameter_node_parent is None:
return parameters
for parameter_node in parameter_node_parent:
name = parameter_node.attrib.get("name")
View it on GitLab: https://salsa.debian.org/debian-gis-team/cdsetool/-/commit/914397e76ccb9b06d15d598280abfcc375ada371
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/cdsetool/-/commit/914397e76ccb9b06d15d598280abfcc375ada371
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20240803/d63f125d/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list