[Git][debian-gis-team/eodag][master] 5 commits: New upstream version 4.9.1+ds
Antonio Valentino (@antonio.valentino)
gitlab at salsa.debian.org
Fri Sep 25 07:08:22 BST 2026
Antonio Valentino pushed to branch master at Debian GIS Project / eodag
Commits:
6592ebb1 by Antonio Valentino at 2026-09-25T05:55:37+00:00
New upstream version 4.9.1+ds
- - - - -
a852b684 by Antonio Valentino at 2026-09-25T05:55:45+00:00
Update upstream source from tag 'upstream/4.9.1+ds'
Update to upstream version '4.9.1+ds'
with Debian dir 997ec66e5782146ce91e7797c8983c5a9a66c3ea
- - - - -
c382e0ae by Antonio Valentino at 2026-09-25T05:56:22+00:00
New upstream release
- - - - -
ea0ab848 by Antonio Valentino at 2026-09-25T05:57:19+00:00
Refresh all patches
- - - - -
54d8796b by Antonio Valentino at 2026-09-25T05:57:43+00:00
Set distribution to unstable
- - - - -
21 changed files:
- CHANGES.rst
- debian/changelog
- debian/patches/0001-System-importlib.metadata.patch
- debian/patches/0002-no-mypy-boto3-s3.patch
- docs/getting_started_guide/configure.rst
- docs/providers.rst
- eodag/api/core.py
- eodag/config.py
- eodag/plugins/authentication/aws_auth.py
- eodag/plugins/download/aws.py
- eodag/resources/ext_collections.json
- eodag/resources/providers/creodias.yml
- eodag/utils/s3.py
- pyproject.toml
- tests/context.py
- tests/test_config.py
- tests/units/test_auth_plugins.py
- tests/units/test_core.py
- tests/units/test_download_plugins.py
- tests/units/test_search_plugins.py
- tests/units/test_utils_s3.py
Changes:
=====================================
CHANGES.rst
=====================================
@@ -3,6 +3,26 @@ Release history
===============
+v4.9.1 (2026-09-23)
+===================
+
+Bug Fixes
+---------
+
+* **core**: Ext collections fetch env var (`#2362`_, `cc50938`_)
+
+* **plugins**: Aws prefix filtering for siblings and dir markers (`#2342`_, `7ab8858`_)
+
+* **plugins**: Creodias and creodias_s3 auth errors handling (`#2357`_, `4bbf5bb`_)
+
+.. _#2342: https://github.com/CS-SI/eodag/pull/2342
+.. _#2357: https://github.com/CS-SI/eodag/pull/2357
+.. _#2362: https://github.com/CS-SI/eodag/pull/2362
+.. _4bbf5bb: https://github.com/CS-SI/eodag/commit/4bbf5bbbe2eaf6b7289f144cce6da69897395201
+.. _7ab8858: https://github.com/CS-SI/eodag/commit/7ab885874059f560e819a37d636b7dcdeb90ca95
+.. _cc50938: https://github.com/CS-SI/eodag/commit/cc509387abb1f8baa4c99f1039a76d3d0b827d17
+
+
v4.9.0 (2026-09-16)
===================
=====================================
debian/changelog
=====================================
@@ -1,3 +1,11 @@
+eodag (4.9.1+ds-1) unstable; urgency=medium
+
+ * New upstream release.
+ * debian/pathes:
+ - Refresh all patches.
+
+ -- Antonio Valentino <antonio.valentino at tiscali.it> Fri, 25 Sep 2026 05:57:24 +0000
+
eodag (4.9.0+ds-1) unstable; urgency=medium
* New upstream release.
=====================================
debian/patches/0001-System-importlib.metadata.patch
=====================================
@@ -65,7 +65,7 @@ index c2f6f64..975954e 100644
"""Return provider config paths exposed by an external plugin distribution."""
config_paths: list[pathlib.Path] = []
diff --git a/pyproject.toml b/pyproject.toml
-index 55397de..0fd91e9 100644
+index 872fa49..155a74b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,7 +39,6 @@ dependencies = [
@@ -99,10 +99,10 @@ index 20afa62..1848ac9 100644
default_libs = stdlib_list()
except FileNotFoundError:
diff --git a/tests/units/test_core.py b/tests/units/test_core.py
-index ab427aa..aec8c21 100644
+index a2149d4..9ac72eb 100644
--- a/tests/units/test_core.py
+++ b/tests/units/test_core.py
-@@ -1691,7 +1691,7 @@ class TestCore(TestCoreBase):
+@@ -1690,7 +1690,7 @@ class TestCore(TestCoreBase):
if "api" in provider_conf:
self.assertEqual(set(provider_conf["api"]), {"credentials"})
=====================================
debian/patches/0002-no-mypy-boto3-s3.patch
=====================================
@@ -8,7 +8,7 @@ Forwarded: not-needed
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/tests/units/test_auth_plugins.py b/tests/units/test_auth_plugins.py
-index 7dedb15..8458c8d 100644
+index 5b56aef..b3c2b1b 100644
--- a/tests/units/test_auth_plugins.py
+++ b/tests/units/test_auth_plugins.py
@@ -26,7 +26,12 @@ from unittest import mock
=====================================
docs/getting_started_guide/configure.rst
=====================================
@@ -339,10 +339,6 @@ commands:
.. code-block:: python
- dag.providers_config["creodias"].auth.credentials["totp"] = "PLEASE_CHANGE_ME"
-
- # OR
-
dag.update_providers_config(
"""
creodias:
@@ -357,7 +353,8 @@ will be stored and used if further authentication tries fail:
.. code-block:: python
- dag._plugins_manager.get_auth_plugin("creodias").authenticate()
+ eoproduct: eodag.EOProduct # EOProduct coming from a search result on creodias
+ eoproduct.downloader_auth.authenticate()
Please note that authentication mechanism is already included in
`download methods <../notebooks/api_user_guide/7_download.ipynb>`_ , so you could
=====================================
docs/providers.rst
=====================================
@@ -641,6 +641,9 @@ No credentials are needed
<getting_started_guide/configure.rst#authenticate-using-an-otp-one-time-password-two-factor-authentication>`__
to see how to proceed.
+ Some collections, like CopDEM ones, require additional license agreement acceptance. Check them on your
+ `creodias personal profile / agreements section <https://new.cloudferro.com/panel/profile/personal>`__.
+
----
**creodias_s3**
=====================================
eodag/api/core.py
=====================================
@@ -40,6 +40,7 @@ from eodag.api.product.metadata_mapping import mtd_cfg_as_conversion_and_querypa
from eodag.api.provider import Provider, ProvidersDict
from eodag.api.search_result import SearchResult
from eodag.config import (
+ EXT_COLLECTIONS_CONF_URI,
PLUGINS_TOPICS_KEYS,
EODAGSettings,
PluginConfig,
@@ -554,13 +555,17 @@ class EODataAccessGateway:
already_fetched = False
if not already_fetched:
+ is_conf_customized = (
+ self.settings.ext_collections_cfg_uri != EXT_COLLECTIONS_CONF_URI
+ )
+
# get ext_collections conf
ext_collections_conf = get_ext_collections_conf(
self.settings.ext_collections_cfg_uri
)
- if not ext_collections_conf:
- # empty ext_collections conf
+ if not is_conf_customized and not ext_collections_conf:
+ # empty ext_collections conf and URI untouched by the user
ext_collections_conf = (
self.discover_collections(provider=provider) or {}
)
=====================================
eodag/config.py
=====================================
@@ -31,7 +31,13 @@ import yaml
import yaml.parser
from annotated_types import Gt
from jsonpath_ng import JSONPath
-from pydantic import BeforeValidator, Field, computed_field, model_validator
+from pydantic import (
+ AliasChoices,
+ BeforeValidator,
+ Field,
+ computed_field,
+ model_validator,
+)
from pydantic_settings import BaseSettings, SettingsConfigDict
from typing_extensions import TypedDict
@@ -133,7 +139,10 @@ class EODAGSettings(BaseSettings):
ext_collections_cfg_uri: str = Field(
default=EXT_COLLECTIONS_CONF_URI,
- validation_alias="EODAG_EXT_COLLECTIONS_CFG_FILE",
+ validation_alias=AliasChoices(
+ "EODAG_EXT_COLLECTIONS_CFG_URI",
+ "EODAG_EXT_COLLECTIONS_CFG_FILE",
+ ),
description=(
"URI of the external collections configuration. "
"Supports HTTP(S), ``file://`` URIs and local filesystem paths."
@@ -186,6 +195,13 @@ class EODAGSettings(BaseSettings):
DeprecationWarning,
stacklevel=5,
)
+ if "EODAG_EXT_COLLECTIONS_CFG_FILE" in os.environ:
+ warnings.warn(
+ "EODAG_EXT_COLLECTIONS_CFG_FILE is deprecated. "
+ "Use EODAG_EXT_COLLECTIONS_CFG_URI instead.",
+ DeprecationWarning,
+ stacklevel=5,
+ )
return self
=====================================
eodag/plugins/authentication/aws_auth.py
=====================================
@@ -165,6 +165,22 @@ class AwsAuth(Authentication):
self.s3_resource = self._create_s3_resource()
return self.s3_resource
+ def get_bucket_objects(self, bucket_name: str) -> BucketObjectsCollection:
+ """Get the object collection for an S3 bucket.
+
+ The returned collection is configured for requester-pays buckets when
+ required by the authentication configuration.
+
+ :param bucket_name: Name of the S3 bucket
+ :returns: Collection of objects in the bucket, filtered for requester-pays if necessary
+ """
+ if not self.s3_resource:
+ self.s3_resource = self._create_s3_resource()
+ objects = self.s3_resource.Bucket(bucket_name).objects
+ if self.config.requester_pays:
+ return objects.filter(RequestPayer="requester")
+ return objects
+
def _get_authenticated_objects(
self, bucket_name: str, prefix: str
) -> BucketObjectsCollection:
@@ -174,15 +190,8 @@ class AwsAuth(Authentication):
:param prefix: Prefix used to filter objects
:returns: The boto3 authenticated objects
"""
- if not self.s3_resource:
- self.s3_resource = self._create_s3_resource()
try:
- if self.config.requester_pays:
- objects = self.s3_resource.Bucket(bucket_name).objects.filter(
- RequestPayer="requester"
- )
- else:
- objects = self.s3_resource.Bucket(bucket_name).objects
+ objects = self.get_bucket_objects(bucket_name)
list(objects.filter(Prefix=prefix).limit(1))
if objects:
logger.debug(
@@ -217,7 +226,7 @@ class AwsAuth(Authentication):
:return: authenticated objects per bucket
"""
- authenticated_objects: dict[str, Any] = {}
+ authenticated_objects: dict[str, BucketObjectsCollection] = {}
auth_error_messages: set[str] = set()
for _, pack in enumerate(bucket_names_and_prefixes):
=====================================
eodag/plugins/download/aws.py
=====================================
@@ -55,17 +55,22 @@ from eodag.utils.exceptions import (
AuthenticationError,
DownloadError,
MisconfiguredError,
- NoMatchingCollection,
NotAvailableError,
TimeOutError,
)
-from eodag.utils.s3 import S3FileInfo, open_s3_zipped_object, stream_download_from_s3
+from eodag.utils.s3 import (
+ S3FileInfo,
+ list_files_in_s3_prefix,
+ open_s3_zipped_object,
+ stream_download_from_s3,
+)
from .base import Download
if TYPE_CHECKING:
from mypy_boto3_s3 import S3ServiceResource
from mypy_boto3_s3.client import S3Client
+ from mypy_boto3_s3.service_resource import BucketObjectsCollection, ObjectSummary
from eodag.api.product import EOProduct
from eodag.config import PluginConfig
@@ -256,7 +261,6 @@ class AwsDownload(Download):
file or with environment variables.
:returns: The absolute path to the downloaded product in the local filesystem
"""
-
if progress_callback is None:
logger.info(
"Progress bar unavailable, please call product.download() instead of plugin.download()"
@@ -430,9 +434,7 @@ class AwsDownload(Download):
progress_callback: ProgressCallback,
executor: ThreadPoolExecutor,
):
- """
- Download file in zip from a prefix like `foo/bar.zip!file.txt`
- """
+ """Download file in zip from a prefix like `foo/bar.zip!file.txt`."""
if downloader_auth.s3_resource is None:
logger.debug("Cannot check files in s3 zip without s3 resource")
return bucket_names_and_prefixes
@@ -504,9 +506,9 @@ class AwsDownload(Download):
progress_callback: ProgressCallback,
**kwargs: Unpack[DownloadConf],
) -> tuple[Optional[str], Optional[str]]:
- """
- Preparation for the download:
+ """Prepare the download.
+ It means:
- check if file was already downloaded
- get file path
- create directories
@@ -533,8 +535,7 @@ class AwsDownload(Download):
return product_local_path, record_filename
def _configure_safe_build(self, build_safe: bool, product: EOProduct):
- """
- Updates the product properties with fetch metadata if safe build is enabled
+ """Update the product properties with fetch metadata if safe build is enabled.
:param build_safe: if safe build is enabled
:param product: product to be updated
@@ -579,8 +580,7 @@ class AwsDownload(Download):
ignore_assets: bool,
complementary_url_keys: list[str],
) -> list[tuple[str, Optional[str]]]:
- """
- Retrieves the bucket names and path prefixes for the assets
+ """Retrieve the bucket names and path prefixes for the assets.
:param product: product for which the assets shall be downloaded
:param asset_filter: text for which the assets should be filtered
@@ -635,14 +635,13 @@ class AwsDownload(Download):
def _get_unique_products(
self,
bucket_names_and_prefixes: list[tuple[str, Optional[str]]],
- authenticated_objects: dict[str, Any],
+ authenticated_objects: dict[str, BucketObjectsCollection],
asset_filter: Optional[str],
ignore_assets: bool,
product: EOProduct,
raise_error: bool = True,
- ) -> set[Any]:
- """
- Retrieve unique product chunks based on authenticated objects and asset filters
+ ) -> set[ObjectSummary]:
+ """Retrieve unique product chunks based on authenticated objects and asset filters.
:param bucket_names_and_prefixes: list of bucket names and corresponding path prefixes
:param authenticated_objects: available objects per bucket
@@ -652,12 +651,15 @@ class AwsDownload(Download):
:param raise_error: raise error if there is nothing to download
:return: set of product chunks that can be downloaded
"""
- product_chunks: list[Any] = []
+ product_chunks: list[ObjectSummary] = []
for bucket_name, prefix in bucket_names_and_prefixes:
# unauthenticated items filtered out
if bucket_name in authenticated_objects.keys():
product_chunks.extend(
- authenticated_objects[bucket_name].filter(Prefix=prefix)
+ list_files_in_s3_prefix(
+ prefix or "",
+ authenticated_objects[bucket_name],
+ )
)
unique_product_chunks = set(product_chunks)
@@ -672,12 +674,15 @@ class AwsDownload(Download):
)
)
if not unique_product_chunks and raise_error:
- raise NotAvailableError(
- rf"No file basename matching re.fullmatch(r'{asset_filter}') was found in {product.remote_location}"
+ msg = (
+ rf"No file basename matching re.fullmatch(r'{asset_filter}') was found in "
+ f"{product.remote_location}"
)
+ raise NotAvailableError(msg)
if not unique_product_chunks and raise_error:
- raise NoMatchingCollection("No product found to download.")
+ msg = f"No downloadable files found for {product}."
+ raise NotAvailableError(msg)
return unique_product_chunks
@@ -691,8 +696,7 @@ class AwsDownload(Download):
timeout: float = DEFAULT_DOWNLOAD_TIMEOUT,
**kwargs: Unpack[DownloadConf],
) -> StreamResponse:
- """
- Stream EO product data as a FastAPI-compatible `StreamResponse`, with support for partial downloads,
+ """Stream EO product data as a FastAPI-compatible `StreamResponse`, with support for partial downloads,
asset filtering, and on-the-fly compression.
This method streams data from one or more S3 objects that belong to a given EO product.
@@ -855,7 +859,7 @@ class AwsDownload(Download):
def get_product_bucket_name_and_prefix(
self, product: EOProduct, url: Optional[str] = None
) -> tuple[str, Optional[str]]:
- """Extract bucket name and prefix from product URL
+ """Extract bucket name and prefix from product URL.
:param product: The EO product to download
:param url: (optional) URL to use as product.location
@@ -880,7 +884,7 @@ class AwsDownload(Download):
return bucket, prefix
def check_manifest_file_list(self, product_path: str) -> None:
- """Checks if products listed in manifest.safe exist"""
+ """Check if products listed in manifest.safe exist."""
manifest_path_list = [
os.path.join(d, x)
for d, _, f in os.walk(product_path)
@@ -904,7 +908,7 @@ class AwsDownload(Download):
logger.warning("SAFE build: %s is missing" % safe_file.get("href"))
def finalize_s2_safe_product(self, product_path: str) -> None:
- """Add missing dirs to downloaded product"""
+ """Add missing dirs to downloaded product."""
try:
logger.debug("Finalize SAFE product")
manifest_path_list = [
@@ -970,7 +974,7 @@ class AwsDownload(Download):
dir_prefix: Optional[str] = None,
build_safe: bool = False,
) -> str:
- """Get chunk SAFE destination path"""
+ """Get chunk SAFE destination path."""
if not build_safe:
if dir_prefix is None:
dir_prefix = chunk.key
=====================================
eodag/resources/ext_collections.json
=====================================
The diff for this file was not included because it is too large.
=====================================
eodag/resources/providers/creodias.yml
=====================================
@@ -246,6 +246,7 @@ creodias:
type: HTTPDownload
extract: true
order_enabled: false
+ auth_error_code: [401, 403]
archive_depth: 2
ssl_verify: true
auth:
=====================================
eodag/utils/s3.py
=====================================
@@ -51,6 +51,7 @@ if TYPE_CHECKING:
from zipfile import ZipInfo
from mypy_boto3_s3.client import S3Client
+ from mypy_boto3_s3.service_resource import BucketObjectsCollection, ObjectSummary
from eodag.api.product import EOProduct # type: ignore
@@ -526,6 +527,42 @@ def stream_download_from_s3(
)
+def list_files_in_s3_prefix(
+ prefix: str,
+ objects: BucketObjectsCollection,
+) -> list[ObjectSummary]:
+ """List files matching an S3 object key or a descendant prefix.
+
+ S3 prefix matching is lexical, so sibling keys such as ``product.txt`` are
+ excluded when listing ``product``. Directory marker objects are excluded,
+ except for a standalone exact object. Some S3-compatible services require
+ a trailing slash to list descendants, which is retried when needed.
+
+ :param prefix: S3 object key prefix to list
+ :param objects: Collection of S3 objects to filter
+ :returns: List of S3 objects matching the prefix criteria
+ """
+ descendant_prefix = f"{prefix.rstrip('/')}/"
+ matching_objects = list(objects.filter(Prefix=prefix))
+ if prefix and not matching_objects:
+ matching_objects = list(objects.filter(Prefix=descendant_prefix))
+
+ has_descendants = any(
+ item.key.startswith(descendant_prefix) and not item.key.endswith("/")
+ for item in matching_objects
+ )
+ return [
+ item
+ for item in matching_objects
+ if not item.key.endswith("/")
+ and (
+ not prefix
+ or (item.key == prefix and not (has_descendants and item.size == 0))
+ or item.key.startswith(descendant_prefix)
+ )
+ ]
+
+
def update_assets_from_s3(
product: EOProduct,
auth: AwsAuth,
@@ -578,29 +615,25 @@ def update_assets_from_s3(
}
else:
# List files in prefix
- s3_objects = s3_client.list_objects(
- Bucket=bucket, Prefix=prefix, MaxKeys=300
- )
- for item_s3 in s3_objects.get("Contents", []):
+ bucket_objects = auth.get_bucket_objects(bucket)
+ for item_s3 in list_files_in_s3_prefix(prefix, bucket_objects):
- url = "s3://{bucket}/{key}".format(bucket=bucket, key=item_s3["Key"])
+ url = "s3://{bucket}/{key}".format(bucket=bucket, key=item_s3.key)
key, roles = product.driver.guess_asset_key_and_roles(url, product)
if key is not None:
asset_data: dict[str, Any] = {
"title": key,
"roles": roles,
"href": url,
- "type": guess_file_type(item_s3["Key"]),
+ "type": guess_file_type(item_s3.key),
}
- etag = item_s3.get("ETag")
+ etag = item_s3.e_tag
# multipart-upload ETags ("<hash>-N") are not MD5 digests
if isinstance(etag, str) and "-" not in etag:
asset_data["file:checksum"] = etag.strip('"')
- size = item_s3.get("Size")
- if size is not None:
- asset_data["file:size"] = size
+ asset_data["file:size"] = item_s3.size
- if last_modified := to_iso_utc_string(item_s3.get("LastModified")):
+ if last_modified := to_iso_utc_string(item_s3.last_modified):
asset_data["updated"] = last_modified
assets_data[key] = asset_data
=====================================
pyproject.toml
=====================================
@@ -212,7 +212,7 @@ include = ["eodag*"]
"*" = ["LICENSE", "NOTICE", "py.typed"]
[tool.setuptools_scm]
-fallback_version = "4.9.1.dev0"
+fallback_version = "4.9.2.dev0"
[tool.isort]
multi_line_output = 3
@@ -291,6 +291,7 @@ addopts = "--doctest-modules --disable-socket --allow-unix-socket"
log_cli = false
filterwarnings = [
"""ignore:
- .*(Call to deprecated eodag|locations_conf_path|user_conf_file_path|EODAG_PROVIDERS_CFG_FILE).*:
+ .*(Call to deprecated eodag|locations_conf_path|user_conf_file_path|EODAG_PROVIDERS_CFG_FILE|\
+ EODAG_EXT_COLLECTIONS_CFG_FILE).*:
DeprecationWarning""",
]
=====================================
tests/context.py
=====================================
@@ -117,6 +117,7 @@ from eodag.utils.requests import LocalFileAdapter, fetch_json
from eodag.utils.import_system import import_all_modules, patch_owslib_requests
from eodag.utils.notebook import NotebookWidgets, check_ipython, check_notebook
from eodag.utils.s3 import (
+ list_files_in_s3_prefix,
list_files_in_s3_zipped_object,
update_assets_from_s3,
open_s3_zipped_object,
=====================================
tests/test_config.py
=====================================
@@ -286,6 +286,28 @@ class TestConfigFunctions(unittest.TestCase):
# priority is set to 0 for all providers
self.assertEqual(value.priority, 0)
+ def test_ext_collections_config_uri_environment_variables(self):
+ """Both current and deprecated environment variable names are supported."""
+ try:
+ os.environ["EODAG_EXT_COLLECTIONS_CFG_URI"] = "new-uri"
+ self.assertEqual(config.EODAGSettings().ext_collections_cfg_uri, "new-uri")
+
+ os.environ.pop("EODAG_EXT_COLLECTIONS_CFG_URI")
+ os.environ["EODAG_EXT_COLLECTIONS_CFG_FILE"] = "deprecated-uri"
+ with self.assertWarnsRegex(
+ DeprecationWarning,
+ "EODAG_EXT_COLLECTIONS_CFG_FILE is deprecated",
+ ):
+ self.assertEqual(
+ config.EODAGSettings().ext_collections_cfg_uri, "deprecated-uri"
+ )
+
+ os.environ["EODAG_EXT_COLLECTIONS_CFG_URI"] = "new-uri"
+ self.assertEqual(config.EODAGSettings().ext_collections_cfg_uri, "new-uri")
+ finally:
+ os.environ.pop("EODAG_EXT_COLLECTIONS_CFG_URI", None)
+ os.environ.pop("EODAG_EXT_COLLECTIONS_CFG_FILE", None)
+
def test_load_provider_configs_file_precedence(self):
"""load_provider_configs must prioritize providers_cfg_file over providers_cfg_dir."""
providers_cfg_file_override = (
=====================================
tests/units/test_auth_plugins.py
=====================================
@@ -854,6 +854,29 @@ class TestAuthPluginAwsAuth(BaseAuthPluginTest):
auth_objects = plugin.authenticate_objects(buckets_prefixes)
self.assertDictEqual({"a": auth_objects_a, "b": auth_objects_b}, auth_objects)
+ @mock.patch(
+ "eodag.plugins.authentication.aws_auth.AwsAuth._create_s3_resource",
+ autospec=True,
+ )
+ def test_plugins_auth_aws_get_bucket_objects_requester_pays(
+ self, mock_create_s3_resource
+ ):
+ """get_bucket_objects must configure requester-pays collections."""
+ plugin = self.get_auth_plugin("provider_with_auth_keys")
+ plugin.config.requester_pays = True
+ plugin.s3_resource = None
+ bucket_objects = (
+ mock_create_s3_resource.return_value.Bucket.return_value.objects
+ )
+
+ objects = plugin.get_bucket_objects("requester-pays-bucket")
+
+ mock_create_s3_resource.return_value.Bucket.assert_called_once_with(
+ "requester-pays-bucket"
+ )
+ bucket_objects.filter.assert_called_once_with(RequestPayer="requester")
+ self.assertIs(objects, bucket_objects.filter.return_value)
+
@mock.patch(
"eodag.plugins.authentication.aws_auth.AwsAuth._create_s3_resource",
autospec=True,
=====================================
tests/units/test_core.py
=====================================
@@ -1558,11 +1558,11 @@ class TestCore(TestCoreBase):
mock_get_ext_collections_conf.return_value = {}
- # an empty configured URI still falls back to discovery
+ # disabled collections discovery
self.dag.settings.ext_collections_cfg_uri = ""
self.dag.fetch_collections_list()
mock_get_ext_collections_conf.assert_called_with("")
- mock_discover_collections.assert_called_once_with(self.dag, provider=None)
+ mock_discover_collections.assert_not_called()
mock_discover_collections.reset_mock()
@@ -1589,7 +1589,7 @@ class TestCore(TestCoreBase):
# default settings still fall back to discover_collections
self.dag.fetch_collections_list()
- mock_discover_collections.assert_called_once_with(self.dag, provider=None)
+ mock_discover_collections.assert_not_called()
mock_discover_collections.reset_mock()
@@ -1611,7 +1611,6 @@ class TestCore(TestCoreBase):
self.assertEqual(
mock_discover_collections.call_args_list,
[
- mock.call(self.dag, provider=None),
mock.call(self.dag, provider="earth_search"),
mock.call(self.dag, provider="foo_provider"),
],
=====================================
tests/units/test_download_plugins.py
=====================================
@@ -35,7 +35,6 @@ from eodag.utils import MockResponse, ProgressCallback
from eodag.utils.exceptions import (
DownloadError,
MisconfiguredError,
- NoMatchingCollection,
QuotaExceededError,
ValidationError,
)
@@ -115,9 +114,10 @@ class BaseDownloadPluginTest(unittest.TestCase):
class TestDownloadPluginBase(BaseDownloadPluginTest):
- def test_plugins_download_base_prepare_download_existing(self):
- """Download._prepare_download must detect if product destination already exists"""
+ """Test cases for the base download plugin."""
+ def test_plugins_download_base_prepare_download_existing(self):
+ """Download._prepare_download must detect if product destination already exists."""
product_file = NamedTemporaryFile()
self.product.location = path_to_uri(product_file.name)
Path(product_file.name).touch()
@@ -132,8 +132,7 @@ class TestDownloadPluginBase(BaseDownloadPluginTest):
self.assertIn("Product already present on this platform", str(cm.output))
def test_plugins_download_base_prepare_download_record_file(self):
- """Download._prepare_download must check existing record files"""
-
+ """Download._prepare_download must check existing record files."""
self.product.location = self.product.remote_location = "http://foo.bar"
self.product.collection = "foo"
@@ -189,8 +188,7 @@ class TestDownloadPluginBase(BaseDownloadPluginTest):
self.assertIn("Product already downloaded", str(cm.output))
def test_plugins_download_base_prepare_download_record_file_collision_dir(self):
- """Download._prepare_download must detect already-downloaded dir without collision suffix"""
-
+ """Download._prepare_download must detect already-downloaded dir without collision suffix."""
self.product.properties["title"] = "title to sanitïze"
self.product.properties["id"] = "id alsô"
self.product.location = self.product.remote_location = "http://foo.bar"
@@ -221,8 +219,7 @@ class TestDownloadPluginBase(BaseDownloadPluginTest):
)
def test_plugins_download_base_prepare_download_no_url(self):
- """Download._prepare_download must return None when no download url"""
-
+ """Download._prepare_download must return None when no download url."""
self.assertEqual(self.product.remote_location, "")
plugin = self.get_download_plugin(self.product)
@@ -233,8 +230,7 @@ class TestDownloadPluginBase(BaseDownloadPluginTest):
self.assertIsNone(record_filename)
def test_plugins_download_base_prepare_download_collision_avoidance(self):
- """Download._prepare_download must use collision avoidance suffix"""
-
+ """Download._prepare_download must use collision avoidance suffix."""
self.product.properties["title"] = "needs sanitïze"
self.product.properties["id"] = "alsô.zip"
self.product.location = self.product.remote_location = "somewhere"
@@ -246,7 +242,7 @@ class TestDownloadPluginBase(BaseDownloadPluginTest):
self.assertEqual(fs_path, os.path.join(gettempdir(), "needs_sanitize-also.zip"))
def test_plugins_download_base_prepare_download_dir_permission(self):
- """Download._prepare_download must check output directory permissions"""
+ """Download._prepare_download must check output directory permissions."""
if os.name == "nt":
self.skipTest("windows permissions too complex to set for this test")
@@ -261,7 +257,7 @@ class TestDownloadPluginBase(BaseDownloadPluginTest):
self.assertIn("Unable to create records directory", str(cm.output))
def test_plugins_download_base_finalize_extract_not_complete(self):
- """Download._finalize must not delete archive if extract is True but file is not an archive"""
+ """Download._finalize must not delete archive if extract is True but file is not an archive."""
plugin = self.get_download_plugin(self.product)
with TemporaryDirectory() as output_dir:
@@ -277,7 +273,7 @@ class TestDownloadPluginBase(BaseDownloadPluginTest):
self.assertTrue(os.path.isfile(fs_path))
def test_plugins_download_base_finalize_extract_complete(self):
- """Download._finalize must delete archive if extract is True and file is an archive"""
+ """Download._finalize must delete archive if extract is True and file is an archive."""
plugin = self.get_download_plugin(self.product)
with TemporaryDirectory() as output_dir:
@@ -299,6 +295,8 @@ class TestDownloadPluginBase(BaseDownloadPluginTest):
class TestDownloadPluginHttp(BaseDownloadPluginTest):
+ """Test cases for the HTTP download plugin."""
+
def _dummy_product(
self, provider: str, properties: dict[str, Any], collection: str
):
@@ -330,8 +328,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
@responses.activate
def test_plugins_download_http_zip_file_ok(self):
- """HTTPDownload.download() must keep the output as it is when it is a zip file"""
-
+ """HTTPDownload.download() must keep the output as it is when it is a zip file."""
provider = "creodias"
download_url = (
"https://zipper.creodias.eu/download/8ff765a2-e089-465d-a48f-cc27008a0962"
@@ -412,9 +409,10 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_nonzip_file_with_zip_extension_ok(
self, mock_stream, mock_stream_download
):
- """HTTPDownload.download() must create an output directory
- when the result is a non-zip file with a '.zip' outputs extension"""
+ """HTTPDownload.download() must create an output directory in certain conditions.
+ One is when the result is a non-zip file with a '.zip' outputs extension.
+ """
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -458,9 +456,10 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_file_without_zip_extension_ok(
self, mock_stream, mock_stream_download
):
- """HTTPDownload.download() must create an output directory
- when the result is a file without a '.zip' outputs extension"""
+ """HTTPDownload.download() must create an output directory in certain conditions.
+ One is when the result is a file without a '.zip' outputs extension.
+ """
# we use a provider having '.nc' as outputs file extension in its configuration
product = EOProduct(
"meteoblue",
@@ -518,8 +517,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_ignore_assets(
self, mock_requests_get, mock_requests_head, mock_stream, mock_stream_download
):
- """HTTPDownload.download() must ignore assets if configured to"""
-
+ """HTTPDownload.download() must ignore assets if configured to."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = (
"http://somewhere/download_from_location"
@@ -583,8 +581,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_ignore_assets_without_ssl(
self, mock_requests_get, mock_requests_head
):
- """HTTPDownload.download() must ignore assets if configured to"""
-
+ """HTTPDownload.download() must ignore assets if configured to."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = (
"http://somewhere/download_from_location"
@@ -628,8 +625,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_ignore_assets_product_config_true(
self, mock_stream_download, mock_stream, mock_download_assets
):
- """HTTPDownload.download() must support product-level ignore_assets=True"""
-
+ """HTTPDownload.download() must support product-level ignore_assets=True."""
product = self._dummy_downloadable_product(
"dummy_provider",
{
@@ -665,8 +661,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_ignore_assets_product_config_false(
self, mock_stream_download, mock_download_assets
):
- """HTTPDownload.download() must support product-level ignore_assets=False"""
-
+ """HTTPDownload.download() must support product-level ignore_assets=False."""
product = self._dummy_downloadable_product(
"dummy_provider",
{
@@ -699,8 +694,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_assets_filename_from_href(
self, mock_requests_get, mock_requests_head
):
- """HTTPDownload.download() must create an outputfile"""
-
+ """HTTPDownload.download() must create an output file."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -752,8 +746,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_assets_filename_from_get(
self, mock_requests_get, mock_requests_head
):
- """HTTPDownload.download() must create an outputfile"""
-
+ """HTTPDownload.download() must create an outputfile."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -785,8 +778,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_assets_error(
self, mock_requests_get, mock_requests_head, mock_asset_size
):
- """HTTPDownload.download() must create an outputfile"""
-
+ """HTTPDownload.download() must create an outputfile."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -810,8 +802,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_assets_interrupt(
self, mock_requests_get, mock_requests_head, mock_progress_callback
):
- """HTTPDownload.download() must download assets to a temporary file"""
-
+ """HTTPDownload.download() must download assets to a temporary file."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -865,8 +856,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_assets_stream_zip_interrupt(
self, mock_requests_get, mock_requests_head, mock_progress_callback
):
- """HTTPDownload.stream_download must raise an error if an error is returned by the provider"""
-
+ """HTTPDownload.stream_download must raise an error if an error is returned by the provider."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -896,8 +886,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_assets_too_many_requests_error(
self, mock_requests_get, mock_requests_head, mock_asset_size
):
- """HTTPDownload.download() must handle a 429 (Too many requests) error"""
-
+ """HTTPDownload.download() must handle a 429 (Too many requests) error."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -913,8 +902,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
plugin.download(self.product, output_dir=self.output_dir)
def test_plugins_download_http_stream_dict_misconfigured(self):
- """HTTPDownload.stream_download() must raise an error if misconfigured"""
-
+ """HTTPDownload.stream_download() must raise an error if misconfigured."""
plugin = self.get_download_plugin(self.product)
with self.assertRaises(MisconfiguredError):
# Wrong auth instance
@@ -924,8 +912,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
)
def test_stream_download_single_asset(self):
- """HTTPDownload.stream_download() must return a response with a single asset"""
-
+ """HTTPDownload.stream_download() must return a response with a single asset."""
plugin = self.get_download_plugin(self.product)
asset = mock.Mock()
@@ -951,8 +938,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
self.assertEqual(response.content, [b"chunk1"])
def test_stream_download_multiple_assets_zip(self):
- """HTTPDownload.stream_download() must return a zipped response with multiple assets"""
-
+ """HTTPDownload.stream_download() must return a zipped response with multiple assets."""
plugin = self.get_download_plugin(self.product)
asset1 = mock.Mock(filename="file1.txt")
@@ -999,8 +985,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
self.assertIn(".zip", response.headers["Content-Disposition"])
def test_stream_download_asset_not_available(self):
- """HTTPDownload.stream_download() must raise NotAvailableError if asset not available"""
-
+ """HTTPDownload.stream_download() must raise NotAvailableError if asset not available."""
plugin = self.get_download_plugin(self.product)
self.product.assets = mock.Mock()
@@ -1013,8 +998,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
)
def test_stream_download_single_asset_with_type(self):
- """HTTPDownload.stream_download() must return a response with a single asset and its type"""
-
+ """HTTPDownload.stream_download() must return a response with a single asset and its type."""
plugin = self.get_download_plugin(self.product)
asset = mock.Mock()
@@ -1044,8 +1028,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
self.assertEqual(response.content, [b"chunk1"])
def test_stream_download_fallback_to_product(self):
- """HTTPDownload.stream_download() must return a response with product headers if no asset headers"""
-
+ """HTTPDownload.stream_download() must return a response with product headers if no asset headers."""
plugin = self.get_download_plugin(self.product)
self.product.assets = mock.Mock()
@@ -1113,8 +1096,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
self.assertIn("foo.zip", response.headers["Content-Disposition"])
def test_stream_download_product_empty_raises(self):
- """HTTPDownload.stream_download() must raise NotAvailableError if no asset and no product headers"""
-
+ """HTTPDownload.stream_download() must raise NotAvailableError if no asset and no product headers."""
plugin = self.get_download_plugin(self.product)
self.product.assets = mock.Mock()
@@ -1134,8 +1116,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_assets_resume(
self, mock_requests_get, mock_requests_head
):
- """HTTPDownload.download() must resume the interrupted download of assets"""
-
+ """HTTPDownload.download() must resume the interrupted download of assets."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -1179,8 +1160,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_asset_filter(
self, mock_requests_get, mock_requests_head
):
- """HTTPDownload.download() must create an outputfile"""
-
+ """HTTPDownload.download() must create an outputfile."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -1220,8 +1200,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_assets_filename_from_head(
self, mock_requests_get, mock_requests_head
):
- """HTTPDownload.download() must create an outputfile"""
-
+ """HTTPDownload.download() must create an outputfile."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -1253,8 +1232,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_assets_size(
self, mock_requests_get, mock_requests_head, mock_progress_callback_reset
):
- """HTTPDownload.download() must get assets sizes"""
-
+ """HTTPDownload.download() must get assets sizes."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.assets.clear()
@@ -1358,8 +1336,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_one_local_asset(
self,
):
- """HTTPDownload.download() must handle one local asset"""
-
+ """HTTPDownload.download() must handle one local asset."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -1383,8 +1360,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_several_local_assets(
self,
):
- """HTTPDownload.download() must handle several local assets"""
-
+ """HTTPDownload.download() must handle several local assets."""
plugin = self.get_download_plugin(self.product)
self.product.location = self.product.remote_location = "http://somewhere"
self.product.properties["id"] = "someproduct"
@@ -1426,8 +1402,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_order_download_cop_ads(
self,
):
- """HTTPDownload.download must order the product if needed"""
-
+ """HTTPDownload.download must order the product if needed."""
self.product.provider = "cop_ads"
self.product.collection = "CAMS_EAC4"
product_dataset = "cams-global-reanalysis-eac4"
@@ -1523,7 +1498,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
@mock.patch("eodag.plugins.download.http.requests.request", autospec=True)
def test_plugins_download_http_order_get(self, mock_request):
- """HTTPDownload._order() must request using eodag:order_link and GET protocol"""
+ """HTTPDownload._order() must request using eodag:order_link and GET protocol."""
plugin = self.get_download_plugin(self.product)
self.product.properties["eodag:download_link"] = (
"https://copernicus.nci.org.au/dummy"
@@ -1557,9 +1532,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
@mock.patch("eodag.plugins.download.http.requests.request", autospec=True)
def test_plugins_download_http_order_get_raises_if_request_500(self, mock_request):
- """HTTPDownload._order() must raise an error if request to backend
- provider failed"""
-
+ """HTTPDownload._order() must raise an error if request to backend provider failed."""
# Configure mock to raise an error
mock_request.return_value = MockResponse(status_code=500)
@@ -1622,7 +1595,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
@mock.patch("eodag.plugins.download.http.requests.request", autospec=True)
def test_plugins_download_http_order_post(self, mock_request):
- """HTTPDownload._order() must request using eodag:order_link and POST protocol"""
+ """HTTPDownload._order() must request using eodag:order_link and POST protocol."""
plugin = self.get_download_plugin(self.product)
self.product.properties["eodag:download_link"] = (
"https://copernicus.nci.org.au/dummy"
@@ -1679,7 +1652,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
)
def test_plugins_download_http_order_status(self):
- """HTTPDownload._order_status() must request status using eodag:status_link"""
+ """HTTPDownload._order_status() must request status using eodag:status_link."""
plugin = self.get_download_plugin(self.product)
plugin.config.order_status = {
"metadata_mapping": {
@@ -1719,7 +1692,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
@mock.patch("eodag.plugins.download.http.requests.request", autospec=True)
def test_plugins_download_http_order_status_from_head_headers(self, mock_request):
- """HTTPDownload._order_status() must not parse an empty HEAD response"""
+ """HTTPDownload._order_status() must not parse an empty HEAD response."""
plugin = HTTPDownload(
provider=self.product.provider,
config=PluginConfig.from_mapping(
@@ -1760,9 +1733,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_order_status_get_raises_if_request_500(
self, mock_request
):
- """HTTPDownload._order() must raise an error if request to backend
- provider failed"""
-
+ """HTTPDownload._order() must raise an error if request to backend provider failed."""
# Configure mock to raise an error
mock_request.return_value = MockResponse(status_code=500)
@@ -1887,7 +1858,7 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
def test_plugins_download_http_order_status_search_again_raises_if_request_failed(
self,
):
- """HTTPDownload._order_status() must raise an error if the search request after success failed"""
+ """HTTPDownload._order_status() must raise an error if the search request after success failed."""
plugin = self.get_download_plugin(self.product)
plugin.config.order_status = {
"metadata_mapping": {"eodag:order_status": "$.json.status"},
@@ -1938,6 +1909,8 @@ class TestDownloadPluginHttp(BaseDownloadPluginTest):
class TestDownloadPluginHttpRetry(BaseDownloadPluginTest):
+ """Test cases for the HTTP download plugin with retry logic."""
+
def setUp(self):
super(TestDownloadPluginHttpRetry, self).setUp()
@@ -1947,7 +1920,7 @@ class TestDownloadPluginHttpRetry(BaseDownloadPluginTest):
self.product.properties["order:status"] = OFFLINE_STATUS
def test_plugins_download_http_retry_error_timeout(self):
- """HTTPDownload.download() must retry on error until timeout"""
+ """HTTPDownload.download() must retry on error until timeout."""
@responses.activate(registry=responses.registries.FirstMatchRegistry)
def run():
@@ -1983,7 +1956,7 @@ class TestDownloadPluginHttpRetry(BaseDownloadPluginTest):
run()
def test_plugins_download_http_retry_notready_timeout(self):
- """HTTPDownload.download() must retry if not ready until timeout"""
+ """HTTPDownload.download() must retry if not ready until timeout."""
@responses.activate(registry=responses.registries.FirstMatchRegistry)
def run():
@@ -2004,7 +1977,7 @@ class TestDownloadPluginHttpRetry(BaseDownloadPluginTest):
run()
def test_plugins_download_http_retry_ok(self):
- """HTTPDownload.download() must retry until request succeeds"""
+ """HTTPDownload.download() must retry until request succeeds."""
@responses.activate(registry=responses.registries.OrderedRegistry)
def run():
@@ -2034,7 +2007,7 @@ class TestDownloadPluginHttpRetry(BaseDownloadPluginTest):
run()
def test_plugins_download_http_retry_short_timeout(self):
- """HTTPDownload.download() must not retry on very short timeout"""
+ """HTTPDownload.download() must not retry on very short timeout."""
@responses.activate(registry=responses.registries.FirstMatchRegistry)
def run():
@@ -2055,7 +2028,7 @@ class TestDownloadPluginHttpRetry(BaseDownloadPluginTest):
run()
def test_plugins_download_http_retry_once_timeout(self):
- """HTTPDownload.download() must retry once if wait time is equal to timeout"""
+ """HTTPDownload.download() must retry once if wait time is equal to timeout."""
@responses.activate(registry=responses.registries.FirstMatchRegistry)
def run():
@@ -2076,7 +2049,7 @@ class TestDownloadPluginHttpRetry(BaseDownloadPluginTest):
run()
def test_plugins_download_http_retry_timeout_disabled(self):
- """HTTPDownload.download() must not retry on error if timeout is disabled"""
+ """HTTPDownload.download() must not retry on error if timeout is disabled."""
@responses.activate(registry=responses.registries.FirstMatchRegistry)
def run():
@@ -2095,6 +2068,8 @@ class TestDownloadPluginHttpRetry(BaseDownloadPluginTest):
class TestDownloadPluginAws(BaseDownloadPluginTest):
+ """Test cases for the AWS download plugin."""
+
def setUp(self):
super(TestDownloadPluginAws, self).setUp()
self.product = EOProduct(
@@ -2112,8 +2087,7 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
)
def test_plugins_download_aws_get_bucket_prefix(self):
- """AwsDownload.get_product_bucket_name_and_prefix() must extract bucket & prefix from location"""
-
+ """AwsDownload.get_product_bucket_name_and_prefix() must extract bucket & prefix from location."""
plugin = self.get_download_plugin(self.product)
plugin.config.products["S2_MSI_L2A"]["default_bucket"] = "default_bucket"
@@ -2131,7 +2105,7 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
self.assertEqual((bucket, prefix), ("default_bucket", "somewhere/else"))
def test_plugins_download_aws_get_commonpath(self):
- """AwsDownload._get_commonpath() must return common chunk destination path"""
+ """AwsDownload._get_commonpath() must return common chunk destination path."""
plugin = self.get_download_plugin(self.product)
product_chunks = {
mock.Mock(key="path/to/some/product/file1.tif"),
@@ -2144,9 +2118,120 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
self.assertEqual(common_path, "path/to/some/product")
+ def test_plugins_download_aws_ignores_directory_markers_and_sibling_files(self):
+ """AwsDownload._get_unique_products() must ignore S3 directory markers and sibling files.
+
+ Objects that should be included are only:
+ - exact object at the product prefix which does not have any child object, even if it has zero size
+ - exact object at the product prefix which has children objects, but it must have a non-zero size
+ - actual product files (child below the product folder), even if they have zero size.
+ """
+ plugin = self.get_download_plugin(self.product)
+
+ # Test the case where the S3 bucket contains a mix of directory markers, actual product files, and sibling files
+
+ # Setup mock S3 objects representing directory markers and actual files:
+ # - Exact directory marker (zero-size object at the prefix itself)
+ # - Exact directory marker with non-zero size (exact object at the prefix itself)
+ # - Slash directory marker (object with trailing slash)
+ # - Actual product file with non-zero size (child below the product folder)
+ # - Actual product file with zero size (child below the product folder)
+ # - Sibling file outside the product directory with similar prefix
+ exact_directory_marker_zero_size = mock.Mock(key="path/to/some/product", size=0)
+ exact_directory_marker_non_zero_size = mock.Mock(
+ key="path/to/some/product", size=1
+ )
+ slash_directory_marker = mock.Mock(key="path/to/some/product/", size=1)
+ product_file_zero_size = mock.Mock(key="path/to/some/product/file2.tif", size=0)
+ product_file_non_zero_size = mock.Mock(
+ key="path/to/some/product/file1.tif", size=1
+ )
+ sibling_file = mock.Mock(key="path/to/some/product.txt", size=1)
+ bucket_objects = mock.Mock()
+ bucket_objects.filter.side_effect = lambda Prefix: [
+ chunk
+ for chunk in (
+ exact_directory_marker_zero_size,
+ exact_directory_marker_non_zero_size,
+ slash_directory_marker,
+ product_file_zero_size,
+ product_file_non_zero_size,
+ sibling_file,
+ )
+ if chunk.key.startswith(Prefix)
+ ]
+
+ product_chunks = plugin._get_unique_products(
+ [("somebucket", "path/to/some/product")],
+ {"somebucket": bucket_objects},
+ None,
+ False,
+ self.product,
+ )
+
+ bucket_objects.filter.assert_called_once_with(Prefix="path/to/some/product")
+
+ # Check the files that should be included in the product chunks are:
+ # - child below the product directory (actual product files), zero size included
+ # - non-empty folder with children objects (not a directory marker)
+ self.assertSetEqual(
+ product_chunks,
+ {
+ product_file_non_zero_size,
+ product_file_zero_size,
+ exact_directory_marker_non_zero_size,
+ },
+ )
+
+ # Test the case where the exact zero-size object is the only one
+ # matching the prefix (it does not have any child object).
+ bucket_objects.reset_mock()
+ bucket_objects.filter.side_effect = lambda Prefix: (
+ []
+ if Prefix == "path/to/some/product"
+ else [exact_directory_marker_zero_size]
+ )
+
+ product_chunks = plugin._get_unique_products(
+ [("somebucket", "path/to/some/product")],
+ {"somebucket": bucket_objects},
+ None,
+ False,
+ self.product,
+ )
+
+ # Check that when an exact object does not have any child object, it is still included
+ # in the product chunks, even it is zero-size (it is not an directory marker anymore).
+ self.assertSetEqual(product_chunks, {exact_directory_marker_zero_size})
+
+ def test_plugins_download_aws_falls_back_to_slash_prefix(self):
+ """AwsDownload._get_unique_products() must support backends requiring slash prefix."""
+ plugin = self.get_download_plugin(self.product)
+ product_file = mock.Mock(key="path/to/some/product/file.tif", size=1)
+ bucket_objects = mock.Mock()
+ bucket_objects.filter.side_effect = lambda Prefix: (
+ [] if Prefix == "path/to/some/product" else [product_file]
+ )
+
+ product_chunks = plugin._get_unique_products(
+ [("somebucket", "path/to/some/product")],
+ {"somebucket": bucket_objects},
+ None,
+ False,
+ self.product,
+ )
+
+ bucket_objects.filter.assert_has_calls(
+ [
+ mock.call(Prefix="path/to/some/product"),
+ mock.call(Prefix="path/to/some/product/"),
+ ]
+ )
+ self.assertSetEqual(product_chunks, {product_file})
+
@mock.patch("eodag.plugins.download.aws.stream_download_from_s3", autospec=True)
def test_plugins_download_aws_stream_download(self, mock_stream_download_from_s3):
- """AwsDownload.stream_download() must stream S3 objects with flattened paths"""
+ """AwsDownload.stream_download() must stream S3 objects with flattened paths."""
expected_response = StreamResponse(iter([b"content"]))
mock_stream_download_from_s3.return_value = expected_response
plugin = self.get_download_plugin(self.product)
@@ -2261,7 +2346,7 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
mock_flatten_top_directories: mock.Mock,
mock__get_unique_products: mock.Mock,
):
- """AwsDownload.download() must not call safe build methods if not needed"""
+ """AwsDownload.download() must not call safe build methods if not needed."""
mock_aws_auth_init.return_value = None
plugin = self.get_download_plugin(self.product)
@@ -2316,8 +2401,7 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
mock_flatten_top_directories: mock.Mock,
mock__get_unique_products: mock.Mock,
):
- """AwsDownload.download() must not call safe build methods if not needed"""
-
+ """AwsDownload.download() must not call safe build methods if not needed."""
mock_aws_auth_init.return_value = None
plugin = self.get_download_plugin(self.product)
auth_plugin = self.get_auth_plugin(plugin, self.product)
@@ -2370,7 +2454,7 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
mock_s3_resource: mock.Mock,
mock_s3_session: mock.Mock,
):
- """AwsDownload.download() must handle files in zip"""
+ """AwsDownload.download() must handle files in zip."""
def _open_zip(*args, **kwargs):
return (
@@ -2451,8 +2535,7 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
mock_check_manifest_file_list,
mock_flatten_top_directories,
):
- """AwsDownload.download() must call safe build methods if needed"""
-
+ """AwsDownload.download() must call safe build methods if needed."""
mock_aws_auth_init.return_value = None
plugin = self.get_download_plugin(self.product)
auth_plugin = self.get_auth_plugin(plugin, self.product)
@@ -2537,8 +2620,7 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
mock_check_manifest_file_list,
mock_flatten_top_directories,
):
- """AwsDownload.download() must call safe build methods if needed"""
-
+ """AwsDownload.download() must call safe build methods if needed."""
mock_aws_auth_init.return_value = None
plugin = self.get_download_plugin(self.product)
auth_plugin = self.get_auth_plugin(plugin, self.product)
@@ -2618,8 +2700,7 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
mock_aws_auth_init,
mock_get_authenticated_objects: mock.Mock,
):
- """AwsDownload.download() must fail if no product chunk is available"""
-
+ """AwsDownload.download() must fail if no product chunk is available."""
mock_aws_auth_init.return_value = None
plugin = self.get_download_plugin(self.product)
auth_plugin = self.get_auth_plugin(plugin, self.product)
@@ -2630,7 +2711,7 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
plugin.config.products[self.product.collection]["build_safe"] = False
plugin.config.flatten_top_dirs = True
- with self.assertRaises(NoMatchingCollection):
+ with self.assertRaises(NotAvailableError):
plugin.download(self.product, outputs_prefix=self.output_dir)
@mock.patch(
@@ -2651,8 +2732,7 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
mock_s3_session: mock.Mock,
mock_s3_client: mock.Mock,
):
- """AwsDownload.get_rio_env() must return rio env dict"""
-
+ """AwsDownload.get_rio_env() must return rio env dict."""
self.product.properties["eodag:download_link"] = "s3://some-bucket/some/prefix"
plugin = self.get_download_plugin(self.product)
=====================================
tests/units/test_search_plugins.py
=====================================
@@ -3414,105 +3414,122 @@ class TestSearchPluginCreodiasS3Search(BaseSearchPluginTest):
def setUp(self):
super(TestSearchPluginCreodiasS3Search, self).setUp()
self.provider = "creodias_s3"
+ with open(
+ Path(TEST_RESOURCES_PATH) / "provider_responses/creodias_s3_objects.json"
+ ) as f:
+ self.s3_objects = json.load(f)["Contents"]
+ with open(Path(TEST_RESOURCES_PATH) / "eodag_search_result_creodias.json") as f:
+ self.search_result = json.load(f)
+
+ def _get_search_results(self, mock_request):
+ mock_request.return_value = MockResponse(self.search_result, 200)
+ search_plugin = self.get_search_plugin("S1_SAR_GRD", self.provider)
+ return search_plugin.query(collection="S1_SAR_GRD")
+
+ def _get_downloader_and_auth(self, product):
+ downloader = self.plugins_manager.get_download_plugin(product)
+ auth = self.plugins_manager.get_auth_plugin(downloader, product)
+ auth.config.credentials = {
+ "aws_access_key_id": "foo",
+ "aws_secret_access_key": "bar",
+ }
+ auth.s3_resource = boto3.resource(
+ "s3", aws_access_key_id="foo", aws_secret_access_key="bar"
+ )
+ return downloader, auth
+
+ def _stub_s3_list_objects_for_product(
+ self, product, auth, object_entries
+ ) -> Stubber:
+ bucket, prefix = product.remote_location.removeprefix("s3://").split("/", 1)
+ response = {
+ "Contents": [
+ {**item, "Key": f"{prefix}/{item['Key'].rsplit('/', 1)[-1]}"}
+ for item in object_entries
+ ]
+ }
+ stubber = Stubber(auth.s3_resource.meta.client)
+ for requested_prefix in (prefix, f"{prefix}/"):
+ stubber.add_response(
+ "list_objects",
+ response if object_entries else {},
+ {"Bucket": bucket, "Prefix": requested_prefix},
+ )
+ return stubber
+
+ def _register_product_with_stubbed_s3_objects(self, product, object_entries):
+ downloader, auth = self._get_downloader_and_auth(product)
+ stubber = self._stub_s3_list_objects_for_product(product, auth, object_entries)
+ stubber.activate()
+ try:
+ product.register_downloader(downloader, auth)
+ finally:
+ stubber.deactivate()
+ return product
- @mock.patch(
- "eodag.plugins.authentication.aws_auth.AwsAuth.get_s3_client", autospec=True
- )
@mock.patch(
"eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True
)
- def test_plugins_search_creodias_s3_links(self, mock_request, mock_s3_client):
- # s3 links should be added to products with register_downloader
- search_plugin = self.get_search_plugin("S1_SAR_GRD", self.provider)
- client = boto3.client("s3", aws_access_key_id="a", aws_secret_access_key="b")
- mock_s3_client.return_value = client
- stubber = Stubber(client)
- s3_response_file = (
- Path(TEST_RESOURCES_PATH) / "provider_responses/creodias_s3_objects.json"
+ def test_plugins_search_creodias_s3_links(self, mock_request):
+ """Registering a product must add its S3 object links as assets."""
+ product = self._register_product_with_stubbed_s3_objects(
+ self._get_search_results(mock_request).data[0], self.s3_objects
)
- with open(s3_response_file) as f:
- list_objects_response = json.load(f)
- creodias_search_result_file = (
- Path(TEST_RESOURCES_PATH) / "eodag_search_result_creodias.json"
- )
- with open(creodias_search_result_file) as f:
- creodias_search_result = json.load(f)
- mock_request.return_value = MockResponse(creodias_search_result, 200)
- res = search_plugin.query(collection="S1_SAR_GRD")
- for product in res.data:
- download_plugin = self.plugins_manager.get_download_plugin(product)
- auth_plugin = self.plugins_manager.get_auth_plugin(download_plugin, product)
- stubber.add_response("list_objects", list_objects_response)
- stubber.activate()
- # fails if credentials are missing
- auth_plugin.config.credentials = {
- "aws_access_key_id": "",
- "aws_secret_access_key": "",
- }
- with self.assertRaisesRegex(
- MisconfiguredError,
- r"^Incomplete credentials .* \['aws_access_key_id', 'aws_secret_access_key'\]$",
- ):
- product.register_downloader(download_plugin, auth_plugin)
- auth_plugin.config.credentials = {
- "aws_access_key_id": "foo",
- "aws_secret_access_key": "bar",
- }
- product.register_downloader(download_plugin, auth_plugin)
- assets = res.data[0].assets
+ assets = product.assets
self.assertEqual(3, len(assets))
- # check if s3 links have been created correctly
for asset in assets.values():
- self.assertIn("s3://eodata/Sentinel-1/SAR/GRD/2014/10/10", asset["href"])
-
- # no occur should occur and assets should be empty if list_objects does not have content
- # (this situation will occur if the product does not have assets but is a tar file)
- stubber.add_response("list_objects", {})
- download_plugin = self.plugins_manager.get_download_plugin(res.data[0])
- auth_plugin = self.plugins_manager.get_auth_plugin(download_plugin, res.data[0])
- res.data[0].driver = None
- res.data[0].assets = AssetsDict(res.data[0])
- res.data[0].register_downloader(download_plugin, auth_plugin)
- self.assertIsNotNone(res.data[0].driver)
- self.assertEqual(0, len(res.data[0].assets))
+ self.assertTrue(asset["href"].startswith(product.remote_location))
@mock.patch(
- "eodag.plugins.authentication.aws_auth.AwsAuth.get_s3_client", autospec=True
+ "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True
)
+ def test_plugins_search_creodias_s3_links_empty(self, mock_request):
+ """Registering a product with no S3 objects must leave assets empty."""
+ product = self._get_search_results(mock_request).data[0]
+ product.driver = None
+ product.assets = AssetsDict(product)
+
+ self._register_product_with_stubbed_s3_objects(product, [])
+
+ self.assertIsNotNone(product.driver)
+ self.assertEqual(0, len(product.assets))
+
@mock.patch(
"eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True
)
- def test_plugins_search_creodias_s3_client_error(
- self, mock_request, mock_s3_client
- ):
- # request error should be raised when there is an error when fetching data from the s3
- search_plugin = self.get_search_plugin("S1_SAR_GRD", self.provider)
- client = boto3.client("s3", aws_access_key_id="a", aws_secret_access_key="b")
- mock_s3_client.return_value = client
- stubber = Stubber(client)
+ def test_plugins_search_creodias_s3_links_require_credentials(self, mock_request):
+ """Registering a Creodias S3 product requires complete AWS credentials."""
+ product = self._get_search_results(mock_request).data[0]
+ downloader = self.plugins_manager.get_download_plugin(product)
+ auth = self.plugins_manager.get_auth_plugin(downloader, product)
+ auth.config.credentials = {"aws_access_key_id": "", "aws_secret_access_key": ""}
+
+ with self.assertRaisesRegex(
+ MisconfiguredError,
+ r"^Incomplete credentials .* \['aws_access_key_id', 'aws_secret_access_key'\]$",
+ ):
+ product.register_downloader(downloader, auth)
- creodias_search_result_file = (
- Path(TEST_RESOURCES_PATH) / "eodag_search_result_creodias.json"
+ @mock.patch(
+ "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True
+ )
+ def test_plugins_search_creodias_s3_client_error(self, mock_request):
+ # request error should be raised when there is an error when fetching data from the s3
+ product = self._get_search_results(mock_request).data[0]
+ downloader, auth = self._get_downloader_and_auth(product)
+ bucket, prefix = product.remote_location.removeprefix("s3://").split("/", 1)
+ stubber = Stubber(auth.s3_resource.meta.client)
+ stubber.add_client_error(
+ "list_objects", expected_params={"Bucket": bucket, "Prefix": prefix}
)
- with open(creodias_search_result_file) as f:
- creodias_search_result = json.load(f)
- mock_request.return_value = MockResponse(creodias_search_result, 200)
with self.assertRaises(NotAvailableError):
- res = search_plugin.query(collection="S1_SAR_GRD")
- for product in res.data:
- download_plugin = self.plugins_manager.get_download_plugin(product)
- auth_plugin = self.plugins_manager.get_auth_plugin(
- download_plugin, product
- )
- auth_plugin.config.credentials = {
- "aws_access_key_id": "foo",
- "aws_secret_access_key": "bar",
- }
- stubber.add_client_error("list_objects")
- stubber.activate()
- product.register_downloader(download_plugin, auth_plugin)
+ stubber.activate()
+ try:
+ product.register_downloader(downloader, auth)
+ finally:
+ stubber.deactivate()
class TestSearchPluginECMWFSearch(unittest.TestCase):
=====================================
tests/units/test_utils_s3.py
=====================================
@@ -21,6 +21,7 @@ from tests.context import (
_compute_file_ranges,
_prepare_file_in_zip,
file_position_from_s3_zip,
+ list_files_in_s3_prefix,
list_files_in_s3_zipped_object,
open_s3_zipped_object,
stream_download_from_s3,
@@ -177,6 +178,23 @@ class TestUtilsS3(TestCase):
self.assertIsInstance(cd_data, bytes)
self.assertEqual(len(zip_file.filelist), 6)
+ def test_utils_s3_list_files_in_s3_prefix(self):
+ """list_files_in_s3_prefix must exclude directory markers and siblings."""
+ self.s3_client.put_object(Bucket="mybucket", Key="test-prefix", Body=b"")
+ self.s3_client.put_object(Bucket="mybucket", Key="test-prefix/", Body=b"")
+ self.s3_client.put_object(
+ Bucket="mybucket", Key="test-prefix/file.txt", Body=b"content"
+ )
+ self.s3_client.put_object(
+ Bucket="mybucket", Key="test-prefix.txt", Body=b"sibling"
+ )
+
+ objects = list_files_in_s3_prefix(
+ "test-prefix", self.auth_plugin.get_bucket_objects("mybucket")
+ )
+
+ self.assertListEqual([item.key for item in objects], ["test-prefix/file.txt"])
+
def test_utils_s3_update_assets_from_s3_zip(self):
"""update_assets_from_s3 must update the assets of a product from a zipped object stored in S3"""
update_assets_from_s3(
View it on GitLab: https://salsa.debian.org/debian-gis-team/eodag/-/compare/9db5345dbfbbacd4a33d4ec0b6528c855645c0ef...54d8796b001e1e0defee73bd91330263a075ef0e
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/eodag/-/compare/9db5345dbfbbacd4a33d4ec0b6528c855645c0ef...54d8796b001e1e0defee73bd91330263a075ef0e
You're receiving this email because of your account on salsa.debian.org. Manage all notifications: https://salsa.debian.org/-/profile/notifications | Help: https://salsa.debian.org/help
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20260925/50ada52b/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list