[Git][debian-gis-team/eodag][upstream] New upstream version 4.7.2+ds
Antonio Valentino (@antonio.valentino)
gitlab at salsa.debian.org
Fri Aug 28 19:41:03 BST 2026
Antonio Valentino pushed to branch upstream at Debian GIS Project / eodag
Commits:
5229ace2 by Antonio Valentino at 2026-08-28T16:09:10+00:00
New upstream version 4.7.2+ds
- - - - -
26 changed files:
- .github/workflows/benchmark.yml
- CHANGES.rst
- eodag/config.py
- eodag/plugins/authentication/base.py
- eodag/plugins/download/aws.py
- eodag/plugins/manager.py
- eodag/plugins/search/csw.py
- eodag/plugins/search/qssearch.py
- eodag/resources/ext_collections.json
- eodag/resources/providers/cop_dataspace_s3.yml
- eodag/resources/providers/creodias_s3.yml
- eodag/resources/providers/fedeo_ceda.yml
- eodag/resources/user_conf_template.yml
- eodag/utils/__init__.py
- pyproject.toml
- tests/context.py
- tests/integration/test_core_search.py
- tests/test_cli.py
- tests/units/test_auth_plugins.py
- tests/units/test_core.py
- tests/units/test_download_plugins.py
- tests/units/test_search_plugins.py
- tests/units/test_search_types.py
- tests/units/test_stac_reader.py
- tests/units/test_utils.py
- utils/ext_product_types_cmp.py
Changes:
=====================================
.github/workflows/benchmark.yml
=====================================
@@ -2,7 +2,6 @@ name: Run benchmark
on:
pull_request:
- branches: [develop, v5_download_link_asset]
permissions:
contents: read
@@ -26,15 +25,17 @@ jobs:
cache-dependency-glob: |
pyproject.toml
tox.ini
- - name: Run benchmark on develop and current ref, then compare
+ - name: Run benchmark on target branch and current ref, then compare
shell: bash
+ env:
+ TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
run: |
set -euo pipefail
CURRENT_SHA="$(git rev-parse HEAD)"
- echo "Running benchmark on develop"
- git checkout --detach origin/develop
+ echo "Running benchmark on ${TARGET_BRANCH}"
+ git checkout --detach "origin/${TARGET_BRANCH}"
uvx --python 3.14 --with tox-uv tox -e benchmark -- --benchmark-json baseline.json
echo "Running benchmark on current ref"
@@ -51,7 +52,7 @@ jobs:
{
echo "## Benchmark comparison"
echo
- echo "Compared baseline: \`origin/develop\`"
+ echo "Compared baseline: \`origin/${TARGET_BRANCH}\`"
echo "Compared candidate: ${CURRENT_SHA}"
echo
echo '```text'
=====================================
CHANGES.rst
=====================================
@@ -3,6 +3,55 @@ Release history
===============
+v4.7.2 (2026-08-28)
+===================
+
+Bug Fixes
+---------
+
+* **plugins**: CSWSearch products build and supported geometries (`#2325`_, `a00249b`_)
+
+* **plugins**: Retry PostJsonSearch requests (`#2328`_, `8edf378`_)
+
+* **providers**: Cop_dataspace_s3 and creodias_s3 auth matching url (`#2332`_, `f368e7c`_)
+
+* **providers**: Disable ssl verify on fedeo_ceda (`#2331`_, `5ddd4e4`_)
+
+Continuous Integration
+----------------------
+
+* Always use PR target branch as benchmark reference (`#2317`_, `fafcb36`_)
+
+* Empty fetched product types fix (`#2333`_, `8239d4b`_)
+
+Refactoring
+-----------
+
+* **config**: Simplify user config template (`#2321`_, `c808210`_)
+
+Testing
+-------
+
+* Add more tests for core, plugins and utils (`#2326`_, `df8bb25`_)
+
+.. _#2317: https://github.com/CS-SI/eodag/pull/2317
+.. _#2321: https://github.com/CS-SI/eodag/pull/2321
+.. _#2325: https://github.com/CS-SI/eodag/pull/2325
+.. _#2326: https://github.com/CS-SI/eodag/pull/2326
+.. _#2328: https://github.com/CS-SI/eodag/pull/2328
+.. _#2331: https://github.com/CS-SI/eodag/pull/2331
+.. _#2332: https://github.com/CS-SI/eodag/pull/2332
+.. _#2333: https://github.com/CS-SI/eodag/pull/2333
+.. _5ddd4e4: https://github.com/CS-SI/eodag/commit/5ddd4e41011bc74488eef6f6f2e6382f22db72e3
+.. _8239d4b: https://github.com/CS-SI/eodag/commit/8239d4b2e2a00c1208fb61e800a40c16772078a7
+.. _8edf378: https://github.com/CS-SI/eodag/commit/8edf378467ce7e20af7acf6582923e78121bdef6
+.. _a00249b: https://github.com/CS-SI/eodag/commit/a00249b42fbec89345f902b3ecf1560a0b576052
+.. _c808210: https://github.com/CS-SI/eodag/commit/c8082108834159c83b21fda94048ddcc952dc4b7
+.. _df8bb25: https://github.com/CS-SI/eodag/commit/df8bb2590f8edcc673200f135bb2c46d1bf834d1
+.. _f368e7c: https://github.com/CS-SI/eodag/commit/f368e7c11f8017a0247903b667afe369b0ceb91a
+.. _fafcb36: https://github.com/CS-SI/eodag/commit/fafcb3663925e5fd8b85d5d8845b787bc00b81c3
+
+
v4.7.1 (2026-08-20)
===================
=====================================
eodag/config.py
=====================================
@@ -647,7 +647,11 @@ class PluginConfig(yaml.YAMLObject):
)
def matches_target_auth(self, target_config: Self):
- """Check if the target auth configuration matches this one"""
+ """Check whether this auth configuration can share credentials with another.
+
+ Matching criteria must be identical: when both ``matching_url`` and
+ ``matching_conf`` are configured, both must match.
+ """
target_matching_conf = getattr(target_config, "matching_conf", {})
target_matching_url = getattr(target_config, "matching_url", None)
=====================================
eodag/plugins/authentication/base.py
=====================================
@@ -36,10 +36,13 @@ class Authentication(PluginTopic):
:param provider: provider name
:param config: Authentication plugin configuration:
- * :attr:`~eodag.config.PluginConfig.matching_url` (``str``): URL pattern to match with search plugin endpoint or
- download link
+ * :attr:`~eodag.config.PluginConfig.matching_url` (``str``): URL pattern to match with a search plugin endpoint
+ or product download link
* :attr:`~eodag.config.PluginConfig.matching_conf` (``dict[str, Any]``): Part of the search or download plugin
- configuration that needs authentication and helps identifying it
+ configuration that needs authentication and helps identifying it.
+
+ At runtime, an auth plugin matches when either criterion matches.
+ When credentials are shared between provider configurations, all configured matching criteria must be identical.
"""
entrypoint_group = "auth"
=====================================
eodag/plugins/download/aws.py
=====================================
@@ -806,8 +806,10 @@ class AwsDownload(Download):
if flatten_top_dirs:
rel_path = os.path.join(
product.properties["title"],
- re.sub(rf"^{common_path}/?", "", rel_path),
+ re.sub(rf"^{re.escape(common_path)}/?", "", rel_path),
)
+ # Normalize to forward slashes for S3 path consistency across platforms
+ rel_path = rel_path.replace("\\", "/")
asset_match = assets_by_path.get(f"{obj.bucket_name}/{obj.key}")
data_type = asset_match.get("type") if asset_match else None
@@ -847,7 +849,8 @@ class AwsDownload(Download):
chunk_paths.append(
self.get_chunk_dest_path(product, product_chunk, build_safe=build_safe)
)
- return os.path.commonpath(chunk_paths)
+ # Normalize to forward slashes for S3 path consistency across platforms
+ return os.path.commonpath(chunk_paths).replace("\\", "/")
def get_product_bucket_name_and_prefix(
self, product: EOProduct, url: Optional[str] = None
=====================================
eodag/plugins/manager.py
=====================================
@@ -319,13 +319,16 @@ class PluginManager:
matching_url: Optional[str] = None,
matching_conf: Optional[PluginConfig] = None,
) -> Iterator[Authentication]:
- """Build and return the authentication plugin for the given collection and
- provider
+ """Build and return authentication plugins matching the given criteria.
+
+ An auth plugin matches when either its ``matching_url`` pattern matches
+ ``matching_url`` or its ``matching_conf`` is a subset of ``matching_conf``.
+ The requested provider is considered first, followed by other providers.
:param provider: The provider for which to get the authentication plugin
:param matching_url: url to compare with plugin matching_url pattern
:param matching_conf: configuration to compare with plugin matching_conf
- :returns: All the Authentication plugins for the given criteria
+ :returns: Authentication plugins matching the given criteria
"""
auth_conf: Optional[PluginConfig] = None
=====================================
eodag/plugins/search/csw.py
=====================================
@@ -199,14 +199,16 @@ class CSWSearch(Search):
self.config.search_definition.get("resource_location_filter", "")
)
for ref in rec.references:
- if ref["scheme"] in SUPPORTED_REFERENCE_SCHEMES:
- if resource_filter.pattern and resource_filter.search(ref["url"]):
- download_url = ref["url"]
- else:
- download_url = ref["url"] # noqa
- break
+ if ref["scheme"] not in SUPPORTED_REFERENCE_SCHEMES:
+ continue
+ if resource_filter.pattern and not resource_filter.search(ref["url"]):
+ continue
+ download_url = ref["url"]
+ break
properties = properties_from_xml(rec.xml, self.config.metadata_mapping)
- if not properties["geometry"]:
+ if download_url:
+ properties.setdefault("eodag:download_link", download_url)
+ if not properties.get("geometry"):
bbox = rec.bbox_wgs84
if not bbox:
code = "EPSG:4326"
@@ -262,9 +264,11 @@ class CSWSearch(Search):
# `footprint`
fp = params.get("geometry")
if fp:
- constraints.append(
- BBox([fp["lonmin"], fp["latmin"], fp["lonmax"], fp["latmax"]])
- )
+ if hasattr(fp, "bounds"):
+ bbox = fp.bounds
+ else:
+ bbox = [fp["lonmin"], fp["latmin"], fp["lonmax"], fp["latmax"]]
+ constraints.append(BBox(bbox))
# dates
start, end = (
=====================================
eodag/plugins/search/qssearch.py
=====================================
@@ -2025,6 +2025,13 @@ class PostJsonSearch(QueryStringSearch):
exception_message = prep.exception_message
timeout = getattr(self.config, "timeout", DEFAULT_SEARCH_TIMEOUT)
ssl_verify = getattr(self.config, "ssl_verify", True)
+ retry_total = getattr(self.config, "retry_total", REQ_RETRY_TOTAL)
+ retry_backoff_factor = getattr(
+ self.config, "retry_backoff_factor", REQ_RETRY_BACKOFF_FACTOR
+ )
+ retry_status_forcelist = getattr(
+ self.config, "retry_status_forcelist", REQ_RETRY_STATUS_FORCELIST
+ )
try:
# auth if needed
RequestsKwargs = TypedDict(
@@ -2054,7 +2061,16 @@ class PostJsonSearch(QueryStringSearch):
logger.debug("Query kwargs: %s" % geojson.dumps(kwargs))
except TypeError:
logger.debug("Query kwargs: %s" % kwargs)
- response = requests.post(
+ session = requests.Session()
+ retries = Retry(
+ total=retry_total,
+ backoff_factor=retry_backoff_factor,
+ status_forcelist=retry_status_forcelist,
+ allowed_methods={"POST"},
+ raise_on_status=False,
+ )
+ session.mount(url, HTTPAdapter(max_retries=retries))
+ response = session.post(
url,
json=prep.query_params,
headers=USER_AGENT,
=====================================
eodag/resources/ext_collections.json
=====================================
The diff for this file was not included because it is too large.
=====================================
eodag/resources/providers/cop_dataspace_s3.yml
=====================================
@@ -251,8 +251,7 @@ cop_dataspace_s3:
auth_error_code: 403
s3_endpoint: 'https://eodata.dataspace.copernicus.eu'
support_presign_url: False
- matching_conf:
- s3_endpoint: 'https://eodata.dataspace.copernicus.eu'
+ matching_url: 's3://eodata'
products:
# S2
S2_MSI_L1C:
=====================================
eodag/resources/providers/creodias_s3.yml
=====================================
@@ -251,8 +251,7 @@ creodias_s3:
auth_error_code: 403
s3_endpoint: 'https://eodata.cloudferro.com'
support_presign_url: False
- matching_conf:
- s3_endpoint: 'https://eodata.cloudferro.com'
+ matching_url: 's3://eodata'
products:
# S1
S1_AUX_GNSSRD:
=====================================
eodag/resources/providers/fedeo_ceda.yml
=====================================
@@ -7,8 +7,9 @@ fedeo_ceda:
search:
type: StacSearch
api_endpoint: 'https://fedeo.ceos.org/search'
- ssl_verify: true
+ ssl_verify: false
timeout: 60
+ retry_status_forcelist: [400, 401, 429, 500, 502, 503, 504]
pagination:
next_page_url_tpl: '{url}?startRecord={next_page_token}'
next_page_query_obj: '{{"limit":{limit}}}'
=====================================
eodag/resources/user_conf_template.yml
=====================================
@@ -15,9 +15,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+# Set priority and credentials here. See the documentation for the complete
+# provider configuration.
aws_eos:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
search_auth:
credentials:
api_key:
@@ -26,70 +27,39 @@ aws_eos:
aws_access_key_id:
aws_secret_access_key:
aws_profile:
- download:
- output_dir:
cop_ads:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- extract:
- output_dir:
auth:
credentials:
apikey:
cop_cds:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- extract:
- output_dir:
auth:
credentials:
apikey:
cop_dataspace:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- extract:
- output_dir:
auth:
credentials:
username:
password:
cop_dataspace_s3:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- output_dir:
auth:
credentials:
aws_access_key_id:
aws_secret_access_key:
cop_ewds:
priority: # Lower value means lower priority (Default: 0)
- download:
- output_dir:
auth:
credentials:
apikey:
cop_ghsl:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- extract:
- output_dir:
cop_marine:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- extract:
- output_dir:
creodias:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- extract:
- output_dir:
auth:
credentials:
username:
@@ -97,81 +67,56 @@ creodias:
totp: # set totp dynamically, see https://eodag.readthedocs.io/en/latest/getting_started_guide/configure.html#authenticate-using-an-otp-one-time-password-two-factor-authentication
creodias_s3:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- output_dir:
auth:
credentials:
aws_access_key_id:
aws_secret_access_key:
dedl:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- output_dir:
auth:
credentials:
username:
password:
dedt_lumi:
priority: # Lower value means lower priority (Default: 0)
- search:
- download:
- output_dir:
auth:
credentials:
username:
password:
dedt_mn5:
priority: # Lower value means lower priority (Default: 0)
- search:
- download:
- output_dir:
auth:
credentials:
username:
password:
dedt_leonardo:
priority: # Lower value means lower priority (Default: 0)
- search:
- download:
- output_dir:
auth:
credentials:
username:
password:
dlr_eoc_geoservice:
priority: # Lower value means lower priority (Default: 0)
- search:
- download:
- output_dir:
auth:
credentials:
username:
password:
earth_search:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
auth:
credentials:
aws_access_key_id:
aws_secret_access_key:
aws_profile:
- download:
- output_dir:
earth_search_gcs:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
auth:
credentials:
aws_access_key_id:
aws_secret_access_key:
- download:
- output_dir:
ecmwf:
priority: # Lower value means lower priority (Default: 0)
api:
- output_dir:
credentials:
username:
password:
@@ -183,78 +128,47 @@ eocat:
password:
eumetsat_ds:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- output_dir:
auth:
credentials:
username:
password:
- download:
- output_dir:
fedeo_ceda:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- extract:
- output_dir:
geodes:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
auth:
credentials:
apikey:
- download:
- extract:
- output_dir:
geodes_s3:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
auth:
credentials:
aws_access_key_id:
aws_secret_access_key:
aws_session_token:
- download:
- extract:
- output_dir:
hydroweb_next:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
auth:
credentials:
apikey:
- download:
- output_dir:
meteoblue:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
auth:
credentials:
apikey:
- download:
- output_dir:
planetary_computer:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
auth:
credentials:
apikey:
- download:
- output_dir:
sara:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- extract:
- output_dir:
- delete_archive:
auth:
credentials:
username:
password:
theia:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
auth:
credentials:
access_key:
@@ -262,46 +176,30 @@ theia:
usgs:
priority: # Lower value means lower priority (Default: 0)
api:
- extract:
- output_dir:
- dl_url_params:
- product_location_scheme:
credentials:
username:
password:
usgs_satapi_aws:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
auth:
credentials:
aws_access_key_id:
aws_secret_access_key:
aws_profile:
- download:
- output_dir:
wekeo_cmems:
priority: # Lower value means lower priority (Default: 0)
- search:
- download:
- output_dir:
auth:
credentials:
username:
password:
wekeo_ecmwf:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- output_dir:
auth:
credentials:
username:
password:
wekeo_main:
priority: # Lower value means lower priority (Default: 0)
- search: # Search parameters configuration
- download:
- output_dir:
auth:
credentials:
username:
=====================================
eodag/utils/__init__.py
=====================================
@@ -424,6 +424,11 @@ def mutate_dict_in_place(func: Callable[[Any], Any], mapping: dict[Any, Any]) ->
allowing to also modify values of nested dicts that may be level-1 values of
mapping.
+ >>> mapping = {"a": 1, "nested": {"b": 2}}
+ >>> mutate_dict_in_place(lambda value: value * 10, mapping)
+ >>> mapping
+ {'a': 10, 'nested': {'b': 20}}
+
:param func: A function to apply to each value of mapping which is not a dict object
:param mapping: A Python dict object
:returns: None
@@ -506,7 +511,14 @@ def merge_mappings(mapping1: dict[Any, Any], mapping2: dict[Any, Any]) -> None:
def maybe_generator(obj: Any) -> Iterator[Any]:
"""Generator function that get an arbitrary object and generate values from it if
- the object is a generator."""
+ the object is a generator.
+
+ >>> list(maybe_generator(value for value in (1, 2, 3)))
+ [1, 2, 3]
+ >>> list(maybe_generator("value"))
+ ['value']
+ """
+
if isinstance(obj, types.GeneratorType):
for elt in obj:
yield elt
@@ -1003,6 +1015,8 @@ def string_to_jsonpath(*args: Any, force: bool = False) -> Union[str, JSONPath]:
... Slice(start=None, end=None, step=None),
... )
True
+ >>> string_to_jsonpath("$.foo[bar]")
+ Child(Child(Root(), Fields('foo')), Fields('bar'))
:param args: Last arg as input string value, to be converted
:param force: force conversion even if input string is not detected as a :class:`jsonpath_ng.JSONPath`
@@ -1655,6 +1669,8 @@ def guess_file_type(file: str) -> str:
'image/tiff'
>>> guess_file_type('foo.grib')
'application/x-grib'
+ >>> guess_file_type('foo.unknown_eodag_extension')
+ 'application/octet-stream'
:param file: file url or path
:returns: guessed mime type
@@ -1828,6 +1844,9 @@ def get_collection_dates(
>>> get_collection_dates({})
(None, None)
+
+ >>> get_collection_dates({"extent": {"temporal": {"interval": []}}})
+ (None, None)
"""
extent_interval = (
collection_dict.get("extent", {})
=====================================
pyproject.toml
=====================================
@@ -211,7 +211,7 @@ include = ["eodag*"]
"*" = ["LICENSE", "NOTICE", "py.typed"]
[tool.setuptools_scm]
-fallback_version = "4.7.2.dev0"
+fallback_version = "4.7.3.dev0"
[tool.isort]
multi_line_output = 3
=====================================
tests/context.py
=====================================
@@ -45,21 +45,21 @@ from eodag.api.product.metadata_mapping import (
from eodag.api.collection import Collection, CollectionsDict, CollectionsList
from eodag.api.search_result import SearchResult
from eodag.cli import download, eodag_cli, list_col, search_crunch
+from eodag.api.provider import Provider, ProviderConfig, ProvidersDict
from eodag.config import (
- load_default_config,
- load_stac_provider_config,
- get_ext_collections_conf,
AUTH_TOPIC_KEYS,
EXT_COLLECTIONS_CONF_URI,
+ PluginConfig,
+ get_ext_collections_conf,
+ load_default_config,
+ load_stac_provider_config,
)
-from eodag.api.provider import ProviderConfig, ProvidersDict, Provider
-from eodag.config import PluginConfig
from eodag.plugins.apis.ecmwf import EcmwfApi
from eodag.plugins.authentication.base import Authentication
-from eodag.plugins.authentication.aws_auth import AwsAuth
-from eodag.plugins.authentication.header import HeaderAuth
+from eodag.plugins.authentication.aws_auth import AwsAuth, raise_if_auth_error
+from eodag.plugins.authentication.header import HeaderAuth, HTTPHeaderAuth
from eodag.plugins.authentication.openid_connect import CodeAuthorizedAuth
-from eodag.plugins.authentication.header import HTTPHeaderAuth
+from eodag.plugins.authentication.token_exchange import OIDCTokenExchangeAuth
from eodag.plugins.authentication.qsauth import HttpQueryStringAuth
from eodag.plugins.base import PluginTopic
from eodag.plugins.crunch.filter_date import FilterDate
@@ -73,10 +73,12 @@ from eodag.plugins.download.base import (
)
from eodag.plugins.download.http import HTTPDownload
from eodag.plugins.manager import PluginManager
-from eodag.plugins.search import PreparedSearch
from eodag.plugins.search.base import Search
from eodag.plugins.search.build_search_result import ecmwf_temporal_to_eodag
+from eodag.plugins.search.csw import CSWSearch
+from eodag.plugins.search import PreparedSearch
from eodag.plugins.search.qssearch import QueryStringSearch
+from eodag.types.bbox import BBox
from eodag.types import model_fields_to_annotated
from eodag.types.queryables import CommonQueryables, Queryables, QueryablesDict
from eodag.utils import (
@@ -87,6 +89,8 @@ from eodag.utils import (
DEFAULT_SEARCH_TIMEOUT,
USER_AGENT,
get_bucket_name_and_prefix,
+ get_geometry_from_ecmwf_area,
+ get_geometry_from_ecmwf_feature,
get_geometry_from_various,
makedirs,
merge_mappings,
@@ -109,7 +113,9 @@ from eodag.utils import (
from eodag.utils.yaml import cached_yaml_load_all
from eodag.utils.dates import get_timestamp, to_iso_utc_string
from eodag.utils.env import is_env_var_true
-from eodag.utils.requests import fetch_json
+from eodag.utils.requests import LocalFileAdapter, fetch_json
+from eodag.utils.import_system import import_all_modules, patch_owslib_requests
+from eodag.utils.notebook import NotebookWidgets, check_ipython, check_notebook
from eodag.utils.s3 import (
list_files_in_s3_zipped_object,
update_assets_from_s3,
@@ -142,5 +148,5 @@ from eodag.utils.exceptions import (
)
from eodag.utils.stac_reader import fetch_stac_items, _TextOpener
from tests import TEST_RESOURCES_PATH
-from usgs.api import USGSAuthExpiredError, USGSError
from usgs.api import TMPFILE as USGS_TMPFILE
+from usgs.api import USGSAuthExpiredError, USGSError
=====================================
tests/integration/test_core_search.py
=====================================
@@ -95,7 +95,7 @@ class TestCoreSearch(unittest.TestCase):
autospec=True,
)
@mock.patch(
- "eodag.plugins.search.qssearch.requests.post",
+ "eodag.plugins.search.qssearch.requests.Session.post",
autospec=True,
side_effect=RequestException,
)
@@ -127,7 +127,7 @@ class TestCoreSearch(unittest.TestCase):
autospec=True,
)
@mock.patch(
- "eodag.plugins.search.qssearch.requests.post",
+ "eodag.plugins.search.qssearch.requests.Session.post",
autospec=True,
side_effect=RequestException,
)
@@ -230,7 +230,7 @@ class TestCoreSearch(unittest.TestCase):
autospec=True,
)
@mock.patch(
- "eodag.plugins.search.qssearch.requests.post",
+ "eodag.plugins.search.qssearch.requests.Session.post",
autospec=True,
side_effect=RequestException,
)
@@ -272,7 +272,7 @@ class TestCoreSearch(unittest.TestCase):
side_effect=RequestException,
)
@mock.patch(
- "eodag.plugins.search.qssearch.requests.post",
+ "eodag.plugins.search.qssearch.requests.Session.post",
autospec=True,
side_effect=RequestException,
)
@@ -822,7 +822,7 @@ class TestCoreSearch(unittest.TestCase):
self.assertEqual("DEDT_LUMI_123-456", result[0].properties["title"])
@mock.patch(
- "eodag.plugins.search.qssearch.requests.post",
+ "eodag.plugins.search.qssearch.requests.Session.post",
autospec=True,
)
@mock.patch(
=====================================
tests/test_cli.py
=====================================
@@ -57,12 +57,13 @@ class TestEodagCli(unittest.TestCase):
@contextmanager
def user_conf(self, conf_file="user_conf.yml", content=b"key: to unused conf"):
"""Utility method"""
- with self.runner.isolated_filesystem():
- with open(conf_file, "wb") as fd:
+ with TemporaryDirectory() as tmp_dir:
+ conf_file_path = os.path.join(tmp_dir, conf_file)
+ with open(conf_file_path, "wb") as fd:
fd.write(
content if isinstance(content, bytes) else content.encode("utf-8")
)
- yield conf_file
+ yield conf_file_path
def setUp(self):
super(TestEodagCli, self).setUp()
=====================================
tests/units/test_auth_plugins.py
=====================================
@@ -20,6 +20,7 @@ import datetime as dt
import pickle
import unittest
from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
from unittest import mock
import boto3
@@ -34,16 +35,19 @@ from requests.exceptions import RequestException
from eodag.api.product._product import EOProduct
from eodag.api.provider import ProvidersDict
from eodag.plugins.authentication.eoiam import _EOIAMSessionAuth
-from eodag.plugins.authentication.openid_connect import CodeAuthorizedAuth
from eodag.utils import MockResponse
-from eodag.utils.exceptions import RequestError
from tests.context import (
HTTP_REQ_TIMEOUT,
USER_AGENT,
AuthenticationError,
+ CodeAuthorizedAuth,
HeaderAuth,
MisconfiguredError,
+ OIDCTokenExchangeAuth,
PluginManager,
+ RequestError,
+ TimeOutError,
+ raise_if_auth_error,
)
@@ -885,6 +889,59 @@ class TestAuthPluginAwsAuth(BaseAuthPluginTest):
self.assertIn("Expires", url)
+class TestAuthPluginTokenExchange(unittest.TestCase):
+ def test_plugins_auth_oidc_token_exchange_success_and_timeout(self):
+ """OIDC token exchange returns a token and closes sessions on timeout."""
+ plugin = OIDCTokenExchangeAuth.__new__(OIDCTokenExchangeAuth)
+ response = mock.Mock()
+ response.json.return_value = {"access_token": "target-token"}
+ session = mock.Mock()
+ session.post.return_value = response
+ plugin.subject = mock.Mock(
+ session=session,
+ authenticate=mock.Mock(
+ return_value=CodeAuthorizedAuth("subject-token", where="header")
+ ),
+ )
+ plugin.config = SimpleNamespace(
+ subject_issuer="issuer",
+ token_uri="https://token.test",
+ client_id="client",
+ audience="audience",
+ token_key="access_token",
+ ssl_verify=False,
+ )
+ result = plugin.authenticate()
+ self.assertEqual(result.token, "target-token")
+ session.close.assert_called_once_with()
+ self.assertEqual(
+ session.post.call_args.kwargs["data"]["grant_type"], plugin.GRANT_TYPE
+ )
+
+ session.post.side_effect = requests.exceptions.Timeout()
+ with self.assertRaises(TimeOutError):
+ plugin.authenticate()
+ self.assertEqual(session.close.call_count, 2)
+
+
+class TestAwsAuthHelpers(unittest.TestCase):
+ def test_plugins_auth_aws_auth_error_only_for_known_credential_errors(self):
+ """AWS credential errors raise only for recognized credential messages."""
+ from botocore.exceptions import ClientError
+
+ error = ClientError(
+ {
+ "Error": {"Code": "AccessDenied", "Message": "bad key"},
+ "ResponseMetadata": {"HTTPStatusCode": 403},
+ },
+ "GetObject",
+ )
+ with self.assertRaises(AuthenticationError):
+ raise_if_auth_error(error, "provider")
+ error.response["Error"]["Message"] = "not a credential problem"
+ raise_if_auth_error(error, "provider")
+
+
class TestAuthPluginEOIAMAuth(BaseAuthPluginTest):
@classmethod
def setUpClass(cls):
=====================================
tests/units/test_core.py
=====================================
@@ -47,6 +47,7 @@ from tests import TEST_RESOURCES_PATH, TEST_RESOURCES_PROVIDERS_PATH
from tests.context import (
DEFAULT_LIMIT,
DEFAULT_MAX_LIMIT,
+ AuthenticationError,
CommonQueryables,
EODataAccessGateway,
EOProduct,
@@ -1186,6 +1187,28 @@ class TestCore(TestCoreBase):
self.dag.update_collections_list(ext_collections_conf)
self.assertNotIn("earth_search", self.dag._providers)
+ def test_update_collections_list_unsupported_provider(self):
+ """Core api.update_collections_list must ignore providers raising UnsupportedProvider"""
+ with open(os.path.join(TEST_RESOURCES_PATH, "ext_collections.json")) as f:
+ ext_collections_conf = json.load(f)
+ provider_conf = self.dag._providers.configs["earth_search"]
+ with (
+ mock.patch.object(
+ self.dag._providers.__class__,
+ "__getitem__",
+ side_effect=UnsupportedProvider("earth_search"),
+ ),
+ mock.patch.object(
+ self.dag._plugins_manager,
+ "build_collection_to_provider_config_map",
+ ),
+ ):
+ self.dag.update_collections_list(ext_collections_conf)
+
+ self.assertIs(self.dag._providers.configs["earth_search"], provider_conf)
+ self.assertNotIn("foo", self.dag.collections_config)
+ self.assertNotIn("bar", self.dag.collections_config)
+
@mock.patch(
"eodag.plugins.search.qssearch.QueryStringSearch.discover_collections",
autospec=True,
@@ -1617,6 +1640,22 @@ class TestCore(TestCoreBase):
os.environ.pop("EODAG__SARA__SEARCH__NEED_AUTH", None)
os.environ.pop("EODAG__SARA__AUTH__CREDENTIALS__USERNAME", None)
+ def test_user_conf_template_keeps_priority_and_credentials(self):
+ """The user configuration template should not include runtime settings."""
+ template_path = res_files("eodag") / "resources" / "user_conf_template.yml"
+ with template_path.open(encoding="utf-8") as file:
+ user_conf = yaml.safe_load(file)
+
+ for provider_conf in user_conf.values():
+ self.assertIn("priority", provider_conf)
+ self.assertTrue(
+ set(provider_conf).issubset(
+ {"priority", "auth", "search_auth", "download_auth", "api"}
+ )
+ )
+ if "api" in provider_conf:
+ self.assertEqual(set(provider_conf["api"]), {"credentials"})
+
@mock.patch("eodag.plugins.manager.importlib_metadata.entry_points", autospec=True)
def test_prune_providers_list_skipped_plugin(self, mock_iter_ep):
"""Providers needing skipped plugin must be pruned on init"""
@@ -2195,6 +2234,50 @@ class TestCore(TestCoreBase):
self.assertIn(original_url, form_url)
mock__fetch_data.reset_mock()
+ @mock.patch(
+ "eodag.plugins.search.qssearch.StacSearch.discover_queryables",
+ autospec=True,
+ return_value={},
+ )
+ @mock.patch("eodag.plugins.manager.PluginManager.get_auth_plugin", autospec=True)
+ def test_list_queryables_logs_authentication_error(
+ self, mock_get_auth_plugin, mock_discover_queryables
+ ):
+ """list_queryables must ignore auth errors and log them at debug level"""
+ search_plugin = mock.Mock(provider="dummy_provider")
+ search_plugin.config.need_auth = True
+ search_plugin.list_queryables.return_value = QueryablesDict(
+ additional_properties=False
+ )
+ auth_plugin = mock.Mock()
+ auth_plugin.authenticate.side_effect = AuthenticationError("auth failed")
+ mock_get_auth_plugin.return_value = auth_plugin
+
+ with (
+ mock.patch.object(
+ self.dag,
+ "list_collections",
+ return_value=[mock.Mock(id="S2_MSI_L1C")],
+ ),
+ mock.patch.object(
+ self.dag,
+ "_attach_collection_config",
+ ),
+ mock.patch.object(
+ self.dag._plugins_manager,
+ "get_search_plugins",
+ return_value=[search_plugin],
+ ),
+ self.assertLogs("eodag.core", level="DEBUG") as cm,
+ ):
+ queryables = self.dag.list_queryables(provider="dummy_provider")
+
+ self.assertIsInstance(queryables, QueryablesDict)
+ self.assertIn(
+ "queryables from provider dummy_provider could not be fetched due to an authentication error",
+ str(cm.output),
+ )
+
def test_queryables_repr(self):
"""The HTML representation of queryables must be correct"""
queryables = self.dag.list_queryables(
@@ -3366,6 +3449,83 @@ class TestCoreSearch(TestCoreBase):
self.assertEqual(found.number_matched, 1)
self.assertEqual(len(found), 1)
+ @mock.patch(
+ "eodag.plugins.manager.PluginManager.get_search_plugins",
+ autospec=True,
+ )
+ def test__search_by_id_handles_plugin_exceptions(self, mock_get_search_plugins):
+ """_search_by_id must store plugin exceptions and return empty results"""
+ search_plugin = mock.Mock(provider="dummy_provider")
+ search_plugin.config.pagination = {"max_limit": 100}
+ mock_get_search_plugins.return_value = [search_plugin]
+
+ with mock.patch.object(
+ self.dag,
+ "search_iter_page_plugin",
+ side_effect=RequestError("search failed"),
+ ):
+ found = self.dag._search_by_id(uid="foo", provider="dummy_provider")
+
+ self.assertEqual(len(found), 0)
+ self.assertEqual(found.number_matched, 0)
+ self.assertEqual(len(found.errors), 1)
+ self.assertEqual(found.errors[0][0], "dummy_provider")
+ self.assertIsInstance(found.errors[0][1], RequestError)
+
+ @mock.patch(
+ "eodag.plugins.manager.PluginManager.get_search_plugins",
+ autospec=True,
+ )
+ def test__search_by_id_raises_plugin_exceptions(self, mock_get_search_plugins):
+ """_search_by_id must raise plugin exceptions when raise_errors is True"""
+ search_plugin = mock.Mock(provider="dummy_provider")
+ search_plugin.config.pagination = {"max_limit": 100}
+ mock_get_search_plugins.return_value = [search_plugin]
+
+ with mock.patch.object(
+ self.dag,
+ "search_iter_page_plugin",
+ side_effect=RequestError("search failed"),
+ ):
+ with self.assertRaises(RequestError):
+ self.dag._search_by_id(
+ uid="foo", provider="dummy_provider", raise_errors=True
+ )
+
+ @mock.patch(
+ "eodag.plugins.manager.PluginManager.get_search_plugins",
+ autospec=True,
+ )
+ def test__search_by_id_guesses_collection_and_resets_driver(
+ self, mock_get_search_plugins
+ ):
+ """_search_by_id must guess missing collection and reset product driver"""
+ search_plugin = mock.Mock(provider="dummy_provider")
+ search_plugin.config.pagination = {"max_limit": 100}
+ mock_get_search_plugins.return_value = [search_plugin]
+ product = EOProduct("dummy_provider", {"id": "foo"})
+ product.collection = None
+ initial_driver = product.driver
+
+ with (
+ mock.patch.object(
+ self.dag,
+ "search_iter_page_plugin",
+ return_value=iter([SearchResult([product], 1)]),
+ ),
+ mock.patch.object(
+ self.dag,
+ "guess_collection",
+ return_value=[mock.Mock(id="S2_MSI_L1C")],
+ ) as mock_guess_collection,
+ ):
+ found = self.dag._search_by_id(uid="foo", provider="dummy_provider")
+
+ self.assertEqual(found.number_matched, 1)
+ self.assertEqual(found[0].collection, "S2_MSI_L1C")
+ self.assertIsNot(found[0].driver, initial_driver)
+ mock_guess_collection.assert_called_once_with(**product.properties)
+
@mock.patch("eodag.plugins.search.qssearch.QueryStringSearch", autospec=True)
def test__do_search_support_itemsperpage_higher_than_maximum(self, search_plugin):
"""_do_search must support itemsperpage higher than maximum"""
@@ -3723,6 +3883,132 @@ class TestCoreSearch(TestCoreBase):
validate=False,
)
+ @mock.patch("eodag.api.core.EODataAccessGateway._do_search", autospec=True)
+ @mock.patch("eodag.api.core.EODataAccessGateway._prepare_search", autospec=True)
+ def test_search_warns_on_deprecated_page_and_items_per_page(
+ self, mock_prepare_search, mock_do_search
+ ):
+ """search must warn when deprecated page and items_per_page are used"""
+ search_plugin = mock.Mock(provider="cop_dataspace")
+ search_plugin.config.pagination = {}
+ mock_prepare_search.return_value = (
+ [search_plugin],
+ {"collection": "S2_MSI_L1C"},
+ )
+ mock_do_search.return_value = self.search_results
+
+ with pytest.warns(DeprecationWarning) as warnings_records:
+ self.dag.search(
+ page=2,
+ items_per_page=3,
+ validate=False,
+ collection="S2_MSI_L1C",
+ )
+
+ warning_messages = [str(record.message) for record in warnings_records]
+ self.assertTrue(
+ any("deprecated search parameter 'page'" in msg for msg in warning_messages)
+ )
+ self.assertTrue(
+ any(
+ "deprecated search parameter 'items_per_page'" in msg
+ for msg in warning_messages
+ )
+ )
+ mock_do_search.assert_called_once_with(
+ self.dag,
+ search_plugin,
+ count=False,
+ raise_errors=False,
+ validate=False,
+ collection="S2_MSI_L1C",
+ page=2,
+ limit=3,
+ )
+
+ @mock.patch("eodag.api.core.EODataAccessGateway.search_iter_page_plugin")
+ @mock.patch("eodag.api.core.EODataAccessGateway._prepare_search")
+ def test_search_iter_page_warns_on_deprecated_items_per_page(
+ self, mock_prepare_search, mock_search_iter_page_plugin
+ ):
+ """search_iter_page must warn when deprecated items_per_page is used"""
+ search_plugin = mock.Mock(provider="cop_dataspace")
+ mock_prepare_search.return_value = (
+ [search_plugin],
+ {"collection": "S2_MSI_L1C"},
+ )
+ mock_search_iter_page_plugin.return_value = iter([self.search_results])
+
+ with pytest.warns(
+ DeprecationWarning, match="deprecated search parameter 'items_per_page'"
+ ):
+ page_iterator = self.dag.search_iter_page(
+ items_per_page=3, collection="S2_MSI_L1C"
+ )
+
+ self.assertEqual(list(page_iterator), [self.search_results])
+ mock_search_iter_page_plugin.assert_called_once_with(
+ limit=3,
+ search_plugin=search_plugin,
+ collection="S2_MSI_L1C",
+ )
+
+ @mock.patch("eodag.api.core.EODataAccessGateway.search")
+ def test_search_all_warns_on_deprecated_items_per_page(self, mock_search):
+ """search_all must warn when deprecated items_per_page is used"""
+ mock_search.return_value = SearchResult([])
+
+ with pytest.warns(
+ DeprecationWarning, match="deprecated search parameter 'items_per_page'"
+ ):
+ results = self.dag.search_all(
+ items_per_page=3,
+ collection="S2_MSI_L1C",
+ )
+
+ self.assertEqual(len(results), 0)
+ mock_search.assert_called_once_with(
+ limit=3,
+ start=None,
+ end=None,
+ geom=None,
+ locations=None,
+ collection="S2_MSI_L1C",
+ )
+
+ @mock.patch("eodag.api.core.EODataAccessGateway._do_search", autospec=True)
+ def test_search_iter_page_plugin_warns_on_deprecated_items_per_page(
+ self, mock_do_search
+ ):
+ """search_iter_page_plugin must warn when deprecated items_per_page is used"""
+ search_plugin = mock.Mock(provider="cop_dataspace")
+ mock_do_search.return_value = SearchResult([])
+
+ with pytest.warns(DeprecationWarning) as warnings_records:
+ list(
+ self.dag.search_iter_page_plugin(
+ search_plugin=search_plugin,
+ items_per_page=3,
+ collection="S2_MSI_L1C",
+ )
+ )
+
+ warning_messages = [str(record.message) for record in warnings_records]
+ self.assertTrue(
+ any(
+ "deprecated search parameter 'items_per_page'" in msg
+ for msg in warning_messages
+ )
+ )
+ mock_do_search.assert_called_once_with(
+ self.dag,
+ search_plugin,
+ raise_errors=True,
+ collection="S2_MSI_L1C",
+ page=1,
+ limit=3,
+ )
+
@mock.patch("eodag.api.core.EODataAccessGateway.search_iter_page_plugin")
@mock.patch("eodag.api.core.EODataAccessGateway._prepare_search")
def test_search_iter_page_requesterror_retry(
@@ -4307,6 +4593,23 @@ class TestCoreSearch(TestCoreBase):
),
)
+ @mock.patch("eodag.api.search_result.SearchResult.next_page")
+ @mock.patch("eodag.api.core.EODataAccessGateway.search")
+ def test_search_all_ignores_next_page_request_error(
+ self, mock_search, mock_next_page
+ ):
+ """search_all must return partial results when pagination raises RequestError"""
+ mock_search.return_value = SearchResult([self.search_results.data[0]], 1)
+ mock_next_page.side_effect = RequestError("next page failed")
+
+ with self.assertLogs("eodag.core", level="WARNING") as cm:
+ results = self.dag.search_all(collection="S2_MSI_L1C")
+
+ self.assertEqual(len(results), 1)
+ self.assertEqual(results.number_matched, 1)
+ self.assertTrue(results.raise_errors)
+ self.assertIn("but it may be incomplete", str(cm.output))
+
@mock.patch(
"eodag.api.core.EODataAccessGateway._do_search",
autospec=True,
@@ -4610,6 +4913,20 @@ class TestCoreProductAlias(TestCoreBase):
with self.assertRaises(NoMatchingCollection):
self.dag.get_collection_from_alias("JUST_A_TYPE")
+ def test_get_collection_from_alias_multiple_matches(self):
+ """get_collection_from_alias must raise NoMatchingCollection if alias is ambiguous"""
+ products = self.dag.collections_config
+ products["S2_MSI_L2A_ALIAS"] = Collection.create_with_dag(
+ self.dag,
+ alias="S2_MSI_ALIAS",
+ **products["S2_MSI_L2A"].model_dump(exclude={"alias"}),
+ )
+
+ with self.assertRaises(NoMatchingCollection):
+ self.dag.get_collection_from_alias("S2_MSI_ALIAS")
+
+ products.pop("S2_MSI_L2A_ALIAS")
+
class TestCoreProviderGroup(TestCoreBase):
# create a group with a provider which has collection discovery mechanism
@@ -4644,6 +4961,17 @@ class TestCoreProviderGroup(TestCoreBase):
self.assertCountEqual(self.dag.providers.groups, providers)
+ def test_available_providers(self) -> None:
+ """available_providers must list available provider names sorted like providers"""
+ self.assertEqual(self.dag.available_providers(), self.dag.providers.names)
+
+ def test_available_providers_for_collection(self) -> None:
+ """available_providers must filter providers by collection"""
+ self.assertEqual(
+ self.dag.available_providers(collection="S2_MSI_L1C"),
+ self.dag.providers.filter("S2_MSI_L1C").names,
+ )
+
def test_list_collections(self) -> None:
"""
List the collections for the provider group.
=====================================
tests/units/test_download_plugins.py
=====================================
@@ -52,6 +52,8 @@ from tests.context import (
PluginConfig,
PluginManager,
ProvidersDict,
+ S3FileInfo,
+ StreamResponse,
load_default_config,
path_to_uri,
uri_to_path,
@@ -2077,6 +2079,104 @@ class TestDownloadPluginAws(BaseDownloadPluginTest):
)
self.assertEqual((bucket, prefix), ("default_bucket", "somewhere/else"))
+ def test_plugins_download_aws_get_commonpath(self):
+ """AwsDownload._get_commonpath() must return common chunk destination path"""
+ plugin = self.get_download_plugin(self.product)
+ product_chunks = {
+ mock.Mock(key="path/to/some/product/file1.tif"),
+ mock.Mock(key="path/to/some/product/sub/file2.tif"),
+ }
+
+ common_path = plugin._get_commonpath(
+ self.product, product_chunks, build_safe=False
+ )
+
+ self.assertEqual(common_path, "path/to/some/product")
+
+ @mock.patch("eodag.plugins.download.aws.stream_download_from_s3", autospec=True)
+ def test_plugins_download_aws_stream_download(self, mock_stream_download_from_s3):
+ """AwsDownload.stream_download() must stream S3 objects with flattened paths"""
+ expected_response = StreamResponse(iter([b"content"]))
+ mock_stream_download_from_s3.return_value = expected_response
+ plugin = self.get_download_plugin(self.product)
+ plugin.config.flatten_top_dirs = True
+ plugin.config.products[self.product.collection]["build_safe"] = False
+ plugin.config.products[self.product.collection]["complementary_url_key"] = []
+ s3_client = mock.Mock()
+ s3_resource = mock.Mock()
+ s3_resource.meta.client = s3_client
+ product_chunks = [
+ mock.Mock(
+ bucket_name="somebucket",
+ key="path/to/some/product/file1.tif",
+ size=1,
+ ),
+ mock.Mock(
+ bucket_name="somebucket",
+ key="path/to/some/product/sub/file2.tif",
+ size=2,
+ ),
+ ]
+ authenticated_objects = {"somebucket": mock.Mock()}
+ authenticated_objects["somebucket"].filter.side_effect = lambda Prefix: [
+ chunk for chunk in product_chunks if chunk.key.startswith(Prefix)
+ ]
+ auth_plugin = mock.Mock()
+ auth_plugin.authenticate_objects.return_value = authenticated_objects
+ self.product.downloader_auth = auth_plugin
+ self.product.assets.clear()
+ self.product.assets.update(
+ {
+ "file1": {
+ "href": "s3://somebucket/path/to/some/product/file1.tif",
+ "type": "image/tiff",
+ },
+ "file2": {
+ "href": "s3://somebucket/path/to/some/product/sub/file2.tif",
+ },
+ }
+ )
+
+ response = plugin.stream_download(
+ self.product,
+ auth=s3_resource,
+ byte_range=(0, 10),
+ compress="zip",
+ )
+
+ self.assertIs(response, expected_response)
+ auth_plugin.authenticate_objects.assert_called_once_with(
+ [
+ ("somebucket", "path/to/some/product/file1.tif"),
+ ("somebucket", "path/to/some/product/sub/file2.tif"),
+ ]
+ )
+ mock_stream_download_from_s3.assert_called_once()
+ args = mock_stream_download_from_s3.call_args.args
+ self.assertIs(args[0], s3_client)
+ files_info = sorted(args[1], key=lambda file_info: file_info.key)
+ self.assertEqual(
+ files_info,
+ [
+ S3FileInfo(
+ key="path/to/some/product/file1.tif",
+ size=1,
+ bucket_name="somebucket",
+ rel_path="dummy_product/file1.tif",
+ data_type="image/tiff",
+ ),
+ S3FileInfo(
+ key="path/to/some/product/sub/file2.tif",
+ size=2,
+ bucket_name="somebucket",
+ rel_path="dummy_product/sub/file2.tif",
+ ),
+ ],
+ )
+ self.assertEqual(args[2], (0, 10))
+ self.assertEqual(args[3], "zip")
+ self.assertEqual(args[4], "dummy_product")
+
@mock.patch(
"eodag.plugins.download.aws.AwsDownload._get_unique_products", autospec=True
)
=====================================
tests/units/test_search_plugins.py
=====================================
@@ -25,12 +25,15 @@ import unittest
from copy import deepcopy as copy_deepcopy
from importlib import import_module
from pathlib import Path
+from types import SimpleNamespace
from typing import Annotated, Literal, Union, get_args, get_origin
from unittest import mock
from unittest.mock import call
+from urllib.parse import quote_plus
import boto3
import botocore
+import geojson
import pytest
import requests
import responses
@@ -46,6 +49,11 @@ from eodag.api.product import AssetsDict
from eodag.api.product.metadata_mapping import get_queryable_from_provider
from eodag.api.provider import Provider, ProvidersDict
from eodag.api.search_result import RawSearchResult
+from eodag.plugins.search.build_search_result import (
+ _check_id,
+ _request_params_to_properties,
+ _update_properties_from_element,
+)
from eodag.plugins.search.cop_ghsl import (
_convert_bbox_to_lonlat_EPSG3035,
_convert_bbox_to_lonlat_mollweide,
@@ -54,6 +62,7 @@ from eodag.plugins.search.cop_ghsl import (
)
from eodag.utils import deepcopy
from eodag.utils.exceptions import (
+ DownloadError,
PluginImplementationError,
QuotaExceededError,
UnsupportedCollection,
@@ -61,14 +70,17 @@ from eodag.utils.exceptions import (
)
from tests.context import (
DEFAULT_SEARCH_TIMEOUT,
+ GENERIC_COLLECTION,
HTTP_REQ_TIMEOUT,
NOT_AVAILABLE,
TEST_RESOURCES_PATH,
USER_AGENT,
AuthenticationError,
+ CSWSearch,
EOProduct,
MisconfiguredError,
NotAvailableError,
+ PluginConfig,
PluginManager,
PreparedSearch,
QueryablesDict,
@@ -1162,6 +1174,119 @@ class TestSearchPluginQueryStringSearch(BaseSearchPluginTest):
with pytest.raises(MisconfiguredError):
search_plugin.count_hits("http://fake.url")
+ def test_plugins_search_querystringsearch_request_without_exception_message_logs_and_raises(
+ self,
+ ):
+ """QueryStringSearch._request must log the default fallback message and raise RequestError."""
+ prep = PreparedSearch(url="https://example.test/search")
+ prep.query_params = {}
+ with mock.patch(
+ "eodag.plugins.search.qssearch.requests.Session.get",
+ side_effect=requests.RequestException("boom"),
+ ):
+ with self.assertLogs("eodag.search.qssearch", level="ERROR") as cm:
+ with self.assertRaises(RequestError):
+ self.sara_search_plugin._request(prep)
+ self.assertIn("Skipping error while requesting", "\n".join(cm.output))
+
+ def test_plugins_search_querystringsearch_build_raw_search_results_sets_next_page_token(
+ self,
+ ):
+ """Raw search result building must extract the next page marker from a `next_page_query_obj` response."""
+ prep = PreparedSearch(limit=2, next_page_token_key="page")
+ prep.query_params = {"foo": "bar"}
+ prep.collection_def_params = {}
+ prep.next_page_token = 2
+ self.sara_search_plugin.config.pagination["next_page_query_obj_key_path"] = (
+ "$.data.next"
+ )
+ raw = self.sara_search_plugin._build_raw_search_results(
+ results=[{"id": "A"}],
+ resp_as_json={"data": {"next": {"page": 3}}},
+ search_kwargs={},
+ limit=2,
+ prep=prep,
+ )
+ self.assertEqual(raw.next_page_token, 3)
+ self.assertEqual(raw.next_page_token_key, "page")
+
+ def test_plugins_search_querystringsearch_init_raises_on_empty_metadata_mapping_from_product(
+ self,
+ ):
+ """QueryStringSearch.__init__ must reject an empty metadata_mapping inherited from another product."""
+ provider = "earth_search"
+ plugin_cfg = copy_deepcopy(self.get_search_plugin(provider=provider).config)
+ plugin_cfg.products["S1_SAR_GRD"][
+ "metadata_mapping_from_product"
+ ] = "S2_MSI_L1C"
+ plugin_cfg.products["S2_MSI_L1C"]["metadata_mapping"] = {}
+
+ with self.assertRaises(MisconfiguredError):
+ QueryStringSearch(provider, plugin_cfg)
+
+ def test_plugins_search_querystringsearch_clear_resets_pagination_state(self):
+ """QueryStringSearch.clear must reset URLs, parameters, and page state."""
+ self.sara_search_plugin.search_urls = ["https://example.test"]
+ self.sara_search_plugin.query_params = {"foo": "bar"}
+ self.sara_search_plugin.query_string = "foo=bar"
+ self.sara_search_plugin.next_page_url = "https://example.test/next"
+ self.sara_search_plugin.next_page_query_obj = {"page": 2}
+ self.sara_search_plugin.next_page_merge = {"features": []}
+
+ self.sara_search_plugin.clear()
+
+ self.assertEqual(self.sara_search_plugin.search_urls, [])
+ self.assertEqual(self.sara_search_plugin.query_params, {})
+ self.assertEqual(self.sara_search_plugin.query_string, "")
+ self.assertIsNone(self.sara_search_plugin.next_page_url)
+ self.assertIsNone(self.sara_search_plugin.next_page_query_obj)
+ self.assertIsNone(self.sara_search_plugin.next_page_merge)
+
+ def test_plugins_search_querystringsearch_generic_collection_returns_empty(self):
+ """QueryStringSearch must not search the internal generic collection."""
+ result = self.sara_search_plugin.query(
+ prep=PreparedSearch(count=True), collection=GENERIC_COLLECTION
+ )
+ self.assertEqual(result.data, [])
+ self.assertEqual(result.number_matched, 0)
+
+ def test_plugins_search_querystringsearch_collect_urls_requires_template(self):
+ """Numeric pagination must reject configurations without a URL template."""
+ self.assertEqual(
+ self.sara_search_plugin.config.pagination["next_page_url_tpl"],
+ "{url}?{search}&maxRecords={limit}&page={next_page_token}",
+ )
+ self.sara_search_plugin.config.pagination.pop("next_page_url_tpl", None)
+ prep = PreparedSearch(limit=2, count=False)
+ prep.query_string = ""
+ prep.query_params = {}
+ with self.assertRaises(MisconfiguredError):
+ self.sara_search_plugin.collect_search_urls(
+ prep, collection=self.collection
+ )
+
+ def test_plugins_search_querystringsearch_collect_urls_formats_collection_endpoint(
+ self,
+ ):
+ """Pagination URL formatting must substitute the provider collection."""
+ self.assertEqual(
+ self.sara_search_plugin.config.api_endpoint,
+ "https://copernicus.nci.org.au/sara.server/1.0/api/collections/{_collection}/search.json",
+ )
+ prep = PreparedSearch(limit=None)
+ prep.query_string = ""
+ prep.query_params = {}
+ urls, _ = self.sara_search_plugin.collect_search_urls(
+ prep, collection=self.collection
+ )
+ self.assertEqual(
+ urls,
+ [
+ "https://copernicus.nci.org.au/sara.server/1.0/api/collections/"
+ "S2_MSI_L1C/search.json"
+ ],
+ )
+
class TestSearchPluginPostJsonSearch(BaseSearchPluginTest):
def setUp(self):
@@ -1197,6 +1322,27 @@ class TestSearchPluginPostJsonSearch(BaseSearchPluginTest):
run()
+ def test_plugins_search_postjsonsearch_retries_status_errors(self):
+ """A PostJsonSearch request must retry configured status errors."""
+ search_plugin = self.get_search_plugin(provider="fedeo_ceda")
+ search_plugin.config.retry_total = 1
+ search_plugin.config.retry_backoff_factor = 0
+ url = search_plugin.config.api_endpoint
+ prep = PreparedSearch(url=url)
+ prep.query_params = {}
+
+ @responses.activate(registry=responses.registries.FirstMatchRegistry)
+ def run():
+ responses.add(responses.POST, url, status=400)
+ responses.add(responses.POST, url, status=200)
+
+ response = search_plugin._request(prep)
+
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(len(responses.calls), 2)
+
+ run()
+
def test_plugins_search_postjsonsearch_request_auth_error(self):
"""A query with a PostJsonSearch must handle auth errors"""
@@ -1218,7 +1364,7 @@ class TestSearchPluginPostJsonSearch(BaseSearchPluginTest):
run()
- @mock.patch("eodag.plugins.search.qssearch.requests.post", autospec=True)
+ @mock.patch("eodag.plugins.search.qssearch.requests.Session.post", autospec=True)
def test_plugins_search_postjsonsearch_search_quota_exceeded(self, mock__request):
"""A query with a PostJsonSearch must handle a 429 response returned by the provider"""
@@ -1336,7 +1482,7 @@ class TestSearchPluginPostJsonSearch(BaseSearchPluginTest):
"eodag.plugins.search.qssearch.QueryStringSearch.normalize_results",
autospec=True,
)
- @mock.patch("eodag.plugins.search.qssearch.requests.post", autospec=True)
+ @mock.patch("eodag.plugins.search.qssearch.requests.Session.post", autospec=True)
def test_plugins_search_postjsonsearch_search_cloudcover_awseos(
self, mock_requests_post, mock_normalize_results
):
@@ -1404,7 +1550,7 @@ class TestSearchPluginPostJsonSearch(BaseSearchPluginTest):
)
self.assertNotIn("bar", products.data[0].properties)
- @mock.patch("eodag.plugins.search.qssearch.requests.post", autospec=True)
+ @mock.patch("eodag.plugins.search.qssearch.requests.Session.post", autospec=True)
@mock.patch(
"eodag.plugins.search.qssearch.PostJsonSearch.normalize_results", autospec=True
)
@@ -1427,6 +1573,7 @@ class TestSearchPluginPostJsonSearch(BaseSearchPluginTest):
},
)
mock_request.assert_called_with(
+ mock.ANY,
"https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/search",
json={
"year": "2020",
@@ -1448,6 +1595,7 @@ class TestSearchPluginPostJsonSearch(BaseSearchPluginTest):
start_datetime="2021-02-01T03:00:00Z",
)
mock_request.assert_called_with(
+ mock.ANY,
"https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/search",
json={
"year": ["2021"],
@@ -1498,6 +1646,7 @@ class TestSearchPluginPostJsonSearch(BaseSearchPluginTest):
)
search_plugin.query(collection="ERA5_SL", prep=PreparedSearch())
mock_request.assert_called_with(
+ mock.ANY,
"https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/search",
json={
"dataset_id": "EO:ECMWF:DAT:REANALYSIS_ERA5_SINGLE_LEVELS",
@@ -1540,6 +1689,7 @@ class TestSearchPluginPostJsonSearch(BaseSearchPluginTest):
)
search_plugin.query(collection="CAMS_EAC4", prep=PreparedSearch())
mock_request.assert_called_with(
+ mock.ANY,
"https://gateway.prod.wekeo2.eu/hda-broker/api/v1/dataaccess/search",
json={
"dataset_id": "EO:ECMWF:DAT:CAMS_GLOBAL_REANALYSIS_EAC4",
@@ -1650,6 +1800,70 @@ class TestSearchPluginPostJsonSearch(BaseSearchPluginTest):
}
_test_query_params(search_criteria, raw_result, expected_query_params)
+ def test_plugins_search_postjsonsearch_request_rejects_empty_url(self):
+ """PostJsonSearch must reject requests without a URL."""
+ with self.assertRaises(ValidationError):
+ self.awseos_search_plugin._request(PreparedSearch())
+
+ @mock.patch("eodag.plugins.search.qssearch.requests.Session.post", autospec=True)
+ def test_plugins_search_postjsonsearch_request_translates_errors(self, mock_post):
+ """PostJsonSearch must translate timeout, auth, quota, and generic request errors."""
+ response = requests.Response()
+ response.status_code = 403
+ response._content = b"forbidden"
+ mock_post.return_value = response
+ self.assertEqual(
+ self.awseos_search_plugin.config.auth_error_code,
+ [402, 403],
+ )
+ prep = PreparedSearch(url=self.awseos_url)
+ prep.query_params = {}
+ with self.assertRaises(AuthenticationError):
+ self.awseos_search_plugin._request(prep)
+
+ response.status_code = 429
+ with self.assertRaises(QuotaExceededError):
+ self.awseos_search_plugin._request(prep)
+
+ mock_post.side_effect = requests.exceptions.RequestException("boom")
+ with self.assertRaises(RequestError):
+ self.awseos_search_plugin._request(prep)
+
+ mock_post.side_effect = requests.exceptions.Timeout()
+ with self.assertRaises(TimeOutError):
+ self.awseos_search_plugin._request(prep)
+
+ def test_plugins_search_postjsonsearch_collect_search_urls_missing_api_format_key_raises(
+ self,
+ ):
+ """PostJsonSearch.collect_search_urls must reject a missing API endpoint format key."""
+ prep = PreparedSearch(limit=2, count=False, auth_plugin=self.awseos_auth_plugin)
+ prep.query_params = {}
+ self.awseos_search_plugin.config.api_endpoint = "https://example.test/{missing}"
+ with self.assertRaises(MisconfiguredError):
+ self.awseos_search_plugin.collect_search_urls(
+ prep, collection=self.collection
+ )
+
+ @mock.patch("eodag.plugins.search.qssearch.PostJsonSearch._request", autospec=True)
+ def test_plugins_search_postjsonsearch_query_accepts_dc_qs_payload(
+ self, mock_request
+ ):
+ """_dc_qs should decode serialized provider payloads and send them as the request body."""
+ payload = {"foo": "bar", "page": 1}
+ mock_request.return_value = mock.Mock()
+ mock_request.return_value.json.return_value = {"features": []}
+
+ self.awseos_search_plugin.query(
+ prep=PreparedSearch(count=False),
+ collection=self.collection,
+ _dc_qs=quote_plus(json.dumps(payload)),
+ )
+
+ query_params = mock_request.call_args[0][1].query_params
+ self.assertEqual(query_params["foo"], payload["foo"])
+ self.assertIn("page", query_params)
+
class TestSearchPluginODataV4Search(BaseSearchPluginTest):
def setUp(self):
@@ -1897,6 +2111,21 @@ class TestSearchPluginODataV4Search(BaseSearchPluginTest):
# products count non extracted from search results as count endpoint is specified
self.assertFalse(hasattr(self.onda_search_plugin, "total_items_nb"))
+ @mock.patch("eodag.plugins.search.qssearch.requests.get", autospec=True)
+ @mock.patch(
+ "eodag.plugins.search.qssearch.QueryStringSearch.do_search", autospec=True
+ )
+ def test_plugins_search_odatav4search_do_search_timeout(
+ self, mock_parent_do_search, mock_requests_get
+ ):
+ """ODataV4Search.do_search must raise TimeOutError when metadata fetch times out."""
+ self.onda_search_plugin.config.per_product_metadata_query = True
+ mock_parent_do_search.return_value = [{"id": "product-1"}]
+ mock_requests_get.side_effect = requests.exceptions.Timeout()
+
+ with self.assertRaises(TimeOutError):
+ self.onda_search_plugin.do_search(PreparedSearch())
+
@mock.patch("eodag.plugins.search.qssearch.requests.get", autospec=True)
@mock.patch(
"eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True
@@ -1904,7 +2133,7 @@ class TestSearchPluginODataV4Search(BaseSearchPluginTest):
def test_plugins_search_odatav4search_count_and_search_onda_per_product_metadata_query_request_error(
self, mock__request, mock_requests_get
):
- """A query with a ODataV4Search (here onda) must handle requests errors for query per product metadata""" # noqa
+ """A query with a ODataV4Search (here onda) must handle requests errors for query per product metadata, including quota responses.""" # noqa
# per_product_metadata_query parameter is updated to True if it is necessary
per_product_metadata_query = (
self.onda_search_plugin.config.per_product_metadata_query
@@ -1925,7 +2154,10 @@ class TestSearchPluginODataV4Search(BaseSearchPluginTest):
mock_requests_get.return_value.json.return_value = dict(
value=[dict(id="dummy_metadata", value="dummy_metadata_val")]
)
- mock_requests_get.side_effect = RequestException()
+ mock_requests_get.side_effect = [
+ RequestException(response=mock.Mock(status_code=429)),
+ RequestException(),
+ ]
with self.assertLogs(level="ERROR") as cm:
self.onda_search_plugin.query(
@@ -1952,8 +2184,9 @@ class TestSearchPluginODataV4Search(BaseSearchPluginTest):
error_message_indexes_list = [
i.start() for i in re.finditer(error_message, str(cm.output))
]
- # we check that two errors have been logged, one per product
+ # we check that two errors have been logged, one per product, including a quota warning for 429s
self.assertEqual(len(error_message_indexes_list), 2)
+ self.assertIn("Too many requests on provider", str(cm.output))
@mock.patch(
"eodag.plugins.search.qssearch.QueryStringSearch.normalize_results",
@@ -2214,7 +2447,7 @@ class TestSearchPluginStacSearch(BaseSearchPluginTest):
self.assertIn("some-asset", products[0].assets)
self.assertEqual(products[0].assets["some-asset"]["title"], "My custom title")
- @mock.patch("eodag.plugins.search.qssearch.requests.post", autospec=True)
+ @mock.patch("eodag.plugins.search.qssearch.requests.Session.post", autospec=True)
def test_plugins_search_stacsearch_opened_time_intervals(self, mock_requests_post):
"""Opened time intervals must be handled by StacSearch plugin"""
mock_requests_post.return_value = mock.Mock()
@@ -2326,6 +2559,28 @@ class TestSearchPluginStacSearch(BaseSearchPluginTest):
"MGRS-31TCJ",
)
+ def test_plugins_search_stacsearch_discover_queryables_requires_fetch_url(self):
+ """StacSearch.discover_queryables must reject configurations without a queryables URL."""
+ plugin = self.get_search_plugin(provider="wekeo_main")
+ plugin.config.discover_queryables = {
+ "fetch_url": None,
+ "collection_fetch_url": None,
+ }
+ with self.assertRaises(NotImplementedError):
+ plugin.discover_queryables(collection="COP_DEM_GLO90_DGED")
+
+ @mock.patch(
+ "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True
+ )
+ def test_plugins_search_stacsearch_discover_queryables_request_error(
+ self, mock_request
+ ):
+ """StacSearch.discover_queryables must raise RequestError on provider request failure."""
+ mock_request.side_effect = RequestError("boom")
+ plugin = self.get_search_plugin(provider="wekeo_main")
+ with self.assertRaises(RequestError):
+ plugin.discover_queryables(collection="COP_DEM_GLO90_DGED")
+
@mock.patch(
"eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True
)
@@ -3040,7 +3295,7 @@ class TestSearchPluginMeteoblueSearch(BaseSearchPluginTest):
self.auth_plugin.config.credentials = {"cred": "entials"}
self.auth = self.auth_plugin.authenticate()
- @mock.patch("eodag.plugins.search.qssearch.requests.post", autospec=True)
+ @mock.patch("eodag.plugins.search.qssearch.requests.Session.post", autospec=True)
def test_plugins_search_buildpostsearchresult_count_and_search(
self, mock_requests_post
):
@@ -3056,6 +3311,7 @@ class TestSearchPluginMeteoblueSearch(BaseSearchPluginTest):
)
mock_requests_post.assert_called_with(
+ mock.ANY,
self.search_plugin.config.api_endpoint,
json=mock.ANY,
headers=USER_AGENT,
@@ -3605,6 +3861,232 @@ class TestSearchPluginECMWFSearch(unittest.TestCase):
("2022-02-15T00:00:00.000Z", "2022-02-15T00:00:00.000Z"),
)
+ def test_plugins_search_ecmwfsearch_update_properties_from_element(self):
+ """_update_properties_from_element must build JSON schema fragments"""
+ prop = {}
+ _update_properties_from_element(
+ prop,
+ {"type": "StringListWidget", "help": "Choose several values"},
+ ["b", "a"],
+ )
+ self.assertDictEqual(
+ prop,
+ {
+ "type": "array",
+ "items": {"type": "string", "enum": ["a", "b"]},
+ "description": "Choose several values",
+ },
+ )
+
+ prop = {}
+ _update_properties_from_element(
+ prop,
+ {"type": "DateRangeWidget"},
+ ["2020-01-01/2020-01-31"],
+ )
+ self.assertEqual(prop["type"], "string")
+ self.assertEqual(prop["enum"], ["2020-01-01/2020-01-31"])
+ self.assertEqual(
+ prop["description"], "date formatted like yyyy-mm-dd/yyyy-mm-dd"
+ )
+
+ prop = {}
+ _update_properties_from_element(prop, {"type": "GeographicExtentWidget"}, [])
+ self.assertEqual(prop["type"], "array")
+ self.assertEqual(prop["minItems"], 4)
+ self.assertEqual(len(prop["items"]), 4)
+
+ prop = {}
+ _update_properties_from_element(prop, {"type": "GeographicLocationWidget"}, [])
+ self.assertEqual(prop["type"], "object")
+ self.assertIn("longitude", prop["properties"])
+ self.assertIn("latitude", prop["properties"])
+
+ def test_plugins_search_ecmwfsearch_queryables_by_values(self):
+ """queryables_by_values must expose defaults, aliases and required fields"""
+ queryables = self.search_plugin.queryables_by_values(
+ {"variable": ["a", "b"], "product_type": ["analysis"]},
+ ["variable"],
+ {"product_type": "analysis"},
+ )
+
+ self.assertIn("ecmwf_variable", queryables)
+ self.assertIn("ecmwf_product_type", queryables)
+ variable_field = get_args(queryables["ecmwf_variable"])[1]
+ product_type_field = get_args(queryables["ecmwf_product_type"])[1]
+ self.assertTrue(variable_field.is_required())
+ self.assertFalse(product_type_field.is_required())
+ self.assertEqual("analysis", product_type_field.get_default())
+ self.assertEqual("ecmwf:variable", variable_field.serialization_alias)
+
+ def test_plugins_search_wekeo_ecmwf_build_query_string_with_empty_dc_qs(self):
+ """WekeoECMWFSearch.build_query_string must ignore _dc_qs=None"""
+ search_plugin = self.get_search_plugin(provider="wekeo_ecmwf")
+
+ query_params, query_string = search_plugin.build_query_string(
+ "ERA5_SL",
+ {
+ "dataset_id": "EO:ECMWF:DAT:REANALYSIS_ERA5_SINGLE_LEVELS",
+ "_dc_qs": None,
+ },
+ )
+
+ self.assertNotIn("_dc_qs", query_params)
+ self.assertNotIn("_dc_qs", query_string)
+
+ def test_plugins_search_ecmwfsearch_preprocess_search_params_with_dc_qs(self):
+ """_preprocess_search_params must decode date and area from _dc_qs"""
+ dc_query_params = {
+ "date": "2020-01-01/to/2020-01-02",
+ "area": "44/1/43/2",
+ }
+ dc_qs = quote_plus(geojson.dumps(dc_query_params))
+
+ params = self.search_plugin._preprocess_search_params(
+ {
+ "_dc_qs": dc_qs,
+ "ecmwf:variable": "temperature",
+ }
+ )
+
+ self.assertEqual(params["start_datetime"], "2020-01-01")
+ self.assertEqual(params["end_datetime"], "2020-01-02")
+ self.assertEqual(params["variable"], "temperature")
+ self.assertEqual(params["_dc_qs"], dc_qs)
+ self.assertEqual(params["geometry"].bounds, (43.0, 1.0, 44.0, 2.0))
+
+ def test_plugins_search_ecmwfsearch_normalize_results_with_dc_qs_and_result(self):
+ """normalize_results must use _dc_qs while preserving non-empty result properties"""
+ dc_query_params = {
+ "variable": "temperature",
+ "area": [44.0, 1.0, 43.0, 2.0],
+ "format": "grib",
+ }
+ raw_search_results = RawSearchResult(
+ [
+ {
+ "dataset": self.product_dataset,
+ "date": "2020-01-01/2020-01-02",
+ "time": "00:00",
+ "area": [90.0, -180.0, -90.0, 180.0],
+ "__hidden": "ignored",
+ "eodag:request_params": {"product_type": "analysis"},
+ }
+ ]
+ )
+ raw_search_results.query_params = {"page": 1}
+ raw_search_results.collection_def_params = {}
+
+ product = self.search_plugin.normalize_results(
+ raw_search_results,
+ collection=self.collection,
+ _dc_qs=quote_plus(geojson.dumps(dc_query_params)),
+ )[0]
+
+ self.assertEqual(product.properties["ecmwf:dataset"], self.product_dataset)
+ self.assertEqual(product.properties["ecmwf:product_type"], "analysis")
+ self.assertNotIn("__hidden", product.properties)
+ self.assertEqual(product.geometry.bounds, (1.0, 43.0, 2.0, 44.0))
+ self.assertEqual(
+ product.properties["start_datetime"], "2020-01-01T00:00:00.000Z"
+ )
+ self.assertEqual(product.properties["end_datetime"], "2020-01-02T00:00:00.000Z")
+ self.assertNotIn("_dc_qs", product.properties)
+ self.assertNotIn("ecmwf:area", product.properties)
+
+ def test_plugins_search_ecmwfsearch_check_id_error_handling(self):
+ """_check_id must translate order status errors into ValidationError"""
+ product = EOProduct("cop_ads", {"id": "generated", "geometry": "POINT (0 0)"})
+ product.search_kwargs = {"id": "123"}
+ product.collection = "ERA5_SL"
+ downloader = mock.Mock()
+ downloader.config.order_on_response = {"metadata_mapping": {}}
+ product.downloader = downloader
+
+ self.assertIs(_check_id(product), product)
+ downloader._order_status.assert_not_called()
+
+ downloader.config.order_on_response = {"metadata_mapping": {"foo": "bar"}}
+ downloader._order_status.side_effect = DownloadError(
+ "order status could not be checked"
+ )
+
+ with self.assertRaises(ValidationError) as context:
+ _check_id(product)
+
+ self.assertIn(
+ "Requested data is not available on cop_ads (123).",
+ context.exception.message,
+ )
+
+ downloader._order_status.side_effect = RuntimeError("boom")
+ with self.assertRaises(ValidationError) as context:
+ _check_id(product)
+
+ self.assertEqual("boom", context.exception.message)
+
+ def test_plugins_search_ecmwfsearch_request_params_to_properties_geometries(self):
+ """_request_params_to_properties must convert supported ECMWF geometry request params"""
+ cases = [
+ (
+ "feature",
+ {
+ "type": "polygon",
+ "shape": [[1, 43], [1, 44], [2, 44], [2, 43], [1, 43]],
+ },
+ (43.0, 1.0, 44.0, 2.0),
+ ),
+ ("area", [44.0, 1.0, 43.0, 2.0], (1.0, 43.0, 2.0, 44.0)),
+ ("location", {"latitude": 43.5, "longitude": 1.5}, (1.5, 43.5, 1.5, 43.5)),
+ ]
+
+ for key, geometry_value, expected_bounds in cases:
+ product = EOProduct(
+ "cop_ads",
+ {
+ "id": key,
+ "geometry": "POINT (0 0)",
+ "eodag:request_params": {
+ key: geometry_value,
+ "date": "2020-01-01/2020-01-02",
+ "variable": "temperature",
+ },
+ },
+ )
+
+ _request_params_to_properties(product)
+
+ self.assertEqual(product.geometry.bounds, expected_bounds)
+ self.assertEqual(product.properties["ecmwf:variable"], "temperature")
+ self.assertEqual(
+ product.properties["start_datetime"], "2020-01-01T00:00:00.000Z"
+ )
+ self.assertEqual(
+ product.properties["end_datetime"], "2020-01-02T00:00:00.000Z"
+ )
+
+ def test_plugins_search_wekeo_ecmwf_do_search_with_order_id(self):
+ """WekeoECMWFSearch.do_search must fake raw results for non-ORDERABLE ids"""
+ search_plugin = self.get_search_plugin(provider="wekeo_ecmwf")
+ prep = PreparedSearch()
+ prep.query_params = {"foo": "bar"}
+ prep.collection_def_params = {
+ "dataset_id": "EO:ECMWF:DAT:REANALYSIS_ERA5_SINGLE_LEVELS"
+ }
+
+ with mock.patch.object(search_plugin, "_request") as mock_request:
+ raw_results = search_plugin.do_search(
+ prep=prep, id="123", collection="ERA5_SL"
+ )
+
+ self.assertEqual([{}], raw_results.data)
+ self.assertEqual(
+ {"id": "123", "collection": "ERA5_SL"}, raw_results.search_params
+ )
+ self.assertEqual(prep.query_params, raw_results.query_params)
+ self.assertEqual(prep.collection_def_params, raw_results.collection_def_params)
+ mock_request.assert_not_called()
+
def test_plugins_search_ecmwfsearch_normalize_results(self):
"""ECMWFSearch should add request params to properties and set
start/end datetime if year/month/day/time are present in normalize_results"""
@@ -5096,6 +5578,122 @@ class TestSearchPluginCopMarineSearch(BaseSearchPluginTest):
id="item_20200204_20200205_niznjvnqkrf_20210101",
)
+ def test_plugins_search_cop_marine_query_pagination_disabled(self):
+ """CopMarineSearch.query must only return one page when pagination is disabled"""
+ search_plugin = self.get_search_plugin("PRODUCT_A", self.provider)
+
+ for prep in [
+ mock.Mock(limit=1, page=None, next_page_token=None, count=True),
+ mock.Mock(limit=None, page=1, next_page_token=None, count=True),
+ mock.Mock(limit=0, page=2, next_page_token=None, count=True),
+ ]:
+ result = search_plugin.query(prep=prep, collection="PRODUCT_A")
+
+ self.assertEqual([], result.data)
+ self.assertEqual(0, result.number_matched)
+
+ def test_plugins_search_cop_marine_query_skips_invalid_s3_url(self):
+ """CopMarineSearch.query must skip datasets with invalid bucket or prefix"""
+ search_plugin = self.get_search_plugin("PRODUCT_A", self.provider)
+
+ with (
+ mock.patch.object(
+ search_plugin,
+ "_get_collection_info",
+ return_value=(self.product_data, [self.dataset1_data]),
+ ),
+ mock.patch(
+ "eodag.plugins.search.cop_marine.get_bucket_name_and_prefix",
+ return_value=(None, None),
+ ),
+ mock.patch(
+ "eodag.plugins.search.cop_marine._get_s3_client"
+ ) as mock_get_s3_client,
+ self.assertLogs("eodag.search.cop_marine", level="WARNING") as cm,
+ ):
+ result = search_plugin.query(
+ prep=PreparedSearch(limit=1, count=True), collection="PRODUCT_A"
+ )
+
+ self.assertEqual([], result.data)
+ self.assertEqual(0, result.number_matched)
+ mock_get_s3_client.assert_not_called()
+ self.assertIn("Unable to get bucket and prefix", str(cm.output))
+
+ def test_plugins_search_cop_marine_query_direct_nc_asset(self):
+ """CopMarineSearch.query must create a product when the collection path is a nc file"""
+ search_plugin = self.get_search_plugin("PRODUCT_A", self.provider)
+ dataset_item = deepcopy(self.dataset1_data)
+ dataset_item["assets"]["native"]["href"] = (
+ "https://s3.test.com/bucket1/native/PRODUCT_A/dataset-number-one/"
+ "item_20200102_20200103_direct_20210101.nc"
+ )
+ s3_client = mock.Mock()
+ s3_client.head_object.return_value = {
+ "ResponseMetadata": {
+ "HTTPStatusCode": 200,
+ "HTTPHeaders": {
+ "content-length": "123",
+ "etag": '"d41d8cd98f00b204e9800998ecf8427e"',
+ "last-modified": dt.datetime(2020, 1, 4, tzinfo=dt.timezone.utc),
+ },
+ }
+ }
+
+ with (
+ mock.patch.object(
+ search_plugin,
+ "_get_collection_info",
+ return_value=(self.product_data, [dataset_item]),
+ ),
+ mock.patch(
+ "eodag.plugins.search.cop_marine._get_s3_client",
+ return_value=s3_client,
+ ),
+ ):
+ result = search_plugin.query(
+ prep=PreparedSearch(limit=1, count=True),
+ collection="PRODUCT_A",
+ start_datetime="2020-01-01T00:00:00Z",
+ end_datetime="2020-01-31T00:00:00Z",
+ )
+
+ self.assertEqual(1, result.number_matched)
+ self.assertEqual(1, len(result.data))
+ product = result.data[0]
+ self.assertEqual(
+ "item_20200102_20200103_direct_20210101", product.properties["id"]
+ )
+ self.assertEqual("native", next(iter(product.assets.keys())))
+ asset = product.assets["native"]
+ self.assertEqual(123, asset["file:size"])
+ self.assertEqual("d41d8cd98f00b204e9800998ecf8427e", asset["file:checksum"])
+ self.assertEqual("2020-01-04T00:00:00.000Z", asset["updated"])
+
+ def test_plugins_search_cop_marine_query_returns_empty_without_s3_contents(self):
+ """CopMarineSearch.query must return an empty counted result if S3 has no Contents"""
+ search_plugin = self.get_search_plugin("PRODUCT_A", self.provider)
+ s3_client = mock.Mock()
+ s3_client.list_objects.return_value = {}
+
+ with (
+ mock.patch.object(
+ search_plugin,
+ "_get_collection_info",
+ return_value=(self.product_data, [self.dataset1_data]),
+ ),
+ mock.patch(
+ "eodag.plugins.search.cop_marine._get_s3_client",
+ return_value=s3_client,
+ ),
+ ):
+ result = search_plugin.query(
+ prep=PreparedSearch(limit=1, count=True), collection="PRODUCT_A"
+ )
+
+ self.assertEqual([], result.data)
+ self.assertEqual(0, result.number_matched)
+
@mock.patch("eodag.plugins.search.cop_marine.requests.get")
def test_plugins_search_cop_marine_normalize_results(self, mock_requests_get):
"""Normalized query results must include asset information fetched from S3"""
@@ -5690,6 +6288,20 @@ class TestSearchPluginCopGhslSearch(BaseSearchPluginTest):
self.assertIn("month", params)
self.assertListEqual(["08", "09", "10", "11"], params["month"])
+ def test_plugins_search_cop_ghsl_get_start_and_end_from_year(self):
+ """_get_start_and_end_from_properties must use a full year date interval"""
+ plugin = next(self.plugins_manager.get_search_plugins(provider="cop_ghsl"))
+
+ datetimes = plugin._get_start_and_end_from_properties({"year": "2020"})
+
+ self.assertDictEqual(
+ datetimes,
+ {
+ "start_date": "2020-01-01T00:00:00.000Z",
+ "end_date": "2020-12-31T23:59:59.000Z",
+ },
+ )
+
@mock.patch("eodag.plugins.search.cop_ghsl.CopGhslSearch._fetch_constraints")
def test_plugins_search_cop_ghsl_check_input_parameters_valid(
self, mock_fetch_constraints
@@ -5836,6 +6448,152 @@ class TestSearchPluginCopGhslSearch(BaseSearchPluginTest):
]
)
+ @mock.patch("eodag.plugins.search.cop_ghsl.CopGhslSearch._fetch_constraints")
+ @mock.patch("eodag.plugins.search.cop_ghsl.requests.get")
+ def test_plugins_search_cop_ghsl_get_tiles_for_filters_exceptions(
+ self, mock_requests_get, mock_fetch_constraints
+ ):
+ """_get_tiles_for_filters must handle missing config and request errors"""
+ mock_fetch_constraints.return_value = {"constraints": self.constraints}
+ collection = "GHS_BUILT_S"
+ plugin = next(
+ self.plugins_manager.get_search_plugins(
+ collection=collection, provider="cop_ghsl"
+ )
+ )
+ params = {
+ "year": "2000",
+ "proj:code": "EPSG:4326",
+ "tile_size": "3ss",
+ "collection": collection,
+ }
+
+ with self.assertRaises(MisconfiguredError):
+ plugin._get_tiles_for_filters({}, deepcopy(params))
+
+ product_type_config = deepcopy(plugin.config.products.get(collection, {}))
+ mock_requests_get.return_value = MockResponse({}, status_code=404)
+ self.assertIsNone(
+ plugin._get_tiles_for_filters(product_type_config, deepcopy(params))
+ )
+
+ product_type_config = deepcopy(plugin.config.products.get(collection, {}))
+ mock_requests_get.side_effect = requests.exceptions.Timeout()
+ with self.assertRaises(TimeOutError):
+ plugin._get_tiles_for_filters(product_type_config, deepcopy(params))
+
+ product_type_config = deepcopy(plugin.config.products.get(collection, {}))
+ mock_requests_get.side_effect = requests.exceptions.RequestException("boom")
+ with self.assertRaises(RequestError):
+ plugin._get_tiles_for_filters(product_type_config, deepcopy(params))
+
+ @mock.patch("eodag.plugins.search.cop_ghsl.requests.get")
+ def test_plugins_search_cop_ghsl_fetch_constraints(self, mock_requests_get):
+ """_fetch_constraints must return provider constraints and handle failures"""
+ plugin = next(self.plugins_manager.get_search_plugins(provider="cop_ghsl"))
+ constraints = {"constraints": self.constraints}
+
+ mock_requests_get.return_value = MockResponse(constraints, status_code=200)
+ self.assertDictEqual(plugin._fetch_constraints("TEST_CONSTRAINTS"), constraints)
+ mock_requests_get.assert_called_once_with(
+ "https://s3.central.data.destination-earth.eu/swift/v1/constraints/cop_ghsl_dev/TEST_CONSTRAINTS.json",
+ timeout=HTTP_REQ_TIMEOUT,
+ headers=USER_AGENT,
+ )
+
+ mock_requests_get.reset_mock()
+ mock_requests_get.return_value = MockResponse({}, status_code=404)
+ self.assertDictEqual(
+ plugin._fetch_constraints("TEST_CONSTRAINTS_404"), {"constraints": {}}
+ )
+
+ mock_requests_get.side_effect = requests.exceptions.Timeout()
+ with self.assertRaises(TimeOutError):
+ plugin._fetch_constraints("TEST_CONSTRAINTS_TIMEOUT")
+
+ mock_requests_get.side_effect = requests.exceptions.RequestException("boom")
+ with self.assertRaises(RequestError):
+ plugin._fetch_constraints("TEST_CONSTRAINTS_ERROR")
+
+ def test_plugins_search_cop_ghsl_query(self):
+ """query must create SearchResult for tiled and non-tiled Cop GHSL products"""
+ collection = "GHS_BUILT_S"
+ plugin = next(
+ self.plugins_manager.get_search_plugins(
+ collection=collection, provider="cop_ghsl"
+ )
+ )
+ product = EOProduct(
+ "cop_ghsl",
+ {"id": "product-id", "geometry": "POINT (0 0)", "title": "product-id"},
+ collection=collection,
+ )
+ tiles = {"2000": [{"tileID": "R3_C3", "BBox": [0, 0, 1, 1]}]}
+
+ with (
+ mock.patch.object(
+ plugin,
+ "_get_tiles_for_filters",
+ return_value=(tiles, "lat/lon"),
+ ) as mock_get_tiles,
+ mock.patch.object(
+ plugin,
+ "_fetch_constraints",
+ return_value={"additional_filter": "classification"},
+ ) as mock_fetch_constraints,
+ mock.patch.object(
+ plugin,
+ "_create_products_from_tiles",
+ return_value=([product], 1),
+ ) as mock_create_products_from_tiles,
+ ):
+ result = plugin.query(
+ prep=PreparedSearch(limit=1, count=True),
+ collection=collection,
+ year="2000",
+ classification="TOTAL",
+ )
+
+ self.assertEqual([product], result.data)
+ self.assertEqual(1, result.number_matched)
+ self.assertEqual("page", result.next_page_token_key)
+ self.assertEqual("2", result.next_page_token)
+ self.assertTrue(result.raise_errors)
+ mock_get_tiles.assert_called_once()
+ mock_fetch_constraints.assert_called_once_with(collection)
+ mock_create_products_from_tiles.assert_called_once_with(
+ tiles,
+ "lat/lon",
+ collection,
+ mock.ANY,
+ additional_filter="classification",
+ need_count=True,
+ )
+
+ with (
+ mock.patch.object(
+ plugin,
+ "_get_tiles_for_filters",
+ return_value=None,
+ ),
+ mock.patch.object(
+ plugin,
+ "_create_products_without_tiles",
+ return_value=([product], 1),
+ ) as mock_create_products_without_tiles,
+ ):
+ result = plugin.query(
+ prep=PreparedSearch(collection=collection, limit=1, count=True),
+ year="2000",
+ )
+
+ self.assertEqual([product], result.data)
+ self.assertEqual(1, result.number_matched)
+ mock_create_products_without_tiles.assert_called_once()
+
+ with self.assertRaises(MisconfiguredError):
+ plugin.query(prep=PreparedSearch(), collection=[collection])
+
@mock.patch("eodag.plugins.search.cop_ghsl.CopGhslSearch._fetch_constraints")
@mock.patch("eodag.plugins.search.cop_ghsl.requests.get")
def test_plugins_search_cop_ghsl_get_tile_from_product_id(
@@ -6010,6 +6768,61 @@ class TestSearchPluginCopGhslSearch(BaseSearchPluginTest):
)
self.assertEqual(geometry, products[0].geometry)
+ def test_plugins_search_cop_ghsl_create_products_from_tiles_mollweide_bbox(self):
+ """_create_products_from_tiles must convert Mollweide metre bboxes"""
+ bbox = ["-6 041 000", "7 000 000", "-5 041 000", "6 000 000"]
+ tiles = {"2000": [{"tileID": "R3_C3", "BBox": bbox}]}
+ collection = "GHS_BUILT_S"
+ plugin = next(
+ self.plugins_manager.get_search_plugins(
+ collection=collection, provider="cop_ghsl"
+ )
+ )
+ params = deepcopy(plugin.config.products.get(collection, {}))
+ params["year"] = "2000"
+ params["proj:code"] = "EPSG:54009"
+ params["tile_size"] = "10m"
+ params["classification"] = "TOTAL"
+ params["per_page"] = 5
+ params["page"] = 1
+
+ products, count = plugin._create_products_from_tiles(
+ tiles, "metres", collection, params, "classification", need_count=True
+ )
+
+ self.assertEqual(count, 1)
+ self.assertEqual(len(products), 1)
+ expected_geometry = get_geometry_from_various(
+ geometry=_convert_bbox_to_lonlat_mollweide(bbox)
+ )
+ self.assertEqual(expected_geometry, products[0].geometry)
+
+ def test_plugins_search_cop_ghsl_create_products_from_tiles_epsg3035_bbox(self):
+ """_create_products_from_tiles must convert EPSG:3035 metre bboxes"""
+ bbox = ["1,944,000", "1,042,000", "2,044,000", "942,000"]
+ tiles = {"2015": [{"tileID": "R3_C3", "BBox_3035": bbox}]}
+ collection = "GHS_ESM"
+ plugin = next(
+ self.plugins_manager.get_search_plugins(
+ collection=collection, provider="cop_ghsl"
+ )
+ )
+ params = deepcopy(plugin.config.products.get(collection, {}))
+ params["tile_size"] = "10m"
+ params["per_page"] = 5
+ params["page"] = 1
+
+ products, count = plugin._create_products_from_tiles(
+ tiles, "metres", collection, params, need_count=True
+ )
+
+ self.assertEqual(count, 1)
+ self.assertEqual(len(products), 1)
+ expected_geometry = get_geometry_from_various(
+ geometry=_convert_bbox_to_lonlat_EPSG3035(bbox)
+ )
+ self.assertEqual(expected_geometry, products[0].geometry)
+
def test_plugins_search_cop_ghsl_create_products_without_tiles(self):
"""test if products are created correctly for product types without tiles"""
@@ -6213,3 +7026,211 @@ class TestSearchPluginEumetsatDsSearch(BaseSearchPluginTest):
},
},
)
+
+
+class TestSearchPluginCSWSearch(unittest.TestCase):
+ def _get_plugin(self, search_definition):
+ return CSWSearch(
+ "provider",
+ PluginConfig.from_mapping(
+ {
+ "type": "CSWSearch",
+ "api_endpoint": "https://csw.example",
+ "search_definition": search_definition,
+ "metadata_mapping": {"title": "//title/text()"},
+ "products": {"collection": {"collection": "provider-collection"}},
+ }
+ ),
+ )
+
+ def test_csw_query_constraints_cover_matching_geometry_and_dates(self):
+ """CSW constraints support matching modes, geometry, and date filters."""
+ plugin = self._get_plugin(
+ {
+ "collection_tags": [],
+ "date_tags": {"start": "begin", "end": "finish"},
+ }
+ )
+ for matching, expected in (
+ ("prefix", "collection%"),
+ ("postfix", "%collection"),
+ ("exact", "collection"),
+ ("unknown", "%collection%"),
+ ):
+ constraints = plugin._CSWSearch__convert_query_params(
+ {"name": "title", "matching": matching}, "collection", {}
+ )
+ self.assertEqual(constraints[0].literal, expected)
+ constraints = plugin._CSWSearch__convert_query_params(
+ {"name": "title"},
+ "collection",
+ {
+ "geometry": {"lonmin": 1, "latmin": 2, "lonmax": 3, "latmax": 4},
+ "start_datetime": "2020-01-01",
+ "end_datetime": "2020-01-02",
+ },
+ )
+ self.assertEqual(len(constraints), 1)
+ self.assertEqual(len(constraints[0]), 4)
+
+ @mock.patch("eodag.plugins.search.csw.properties_from_xml")
+ def test_csw_build_product_selects_filtered_reference_and_download_link(
+ self, properties_from_xml
+ ):
+ """CSW products select filtered references and expose download links."""
+ plugin = self._get_plugin(
+ {"collection_tags": [], "resource_location_filter": "preferred"}
+ )
+ properties_from_xml.return_value = {"geometry": None}
+ record = SimpleNamespace(
+ xml=b"<record />",
+ bbox_wgs84=(1, 2, 3, 4),
+ references=[
+ {"scheme": "WWW:DOWNLOAD-1.0-http--download", "url": "other"},
+ {"scheme": "WWW:DOWNLOAD-1.0-http--download", "url": "preferred"},
+ ],
+ )
+ product = plugin._CSWSearch__build_product(record, "collection")
+ self.assertEqual(product.location, "preferred")
+ self.assertEqual(product.properties["eodag:download_link"], "preferred")
+ self.assertEqual(product.geometry.bounds, (1.0, 2.0, 3.0, 4.0))
+
+ def test_csw_constraints_accept_shapely_geometry(self):
+ """CSW constraints accept Shapely geometry bounds."""
+ from shapely.geometry import box
+
+ plugin = self._get_plugin({"collection_tags": []})
+ constraints = plugin._CSWSearch__convert_query_params(
+ {"name": "title"}, "collection", {"geometry": box(1, 2, 3, 4)}
+ )
+ self.assertEqual(constraints[0][1].bbox, (1.0, 2.0, 3.0, 4.0))
+
+ def test_csw_clear_resets_catalog(self):
+ """CSW clear resets the cached catalog."""
+ plugin = self._get_plugin({"collection_tags": []})
+ plugin.catalog = object()
+ plugin.clear()
+ self.assertIsNone(plugin.catalog)
+
+ def test_csw_query_without_collection_returns_empty_result(self):
+ """CSW query without a collection returns an empty counted result."""
+ plugin = self._get_plugin({"collection_tags": []})
+ result = plugin.query(prep=PreparedSearch(count=True))
+ self.assertEqual(result.data, [])
+ self.assertEqual(result.number_matched, 0)
+
+ @mock.patch("eodag.plugins.search.csw.CatalogueServiceWeb")
+ def test_csw_init_catalog_uses_credentials_and_caches_result(
+ self, catalogue_service
+ ):
+ """CSW catalog initialization passes credentials and avoids recreation."""
+ plugin = self._get_plugin({"collection_tags": []})
+ plugin.config.api_endpoint = "https://csw.example"
+ plugin.config.version = "2.0.2"
+ plugin._CSWSearch__init_catalog("user", "password")
+ catalogue_service.assert_called_once_with(
+ "https://csw.example",
+ version="2.0.2",
+ username="user",
+ password="password",
+ )
+ plugin._CSWSearch__init_catalog("other", "credentials")
+ catalogue_service.assert_called_once()
+
+ @mock.patch(
+ "eodag.plugins.search.csw.CatalogueServiceWeb",
+ side_effect=RuntimeError("catalog unavailable"),
+ )
+ def test_csw_init_catalog_failure_leaves_catalog_unset(self, catalogue_service):
+ """CSW catalog initialization failures are logged and leave no catalog."""
+ plugin = self._get_plugin({"collection_tags": []})
+ plugin.config.api_endpoint = "https://csw.example"
+ with self.assertLogs("eodag.search.csw", level="WARNING"):
+ plugin._CSWSearch__init_catalog()
+ self.assertIsNone(plugin.catalog)
+
+ @mock.patch("eodag.plugins.search.csw.CatalogueServiceWeb")
+ def test_csw_query_continues_after_exception_report(self, catalogue_service):
+ """CSW query skips failed collection tags and returns remaining results."""
+ from owslib.ows import ExceptionReport
+
+ plugin = self._get_plugin(
+ {"collection_tags": [{"name": "title"}, {"name": "alternate"}]}
+ )
+ exception_report = ExceptionReport.__new__(ExceptionReport)
+ catalog = mock.Mock(records={})
+ catalog.getrecords2.side_effect = [exception_report, None]
+ catalogue_service.return_value = catalog
+ result = plugin.query(collection="collection")
+ self.assertEqual(result.data, [])
+ self.assertEqual(catalog.getrecords2.call_count, 2)
+
+
+class TestSearchPluginStacListAssets(BaseSearchPluginTest):
+ @mock.patch("eodag.plugins.search.stac_list_assets.update_assets_from_s3")
+ def test_plugins_search_stac_list_assets_register_downloader_updates_assets(
+ self, mock_update_assets_from_s3
+ ):
+ """StacListAssets must patch the product downloader registration to refresh S3 assets."""
+ search_plugin = self.get_search_plugin(provider="geodes_s3")
+
+ products = search_plugin.normalize_results(
+ [
+ {
+ "id": "foo",
+ "geometry": {"type": "Point", "coordinates": [0.0, 0.0]},
+ "properties": {
+ "identifier": "foo",
+ "start_datetime": "2020-01-01T00:00:00Z",
+ "end_datetime": "2020-01-02T00:00:00Z",
+ },
+ "assets": {
+ "data": {
+ "href": "s3://bucket/path/data.tif",
+ "roles": ["data"],
+ "title": "data",
+ }
+ },
+ }
+ ]
+ )
+
+ self.assertEqual(len(products), 1)
+ product = products[0]
+ self.assertTrue(hasattr(product, "register_downloader_only"))
+ self.assertIsNot(product.register_downloader, product.register_downloader_only)
+
+ downloader = mock.Mock()
+ downloader.config = mock.Mock(s3_endpoint="https://s3.example.com")
+ authenticator = mock.Mock()
+
+ product.register_downloader(downloader, authenticator)
+
+ self.assertIs(product.downloader, downloader)
+ self.assertIs(product.downloader_auth, authenticator)
+ mock_update_assets_from_s3.assert_called_once_with(
+ product, authenticator, "https://s3.example.com"
+ )
+
+ @mock.patch(
+ "eodag.plugins.search.stac_list_assets.update_assets_from_s3",
+ side_effect=botocore.exceptions.BotoCoreError(),
+ )
+ def test_plugins_search_stac_list_assets_register_downloader_request_error(
+ self, mock_update_assets_from_s3
+ ):
+ """S3 asset refresh failures are exposed as RequestError."""
+ search_plugin = self.get_search_plugin(provider="geodes_s3")
+ product = search_plugin.normalize_results(
+ [
+ {
+ "id": "foo",
+ "geometry": {"type": "Point", "coordinates": [0.0, 0.0]},
+ "properties": {"identifier": "foo"},
+ }
+ ]
+ )[0]
+ downloader = mock.Mock(config=mock.Mock(s3_endpoint="https://s3.example.com"))
+ with self.assertRaises(RequestError):
+ product.register_downloader(downloader, mock.Mock())
+ mock_update_assets_from_s3.assert_called_once()
=====================================
tests/units/test_search_types.py
=====================================
@@ -25,6 +25,7 @@ from typing_extensions import get_args, get_origin
from eodag.types import json_field_definition_to_python, queryables, search_args
from eodag.utils.exceptions import ValidationError as EodagValidationError
+from tests.context import BBox
class TestStacSearch(unittest.TestCase):
@@ -100,6 +101,38 @@ class TestStacSearch(unittest.TestCase):
)
+class TestBBox(unittest.TestCase):
+ def test_bbox_valid_inputs_and_polygon(self):
+ """BBox accepts supported inputs and produces the expected polygon."""
+ values = (
+ [1, 2, 3, 4],
+ (1, 2, 3, 4),
+ {"lonmin": 1, "latmin": 2, "lonmax": 3, "latmax": 4},
+ )
+ for value in values:
+ bbox = BBox(value)
+ self.assertEqual(
+ (bbox.lonmin, bbox.latmin, bbox.lonmax, bbox.latmax),
+ (1, 2, 3, 4),
+ )
+ self.assertEqual(bbox.to_polygon().bounds, (1.0, 2.0, 3.0, 4.0))
+
+ def test_bbox_rejects_invalid_shape_and_coordinates(self):
+ """BBox rejects invalid dimensions, coordinate ranges, and ordering."""
+ with self.assertRaises(ValueError):
+ BBox([1, 2, 3])
+ for value in (
+ [-181, 0, 1, 1],
+ [0, -91, 1, 1],
+ [0, 0, 181, 1],
+ [0, 0, 1, 91],
+ [2, 0, 1, 1],
+ [0, 2, 1, 1],
+ ):
+ with self.assertRaises(ValidationError):
+ BBox(value)
+
+
class TestQueryables(unittest.TestCase):
def setUp(self):
super(TestQueryables, self).setUp()
=====================================
tests/units/test_stac_reader.py
=====================================
@@ -17,6 +17,7 @@
# limitations under the License.
import os
import unittest
+from unittest import mock
from tests import TEST_RESOURCES_PATH
from tests.context import STACOpenerError, _TextOpener, fetch_stac_items
@@ -79,3 +80,10 @@ class TestStacReader(unittest.TestCase):
"http://data.example.org/",
True,
)
+
+ def test_stac_reader_text_opener_falls_back_to_http(self):
+ """The STAC text opener falls back when local reading fails."""
+ opener = _TextOpener(timeout=3, ssl_verify=True)
+ opener.openers[0] = mock.Mock(side_effect=STACOpenerError("not local"))
+ opener.openers[1] = mock.Mock(return_value={"id": "item"})
+ self.assertEqual(opener("file.json", as_json=True), {"id": "item"})
=====================================
tests/units/test_utils.py
=====================================
@@ -18,37 +18,58 @@
import copy
import datetime as dt
+import json
import logging
import os
import ssl
import sys
+import tempfile
import unittest
+import warnings
from contextlib import closing
from io import StringIO
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest import mock
+import requests
+from click.exceptions import BadParameter
from dateutil import parser as dateutil_parser
from requests.exceptions import RequestException
from shapely.geometry import Point, Polygon
-from eodag.utils import get_geometry_from_ecmwf_area, get_geometry_from_ecmwf_feature
+import eodag.utils as utils
+from eodag.utils import (
+ _build_float_range_cls,
+ _deprecated_class,
+ format_string,
+ nested_pairs2dict,
+)
+from eodag.utils.exceptions import MisconfiguredError
from eodag.utils.logging import TqdmLoggingHandler
from tests.context import (
HTTP_REQ_TIMEOUT,
USER_AGENT,
DownloadedCallback,
+ LocalFileAdapter,
+ NotebookWidgets,
ProgressCallback,
RequestError,
+ TimeOutError,
+ check_ipython,
+ check_notebook,
deepcopy,
fetch_json,
flatten_top_directories,
get_bucket_name_and_prefix,
+ get_geometry_from_ecmwf_area,
+ get_geometry_from_ecmwf_feature,
get_ssl_context,
get_timestamp,
+ import_all_modules,
is_env_var_true,
merge_mappings,
+ patch_owslib_requests,
path_to_uri,
setup_logging,
uri_to_path,
@@ -71,6 +92,79 @@ class TestUtils(unittest.TestCase):
logger.handlers = []
logger.level = 0
+ def test_build_float_range_cls(self):
+ """Test FloatRange conversion and range validation."""
+ float_range = _build_float_range_cls()
+ parameter_type = float_range(0, 100)
+
+ self.assertEqual(parameter_type.convert("42.5", None, None), 42.5)
+ with self.assertRaises(BadParameter):
+ parameter_type.convert("-1", None, None)
+ with self.assertRaises(BadParameter):
+ parameter_type.convert("101", None, None)
+
+ self.assertEqual(float_range(max=100).convert(42, None, None), 42.0)
+ self.assertEqual(float_range(min=0).convert(42, None, None), 42.0)
+
+ def test_utils_getattr_float_range(self):
+ """Test __getattr__ lazily creates and caches FloatRange."""
+ previous_float_range = utils.__dict__.pop("FloatRange", None)
+ try:
+ float_range = getattr(utils, "FloatRange")
+ self.assertIs(getattr(utils, "FloatRange"), float_range)
+ self.assertEqual(float_range(0, 1).convert("0.5", None, None), 0.5)
+ with self.assertRaises(AttributeError):
+ getattr(utils, "missing_attribute")
+ finally:
+ utils.__dict__.pop("FloatRange", None)
+ if previous_float_range is not None:
+ utils.__dict__["FloatRange"] = previous_float_range
+
+ def test_deprecated_class(self):
+ """Test _deprecated_class preserves identity and warns on constructors."""
+
+ class Example:
+ def __init__(self, value):
+ self.value = value
+
+ @classmethod
+ def model_validate(cls, value):
+ return cls(value)
+
+ decorated_class = _deprecated_class(reason="legacy", version="3.0")(Example)
+ self.assertIs(decorated_class, Example)
+
+ with warnings.catch_warnings(record=True) as caught_warnings:
+ warnings.simplefilter("always")
+ instance = Example("value")
+ self.assertEqual(instance.value, "value")
+ self.assertEqual(len(caught_warnings), 1)
+ self.assertIn(
+ "Example (legacy) -- Deprecated since v3.0", str(caught_warnings[0].message)
+ )
+
+ with warnings.catch_warnings(record=True) as caught_warnings:
+ warnings.simplefilter("always")
+ validated = Example.model_validate("validated")
+ self.assertEqual(validated.value, "validated")
+ self.assertEqual(len(caught_warnings), 2)
+
+ def test_format_string_exception_handling(self):
+ """Test format_string handles malformed and colon-containing formats."""
+ with self.assertRaisesRegex(MisconfiguredError, "Unable to format"):
+ format_string(None, "{invalid", value="unused")
+
+ self.assertEqual(
+ format_string(None, "{foo:bar}", **{"foo:bar": "value"}),
+ "value",
+ )
+
+ def test_nested_pairs2dict_value_error(self):
+ """Test nested_pairs2dict returns malformed pairs unchanged."""
+ pairs = [["valid", "pair"], ["invalid"]]
+
+ self.assertIs(nested_pairs2dict(pairs), pairs)
+
def test_utils_get_timestamp(self):
"""Test get_timestamp returns correct UNIX timestamp for various date formats"""
# Date to timestamp to date, this assumes the date is in UTC
@@ -500,6 +594,33 @@ class TestUtils(unittest.TestCase):
)
)
+ def test_get_geometry_from_ecmwf_feature_exceptions(self):
+ """ECMWF feature validation raises TypeError for invalid geometries."""
+ invalid_geometries = [
+ None,
+ {},
+ {"type": "polygon"},
+ {"type": "polygon", "shape": "invalid"},
+ {"type": "boundingbox"},
+ {"type": "boundingbox", "points": "invalid"},
+ {"type": "position"},
+ {"type": "position", "points": []},
+ {"type": "trajectory"},
+ {"type": "trajectory", "points": [[43.0, 1.0]]},
+ {
+ "type": "trajectory",
+ "points": [[43.0, 1.0], [43.5, 1.5]],
+ },
+ {"type": "circle"},
+ {"type": "circle", "center": [43.5, 1.5]},
+ {"type": "unsupported"},
+ ]
+
+ for geometry in invalid_geometries:
+ with self.subTest(geometry=geometry):
+ with self.assertRaises(TypeError):
+ get_geometry_from_ecmwf_feature(geometry)
+
def test_get_geometry_from_ecmwf_area_accepts_list_and_string(self):
"""``get_geometry_from_ecmwf_area`` must accept both list and slash-separated string formats."""
# list format: [max_lat, min_lon, min_lat, max_lon]
@@ -522,3 +643,87 @@ class TestUtils(unittest.TestCase):
# invalid string: non-numeric content
with self.assertRaises(ValueError):
get_geometry_from_ecmwf_area("a/b/c/d")
+
+ def test_patch_owslib_requests_restores_functions(self):
+ """OWSLib request patches apply verification and restore originals."""
+ import owslib.util
+
+ original_request = owslib.util.requests.request
+ original_post = owslib.util.requests.post
+ with patch_owslib_requests(verify=False):
+ self.assertFalse(owslib.util.requests.request.keywords["verify"])
+ self.assertFalse(owslib.util.requests.post.keywords["verify"])
+ self.assertIs(owslib.util.requests.request, original_request)
+ self.assertIs(owslib.util.requests.post, original_post)
+ with self.assertRaisesRegex(RuntimeError, "boom"):
+ with patch_owslib_requests():
+ raise RuntimeError("boom")
+ self.assertIs(owslib.util.requests.request, original_request)
+
+ def test_import_all_modules_honors_exclude(self):
+ """Module discovery skips excluded entries."""
+ from types import SimpleNamespace
+
+ package = SimpleNamespace(__name__="test_package", __path__=["unused"])
+ modules = [
+ (None, "module", False),
+ (None, "subpackage", True),
+ (None, "excluded", False),
+ ]
+ with (
+ mock.patch(
+ "eodag.utils.import_system.pkgutil.iter_modules", return_value=modules
+ ),
+ mock.patch(
+ "eodag.utils.import_system.importlib.import_module"
+ ) as import_module,
+ ):
+ import_all_modules(package, depth=1, exclude=("excluded",))
+ import_module.assert_called_once_with(".module", package="test_package")
+
+ def test_notebook_detection_and_non_notebook_widgets(self):
+ """Notebook widgets are no-ops outside a notebook."""
+ self.assertFalse(check_ipython())
+ self.assertFalse(check_notebook())
+ widgets = NotebookWidgets()
+ self.assertIsNone(widgets.display_html("ignored"))
+ self.assertIsNone(widgets.clear_html())
+
+ def test_notebook_detection_shells(self):
+ """Notebook detection distinguishes Jupyter and terminal IPython shells."""
+ with mock.patch("eodag.utils.notebook.get_ipython", create=True) as get_ipython:
+ get_ipython.return_value.__class__.__name__ = "ZMQInteractiveShell"
+ self.assertTrue(check_notebook())
+ get_ipython.return_value.__class__.__name__ = "TerminalInteractiveShell"
+ self.assertFalse(check_notebook())
+
+ def test_local_file_adapter_statuses_and_fetch_json(self):
+ """Local file requests return expected statuses and parse JSON."""
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=".json", delete=False
+ ) as file:
+ json.dump({"value": 1}, file)
+ path = file.name
+ try:
+ self.assertEqual(fetch_json(path), {"value": 1})
+ self.assertEqual(LocalFileAdapter._chkpath("put", path)[0], 501)
+ self.assertEqual(LocalFileAdapter._chkpath("patch", path)[0], 405)
+ self.assertEqual(LocalFileAdapter._chkpath("get", path)[0], 200)
+ self.assertEqual(
+ LocalFileAdapter._chkpath("get", path + "-missing")[0], 404
+ )
+ self.assertEqual(
+ LocalFileAdapter._chkpath("get", os.path.dirname(path))[0], 400
+ )
+ finally:
+ os.unlink(path)
+
+ @mock.patch("eodag.utils.requests.requests.sessions.Session.get", autospec=True)
+ def test_fetch_json_timeout_and_request_error(self, mock_get):
+ """fetch_json translates timeout and request failures to EODAG errors."""
+ mock_get.side_effect = requests.exceptions.Timeout()
+ with self.assertRaises(TimeOutError):
+ fetch_json("https://example.test")
+ mock_get.side_effect = requests.exceptions.RequestException()
+ with self.assertRaises(RequestError):
+ fetch_json("https://example.test")
=====================================
utils/ext_product_types_cmp.py
=====================================
@@ -126,8 +126,8 @@ def compare_product_types(
# Compare each provider
common_providers = providers1 & providers2
for provider in sorted(common_providers):
- provider_data1 = data1[provider]
- provider_data2 = data2[provider]
+ provider_data1 = data1[provider] or {}
+ provider_data2 = data2[provider] or {}
# Compare providers_config items content (no list diff, just content)
providers_config1 = set(provider_data1.get("providers_config", {}).keys())
View it on GitLab: https://salsa.debian.org/debian-gis-team/eodag/-/commit/5229ace2b1c6da23af1cd887fcc4e68b99ecf554
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/eodag/-/commit/5229ace2b1c6da23af1cd887fcc4e68b99ecf554
You're receiving this email because of your account on salsa.debian.org. Manage all notifications: https://salsa.debian.org/-/profile/notifications | Help: https://salsa.debian.org/help
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20260828/5acd0ee6/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list