[Git][debian-gis-team/asf-search][master] 5 commits: New upstream version 7.1.1
Antonio Valentino (@antonio.valentino)
gitlab at salsa.debian.org
Tue Apr 30 17:01:28 BST 2024
Antonio Valentino pushed to branch master at Debian GIS Project / asf-search
Commits:
cd8260ce by Antonio Valentino at 2024-04-30T15:56:30+00:00
New upstream version 7.1.1
- - - - -
6d298187 by Antonio Valentino at 2024-04-30T15:56:34+00:00
Update upstream source from tag 'upstream/7.1.1'
Update to upstream version '7.1.1'
with Debian dir a72a7566d670ff831e1345dee09f79a613e4debc
- - - - -
61d9240f by Antonio Valentino at 2024-04-30T15:57:20+00:00
New upstream release
- - - - -
76a2d28d by Antonio Valentino at 2024-04-30T15:58:23+00:00
Drop dependency on python3-dateutil
- - - - -
59931e80 by Antonio Valentino at 2024-04-30T15:58:46+00:00
Set distribution to unstable
- - - - -
15 changed files:
- CHANGELOG.md
- asf_search/ASFProduct.py
- asf_search/ASFSearchOptions/ASFSearchOptions.py
- asf_search/Products/ARIAS1GUNWProduct.py
- asf_search/Products/NISARProduct.py
- asf_search/Products/OPERAS1Product.py
- asf_search/WKT/RepairEntry.py
- asf_search/WKT/validate_wkt.py
- asf_search/baseline/calc.py
- asf_search/baseline/stack.py
- asf_search/search/search.py
- asf_search/search/search_generator.py
- debian/changelog
- debian/control
- setup.py
Changes:
=====================================
CHANGELOG.md
=====================================
@@ -25,15 +25,24 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-->
+------
+## [v7.1.1](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.0...v7.1.1)
+### Changed
+- Uses `ciso8601.parse_datetime()` in baseline calculations, speeds up calculations on larger stacks
+### Added
+- Adds `ASF_LOGGER` logging in `search_generator()` and related methods
+### Fixed
+- `ASFProduct.get_sort_keys()` will no longer returns `None` if missing sort key, defaults to empty string
+
------
## [v7.1.0](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.9...v7.1.0)
### Added
- Improved logging in `ASFSession` authentication methods
-
### Changed
- Uses `ciso8601` module for parsing dates from CMR response, significant performance improvement post-query
- `ASFSession` now allows for authorized user access to hidden/restricted CMR datasets via `auth_with_creds()` or `auth_with_cookiejar()` authentication methods (previously only supported via `auth_with_token()` method)
- `ASFSession.auth_with_token()` now authenticates directly against EDL endpoint
+- UMM Platform ShortName used as final fallback criteria for product subclass assignment
------
## [v7.0.9](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.8...v7.0.9)
=====================================
asf_search/ASFProduct.py
=====================================
@@ -284,12 +284,32 @@ class ASFProduct:
"""
return ASFProduct._base_properties
- def get_sort_keys(self) -> Tuple:
+ def get_sort_keys(self) -> Tuple[str, str]:
"""
Returns tuple of primary and secondary date values used for sorting final search results
+ Any subclasses must return string for final `sort()` to work
"""
- return (self.properties.get('stopTime'), self.properties.get('fileID', 'sceneName'))
-
+ # `sort()` will raise an error when comparing `NoneType`,
+ # using self._read_property() to wrap standard `dict.get()` for possible `None` values
+ primary_key = self._read_property(key='stopTime', default='')
+ secondary_key = self._read_property(
+ key='fileID',
+ default=self._read_property('sceneName', '')
+ )
+
+ return (primary_key, secondary_key)
+
+ def _read_property(self, key: str, default: Any = None) -> Any:
+ """
+ Helper method wraps `properties.get()`.
+ Since a property can be `None`, if the key exists `dict.get('key', 'default')` will never return the default
+ """
+ output = default
+ if (value:=self.properties.get(key)) is not None:
+ output = value
+
+ return output
+
@final
@staticmethod
def umm_get(item: Dict, *args):
=====================================
asf_search/ASFSearchOptions/ASFSearchOptions.py
=====================================
@@ -69,7 +69,7 @@ class ASFSearchOptions:
"""
What to display if `print(opts)` is called.
"""
- return json.dumps(dict(self), indent=4)
+ return json.dumps(dict(self), indent=4, default=str)
# Default is set to '...', since 'None' is a very valid value here
def pop(self, key, default=...):
=====================================
asf_search/Products/ARIAS1GUNWProduct.py
=====================================
@@ -1,5 +1,6 @@
from typing import Dict
from asf_search import ASFSession
+from asf_search.ASFProduct import ASFProduct
from asf_search.ASFSearchOptions import ASFSearchOptions
from asf_search.Products import S1Product
from asf_search.CMR.translate import try_parse_float
@@ -54,4 +55,13 @@ class ARIAS1GUNWProduct(S1Product):
"""
Returns the product type to search for when building a baseline stack.
"""
- return None
\ No newline at end of file
+ return None
+
+ @staticmethod
+ def is_ARIAS1GUNWProduct(item: Dict) -> bool:
+ platform = ASFProduct.umm_get(item['umm'], 'Platforms', 0, 'ShortName')
+ if platform in ['SENTINEL-1A', 'SENTINEL-1B']:
+ asf_platform = ASFProduct.umm_get(item['umm'], 'AdditionalAttributes', ('Name', 'ASF_PLATFORM'), 'Values', 0)
+ return 'Sentinel-1 Interferogram' in asf_platform
+
+ return False
=====================================
asf_search/Products/NISARProduct.py
=====================================
@@ -1,4 +1,4 @@
-from typing import Dict, Union
+from typing import Dict, Tuple, Union
from asf_search import ASFSearchOptions, ASFSession, ASFStackableProduct
from asf_search.CMR.translate import try_parse_float, try_parse_int, try_round_float
from asf_search.constants import PRODUCT_TYPE
@@ -48,10 +48,10 @@ class NISARProduct(ASFStackableProduct):
**NISARProduct._base_properties
}
- def get_sort_keys(self):
+ def get_sort_keys(self) -> Tuple[str, str]:
keys = super().get_sort_keys()
-
- if keys[0] is None:
- return (self.properties.get('processingDate', ''), keys[1])
+
+ if keys[0] == '':
+ return (self._read_property('processingDate', ''), keys[1])
return keys
=====================================
asf_search/Products/OPERAS1Product.py
=====================================
@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, Tuple
from asf_search import ASFSearchOptions, ASFSession
from asf_search.CMR.translate import try_parse_date
from asf_search.Products import S1Product
@@ -71,10 +71,10 @@ class OPERAS1Product(S1Product):
"""
return None
- def get_sort_keys(self):
+ def get_sort_keys(self) -> Tuple[str, str]:
keys = super().get_sort_keys()
- if keys[0] is None:
- keys = self.properties.get('validityStartDate'), keys[1]
+ if keys[0] == '':
+ return (self._read_property('validityStartDate', ''), keys[1])
return keys
=====================================
asf_search/WKT/RepairEntry.py
=====================================
@@ -4,4 +4,4 @@ class RepairEntry:
self.report = report
def __str__(self) -> str:
- return f'{self.report_type}\n\t{self.report}'
+ return f"{self.report_type}: {self.report}"
=====================================
asf_search/WKT/validate_wkt.py
=====================================
@@ -11,7 +11,7 @@ from .RepairEntry import RepairEntry
from asf_search.exceptions import ASFWKTError
-def validate_wkt(aoi: Union[str, BaseGeometry]) -> Tuple[BaseGeometry, List[RepairEntry]]:
+def validate_wkt(aoi: Union[str, BaseGeometry]) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]:
"""
Param aoi: the WKT string or Shapely Geometry to validate and prepare for the CMR query
Validates the given area of interest, and returns a validated and simplified WKT string
@@ -52,7 +52,7 @@ def _search_wkt_prep(shape: BaseGeometry):
if isinstance(shape, Polygon):
return orient(Polygon(shape.exterior), sign=1.0)
-def _simplify_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, List[RepairEntry]]:
+def _simplify_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]:
"""
param geometry: AOI Shapely Geometry to be prepped for CMR
prepares geometry for CMR by:
@@ -165,7 +165,7 @@ def _counter_clockwise_reorientation(geometry: Union[Point, LineString, Polygon]
return reoriented, None
-def _get_clamped_and_wrapped_geometry(shape: BaseGeometry) -> Tuple[BaseGeometry, List[RepairEntry]]:
+def _get_clamped_and_wrapped_geometry(shape: BaseGeometry) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]:
"""
param geometry: Shapely geometry to clamp
Clamps geometry to +/-90 latitude and wraps longitude +/-180
=====================================
asf_search/baseline/calc.py
=====================================
@@ -2,7 +2,7 @@ from math import sqrt, cos, sin, radians
from typing import List
import numpy as np
-from dateutil.parser import parse
+from ciso8601 import parse_datetime
from asf_search import ASFProduct
# WGS84 constants
@@ -23,17 +23,17 @@ def calculate_perpendicular_baselines(reference: str, stack: List[ASFProduct]):
baselineProperties['noStateVectors'] = True
continue
- asc_node_time = parse(baselineProperties['ascendingNodeTime']).timestamp()
+ asc_node_time = parse_datetime(baselineProperties['ascendingNodeTime']).timestamp()
- start = parse(product.properties['startTime']).timestamp()
- end = parse(product.properties['stopTime']).timestamp()
+ start = parse_datetime(product.properties['startTime']).timestamp()
+ end = parse_datetime(product.properties['stopTime']).timestamp()
center = start + ((end - start) / 2)
baselineProperties['relative_start_time'] = start - asc_node_time
baselineProperties['relative_center_time'] = center - asc_node_time
baselineProperties['relative_end_time'] = end - asc_node_time
- t_pre = parse(positionProperties['prePositionTime']).timestamp()
- t_post = parse(positionProperties['postPositionTime']).timestamp()
+ t_pre = parse_datetime(positionProperties['prePositionTime']).timestamp()
+ t_post = parse_datetime(positionProperties['postPositionTime']).timestamp()
product.baseline['relative_sv_pre_time'] = t_pre - asc_node_time
product.baseline['relative_sv_post_time'] = t_post - asc_node_time
=====================================
asf_search/baseline/stack.py
=====================================
@@ -1,5 +1,5 @@
from typing import Tuple, List
-from dateutil.parser import parse
+from ciso8601 import parse_datetime
import pytz
from .calc import calculate_perpendicular_baselines
@@ -66,12 +66,12 @@ def calculate_temporal_baselines(reference: ASFProduct, stack: ASFSearchResults)
:param stack: The stack to operate on.
:return: None, as the operation occurs in-place on the stack provided.
"""
- reference_time = parse(reference.properties['startTime'])
+ reference_time = parse_datetime(reference.properties['startTime'])
if reference_time.tzinfo is None:
reference_time = pytz.utc.localize(reference_time)
for secondary in stack:
- secondary_time = parse(secondary.properties['startTime'])
+ secondary_time = parse_datetime(secondary.properties['startTime'])
if secondary_time.tzinfo is None:
secondary_time = pytz.utc.localize(secondary_time)
secondary.properties['temporalBaseline'] = (secondary_time.date() - reference_time.date()).days
=====================================
asf_search/search/search.py
=====================================
@@ -2,7 +2,7 @@ from typing import Union, Sequence, Tuple
from copy import copy
import datetime
-from asf_search import ASFSearchResults
+from asf_search import ASF_LOGGER, ASFSearchResults
from asf_search.ASFSearchOptions import ASFSearchOptions
from asf_search.search.search_generator import search_generator
@@ -99,6 +99,9 @@ def search(
results.searchComplete = page.searchComplete
results.searchOptions = page.searchOptions
- results.sort(key=lambda p: p.get_sort_keys(), reverse=True)
+ try:
+ results.sort(key=lambda p: p.get_sort_keys(), reverse=True)
+ except TypeError as exc:
+ ASF_LOGGER.warning(f"Failed to sort final results, leaving results unsorted. Reason: {exc}")
return results
=====================================
asf_search/search/search_generator.py
=====================================
@@ -1,4 +1,3 @@
-import logging
from typing import Dict, Generator, Union, Sequence, Tuple, List
from copy import copy
from requests.exceptions import HTTPError
@@ -6,9 +5,8 @@ from requests import ReadTimeout, Response
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential, wait_fixed
import datetime
import dateparser
-import warnings
-from asf_search import __version__
+from asf_search import ASF_LOGGER, __version__
from asf_search.ASFSearchResults import ASFSearchResults
from asf_search.ASFSearchOptions import ASFSearchOptions
@@ -22,7 +20,7 @@ from asf_search.constants import INTERNAL
from asf_search.WKT.validate_wkt import validate_wkt
from asf_search.search.error_reporting import report_search_error
import asf_search.Products as ASFProductType
-
+from shapely.geometry.base import BaseGeometry
def search_generator(
absoluteOrbit: Union[int, Tuple[int, int], range, Sequence[Union[int, Tuple[int, int], range]]] = None,
@@ -82,27 +80,42 @@ def search_generator(
(getattr(opts, 'granule_list', False) or getattr(opts, 'product_list', False)):
raise ValueError("Cannot use maxResults along with product_list/granule_list.")
+ ASF_LOGGER.debug(f'SEARCH: preprocessing opts: {opts}')
preprocess_opts(opts)
+ ASF_LOGGER.debug(f'SEARCH: preprocessed opts: {opts}')
+
+ ASF_LOGGER.info(f'SEARCH: Using search opts {opts}')
url = '/'.join(s.strip('/') for s in [f'https://{opts.host}', f'{INTERNAL.CMR_GRANULE_PATH}'])
total = 0
queries = build_subqueries(opts)
- for query in queries:
+
+ ASF_LOGGER.info(f'SEARCH: Using cmr endpoint: "{url}"')
+ ASF_LOGGER.debug(f'SEARCH: Built {len(queries)} subqueries')
+
+ for subquery_idx, query in enumerate(queries):
+ ASF_LOGGER.info(f'SUBQUERY {subquery_idx + 1}: Beginning subquery with opts: {query}')
+
+ ASF_LOGGER.debug(f'TRANSLATION: Translating subquery:\n{query}')
translated_opts = translate_opts(query)
+ ASF_LOGGER.debug(f'TRANSLATION: Subquery translated to cmr keywords:\n{translated_opts}')
cmr_search_after_header = ""
subquery_count = 0
+ page_number = 1
while(cmr_search_after_header is not None):
try:
+ ASF_LOGGER.debug(f'SUBQUERY {subquery_idx + 1}: Fetching page {page_number}')
items, subquery_max_results, cmr_search_after_header = query_cmr(opts.session, url, translated_opts, subquery_count)
except (ASFSearchError, CMRIncompleteError) as e:
message = str(e)
- logging.error(message)
+ ASF_LOGGER.error(message)
report_search_error(query, message)
opts.session.headers.pop('CMR-Search-After', None)
return
+ ASF_LOGGER.debug(f'SUBQUERY {subquery_idx + 1}: Page {page_number} fetched, returned {len(items)} items.')
opts.session.headers.update({'CMR-Search-After': cmr_search_after_header})
last_page = process_page(items, maxResults, subquery_max_results, total, subquery_count, opts)
subquery_count += len(last_page)
@@ -112,13 +125,18 @@ def search_generator(
if last_page.searchComplete:
if total == maxResults: # the user has as many results as they wanted
+ ASF_LOGGER.info(f'SEARCH COMPLETE: MaxResults ({maxResults}) reached')
opts.session.headers.pop('CMR-Search-After', None)
return
else: # or we've gotten all possible results for this subquery
+ ASF_LOGGER.info(f'SUBQUERY {subquery_idx + 1} COMPLETE: results exhausted for subquery')
cmr_search_after_header = None
+
+ page_number += 1
opts.session.headers.pop('CMR-Search-After', None)
+ ASF_LOGGER.info(f'SEARCH COMPLETE: results exhausted for search opts {opts}')
@retry(reraise=True,
retry=retry_if_exception_type(CMRIncompleteError),
@@ -180,8 +198,10 @@ def preprocess_opts(opts: ASFSearchOptions):
def wrap_wkt(opts: ASFSearchOptions):
if opts.intersectsWith is not None:
- wrapped, _, __ = validate_wkt(opts.intersectsWith)
+ wrapped, _, repairs = validate_wkt(opts.intersectsWith)
opts.intersectsWith = wrapped.wkt
+ if len(repairs):
+ ASF_LOGGER.warning(f"WKT REPAIR/VALIDATION: The following repairs were performed on the provided AOI:\n{[str(repair) for repair in repairs]}")
def set_default_dates(opts: ASFSearchOptions):
@@ -192,7 +212,7 @@ def set_default_dates(opts: ASFSearchOptions):
# If both are used, make sure they're in the right order:
if opts.start is not None and opts.end is not None:
if opts.start > opts.end:
- warnings.warn(f"Start date ({opts.start}) is after end date ({opts.end}). Switching the two.")
+ ASF_LOGGER.warning(f"Start date ({opts.start}) is after end date ({opts.end}). Switching the two.")
opts.start, opts.end = opts.end, opts.start
# Can't do this sooner, since you need to compare start vs end:
if opts.start is not None:
@@ -253,7 +273,7 @@ def as_ASFProduct(item: Dict, session: ASFSession) -> ASFProduct:
if subclass is not None:
return subclass(item, session=session)
- # or if the key matches one of the shortnames in any of our datasets
+ # if the key matches one of the shortnames in any of our datasets
for dataset, collections in dataset_collections.items():
if collections.get(product_type_key) is not None:
subclass = dataset_to_product_types.get(dataset)
@@ -261,7 +281,21 @@ def as_ASFProduct(item: Dict, session: ASFSession) -> ASFProduct:
return subclass(item, session=session)
break # dataset exists, but is not in dataset_to_product_types yet
- return ASFProduct(item, session=session)
+ # If the platform exists, try to match it
+ platform = _get_platform(item=item)
+ if ASFProductType.ARIAS1GUNWProduct.is_ARIAS1GUNWProduct(item=item):
+ return dataset_to_product_types.get('ARIA S1 GUNW')(item, session=session)
+ elif (subclass := dataset_to_product_types.get(platform)) is not None:
+ return subclass(item, session=session)
+
+ output = ASFProduct(item, session=session)
+
+ granule_concept_id = output.meta.get('concept-id', 'Missing Granule Concept ID')
+ fileID = output.properties.get('fileID', output.properties.get('sceneName', 'fileID and sceneName Missing'))
+
+ ASF_LOGGER.warning(f'Failed to find corresponding ASFProduct subclass for \
+ Product: "{fileID}", Granule Concept ID: "{granule_concept_id}", default to "ASFProduct"')
+ return output
def _get_product_type_key(item: Dict) -> str:
"""Match the umm response to the right ASFProduct subclass by returning one of the following:
@@ -272,16 +306,17 @@ def _get_product_type_key(item: Dict) -> str:
collection_shortName = ASFProduct.umm_get(item['umm'], 'CollectionReference', 'ShortName')
if collection_shortName is None:
- platform_shortname = ASFProduct.umm_get(item['umm'], 'Platforms', 0, 'ShortName')
- if platform_shortname in ['SENTINEL-1A', 'SENTINEL-1B']:
- asf_platform = ASFProduct.umm_get(item['umm'], 'AdditionalAttributes', ('Name', 'ASF_PLATFORM'), 'Values', 0)
- if 'Sentinel-1 Interferogram' in asf_platform:
- return 'ARIA S1 GUNW'
+ platform = _get_platform(item=item)
+ if ASFProductType.ARIAS1GUNWProduct.is_ARIAS1GUNWProduct(item=item):
+ return 'ARIA S1 GUNW'
- return platform_shortname
+ return platform
return collection_shortName
+def _get_platform(item: Dict):
+ return ASFProduct.umm_get(item['umm'], 'Platforms', 0, 'ShortName')
+
# Maps datasets from DATASET.py and collection/platform shortnames to ASFProduct subclasses
dataset_to_product_types = {
'SENTINEL-1': ASFProductType.S1Product,
=====================================
debian/changelog
=====================================
@@ -1,3 +1,11 @@
+asf-search (7.1.1-1) unstable; urgency=medium
+
+ * New upstream release.
+ * debian/control:
+ - Drop dependency on python3-dateutil.
+
+ -- Antonio Valentino <antonio.valentino at tiscali.it> Tue, 30 Apr 2024 15:58:30 +0000
+
asf-search (7.1.0-1) unstable; urgency=medium
* New upstream release.
=====================================
debian/control
=====================================
@@ -9,7 +9,6 @@ Build-Depends: debhelper-compat (= 13),
python3-all,
python3-ciso8601,
python3-dateparser,
- python3-dateutil,
python3-importlib-metadata,
python3-numpy,
python3-remotezip,
=====================================
setup.py
=====================================
@@ -4,7 +4,6 @@ from setuptools import find_packages, setup
requirements = [
"requests",
"shapely",
- "python-dateutil",
"pytz",
"importlib_metadata",
"numpy",
@@ -14,8 +13,8 @@ requirements = [
]
test_requirements = [
- "pytest < 7.2.0",
- "pytest-automation",
+ "pytest==8.1.1",
+ "pytest-automation==3.0.0",
"pytest-cov",
"pytest-xdist",
"coverage",
@@ -54,7 +53,7 @@ setup(
license='BSD',
license_files=('LICENSE',),
classifiers=[
- "Development Status :: 2 - Pre-Alpha",
+ "Development Status :: 5 - Production/Stable",
"License :: OSI Approved :: BSD License",
"Operating System :: OS Independent",
"Intended Audience :: Developers",
@@ -63,6 +62,9 @@ setup(
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
"Topic :: Software Development",
"Topic :: Scientific/Engineering :: Atmospheric Science",
"Topic :: Scientific/Engineering :: GIS",
View it on GitLab: https://salsa.debian.org/debian-gis-team/asf-search/-/compare/b1134f5b5fd35d23deaf5275cbf58bea7adfbe34...59931e8048a814d1fb6cc7ada297c0a63e2a72ab
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/asf-search/-/compare/b1134f5b5fd35d23deaf5275cbf58bea7adfbe34...59931e8048a814d1fb6cc7ada297c0a63e2a72ab
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20240430/79dee749/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list