[Git][debian-gis-team/asf-search][upstream] New upstream version 7.1.1

Antonio Valentino (@antonio.valentino) gitlab at salsa.debian.org
Tue Apr 30 17:01:41 BST 2024



Antonio Valentino pushed to branch upstream at Debian GIS Project / asf-search


Commits:
cd8260ce by Antonio Valentino at 2024-04-30T15:56:30+00:00
New upstream version 7.1.1
- - - - -


13 changed files:

- CHANGELOG.md
- asf_search/ASFProduct.py
- asf_search/ASFSearchOptions/ASFSearchOptions.py
- asf_search/Products/ARIAS1GUNWProduct.py
- asf_search/Products/NISARProduct.py
- asf_search/Products/OPERAS1Product.py
- asf_search/WKT/RepairEntry.py
- asf_search/WKT/validate_wkt.py
- asf_search/baseline/calc.py
- asf_search/baseline/stack.py
- asf_search/search/search.py
- asf_search/search/search_generator.py
- setup.py


Changes:

=====================================
CHANGELOG.md
=====================================
@@ -25,15 +25,24 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 -
 
 -->
+------
+## [v7.1.1](https://github.com/asfadmin/Discovery-asf_search/compare/v7.1.0...v7.1.1)
+### Changed
+- Uses `ciso8601.parse_datetime()` in baseline calculations, speeds up calculations on larger stacks
+### Added
+- Adds `ASF_LOGGER` logging in `search_generator()` and related methods
+### Fixed
+- `ASFProduct.get_sort_keys()` will no longer returns `None` if missing sort key, defaults to empty string 
+
 ------
 ## [v7.1.0](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.9...v7.1.0)
 ### Added
 - Improved logging in `ASFSession` authentication methods
-
 ### Changed
 - Uses `ciso8601` module for parsing dates from CMR response, significant performance improvement post-query
 - `ASFSession` now allows for authorized user access to hidden/restricted CMR datasets via `auth_with_creds()` or `auth_with_cookiejar()` authentication methods (previously only supported via `auth_with_token()` method)
 - `ASFSession.auth_with_token()` now authenticates directly against EDL endpoint
+- UMM Platform ShortName used as final fallback criteria for product subclass assignment
 
 ------
 ## [v7.0.9](https://github.com/asfadmin/Discovery-asf_search/compare/v7.0.8...v7.0.9)


=====================================
asf_search/ASFProduct.py
=====================================
@@ -284,12 +284,32 @@ class ASFProduct:
         """
         return ASFProduct._base_properties
 
-    def get_sort_keys(self) -> Tuple:
+    def get_sort_keys(self) -> Tuple[str, str]:
         """
         Returns tuple of primary and secondary date values used for sorting final search results
+        Any subclasses must return string for final `sort()` to work
         """
-        return (self.properties.get('stopTime'), self.properties.get('fileID', 'sceneName'))
-
+        # `sort()` will raise an error when comparing `NoneType`,
+        # using self._read_property() to wrap standard `dict.get()` for possible `None` values
+        primary_key = self._read_property(key='stopTime', default='')
+        secondary_key = self._read_property(
+            key='fileID', 
+            default=self._read_property('sceneName', '')
+        )
+        
+        return (primary_key, secondary_key)
+    
+    def _read_property(self, key: str, default: Any = None) -> Any:
+        """
+        Helper method wraps `properties.get()`.
+        Since a property can be `None`, if the key exists `dict.get('key', 'default')` will never return the default
+        """
+        output = default
+        if (value:=self.properties.get(key)) is not None:
+            output = value
+        
+        return output
+            
     @final
     @staticmethod
     def umm_get(item: Dict, *args):


=====================================
asf_search/ASFSearchOptions/ASFSearchOptions.py
=====================================
@@ -69,7 +69,7 @@ class ASFSearchOptions:
         """
         What to display if `print(opts)` is called.
         """
-        return json.dumps(dict(self), indent=4)
+        return json.dumps(dict(self), indent=4, default=str)
 
     # Default is set to '...', since 'None' is a very valid value here
     def pop(self, key, default=...):


=====================================
asf_search/Products/ARIAS1GUNWProduct.py
=====================================
@@ -1,5 +1,6 @@
 from typing import Dict
 from asf_search import ASFSession
+from asf_search.ASFProduct import ASFProduct
 from asf_search.ASFSearchOptions import ASFSearchOptions
 from asf_search.Products import S1Product
 from asf_search.CMR.translate import try_parse_float
@@ -54,4 +55,13 @@ class ARIAS1GUNWProduct(S1Product):
         """
         Returns the product type to search for when building a baseline stack.
         """
-        return None
\ No newline at end of file
+        return None
+
+    @staticmethod
+    def is_ARIAS1GUNWProduct(item: Dict) -> bool:
+        platform = ASFProduct.umm_get(item['umm'], 'Platforms', 0, 'ShortName')
+        if platform in ['SENTINEL-1A', 'SENTINEL-1B']:
+            asf_platform = ASFProduct.umm_get(item['umm'], 'AdditionalAttributes', ('Name', 'ASF_PLATFORM'), 'Values', 0)
+            return 'Sentinel-1 Interferogram' in asf_platform
+
+        return False


=====================================
asf_search/Products/NISARProduct.py
=====================================
@@ -1,4 +1,4 @@
-from typing import Dict, Union
+from typing import Dict, Tuple, Union
 from asf_search import ASFSearchOptions, ASFSession, ASFStackableProduct
 from asf_search.CMR.translate import try_parse_float, try_parse_int, try_round_float
 from asf_search.constants import PRODUCT_TYPE
@@ -48,10 +48,10 @@ class NISARProduct(ASFStackableProduct):
             **NISARProduct._base_properties
         }
 
-    def get_sort_keys(self):
+    def get_sort_keys(self) -> Tuple[str, str]:
         keys = super().get_sort_keys()
-
-        if keys[0] is None:
-            return (self.properties.get('processingDate', ''), keys[1])
+        
+        if keys[0] == '':
+            return (self._read_property('processingDate', ''), keys[1])
 
         return keys


=====================================
asf_search/Products/OPERAS1Product.py
=====================================
@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, Tuple
 from asf_search import ASFSearchOptions, ASFSession
 from asf_search.CMR.translate import try_parse_date
 from asf_search.Products import S1Product
@@ -71,10 +71,10 @@ class OPERAS1Product(S1Product):
         """
         return None
 
-    def get_sort_keys(self):
+    def get_sort_keys(self) -> Tuple[str, str]:
         keys = super().get_sort_keys()
 
-        if keys[0] is None:
-            keys = self.properties.get('validityStartDate'), keys[1]
+        if keys[0] == '':
+            return (self._read_property('validityStartDate', ''), keys[1])
 
         return keys


=====================================
asf_search/WKT/RepairEntry.py
=====================================
@@ -4,4 +4,4 @@ class RepairEntry:
         self.report = report
     
     def __str__(self) -> str:
-        return f'{self.report_type}\n\t{self.report}'
+        return f"{self.report_type}: {self.report}"


=====================================
asf_search/WKT/validate_wkt.py
=====================================
@@ -11,7 +11,7 @@ from .RepairEntry import RepairEntry
 from asf_search.exceptions import ASFWKTError
 
 
-def validate_wkt(aoi: Union[str, BaseGeometry]) -> Tuple[BaseGeometry, List[RepairEntry]]:
+def validate_wkt(aoi: Union[str, BaseGeometry]) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]:
     """
     Param aoi: the WKT string or Shapely Geometry to validate and prepare for the CMR query
     Validates the given area of interest, and returns a validated and simplified WKT string
@@ -52,7 +52,7 @@ def _search_wkt_prep(shape: BaseGeometry):
     if isinstance(shape, Polygon):
         return orient(Polygon(shape.exterior), sign=1.0)
 
-def _simplify_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, List[RepairEntry]]:
+def _simplify_geometry(geometry: BaseGeometry) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]:
     """
     param geometry: AOI Shapely Geometry to be prepped for CMR 
     prepares geometry for CMR by:
@@ -165,7 +165,7 @@ def _counter_clockwise_reorientation(geometry: Union[Point, LineString, Polygon]
     return reoriented, None
 
 
-def _get_clamped_and_wrapped_geometry(shape: BaseGeometry) -> Tuple[BaseGeometry, List[RepairEntry]]:
+def _get_clamped_and_wrapped_geometry(shape: BaseGeometry) -> Tuple[BaseGeometry, BaseGeometry, List[RepairEntry]]:
     """
     param geometry: Shapely geometry to clamp    
     Clamps geometry to +/-90 latitude and wraps longitude +/-180


=====================================
asf_search/baseline/calc.py
=====================================
@@ -2,7 +2,7 @@ from math import sqrt, cos, sin, radians
 from typing import List
 
 import numpy as np
-from dateutil.parser import parse
+from ciso8601 import parse_datetime
 
 from asf_search import ASFProduct
 # WGS84 constants
@@ -23,17 +23,17 @@ def calculate_perpendicular_baselines(reference: str, stack: List[ASFProduct]):
             baselineProperties['noStateVectors'] = True
             continue
 
-        asc_node_time = parse(baselineProperties['ascendingNodeTime']).timestamp()
+        asc_node_time = parse_datetime(baselineProperties['ascendingNodeTime']).timestamp()
 
-        start = parse(product.properties['startTime']).timestamp()
-        end = parse(product.properties['stopTime']).timestamp()
+        start = parse_datetime(product.properties['startTime']).timestamp()
+        end = parse_datetime(product.properties['stopTime']).timestamp()
         center = start + ((end - start) / 2)
         baselineProperties['relative_start_time'] = start - asc_node_time
         baselineProperties['relative_center_time'] = center - asc_node_time
         baselineProperties['relative_end_time'] = end - asc_node_time
 
-        t_pre = parse(positionProperties['prePositionTime']).timestamp()
-        t_post = parse(positionProperties['postPositionTime']).timestamp()
+        t_pre = parse_datetime(positionProperties['prePositionTime']).timestamp()
+        t_post = parse_datetime(positionProperties['postPositionTime']).timestamp()
         product.baseline['relative_sv_pre_time'] = t_pre - asc_node_time
         product.baseline['relative_sv_post_time'] = t_post - asc_node_time
 


=====================================
asf_search/baseline/stack.py
=====================================
@@ -1,5 +1,5 @@
 from typing import Tuple, List
-from dateutil.parser import parse
+from ciso8601 import parse_datetime
 import pytz
 
 from .calc import calculate_perpendicular_baselines
@@ -66,12 +66,12 @@ def calculate_temporal_baselines(reference: ASFProduct, stack: ASFSearchResults)
     :param stack: The stack to operate on.
     :return: None, as the operation occurs in-place on the stack provided.
     """
-    reference_time = parse(reference.properties['startTime'])
+    reference_time = parse_datetime(reference.properties['startTime'])
     if reference_time.tzinfo is None:
         reference_time = pytz.utc.localize(reference_time)
 
     for secondary in stack:
-        secondary_time = parse(secondary.properties['startTime'])
+        secondary_time = parse_datetime(secondary.properties['startTime'])
         if secondary_time.tzinfo is None:
             secondary_time = pytz.utc.localize(secondary_time)
         secondary.properties['temporalBaseline'] = (secondary_time.date() - reference_time.date()).days


=====================================
asf_search/search/search.py
=====================================
@@ -2,7 +2,7 @@ from typing import Union, Sequence, Tuple
 from copy import copy
 import datetime
 
-from asf_search import ASFSearchResults
+from asf_search import ASF_LOGGER, ASFSearchResults
 from asf_search.ASFSearchOptions import ASFSearchOptions
 from asf_search.search.search_generator import search_generator
 
@@ -99,6 +99,9 @@ def search(
         results.searchComplete = page.searchComplete
         results.searchOptions = page.searchOptions
     
-    results.sort(key=lambda p: p.get_sort_keys(), reverse=True)
+    try:
+        results.sort(key=lambda p: p.get_sort_keys(), reverse=True)
+    except TypeError as exc:
+        ASF_LOGGER.warning(f"Failed to sort final results, leaving results unsorted. Reason: {exc}")
     
     return results


=====================================
asf_search/search/search_generator.py
=====================================
@@ -1,4 +1,3 @@
-import logging
 from typing import Dict, Generator, Union, Sequence, Tuple, List
 from copy import copy
 from requests.exceptions import HTTPError
@@ -6,9 +5,8 @@ from requests import ReadTimeout, Response
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential, wait_fixed
 import datetime
 import dateparser
-import warnings
 
-from asf_search import __version__
+from asf_search import ASF_LOGGER, __version__
 
 from asf_search.ASFSearchResults import ASFSearchResults
 from asf_search.ASFSearchOptions import ASFSearchOptions
@@ -22,7 +20,7 @@ from asf_search.constants import INTERNAL
 from asf_search.WKT.validate_wkt import validate_wkt
 from asf_search.search.error_reporting import report_search_error
 import asf_search.Products as ASFProductType
-
+from shapely.geometry.base import BaseGeometry
 
 def search_generator(
         absoluteOrbit: Union[int, Tuple[int, int], range, Sequence[Union[int, Tuple[int, int], range]]] = None,
@@ -82,27 +80,42 @@ def search_generator(
         (getattr(opts, 'granule_list', False) or getattr(opts, 'product_list', False)):
             raise ValueError("Cannot use maxResults along with product_list/granule_list.")
     
+    ASF_LOGGER.debug(f'SEARCH: preprocessing opts: {opts}')
     preprocess_opts(opts)
+    ASF_LOGGER.debug(f'SEARCH: preprocessed opts: {opts}')
+    
+    ASF_LOGGER.info(f'SEARCH: Using search opts {opts}')
 
     url = '/'.join(s.strip('/') for s in [f'https://{opts.host}', f'{INTERNAL.CMR_GRANULE_PATH}'])
     total = 0
 
     queries = build_subqueries(opts)
-    for query in queries:
+
+    ASF_LOGGER.info(f'SEARCH: Using cmr endpoint: "{url}"')
+    ASF_LOGGER.debug(f'SEARCH: Built {len(queries)} subqueries')
+    
+    for subquery_idx, query in enumerate(queries):
+        ASF_LOGGER.info(f'SUBQUERY {subquery_idx + 1}: Beginning subquery with opts: {query}')
+
+        ASF_LOGGER.debug(f'TRANSLATION: Translating subquery:\n{query}')
         translated_opts = translate_opts(query)
+        ASF_LOGGER.debug(f'TRANSLATION: Subquery translated to cmr keywords:\n{translated_opts}')
         cmr_search_after_header = ""
         subquery_count = 0
 
+        page_number = 1
         while(cmr_search_after_header is not None):
             try:
+                ASF_LOGGER.debug(f'SUBQUERY {subquery_idx + 1}: Fetching page {page_number}')
                 items, subquery_max_results, cmr_search_after_header = query_cmr(opts.session, url, translated_opts, subquery_count)
             except (ASFSearchError, CMRIncompleteError) as e:
                 message = str(e)
-                logging.error(message)
+                ASF_LOGGER.error(message)
                 report_search_error(query, message)
                 opts.session.headers.pop('CMR-Search-After', None)
                 return
 
+            ASF_LOGGER.debug(f'SUBQUERY {subquery_idx + 1}: Page {page_number} fetched, returned {len(items)} items.')
             opts.session.headers.update({'CMR-Search-After': cmr_search_after_header})
             last_page = process_page(items, maxResults, subquery_max_results, total, subquery_count, opts)
             subquery_count += len(last_page)
@@ -112,13 +125,18 @@ def search_generator(
 
             if last_page.searchComplete:
                 if total == maxResults: # the user has as many results as they wanted
+                    ASF_LOGGER.info(f'SEARCH COMPLETE: MaxResults ({maxResults}) reached')
                     opts.session.headers.pop('CMR-Search-After', None)
                     return
                 else: # or we've gotten all possible results for this subquery
+                    ASF_LOGGER.info(f'SUBQUERY {subquery_idx + 1} COMPLETE: results exhausted for subquery')
                     cmr_search_after_header = None
+            
+            page_number += 1
 
         opts.session.headers.pop('CMR-Search-After', None)
 
+    ASF_LOGGER.info(f'SEARCH COMPLETE: results exhausted for search opts {opts}')
 
 @retry(reraise=True,
        retry=retry_if_exception_type(CMRIncompleteError),
@@ -180,8 +198,10 @@ def preprocess_opts(opts: ASFSearchOptions):
 
 def wrap_wkt(opts: ASFSearchOptions):
     if opts.intersectsWith is not None:
-        wrapped, _, __ = validate_wkt(opts.intersectsWith)
+        wrapped, _, repairs = validate_wkt(opts.intersectsWith)
         opts.intersectsWith = wrapped.wkt
+        if len(repairs):
+            ASF_LOGGER.warning(f"WKT REPAIR/VALIDATION: The following repairs were performed on the provided AOI:\n{[str(repair) for repair in repairs]}")
 
 
 def set_default_dates(opts: ASFSearchOptions):
@@ -192,7 +212,7 @@ def set_default_dates(opts: ASFSearchOptions):
     # If both are used, make sure they're in the right order:
     if opts.start is not None and opts.end is not None:
         if opts.start > opts.end:
-            warnings.warn(f"Start date ({opts.start}) is after end date ({opts.end}). Switching the two.")
+            ASF_LOGGER.warning(f"Start date ({opts.start}) is after end date ({opts.end}). Switching the two.")
             opts.start, opts.end = opts.end, opts.start
     # Can't do this sooner, since you need to compare start vs end:
     if opts.start is not None:
@@ -253,7 +273,7 @@ def as_ASFProduct(item: Dict, session: ASFSession) -> ASFProduct:
     if subclass is not None:
         return subclass(item, session=session)
 
-    # or if the key matches one of the shortnames in any of our datasets
+    # if the key matches one of the shortnames in any of our datasets
     for dataset, collections in dataset_collections.items():
         if collections.get(product_type_key) is not None:
             subclass = dataset_to_product_types.get(dataset)
@@ -261,7 +281,21 @@ def as_ASFProduct(item: Dict, session: ASFSession) -> ASFProduct:
                 return subclass(item, session=session)
             break # dataset exists, but is not in dataset_to_product_types yet
 
-    return ASFProduct(item, session=session)
+    # If the platform exists, try to match it
+    platform = _get_platform(item=item)
+    if ASFProductType.ARIAS1GUNWProduct.is_ARIAS1GUNWProduct(item=item):
+        return dataset_to_product_types.get('ARIA S1 GUNW')(item, session=session)
+    elif (subclass := dataset_to_product_types.get(platform)) is not None:
+        return subclass(item, session=session)
+    
+    output = ASFProduct(item, session=session)
+    
+    granule_concept_id = output.meta.get('concept-id', 'Missing Granule Concept ID')
+    fileID = output.properties.get('fileID', output.properties.get('sceneName', 'fileID and sceneName Missing'))
+
+    ASF_LOGGER.warning(f'Failed to find corresponding ASFProduct subclass for \
+                       Product: "{fileID}", Granule Concept ID: "{granule_concept_id}", default to "ASFProduct"')
+    return output
 
 def _get_product_type_key(item: Dict) -> str:
     """Match the umm response to the right ASFProduct subclass by returning one of the following:
@@ -272,16 +306,17 @@ def _get_product_type_key(item: Dict) -> str:
     collection_shortName = ASFProduct.umm_get(item['umm'], 'CollectionReference', 'ShortName')
 
     if collection_shortName is None:
-        platform_shortname = ASFProduct.umm_get(item['umm'], 'Platforms', 0, 'ShortName')
-        if platform_shortname in ['SENTINEL-1A', 'SENTINEL-1B']:
-            asf_platform = ASFProduct.umm_get(item['umm'], 'AdditionalAttributes', ('Name', 'ASF_PLATFORM'), 'Values', 0)
-            if 'Sentinel-1 Interferogram' in asf_platform:
-                return 'ARIA S1 GUNW'
+        platform = _get_platform(item=item)
+        if ASFProductType.ARIAS1GUNWProduct.is_ARIAS1GUNWProduct(item=item):
+            return 'ARIA S1 GUNW'
 
-        return platform_shortname
+        return platform
 
     return collection_shortName
 
+def _get_platform(item: Dict):
+    return ASFProduct.umm_get(item['umm'], 'Platforms', 0, 'ShortName')
+
 # Maps datasets from DATASET.py and collection/platform shortnames to ASFProduct subclasses
 dataset_to_product_types = {
     'SENTINEL-1': ASFProductType.S1Product,


=====================================
setup.py
=====================================
@@ -4,7 +4,6 @@ from setuptools import find_packages, setup
 requirements = [
     "requests",
     "shapely",
-    "python-dateutil",
     "pytz",
     "importlib_metadata",
     "numpy",
@@ -14,8 +13,8 @@ requirements = [
 ]
 
 test_requirements = [
-    "pytest < 7.2.0",
-    "pytest-automation",
+    "pytest==8.1.1",
+    "pytest-automation==3.0.0",
     "pytest-cov",
     "pytest-xdist",
     "coverage",
@@ -54,7 +53,7 @@ setup(
     license='BSD',
     license_files=('LICENSE',),
     classifiers=[
-        "Development Status :: 2 - Pre-Alpha",
+        "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
         "Intended Audience :: Developers",
@@ -63,6 +62,9 @@ setup(
         "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Topic :: Software Development",
         "Topic :: Scientific/Engineering :: Atmospheric Science",
         "Topic :: Scientific/Engineering :: GIS",



View it on GitLab: https://salsa.debian.org/debian-gis-team/asf-search/-/commit/cd8260cef78c93db3f4a7fc135c626e79466c91f

-- 
View it on GitLab: https://salsa.debian.org/debian-gis-team/asf-search/-/commit/cd8260cef78c93db3f4a7fc135c626e79466c91f
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20240430/002f4233/attachment-0001.htm>


More information about the Pkg-grass-devel mailing list