[Git][debian-gis-team/asf-search][upstream] New upstream version 6.7.0
Antonio Valentino (@antonio.valentino)
gitlab at salsa.debian.org
Sat Nov 11 07:57:31 GMT 2023
Antonio Valentino pushed to branch upstream at Debian GIS Project / asf-search
Commits:
5065ffa9 by Antonio Valentino at 2023-11-11T07:54:22+00:00
New upstream version 6.7.0
- - - - -
17 changed files:
- CHANGELOG.md
- asf_search/ASFProduct.py
- asf_search/ASFSearchOptions/ASFSearchOptions.py
- asf_search/ASFSearchOptions/validator_map.py
- asf_search/CMR/__init__.py
- + asf_search/CMR/datasets.py
- asf_search/CMR/field_map.py
- asf_search/CMR/subquery.py
- asf_search/CMR/translate.py
- asf_search/constants/INTERNAL.py
- asf_search/download/download.py
- asf_search/search/search.py
- asf_search/search/search_generator.py
- tests/Search/test_search.py
- tests/pytest-config.yml
- tests/pytest-managers.py
- tests/yml_tests/test_search.yml
Changes:
=====================================
CHANGELOG.md
=====================================
@@ -25,6 +25,22 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-->
+------
+## [v6.7.0](https://github.com/asfadmin/Discovery-asf_search/compare/v6.6.3...v6.7.0)
+### Added
+- Adds new `dataset` keyword to `search()` as an alternative to `platform`. Allows users to get results from multiple platforms at once in a single page
+- Adds `operaBurstID` keyword to `search()`
+- Adds OPERA-S1 param `operaBurstID` to `ASFProduct.properties`, and adds Opera product urls to `additionalUrls`
+- OPERA-S1 RTC product `polarization` now shows both polarizations as list
+- adds `frameNumber` properties support for new `Sentinel-1 Interferogram` products
+- added `CMR_TIMEOUT` constant. This is the amount of time in seconds to wait without seeing *any* data. (Default=30)
+
+### Changed
+- Changes `CMR_FORMAT_EXT` constant from `umm_json_v1_4` to `umm_json`, umm returned from CMR will now be in latest umm format by default
+
+### Fixed
+- ERS-1, ERS-2, JERS-1, and RADARSAT-1 now assign `FRAME_NUMBER` to the `frameNumber` properties field
+
------
## [v6.6.3](https://github.com/asfadmin/Discovery-asf_search/compare/v6.6.2...v6.6.3)
### Fixed
=====================================
asf_search/ASFProduct.py
=====================================
@@ -1,7 +1,10 @@
+import os
import warnings
from shapely.geometry import shape, Point, Polygon, mapping
import json
+from urllib import parse
+
from asf_search import ASFSession, ASFSearchResults
from asf_search.ASFSearchOptions import ASFSearchOptions
from asf_search.download import download_url
@@ -64,10 +67,14 @@ class ASFProduct:
def get_additional_urls():
output = []
- base_filename = '.'.join(default_filename.split('.')[:-1])
for url in self.properties['additionalUrls']:
- extension = url.split('.')[-1]
- urls.append((f"{base_filename}.{extension}", url))
+ if self.properties['processingLevel'] == 'BURST':
+ # Burst XML filenames are just numbers, this makes it more indentifiable
+ file_name = '.'.join(default_filename.split('.')[:-1]) + url.split('.')[-1]
+ else:
+ # otherwise just use the name found in the url
+ file_name = os.path.split(parse.urlparse(url).path)[1]
+ urls.append((f"{file_name}", url))
return output
=====================================
asf_search/ASFSearchOptions/ASFSearchOptions.py
=====================================
@@ -114,7 +114,7 @@ class ASFSearchOptions:
# Spit out warning if the value is something other than the default:
if not self._is_val_default(key):
msg = f'While merging search options, existing option {key}:{getattr(self, key, None)} overwritten by kwarg with value {kwargs[key]}'
- ASF_LOGGER.warging(msg)
+ ASF_LOGGER.warning(msg)
warnings.warn(msg)
self.__setattr__(key, kwargs[key])
=====================================
asf_search/ASFSearchOptions/validator_map.py
=====================================
@@ -57,12 +57,14 @@ validator_map = {
'instrument': parse_string,
'collections': parse_string_list,
'temporalBaselineDays': parse_string_list,
-
+ 'operaBurstID': parse_string_list,
+ 'absoluteBurstID': parse_int_list,
+ 'relativeBurstID': parse_int_list,
+ 'fullBurstID': parse_string_list,
+ 'dataset': parse_string_list,
+
# Config parameters Parser
'session': parse_session,
'host': parse_string,
- 'provider': parse_string,
- 'absoluteBurstID': parse_int_list,
- 'relativeBurstID': parse_int_list,
- 'fullBurstID': parse_string_list
+ 'provider': parse_string
}
=====================================
asf_search/CMR/__init__.py
=====================================
@@ -1,4 +1,5 @@
from .MissionList import get_campaigns
from .subquery import build_subqueries
from .translate import translate_product, translate_opts, get_additional_fields
-from .field_map import field_map
\ No newline at end of file
+from .field_map import field_map
+from .datasets import dataset_collections
\ No newline at end of file
=====================================
asf_search/CMR/datasets.py
=====================================
@@ -0,0 +1,327 @@
+dataset_collections = {
+ "SENTINEL-1": {
+ "SENTINEL-1A_SLC": ["C1214470488-ASF", "C1205428742-ASF", "C1234413245-ASFDEV"],
+ "SENTINEL-1B_SLC": ["C1327985661-ASF", "C1216244348-ASF", "C1234413263-ASFDEV"],
+ "SENTINEL-1A_DP_GRD_HIGH": [
+ "C1214470533-ASF",
+ "C1212201032-ASF",
+ "C1234413229-ASFDEV",
+ ],
+ "SENTINEL-1A_DP_META_GRD_HIGH": [
+ "C1214470576-ASF",
+ "C1212209226-ASF",
+ "C1234413232-ASFDEV",
+ ],
+ "SENTINEL-1B_DP_GRD_HIGH": [
+ "C1327985645-ASF",
+ "C1216244589-ASF",
+ "C1234413247-ASFDEV",
+ ],
+ "SENTINEL-1A_META_SLC": [
+ "C1214470496-ASF",
+ "C1208117434-ASF",
+ "C1234413236-ASFDEV",
+ ],
+ "SENTINEL-1A_META_RAW": [
+ "C1214470532-ASF",
+ "C1208115009-ASF",
+ "C1234413235-ASFDEV",
+ ],
+ "SENTINEL-1A_OCN": ["C1214472977-ASF", "C1212212560-ASF", "C1234413237-ASFDEV"],
+ "SENTINEL-1A_DP_META_GRD_MEDIUM": [
+ "C1214472336-ASF",
+ "C1212212493-ASF",
+ "C1234413233-ASFDEV",
+ ],
+ "SENTINEL-1A_META_OCN": [
+ "C1266376001-ASF",
+ "C1215704763-ASF",
+ "C1234413234-ASFDEV",
+ ],
+ "SENTINEL-1A_SP_META_GRD_HIGH": [
+ "C1214470732-ASF",
+ "C1212158326-ASF",
+ "C1234413243-ASFDEV",
+ ],
+ "SENTINEL-1B_DP_GRD_MEDIUM": [
+ "C1327985660-ASF",
+ "C1216244594-ASF",
+ "C1234413248-ASFDEV",
+ ],
+ "SENTINEL-1B_DP_META_GRD_HIGH": [
+ "C1327985741-ASF",
+ "C1216244601-ASF",
+ "C1234413250-ASFDEV",
+ ],
+ "SENTINEL-1B_DP_META_GRD_MEDIUM": [
+ "C1327985578-ASF",
+ "C1216244591-ASF",
+ "C1234413251-ASFDEV",
+ ],
+ "SENTINEL-1B_META_RAW": [
+ "C1327985650-ASF",
+ "C1216244595-ASF",
+ "C1234413253-ASFDEV",
+ ],
+ "SENTINEL-1B_META_SLC": [
+ "C1327985617-ASF",
+ "C1216244585-ASF",
+ "C1234413254-ASFDEV",
+ ],
+ "SENTINEL-1B_OCN": ["C1327985579-ASF", "C1216244593-ASF", "C1234413255-ASFDEV"],
+ "SENTINEL-1B_SP_META_GRD_HIGH": [
+ "C1327985619-ASF",
+ "C1216244587-ASF",
+ "C1234413261-ASFDEV",
+ ],
+ "SENTINEL-1A_SP_GRD_MEDIUM": [
+ "C1214472994-ASF",
+ "C1212158318-ASF",
+ "C1234413241-ASFDEV",
+ ],
+ "SENTINEL-1A_SP_META_GRD_MEDIUM": [
+ "C1214473170-ASF",
+ "C1212233976-ASF",
+ "C1234413244-ASFDEV",
+ ],
+ "SENTINEL-1B_META_OCN": [
+ "C1327985646-ASF",
+ "C1216244590-ASF",
+ "C1234413252-ASFDEV",
+ ],
+ "SENTINEL-1B_SP_GRD_MEDIUM": [
+ "C1327985740-ASF",
+ "C1216244600-ASF",
+ "C1234413259-ASFDEV",
+ ],
+ "SENTINEL-1B_SP_META_GRD_MEDIUM": [
+ "C1327985739-ASF",
+ "C1216244598-ASF",
+ "C1234413262-ASFDEV",
+ ],
+ "SENTINEL-1A_RAW": ["C1214470561-ASF", "C1205264459-ASF", "C1234413238-ASFDEV"],
+ "SENTINEL-1A_DP_GRD_MEDIUM": [
+ "C1214471521-ASF",
+ "C1212209035-ASF",
+ "C1234413230-ASFDEV",
+ ],
+ "SENTINEL-1A_SP_GRD_HIGH": [
+ "C1214470682-ASF",
+ "C1212158327-ASF",
+ "C1234413240-ASFDEV",
+ ],
+ "SENTINEL-1B_RAW": ["C1327985647-ASF", "C1216244592-ASF", "C1234413256-ASFDEV"],
+ "SENTINEL-1A_DP_GRD_FULL": [
+ "C1214471197-ASF",
+ "C1212200781-ASF",
+ "C1234413228-ASFDEV",
+ ],
+ "SENTINEL-1A_DP_META_GRD_FULL": [
+ "C1214471960-ASF",
+ "C1212209075-ASF",
+ "C1234413231-ASFDEV",
+ ],
+ "SENTINEL-1A_SP_GRD_FULL": ["C1214472978-ASF", "C1234413239-ASFDEV"],
+ "SENTINEL-1A_SP_META_GRD_FULL": ["C1214473165-ASF", "C1234413242-ASFDEV"],
+ "SENTINEL-1B_DP_GRD_FULL": [
+ "C1327985697-ASF",
+ "C1216244597-ASF",
+ "C1234413246-ASFDEV",
+ ],
+ "SENTINEL-1B_DP_META_GRD_FULL": [
+ "C1327985651-ASF",
+ "C1216244596-ASF",
+ "C1234413249-ASFDEV",
+ ],
+ "SENTINEL-1B_SP_GRD_FULL": [
+ "C1327985644-ASF",
+ "C1216244588-ASF",
+ "C1234413257-ASFDEV",
+ ],
+ "SENTINEL-1B_SP_GRD_HIGH": [
+ "C1327985571-ASF",
+ "C1216244586-ASF",
+ "C1234413258-ASFDEV",
+ ],
+ "SENTINEL-1B_SP_META_GRD_FULL": [
+ "C1327985674-ASF",
+ "C1216244599-ASF",
+ "C1234413260-ASFDEV",
+ ],
+ "S1_Bursts": ["C1244552887-ASFDEV"],
+ "SENTINEL-1_BURSTS_DEV10": ["C1257175154-ASFDEV"],
+ "Sentinel-1_Burst_Map": ["C1244598379-ASFDEV"],
+ "Various Browse Images": ["C1240784657-ASFDEV"],
+ },
+ "OPERA-S1": {
+ "OPERA_L2_CSLC-S1_V1": ["C2777443834-ASF", "C1259976861-ASF"],
+ "OPERA_L2_RTC-S1_V1": ["C2777436413-ASF", "C1259974840-ASF"],
+ "OPERA_L2_CSLC-S1-STATIC_PROVISIONAL_V0": ["C1258354200-ASF"],
+ "OPERA_L2_CSLC-S1-STATIC_V1": ["C1259982010-ASF"],
+ "OPERA_L2_CSLC-S1_PROVISIONAL_V0": ["C1257995185-ASF"],
+ "OPERA_L2_RTC-S1-STATIC_PROVISIONAL_V0": ["C1258354201-ASF"],
+ "OPERA_L2_RTC-S1-STATIC_V1": ["C1259981910-ASF"],
+ "OPERA_L2_RTC-S1_PROVISIONAL_V0": ["C1257995186-ASF"],
+ "OPERA_L2_CSLC-S1-STATIC_CALVAL_V1": ["C1260726384-ASF"],
+ "OPERA_L2_CSLC-S1_CALVAL_V1": ["C1260721945-ASF"],
+ "OPERA_L2_RTC-S1_CALVAL_V1": ["C1260721853-ASF"],
+ "OPERA_L2_RTC-S1-STATIC_CALVAL_V1": ["C1260726378-ASF"],
+ },
+ "SLC-BURST": {"SENTINEL-1_BURSTS": ["C2709161906-ASF", "C1257024016-ASF"]},
+ "ALOS PALSAR": {
+ "ALOS_PSR_RTC_HIGH": ["C1206487504-ASF", "C1207181535-ASF"],
+ "ALOS_PSR_L1.5": ["C1206485940-ASF", "C1205261223-ASF"],
+ "ALOS_PSR_RTC_LOW": ["C1206487217-ASF", "C1208013295-ASF"],
+ "ALOS_PSR_KMZ": ["C1206156901-ASF", "C1207019609-ASF"],
+ "ALOS_PSR_L1.0": ["C1206485320-ASF"],
+ "ALOS_PSR_L1.1": ["C1206485527-ASF", "C1207710476-ASF", "C1239611505-ASFDEV"],
+ "ALOS_PSR_L2.2": ["C2011599335-ASF", "C1239927797-ASF", "C1238733834-ASFDEV"],
+ "ALOS_PALSAR_INSAR_METADATA": ["C1229740239-ASF"],
+ },
+ "ALOS AVNIR-2": {
+ "ALOS_AVNIR_OBS_ORI": [
+ "C1808440897-ASF",
+ "C1233629671-ASF",
+ "C1234413224-ASFDEV",
+ ],
+ "ALOS_AVNIR_OBS_ORI_BROWSE": ["C1234712303-ASF"],
+ },
+ "SIR-C": {
+ "STS-59_BROWSE_GRD": [
+ "C1661710578-ASF",
+ "C1226557819-ASF",
+ "C1234413264-ASFDEV",
+ ],
+ "STS-59_BROWSE_SLC": [
+ "C1661710581-ASF",
+ "C1226557809-ASF",
+ "C1234413265-ASFDEV",
+ ],
+ "STS-59_GRD": ["C1661710583-ASF", "C1226557808-ASF", "C1234413266-ASFDEV"],
+ "STS-59_META_GRD": ["C1661710586-ASF", "C1226557810-ASF", "C1234413267-ASFDEV"],
+ "STS-59_META_SLC": ["C1661710588-ASF", "C1226557811-ASF", "C1234413268-ASFDEV"],
+ "STS-59_SLC": ["C1661710590-ASF", "C1226557812-ASF", "C1234413269-ASFDEV"],
+ "STS-68_BROWSE_GRD": [
+ "C1661710593-ASF",
+ "C1226557813-ASF",
+ "C1234413270-ASFDEV",
+ ],
+ "STS-68_BROWSE_SLC": [
+ "C1661710596-ASF",
+ "C1226557814-ASF",
+ "C1234413271-ASFDEV",
+ ],
+ "STS-68_GRD": ["C1661710597-ASF", "C1226557815-ASF", "C1234413272-ASFDEV"],
+ "STS-68_META_GRD": ["C1661710600-ASF", "C1226557816-ASF", "C1234413273-ASFDEV"],
+ "STS-68_META_SLC": ["C1661710603-ASF", "C1226557817-ASF", "C1234413274-ASFDEV"],
+ "STS-68_SLC": ["C1661710604-ASF", "C1226557818-ASF", "C1234413275-ASFDEV"],
+ },
+ "ARIA S1 GUNW": {
+ "SENTINEL-1_INTERFEROGRAMS": ["C1595422627-ASF", "C1225776654-ASF"],
+ "SENTINEL-1_INTERFEROGRAMS_AMPLITUDE": ["C1596065640-ASF", "C1225776655-ASF"],
+ "SENTINEL-1_INTERFEROGRAMS_COHERENCE": ["C1596065639-ASF", "C1225776657-ASF"],
+ "SENTINEL-1_INTERFEROGRAMS_CONNECTED_COMPONENTS": [
+ "C1596065641-ASF",
+ "C1225776658-ASF",
+ ],
+ "SENTINEL-1_INTERFEROGRAMS_UNWRAPPED_PHASE": [
+ "C1595765183-ASF",
+ "C1225776659-ASF",
+ ],
+ },
+ "SMAP": {
+ "SPL1A_RO_METADATA_003": ["C1243122884-ASF", "C1233103964-ASF"],
+ "SPL1A_RO_QA_003": ["C1243124139-ASF", "C1216074923-ASF"],
+ "SPL1A_001": ["C1214473171-ASF", "C1212243761-ASF"],
+ "SPL1A_002": ["C1243149604-ASF", "C1213091807-ASF"],
+ "SPL1A_METADATA_001": ["C1214473426-ASF", "C1212243437-ASF"],
+ "SPL1A_METADATA_002": ["C1243119801-ASF", "C1213096699-ASF"],
+ "SPL1A_QA_001": ["C1214473839-ASF", "C1212249653-ASF"],
+ "SPL1A_QA_002": ["C1243133204-ASF", "C1213101573-ASF"],
+ "SPL1A_RO_001": ["C1243197402-ASF"],
+ "SPL1A_RO_002": ["C1243215430-ASF", "C1213136240-ASF"],
+ "SPL1A_RO_003": ["C1243124754-ASF", "C1216074755-ASF"],
+ "SPL1A_RO_METADATA_001": ["C1243141638-ASF", "C1213136752-ASF"],
+ "SPL1A_RO_METADATA_002": ["C1243162394-ASF", "C1213136799-ASF"],
+ "SPL1A_RO_QA_001": ["C1243168733-ASF", "C1213136709-ASF"],
+ "SPL1A_RO_QA_002": ["C1243168866-ASF", "C1213136844-ASF"],
+ "SPL1B_SO_LoRes_001": ["C1214473308-ASF", "C1212249811-ASF"],
+ "SPL1B_SO_LoRes_002": ["C1243253631-ASF", "C1213125007-ASF"],
+ "SPL1B_SO_LoRes_003": ["C1243133445-ASF", "C1216074919-ASF"],
+ "SPL1B_SO_LoRes_METADATA_001": ["C1214473550-ASF", "C1212196951-ASF"],
+ "SPL1B_SO_LoRes_METADATA_002": ["C1243197502-ASF", "C1213115690-ASF"],
+ "SPL1B_SO_LoRes_METADATA_003": ["C1243126328-ASF", "C1216074758-ASF"],
+ "SPL1B_SO_LoRes_QA_001": ["C1214474243-ASF", "C1212243666-ASF"],
+ "SPL1B_SO_LoRes_QA_002": ["C1243216659-ASF", "C1213115896-ASF"],
+ "SPL1B_SO_LoRes_QA_003": ["C1243129847-ASF", "C1216074761-ASF"],
+ "SPL1C_S0_HiRes_001": ["C1214473367-ASF", "C1212250364-ASF"],
+ "SPL1C_S0_HiRes_002": ["C1243268956-ASF", "C1213134622-ASF"],
+ "SPL1C_S0_HiRes_003": ["C1243144528-ASF", "C1216074770-ASF"],
+ "SPL1C_S0_HiRes_METADATA_001": ["C1214473624-ASF", "C1212246173-ASF"],
+ "SPL1C_S0_HiRes_METADATA_002": ["C1243228612-ASF", "C1213125156-ASF"],
+ "SPL1C_S0_HiRes_METADATA_003": ["C1243136142-ASF", "C1216074764-ASF"],
+ "SPL1C_S0_HiRes_QA_001": ["C1214474435-ASF", "C1212249773-ASF"],
+ "SPL1C_S0_HiRes_QA_002": ["C1243255360-ASF", "C1213134486-ASF"],
+ "SPL1C_S0_HiRes_QA_003": ["C1243140611-ASF", "C1233101609-ASF"],
+ "SPL1A_003": ["C1216074922-ASF"],
+ "SPL1A_METADATA_003": ["C1216074750-ASF"],
+ "SPL1A_QA_003": ["C1216074751-ASF"],
+ },
+ "UAVSAR": {
+ "UAVSAR_POL_META": ["C1214353986-ASF", "C1210487703-ASF"],
+ "UAVSAR_INSAR_META": ["C1214336717-ASF", "C1212030772-ASF"],
+ "UAVSAR_INSAR_INT": ["C1214336045-ASF", "C1212001698-ASF"],
+ "UAVSAR_INSAR_AMP": ["C1214335430-ASF", "C1206116665-ASF"],
+ "UAVSAR_INSAR_AMP_GRD": ["C1214335471-ASF", "C1206132445-ASF"],
+ "UAVSAR_INSAR_DEM": ["C1214335903-ASF", "C1211962154-ASF"],
+ "UAVSAR_INSAR_INT_GRD": ["C1214336154-ASF", "C1212005594-ASF"],
+ "UAVSAR_INSAR_KMZ": ["C1214336554-ASF", "C1212019993-ASF"],
+ "UAVSAR_POL_DEM": ["C1214353593-ASF", "C1207638502-ASF"],
+ "UAVSAR_POL_INC": ["C1214353754-ASF", "C1210025872-ASF"],
+ "UAVSAR_POL_KMZ": ["C1214353859-ASF", "C1210485039-ASF"],
+ "UAVSAR_POL_ML_CMPLX_GRD": ["C1214337770-ASF", "C1207188317-ASF"],
+ "UAVSAR_POL_ML_CMPLX_GRD_3X3": ["C1214354144-ASF", "C1210546638-ASF"],
+ "UAVSAR_POL_ML_CMPLX_GRD_5X5": ["C1214354235-ASF", "C1206122195-ASF"],
+ "UAVSAR_POL_ML_CMPLX_SLANT": ["C1214343609-ASF", "C1209970710-ASF"],
+ "UAVSAR_POL_PAULI": ["C1214354031-ASF", "C1207038647-ASF"],
+ "UAVSAR_POL_SLOPE": ["C1214408428-ASF", "C1210599503-ASF"],
+ "UAVSAR_POL_STOKES": ["C1214419355-ASF", "C1210599673-ASF"],
+ },
+ "RADARSAT-1": {
+ "RSAT-1_L0": ["C1206897141-ASF"],
+ "RSAT-1_L1": ["C1206936391-ASF", "C1205181982-ASF"],
+ "RSAT-1_POLAR_YEAR_ANTARCTICA_L1": ["C1215670813-ASF"],
+ "RSAT-1_POLAR_YEAR_GREENLAND_L0": ["C1215709884-ASF"],
+ "RSAT-1_POLAR_YEAR_GREENLAND_L1": ["C1215709880-ASF"],
+ "RSAT-1_POLAR_YEAR_KAMCHATKA_L1": ["C1215714443-ASF"],
+ "RSAT-1_POLAR_YEAR_SEA_ICE_MIN_MAX_L1": ["C1215775284-ASF"],
+ "RSAT-1_POLAR_YEAR_TOOLIK_L1": ["C1215614037-ASF"],
+ },
+ "ERS": {
+ "ERS-1_L0": ["C1210197768-ASF", "C1205261222-ASF"],
+ "ERS-1_L1": ["C1211627521-ASF", "C1205302527-ASF"],
+ "ERS-2_L0": ["C1208794942-ASF", "C1207143701-ASF"],
+ "ERS-2_L1": ["C1209373626-ASF", "C1207144966-ASF"],
+ },
+ "JERS-1": {
+ "JERS-1_L0": ["C1208662092-ASF", "C1207175327-ASF"],
+ "JERS-1_L1": ["C1207933168-ASF", "C1207177736-ASF"],
+ },
+ "AIRSAR": {
+ "AIRSAR_POL_3FP": ["C1213921661-ASF", "C1205256880-ASF"],
+ "AIRSAR_INT_JPG": ["C1213921626-ASF", "C1000000306-ASF"],
+ "AIRSAR_POL_SYN_3FP": ["C1213928843-ASF", "C1208713702-ASF"],
+ "AIRSAR_TOP_C-DEM_STOKES": ["C1213927035-ASF", "C1208707768-ASF"],
+ "AIRSAR_TOP_DEM": ["C179001730-ASF", "C1208655639-ASF"],
+ "AIRSAR_TOP_DEM_C": ["C1213925022-ASF", "C1208680681-ASF"],
+ "AIRSAR_TOP_DEM_L": ["C1213926419-ASF", "C1208691361-ASF"],
+ "AIRSAR_TOP_DEM_P": ["C1213926777-ASF", "C1208703384-ASF"],
+ "AIRSAR_TOP_L-STOKES": ["C1213927939-ASF"],
+ "AIRSAR_TOP_P-STOKES": ["C1213928209-ASF"],
+ "AIRSAR_INT": ["C1208652494-ASF"],
+ },
+ "SEASAT": {
+ "SEASAT_SAR_L1_TIFF": ["C1206500826-ASF", "C1206752770-ASF"],
+ "SEASAT_SAR_L1_HDF5": ["C1206500991-ASF", "C1206144699-ASF"],
+ },
+}
=====================================
asf_search/CMR/field_map.py
=====================================
@@ -2,7 +2,6 @@ field_map = {
# API parameter CMR keyword CMR format strings
'absoluteOrbit': {'key': 'orbit_number', 'fmt': '{0}'},
'asfFrame': {'key': 'attribute[]', 'fmt': 'int,FRAME_NUMBER,{0}'},
- 'asfPlatform': {'key': 'attribute[]', 'fmt': 'string,ASF_PLATFORM,{0}'},
'maxBaselinePerp': {'key': 'attribute[]', 'fmt': 'float,INSAR_BASELINE,,{0}'},
'minBaselinePerp': {'key': 'attribute[]', 'fmt': 'float,INSAR_BASELINE,{0},'},
'bbox': {'key': 'bounding_box', 'fmt': '{0}'},
@@ -42,4 +41,7 @@ field_map = {
'absoluteBurstID': {'key': 'attribute[]', 'fmt': 'int,BURST_ID_ABSOLUTE,{0}'},
'relativeBurstID': {'key': 'attribute[]', 'fmt': 'int,BURST_ID_RELATIVE,{0}'},
'fullBurstID': {'key': 'attribute[]', 'fmt': 'string,BURST_ID_FULL,{0}'},
+
+ # OPERA-S1 field
+ 'operaBurstID': {'key': 'attribute[]', 'fmt': 'string,OPERA_BURST_ID,{0}'},
}
\ No newline at end of file
=====================================
asf_search/CMR/subquery.py
=====================================
@@ -22,7 +22,7 @@ def build_subqueries(opts: ASFSearchOptions) -> List[ASFSearchOptions]:
if params.get('product_list') is not None:
params['product_list'] = chunk_list(params['product_list'], CMR_PAGE_SIZE)
- list_param_names = ['platform', 'season', 'collections'] # these parameters will dodge the subquery system
+ list_param_names = ['platform', 'season', 'collections', 'dataset'] # these parameters will dodge the subquery system
skip_param_names = ['maxResults']# these params exist in opts, but shouldn't be passed on to subqueries at ALL
params = dict([ (k, v) for k, v in params.items() if k not in skip_param_names ])
=====================================
asf_search/CMR/translate.py
=====================================
@@ -7,6 +7,7 @@ from shapely import wkt
from shapely.geometry import Polygon
from shapely.geometry.base import BaseGeometry
from .field_map import field_map
+from .datasets import dataset_collections
import logging
@@ -46,7 +47,20 @@ def translate_opts(opts: ASFSearchOptions) -> list:
# If you need to use the temporal key:
if any(key in dict_opts for key in ['start', 'end', 'season']):
dict_opts = fix_date(dict_opts)
-
+
+ if 'dataset' in dict_opts:
+ if 'collections' not in dict_opts:
+ dict_opts['collections'] = []
+
+ for dataset in dict_opts['dataset']:
+ if collections_by_short_name := dataset_collections.get(dataset):
+ for concept_ids in collections_by_short_name.values():
+ dict_opts['collections'].extend(concept_ids)
+ else:
+ raise ValueError(f'Could not find dataset named "{dataset}" provided for dataset keyword.')
+
+ dict_opts.pop('dataset')
+
# convert the above parameters to a list of key/value tuples
cmr_opts = []
for (key, val) in dict_opts.items():
@@ -88,9 +102,16 @@ def translate_opts(opts: ASFSearchOptions) -> list:
def should_use_asf_frame(cmr_opts):
asf_frame_platforms = ['SENTINEL-1A', 'SENTINEL-1B', 'ALOS']
+ asf_frame_datasets = ['SENTINEL-1', 'OPERA-S1', 'SLC-BURST', 'ALOS PALSAR', 'ALOS AVNIR-2']
+
+ asf_frame_collections = []
+ for dataset in asf_frame_datasets:
+ for concept_ids in dataset_collections.get(dataset).values():
+ asf_frame_collections.extend(concept_ids)
return any([
p[0] == 'platform[]' and p[1].upper() in asf_frame_platforms
+ or p[0] == 'echo_collection_id[]' and p[1] in asf_frame_collections
for p in cmr_opts
])
@@ -199,7 +220,7 @@ def translate_product(item: dict) -> dict:
if properties['platform'] is None:
properties['platform'] = get(umm, 'Platforms', 0, 'ShortName')
- asf_frame_platforms = ['Sentinel-1A', 'Sentinel-1B', 'ALOS', 'SENTINEL-1A', 'SENTINEL-1B']
+ asf_frame_platforms = ['Sentinel-1A', 'Sentinel-1B', 'ALOS', 'SENTINEL-1A', 'SENTINEL-1B', 'Sentinel-1 Interferogram', 'Sentinel-1 Interferogram (BETA)', 'ERS-1', 'ERS-2', 'JERS-1', 'RADARSAT-1']
if properties['platform'] in asf_frame_platforms:
properties['frameNumber'] = cast(int, get(umm, 'AdditionalAttributes', ('Name', 'FRAME_NUMBER'), 'Values', 0))
else:
@@ -226,6 +247,20 @@ def translate_product(item: dict) -> dict:
properties['fileName'] = properties['fileID'] + '.' + urls[0].split('.')[-1]
properties['additionalUrls'] = [urls[1]]
+
+ if (fileID:=properties.get('fileID')):
+ if fileID.startswith('OPERA'):
+ properties['beamMode'] = get(umm, 'AdditionalAttributes', ('Name', 'BEAM_MODE'), 'Values', 0)
+ accessUrls = [*get(umm, 'RelatedUrls', ('Type', [('GET DATA', 'URL')]), 0), *get(umm, 'RelatedUrls', ('Type', [('EXTENDED METADATA', 'URL')]), 0)]
+ properties['additionalUrls'] = sorted([url for url in list(set(accessUrls)) if not url.endswith('.md5')
+ and not url.startswith('s3://')
+ and not 's3credentials' in url
+ and not url.endswith('.png')
+ and url != properties['url']])
+ properties['polarization'] = get(umm, 'AdditionalAttributes', ('Name', 'POLARIZATION'), 'Values')
+
+ properties['operaBurstID'] = get(umm, 'AdditionalAttributes', ('Name', 'OPERA_BURST_ID'), 'Values', 0)
+
return {'geometry': geometry, 'properties': properties, 'type': 'Feature', 'baseline': baseline}
def get_additional_fields(umm, *field_path):
=====================================
asf_search/constants/INTERNAL.py
=====================================
@@ -1,7 +1,8 @@
ASF_AUTH_HOST = 'auth.asf.alaska.edu'
CMR_HOST = 'cmr.earthdata.nasa.gov'
-CMR_FORMAT_EXT = 'umm_json_v1_4'
+CMR_TIMEOUT = 30
+CMR_FORMAT_EXT = 'umm_json'
CMR_GRANULE_PATH = f'/search/granules.{CMR_FORMAT_EXT}'
CMR_COLLECTIONS = f'/search/collections'
CMR_COLLECTIONS_PATH = f'{CMR_COLLECTIONS}.{CMR_FORMAT_EXT}'
=====================================
asf_search/download/download.py
=====================================
@@ -1,7 +1,7 @@
from typing import Iterable
from multiprocessing import Pool
import os.path
-import urllib.parse
+from urllib import parse
from requests import Response
from requests.exceptions import HTTPError
import warnings
@@ -56,7 +56,7 @@ def download_url(url: str, path: str, filename: str = None, session: ASFSession
"""
if filename is None:
- filename = os.path.split(urllib.parse.urlparse(url).path)[1]
+ filename = os.path.split(parse.urlparse(url).path)[1]
if not os.path.isdir(path):
raise ASFDownloadError(f'Error downloading {url}: directory not found: {path}')
@@ -84,7 +84,7 @@ def remotezip(url: str, session: ASFSession) -> RemoteZip:
return RemoteZip(url, session=session)
def strip_auth_if_aws(r, *args, **kwargs):
- if 300 <= r.status_code <= 399 and 'amazonaws.com' in urllib.parse.urlparse(r.headers['location']).netloc:
+ if 300 <= r.status_code <= 399 and 'amazonaws.com' in parse.urlparse(r.headers['location']).netloc:
location = r.headers['location']
r.headers.clear()
r.headers['location'] = location
=====================================
asf_search/search/search.py
=====================================
@@ -40,6 +40,8 @@ def search(
fullBurstID: Union[str, Iterable[str]] = None,
collections: Union[str, Iterable[str]] = None,
temporalBaselineDays: Union[str, Iterable[str]] = None,
+ operaBurstID: Union[str, Iterable[str]] = None,
+ dataset: Union[str, Iterable[str]] = None,
maxResults: int = None,
opts: ASFSearchOptions = None,
) -> ASFSearchResults:
=====================================
asf_search/search/search_generator.py
=====================================
@@ -54,6 +54,8 @@ def search_generator(
fullBurstID: Union[str, Iterable[str]] = None,
collections: Union[str, Iterable[str]] = None,
temporalBaselineDays: Union[str, Iterable[str]] = None,
+ operaBurstID: Union[str, Iterable[str]] = None,
+ dataset: Union[str, Iterable[str]] = None,
maxResults: int = None,
opts: ASFSearchOptions = None,
) -> Generator[ASFSearchResults, None, None]:
@@ -145,7 +147,7 @@ def process_page(items: List[ASFProduct], max_results: int, subquery_max_results
)
def get_page(session: ASFSession, url: str, translated_opts: list) -> Response:
try:
- response = session.post(url=url, data=translated_opts, timeout=30)
+ response = session.post(url=url, data=translated_opts, timeout=INTERNAL.CMR_TIMEOUT)
response.raise_for_status()
except HTTPError as exc:
error_message = f'HTTP {response.status_code}: {response.json()["errors"]}'
@@ -154,7 +156,7 @@ def get_page(session: ASFSession, url: str, translated_opts: list) -> Response:
if 500 <= response.status_code <= 599:
raise ASFSearch5xxError(error_message) from exc
except ReadTimeout as exc:
- raise ASFSearchError(f'Connection Error (Timeout): CMR took too long to respond ({url=})') from exc
+ raise ASFSearchError(f'Connection Error (Timeout): CMR took too long to respond. Set asf constant "CMR_TIMEOUT" to increase. ({url=}, timeout={INTERNAL.CMR_TIMEOUT})') from exc
return response
=====================================
tests/Search/test_search.py
=====================================
@@ -1,15 +1,15 @@
from numbers import Number
from asf_search import ASFSearchOptions
from asf_search.ASFProduct import ASFProduct
+from asf_search.CMR.translate import get
from asf_search.constants import INTERNAL
from asf_search.exceptions import ASFSearchError
from asf_search.search import search
from asf_search.ASFSearchResults import ASFSearchResults
-
-import requests
-
+from asf_search.CMR import dataset_collections
from pytest import raises
-
+from typing import List
+import requests
import requests_mock
def run_test_ASFSearchResults(search_resp):
@@ -75,4 +75,19 @@ def run_test_search_http_error(search_parameters, status_code: Number, report: s
with raises(ASFSearchError):
results.raise_if_incomplete()
-
+def run_test_dataset_search(datasets: List):
+ if any(dataset for dataset in datasets if dataset_collections.get(dataset) is None):
+ with raises(ValueError):
+ search(dataset=datasets, maxResults=1)
+ else:
+ for dataset in datasets:
+ valid_shortnames = list(dataset_collections.get(dataset))
+
+ response = search(dataset=dataset, maxResults=250)
+
+ # Get collection shortName of all granules
+ shortNames = list(set([shortName for product in response if (shortName:=get(product.umm, 'CollectionReference', 'ShortName')) is not None]))
+
+ # and check that results are limited to the expected datasets by their shortname
+ for shortName in shortNames:
+ assert shortName in valid_shortnames
=====================================
tests/pytest-config.yml
=====================================
@@ -199,6 +199,11 @@ test_types:
required_keys: results
method: test_output_format
+- For running dataset keyword tests:
+ required_in_title: search-dataset
+ required_keys: dataset
+ method: test_search_dataset
+
- For running jupyter notebook example tests:
required_keys: notebook
method: test_notebook_examples
=====================================
tests/pytest-managers.py
=====================================
@@ -7,7 +7,7 @@ from ASFSearchOptions.test_ASFSearchOptions import run_test_ASFSearchOptions
from ASFSearchResults.test_ASFSearchResults import run_test_output_format, run_test_ASFSearchResults_intersection
from ASFSession.test_ASFSession import run_auth_with_cookiejar, run_auth_with_creds, run_auth_with_token, run_test_asf_session_rebuild_auth
from BaselineSearch.test_baseline_search import *
-from Search.test_search import run_test_ASFSearchResults, run_test_search, run_test_search_http_error
+from Search.test_search import run_test_ASFSearchResults, run_test_dataset_search, run_test_search, run_test_search_http_error
from Search.test_search_generator import run_test_search_generator, run_test_search_generator_multi
from CMR.test_MissionList import run_test_get_project_names
@@ -417,6 +417,10 @@ def test_ASFSearchResults_intersection(**kwargs) -> None:
wkt = get_resource(kwargs['test_info']['wkt'])
run_test_ASFSearchResults_intersection(wkt)
+def test_search_dataset(**kwargs) -> None:
+ dataset = get_resource(kwargs['test_info']['dataset'])
+ run_test_dataset_search(dataset)
+
def test_serialization(**args) -> None:
test_info = args['test_info']
product = get_resource(test_info.get('product'))
=====================================
tests/yml_tests/test_search.yml
=====================================
@@ -42,3 +42,15 @@ tests:
platform: "Sentinel-1"
status_code: 500
report: "Server Error: This is a Test Error"
+
+- test-search-dataset S1 Datasets:
+ dataset: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1']
+
+- test-search-dataset S1 Datasets and non-S1:
+ dataset: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1', 'UAVSAR']
+
+- test-search-dataset fake dataset:
+ dataset: 'FAKE-DATASET-V1'
+
+- test-search-dataset S1 Datasets and fake dataset:
+ dataset: ['SENTINEL-1', 'SLC-BURST', 'OPERA-S1', 'FAKE-DATASET-V2']
View it on GitLab: https://salsa.debian.org/debian-gis-team/asf-search/-/commit/5065ffa9b1c1c2f880bbf3103bd3d9990604444c
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/asf-search/-/commit/5065ffa9b1c1c2f880bbf3103bd3d9990604444c
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20231111/6f9e6119/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list