[Git][debian-gis-team/xarray-safe-s1][upstream] New upstream version 2024.04.19.post1

Antonio Valentino (@antonio.valentino) gitlab at salsa.debian.org
Sun Apr 28 08:45:43 BST 2024



Antonio Valentino pushed to branch upstream at Debian GIS Project / xarray-safe-s1


Commits:
01be2fe6 by Antonio Valentino at 2024-04-28T07:36:33+00:00
New upstream version 2024.04.19.post1
- - - - -


10 changed files:

- .github/workflows/publish.yml
- .gitignore
- README.md
- + highleveltests/open_SLC_IW.py
- + highleveltests/open_SLC_IW_S3.py
- safe_s1/__init__.py
- + safe_s1/getconfig.py
- safe_s1/metadata.py → safe_s1/reader.py
- safe_s1/sentinel1_xml_mappings.py
- test/test_s1reader.py


Changes:

=====================================
.github/workflows/publish.yml
=====================================
@@ -28,7 +28,7 @@ jobs:
           twine check dist/*
           pip install dist/*.whl
       - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish at 2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf
+        uses: pypa/gh-action-pypi-publish at 81e9d935c883d0b210363ab89cf05f3894778450
         with:
           password: ${{ secrets.pypi_token }}
           repository_url: https://upload.pypi.org/legacy/


=====================================
.gitignore
=====================================
@@ -19,3 +19,5 @@ __pycache__/
 .coverage.*
 .cache
 /docs/_build/
+localconfig.yml
+.idea


=====================================
README.md
=====================================
@@ -152,4 +152,11 @@ DataTree('None', parent=None)
 
 ```
 
+Example of usage for S3: access
 
+```pycon
+url = 's3:///eodata/..../.SAFE'
+storage_options = {"anon": False, "client_kwargs": {"endpoint_url": 'https://'+entrypoint_url, 'aws_access_key_id':access_key,
+    'aws_secret_access_key':secret_key}}
+reader = Sentinel1Reader(url,backend_kwargs={"storage_options": storage_options})
+```


=====================================
highleveltests/open_SLC_IW.py
=====================================
@@ -0,0 +1,13 @@
+from safe_s1 import Sentinel1Reader, getconfig
+import time
+conf = getconfig.get_config()
+subswath = conf['nfs_iw_grd_path']
+print(subswath)
+t0 = time.time()
+sub_reader = Sentinel1Reader(subswath)
+elapse_t = time.time()-t0
+
+dt = sub_reader.datatree
+print('out of the reader')
+print(dt)
+print('time to read the SAFE through S3: %1.2f sec'%elapse_t)


=====================================
highleveltests/open_SLC_IW_S3.py
=====================================
@@ -0,0 +1,36 @@
+# see https://stackoverflow.com/questions/69624867/no-such-file-error-when-trying-to-create-local-cache-of-s3-object
+from safe_s1 import Sentinel1Reader,getconfig
+import pdb
+import os
+import time
+import logging
+import fsspec
+logging.basicConfig(level=logging.INFO)
+logging.info('test start')
+conf = getconfig.get_config()
+access_key = conf['access_key']
+secret_key = conf['secret_key']
+entrypoint_url = conf['entrypoint_url']
+s3 = fsspec.filesystem("s3", anon=False,
+      key=access_key,
+      secret=secret_key,
+      endpoint_url='https://'+entrypoint_url)
+
+# this syntaxe works we can get content xml files but I would have to precise which subswath I want to decode in case of SLC
+# safe2 = 's3:///eodata/Sentinel-1/SAR/SLC/2019/10/13/S1B_IW_SLC__1SDV_20191013T155948_20191013T160015_018459_022C6B_13A2.SAFE'
+safe2 = 's3:///eodata/Sentinel-1/SAR/IW_GRDH_1S/2024/04/18/S1A_IW_GRDH_1SSH_20240418T080141_20240418T080210_053485_067D74_C073.SAFE'
+# safe2 = conf['s3_iw_grd_path']
+option = 'kwargs'
+if option == 'kwargs':
+    storage_options = {"anon": False, "client_kwargs": {"endpoint_url": 'https://'+entrypoint_url, 'aws_access_key_id':access_key,
+    'aws_secret_access_key':secret_key}}
+    t0 = time.time()
+    sub_reader = Sentinel1Reader(safe2,backend_kwargs={"storage_options": storage_options})
+    elapse_t = time.time()-t0
+    print('time to read the SAFE through S3: %1.2f sec'%elapse_t)
+else:
+    # this solution is not supported.
+    sub_reader = Sentinel1Reader(s3.get_mapper(safe2)) # botocore.errorfactory.NoSuchKey: An error occurred (NoSuchKey) when calling the GetObject operation: Unknown
+dt = sub_reader.datatree
+print('out of the reader')
+print(dt)


=====================================
safe_s1/__init__.py
=====================================
@@ -1,7 +1,12 @@
-from importlib.metadata import version
-from .metadata import Sentinel1Reader
-
+import traceback
+#import safe_s1
+from safe_s1.reader import Sentinel1Reader
 try:
-    __version__ = version("safe_s1")
+    from importlib import metadata
+except ImportError: # for Python<3.8
+    import importlib_metadata as metadata
+try: 
+    __version__ = metadata.version("xarray-safe-s1")
 except Exception:
+    print('trace',traceback.format_exc())
     __version__ = "999"


=====================================
safe_s1/getconfig.py
=====================================
@@ -0,0 +1,23 @@
+import yaml
+import os
+import logging
+import safe_s1
+from pathlib import Path
+# determine the config file we will use (config.yml by default, and a local config if one is present) and retrieve
+# the products names
+def get_config():
+    local_config_pontential_path = os.path.join(os.path.dirname(safe_s1.__file__), 'localconfig.yml')
+    logging.info('potential local config: %s',local_config_pontential_path)
+    #local_config_pontential_path = Path(os.path.join('~', 'xarray-safe-s1', 'localconfig.yml')).expanduser()
+    if os.path.exists(local_config_pontential_path):
+        logging.info('localconfig used')
+        config_path = local_config_pontential_path
+        with open(config_path) as config_content:
+            conf = yaml.load(config_content, Loader=yaml.SafeLoader)
+    else:
+        logging.info('default config')
+        config_path = Path(os.path.join(os.path.dirname(safe_s1.__file__), 'config.yml'))
+        with open(config_path) as config_content:
+            conf = yaml.load(config_content, Loader=yaml.SafeLoader)
+    return conf
+


=====================================
safe_s1/metadata.py → safe_s1/reader.py
=====================================
@@ -1,6 +1,6 @@
 import os
 import re
-
+import pdb
 import dask
 import fsspec
 import numpy as np
@@ -21,7 +21,7 @@ class Sentinel1Reader:
 
     def __init__(self, name, backend_kwargs=None):
         if not isinstance(name, (str, os.PathLike)):
-            raise ValueError(f"cannot deal with object of type {type(name)}: {name}")
+           raise ValueError(f"cannot deal with object of type {type(name)}: {name}")
         # gdal dataset name
         if not name.startswith('SENTINEL1_DS:'):
             name = 'SENTINEL1_DS:%s:' % name
@@ -39,10 +39,13 @@ class Sentinel1Reader:
         """Dataset path"""
         self.safe = os.path.basename(self.path)
 
+        self.path = os.fspath(self.path)
+
         if backend_kwargs is None:
             backend_kwargs = {}
-        self.path = os.fspath(self.path)
+
         storage_options = backend_kwargs.get("storage_options", {})
+
         mapper = fsspec.get_mapper(self.path, **storage_options)
         self.xml_parser = XmlParser(
             xpath_mappings=sentinel1_xml_mappings.xpath_mappings,
@@ -89,7 +92,6 @@ class Sentinel1Reader:
             'geolocationGrid': None,
         }
         if not self.multidataset:
-
             self._dict = {
                 'geolocationGrid': self.geoloc,
                 'orbit': self.orbit,
@@ -105,6 +107,9 @@ class Sentinel1Reader:
             }
             self.dt = datatree.DataTree.from_dict(self._dict)
             assert self.dt==self.datatree
+        else:
+            print('multidataset')
+            raise Exception()
 
     def load_digital_number(self, resolution=None, chunks=None, resampling=rasterio.enums.Resampling.rms):
         """


=====================================
safe_s1/sentinel1_xml_mappings.py
=====================================
@@ -472,7 +472,7 @@ def df_files(annotation_files, measurement_files, noise_files, calibration_files
 def xsd_files_func(xsd_product_file):
     """
     return a xarray Dataset with path of the different xsd files
-    :param xsd_product:
+    :param xsd_product: str
     :return:
     """
     ds = xr.Dataset()


=====================================
test/test_s1reader.py
=====================================
@@ -1,9 +1,6 @@
-import safe_s1
-from safe_s1 import sentinel1_xml_mappings, Sentinel1Reader
-import os
+from safe_s1 import sentinel1_xml_mappings, Sentinel1Reader, getconfig
 import logging
-from pathlib import Path
-import yaml
+
 
 logging.basicConfig()
 logging.captureWarnings(True)
@@ -11,20 +8,8 @@ logging.captureWarnings(True)
 logger = logging.getLogger('s1_reader_test')
 logger.setLevel(logging.DEBUG)
 
-
-# determine the config file we will use (config.yml by default, and a local config if one is present) and retrieve
-# the products names
-local_config_pontential_path = Path(os.path.join('~', 'xarray-safe-s1', 'localconfig.yml')).expanduser()
-if local_config_pontential_path.exists():
-    config_path = local_config_pontential_path
-    with open(config_path) as config_content:
-        products = yaml.load(config_content, Loader=yaml.SafeLoader)['product_paths']
-else:
-    config_path = Path(os.path.join(os.path.dirname(safe_s1.__file__), 'config.yml'))
-    with open(config_path) as config_content:
-        raw_products = yaml.load(config_content, Loader=yaml.SafeLoader)['product_paths']
-    products = [sentinel1_xml_mappings.get_test_file(filename) for filename in raw_products]
-
+conf = getconfig.get_config()
+products = [sentinel1_xml_mappings.get_test_file(filename) for filename in conf['product_paths']]
 
 # Try to apply the reader on different products
 def test_reader():



View it on GitLab: https://salsa.debian.org/debian-gis-team/xarray-safe-s1/-/commit/01be2fe6e723fc78da07326b4bf6a0a294efba1d

-- 
View it on GitLab: https://salsa.debian.org/debian-gis-team/xarray-safe-s1/-/commit/01be2fe6e723fc78da07326b4bf6a0a294efba1d
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20240428/b1fa9ed7/attachment-0001.htm>


More information about the Pkg-grass-devel mailing list