[Git][debian-gis-team/xarray-safe-s1][master] 4 commits: New upstream version 2024.04.19.post1
Antonio Valentino (@antonio.valentino)
gitlab at salsa.debian.org
Sun Apr 28 08:45:55 BST 2024
Antonio Valentino pushed to branch master at Debian GIS Project / xarray-safe-s1
Commits:
01be2fe6 by Antonio Valentino at 2024-04-28T07:36:33+00:00
New upstream version 2024.04.19.post1
- - - - -
e4db2002 by Antonio Valentino at 2024-04-28T07:36:33+00:00
Update upstream source from tag 'upstream/2024.04.19.post1'
Update to upstream version '2024.04.19.post1'
with Debian dir 9ce2ff3ba89e6b3d24dba24d007d2b920940e8bd
- - - - -
cdf0ef83 by Antonio Valentino at 2024-04-28T07:38:49+00:00
New upstream release
- - - - -
9472909a by Antonio Valentino at 2024-04-28T07:41:43+00:00
Set distribution to suntable
- - - - -
11 changed files:
- .github/workflows/publish.yml
- .gitignore
- README.md
- debian/changelog
- + highleveltests/open_SLC_IW.py
- + highleveltests/open_SLC_IW_S3.py
- safe_s1/__init__.py
- + safe_s1/getconfig.py
- safe_s1/metadata.py → safe_s1/reader.py
- safe_s1/sentinel1_xml_mappings.py
- test/test_s1reader.py
Changes:
=====================================
.github/workflows/publish.yml
=====================================
@@ -28,7 +28,7 @@ jobs:
twine check dist/*
pip install dist/*.whl
- name: Publish to PyPI
- uses: pypa/gh-action-pypi-publish at 2f6f737ca5f74c637829c0f5c3acd0e29ea5e8bf
+ uses: pypa/gh-action-pypi-publish at 81e9d935c883d0b210363ab89cf05f3894778450
with:
password: ${{ secrets.pypi_token }}
repository_url: https://upload.pypi.org/legacy/
=====================================
.gitignore
=====================================
@@ -19,3 +19,5 @@ __pycache__/
.coverage.*
.cache
/docs/_build/
+localconfig.yml
+.idea
=====================================
README.md
=====================================
@@ -152,4 +152,11 @@ DataTree('None', parent=None)
```
+Example of usage for S3: access
+```pycon
+url = 's3:///eodata/..../.SAFE'
+storage_options = {"anon": False, "client_kwargs": {"endpoint_url": 'https://'+entrypoint_url, 'aws_access_key_id':access_key,
+ 'aws_secret_access_key':secret_key}}
+reader = Sentinel1Reader(url,backend_kwargs={"storage_options": storage_options})
+```
=====================================
debian/changelog
=====================================
@@ -1,3 +1,9 @@
+xarray-safe-s1 (2024.04.19.post1-1) unstable; urgency=medium
+
+ * New upstrean release.
+
+ -- Antonio Valentino <antonio.valentino at tiscali.it> Sun, 28 Apr 2024 07:41:26 +0000
+
xarray-safe-s1 (2024.01.09-1) unstable; urgency=medium
* New upstream release.
=====================================
highleveltests/open_SLC_IW.py
=====================================
@@ -0,0 +1,13 @@
+from safe_s1 import Sentinel1Reader, getconfig
+import time
+conf = getconfig.get_config()
+subswath = conf['nfs_iw_grd_path']
+print(subswath)
+t0 = time.time()
+sub_reader = Sentinel1Reader(subswath)
+elapse_t = time.time()-t0
+
+dt = sub_reader.datatree
+print('out of the reader')
+print(dt)
+print('time to read the SAFE through S3: %1.2f sec'%elapse_t)
=====================================
highleveltests/open_SLC_IW_S3.py
=====================================
@@ -0,0 +1,36 @@
+# see https://stackoverflow.com/questions/69624867/no-such-file-error-when-trying-to-create-local-cache-of-s3-object
+from safe_s1 import Sentinel1Reader,getconfig
+import pdb
+import os
+import time
+import logging
+import fsspec
+logging.basicConfig(level=logging.INFO)
+logging.info('test start')
+conf = getconfig.get_config()
+access_key = conf['access_key']
+secret_key = conf['secret_key']
+entrypoint_url = conf['entrypoint_url']
+s3 = fsspec.filesystem("s3", anon=False,
+ key=access_key,
+ secret=secret_key,
+ endpoint_url='https://'+entrypoint_url)
+
+# this syntaxe works we can get content xml files but I would have to precise which subswath I want to decode in case of SLC
+# safe2 = 's3:///eodata/Sentinel-1/SAR/SLC/2019/10/13/S1B_IW_SLC__1SDV_20191013T155948_20191013T160015_018459_022C6B_13A2.SAFE'
+safe2 = 's3:///eodata/Sentinel-1/SAR/IW_GRDH_1S/2024/04/18/S1A_IW_GRDH_1SSH_20240418T080141_20240418T080210_053485_067D74_C073.SAFE'
+# safe2 = conf['s3_iw_grd_path']
+option = 'kwargs'
+if option == 'kwargs':
+ storage_options = {"anon": False, "client_kwargs": {"endpoint_url": 'https://'+entrypoint_url, 'aws_access_key_id':access_key,
+ 'aws_secret_access_key':secret_key}}
+ t0 = time.time()
+ sub_reader = Sentinel1Reader(safe2,backend_kwargs={"storage_options": storage_options})
+ elapse_t = time.time()-t0
+ print('time to read the SAFE through S3: %1.2f sec'%elapse_t)
+else:
+ # this solution is not supported.
+ sub_reader = Sentinel1Reader(s3.get_mapper(safe2)) # botocore.errorfactory.NoSuchKey: An error occurred (NoSuchKey) when calling the GetObject operation: Unknown
+dt = sub_reader.datatree
+print('out of the reader')
+print(dt)
=====================================
safe_s1/__init__.py
=====================================
@@ -1,7 +1,12 @@
-from importlib.metadata import version
-from .metadata import Sentinel1Reader
-
+import traceback
+#import safe_s1
+from safe_s1.reader import Sentinel1Reader
try:
- __version__ = version("safe_s1")
+ from importlib import metadata
+except ImportError: # for Python<3.8
+ import importlib_metadata as metadata
+try:
+ __version__ = metadata.version("xarray-safe-s1")
except Exception:
+ print('trace',traceback.format_exc())
__version__ = "999"
=====================================
safe_s1/getconfig.py
=====================================
@@ -0,0 +1,23 @@
+import yaml
+import os
+import logging
+import safe_s1
+from pathlib import Path
+# determine the config file we will use (config.yml by default, and a local config if one is present) and retrieve
+# the products names
+def get_config():
+ local_config_pontential_path = os.path.join(os.path.dirname(safe_s1.__file__), 'localconfig.yml')
+ logging.info('potential local config: %s',local_config_pontential_path)
+ #local_config_pontential_path = Path(os.path.join('~', 'xarray-safe-s1', 'localconfig.yml')).expanduser()
+ if os.path.exists(local_config_pontential_path):
+ logging.info('localconfig used')
+ config_path = local_config_pontential_path
+ with open(config_path) as config_content:
+ conf = yaml.load(config_content, Loader=yaml.SafeLoader)
+ else:
+ logging.info('default config')
+ config_path = Path(os.path.join(os.path.dirname(safe_s1.__file__), 'config.yml'))
+ with open(config_path) as config_content:
+ conf = yaml.load(config_content, Loader=yaml.SafeLoader)
+ return conf
+
=====================================
safe_s1/metadata.py → safe_s1/reader.py
=====================================
@@ -1,6 +1,6 @@
import os
import re
-
+import pdb
import dask
import fsspec
import numpy as np
@@ -21,7 +21,7 @@ class Sentinel1Reader:
def __init__(self, name, backend_kwargs=None):
if not isinstance(name, (str, os.PathLike)):
- raise ValueError(f"cannot deal with object of type {type(name)}: {name}")
+ raise ValueError(f"cannot deal with object of type {type(name)}: {name}")
# gdal dataset name
if not name.startswith('SENTINEL1_DS:'):
name = 'SENTINEL1_DS:%s:' % name
@@ -39,10 +39,13 @@ class Sentinel1Reader:
"""Dataset path"""
self.safe = os.path.basename(self.path)
+ self.path = os.fspath(self.path)
+
if backend_kwargs is None:
backend_kwargs = {}
- self.path = os.fspath(self.path)
+
storage_options = backend_kwargs.get("storage_options", {})
+
mapper = fsspec.get_mapper(self.path, **storage_options)
self.xml_parser = XmlParser(
xpath_mappings=sentinel1_xml_mappings.xpath_mappings,
@@ -89,7 +92,6 @@ class Sentinel1Reader:
'geolocationGrid': None,
}
if not self.multidataset:
-
self._dict = {
'geolocationGrid': self.geoloc,
'orbit': self.orbit,
@@ -105,6 +107,9 @@ class Sentinel1Reader:
}
self.dt = datatree.DataTree.from_dict(self._dict)
assert self.dt==self.datatree
+ else:
+ print('multidataset')
+ raise Exception()
def load_digital_number(self, resolution=None, chunks=None, resampling=rasterio.enums.Resampling.rms):
"""
=====================================
safe_s1/sentinel1_xml_mappings.py
=====================================
@@ -472,7 +472,7 @@ def df_files(annotation_files, measurement_files, noise_files, calibration_files
def xsd_files_func(xsd_product_file):
"""
return a xarray Dataset with path of the different xsd files
- :param xsd_product:
+ :param xsd_product: str
:return:
"""
ds = xr.Dataset()
=====================================
test/test_s1reader.py
=====================================
@@ -1,9 +1,6 @@
-import safe_s1
-from safe_s1 import sentinel1_xml_mappings, Sentinel1Reader
-import os
+from safe_s1 import sentinel1_xml_mappings, Sentinel1Reader, getconfig
import logging
-from pathlib import Path
-import yaml
+
logging.basicConfig()
logging.captureWarnings(True)
@@ -11,20 +8,8 @@ logging.captureWarnings(True)
logger = logging.getLogger('s1_reader_test')
logger.setLevel(logging.DEBUG)
-
-# determine the config file we will use (config.yml by default, and a local config if one is present) and retrieve
-# the products names
-local_config_pontential_path = Path(os.path.join('~', 'xarray-safe-s1', 'localconfig.yml')).expanduser()
-if local_config_pontential_path.exists():
- config_path = local_config_pontential_path
- with open(config_path) as config_content:
- products = yaml.load(config_content, Loader=yaml.SafeLoader)['product_paths']
-else:
- config_path = Path(os.path.join(os.path.dirname(safe_s1.__file__), 'config.yml'))
- with open(config_path) as config_content:
- raw_products = yaml.load(config_content, Loader=yaml.SafeLoader)['product_paths']
- products = [sentinel1_xml_mappings.get_test_file(filename) for filename in raw_products]
-
+conf = getconfig.get_config()
+products = [sentinel1_xml_mappings.get_test_file(filename) for filename in conf['product_paths']]
# Try to apply the reader on different products
def test_reader():
View it on GitLab: https://salsa.debian.org/debian-gis-team/xarray-safe-s1/-/compare/60bf14455b20e0bff82759575accfbafe41353db...9472909a74e9ed2117e86781e7b4484e52e5d64f
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/xarray-safe-s1/-/compare/60bf14455b20e0bff82759575accfbafe41353db...9472909a74e9ed2117e86781e7b4484e52e5d64f
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20240428/04d5d54b/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list