[Git][debian-gis-team/xradarsat2][upstream] New upstream version 2024.11.12+ds

Antonio Valentino (@antonio.valentino) gitlab at salsa.debian.org
Mon Nov 25 15:10:18 GMT 2024



Antonio Valentino pushed to branch upstream at Debian GIS Project / xradarsat2


Commits:
e8ba3a55 by Antonio Valentino at 2024-11-15T16:24:19+00:00
New upstream version 2024.11.12+ds
- - - - -


13 changed files:

- .git_archival.txt
- .github/workflows/publish.yml
- .gitignore
- .pre-commit-config.yaml
- README.md
- docs/doc_xradarsat2.ipynb
- pyproject.toml
- src/xradarsat2/__init__.py
- src/xradarsat2/radarSat2_tiff_reader.py
- src/xradarsat2/radarSat2_xarray_reader.py
- + src/xradarsat2/utils.py
- + src/xradarsat2/xradarsat2-config.yaml
- + test/test_opening_datatree_radarsat2.py


Changes:

=====================================
.git_archival.txt
=====================================
@@ -1,4 +1,4 @@
-node: 15e4ffe83bacad29251b9f52b96b4d6b53d32271
-node-date: 2023-06-13T11:09:46+02:00
-describe-name: 2023.06.13
-ref-names: tag: 2023.06.13
+node: de1141391ab0b197ade12e6427804c6e89b95d80
+node-date: 2024-11-12T14:26:55+01:00
+describe-name: 2024.11.12
+ref-names: HEAD -> main, tag: 2024.11.12


=====================================
.github/workflows/publish.yml
=====================================
@@ -12,9 +12,9 @@ jobs:
       contents: 'read'
       id-token: 'write'
     steps:
-      - uses: actions/checkout at v3
+      - uses: actions/checkout at v4
       - name: Set up Python
-        uses: actions/setup-python at v4
+        uses: actions/setup-python at v5
         with:
           python-version: '3.x'
       - name: Install dependencies
@@ -28,7 +28,7 @@ jobs:
           twine check dist/*
           pip install dist/*.whl
       - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish at a56da0b891b3dc519c7ee3284aff1fad93cc8598
+        uses: pypa/gh-action-pypi-publish at 15c56dba361d8335944d31a2ecd17d700fc7bcbc
         with:
           password: ${{ secrets.pypi_token }}
           repository_url: https://upload.pypi.org/legacy/


=====================================
.gitignore
=====================================
@@ -129,3 +129,4 @@ tmp/
 /.idea/
 
 dask-worker-space/
+localxradarsat2-config.yaml
\ No newline at end of file


=====================================
.pre-commit-config.yaml
=====================================
@@ -1,21 +1,41 @@
+ci:
+  autoupdate_schedule: monthly
+
+# https://pre-commit.com/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
-      - id: check-toml
-      - id: check-yaml
-  - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
-    hooks:
-      - id: flake8
-  - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
-    hooks:
-      - id: isort
+      - id: check-docstring-first
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 24.10.0
     hooks:
       - id: black
-      - id: black-jupyter
+  - repo: https://github.com/keewis/blackdoc
+    rev: v0.3.9
+    hooks:
+      - id: blackdoc
+        additional_dependencies: ["black==24.10.0"]
+      - id: blackdoc-autoupdate-black
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.7.1
+    hooks:
+      - id: ruff
+        args: [--fix]
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.7.1
+    hooks:
+      - id: nbstripout
+        args: [--extra-keys=metadata.kernelspec metadata.language_info.version]
+  - repo: https://github.com/rbubley/mirrors-prettier
+    rev: v3.3.3
+    hooks:
+      - id: prettier
+  - repo: https://github.com/ComPWA/taplo-pre-commit
+    rev: v0.9.3
+    hooks:
+      - id: taplo-format
+      - id: taplo-lint
+        args: [--no-schema]


=====================================
README.md
=====================================
@@ -2,11 +2,8 @@
 
 radarSat2 Level 1 python reader for efficient xarray/dask based processor
 
-
-
 # Install
 
-
 ```
 conda install -c conda-forge xradarsat2
 ```


=====================================
docs/doc_xradarsat2.ipynb
=====================================
The diff for this file was not included because it is too large.

=====================================
pyproject.toml
=====================================
@@ -30,8 +30,7 @@ classifiers = [
 dependencies = [
     'xmltodict',
     'numpy',
-    'xarray',
-    'xarray-datatree',
+    "xarray>=2024.10.0",
     'rasterio',
     'rioxarray',
     'dask',


=====================================
src/xradarsat2/__init__.py
=====================================
@@ -1,2 +1,8 @@
-from .radarSat2_xarray_reader import load_digital_number  # noqa: F401
-from .radarSat2_xarray_reader import rs2_reader  # noqa: F401
+from xradarsat2.radarSat2_xarray_reader import load_digital_number  # noqa: F401
+from xradarsat2.radarSat2_xarray_reader import rs2_reader  # noqa: F401
+from importlib.metadata import version
+
+try:
+    __version__ = version("xradarsat2")
+except Exception:
+    __version__ = "999"


=====================================
src/xradarsat2/radarSat2_tiff_reader.py
=====================================
@@ -2,15 +2,13 @@ import glob
 import os
 
 import dask
-import datatree
 import numpy as np
-import radarSat2_xarray_reader
 import rasterio
 import rioxarray
 import xarray as xr
 import yaml
 from affine import Affine
-from xsar.utils import get_glob
+from xradarsat2.utils import get_glob, load_config
 
 # folder_path = "/home/datawork-cersat-public/cache/project/sarwing/data/RS2/L1/VV/2010/288/" \
 #              "RS2_OK72200_PK649463_DK111111_SCWA_20101015_210132_VV_SGF"
@@ -18,21 +16,14 @@ folder_path = (
     "/home/datawork-cersat-public/cache/project/sarwing/data/RS2/L1/VV_VH/2021/137"
     "/RS2_OK129673_PK1136693_DK1093025_SCWA_20210517_010235_VV_VH_SGF"
 )
+conf = load_config()
+folder_path = conf["folder_path"]
 
 
 def list_tiff_files(root_path):
     return glob.glob(os.path.join(root_path, "*imagery*tif"))
 
 
-"""def tiff_reader(root_path):
-    tiff_files_paths = list_tiff_files(root_path)
-    for path in tiff_files_paths:
-        src = rasterio.open(path)
-        print(src.name, src.mode)
-        im = src.read(1)
-        print(im)"""
-
-
 def _load_digital_number(
     root_path,
     dt,
@@ -40,6 +31,15 @@ def _load_digital_number(
     chunks=None,
     resampling=rasterio.enums.Resampling.rms,
 ):
+    """
+
+    :param root_path:
+    :param dt:
+    :param resolution:
+    :param chunks:
+    :param resampling:
+    :return:
+    """
     tiff_files = list_tiff_files(root_path)
     map_dims = {"pol": "band", "line": "y", "sample": "x"}
     if resolution is not None:
@@ -164,15 +164,5 @@ def _load_digital_number(
         ),
     }
     ds = dn.to_dataset(name=var_name)
-    dt["digital_numbers"] = datatree.DataTree(data=ds)
+    dt["digital_numbers"] = xr.DataTree(data=ds)
     return dt
-
-
-if __name__ == "__main__":
-    dt = radarSat2_xarray_reader.rs2_reader(folder_path)
-    _load_digital_number(
-        folder_path,
-        chunks={"line": 5000, "sample": 5000},
-        resolution={"line": 50, "sample": 50},
-        dt=dt,
-    )


=====================================
src/xradarsat2/radarSat2_xarray_reader.py
=====================================
@@ -6,7 +6,6 @@ import re
 import traceback
 
 import dask
-import datatree
 import numpy as np
 import rasterio
 import rioxarray
@@ -14,6 +13,7 @@ import xarray as xr
 import xmltodict
 import yaml
 from affine import Affine
+from datetime import datetime
 
 xpath_dict = {
     "geolocation_grid": {
@@ -296,7 +296,10 @@ def get_dic_orbit_information(dictio):
             ds_attr[key] = content_dict[key]
         elif isinstance(content_dict[key], list):
             for value in content_dict[key]:
-                timestamp.append(np.datetime64(value["timeStamp"]).astype("datetime64[ns]"))
+                dt_stamp = datetime.strptime(
+                    value["timeStamp"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                )
+                timestamp.append(np.array(dt_stamp).astype("datetime64[ns]"))
                 xPosition["values"].append(float(value["xPosition"]["#text"]))
                 xPosition["attr"]["units"] = value["xPosition"]["@units"]
                 yPosition["values"].append(float(value["yPosition"]["#text"]))
@@ -459,7 +462,10 @@ def get_dic_attitude_info(dictio):
             ds_attr[key] = content_dict[key]
         elif isinstance(content_dict[key], list):
             for value in content_dict[key]:
-                timestamp.append(np.datetime64(value["timeStamp"]).astype("datetime64[ns]"))
+                dt_stamp = datetime.strptime(
+                    value["timeStamp"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                )
+                timestamp.append(np.array(dt_stamp).astype("datetime64[ns]"))
                 yaw["values"].append(float(value["yaw"]["#text"]))
                 yaw["attr"]["units"] = value["yaw"]["@units"]
                 roll["values"].append(float(value["roll"]["#text"]))
@@ -567,7 +573,10 @@ def get_dict_doppler_centroid(dictio):
         if key == "dopplerCentroid":
             xpath = os.path.join(xpath, key)
             for value in content_dict[key]:
-                times.append(np.datetime64(value["timeOfDopplerCentroidEstimate"]).astype("datetime64[ns]"))
+                dt_dce = datetime.strptime(
+                    value["timeOfDopplerCentroidEstimate"], "%Y-%m-%dT%H:%M:%S.%fZ"
+                )
+                times.append(np.array(dt_dce).astype("datetime64[ns]"))
                 Ambiguity["values"].append(int(value["dopplerAmbiguity"]))
                 Ambiguity["attr"]["xpath"] = os.path.join(xpath, "dopplerAmbiguity")
                 AmbiguityConfidence["values"].append(
@@ -773,9 +782,9 @@ def get_dic_doppler_rate_values(dictio):
                 RateReferenceTime["attr"]["RateReferenceTime units"] = content_dict[
                     key
                 ]["dopplerRateReferenceTime"]["@units"]
-                RateReferenceTime["attr"][
-                    "dopplerRateReferenceTime_xpath"
-                ] = os.path.join(xpath, "dopplerRateReferenceTime")
+                RateReferenceTime["attr"]["dopplerRateReferenceTime_xpath"] = (
+                    os.path.join(xpath, "dopplerRateReferenceTime")
+                )
                 RateValuesCoefficients["values"].append(
                     [
                         float(x)
@@ -795,9 +804,9 @@ def get_dic_doppler_rate_values(dictio):
                     RateReferenceTime["attr"]["units"] = content_dict[key][
                         "dopplerRateReferenceTime"
                     ]["@units"]
-                    RateReferenceTime["attr"][
-                        "dopplerRateReferenceTime_xpath"
-                    ] = os.path.join(xpath, "dopplerRateReferenceTime")
+                    RateReferenceTime["attr"]["dopplerRateReferenceTime_xpath"] = (
+                        os.path.join(xpath, "dopplerRateReferenceTime")
+                    )
                     RateValuesCoefficients["values"].append(
                         [
                             float(x)
@@ -1295,9 +1304,9 @@ def get_dict_radar_parameters(dictio):
                                 parse_value(value[intern_key])
                             )
                         else:
-                            principal_dic[key]["attr"][
-                                intern_key.replace("@", "")
-                            ] = parse_value(value[intern_key])
+                            principal_dic[key]["attr"][intern_key.replace("@", "")] = (
+                                parse_value(value[intern_key])
+                            )
         elif isinstance(content_dict[key], list):
             prefix_path = os.path.join(xpath, key)
             for value in content_dict[key]:
@@ -1691,7 +1700,7 @@ def create_data_array_lut(dictio, dt):
     ----------
     dictio: dict
         Content of a xml LookUpTable file described as a dictionary
-    dt:datatree.Datatree
+    dt:xr.Datatree
         Contains every dataset of product.xml
 
     Returns
@@ -1733,7 +1742,7 @@ def create_dataset_lut(files, dt, folder_path):
     ----------
     files: List[str]
         Path names of LookUpTables files in the current folder
-    dt: datatree.Datatree
+    dt: xr.Datatree
         Contains every dataset of product.xml
     folder_path: str
         Folder path containing the level 1 files
@@ -1850,7 +1859,7 @@ def load_digital_number(
 
     Parameters
     ----------
-    dt: datatree.Datatree
+    dt: xr.Datatree
         datatree containing every dataset
     resolution: str, dict[str, int], None or number
         Resampling dict like `{'line': 20, 'sample': 20}` where 20 is in pixels.
@@ -1864,7 +1873,7 @@ def load_digital_number(
 
     Returns
     -------
-    datatree.Datatree
+    xr.Datatree
         Initial datatree + dataset (possibly dual-pol), with basic coords/dims naming convention
     """
 
@@ -2025,7 +2034,7 @@ def load_digital_number(
         ),
     }
     ds = dn.to_dataset(name=var_name)
-    dt["digital_numbers"] = datatree.DataTree(data=ds)
+    dt["digital_numbers"] = xr.DataTree(ds)
     return dt
 
 
@@ -2053,7 +2062,8 @@ def get_product_attributes(dic):
         else:
             if isinstance(dic[key], str) and key in useful_attributes:
                 if "Time" in key:
-                    final_dic[key] = np.datetime64(dic[key]).astype("datetime64[ns]")
+                    dt = datetime.strptime(dic[key], "%Y-%m-%dT%H:%M:%S.%fZ")
+                    final_dic[key] = np.array(dt).astype("datetime64[ns]")
                 else:
                     final_dic[key] = dic[key]
     return final_dic
@@ -2146,7 +2156,7 @@ def rs2_reader(folder_path):
 
     Returns
     -------
-    datatree.Datatree
+    xarray.Datatree
         datatree containing every dataset
     """
     # Verify if the product is complete
@@ -2193,9 +2203,9 @@ def rs2_reader(folder_path):
         )
     except Exception:
         pass
-    dt = datatree.DataTree()
-    dt["orbitAndAttitude"] = datatree.DataTree(data=ds_orbit_attitude_info)
-    dt["geolocationGrid"] = datatree.DataTree(data=ds_geo)
+    dt = xr.DataTree()
+    dt["orbitAndAttitude"] = xr.DataTree(ds_orbit_attitude_info)
+    dt["geolocationGrid"] = xr.DataTree(ds_geo)
     dic_doppler_centroid = get_dict_doppler_centroid(dic)
     ds_doppler_centroid = create_dataset_doppler_centroid(
         dic_doppler_centroid["ds_attr"],
@@ -2208,8 +2218,8 @@ def rs2_reader(folder_path):
         dic_doppler_centroid["dopplerCentroidConfidence"],
         folder_path,
     )
-    dt["imageGenerationParameters/doppler/dopplerCentroid"] = datatree.DataTree(
-        data=ds_doppler_centroid
+    dt["imageGenerationParameters/doppler/dopplerCentroid"] = xr.DataTree(
+        ds_doppler_centroid
     )
     dic_doppler_rate_values = get_dic_doppler_rate_values(dic)
     ds_doppler_rate_values = create_dataset_doppler_rate_values(
@@ -2218,8 +2228,8 @@ def rs2_reader(folder_path):
         dic_doppler_rate_values["dopplerRateValuesCoefficients"],
         folder_path,
     )
-    dt["imageGenerationParameters/doppler/dopplerRateValues"] = datatree.DataTree(
-        data=ds_doppler_rate_values
+    dt["imageGenerationParameters/doppler/dopplerRateValues"] = xr.DataTree(
+        ds_doppler_rate_values
     )
     dic_chirp = get_dict_chirp(dic)
     ds_chirp = create_dataset_chirp(
@@ -2235,14 +2245,14 @@ def rs2_reader(folder_path):
         dic_chirp["phaseCoefficients"],
         folder_path,
     )
-    dt["imageGenerationParameters/chirp"] = datatree.DataTree(data=ds_chirp)
+    dt["imageGenerationParameters/chirp"] = xr.DataTree(ds_chirp)
     radar_parameters_dic = get_dict_radar_parameters(dic)
     ds_radar_parameters = create_dataset_radar_parameters(
         radar_parameters_dic, folder_path
     )
-    dt["radarParameters"] = datatree.DataTree(data=ds_radar_parameters)
+    dt["radarParameters"] = xr.DataTree(ds_radar_parameters)
     ds_lut = create_dataset_lut(list_lut_files(folder_path), dt, folder_path)
-    dt["lut"] = datatree.DataTree(data=ds_lut)
+    dt["lut"] = xr.DataTree(ds_lut)
     dt.attrs["product_path"] = folder_path
     dt.attrs |= get_product_attributes(dic)
     dt.attrs |= get_satellite_height(dic)


=====================================
src/xradarsat2/utils.py
=====================================
@@ -0,0 +1,47 @@
+import xradarsat2
+import logging
+import os
+import yaml
+import re
+
+
+def get_glob(strlist):
+    # from list of str, replace diff by '?'
+    def _get_glob(st):
+        stglob = "".join(
+            [
+                "?" if len(charlist) > 1 else charlist[0]
+                for charlist in [list(set(charset)) for charset in zip(*st)]
+            ]
+        )
+        return re.sub(r"\?+", "*", stglob)
+
+    strglob = _get_glob(strlist)
+    if strglob.endswith("*"):
+        strglob += _get_glob(s[::-1] for s in strlist)[::-1]
+        strglob = strglob.replace("**", "*")
+
+    return strglob
+
+
+def load_config():
+    """
+
+    Returns:
+        conf: dict
+    """
+    local_config_path = os.path.join(
+        os.path.dirname(xradarsat2.__file__), "localxradarsat2-config.yaml"
+    )
+
+    if os.path.exists(local_config_path):
+        config_path = local_config_path
+    else:
+        config_path = os.path.join(
+            os.path.dirname(xradarsat2.__file__), "xradarsat2-config.yaml"
+        )
+
+    logging.info("config path: %s", config_path)
+    stream = open(config_path, "r")
+    conf = yaml.load(stream, Loader=yaml.CLoader)
+    return conf


=====================================
src/xradarsat2/xradarsat2-config.yaml
=====================================
@@ -0,0 +1 @@
+folder_path: ./2021/137/RS2_OK129673_PK1136693_DK1093025_SCWA_20210517_010235_VV_VH_SGF


=====================================
test/test_opening_datatree_radarsat2.py
=====================================
@@ -0,0 +1,25 @@
+from xradarsat2.utils import load_config
+import xradarsat2
+import time
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
+logging.debug("start opening RadarSAT-2 product")
+# conf = getconfig.get_config()
+# subswath = conf['product_paths'][0]
+
+t0 = time.time()
+conf = load_config()
+folder_path = conf["folder_path"]
+dt = xradarsat2.rs2_reader(folder_path)
+elapse_t = time.time() - t0
+
+print(type(dt), dt)
+print("out of the reader")
+print(dt)
+print("time to read the SAFE through nfs: %1.2f sec" % elapse_t)
+dt = xradarsat2.load_digital_number(
+    dt, chunks={"pol": "VV", "line": 6000, "sample": 8000}
+)
+print("DN", dt["/digital_numbers"])
+# pdb.set_trace()



View it on GitLab: https://salsa.debian.org/debian-gis-team/xradarsat2/-/commit/e8ba3a558f664d95a5570dad8e7bc9c4968be996

-- 
View it on GitLab: https://salsa.debian.org/debian-gis-team/xradarsat2/-/commit/e8ba3a558f664d95a5570dad8e7bc9c4968be996
You're receiving this email because of your account on salsa.debian.org.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20241125/e13e0ddc/attachment-0001.htm>


More information about the Pkg-grass-devel mailing list