[Git][debian-gis-team/pygac][master] 6 commits: New upstream version 1.7.3
Antonio Valentino (@antonio.valentino)
gitlab at salsa.debian.org
Sun Mar 24 17:01:17 GMT 2024
Antonio Valentino pushed to branch master at Debian GIS Project / pygac
Commits:
0b5163ec by Antonio Valentino at 2024-03-24T16:30:12+00:00
New upstream version 1.7.3
- - - - -
29e00bf6 by Antonio Valentino at 2024-03-24T16:30:15+00:00
Update upstream source from tag 'upstream/1.7.3'
Update to upstream version '1.7.3'
with Debian dir e82a4d05d0d8fdc0935f8067826573e02415e493
- - - - -
cc49b458 by Antonio Valentino at 2024-03-24T16:37:41+00:00
New upstream release
- - - - -
6a0b35b8 by Antonio Valentino at 2024-03-24T16:54:54+00:00
Remove pytest_cache after testing
- - - - -
82011966 by Antonio Valentino at 2024-03-24T16:56:54+00:00
Update dates in d/copyright
- - - - -
981daf9e by Antonio Valentino at 2024-03-24T16:57:20+00:00
Set distribution to unstable
- - - - -
14 changed files:
- .github/workflows/ci.yaml
- CHANGELOG.md
- bin/pygac-convert-patmosx-coefficients
- debian/changelog
- debian/copyright
- debian/rules
- pygac/calibration.py
- pygac/data/calibration.json
- pygac/pod_reader.py
- pygac/reader.py
- pygac/tests/test_calibrate_klm.py
- pygac/tests/test_calibrate_pod.py
- pygac/tests/test_klm.py
- pygac/tests/test_reader.py
Changes:
=====================================
.github/workflows/ci.yaml
=====================================
@@ -10,7 +10,7 @@ jobs:
fail-fast: true
matrix:
os: ["ubuntu-latest"]
- python-version: ["3.8", "3.9", "3.10"]
+ python-version: ["3.10", "3.11", "3.12"]
experimental: [false]
env:
=====================================
CHANGELOG.md
=====================================
@@ -1,3 +1,15 @@
+## Version 1.7.3 (2024/03/19)
+
+
+### Pull Requests Merged
+
+#### Features added
+
+* [PR 125](https://github.com/pytroll/pygac/pull/125) - Support reading EO-SIP LAC data
+
+In this release 1 pull request was closed.
+
+
## Version 1.7.2 (2023/06/26)
=====================================
bin/pygac-convert-patmosx-coefficients
=====================================
@@ -64,10 +64,10 @@ class PatmosxReader:
r'\s*(?P<ch1_gain_switches_count>[eE0-9\.-]+)[^\n]*\n'
r'\s*(?P<ch2_gain_switches_count>[eE0-9\.-]+)[^\n]*\n'
r'\s*(?P<ch3a_gain_switches_count>[eE0-9\.-]+)[^\n]*\n'
- r'\s*(?P<PRT1_0>[eE0-9\.-]+)\s*(?P<PRT1_1>[eE0-9\.-]+)\s*(?P<PRT1_2>[eE0-9\.-]+)\s*(?P<PRT1_3>[eE0-9\.-]+)\s*(?P<PRT1_4>[eE0-9\.-]+)[^\n]*\n'
- r'\s*(?P<PRT2_0>[eE0-9\.-]+)\s*(?P<PRT2_1>[eE0-9\.-]+)\s*(?P<PRT2_2>[eE0-9\.-]+)\s*(?P<PRT2_3>[eE0-9\.-]+)\s*(?P<PRT2_4>[eE0-9\.-]+)[^\n]*\n'
- r'\s*(?P<PRT3_0>[eE0-9\.-]+)\s*(?P<PRT3_1>[eE0-9\.-]+)\s*(?P<PRT3_2>[eE0-9\.-]+)\s*(?P<PRT3_3>[eE0-9\.-]+)\s*(?P<PRT3_4>[eE0-9\.-]+)[^\n]*\n'
- r'\s*(?P<PRT4_0>[eE0-9\.-]+)\s*(?P<PRT4_1>[eE0-9\.-]+)\s*(?P<PRT4_2>[eE0-9\.-]+)\s*(?P<PRT4_3>[eE0-9\.-]+)\s*(?P<PRT4_4>[eE0-9\.-]+)[^\n]*\n'
+ r'\s*(?P<PRT1_0>[eE0-9\.-]+)\,*\s*(?P<PRT1_1>[eE0-9\.-]+)\,*\s*(?P<PRT1_2>[eE0-9\.-]+)\,*\s*(?P<PRT1_3>[eE0-9\.-]+)\,*\s*(?P<PRT1_4>[eE0-9\.-]+)[^\n]*\n'
+ r'\s*(?P<PRT2_0>[eE0-9\.-]+)\,*\s*(?P<PRT2_1>[eE0-9\.-]+)\,*\s*(?P<PRT2_2>[eE0-9\.-]+)\,*\s*(?P<PRT2_3>[eE0-9\.-]+)\,*\s*(?P<PRT2_4>[eE0-9\.-]+)[^\n]*\n'
+ r'\s*(?P<PRT3_0>[eE0-9\.-]+)\,*\s*(?P<PRT3_1>[eE0-9\.-]+)\,*\s*(?P<PRT3_2>[eE0-9\.-]+)\,*\s*(?P<PRT3_3>[eE0-9\.-]+)\,*\s*(?P<PRT3_4>[eE0-9\.-]+)[^\n]*\n'
+ r'\s*(?P<PRT4_0>[eE0-9\.-]+)\,*\s*(?P<PRT4_1>[eE0-9\.-]+)\,*\s*(?P<PRT4_2>[eE0-9\.-]+)\,*\s*(?P<PRT4_3>[eE0-9\.-]+)\,*\s*(?P<PRT4_4>[eE0-9\.-]+)[^\n]*\n'
r'\s*(?P<PRT_weight_1>[eE0-9\.-]+)\s*(?P<PRT_weight_2>[eE0-9\.-]+)\s*(?P<PRT_weight_3>[eE0-9\.-]+)\s*(?P<PRT_weight_4>[eE0-9\.-]+)[^\n]*\n'
r'(?:\s*(?P<sst_mask_day_0>[eE0-9\.-]+),\s*(?P<sst_mask_day_1>[eE0-9\.-]+),\s*(?P<sst_mask_day_2>[eE0-9\.-]+),\s*(?P<sst_mask_day_3>[eE0-9\.-]+)[^\n]*\n)?'
r'(?:\s*(?P<sst_mask_night_0>[eE0-9\.-]+),\s*(?P<sst_mask_night_1>[eE0-9\.-]+),\s*(?P<sst_mask_night_2>[eE0-9\.-]+),\s*(?P<sst_mask_night_3>[eE0-9\.-]+)[^\n]*\n)?'
=====================================
debian/changelog
=====================================
@@ -1,3 +1,12 @@
+pygac (1.7.3-1) unstable; urgency=medium
+
+ * New upstream release (Closes: #1066765).
+ * debian/rules:
+ - Remove .pytest_cache after testing.
+ * Update dates in d/copyright.
+
+ -- Antonio Valentino <antonio.valentino at tiscali.it> Sun, 24 Mar 2024 16:57:04 +0000
+
pygac (1.7.2-2) unstable; urgency=medium
[ Bas Couwenberg ]
=====================================
debian/copyright
=====================================
@@ -18,7 +18,7 @@ Copyright: 2010-2020, Martin Raspaud <martin.raspaud at smhi.se>
License: GPL-3+
Files: debian/*
-Copyright: 2018-2023, Antonio Valentino <antonio.valentino at tiscali.it>
+Copyright: 2018-2024, Antonio Valentino <antonio.valentino at tiscali.it>
License: GPL-3+
License: GPL-3+
=====================================
debian/rules
=====================================
@@ -10,6 +10,7 @@ export DEB_BUILD_MAINT_OPTIONS=hardening=+all
export PYGAC_CONFIG_FILE=${CURDIR}/debian/pygac-test.cfg
export SETUPTOOLS_SCM_PRETEND_VERSION=$(DEB_VERSION_UPSTREAM)
export PYBUILD_TEST_ARGS=--pyargs pygac
+export PYBUILD_AFTER_TEST=${RM} -r {build_dir}/.pytest_cache
%:
dh $@ --buildsystem=pybuild
=====================================
pygac/calibration.py
=====================================
@@ -62,7 +62,11 @@ class Calibrator(object):
'689386c822de18a07194ac7fd71652ea': {
'name': 'PATMOS-x, v2017r1, with provisional coefficients for MetOp-C',
'status': CoeffStatus.PROVISIONAL
- }
+ },
+ 'e8735ec394ecdb87b7edcd261e72d2eb': {
+ 'name': 'PATMOS-x, v2023',
+ 'status': CoeffStatus.PROVISIONAL
+ },
}
fields = [
"dark_count", "gain_switch", "s0", "s1", "s2", "b", # "b0", "b1", "b2",
=====================================
pygac/data/calibration.json
=====================================
@@ -43,23 +43,23 @@
"channel_1": {
"dark_count": 40.43,
"gain_switch": 501.0,
- "s0": 0.11266666666666668,
- "s1": 0.609,
- "s2": -0.029
+ "s0": 0.11133333333333334,
+ "s1": 0.887,
+ "s2": -0.033
},
"channel_2": {
"dark_count": 39.75,
"gain_switch": 500.0,
- "s0": 0.13266666666666668,
- "s1": 0.98,
- "s2": -0.016
+ "s0": 0.13333333333333333,
+ "s1": 0.807,
+ "s2": 0.006
},
"channel_3a": {
"dark_count": 41.8,
"gain_switch": 502.0,
- "s0": 0.1217142857142857,
- "s1": 1.224,
- "s2": -0.033
+ "s0": 0.12457142857142857,
+ "s1": 1.358,
+ "s2": -0.035
},
"channel_3b": {
"b0": 0.0,
@@ -123,22 +123,22 @@
"dark_count": 39.7,
"gain_switch": 501.12,
"s0": 0.11066666666666668,
- "s1": 2.019,
- "s2": -0.201
+ "s1": 1.893,
+ "s2": -0.14
},
"channel_2": {
"dark_count": 40.0,
"gain_switch": 500.82,
- "s0": 0.122,
- "s1": 1.476,
- "s2": -0.137
+ "s0": 0.12266666666666666,
+ "s1": 1.392,
+ "s2": -0.08
},
"channel_3a": {
"dark_count": 40.3,
"gain_switch": 501.32,
- "s0": 0.11485714285714287,
- "s1": 1.748,
- "s2": -0.033
+ "s0": 0.1142857142857143,
+ "s1": 2.605,
+ "s2": -0.189
},
"channel_3b": {
"b0": 0.0,
@@ -201,23 +201,23 @@
"channel_1": {
"dark_count": 40.41,
"gain_switch": 498.68,
- "s0": 0.10866666666666668,
- "s1": 7.424,
- "s2": -2.579
+ "s0": 0.11,
+ "s1": 1.497,
+ "s2": -0.086
},
"channel_2": {
"dark_count": 40.94,
"gain_switch": 500.01,
- "s0": 0.13,
- "s1": -0.9,
- "s2": 1.278
+ "s0": 0.12866666666666668,
+ "s1": 3.982,
+ "s2": -0.51
},
"channel_3a": {
"dark_count": 40.57,
"gain_switch": 498.72,
- "s0": 0.16228571428571428,
- "s1": -45.535,
- "s2": 20.122
+ "s0": 0.12457142857142857,
+ "s1": 5.208,
+ "s2": -0.91
},
"channel_3b": {
"b0": 0.0,
@@ -281,20 +281,20 @@
"dark_count": 39.44,
"gain_switch": null,
"s0": 0.111,
- "s1": 6.087,
- "s2": -1.039
+ "s1": 6.031,
+ "s2": -1.089
},
"channel_2": {
"dark_count": 39.4,
"gain_switch": null,
"s0": 0.137,
- "s1": 0.119,
- "s2": 0.123
+ "s1": -0.006,
+ "s2": 0.179
},
"channel_3a": {
"dark_count": 37.51,
"gain_switch": null,
- "s0": 0.1,
+ "s0": 0.0,
"s1": 0.0,
"s2": 0.0
},
@@ -360,15 +360,15 @@
"dark_count": 40.0,
"gain_switch": null,
"s0": 0.11,
- "s1": 0.632,
- "s2": -0.044
+ "s1": 0.563,
+ "s2": -0.031
},
"channel_2": {
"dark_count": 40.0,
"gain_switch": null,
- "s0": 0.118,
- "s1": -0.037,
- "s2": 0.072
+ "s0": 0.117,
+ "s1": 0.164,
+ "s2": 0.039
},
"channel_3a": {
"dark_count": 40.0,
@@ -438,21 +438,21 @@
"channel_1": {
"dark_count": 41.0,
"gain_switch": null,
- "s0": 0.121,
- "s1": 2.032,
- "s2": -0.032
+ "s0": 0.12,
+ "s1": 2.184,
+ "s2": -0.051
},
"channel_2": {
"dark_count": 40.0,
"gain_switch": null,
- "s0": 0.148,
- "s1": 1.323,
- "s2": -0.008
+ "s0": 0.151,
+ "s1": 0.505,
+ "s2": 0.118
},
"channel_3a": {
"dark_count": 40.0,
"gain_switch": null,
- "s0": 0.1,
+ "s0": 0.0,
"s1": 0.0,
"s2": 0.0
},
@@ -518,20 +518,20 @@
"dark_count": 41.0,
"gain_switch": null,
"s0": 0.121,
- "s1": 3.555,
- "s2": -0.339
+ "s1": 3.559,
+ "s2": -0.334
},
"channel_2": {
"dark_count": 41.0,
"gain_switch": null,
- "s0": 0.152,
- "s1": 0.254,
- "s2": 0.201
+ "s0": 0.148,
+ "s1": 1.342,
+ "s2": 0.096
},
"channel_3a": {
"dark_count": 39.0,
"gain_switch": null,
- "s0": 0.1,
+ "s0": 0.0,
"s1": 0.0,
"s2": 0.0
},
@@ -596,20 +596,20 @@
"channel_1": {
"dark_count": 39.0,
"gain_switch": 500.0,
- "s0": 0.11933333333333333,
- "s1": -0.069,
- "s2": 0.002
+ "s0": 0.12,
+ "s1": -0.241,
+ "s2": 0.012
},
"channel_2": {
"dark_count": 40.0,
"gain_switch": 500.0,
- "s0": 0.13733333333333334,
- "s1": 0.339,
- "s2": -0.01
+ "s0": 0.138,
+ "s1": 0.095,
+ "s2": 0.008
},
"channel_3a": {
"dark_count": 39.0,
- "gain_switch": 500.0,
+ "gain_switch": null,
"s0": 0.1,
"s1": 0.0,
"s2": 0.0
@@ -676,22 +676,22 @@
"dark_count": 39.3,
"gain_switch": 498.96,
"s0": 0.11,
- "s1": 0.839,
- "s2": -0.051
+ "s1": 1.268,
+ "s2": -0.126
},
"channel_2": {
"dark_count": 38.9,
"gain_switch": 500.17,
"s0": 0.11933333333333333,
- "s1": 0.786,
- "s2": -0.031
+ "s1": 0.758,
+ "s2": -0.06
},
"channel_3a": {
"dark_count": 38.4,
"gain_switch": 499.43,
- "s0": 0.10685714285714286,
- "s1": 0.29,
- "s2": -0.294
+ "s0": 0.108,
+ "s1": -0.146,
+ "s2": -0.27
},
"channel_3b": {
"b0": 0.0,
@@ -754,23 +754,23 @@
"channel_1": {
"dark_count": 39.99,
"gain_switch": 501.12,
- "s0": 0.11466666666666665,
- "s1": 1.007,
- "s2": -0.044
+ "s0": 0.116,
+ "s1": 0.517,
+ "s2": 0.028
},
"channel_2": {
"dark_count": 39.09,
"gain_switch": 500.73,
- "s0": 0.14,
- "s1": 1.474,
- "s2": -0.118
+ "s0": 0.14133333333333334,
+ "s1": 0.739,
+ "s2": 0.026
},
"channel_3a": {
"dark_count": 42.09,
"gain_switch": 501.37,
- "s0": 0.11942857142857143,
- "s1": 2.787,
- "s2": -0.292
+ "s0": 0.12,
+ "s1": 3.086,
+ "s2": -0.301
},
"channel_3b": {
"b0": 0.0,
@@ -833,21 +833,21 @@
"channel_1": {
"dark_count": 39.44,
"gain_switch": 500.54,
- "s0": 0.114,
- "s1": 0.603,
- "s2": -0.0
+ "s0": 0.11133333333333334,
+ "s1": 1.13,
+ "s2": -0.017
},
"channel_2": {
"dark_count": 39.4,
"gain_switch": 500.4,
- "s0": 0.128,
- "s1": 0.632,
- "s2": 0.045
+ "s0": 0.124,
+ "s1": 1.39,
+ "s2": 0.011
},
"channel_3a": {
"dark_count": 37.51,
"gain_switch": 500.56,
- "s0": 0.1,
+ "s0": 0.22350000000000014,
"s1": 0.0,
"s2": 0.0
},
@@ -912,21 +912,21 @@
"channel_1": {
"dark_count": 38.8,
"gain_switch": 496.43,
- "s0": 0.108,
- "s1": 0.626,
- "s2": -0.044
+ "s0": 0.10866666666666668,
+ "s1": 0.286,
+ "s2": 0.012
},
"channel_2": {
"dark_count": 39.0,
"gain_switch": 500.37,
"s0": 0.122,
- "s1": 0.95,
- "s2": -0.039
+ "s1": 0.478,
+ "s2": 0.052
},
"channel_3a": {
"dark_count": 39.4,
"gain_switch": 496.11,
- "s0": 0.1,
+ "s0": 0.10771428571385998,
"s1": 0.0,
"s2": 0.0
},
@@ -1005,7 +1005,7 @@
"channel_3a": {
"dark_count": 37.51,
"gain_switch": null,
- "s0": 0.1,
+ "s0": 0.0,
"s1": 0.0,
"s2": 0.0
},
@@ -1084,7 +1084,7 @@
"channel_3a": {
"dark_count": 39.0,
"gain_switch": null,
- "s0": 0.1,
+ "s0": 0.0,
"s1": 0.0,
"s2": 0.0
},
@@ -1149,21 +1149,21 @@
"channel_1": {
"dark_count": 39.44,
"gain_switch": null,
- "s0": 0.119,
- "s1": 6.065,
+ "s0": 0.126,
+ "s1": 2.974,
"s2": 0.0
},
"channel_2": {
"dark_count": 39.4,
"gain_switch": null,
- "s0": 0.136,
- "s1": 7.248,
+ "s0": 0.138,
+ "s1": 5.958,
"s2": 0.0
},
"channel_3a": {
"dark_count": 37.51,
"gain_switch": null,
- "s0": 0.1,
+ "s0": 0.0,
"s1": 0.0,
"s2": 0.0
},
@@ -1228,21 +1228,21 @@
"channel_1": {
"dark_count": 38.0,
"gain_switch": null,
- "s0": 0.108,
- "s1": 4.255,
- "s2": 0.64
+ "s0": 0.107,
+ "s1": 4.694,
+ "s2": 0.51
},
"channel_2": {
"dark_count": 40.0,
"gain_switch": null,
- "s0": 0.122,
- "s1": 0.31,
- "s2": 0.642
+ "s0": 0.121,
+ "s1": 1.147,
+ "s2": 0.428
},
"channel_3a": {
"dark_count": 38.0,
"gain_switch": null,
- "s0": 0.1,
+ "s0": 0.0,
"s1": 0.0,
"s2": 0.0
},
@@ -1321,7 +1321,7 @@
"channel_3a": {
"dark_count": 37.51,
"gain_switch": null,
- "s0": 0.1,
+ "s0": 0.0,
"s1": 0.0,
"s2": 0.0
},
@@ -1382,5 +1382,4 @@
"d4": 0.0
}
}
-}
-
+}
\ No newline at end of file
=====================================
pygac/pod_reader.py
=====================================
@@ -279,7 +279,7 @@ class PODReader(Reader):
# choose the right header depending on the date
with file_opener(fileobj or filename) as fd_:
self.tbm_head, self.head = self.read_header(
- filename, fileobj=fd_)
+ filename, fileobj=fd_, header_date=self.header_date)
if self.tbm_head:
tbm_offset = tbm_header.itemsize
else:
@@ -302,12 +302,14 @@ class PODReader(Reader):
return self.head, self.scans
@classmethod
- def read_header(cls, filename, fileobj=None):
+ def read_header(cls, filename, fileobj=None, header_date="auto"):
"""Read the file header.
Args:
filename (str): Path to GAC/LAC file
fileobj: An open file object to read from. (optional)
+ header_date: date to use to choose the header.
+ Defaults to "auto" to use the data to pick the header corresponding to the date of the file.
Returns:
archive_header (struct): archive header
@@ -332,19 +334,7 @@ class PODReader(Reader):
fd_.seek(0)
tbm_head = None
tbm_offset = 0
- # read header
- head0, = np.frombuffer(
- fd_.read(header0.itemsize),
- dtype=header0, count=1)
- year, jday, _ = cls.decode_timestamps(head0["start_time"])
- start_date = (datetime.date(year, 1, 1) +
- datetime.timedelta(days=int(jday) - 1))
- if start_date < datetime.date(1992, 9, 8):
- header = header1
- elif start_date <= datetime.date(1994, 11, 15):
- header = header2
- else:
- header = header3
+ header = cls.choose_header_based_on_timestamp(header_date, fd_)
fd_.seek(tbm_offset, 0)
# need to copy frombuffer to have write access on head
head, = np.frombuffer(
@@ -354,6 +344,31 @@ class PODReader(Reader):
cls._validate_header(head)
return tbm_head, head
+ @classmethod
+ def choose_header_based_on_timestamp(cls, header_date, fd_):
+ """Choose the header dtype based on the timestamp."""
+ if header_date == "auto":
+ header_date = cls.get_start_date(fd_)
+ if header_date < datetime.date(1992, 9, 8):
+ header = header1
+ elif header_date <= datetime.date(1994, 11, 15):
+ header = header2
+ else:
+ header = header3
+ return header
+
+ @classmethod
+ def get_start_date(cls, fd_):
+ """Get the start time from the filestream."""
+ head0, = np.frombuffer(
+ fd_.read(header0.itemsize),
+ dtype=header0, count=1)
+ year, jday, _ = cls.decode_timestamps(head0["start_time"])
+ header_date = (datetime.date(year, 1, 1) +
+ datetime.timedelta(days=int(jday) - 1))
+
+ return header_date
+
@classmethod
def _validate_header(cls, header):
"""Check if the header belongs to this reader."""
=====================================
pygac/reader.py
=====================================
@@ -86,6 +86,10 @@ class NoTLEData(IndexError):
"""Raised if no TLE data available within time range."""
+class DecodingError(ValueError):
+ """Raised when decoding of some value fails."""
+
+
class Reader(six.with_metaclass(ABCMeta)):
"""Reader for GAC and LAC format, POD and KLM platforms."""
@@ -95,7 +99,7 @@ class Reader(six.with_metaclass(ABCMeta)):
def __init__(self, interpolate_coords=True, adjust_clock_drift=True,
tle_dir=None, tle_name=None, tle_thresh=7, creation_site=None,
- custom_calibration=None, calibration_file=None):
+ custom_calibration=None, calibration_file=None, header_date="auto"):
"""Init the reader.
Args:
@@ -111,6 +115,7 @@ class Reader(six.with_metaclass(ABCMeta)):
custom_calibration: dictionary with a subset of user defined satellite specific
calibration coefficients
calibration_file: path to json file containing default calibrations
+ header_date: the date to use for pod header choice. Defaults to "auto".
"""
self.meta_data = {}
@@ -122,6 +127,7 @@ class Reader(six.with_metaclass(ABCMeta)):
self.creation_site = (creation_site or 'NSS').encode('utf-8')
self.custom_calibration = custom_calibration
self.calibration_file = calibration_file
+ self.header_date = header_date
self.head = None
self.scans = None
self.spacecraft_name = None
@@ -205,22 +211,38 @@ class Reader(six.with_metaclass(ABCMeta)):
filename (str): path to file
"""
filename = str(filename)
- data_set_name = header['data_set_name'].decode(errors='ignore')
- if not cls.data_set_pattern.match(data_set_name):
- LOG.debug('The data_set_name in header %s does not match.'
- ' Use filename instead.' % header['data_set_name'])
+ for encoding in "utf-8", "cp500":
+ data_set_name = header['data_set_name']
+ try:
+ data_set_name = cls._decode_data_set_name(data_set_name, encoding)
+ except DecodingError as err:
+ LOG.debug(str(err))
+ else:
+ header["data_set_name"] = data_set_name
+ break
+ else:
+ LOG.debug(f'The data_set_name in header {header["data_set_name"]} does not match.'
+ ' Use filename instead.')
match = cls.data_set_pattern.search(filename)
if match:
data_set_name = match.group()
- LOG.debug("Set data_set_name, to filename %s"
- % data_set_name)
+ LOG.debug(f"Set data_set_name, to filename {data_set_name}")
header['data_set_name'] = data_set_name.encode()
else:
- LOG.debug("header['data_set_name']=%s; filename='%s'"
- % (header['data_set_name'], filename))
+ LOG.debug(f"header['data_set_name']={header['data_set_name']}; filename='{filename}'")
raise ReaderError('Cannot determine data_set_name!')
return header
+ @classmethod
+ def _decode_data_set_name(cls, data_set_name, encoding):
+ data_set_name = data_set_name.decode(encoding, errors='ignore')
+ if not cls.data_set_pattern.match(data_set_name):
+ raise DecodingError(f'The data_set_name in header {data_set_name} '
+ f'does not seem correct using encoding {encoding}.')
+ else:
+ data_set_name = data_set_name.encode()
+ return data_set_name
+
@classmethod
def _validate_header(cls, header):
"""Check if the header belongs to this reader.
@@ -274,7 +296,7 @@ class Reader(six.with_metaclass(ABCMeta)):
"Expected %d scan lines, but found %d!"
% (count, line_count))
warnings.warn("Unexpected number of scanlines!",
- category=RuntimeWarning)
+ category=RuntimeWarning, stacklevel=2)
self.scans = np.frombuffer(
buffer, dtype=self.scanline_type, count=line_count)
=====================================
pygac/tests/test_calibrate_klm.py
=====================================
@@ -63,12 +63,12 @@ class TestGenericCalibration(unittest.TestCase):
ref3 = calibrate_solar(data, channel, year, jday, cal, corr)
- expected = (np.array([[np.nan, 27.37909518, 110.60103456],
- [0.11943135, 6.03671211, 57.99695154]]),
- np.array([[np.nan, 3.05229160e+01, 1.24811455e+02],
- [1.23011792e-01, 6.82715447e+00, 6.52122414e+01]]),
- np.array([[0., 523.41775, 1034.41775],
- [41., 150., 711.41775]]))
+ expected = (np.array([[np.nan, 27.32328509, 110.84050459],
+ [0.1191198, 6.02096454, 58.0497768]]),
+ np.array([[np.nan, 3.04160070e+01, 1.24374292e+02],
+ [1.22580933e-01, 6.80324179e+00, 6.49838301e+01]]),
+ np.array([[0., 524.33117, 1035.33117],
+ [41., 150., 712.33117]]))
np.testing.assert_allclose(ref1, expected[0])
np.testing.assert_allclose(ref2, expected[1])
np.testing.assert_allclose(ref3, expected[2])
@@ -87,9 +87,9 @@ class TestGenericCalibration(unittest.TestCase):
ict_counts = np.array([[745.3, 397.9, 377.8],
[744.8, 398.1, 378.4],
[745.7, 398., 378.3]])
- space_counts = np.array([[987.3, 992.5, 989.4],
- [986.9, 992.8, 989.6],
- [986.3, 992.3, 988.9]])
+ space_counts = np.array([[987.3, 992.5, 989.4],
+ [986.9, 992.8, 989.6],
+ [986.3, 992.3, 988.9]])
spacecraft_id = "noaa19"
cal = Calibrator(spacecraft_id)
=====================================
pygac/tests/test_calibrate_pod.py
=====================================
@@ -64,10 +64,10 @@ class TestGenericCalibration(unittest.TestCase):
ref3 = calibrate_solar(data, channel, year, jday, cal, corr)
- expected = (np.array([[np.nan, 60.891074, 126.953364],
- [0., 14.091565, 85.195791]]),
- np.array([[np.nan, 72.98262, 152.16334],
- [0., 16.889821, 102.113687]]),
+ expected = (np.array([[np.nan, 60.91525491, 127.00377987],
+ [0., 14.0971609, 85.22962417]]),
+ np.array([[np.nan, 72.51635437, 151.19121018],
+ [0., 16.7819164, 101.46131111]]),
np.array([[-32001., -32001., -32001.],
[-32001., -32001., -32001.]]))
@@ -89,9 +89,9 @@ class TestGenericCalibration(unittest.TestCase):
ict_counts = np.array([[745.3, 397.9, 377.8],
[744.8, 398.1, 378.4],
[745.7, 398., 378.3]])
- space_counts = np.array([[987.3, 992.5, 989.4],
- [986.9, 992.8, 989.6],
- [986.3, 992.3, 988.9]])
+ space_counts = np.array([[987.3, 992.5, 989.4],
+ [986.9, 992.8, 989.6],
+ [986.3, 992.3, 988.9]])
spacecraft_id = "noaa14"
cal = Calibrator(spacecraft_id)
=====================================
pygac/tests/test_klm.py
=====================================
@@ -38,7 +38,7 @@ from pygac.tests.utils import CalledWithArray
class TestKLM:
"""Test the klm reader."""
- def setup(self):
+ def setup_method(self):
"""Set up the tests."""
self.reader = GACKLMReader()
@@ -126,7 +126,7 @@ class TestKLM:
class TestGACKLM:
"""Tests for gac klm."""
- def setup(self):
+ def setup_method(self):
"""Set up the tests."""
self.reader = GACKLMReader()
@@ -150,7 +150,7 @@ class TestGACKLM:
class TestLACKLM:
"""Tests for lac klm."""
- def setup(self):
+ def setup_method(self):
"""Set up the tests."""
self.reader = LACKLMReader()
self.reader.scans = np.ones(100, dtype=scanline)
=====================================
pygac/tests/test_reader.py
=====================================
@@ -24,11 +24,9 @@ import datetime
import os
import sys
import unittest
+import pytest
-try:
- import mock
-except ImportError:
- from unittest import mock
+from unittest import mock
import numpy as np
import numpy.testing
from pygac.gac_reader import GACReader, ReaderError
@@ -36,6 +34,10 @@ from pygac.lac_reader import LACReader
from pygac.pod_reader import POD_QualityIndicator
from pygac.gac_pod import scanline
from pygac.reader import NoTLEData
+from pygac.lac_pod import LACPODReader
+
+from pygac.pod_reader import tbm_header as tbm_header_dtype, header3
+from pygac.lac_pod import scanline as lacpod_scanline
class TestPath(os.PathLike):
@@ -127,9 +129,6 @@ class TestGacReader(unittest.TestCase):
"""Set up the tests."""
self.interpolator = interpolator
self.reader = GACReader()
- # python 2 compatibility
- if sys.version_info.major < 3:
- self.assertRaisesRegex = self.assertRaisesRegexp
def test_filename(self):
"""Test the setter of the filename property."""
@@ -174,29 +173,62 @@ class TestGacReader(unittest.TestCase):
head = {'data_set_name': b'\xea\xf8'}
self.reader._validate_header(head)
- def test__correct_data_set_name(self):
+ def test__correct_data_set_name_ebcdic_encoded_header_invalid_path(self):
+ """Test the data_set_name correction in file header."""
+ inv_filename = 'InvalidFileName'
+ inv_filepath = 'path/to/' + inv_filename
+
+ expected_data_set_name = 'NSS.GHRR.TN.D80001.S0332.E0526.B0627173.WI'
+ val_head = {'data_set_name': 'NSS.GHRR.TN.D80001.S0332.E0526.B0627173.WI'.encode("cp500")}
+ head = self.reader._correct_data_set_name(val_head.copy(), inv_filepath)
+ assert head['data_set_name'] == expected_data_set_name.encode()
+
+ def test__correct_data_set_name_valid_header_and_file(self):
"""Test the data_set_name correction in file header."""
val_filename = 'NSS.GHRR.TN.D80001.S0332.E0526.B0627173.WI'
val_filepath = 'path/to/' + val_filename
val_head = {'data_set_name': b'NSS.GHRR.TN.D80001.S0332.E0526.B0627173.WI'}
- inv_filename = 'InvalidFileName'
- inv_filepath = 'path/to/' + inv_filename
- inv_head = {'data_set_name': b'InvalidDataSetName'}
# Note: always pass a copy to _correct_data_set_name, because
# the input header is modified in place.
# enter a valid data_set_name and filepath
head = self.reader._correct_data_set_name(val_head.copy(), val_filepath)
+ assert head['data_set_name'] == val_filename.encode()
+
+ def test__correct_data_set_name_invalid_header_and_valid_file(self):
+ """Test the data_set_name correction in file header."""
+ val_filename = 'NSS.GHRR.TN.D80001.S0332.E0526.B0627173.WI'
+ val_filepath = 'path/to/' + val_filename
+ inv_head = {'data_set_name': b'InvalidDataSetName'}
+
# enter an invalid data_set_name, but valid filepath
head = self.reader._correct_data_set_name(inv_head.copy(), val_filepath)
- self.assertEqual(head['data_set_name'], val_filename.encode())
- # enter an invalid data_set_name, and invalid filepath
+ assert head['data_set_name'] == val_filename.encode()
+
+ def test__correct_data_set_name_invalid_header_and_file(self):
+ """Test the data_set_name correction in file header."""
+ inv_filename = 'InvalidFileName'
+ inv_filepath = 'path/to/' + inv_filename
+ inv_head = {'data_set_name': b'InvalidDataSetName'}
with self.assertRaisesRegex(ReaderError, 'Cannot determine data_set_name!'):
- head = self.reader._correct_data_set_name(inv_head.copy(), inv_filepath)
+ _ = self.reader._correct_data_set_name(inv_head.copy(), inv_filepath)
+
+ def test__correct_data_set_name_valid_header_invalid_file(self):
+ """Test the data_set_name correction in file header."""
+ val_head = {'data_set_name': b'NSS.GHRR.TN.D80001.S0332.E0526.B0627173.WI'}
+ inv_filename = 'InvalidFileName'
+ inv_filepath = 'path/to/' + inv_filename
+
# enter a valid data_set_name, and an invalid filepath
# should be fine, because the data_set_name is the pefered source
head = self.reader._correct_data_set_name(val_head.copy(), inv_filepath)
- self.assertEqual(head['data_set_name'], val_head['data_set_name'])
- # enter a valid data_set_name, and an FSFile/pathlib object as filepath
+ assert head['data_set_name'] == val_head['data_set_name']
+
+ def test__correct_data_set_name_valid_header_pathlib_file(self):
+ """Test the data_set_name correction in file header."""
+ val_filename = 'NSS.GHRR.TN.D80001.S0332.E0526.B0627173.WI'
+ val_filepath = 'path/to/' + val_filename
+ val_head = {'data_set_name': b'NSS.GHRR.TN.D80001.S0332.E0526.B0627173.WI'}
+
fs_filepath = TestPath(val_filepath)
head = self.reader._correct_data_set_name(val_head.copy(), fs_filepath)
self.assertEqual(head['data_set_name'], val_filename.encode())
@@ -648,3 +680,105 @@ class TestLacReader(unittest.TestCase):
self.reader.correct_scan_line_numbers()
numpy.testing.assert_array_equal(self.reader.scans['scan_line_number'],
expected)
+
+
+ at pytest.fixture
+def pod_file_with_tbm_header(tmp_path):
+ """Create a pod file with a tbm header, and header, and some scanlines."""
+ number_of_scans = 3
+
+ tbm_header = np.zeros(1, dtype=tbm_header_dtype)
+ tbm_header["data_set_name"] = b"BRN.HRPT.NJ.D00322.S0334.E0319.B3031919.BL "
+ tbm_header["select_flag"] = b"S"
+ tbm_header["beginning_latitude"] = b"+77"
+ tbm_header["ending_latitude"] = b"+22"
+ tbm_header["beginning_longitude"] = b"-004"
+ tbm_header["ending_longitude"] = b"+032"
+ tbm_header["start_hour"] = b'AL'
+ tbm_header["start_minute"] = b'L '
+ tbm_header["number_of_minutes"] = b'ALL'
+ tbm_header["appended_data_flag"] = b'Y'
+ tbm_header["channel_select_flag"][0, :5] = b'\x01'
+ tbm_header["sensor_data_word_size"] = b'10'
+
+ header = np.zeros(1, dtype=header3)
+ header["noaa_spacecraft_identification_code"] = 3
+ header["data_type_code"] = 48
+ header["start_time"] = [51522, 181, 62790]
+ header["number_of_scans"] = number_of_scans
+ header["end_time"] = [51522, 195, 42286]
+ header["processing_block_id"] = b'3031919'
+ header["ramp_auto_calibration"] = 0
+ header["number_of_data_gaps"] = 0
+ header["dacs_quality"] = [21, 7, 0, 0, 0, 0]
+ header["calibration_parameter_id"] = 12336
+ header["dacs_status"] = 24
+ header["nadir_earth_location_tolerance"] = 20
+ header["start_of_data_set_year"] = 2000
+ # EBCDIC, aka cp500 encoding
+ header["data_set_name"] = (b"\xc2\xd9\xd5K\xc8\xd9\xd7\xe3K\xd5\xd1K\xc4\xf0\xf0\xf3\xf2\xf2K\xe2\xf0\xf3\xf1\xf9K"
+ b"\xc5\xf0\xf3\xf3\xf4K\xc2\xf3\xf0\xf3\xf1\xf9\xf1\xf9K\xc2\xd3@@")
+ header["year_of_epoch"] = 2000
+ header["julian_day_of_epoch"] = 322
+ header["millisecond_utc_epoch_time_of_day"] = 11864806
+ header["semi_major_axis"] = 7226239
+ header["eccentricity"] = 100496
+ header["inclination"] = 9915900
+ header["argument_of_perigee"] = 2511798
+ header["right_ascension"] = 30366227
+ header["mean_anomaly"] = 7341437
+ header["x_component_of_position_vector"] = 40028760
+ header["y_component_of_position_vector"] = -60151283
+ header["z_component_of_position_vector"] = 0,
+ header["x_dot_component_of_position_vector"] = -990271
+ header["y_dot_component_of_position_vector"] = -646066
+ header["z_dot_component_of_position_vector"] = 7337861
+ header["yaw_fixed_error_correction"] = 0
+ header["roll_fixed_error_correction"] = 0
+ header["pitch_fixed_error_correction"] = 0
+
+ scanlines = np.zeros(number_of_scans, dtype=lacpod_scanline)
+ scanlines["scan_line_number"] = np.arange(number_of_scans) + 1
+ scanlines["time_code"][:, :2] = [51522, 181]
+ scanlines["time_code"][:, 2] = np.arange(62790, 62790 + 166 * number_of_scans, 166)
+ scanlines["quality_indicators"] = 1073741824
+ scanlines["calibration_coefficients"] = [149722896, -23983736, 173651248, -27815402, -1803673, 6985664, -176321840,
+ 666680320, -196992576, 751880640]
+ scanlines["number_of_meaningful_zenith_angles_and_earth_location_appended"] = 51
+ scanlines["solar_zenith_angles"] = [-24, -26, -28, -30, -32, -33, -34, -35, -37, -37, -38, -39, -40, -41, -41, -42,
+ -43, -43, -44, -45, -45, -46, -46, -47, -48, -48, -49, -49, -50, -50, -51, -52,
+ -52, -53, -53, -54, -55, -55, -56, -57, -58, -59, -60, -61, -62, -63, -64, -66,
+ -68, -70, -73]
+ scanlines["earth_location"] = [9929, -36, 9966, 747, 9983, 1415, 9988, 1991, 9984, 2492, 9975, 2931,
+ 9963, 3320, 9948, 3667, 9931, 3979, 9913, 4262, 9895, 4519, 9875, 4755,
+ 9856, 4972, 9836, 5174, 9816, 5362, 9796, 5538, 9775, 5703, 9755, 5860,
+ 9734, 6009, 9713, 6150, 9692, 6286, 9671, 6416, 9650, 6542, 9628, 6663,
+ 9606, 6781, 9583, 6896, 9560, 7009, 9537, 7119, 9513, 7227, 9488, 7334,
+ 9463, 7440, 9437, 7545, 9409, 7650, 9381, 7755, 9351, 7860, 9320,
+ 7966, 9288, 8073, 9253, 8181, 9216, 8291, 9177, 8403, 9135, 8518,
+ 9090, 8637, 9040, 8759, 8986, 8886, 8926, 9019, 8859, 9159, 8784,
+ 9307, 8697, 9466, 8596, 9637, 8476, 9824, 8327, 10033]
+ scanlines["telemetry"] = 0
+ scanlines["sensor_data"] = 99
+ scanlines["add_on_zenith"] = 0
+ scanlines["clock_drift_delta"] = 0
+
+ pod_filename = tmp_path / "image.l1b"
+ offset = 14800
+ fill = np.zeros(offset - header3.itemsize, dtype=np.uint8)
+
+ with open(pod_filename, "wb") as fd_:
+ fd_.write(tbm_header.tobytes())
+ fd_.write(header.tobytes())
+ fd_.write(fill.tobytes())
+ fd_.write(scanlines.tobytes())
+ return pod_filename
+
+
+def test_podlac_eosip(pod_file_with_tbm_header):
+ """Test reading a real podlac file."""
+ reader = LACPODReader(interpolate_coords=False)
+ reader.read(pod_file_with_tbm_header)
+ assert reader.head.itemsize == header3.itemsize
+ # this is broken in eosip pod data, tbm data set name has start and end times reversed.
+ # assert reader.head["data_set_name"] == reader.tbm_head["data_set_name"]
View it on GitLab: https://salsa.debian.org/debian-gis-team/pygac/-/compare/ab2115aefd35fff5ab91467f53b548a010ac6eb7...981daf9e332273624bda28fc71de171ac1c2013b
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/pygac/-/compare/ab2115aefd35fff5ab91467f53b548a010ac6eb7...981daf9e332273624bda28fc71de171ac1c2013b
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20240324/94f64173/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list