[Git][debian-gis-team/pygac][master] 5 commits: New upstream version 1.7.0
Antonio Valentino (@antonio.valentino)
gitlab at salsa.debian.org
Fri Nov 4 07:10:57 GMT 2022
Antonio Valentino pushed to branch master at Debian GIS Project / pygac
Commits:
d91b63cb by Antonio Valentino at 2022-11-04T06:40:51+00:00
New upstream version 1.7.0
- - - - -
172e51ff by Antonio Valentino at 2022-11-04T06:40:55+00:00
Update upstream source from tag 'upstream/1.7.0'
Update to upstream version '1.7.0'
with Debian dir 77114efc6e5d0beab74444f9c7244be2fa363be9
- - - - -
18ccf62a by Antonio Valentino at 2022-11-04T06:41:35+00:00
New upstream release
- - - - -
6e0f4790 by Antonio Valentino at 2022-11-04T06:48:47+00:00
Refresh and renumber all patches
- - - - -
25c844f0 by Antonio Valentino at 2022-11-04T07:06:08+00:00
Set distribution to unstable
- - - - -
15 changed files:
- .github/workflows/ci.yaml
- CHANGELOG.md
- debian/changelog
- debian/patches/0001-Fix-config.patch
- debian/patches/0003-Install-the-tests-sub-package.patch → debian/patches/0002-Install-the-tests-sub-package.patch
- debian/patches/series
- doc/source/usage.rst
- pygac/gac_reader.py
- pygac/klm_reader.py
- pygac/lac_reader.py
- pygac/pod_reader.py
- pygac/reader.py
- pygac/tests/test_klm.py
- pygac/tests/test_reader.py
- setup.py
Changes:
=====================================
.github/workflows/ci.yaml
=====================================
@@ -10,7 +10,7 @@ jobs:
fail-fast: true
matrix:
os: ["ubuntu-latest"]
- python-version: ["3.7", "3.8"]
+ python-version: ["3.8", "3.9", "3.10"]
experimental: [false]
env:
=====================================
CHANGELOG.md
=====================================
@@ -1,4 +1,29 @@
-## Version <1.6.0> (2022/08/04)
+## Version 1.7.0 (2022/10/31)
+
+### Issues Closed
+
+* [Issue 112](https://github.com/pytroll/pygac/issues/112) - Usage documentation uses wrong function name for 'get_reader_class' ([PR 113](https://github.com/pytroll/pygac/pull/113) by [@mraspaud](https://github.com/mraspaud))
+* [Issue 78](https://github.com/pytroll/pygac/issues/78) - Handle completely missing ICT or space counts ([PR 117](https://github.com/pytroll/pygac/pull/117) by [@mraspaud](https://github.com/mraspaud))
+* [Issue 22](https://github.com/pytroll/pygac/issues/22) - Thermal calibration error ([PR 117](https://github.com/pytroll/pygac/pull/117) by [@mraspaud](https://github.com/mraspaud))
+
+In this release 3 issues were closed.
+
+### Pull Requests Merged
+
+#### Bugs fixed
+
+* [PR 117](https://github.com/pytroll/pygac/pull/117) - Fix calibration when 3b is totally deactivated ([78](https://github.com/pytroll/pygac/issues/78), [22](https://github.com/pytroll/pygac/issues/22))
+* [PR 116](https://github.com/pytroll/pygac/pull/116) - Fix typos and deprecations
+* [PR 113](https://github.com/pytroll/pygac/pull/113) - Fix usage typo ([112](https://github.com/pytroll/pygac/issues/112))
+
+#### Features added
+
+* [PR 118](https://github.com/pytroll/pygac/pull/118) - Bump up python versions
+* [PR 115](https://github.com/pytroll/pygac/pull/115) - Add support for frac data
+
+In this release 5 pull requests were closed.
+
+## Version 1.6.0 (2022/08/04)
### Issues Closed
=====================================
debian/changelog
=====================================
@@ -1,8 +1,11 @@
-pygac (1.6.0-2) UNRELEASED; urgency=medium
+pygac (1.7.0-1) unstable; urgency=medium
+ * New upstream release.
* Fix d/copyright formatting.
+ * debian/patches:
+ - refresh and renumber all patches.
- -- Antonio Valentino <antonio.valentino at tiscali.it> Wed, 10 Aug 2022 06:36:05 +0000
+ -- Antonio Valentino <antonio.valentino at tiscali.it> Fri, 04 Nov 2022 07:05:27 +0000
pygac (1.6.0-1) unstable; urgency=medium
=====================================
debian/patches/0001-Fix-config.patch
=====================================
@@ -27,10 +27,10 @@ index 9ca301e..9264fea 100644
return _config
diff --git a/setup.py b/setup.py
-index d544eaf..037ce46 100644
+index 0614742..e0a653f 100644
--- a/setup.py
+++ b/setup.py
-@@ -68,8 +68,8 @@ if __name__ == '__main__':
+@@ -63,8 +63,8 @@ if __name__ == '__main__':
install_requires=requirements,
extras_require=extras_require,
scripts=[os.path.join('bin', item) for item in os.listdir('bin')],
@@ -38,6 +38,6 @@ index d544eaf..037ce46 100644
- ('gapfilled_tles', ['gapfilled_tles/TLE_noaa16.txt'])],
+ data_files=[ # ('etc', ['etc/pygac.cfg.template']),
+ ('share/pygac/gapfilled_tles', ['gapfilled_tles/TLE_noaa16.txt'])],
- python_requires='>=3.6',
+ python_requires='>=3.8',
zip_safe=False
)
=====================================
debian/patches/0003-Install-the-tests-sub-package.patch → debian/patches/0002-Install-the-tests-sub-package.patch
=====================================
@@ -8,10 +8,10 @@ Forwarded: not-needed
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
-index 037ce46..992a497 100644
+index e0a653f..8df8781 100644
--- a/setup.py
+++ b/setup.py
-@@ -60,7 +60,7 @@ if __name__ == '__main__':
+@@ -55,7 +55,7 @@ if __name__ == '__main__':
long_description_content_type='text/markdown',
license='GPLv3',
=====================================
debian/patches/series
=====================================
@@ -1,2 +1,2 @@
0001-Fix-config.patch
-0003-Install-the-tests-sub-package.patch
+0002-Install-the-tests-sub-package.patch
=====================================
doc/source/usage.rst
=====================================
@@ -53,10 +53,10 @@ Alternatively you can also use Pygac directly.
.. code-block:: python
- from pygac import get_reader_cls
-
+ from pygac import get_reader_class
+
filename = 'NSS.GHRR.NP.D15361.S0121.E0315.B3547172.SV'
- reader_cls = get_reader_cls(filename)
+ reader_cls = get_reader_class(filename)
reader = reader_cls(tle_dir='/path/to/tle', tle_name='TLE_%(satname)s.txt')
reader.read(filename)
@@ -82,7 +82,7 @@ it in a directory as you please. Set the environment variable ``PYGAC_CONFIG_FIL
pointing to the file. e.g.
.. code-block:: bash
-
+
PYGAC_CONFIG_FILE=/home/user/pygac.cfg; export PYGAC_CONFIG_FILE
Also adapt the configuration file to your needs. The ``tledir`` parameter should
@@ -93,7 +93,7 @@ Then call ``pygac-run`` on a GAC/LAC file.
.. code-block:: bash
pygac-run testdata/NSS.GHRR.NL.D02187.S1904.E2058.B0921517.GC 0 0
-
+
The last two digits are the start and end scanline numbers, thus specifying the
portion of the GAC orbit that user wants to process. The first scanline number
starts at 0. If zeroes are specified at both locations, then the entire orbit
=====================================
pygac/gac_reader.py
=====================================
@@ -34,11 +34,14 @@ import pygac.pygac_geotiepoints as gtp
LOG = logging.getLogger(__name__)
+
class GACReader(Reader):
"""Reader for GAC data."""
# Scanning frequency (scanlines per millisecond)
scan_freq = 2.0 / 1000.0
+ # Max scanlines
+ max_scanlines = 15000
def __init__(self, *args, **kwargs):
"""Init the GAC reader."""
@@ -48,7 +51,7 @@ class GACReader(Reader):
@classmethod
def _validate_header(cls, header):
- """Check if the header belongs to this reader"""
+ """Check if the header belongs to this reader."""
# call super to enter the Method Resolution Order (MRO)
super(GACReader, cls)._validate_header(header)
LOG.debug("validate header")
=====================================
pygac/klm_reader.py
=====================================
@@ -66,7 +66,6 @@ class KLM_QualityIndicator(IntFlag):
post-January 25, 2006, all spacecraft).
Notes:
-
- Table 8.3.1.3.3.1-1. and Table 8.3.1.4.3.1-1. define bit: 21 as
"frame sync word not valid"
- Table 8.3.1.3.3.2-1. and Table 8.3.1.4.3.2-1. define bit: 21 as
@@ -103,7 +102,7 @@ class KLM_QualityIndicator(IntFlag):
PSEUDO_NOISE = 2**0 # Pseudo noise occurred on this frame
-# GAC header object
+# header object
header = np.dtype([("data_set_creation_site_id", "S3"),
("ascii_blank_=_x20", "S1"),
@@ -721,7 +720,7 @@ class KLMReader(Reader):
@classmethod
def _validate_header(cls, header):
- """Check if the header belongs to this reader"""
+ """Check if the header belongs to this reader."""
# call super to enter the Method Resolution Order (MRO)
super(KLMReader, cls)._validate_header(header)
LOG.debug("validate header")
@@ -795,12 +794,18 @@ class KLMReader(Reader):
def get_ch3_switch(self):
"""Channel 3 identification.
- 0: Channel 3b (Brightness temperature
+ 0: Channel 3b (Brightness temperature)
1: Channel 3a (Reflectance)
2: Transition (No data)
"""
return self.scans["scan_line_bit_field"][:] & 3
+ def _get_ir_channels_to_calibrate(self):
+ ir_channels_to_calibrate = [3, 4, 5]
+ if np.all(self.get_ch3_switch() != 0):
+ ir_channels_to_calibrate = [4, 5]
+ return ir_channels_to_calibrate
+
def postproc(self, channels):
"""Apply KLM specific postprocessing.
@@ -814,6 +819,7 @@ class KLMReader(Reader):
def _adjust_clock_drift(self):
"""Adjust the geolocation to compensate for the clock error.
+
Note:
Clock drift correction is only applied to POD satellites.
On the KLM series, the clock is updated daily.
=====================================
pygac/lac_reader.py
=====================================
@@ -32,11 +32,14 @@ import pygac.pygac_geotiepoints as gtp
LOG = logging.getLogger(__name__)
+
class LACReader(Reader):
"""Reader for LAC data."""
# Scanning frequency (scanlines per millisecond)
scan_freq = 6.0 / 1000.0
+ # Max scanlines
+ max_scanlines = 65535
def __init__(self, *args, **kwargs):
"""Init the LAC reader."""
@@ -46,7 +49,7 @@ class LACReader(Reader):
@classmethod
def _validate_header(cls, header):
- """Check if the header belongs to this reader"""
+ """Check if the header belongs to this reader."""
# call super to enter the Method Resolution Order (MRO)
super(LACReader, cls)._validate_header(header)
LOG.debug("validate header")
@@ -54,5 +57,5 @@ class LACReader(Reader):
# split header into parts
creation_site, transfer_mode, platform_id = (
data_set_name.split('.')[:3])
- if transfer_mode not in ['LHRR', 'HRPT']:
+ if transfer_mode not in ["LHRR", "HRPT", "FRAC"]:
raise ReaderError('Improper transfer mode "%s"!' % transfer_mode)
=====================================
pygac/pod_reader.py
=====================================
@@ -64,6 +64,7 @@ class POD_QualityIndicator(IntFlag):
Source:
POD guide Table 3.1.2.1-2. Format of quality indicators.
"""
+
# POD guide Table 3.1.2.1-2. Format of quality indicators.
FATAL_FLAG = 2**31 # Data should not be used for product generation
TIME_ERROR = 2**30 # A time sequence error was detected while Processing
@@ -420,7 +421,7 @@ class PODReader(Reader):
return self.decode_timestamps(self.scans["time_code"])
def _compute_missing_lonlat(self, missed_utcs):
- """compute lon lat values using pyorbital"""
+ """Compute lon lat values using pyorbital."""
tic = datetime.datetime.now()
scan_rate = datetime.timedelta(milliseconds=1/self.scan_freq).total_seconds()
@@ -560,6 +561,11 @@ class PODReader(Reader):
return prt_counts, ict_counts, space_counts
+ @staticmethod
+ def _get_ir_channels_to_calibrate():
+ ir_channels_to_calibrate = [3, 4, 5]
+ return ir_channels_to_calibrate
+
def postproc(self, channels):
"""No POD specific postprocessing to be done."""
pass
@@ -573,7 +579,7 @@ class PODReader(Reader):
channels[:, :, 3], channels[:, :, 4])
def _get_calibrated_channels_uniform_shape(self):
- """Prepare the channels as input for gac_io.save_gac"""
+ """Prepare the channels as input for gac_io.save_gac."""
_channels = self.get_calibrated_channels()
# prepare input
# maybe there is a better (less memory requiring) method
=====================================
pygac/reader.py
=====================================
@@ -24,9 +24,10 @@
Can't be used as is, has to be subclassed to add specific read functions..
"""
-from abc import ABCMeta, abstractmethod, abstractproperty
+from abc import ABCMeta, abstractmethod
import datetime
import logging
+
import numpy as np
import os
import re
@@ -49,34 +50,35 @@ LOG = logging.getLogger(__name__)
# rpy values from
# here:http://yyy.rsmas.miami.edu/groups/rrsl/pathfinder/Processing/proc_app_a.html
rpy_coeffs = {
- 'noaa7': {'roll': 0.000,
- 'pitch': 0.000,
- 'yaw': 0.000,
- },
- 'noaa9': {'roll': 0.000,
- 'pitch': 0.0025,
- 'yaw': 0.000,
- },
- 'noaa10': {'roll': 0.000,
+ 'noaa7': {'roll': 0.000,
+ 'pitch': 0.000,
+ 'yaw': 0.000,
+ },
+ 'noaa9': {'roll': 0.000,
+ 'pitch': 0.0025,
+ 'yaw': 0.000,
+ },
+ 'noaa10': {'roll': 0.000,
'pitch': 0.000,
- 'yaw': 0.000,
+ 'yaw': 0.000,
},
'noaa11': {'roll': -0.0019,
'pitch': -0.0037,
- 'yaw': 0.000,
+ 'yaw': 0.000,
},
- 'noaa12': {'roll': 0.000,
+ 'noaa12': {'roll': 0.000,
'pitch': 0.000,
- 'yaw': 0.000,
+ 'yaw': 0.000,
},
- 'noaa14': {'roll': 0.000,
+ 'noaa14': {'roll': 0.000,
'pitch': 0.000,
- 'yaw': 0.000,
+ 'yaw': 0.000,
}}
class ReaderError(ValueError):
- """Raised in Reader.read if the given file does not correspond to it"""
+ """Raised in Reader.read if the given file does not correspond to it."""
+
pass
@@ -134,7 +136,7 @@ class Reader(six.with_metaclass(ABCMeta)):
@property
def times(self):
- """The UTCs as datetime.datetime"""
+ """Get the UTCs as datetime.datetime."""
return self.to_datetime(self.utcs)
@property
@@ -196,7 +198,7 @@ class Reader(six.with_metaclass(ABCMeta)):
@classmethod
def _correct_data_set_name(cls, header, filename):
- """Replace invalid data_set_name from header with filename
+ """Replace invalid data_set_name from header with filename.
Args:
header (struct): file header
@@ -221,7 +223,7 @@ class Reader(six.with_metaclass(ABCMeta)):
@classmethod
def _validate_header(cls, header):
- """Check if the header belongs to this reader
+ """Check if the header belongs to this reader.
Note:
according to https://www1.ncdc.noaa.gov/pub/data/satellite/
@@ -258,7 +260,7 @@ class Reader(six.with_metaclass(ABCMeta)):
% header['data_set_name'])
def _read_scanlines(self, buffer, count):
- """Read the scanlines from the given buffer
+ """Read the scanlines from the given buffer.
Args:
buffer (bytes, bytearray): buffer to read from
@@ -304,7 +306,7 @@ class Reader(six.with_metaclass(ABCMeta)):
@classmethod
def fromfile(cls, filename, fileobj=None):
- """Create Reader from file (alternative constructor)
+ """Create Reader from file, alternative constructor.
Args:
filename (str): Path to GAC/LAC file
@@ -323,14 +325,14 @@ class Reader(six.with_metaclass(ABCMeta)):
return instance
def _get_calibrated_channels_uniform_shape(self):
- """Prepare the channels as input for gac_io.save_gac"""
+ """Prepare the channels as input for gac_io.save_gac."""
channels = self.get_calibrated_channels()
assert channels.shape[-1] == 6
return channels
def save(self, start_line, end_line, output_file_prefix="PyGAC", output_dir="./",
avhrr_dir=None, qual_dir=None, sunsatangles_dir=None):
- """Convert the Reader instance content into hdf5 files"""
+ """Convert the Reader instance content into hdf5 files."""
avhrr_dir = avhrr_dir or output_dir
qual_dir = qual_dir or output_dir
sunsatangles_dir = sunsatangles_dir or output_dir
@@ -521,7 +523,10 @@ class Reader(six.with_metaclass(ABCMeta)):
corr
)
prt, ict, space = self.get_telemetry()
- for chan in [3, 4, 5]:
+
+ ir_channels_to_calibrate = self._get_ir_channels_to_calibrate()
+
+ for chan in ir_channels_to_calibrate:
channels[:, :, chan - 6] = calibrate_thermal(
channels[:, :, chan - 6],
prt,
@@ -590,12 +595,14 @@ class Reader(six.with_metaclass(ABCMeta)):
self._mask = self._get_corrupt_mask()
return self._mask
- @abstractproperty
+ @property
+ @abstractmethod
def QFlag(self): # pragma: no cover
"""KLM/POD specific quality indicators."""
raise NotImplementedError
- @abstractproperty
+ @property
+ @abstractmethod
def _quality_indicators_key(self): # pragma: no cover
raise NotImplementedError
@@ -775,16 +782,15 @@ class Reader(six.with_metaclass(ABCMeta)):
and different ranges.
Returns:
-
sat_azi: satellite azimuth angle degree clockwise from north in
range ]-180, 180]
- sat_zentih: satellite zenith angles in degrees in range [0,90]
+ sat_zenith: satellite zenith angles in degrees in range [0,90]
sun_azi: sun azimuth angle degree clockwise from north in range
]-180, 180]
- sun_zentih: sun zenith angles in degrees in range [0,90]
+ sun_zenith: sun zenith angles in degrees in range [0,90]
rel_azi: absolute azimuth angle difference in degrees between sun
and sensor in range [0, 180]
@@ -876,7 +882,7 @@ class Reader(six.with_metaclass(ABCMeta)):
"""Remove scanlines with corrupted scanline numbers.
This includes:
- - Scanline numbers outside the valide range
+ - Scanline numbers outside the valid range
- Scanline numbers deviating more than a certain threshold from the
ideal case (1,2,3,...N)
@@ -894,7 +900,7 @@ class Reader(six.with_metaclass(ABCMeta)):
'n_orig': self.scans['scan_line_number'].copy()}
# Remove scanlines whose scanline number is outside the valid range
- within_range = np.logical_and(self.scans["scan_line_number"] < 15000,
+ within_range = np.logical_and(self.scans["scan_line_number"] < self.max_scanlines,
self.scans["scan_line_number"] >= 0)
self.scans = self.scans[within_range]
@@ -927,7 +933,7 @@ class Reader(six.with_metaclass(ABCMeta)):
# Relatively small variation, keep (almost) everything
thresh = mean_nz_diffs + 3*std_nz_diffs
else:
- # Large variation, filter more agressively. Use median and
+ # Large variation, filter more aggressively. Use median and
# median absolute deviation (MAD) as they are less sensitive to
# outliers. However, allow differences < 500 scanlines as they
# occur quite often.
@@ -1049,7 +1055,8 @@ class Reader(six.with_metaclass(ABCMeta)):
results.update({'tn': tn, 'tcorr': self.utcs, 't0': t0})
return results
- @abstractproperty
+ @property
+ @abstractmethod
def tsm_affected_intervals(self): # pragma: no cover
"""Specify time intervals being affected by the scan motor problem.
@@ -1131,7 +1138,7 @@ class Reader(six.with_metaclass(ABCMeta)):
raise NotImplementedError
def get_attitude_coeffs(self):
- """Return the roll, pitch, yaw values"""
+ """Return the roll, pitch, yaw values."""
if self._rpy is None:
if "constant_yaw_attitude_error" in self.head.dtype.fields:
rpy = np.deg2rad([self.head["constant_roll_attitude_error"] / 1e3,
=====================================
pygac/tests/test_klm.py
=====================================
@@ -24,26 +24,23 @@
"""Test the GAC KLM reader."""
import datetime as dt
+from unittest import mock
+
import numpy as np
import numpy.testing
-import unittest
-import sys
-try:
- from unittest import mock
-except ImportError:
- import mock
-
-from pygac import gac_klm
+
+from pygac.gac_klm import GACKLMReader
+from pygac.klm_reader import header
+from pygac.lac_klm import LACKLMReader, scanline
from pygac.tests.utils import CalledWithArray
-class TestKLM(unittest.TestCase):
+class TestKLM:
+ """Test the klm reader."""
- def setUp(self):
- self.reader = gac_klm.GACKLMReader()
- # python 2 compatibility
- if sys.version_info.major < 3:
- self.assertRaisesRegex = self.assertRaisesRegexp
+ def setup(self):
+ """Set up the tests."""
+ self.reader = GACKLMReader()
def test_get_lonlat(self):
"""Test readout of lon/lat coordinates."""
@@ -68,14 +65,14 @@ class TestKLM(unittest.TestCase):
'start_of_data_set_utc_time_of_day': np.array([123456])
}
time = self.reader.get_header_timestamp()
- self.assertEqual(time, dt.datetime(2019, 5, 3, 0, 2, 3, 456000))
+ assert time == dt.datetime(2019, 5, 3, 0, 2, 3, 456000)
def test_get_times(self):
"""Test readout of scanline timestamps."""
self.reader.scans = {'scan_line_year': 1,
'scan_line_day_of_year': 2,
'scan_line_utc_time_of_day': 3}
- self.assertTupleEqual(self.reader._get_times(), (1, 2, 3))
+ assert self.reader._get_times() == (1, 2, 3)
def test_get_ch3_switch(self):
"""Test channel 3 identification."""
@@ -85,10 +82,10 @@ class TestKLM(unittest.TestCase):
numpy.testing.assert_array_equal(
self.reader.get_ch3_switch(), switch_exp)
- @mock.patch('pygac.gac_klm.GACKLMReader.get_ch3_switch')
- def test_postproc(self, get_ch3_switch):
+ def test_postproc(self):
"""Test KLM specific postprocessing."""
- get_ch3_switch.return_value = np.array([0, 1, 2])
+ self.reader.scans = {
+ 'scan_line_bit_field': np.array([0, 1, 2])}
channels = np.array([[[1., 2., 3., 4.],
[1., 2., 3., 4.]],
[[1., 2., 3., 4.],
@@ -105,22 +102,6 @@ class TestKLM(unittest.TestCase):
self.reader.postproc(channels) # masks in-place
numpy.testing.assert_array_equal(channels, masked_exp)
- @mock.patch('pygac.klm_reader.get_tsm_idx')
- def test_get_tsm_pixels(self, get_tsm_idx):
- """Test channel set used for TSM correction."""
- ones = np.ones((409, 100))
- zeros = np.zeros(ones.shape)
- ch1 = 1*ones
- ch2 = 2*ones
- ch4 = 4*ones
- ch5 = 5*ones
- channels = np.dstack((ch1, ch2, zeros, zeros, ch4, ch5))
- self.reader.get_tsm_pixels(channels)
- get_tsm_idx.assert_called_with(CalledWithArray(ch1),
- CalledWithArray(ch2),
- CalledWithArray(ch4),
- CalledWithArray(ch5))
-
def test_quality_indicators(self):
"""Test the quality indicator unpacking."""
reader = self.reader
@@ -137,6 +118,71 @@ class TestKLM(unittest.TestCase):
expected_mask = np.array([False, True, True, False, True], dtype=bool)
numpy.testing.assert_array_equal(reader.mask, expected_mask)
# test individual flags
- self.assertTrue(reader._get_corrupt_mask(flags=QFlag.FATAL_FLAG).any())
+ assert reader._get_corrupt_mask(flags=QFlag.FATAL_FLAG).any()
# count the occurence (everything flagged and last entrance => 2)
- self.assertEqual(reader._get_corrupt_mask(flags=QFlag.FATAL_FLAG).sum(), 2)
+ assert reader._get_corrupt_mask(flags=QFlag.FATAL_FLAG).sum() == 2
+
+
+class TestGACKLM:
+ """Tests for gac klm."""
+
+ def setup(self):
+ """Set up the tests."""
+ self.reader = GACKLMReader()
+
+ @mock.patch('pygac.klm_reader.get_tsm_idx')
+ def test_get_tsm_pixels(self, get_tsm_idx):
+ """Test channel set used for TSM correction."""
+ ones = np.ones((409, 100))
+ zeros = np.zeros(ones.shape)
+ ch1 = 1*ones
+ ch2 = 2*ones
+ ch4 = 4*ones
+ ch5 = 5*ones
+ channels = np.dstack((ch1, ch2, zeros, zeros, ch4, ch5))
+ self.reader.get_tsm_pixels(channels)
+ get_tsm_idx.assert_called_with(CalledWithArray(ch1),
+ CalledWithArray(ch2),
+ CalledWithArray(ch4),
+ CalledWithArray(ch5))
+
+
+class TestLACKLM:
+ """Tests for lac klm."""
+
+ def setup(self):
+ """Set up the tests."""
+ self.reader = LACKLMReader()
+ self.reader.scans = np.ones(100, dtype=scanline)
+ self.reader.head = np.ones(1, dtype=header)[0]
+ self.reader.spacecraft_id = 12
+ self.reader.head["noaa_spacecraft_identification_code"] = self.reader.spacecraft_id
+ self.reader.spacecraft_name = "metopa"
+ self.reader.scans["scan_line_number"] = np.arange(100)
+ # PRT
+ self.reader.scans["telemetry"]["PRT"] = 400
+ self.reader.scans["telemetry"]["PRT"][0::5, :] = 0
+
+ def test_get_ch3_switch(self):
+ """Test channel 3 identification."""
+ self.reader.scans = {
+ 'scan_line_bit_field': np.array([1, 2, 3, 4, 5, 6])}
+ switch_exp = np.array([1, 2, 3, 0, 1, 2])
+ numpy.testing.assert_array_equal(
+ self.reader.get_ch3_switch(), switch_exp)
+
+ def test_calibrate_channels(self):
+ """Test channel calibration."""
+ # ICT
+ self.reader.scans["back_scan"] = 400
+ self.reader.scans["back_scan"][0::5, :] = 0
+ # Space
+ self.reader.scans["space_data"] = 400
+ self.reader.scans["space_data"][0::5, :] = 0
+
+ assert np.any(np.isfinite(self.reader.get_calibrated_channels()))
+
+ def test_calibrate_inactive_3b(self):
+ """Test calibration of an inactive 3b."""
+ channels = self.reader.get_calibrated_channels()
+ assert np.all(np.isnan(channels[:, :, 3]))
=====================================
pygac/tests/test_reader.py
=====================================
@@ -21,9 +21,10 @@
"""Test the readers."""
import datetime
-import unittest
-import sys
import os
+import sys
+import unittest
+
try:
import mock
except ImportError:
@@ -31,20 +32,27 @@ except ImportError:
import numpy as np
import numpy.testing
from pygac.gac_reader import GACReader, ReaderError
+from pygac.lac_reader import LACReader
from pygac.pod_reader import POD_QualityIndicator
from pygac.gac_pod import scanline
from pygac.reader import NoTLEData
class TestPath(os.PathLike):
+ """Fake path class."""
+
def __init__(self, path):
+ """Initialize the path."""
self.path = str(path)
def __fspath__(self):
+ """Return the path."""
return self.path
class FakeGACReader(GACReader):
+ """Fake GAC reader class."""
+
QFlag = POD_QualityIndicator
_quality_indicators_key = "quality_indicators"
tsm_affected_intervals = {None: []}
@@ -52,7 +60,8 @@ class FakeGACReader(GACReader):
across_track = 4
def __init__(self):
- super(FakeGACReader, self).__init__()
+ """Initialize the fake reader."""
+ super().__init__()
self.scan_width = self.across_track
scans = np.zeros(self.along_track, dtype=scanline)
scans["scan_line_number"] = np.arange(self.along_track)
@@ -68,9 +77,11 @@ class FakeGACReader(GACReader):
return year, jday, msec
def get_header_timestamp(self):
+ """Get the header timestamp."""
return datetime.datetime(1970, 1, 1)
def get_telemetry(self):
+ """Get the telemetry."""
prt = 51 * np.ones(self.along_track) # prt threshold is 50
ict = 101 * np.ones((self.along_track, 3)) # ict threshold is 100
space = 101 * np.ones((self.along_track, 3)) # space threshold is 100
@@ -82,17 +93,25 @@ class FakeGACReader(GACReader):
def _get_lonlat(self):
pass
+ @staticmethod
+ def _get_ir_channels_to_calibrate():
+ return [3, 4, 5]
+
def postproc(self, channels):
+ """Postprocess the data."""
pass
def read(self, filename, fileobj=None):
+ """Read the data."""
pass
@classmethod
def read_header(cls, filename, fileobj=None):
+ """Read the header."""
pass
def get_tsm_pixels(self, channels):
+ """Get the tsm pixels."""
pass
@@ -192,6 +211,7 @@ class TestGacReader(unittest.TestCase):
self.reader._get_calibrated_channels_uniform_shape()
def test_get_calibrated_channels(self):
+ """Test getting calibrated channels."""
reader = FakeGACReader()
res = reader.get_calibrated_channels()
expected = np.full(
@@ -325,9 +345,9 @@ class TestGacReader(unittest.TestCase):
utcs2 = np.array([1, 2, 3]).astype('datetime64[ms]')
scanline2 = None
- for utcs, scanline in zip((utcs1, utcs2), (scanline1, scanline2)):
+ for utcs, scan_line in zip((utcs1, utcs2), (scanline1, scanline2)):
self.reader.utcs = utcs
- self.assertEqual(self.reader.get_midnight_scanline(), scanline,
+ self.assertEqual(self.reader.get_midnight_scanline(), scan_line,
msg='Incorrect midnight scanline')
def test_miss_lines(self):
@@ -405,19 +425,19 @@ class TestGacReader(unittest.TestCase):
# Test data correspond to columns 0:2, 201:208 and 407:409. Extracted like this:
# self.lons[0:5, [0, 1, 201, 202, 203, 204, 205, 206, 207, -2, -1]]
self.reader.lons = np.array([[69.41555135, 68.76815744, 28.04133742, 27.94671757, 27.85220562,
- 27.7578125, 27.66354783, 27.5694182, 27.47542957, 2.66416611,
+ 27.7578125, 27.66354783, 27.5694182, 27.47542957, 2.66416611,
2.39739436],
[69.41409536, 68.76979616, 28.00228658, 27.9076628, 27.8131467,
- 27.71875, 27.62448295, 27.53035209, 27.43636312, 2.61727408,
+ 27.71875, 27.62448295, 27.53035209, 27.43636312, 2.61727408,
2.35049275],
[69.42987929, 68.78543423, 27.96407251, 27.86936406, 27.77457923,
- 27.6796875, 27.58467527, 27.48959853, 27.39453053, 2.5704025,
+ 27.6796875, 27.58467527, 27.48959853, 27.39453053, 2.5704025,
2.30362323],
[69.44430772, 68.80064104, 27.91910034, 27.82340242, 27.72794715,
- 27.6328125, 27.53805662, 27.44366144, 27.34959008, 2.53088093,
+ 27.6328125, 27.53805662, 27.44366144, 27.34959008, 2.53088093,
2.26729486],
[69.47408815, 68.8259859, 27.87666513, 27.78224611, 27.68795045,
- 27.59375, 27.49962326, 27.40557682, 27.31162435, 2.48359319,
+ 27.59375, 27.49962326, 27.40557682, 27.31162435, 2.48359319,
2.21976689]])
self.reader.lats = np.array([[71.62830288, 71.67081539, 69.90976034, 69.89297223, 69.87616536,
69.859375, 69.84262997, 69.82593315, 69.80928089, 61.61334632,
@@ -513,31 +533,9 @@ class TestGacReader(unittest.TestCase):
self.reader.mask_tsm_pixels(channels) # masks in-place
numpy.testing.assert_array_equal(channels, masked_exp)
- def _get_scanline_numbers(self):
- """Create artificial scanline numbers with some corruptions.
-
- Returns:
- Corrupted and corrected scanline numbers.
-
- """
- along_track = 12000
- scans = np.zeros(12000, dtype=[("scan_line_number", ">u2")])
- scans["scan_line_number"] = np.arange(1, along_track+1)
-
- # ... with 500 missing scanlines at scanline 8000
- scans["scan_line_number"][8000:] += 500
- corrected = scans["scan_line_number"].copy()
-
- # ... and some spikes here and there
- scans["scan_line_number"][3000] += 1E4
- scans["scan_line_number"][9000] -= 1E4
- corrected = np.delete(corrected, [3000, 9000])
-
- return scans, corrected
-
def test_correct_scan_line_numbers(self):
"""Test scanline number correction."""
- scans, expected = self._get_scanline_numbers()
+ scans, expected = _get_scanline_numbers(14000)
self.reader.scans = scans
self.reader.correct_scan_line_numbers()
numpy.testing.assert_array_equal(self.reader.scans['scan_line_number'],
@@ -549,7 +547,7 @@ class TestGacReader(unittest.TestCase):
header_time = datetime.datetime(2016, 8, 16, 16, 7, 36)
# Create artificial timestamps
- _, scan_line_numbers = self._get_scanline_numbers()
+ _, scan_line_numbers = _get_scanline_numbers(14000)
t0 = np.array([header_time], dtype="datetime64[ms]").astype("i8")[0]
shift = 1000
msecs = t0 + shift + scan_line_numbers / GACReader.scan_freq
@@ -588,6 +586,7 @@ class TestGacReader(unittest.TestCase):
get_miss_lines,
get_midnight_scanline,
get_sun_earth_distance_correction):
+ """Test updating the metadata."""
get_miss_lines.return_value = 'miss_lines'
get_midnight_scanline.return_value = 'midn_line'
get_sun_earth_distance_correction.return_value = 'factor'
@@ -602,3 +601,50 @@ class TestGacReader(unittest.TestCase):
'gac_header': {'foo': 'bar'},
'calib_coeffs_version': 'version'}
self.assertDictEqual(self.reader.meta_data, mda_exp)
+
+
+def _get_scanline_numbers(scanlines_along_track):
+ """Create artificial scanline numbers with some corruptions.
+
+ Returns:
+ Corrupted and corrected scanline numbers.
+
+ """
+ scans = np.zeros(scanlines_along_track, dtype=[("scan_line_number", ">u2")])
+ scans["scan_line_number"] = np.arange(1, scanlines_along_track + 1)
+
+ # ... with 500 missing scanlines at scanline 8000
+ scans["scan_line_number"][8000:] += 500
+ corrected = scans["scan_line_number"].copy()
+
+ # ... and some spikes here and there
+ scans["scan_line_number"][3000] += 1E4
+ scans["scan_line_number"][9000] -= 1E4
+ corrected = np.delete(corrected, [3000, 9000])
+
+ return scans, corrected
+
+
+class TestLacReader(unittest.TestCase):
+ """Test the common LAC Reader."""
+
+ longMessage = True
+
+ @mock.patch.multiple('pygac.lac_reader.LACReader',
+ __abstractmethods__=set())
+ def setUp(self, *mocks):
+ """Set up the tests."""
+ self.reader = LACReader()
+
+ def test_lac_reader_accepts_FRAC(self):
+ """Test the header validation."""
+ head = {'data_set_name': b'NSS.FRAC.M1.D19115.S2352.E0050.B3425758.SV'}
+ self.reader._validate_header(head)
+
+ def test_correct_scan_line_numbers(self):
+ """Test scanline number correction."""
+ scans, expected = _get_scanline_numbers(22000)
+ self.reader.scans = scans
+ self.reader.correct_scan_line_numbers()
+ numpy.testing.assert_array_equal(self.reader.scans['scan_line_number'],
+ expected)
=====================================
setup.py
=====================================
@@ -22,7 +22,6 @@
"""The setup module."""
from setuptools import setup
-import sys
import os
if __name__ == '__main__':
@@ -40,10 +39,6 @@ if __name__ == '__main__':
'dev': ['pytest', 'pre-commit', 'flake8']
}
- if sys.version_info < (3, 7):
- # To parse ISO timestamps in calibration.py
- requirements.append('python-dateutil>=2.8.0')
-
setup(name='pygac',
description='NOAA AVHRR GAC/LAC reader and calibration',
author='The Pytroll Team',
@@ -70,6 +65,6 @@ if __name__ == '__main__':
scripts=[os.path.join('bin', item) for item in os.listdir('bin')],
data_files=[('etc', ['etc/pygac.cfg.template']),
('gapfilled_tles', ['gapfilled_tles/TLE_noaa16.txt'])],
- python_requires='>=3.6',
+ python_requires='>=3.8',
zip_safe=False
)
View it on GitLab: https://salsa.debian.org/debian-gis-team/pygac/-/compare/1ad2ff069d3b7211c460da3212e4b2dd5a0f2971...25c844f055e8d07abe5e2d96c9509f90379b1aab
--
View it on GitLab: https://salsa.debian.org/debian-gis-team/pygac/-/compare/1ad2ff069d3b7211c460da3212e4b2dd5a0f2971...25c844f055e8d07abe5e2d96c9509f90379b1aab
You're receiving this email because of your account on salsa.debian.org.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://alioth-lists.debian.net/pipermail/pkg-grass-devel/attachments/20221104/e5cb3e37/attachment-0001.htm>
More information about the Pkg-grass-devel
mailing list