Bug#1013114: silx: TestRetry fails with scipy 1.8.1-4

[Drew Parsons]

Source: silx
Version: 1.0.0+dfsg-4
Severity: normal

silx debci tests have started failing in testing on amd64 running
against scipy 1.8.1-4 from unstable.

Not clear what the problem is since the test previously passed against
scipy/1.8.1-3. Maybe it's a coincidence, triggered by something else
that migrated to testing at the same time that scipy 1.8.1-4 got
uploaded to unstable. Or maybe this test is intermittently flakey, I
don't know.


Test log reports:

______________________ TestRetry.test_retry_in_subprocess ______________________

options = {'exception': RetryError(''), 'retry_on_error': <function _default_retry_on_error at 0x7f92323de5f0>}

    @contextmanager
    def _handle_exception(options):
        try:
>           yield

/usr/lib/python3/dist-packages/silx/utils/retry.py:63: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = ('/tmp/tmppg678tas/failcounter.txt',)
kw = {'faildelay': 0.1, 'kwcheck': True, 'ncausefailure': 3}
_retry_timeout = 30.299999999999997, _retry_period = 0.01
_retry_on_error = <function _default_retry_on_error at 0x7f92323de5f0>
start_subprocess = <function retry_in_subprocess.<locals>.decorator.<locals>.wrapper.<locals>.start_subprocess at 0x7f92300cad40>
stop_subprocess = <function retry_in_subprocess.<locals>.decorator.<locals>.wrapper.<locals>.stop_subprocess at 0x7f9223d5c310>
p = <SpawnProcess name='SpawnProcess-16' pid=13378 parent=11603 stopped exitcode=-SIGKILL>
queue = <multiprocessing.queues.Queue object at 0x7f91fc1e7eb0>
options = {'exception': RetryError(''), 'retry_on_error': <function _default_retry_on_error at 0x7f92323de5f0>}
result = RetryError('')

    @wraps(method)
    def wrapper(*args, **kw):
        _retry_timeout = kw.pop("retry_timeout", retry_timeout)
        _retry_period = kw.pop("retry_period", retry_period)
        _retry_on_error = kw.pop("retry_on_error", retry_on_error)
    
        ctx = multiprocessing.get_context("spawn")
    
        def start_subprocess():
            queue = ctx.Queue(maxsize=1)
            p = ctx.Process(
                target=_subprocess_main,
                args=(queue, method, retry_on_error) + args,
                kwargs=kw,
            )
            p.start()
            return p, queue
    
        def stop_subprocess(p):
            try:
                p.kill()
            except AttributeError:
                p.terminate()
            p.join()
    
        p, queue = start_subprocess()
        try:
            for options in _retry_loop(
                retry_timeout=_retry_timeout, retry_on_error=_retry_on_error
            ):
                with _handle_exception(options):
                    if not p.is_alive():
                        p, queue = start_subprocess()
                    try:
                        result = queue.get(block=True, timeout=_retry_period)
                    except Empty:
                        pass
                    except ValueError:
                        pass
                    else:
                        if isinstance(result, BaseException):
                            stop_subprocess(p)
>                           raise result
E                           silx.utils.retry.RetryError

/usr/lib/python3/dist-packages/silx/utils/retry.py:256: RetryError

The above exception was the direct cause of the following exception:

self = <silx.utils.test.test_retry.TestRetry testMethod=test_retry_in_subprocess>

    def test_retry_in_subprocess(self):
        ncausefailure = 3
        faildelay = 0.1
        sufficient_timeout = ncausefailure * (faildelay + 10)
        insufficient_timeout = ncausefailure * faildelay * 0.5
    
        kw = {
            "ncausefailure": ncausefailure,
            "faildelay": faildelay,
            "kwcheck": True,
            "retry_timeout": sufficient_timeout,
        }
        with open(self.ctr_file, mode="w") as f:
            f.write("0")
>       self.assertTrue(_wsubmain(self.ctr_file, **kw))

/usr/lib/python3/dist-packages/silx/utils/test/test_retry.py:158: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/lib/python3/dist-packages/silx/utils/retry.py:241: in wrapper
    for options in _retry_loop(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

retry_timeout = 30.299999999999997, retry_period = None
retry_on_error = <function _default_retry_on_error at 0x7f92323de5f0>

    def _retry_loop(retry_timeout=None, retry_period=None, retry_on_error=None):
        """Iterator which is endless or ends with an RetryTimeoutError.
        It yields a dictionary which can be used to influence the loop.
    
        :param num retry_timeout:
        :param num retry_period: sleep before retry
        :param callable or None retry_on_error: checks whether an exception is
                                                eligible for retry
        """
        has_timeout = retry_timeout is not None
        options = {"exception": None, "retry_on_error": retry_on_error}
        if has_timeout:
            t0 = time.time()
        while True:
            yield options
            if retry_period is not None:
                time.sleep(retry_period)
            if has_timeout and (time.time() - t0) > retry_timeout:
>               raise RetryTimeoutError from options.get("exception")
E               silx.utils.retry.RetryTimeoutError

/usr/lib/python3/dist-packages/silx/utils/retry.py:90: RetryTimeoutError