[Python-modules-commits] [python-flaky] 01/03: Import python-flaky_3.0.2.orig.tar.gz
Tristan Seligmann
mithrandi at moszumanska.debian.org
Tue Jan 19 23:38:35 UTC 2016
This is an automated email from the git hooks/post-receive script.
mithrandi pushed a commit to branch master
in repository python-flaky.
commit 0c2dd2ce9a80825c775fe19c2c40f59e5130b1c7
Author: Tristan Seligmann <mithrandi at mithrandi.net>
Date: Wed Jan 20 01:35:32 2016 +0200
Import python-flaky_3.0.2.orig.tar.gz
---
.pylintrc | 22 +-
HISTORY.rst | 25 ++
README.rst | 14 +
flaky.iml | 1 +
flaky/_flaky_plugin.py | 231 +++++++-----
flaky/defaults.py | 1 +
flaky/flaky_decorator.py | 1 +
flaky/flaky_nose_plugin.py | 12 +-
flaky/flaky_pytest_plugin.py | 412 ++++++++++-----------
setup.py | 2 +-
test/test_flaky_plugin.py | 5 +-
test/test_multiprocess_string_io.py | 3 +-
test/test_nose/__init__.py | 3 +
test/{ => test_nose}/test_flaky_nose_plugin.py | 0
test/{ => test_nose}/test_nose_example.py | 0
test/{ => test_nose}/test_nose_options_example.py | 0
test/test_pytest/__init__.py | 3 +
.../pytest_generate_example/__init__.py | 0
.../pytest_generate_example/conftest.py | 3 +
.../test_pytest_generate_example.py | 6 +-
test/{ => test_pytest}/test_flaky_pytest_plugin.py | 45 +--
test/{ => test_pytest}/test_pytest_example.py | 60 ++-
.../test_pytest_options_example.py | 14 +-
test/test_pytest/test_pytester_plugin.py | 20 +
test/test_utils.py | 4 +-
tox.ini | 22 +-
26 files changed, 520 insertions(+), 389 deletions(-)
diff --git a/.pylintrc b/.pylintrc
index dfb8e86..b35c7e6 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -7,9 +7,6 @@
# pygtk.require().
#init-hook=
-# Profiled execution.
-profile=no
-
# Add files or directories to the blacklist. They should be base names, not
# paths.
#ignore=CVS
@@ -67,10 +64,6 @@ reports=no
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
-# Add a comment according to your evaluation note. This is used by the global
-# evaluation report (RP0004).
-comment=no
-
# Template used to display messages. This is a python new-style format string
# used to format the massage information. See doc for all details
msg-template={module}:{line}:{column}: [{msg_id}({symbol}), {obj}] {msg}
@@ -78,9 +71,6 @@ msg-template={module}:{line}:{column}: [{msg_id}({symbol}), {obj}] {msg}
[BASIC]
-# Required attributes for module, separated by a comma
-required-attributes=
-
# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,apply,input
@@ -94,7 +84,7 @@ const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Regular expression which should only match correct function names
-function-rgx=[a-z_][a-z0-9_]{2,50}$
+function-rgx=[a-z_][a-z0-9_]{2,60}$
# Regular expression which should only match correct method names
method-rgx=[a-z_][a-z0-9_]{2,60}$
@@ -134,7 +124,7 @@ docstring-min-length=-1
[FORMAT]
# Maximum number of characters on a single line.
-max-line-length=80
+max-line-length=120
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
@@ -179,10 +169,6 @@ ignore-mixin-members=yes
#ignored-classes=SQLObject
ignored-classes=pytest, _pytest
-# When zope mode is activated, add a predefined set of Zope acquired attributes
-# to generated-members.
-zope=no
-
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E0201 when accessed. Python regular
# expressions are accepted.
@@ -205,10 +191,6 @@ additional-builtins=
[CLASSES]
-# List of interface methods to ignore, separated by a comma. This is used for
-# instance to not check methods defines in Zope's Interface base class.
-ignore-iface-methods=
-
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
diff --git a/HISTORY.rst b/HISTORY.rst
index 299867e..9dd074f 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -6,10 +6,35 @@ Release History
Upcoming
++++++++
+3.0.2 (2015-12-21)
+++++++++++++++++++
+
+**Bugfixes**
+
+- Flaky for pytest no longer passes None for the first 2 arguments to the optional ``rerun_filter``.
+
+
+3.0.1 (2015-12-16)
+++++++++++++++++++
+
+**Bugfixes**
+
+- Flaky for pytest no longer causes errors with the pytester plugin.
+
+3.0.0 (2015-12-14)
+++++++++++++++++++
+
+- Flaky for pytest now reruns test setup and teardown. **This is a possibly breaking change.**
+
+**Bugfixes**
+
+- Bug with nose and multiprocess fixed.
+
2.4.0 (2015-10-27)
++++++++++++++++++
**Bugfixes**
+
- The flaky report is now available under nose with the multiprocessing plugin.
2.3.0 (2015-10-15)
diff --git a/README.rst b/README.rst
index f209071..2635b04 100644
--- a/README.rst
+++ b/README.rst
@@ -100,6 +100,20 @@ after those failures, you can specify a filter function that can tell flaky to f
Flaky will run `test_something` twice, but will only run `test_something_else` once.
+It can also be used to incur a delay between test retries:
+
+.. code-block:: python
+
+ import time
+
+ def delay_rerun(*args):
+ time.sleep(1)
+ return True
+
+ @flaky(rerun_filter=delay_rerun)
+ def test_something_else():
+ ...
+
Activating the plugin
~~~~~~~~~~~~~~~~~~~~~
diff --git a/flaky.iml b/flaky.iml
index 9e14a9a..7778712 100644
--- a/flaky.iml
+++ b/flaky.iml
@@ -5,6 +5,7 @@
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/test" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/flaky" isTestSource="false" />
+ <excludeFolder url="file://$MODULE_DIR$/.cache" />
<excludeFolder url="file://$MODULE_DIR$/.idea" />
<excludeFolder url="file://$MODULE_DIR$/.tox" />
<excludeFolder url="file://$MODULE_DIR$/build" />
diff --git a/flaky/_flaky_plugin.py b/flaky/_flaky_plugin.py
index 69045df..cc329a1 100644
--- a/flaky/_flaky_plugin.py
+++ b/flaky/_flaky_plugin.py
@@ -25,7 +25,9 @@ class _FlakyPlugin(object):
will be written to the flaky report.
:return:
+ The stream used for building the flaky report.
:rtype:
+ :class:`StringIO`
"""
return self._stream
@@ -100,15 +102,65 @@ class _FlakyPlugin(object):
)
self._log_test_failure(name, err, message)
+ def _should_handle_test_error_or_failure(self, test):
+ """
+ Whether or not flaky should handle a test error or failure.
+ Only handle tests marked @flaky.
+ Count remaining retries and compare with number of required successes that have not yet been achieved.
+
+ This method may be called multiple times for the same test run, so it has no side effects.
+
+ :param test:
+ The test that has raised an error
+ :type test:
+ :class:`nose.case.Test` or :class:`Function`
+ :return:
+ True, if the test needs to be rerun; False, otherwise.
+ :rtype:
+ `bool`
+ """
+ if not self._has_flaky_attributes(test):
+ return False
+ flaky_attributes = self._get_flaky_attributes(test)
+ flaky_attributes[FlakyNames.CURRENT_RUNS] += 1
+ has_failed = self._has_flaky_test_failed(flaky_attributes)
+ return not has_failed
+
+ def _will_handle_test_error_or_failure(self, test, name, err):
+ """
+ Whether or not flaky will handle a test error or failure.
+ Returns True if the plugin should handle the test result, and
+ the `rerun_filter` returns True.
+
+ :param test:
+ The test that has raised an error
+ :type test:
+ :class:`nose.case.Test` or :class:`Function`
+ :param name:
+ The name of the test that has raised an error
+ :type name:
+ `unicode`
+ :param err:
+ Information about the test failure (from sys.exc_info())
+ :type err:
+ `tuple` of `type`, :class:`Exception`, `traceback`
+ :return:
+ True, if the test will be rerun by flaky; False, otherwise.
+ :rtype:
+ `bool`
+ """
+ return self._should_handle_test_error_or_failure(test) and self._should_rerun_test(test, name, err)
+
def _handle_test_error_or_failure(self, test, err):
"""
Handle a flaky test error or failure.
- Count remaining retries and compare with number of required successes
- that have not yet been achieved; retry if necessary.
Returning True from this method keeps the test runner from reporting
the test as a failure; this way we can retry and only report as a
failure if we are out of retries.
+
+ This method may only be called once per test run; it changes persisted flaky attributes.
+
:param test:
The test that has raised an error
:type test:
@@ -116,7 +168,7 @@ class _FlakyPlugin(object):
:param err:
Information about the test failure (from sys.exc_info())
:type err:
- `tuple` of `class`, :class:`Exception`, `traceback`
+ `tuple` of `type`, :class:`Exception`, `traceback`
:return:
True, if the test will be rerun;
False, if the test runner should handle it.
@@ -127,33 +179,24 @@ class _FlakyPlugin(object):
_, _, name = self._get_test_declaration_callable_and_name(test)
except AttributeError:
return False
- current_runs = self._get_flaky_attribute(
- test,
- FlakyNames.CURRENT_RUNS,
- )
- if current_runs is None:
- return False
- current_runs += 1
- self._set_flaky_attribute(
- test,
- FlakyNames.CURRENT_RUNS,
- current_runs,
- )
- self._add_flaky_test_failure(test, err)
- flaky = self._get_flaky_attributes(test)
- if not self._has_flaky_test_failed(flaky):
- if self._should_rerun_test(test, name, err):
- self._log_intermediate_failure(err, flaky, name)
- self._rerun_test(test)
- return True
+ if self._has_flaky_attributes(test):
+ self._add_flaky_test_failure(test, err)
+ should_handle = self._should_handle_test_error_or_failure(test)
+ self._increment_flaky_attribute(test, FlakyNames.CURRENT_RUNS)
+ if should_handle:
+ flaky_attributes = self._get_flaky_attributes(test)
+ if self._should_rerun_test(test, name, err):
+ self._log_intermediate_failure(err, flaky_attributes, name)
+ self._mark_test_for_rerun(test)
+ return True
+ else:
+ self._log_test_failure(name, err, self._not_rerun_message)
+ return False
else:
- message = self._not_rerun_message
- self._log_test_failure(name, err, message)
- return False
- else:
- self._report_final_failure(err, flaky, name)
- return False
+ flaky_attributes = self._get_flaky_attributes(test)
+ self._report_final_failure(err, flaky_attributes, name)
+ return False
def _should_rerun_test(self, test, name, err):
"""
@@ -184,16 +227,25 @@ class _FlakyPlugin(object):
rerun_filter = self._get_flaky_attribute(test, FlakyNames.RERUN_FILTER)
return rerun_filter(err, name, test, self)
- def _rerun_test(self, test):
+ def _mark_test_for_rerun(self, test):
"""
- Rerun a flaky test.
+ Mark a flaky test for rerun.
+
:param test:
The test that has raised an error or succeeded
:type test:
- :class:`Function`
+ :class:`nose.case.Test` or :class:`Function`
"""
raise NotImplementedError # pragma: no cover
+ def _should_handle_test_success(self, test):
+ if not self._has_flaky_attributes(test):
+ return False
+ flaky = self._get_flaky_attributes(test)
+ flaky[FlakyNames.CURRENT_PASSES] += 1
+ flaky[FlakyNames.CURRENT_RUNS] += 1
+ return not self._has_flaky_test_succeeded(flaky)
+
def _handle_test_success(self, test):
"""
Handle a flaky test success.
@@ -203,65 +255,55 @@ class _FlakyPlugin(object):
Returning True from this method keeps the test runner from reporting
the test as a success; this way we can retry and only report as a
success if the test has passed the required number of times.
+
:param test:
The test that has raised an error
:type test:
- :class:`nose.case.Test`
+ :class:`nose.case.Test` or :class:`Function`
:return:
- True, if the test will be rerun; False, if nose should handle it.
+ True, if the test will be rerun; False, if the test runner should handle it.
:rtype:
`bool`
"""
- _, _, name = self._get_test_declaration_callable_and_name(test)
- current_runs = self._get_flaky_attribute(
- test,
- FlakyNames.CURRENT_RUNS
- )
- if current_runs is None:
+ try:
+ _, _, name = self._get_test_declaration_callable_and_name(test)
+ except AttributeError:
return False
- current_runs += 1
- current_passes = self._get_flaky_attribute(
- test,
- FlakyNames.CURRENT_PASSES
- )
- current_passes += 1
- self._set_flaky_attribute(
- test,
- FlakyNames.CURRENT_RUNS,
- current_runs
- )
- self._set_flaky_attribute(
- test,
- FlakyNames.CURRENT_PASSES,
- current_passes
- )
- flaky = self._get_flaky_attributes(test)
- need_reruns = not self._has_flaky_test_succeeded(flaky)
- if self._flaky_success_report:
+ need_reruns = self._should_handle_test_success(test)
+
+ if self._has_flaky_attributes(test):
+ flaky = self._get_flaky_attributes(test)
min_passes = flaky[FlakyNames.MIN_PASSES]
- self._stream.writelines([
- ensure_unicode_string(name),
- ' passed {0} out of the required {1} times. '.format(
- current_passes,
- min_passes,
- ),
- ])
- if need_reruns:
- self._stream.write(
- 'Running test again until it passes {0} times.\n'.format(
+ passes = flaky[FlakyNames.CURRENT_PASSES] + 1
+ self._set_flaky_attribute(test, FlakyNames.CURRENT_PASSES, passes)
+ self._increment_flaky_attribute(test, FlakyNames.CURRENT_RUNS)
+
+ if self._flaky_success_report:
+ self._stream.writelines([
+ ensure_unicode_string(name),
+ ' passed {0} out of the required {1} times. '.format(
+ passes,
min_passes,
+ ),
+ ])
+ if need_reruns:
+ self._stream.write(
+ 'Running test again until it passes {0} times.\n'.format(
+ min_passes,
+ )
)
- )
- else:
- self._stream.write('Success!\n')
+ else:
+ self._stream.write('Success!\n')
+
if need_reruns:
- self._rerun_test(test)
+ self._mark_test_for_rerun(test)
return need_reruns
@staticmethod
def add_report_option(add_option):
"""
Add an option to the test runner to suppress the flaky report.
+
:param add_option:
A function that can add an option to the test runner.
Its argspec should equal that of argparse.add_option.
@@ -290,6 +332,7 @@ class _FlakyPlugin(object):
def add_force_flaky_options(add_option):
"""
Add options to the test runner that force all tests to be flaky.
+
:param add_option:
A function that can add an option to the test runner.
Its argspec should equal that of argparse.add_option.
@@ -328,6 +371,7 @@ class _FlakyPlugin(object):
def _add_flaky_report(self, stream):
"""
Baseclass override. Write details about flaky tests to the test report.
+
:param stream:
The test stream to which the report can be written.
:type stream:
@@ -351,6 +395,7 @@ class _FlakyPlugin(object):
def _copy_flaky_attributes(cls, test, test_class):
"""
Copy flaky attributes from the test callable or class to the test.
+
:param test:
The test that is being prepared to run
:type test:
@@ -371,6 +416,7 @@ class _FlakyPlugin(object):
def _get_flaky_attribute(test_item, flaky_attribute):
"""
Gets an attribute describing the flaky test.
+
:param test_item:
The test method from which to get the attribute
:type test_item:
@@ -385,17 +431,14 @@ class _FlakyPlugin(object):
:rtype:
varies
"""
- return getattr(
- test_item,
- flaky_attribute,
- None,
- )
+ return getattr(test_item, flaky_attribute, None)
@staticmethod
def _set_flaky_attribute(test_item, flaky_attribute, value):
"""
Sets an attribute on a flaky test. Uses magic __dict__ since setattr
doesn't work for bound methods.
+
:param test_item:
The test callable on which to set the attribute
:type test_item:
@@ -412,9 +455,26 @@ class _FlakyPlugin(object):
test_item.__dict__[flaky_attribute] = value
@classmethod
+ def _increment_flaky_attribute(cls, test_item, flaky_attribute):
+ """
+ Increments the value of an attribute on a flaky test.
+
+ :param test_item:
+ The test callable on which to set the attribute
+ :type test_item:
+ `callable` or :class:`nose.case.Test` or :class:`Function`
+ :param flaky_attribute:
+ The name of the attribute to set
+ :type flaky_attribute:
+ `unicode`
+ """
+ cls._set_flaky_attribute(test_item, flaky_attribute, cls._get_flaky_attribute(test_item, flaky_attribute) + 1)
+
+ @classmethod
def _has_flaky_attributes(cls, test):
"""
Returns True if the test callable in question is marked as flaky.
+
:param test:
The test that is being prepared to run
:type test:
@@ -423,16 +483,14 @@ class _FlakyPlugin(object):
:rtype:
`bool`
"""
- current_runs = cls._get_flaky_attribute(
- test,
- FlakyNames.CURRENT_RUNS,
- )
+ current_runs = cls._get_flaky_attribute(test, FlakyNames.CURRENT_RUNS)
return current_runs is not None
@classmethod
def _get_flaky_attributes(cls, test_item):
"""
Get all the flaky related attributes from the test.
+
:param test_item:
The test callable from which to get the flaky related attributes.
:type test_item:
@@ -452,6 +510,7 @@ class _FlakyPlugin(object):
def _add_flaky_test_failure(cls, test, err):
"""
Store test error information on the test callable.
+
:param test:
The flaky test on which to update the flaky attributes.
:type test:
@@ -462,17 +521,14 @@ class _FlakyPlugin(object):
`tuple` of `class`, :class:`Exception`, `traceback`
"""
errs = getattr(test, FlakyNames.CURRENT_ERRORS, None) or []
- cls._set_flaky_attribute(
- test,
- FlakyNames.CURRENT_ERRORS,
- errs,
- )
+ cls._set_flaky_attribute(test, FlakyNames.CURRENT_ERRORS, errs)
errs.append(err)
@classmethod
def _has_flaky_test_failed(cls, flaky):
"""
Whether or not the flaky test has failed
+
:param flaky:
Dictionary of flaky attributes
:type flaky:
@@ -498,6 +554,7 @@ class _FlakyPlugin(object):
def _has_flaky_test_succeeded(flaky):
"""
Whether or not the flaky test has succeeded
+
:param flaky:
Dictionary of flaky attributes
:type flaky:
@@ -515,6 +572,7 @@ class _FlakyPlugin(object):
"""
Get the test declaration, the test callable,
and test callable name from the test.
+
:param test:
The test that has raised an error or succeeded
:type test:
@@ -530,6 +588,7 @@ class _FlakyPlugin(object):
def _make_test_flaky(cls, test, max_runs, min_passes):
"""
Make a given test flaky.
+
:param test:
The test in question.
:type test:
diff --git a/flaky/defaults.py b/flaky/defaults.py
index 8991b9b..98782ce 100644
--- a/flaky/defaults.py
+++ b/flaky/defaults.py
@@ -27,6 +27,7 @@ class FilterWrapper(object):
def default_flaky_attributes(max_runs, min_passes, rerun_filter=None):
"""
Returns the default flaky attributes to set on a flaky test.
+
:param max_runs:
The value of the FlakyNames.MAX_RUNS attribute to use.
:type max_runs:
diff --git a/flaky/flaky_decorator.py b/flaky/flaky_decorator.py
index 1c018ba..41c247e 100644
--- a/flaky/flaky_decorator.py
+++ b/flaky/flaky_decorator.py
@@ -10,6 +10,7 @@ def flaky(max_runs=None, min_passes=None, rerun_filter=None):
Decorator used to mark a test as "flaky". When used in conjuction with
the flaky nosetests plugin, will cause the decorated test to be retried
until min_passes successes are achieved out of up to max_runs test runs.
+
:param max_runs:
The maximum number of times the decorated test will be run.
:type max_runs:
diff --git a/flaky/flaky_nose_plugin.py b/flaky/flaky_nose_plugin.py
index 79b1190..8bb1f73 100644
--- a/flaky/flaky_nose_plugin.py
+++ b/flaky/flaky_nose_plugin.py
@@ -1,12 +1,14 @@
# coding: utf-8
from __future__ import unicode_literals
+
import logging
from optparse import OptionGroup
+import os
+
from nose.failure import Failure
from nose.plugins import Plugin
from nose.result import TextTestResult
-import os
from flaky._flaky_plugin import _FlakyPlugin
@@ -69,7 +71,7 @@ class FlakyPlugin(_FlakyPlugin, Plugin):
super(FlakyPlugin, self).configure(options, conf)
if not self.enabled:
return
- is_multiprocess = getattr(options, 'multiprocess_workers', 0) > 0
+ is_multiprocess = int(getattr(options, 'multiprocess_workers', 0)) > 0
self._stream = self._get_stream(is_multiprocess)
self._flaky_result = TextTestResult(self._stream, [], 0)
self._flaky_report = options.flaky_report
@@ -110,7 +112,7 @@ class FlakyPlugin(_FlakyPlugin, Plugin):
test.run(self._flaky_result)
self._test_status.pop(test, None)
- def _rerun_test(self, test):
+ def _mark_test_for_rerun(self, test):
"""
Base class override. Rerun a flaky test.
@@ -186,6 +188,7 @@ class FlakyPlugin(_FlakyPlugin, Plugin):
Returning True from this method keeps the test runner from reporting
the test as a success; this way we can retry and only report as a
success if we have achieved the required number of successes.
+
:param test:
The test that has succeeded
:type test:
@@ -201,6 +204,7 @@ class FlakyPlugin(_FlakyPlugin, Plugin):
def report(self, stream):
"""
Baseclass override. Write details about flaky tests to the test report.
+
:param stream:
The test stream to which the report can be written.
:type stream:
@@ -230,6 +234,7 @@ class FlakyPlugin(_FlakyPlugin, Plugin):
If the test class is marked flaky and the test callable is not, copy
the flaky attributes from the test class to the test callable.
+
:param test:
The test that is being prepared to run
:type test:
@@ -247,6 +252,7 @@ class FlakyPlugin(_FlakyPlugin, Plugin):
def _get_test_callable_name(test):
"""
Get the name of the test callable from the test.
+
:param test:
The test that has raised an error or succeeded
:type test:
diff --git a/flaky/flaky_pytest_plugin.py b/flaky/flaky_pytest_plugin.py
index af0d802..322b752 100644
--- a/flaky/flaky_pytest_plugin.py
+++ b/flaky/flaky_pytest_plugin.py
@@ -1,81 +1,29 @@
# coding: utf-8
from __future__ import unicode_literals
-import py
+import pytest
# pylint:disable=import-error
-from _pytest.runner import CallInfo, Skipped
+from _pytest.runner import CallInfo
# pylint:enable=import-error
from flaky._flaky_plugin import _FlakyPlugin
-def pytest_runtest_protocol(item, nextitem):
- """
- Pytest hook to override how tests are run.
- """
- PLUGIN.run_test(item, nextitem)
- return True
-
-
-def pytest_terminal_summary(terminalreporter):
- """
- Pytest hook to write details about flaky tests to the test report.
- :param terminalreporter:
- Terminal reporter object. Supports stream writing operations.
- :type terminalreporter:
- :class: `TerminalReporter`
- """
- PLUGIN.terminal_summary(terminalreporter)
-
-
-def pytest_addoption(parser):
- """
- Pytest hook to add an option to the argument parser.
- :param parser:
- Parser for command line arguments and ini-file values.
- :type parser:
- :class:`Parser`
- """
- PLUGIN.add_report_option(parser.addoption)
-
- group = parser.getgroup(
- "Force flaky", "Force all tests to be flaky.")
- PLUGIN.add_force_flaky_options(group.addoption)
-
-
class FlakyXdist(object):
+ def __init__(self, plugin):
+ super(FlakyXdist, self).__init__()
+ self._plugin = plugin
+
def pytest_testnodedown(self, node, error):
+ """
+ Pytest hook for responding to a test node shutting down.
+ Copy slave flaky report output so it's available on the master flaky report.
+ """
# pylint: disable=unused-argument, no-self-use
if hasattr(node, 'slaveoutput') and 'flaky_report' in node.slaveoutput:
- PLUGIN.stream.write(node.slaveoutput['flaky_report'])
-
-
-def pytest_configure(config):
- """
- Pytest hook to get information about how the test run has been configured.
- :param config:
- The pytest configuration object for this test run.
- :type config:
- :class:`Configuration`
- """
- PLUGIN.flaky_report = config.option.flaky_report
- PLUGIN.flaky_success_report = config.option.flaky_success_report
- PLUGIN.force_flaky = config.option.force_flaky
- PLUGIN.max_runs = config.option.max_runs
- PLUGIN.min_passes = config.option.min_passes
- PLUGIN.runner = config.pluginmanager.getplugin("runner")
- if config.pluginmanager.hasplugin('xdist'):
- config.pluginmanager.register(FlakyXdist())
- PLUGIN.config = config
- if hasattr(config, 'slaveoutput'):
- config.slaveoutput['flaky_report'] = ''
-
-
-def pytest_sessionfinish():
- if hasattr(PLUGIN.config, 'slaveoutput'):
- PLUGIN.config.slaveoutput['flaky_report'] += PLUGIN.stream.getvalue()
+ self._plugin.stream.write(node.slaveoutput['flaky_report'])
class FlakyPlugin(_FlakyPlugin):
@@ -84,12 +32,188 @@ class FlakyPlugin(_FlakyPlugin):
"""
runner = None
- _info = None
flaky_report = True
force_flaky = False
max_runs = None
min_passes = None
config = None
+ _call_infos = {}
+ _PYTEST_WHEN_CALL = 'call'
+ _PYTEST_OUTCOME_PASSED = 'passed'
+ _PYTEST_OUTCOME_FAILED = 'failed'
+ _PYTEST_EMPTY_STATUS = ('', '', '')
+
+ def pytest_runtest_protocol(self, item, nextitem):
+ """
+ Pytest hook to override how tests are run.
+
+ Runs a test collected by py.test.
+ - First, monkey patches the builtin runner module to call back to
+ FlakyPlugin.call_runtest_hook rather than its own.
+ - Then defers to the builtin runner module to run the test,
+ and repeats the process if the test needs to be rerun.
+ - Reports test results to the flaky report.
+
+ :param item:
+ py.test wrapper for the test function to be run
+ :type item:
+ :class:`Function`
+ :param nextitem:
+ py.test wrapper for the next test function to be run
+ :type nextitem:
+ :class:`Function`
+ :return:
+ True if no further hook implementations should be invoked.
+ :rtype:
+ `bool`
+ """
+ test_instance = self._get_test_instance(item)
+ self._copy_flaky_attributes(item, test_instance)
+ if self.force_flaky and not self._has_flaky_attributes(item):
+ self._make_test_flaky(
+ item,
+ self.max_runs,
+ self.min_passes,
+ )
+ original_call_runtest_hook = self.runner.call_runtest_hook
+ self._call_infos[item] = {}
+ should_rerun = True
+ try:
+ self.runner.call_runtest_hook = self.call_runtest_hook
+ while should_rerun:
+ self.runner.pytest_runtest_protocol(item, nextitem)
+ call_info = self._call_infos.get(item, {}).get(self._PYTEST_WHEN_CALL, None)
+ if call_info is None:
+ return False
+ passed = call_info.excinfo is None
+ if passed:
+ should_rerun = self.add_success(item)
+ else:
+ should_rerun = self.add_failure(item, call_info.excinfo)
+ if not should_rerun:
+ item.excinfo = call_info.excinfo
+ finally:
+ self.runner.call_runtest_hook = original_call_runtest_hook
+ del self._call_infos[item]
+ return True
+
+ def _get_test_name_and_err(self, item):
+ """
+ Get the test name and error tuple from a test item.
+
+ :param item:
+ py.test wrapper for the test function to be run
+ :type item:
+ :class:`Function`
+ :return:
+ The test name and error tuple.
+ :rtype:
+ ((`type`, :class:`Exception`, :class:`Traceback`) or (None, None, None), `unicode`)
+ """
+ _, _, name = self._get_test_declaration_callable_and_name(item)
+ call_info = self._call_infos.get(item, {}).get(self._PYTEST_WHEN_CALL, None)
+ if call_info is not None:
+ err = (call_info.excinfo.type, call_info.excinfo.value, call_info.excinfo.tb)
+ else:
+ err = (None, None, None)
+ return err, name
+
+ @pytest.hookimpl(tryfirst=True, hookwrapper=True)
+ def pytest_runtest_makereport(self, item, call):
+ """
+ Pytest hook to intercept the report for reruns.
+
+ Change the report's outcome to 'passed' if flaky is going to handle the test run.
+ That way, pytest will not mark the run as failed.
+ """
+ outcome = yield
+ if call.when == self._PYTEST_WHEN_CALL:
+ report = outcome.get_result()
+ report.item = item
+ report.original_outcome = report.outcome
+ if report.failed:
+ err, name = self._get_test_name_and_err(item)
+ if self._will_handle_test_error_or_failure(item, name, err):
+ report.outcome = self._PYTEST_OUTCOME_PASSED
+
+ @pytest.hookimpl(tryfirst=True, hookwrapper=True)
+ def pytest_report_teststatus(self, report):
+ """
+ Pytest hook to only add final runs to the report.
+
+ Given a test report, get the correpsonding test status.
+ For tests that flaky is handling, return the empty status
+ so it isn't reported; otherwise, don't change the status.
+ """
+ outcome = yield
+ if report.when == self._PYTEST_WHEN_CALL:
+ item = report.item
+ if report.original_outcome == self._PYTEST_OUTCOME_PASSED:
+ if self._should_handle_test_success(item):
+ outcome.force_result(self._PYTEST_EMPTY_STATUS)
+ elif report.original_outcome == self._PYTEST_OUTCOME_FAILED:
+ err, name = self._get_test_name_and_err(item)
+ if self._will_handle_test_error_or_failure(item, name, err):
+ outcome.force_result(self._PYTEST_EMPTY_STATUS)
+ delattr(report, 'item')
+
+ def pytest_terminal_summary(self, terminalreporter):
+ """
+ Pytest hook to write details about flaky tests to the test report.
+
+ Write details about flaky tests to the test report.
+
+ :param terminalreporter:
+ Terminal reporter object. Supports stream writing operations.
+ :type terminalreporter:
+ :class: `TerminalReporter`
+ """
+ if self.flaky_report:
+ self._add_flaky_report(terminalreporter)
+
+ def pytest_addoption(self, parser):
+ """
+ Pytest hook to add an option to the argument parser.
+
+ :param parser:
+ Parser for command line arguments and ini-file values.
+ :type parser:
+ :class:`Parser`
+ """
+ self.add_report_option(parser.addoption)
+
+ group = parser.getgroup(
+ "Force flaky", "Force all tests to be flaky.")
+ self.add_force_flaky_options(group.addoption)
+
+ def pytest_configure(self, config):
+ """
+ Pytest hook to get information about how the test run has been configured.
+
+ :param config:
+ The pytest configuration object for this test run.
+ :type config:
+ :class:`Configuration`
+ """
+ self.flaky_report = config.option.flaky_report
+ self.flaky_success_report = config.option.flaky_success_report
+ self.force_flaky = config.option.force_flaky
+ self.max_runs = config.option.max_runs
+ self.min_passes = config.option.min_passes
+ self.runner = config.pluginmanager.getplugin("runner")
+ if config.pluginmanager.hasplugin('xdist'):
+ config.pluginmanager.register(FlakyXdist(self))
+ self.config = config
+ if hasattr(config, 'slaveoutput'):
+ config.slaveoutput['flaky_report'] = ''
+
+ def pytest_sessionfinish(self):
+ """
+ Pytest hook to take a final action after the session is complete.
+ Copy flaky report contents so that the master process can read it.
+ """
+ if hasattr(self.config, 'slaveoutput'):
+ self.config.slaveoutput['flaky_report'] += self.stream.getvalue()
@property
def stream(self):
@@ -133,40 +257,12 @@ class FlakyPlugin(_FlakyPlugin):
test_instance = item.parent.obj
return test_instance
- def run_test(self, item, nextitem):
- """
- Runs a test collected by py.test. First, monkey patches the builtin
- runner module to call back to FlakyPlugin.call_runtest_hook rather
- than its own. Then defer to the builtin runner module to run the test.
- :param item:
- py.test wrapper for the test function to be run
- :type item:
- :class:`Function`
- :param nextitem:
- py.test wrapper for the next test function to be run
- :type nextitem:
... 735 lines suppressed ...
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/python-flaky.git
More information about the Python-modules-commits
mailing list