src/antlion/test_decorators.py

#!/usr/bin/env python3
#
# Copyright 2022 The Fuchsia Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from antlion import signals


def __select_last(test_signals, _):
    return test_signals[-1]


def repeated_test(num_passes, acceptable_failures=0,
                  result_selector=__select_last):
    """A decorator that runs a test case multiple times.

    This decorator can be used to run a test multiple times and aggregate the
    data into a single test result. By setting `result_selector`, the user can
    access the returned result of each run, allowing them to average results,
    return the median, or gather and return standard deviation values.

    This decorator should be used on test cases, and should not be used on
    static or class methods. The test case must take in an additional argument,
    `attempt_number`, which returns the current attempt number, starting from
    1.

    Note that any TestSignal intended to abort or skip the test will take
    abort or skip immediately.

    Args:
        num_passes: The number of times the test needs to pass to report the
            test case as passing.
        acceptable_failures: The number of failures accepted. If the failures
            exceeds this number, the test will stop repeating. The maximum
            number of runs is `num_passes + acceptable_failures`. If the test
            does fail, result_selector will still be called.
        result_selector: A lambda that takes in the list of TestSignals and
            returns the test signal to report the test case as. Note that the
            list also contains any uncaught exceptions from the test execution.
    """
    def decorator(func):
        if not func.__name__.startswith('test_'):
            raise ValueError('Tests must start with "test_".')

        def test_wrapper(self):
            num_failures = 0
            num_seen_passes = 0
            test_signals_received = []
            for i in range(num_passes + acceptable_failures):
                try:
                    func(self, i + 1)
                except (signals.TestFailure, signals.TestError,
                        AssertionError) as signal:
                    test_signals_received.append(signal)
                    num_failures += 1
                except signals.TestPass as signal:
                    test_signals_received.append(signal)
                    num_seen_passes += 1
                except (signals.TestSignal, KeyboardInterrupt):
                    raise
                except Exception as signal:
                    test_signals_received.append(signal)
                    num_failures += 1
                else:
                    num_seen_passes += 1
                    test_signals_received.append(signals.TestPass(
                        'Test iteration %s of %s passed without details.' % (
                        i, func.__name__)))

                if num_failures > acceptable_failures:
                    break
                elif num_seen_passes == num_passes:
                    break
                else:
                    self.teardown_test()
                    self.setup_test()

            raise result_selector(test_signals_received, self)

        return test_wrapper

    return decorator