| #!/usr/bin/env python3 |
| # |
| # Copyright 2022 The Fuchsia Authors |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| from antlion import signals |
| |
| |
| def __select_last(test_signals, _): |
| return test_signals[-1] |
| |
| |
| def repeated_test(num_passes, acceptable_failures=0, |
| result_selector=__select_last): |
| """A decorator that runs a test case multiple times. |
| |
| This decorator can be used to run a test multiple times and aggregate the |
| data into a single test result. By setting `result_selector`, the user can |
| access the returned result of each run, allowing them to average results, |
| return the median, or gather and return standard deviation values. |
| |
| This decorator should be used on test cases, and should not be used on |
| static or class methods. The test case must take in an additional argument, |
| `attempt_number`, which returns the current attempt number, starting from |
| 1. |
| |
| Note that any TestSignal intended to abort or skip the test will take |
| abort or skip immediately. |
| |
| Args: |
| num_passes: The number of times the test needs to pass to report the |
| test case as passing. |
| acceptable_failures: The number of failures accepted. If the failures |
| exceeds this number, the test will stop repeating. The maximum |
| number of runs is `num_passes + acceptable_failures`. If the test |
| does fail, result_selector will still be called. |
| result_selector: A lambda that takes in the list of TestSignals and |
| returns the test signal to report the test case as. Note that the |
| list also contains any uncaught exceptions from the test execution. |
| """ |
| def decorator(func): |
| if not func.__name__.startswith('test_'): |
| raise ValueError('Tests must start with "test_".') |
| |
| def test_wrapper(self): |
| num_failures = 0 |
| num_seen_passes = 0 |
| test_signals_received = [] |
| for i in range(num_passes + acceptable_failures): |
| try: |
| func(self, i + 1) |
| except (signals.TestFailure, signals.TestError, |
| AssertionError) as signal: |
| test_signals_received.append(signal) |
| num_failures += 1 |
| except signals.TestPass as signal: |
| test_signals_received.append(signal) |
| num_seen_passes += 1 |
| except (signals.TestSignal, KeyboardInterrupt): |
| raise |
| except Exception as signal: |
| test_signals_received.append(signal) |
| num_failures += 1 |
| else: |
| num_seen_passes += 1 |
| test_signals_received.append(signals.TestPass( |
| 'Test iteration %s of %s passed without details.' % ( |
| i, func.__name__))) |
| |
| if num_failures > acceptable_failures: |
| break |
| elif num_seen_passes == num_passes: |
| break |
| else: |
| self.teardown_test() |
| self.setup_test() |
| |
| raise result_selector(test_signals_received, self) |
| |
| return test_wrapper |
| |
| return decorator |