| #!/usr/bin/env python3 | |
| # | |
| # Copyright 2022 The Fuchsia Authors | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| from antlion import signals | |
| def __select_last(test_signals, _): | |
| return test_signals[-1] | |
| def repeated_test(num_passes, acceptable_failures=0, result_selector=__select_last): | |
| """A decorator that runs a test case multiple times. | |
| This decorator can be used to run a test multiple times and aggregate the | |
| data into a single test result. By setting `result_selector`, the user can | |
| access the returned result of each run, allowing them to average results, | |
| return the median, or gather and return standard deviation values. | |
| This decorator should be used on test cases, and should not be used on | |
| static or class methods. The test case must take in an additional argument, | |
| `attempt_number`, which returns the current attempt number, starting from | |
| 1. | |
| Note that any TestSignal intended to abort or skip the test will take | |
| abort or skip immediately. | |
| Args: | |
| num_passes: The number of times the test needs to pass to report the | |
| test case as passing. | |
| acceptable_failures: The number of failures accepted. If the failures | |
| exceeds this number, the test will stop repeating. The maximum | |
| number of runs is `num_passes + acceptable_failures`. If the test | |
| does fail, result_selector will still be called. | |
| result_selector: A lambda that takes in the list of TestSignals and | |
| returns the test signal to report the test case as. Note that the | |
| list also contains any uncaught exceptions from the test execution. | |
| """ | |
| def decorator(func): | |
| if not func.__name__.startswith("test_"): | |
| raise ValueError('Tests must start with "test_".') | |
| def test_wrapper(self): | |
| num_failures = 0 | |
| num_seen_passes = 0 | |
| test_signals_received = [] | |
| for i in range(num_passes + acceptable_failures): | |
| try: | |
| func(self, i + 1) | |
| except ( | |
| signals.TestFailure, | |
| signals.TestError, | |
| AssertionError, | |
| ) as signal: | |
| test_signals_received.append(signal) | |
| num_failures += 1 | |
| except signals.TestPass as signal: | |
| test_signals_received.append(signal) | |
| num_seen_passes += 1 | |
| except (signals.TestSignal, KeyboardInterrupt): | |
| raise | |
| except Exception as signal: | |
| test_signals_received.append(signal) | |
| num_failures += 1 | |
| else: | |
| num_seen_passes += 1 | |
| test_signals_received.append( | |
| signals.TestPass( | |
| "Test iteration %s of %s passed without details." | |
| % (i, func.__name__) | |
| ) | |
| ) | |
| if num_failures > acceptable_failures: | |
| break | |
| elif num_seen_passes == num_passes: | |
| break | |
| else: | |
| self.teardown_test() | |
| self.setup_test() | |
| raise result_selector(test_signals_received, self) | |
| return test_wrapper | |
| return decorator |