| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import io |
| import json |
| import os |
| import re |
| import shutil |
| import subprocess |
| import sys |
| import tarfile |
| import tempfile |
| import unittest |
| |
| import numpy |
| |
| import perfcompare |
| |
| |
| # Test case helper class for creating temporary directories that will |
| # be cleaned up when the test finishes. |
| class TempDirTestCase(unittest.TestCase): |
| |
| def setUp(self): |
| self._on_teardown = [] |
| |
| def MakeTempDir(self): |
| temp_dir = tempfile.mkdtemp( |
| prefix='tmp_unittest_%s_' % self.__class__.__name__) |
| def tear_down(): |
| shutil.rmtree(temp_dir) |
| self._on_teardown.append(tear_down) |
| return temp_dir |
| |
| def tearDown(self): |
| for func in reversed(self._on_teardown): |
| func() |
| |
| |
| def WriteJsonFile(filename, json_data): |
| with open(filename, 'w') as fh: |
| json.dump(json_data, fh) |
| |
| |
| def ReadGoldenFile(filename): |
| with open(filename, 'r') as fh: |
| data = fh.read() |
| matches = list(re.finditer('\n\n### (.*)\n', data, re.M)) |
| starts = [m.end() for m in matches] |
| ends = [m.start() for m in matches[1:]] + [len(data)] |
| for m, start, end in zip(matches, starts, ends): |
| yield m.group(1), data[start:end] |
| |
| |
| # Helper for checking against test expectations in a golden file. |
| # This provides an implementation of AssertCaseEq() that compares |
| # results against the golden file. |
| class GoldenDataInput(object): |
| |
| def __init__(self, filename): |
| self._cases = dict(ReadGoldenFile(filename)) |
| |
| def AssertCaseEq(self, name, actual): |
| expected = self._cases[name] |
| if expected != actual: |
| raise AssertionError('"%s" != "%s"' % (actual, expected)) |
| |
| |
| # This provides an implementation of AssertCaseEq() that updates the |
| # golden file with new expectations generated by the tests. |
| class GoldenDataOutput(object): |
| |
| def __init__(self): |
| self._cases = {} |
| |
| def AssertCaseEq(self, name, actual): |
| assert name not in self._cases, name |
| self._cases[name] = actual |
| |
| def WriteFile(self, filename): |
| with open(filename, 'w') as fh: |
| for name, data in sorted(self._cases.items()): |
| fh.write('\n\n### %s\n%s' % (name, data)) |
| |
| |
| GOLDEN_FILE = os.path.join(os.path.dirname(__file__), |
| 'perfcompare_test_output.txt') |
| GOLDEN = GoldenDataInput(GOLDEN_FILE) |
| |
| |
| def TestMain(): |
| global GOLDEN |
| if '--generate' in sys.argv: |
| sys.argv.pop(sys.argv.index('--generate')) |
| GOLDEN = GoldenDataOutput() |
| try: |
| unittest.main() |
| finally: |
| GOLDEN.WriteFile(GOLDEN_FILE) |
| else: |
| unittest.main() |
| |
| |
| # Test data from a normal distribution, generated using the following code: |
| # ', '.join('%.4f' % random.gauss(0, 1) for _ in xrange(100)) |
| TEST_VALUES = [ |
| 0.4171, 2.1056, -0.0223, -1.6592, 0.4766, -0.6405, 0.3488, 1.5729, |
| 2.0654, -0.1324, -0.8648, -0.2793, -0.7966, 0.2851, -0.9374, -2.0275, |
| 0.8222, -0.2396, -0.6982, 0.9067, 0.9416, -2.2870, -0.1868, 1.0700, |
| -1.2531, 0.8455, 1.4755, 0.2979, 0.3441, 0.6694, -0.1808, -0.9038, |
| 0.8267, -0.4320, -0.7166, 0.3757, -0.5135, -0.9497, 2.0372, -0.3364, |
| 0.3879, -0.2970, 1.3872, 0.6538, 1.0674, 1.2349, -0.6873, -0.1807, |
| 0.6867, -0.1150, -1.0526, -0.6853, -0.5858, -1.8460, 1.6041, -1.1638, |
| 0.5459, -1.6476, -0.8711, -0.9001, 0.0788, -0.8170, 0.2439, 0.0129, |
| -0.8674, -1.1076, -0.0074, -0.6230, -0.4761, -2.2526, 0.4906, -0.5001, |
| -0.2050, 0.7623, -0.5511, -0.2837, -0.8797, -0.5374, -1.2910, 0.9551, |
| 0.4483, -0.6352, -0.3334, -0.5105, 0.1073, 2.9131, -0.4941, -0.2808, |
| -0.2517, -1.9961, 0.9214, -0.6325, -1.1895, 0.8118, 1.5424, 0.5601, |
| -1.0322, 0.7135, -0.2780, -0.1128] |
| |
| def GenerateTestData(mean, stddev): |
| return [x * stddev + mean for x in TEST_VALUES] |
| |
| |
| # This is an example of a slow running time value for an initial run of a |
| # test. This should be skipped by the software under test. |
| SLOW_INITIAL_RUN = [1e6] |
| |
| |
| class FormatConfidenceIntervalTest(unittest.TestCase): |
| |
| def test_confidence_interval_formatting(self): |
| Format = perfcompare.FormatConfidenceInterval |
| |
| self.assertEqual(Format(12345.6789, 2222), '12346 +/- 2222') |
| self.assertEqual(Format(12345.6789, 0.02222), '12345.679 +/- 0.022') |
| self.assertEqual(Format(12345.6789, 0.07777), '12345.679 +/- 0.078') |
| self.assertEqual(Format(12345.6789, 0.09911), '12345.679 +/- 0.099') |
| # Corner case: rounding 0.09950 to 2 significant figures produces |
| # 0.100, which looks like 3 significant figures rather than 2. |
| self.assertEqual(Format(12345.6789, 0.09950), '12345.679 +/- 0.100') |
| self.assertEqual(Format(12345.6789, 2e-5), '12345.678900 +/- 0.000020') |
| |
| # Corner case: the offset is a power of 10. |
| self.assertEqual(Format(12345.6789, 0.1), '12345.68 +/- 0.10') |
| self.assertEqual(Format(12345.6789, 0.01), '12345.679 +/- 0.010') |
| |
| # Corner case: zero offset. |
| self.assertEqual(Format(12345.6789, 0), '12345.7 +/- 0') |
| |
| # Corner case: negative offset. This does not make sense for a |
| # confidence interval and should not happen, but let's ensure it |
| # gets formatted anyway in case that it useful for debugging. |
| self.assertEqual(Format(12345.6789, -1), '12345.7 +/- -1') |
| |
| # Corner cases: infinity and NaN. |
| self.assertEqual(Format(12345.6789, numpy.inf), '12345.7 +/- inf') |
| self.assertEqual(Format(12345.6789, -numpy.inf), '12345.7 +/- -inf') |
| self.assertEqual(Format(12345.6789, numpy.nan), '12345.7 +/- nan') |
| self.assertEqual(Format(numpy.inf, 0.1234), 'inf +/- 0.12') |
| self.assertEqual(Format(-numpy.inf, 0.1234), '-inf +/- 0.12') |
| self.assertEqual(Format(numpy.nan, 0.1234), 'nan +/- 0.12') |
| |
| |
| # Generate some example perf test data, allowing variation at each level of |
| # the sampling process (per boot, per process, and per iteration within |
| # each process). This follows a random effects model. Returns a list of |
| # lists of lists of values. |
| def GenerateData(mean=1000, |
| stddev_across_boots=0, |
| stddev_across_processes=0, |
| stddev_across_iters=0): |
| it = iter(TEST_VALUES) |
| |
| def GenerateValues(mean, stddev, count): |
| return [next(it) * stddev + mean for _ in range(count)] |
| |
| # This reads 4**3 + 4**2 + 4 = 84 values from TEST_VALUES, so it does |
| # not exceed the number of values in TEST_VALUES. |
| return [[SLOW_INITIAL_RUN |
| + GenerateValues(mean_within_process, stddev_across_iters, 4) |
| for mean_within_process in GenerateValues( |
| mean_within_boot, stddev_across_processes, 4)] |
| for mean_within_boot in GenerateValues( |
| mean, stddev_across_boots, 4)] |
| |
| |
| class StatisticsTest(TempDirTestCase): |
| |
| def ResultsDictForValues(self, run_values): |
| return {'label': 'ExampleTest', |
| 'test_suite': 'example_suite', |
| 'unit': 'nanoseconds', |
| 'values': run_values} |
| |
| # Given data in the format returned by GenerateData(), writes this data |
| # to a temporary directory. |
| def DirOfData(self, data): |
| dir_path = self.MakeTempDir() |
| os.mkdir(os.path.join(dir_path, 'by_boot')) |
| for boot_idx, results_for_boot in enumerate(data): |
| test_dir = os.path.join( |
| dir_path, 'by_boot', 'boot%06d' % boot_idx, 'test-name', |
| 'subdir') |
| os.makedirs(test_dir) |
| for process_idx, run_values in enumerate(results_for_boot): |
| dest_file = os.path.join( |
| test_dir, |
| 'example_process%06d.fuchsiaperf.json' % process_idx) |
| WriteJsonFile(dest_file, [self.ResultsDictForValues(run_values)]) |
| return dir_path |
| |
| # Sanity-check that DirOfData() writes data in the correct format by |
| # reading back some simple test data. |
| def test_readback_of_data(self): |
| data = [[[1, 2], [3, 4]], |
| [[5, 6], [7, 8]]] |
| dataset = perfcompare.MultiBootDataset(self.DirOfData(data)) |
| boot_datasets = list(dataset.GetBootDatasets()) |
| self.assertEqual(len(boot_datasets), 2) |
| self.assertEqual(list(boot_datasets[0].GetProcessDatasets()), |
| [[self.ResultsDictForValues([1, 2])], |
| [self.ResultsDictForValues([3, 4])]]) |
| self.assertEqual(list(boot_datasets[1].GetProcessDatasets()), |
| [[self.ResultsDictForValues([5, 6])], |
| [self.ResultsDictForValues([7, 8])]]) |
| |
| def TarFileOfDir(self, dir_path, write_mode): |
| tar_filename = os.path.join(self.MakeTempDir(), 'out.tar') |
| with tarfile.open(tar_filename, write_mode) as tar: |
| for name in os.listdir(dir_path): |
| tar.add(os.path.join(dir_path, name), arcname=name) |
| return tar_filename |
| |
| def test_readback_of_data_from_tar_file(self): |
| data = [[[1, 2], [3, 4]]] |
| dir_path = self.DirOfData(data) |
| self.assertEqual(len(os.listdir(os.path.join(dir_path, 'by_boot'))), 1) |
| # Test the uncompressed and gzipped cases. |
| for write_mode in ('w', 'w:gz'): |
| tar_filename = self.TarFileOfDir( |
| os.path.join(dir_path, 'by_boot', 'boot000000'), write_mode) |
| boot_dataset = perfcompare.SingleBootDataset(tar_filename) |
| self.assertEqual(list(boot_dataset.GetProcessDatasets()), |
| [[self.ResultsDictForValues([1, 2])], |
| [self.ResultsDictForValues([3, 4])]]) |
| |
| def CheckConfidenceInterval(self, data, interval_string): |
| dir_path = self.DirOfData(data) |
| test_name = 'example_suite: ExampleTest' |
| stats = perfcompare.StatsFromMultiBootDataset( |
| perfcompare.MultiBootDataset(dir_path))[test_name] |
| self.assertEqual(stats.FormatConfidenceInterval(), interval_string) |
| |
| # Test the CIs produced with variation at different levels of the |
| # multi-level sampling process. |
| def test_confidence_intervals(self): |
| self.CheckConfidenceInterval(GenerateData(), '1000 +/- 0 ns') |
| self.CheckConfidenceInterval( |
| GenerateData(stddev_across_boots=100), '1021 +/- 452 ns') |
| self.CheckConfidenceInterval( |
| GenerateData(stddev_across_processes=100), '1012 +/- 151 ns') |
| self.CheckConfidenceInterval( |
| GenerateData(stddev_across_iters=100), '981 +/- 74 ns') |
| |
| # Test the case where just a single value is produced per process run. |
| def test_confidence_interval_with_single_value_per_process(self): |
| self.CheckConfidenceInterval( |
| [[[100]], [[101]]], '100 +/- 32 ns') |
| |
| # If the "before" and "after" results have identical confidence |
| # intervals, that should be treated as "no difference", including when |
| # the CIs are zero-width (as tested here). |
| def test_comparing_equal_zero_width_confidence_intervals(self): |
| dir_path = self.DirOfData([[[200]], [[200]]]) |
| stdout = io.StringIO() |
| perfcompare.Main(['compare_perf', dir_path, dir_path], stdout) |
| output = stdout.getvalue() |
| GOLDEN.AssertCaseEq('comparison_no_change_zero_width_ci', output) |
| |
| |
| class PerfCompareTest(TempDirTestCase): |
| |
| def AddIgnoredFiles(self, dest_dir): |
| # Include a summary.json file to check that we skip reading it. |
| with open(os.path.join(dest_dir, 'summary.json'), 'w') as fh: |
| fh.write('dummy_data') |
| # Include a *.catapult_json file to check that we skip reading these. |
| with open(os.path.join(dest_dir, 'foo.catapult_json'), 'w') as fh: |
| fh.write('dummy_data') |
| |
| def WriteExampleDataDir(self, dir_path, mean=1000, stddev=100, |
| drop_one=False, single_boot=False): |
| results = [('ClockGetTimeExample', GenerateTestData(mean, stddev))] |
| if not drop_one: |
| results.append(('SecondExample', GenerateTestData(2000, 300))) |
| |
| if single_boot: |
| for test_name, values in results: |
| dest_dir = os.path.join(dir_path, 'by_boot', 'boot0') |
| dest_file = os.path.join( |
| dest_dir, '%s.fuchsiaperf.json' % test_name) |
| if not os.path.exists(dest_dir): |
| os.makedirs(dest_dir) |
| self.AddIgnoredFiles(dest_dir) |
| WriteJsonFile( |
| dest_file, |
| [{'label': test_name, |
| 'test_suite': 'fuchsia.example', |
| 'unit': 'nanoseconds', |
| 'values': SLOW_INITIAL_RUN + values}]) |
| else: |
| for test_name, values in results: |
| for idx, value in enumerate(values): |
| dest_dir = os.path.join( |
| dir_path, 'by_boot', 'boot%06d' % idx) |
| dest_file = os.path.join( |
| dest_dir, '%s.fuchsiaperf.json' % test_name) |
| if not os.path.exists(dest_dir): |
| os.makedirs(dest_dir) |
| self.AddIgnoredFiles(dest_dir) |
| WriteJsonFile( |
| dest_file, |
| [{'label': test_name, |
| 'test_suite': 'fuchsia.example', |
| 'unit': 'nanoseconds', |
| 'values': SLOW_INITIAL_RUN + [value]}]) |
| |
| def ExampleDataDir(self, **kwargs): |
| dir_path = self.MakeTempDir() |
| self.WriteExampleDataDir(dir_path, **kwargs) |
| return dir_path |
| |
| def test_reading_results_from_dir(self): |
| dir_path = self.ExampleDataDir() |
| results = perfcompare.StatsFromMultiBootDataset( |
| perfcompare.MultiBootDataset(dir_path)) |
| test_name = 'fuchsia.example: ClockGetTimeExample' |
| self.assertEqual( |
| results[test_name].FormatConfidenceInterval(), |
| '992 +/- 26 ns') |
| |
| # Returns the output of compare_perf when run on the given directories. |
| def ComparePerf(self, before_dir, after_dir): |
| stdout = io.StringIO() |
| perfcompare.Main(['compare_perf', before_dir, after_dir], stdout) |
| return stdout.getvalue() |
| |
| def test_mean_and_stddev(self): |
| values = [10, 5, 15] |
| mean_val, stddev_val = perfcompare.MeanAndStddev(values) |
| self.assertEqual(mean_val, 10.0) |
| self.assertEqual(perfcompare.Mean(values), 10.0) |
| self.assertEqual(stddev_val, 5.0) |
| # Single-value sample. |
| self.assertEqual(perfcompare.MeanAndStddev([123]), (123.0, None)) |
| # Check error cases. |
| self.assertRaises(AssertionError, lambda: perfcompare.Mean([])) |
| self.assertRaises(AssertionError, lambda: perfcompare.MeanAndStddev([])) |
| |
| # Check that data written using the golden file helper reads back |
| # the same. |
| def test_golden_file_write_and_read(self): |
| temp_file = os.path.join(self.MakeTempDir(), 'file') |
| writer = GoldenDataOutput() |
| writer.AssertCaseEq('a_key', 'a_value') |
| writer.AssertCaseEq('b_key', 'line 1\n' 'line 2\n') |
| writer.WriteFile(temp_file) |
| reader = GoldenDataInput(temp_file) |
| reader.AssertCaseEq('a_key', 'a_value') |
| reader.AssertCaseEq('b_key', 'line 1\n' 'line 2\n') |
| self.assertRaises(AssertionError, |
| lambda: reader.AssertCaseEq('a_key', 'other_value')) |
| |
| def test_comparison_no_change(self): |
| before_dir = self.ExampleDataDir() |
| after_dir = self.ExampleDataDir() |
| output = self.ComparePerf(before_dir, after_dir) |
| GOLDEN.AssertCaseEq('comparison_no_change', output) |
| |
| # Test a regression that is large enough to be flagged. |
| def test_comparison_regression(self): |
| before_dir = self.ExampleDataDir(mean=1500, stddev=100) |
| after_dir = self.ExampleDataDir(mean=1600, stddev=100) |
| output = self.ComparePerf(before_dir, after_dir) |
| GOLDEN.AssertCaseEq('comparison_regression', output) |
| |
| # Test an improvement that is large enough to be flagged. |
| def test_comparison_improvement(self): |
| before_dir = self.ExampleDataDir(mean=1500, stddev=100) |
| after_dir = self.ExampleDataDir(mean=1400, stddev=100) |
| output = self.ComparePerf(before_dir, after_dir) |
| GOLDEN.AssertCaseEq('comparison_improvement', output) |
| |
| # Test an improvement that is not large enough to be flagged. |
| def test_comparison_improvement_small(self): |
| before_dir = self.ExampleDataDir(mean=1500, stddev=100) |
| after_dir = self.ExampleDataDir(mean=1450, stddev=100) |
| output = self.ComparePerf(before_dir, after_dir) |
| GOLDEN.AssertCaseEq('comparison_improvement_small', output) |
| |
| def test_adding_test(self): |
| before_dir = self.ExampleDataDir(drop_one=True) |
| after_dir = self.ExampleDataDir() |
| output = self.ComparePerf(before_dir, after_dir) |
| GOLDEN.AssertCaseEq('adding_test', output) |
| |
| def test_removing_test(self): |
| before_dir = self.ExampleDataDir() |
| after_dir = self.ExampleDataDir(drop_one=True) |
| output = self.ComparePerf(before_dir, after_dir) |
| GOLDEN.AssertCaseEq('removing_test', output) |
| |
| def test_display_single_dataset(self): |
| dataset_dir = self.ExampleDataDir() |
| stdout = io.StringIO() |
| perfcompare.Main(['compare_perf', dataset_dir], stdout) |
| output = stdout.getvalue() |
| GOLDEN.AssertCaseEq('display_single_dataset', output) |
| |
| def test_display_three_datasets(self): |
| dataset_dirs = [self.ExampleDataDir(mean=1000), |
| self.ExampleDataDir(mean=2000, drop_one=True), |
| self.ExampleDataDir(mean=3000)] |
| stdout = io.StringIO() |
| perfcompare.Main(['compare_perf'] + dataset_dirs, stdout) |
| output = stdout.getvalue() |
| GOLDEN.AssertCaseEq('display_three_datasets', output) |
| |
| # Test printing a table of point estimates. |
| def test_display_single_boot_single_dataset(self): |
| dataset_dir = self.ExampleDataDir(single_boot=True) |
| stdout = io.StringIO() |
| perfcompare.Main(['compare_perf', dataset_dir], stdout) |
| output = stdout.getvalue() |
| GOLDEN.AssertCaseEq('display_single_boot_single_dataset', output) |
| |
| # Test printing a table of point estimates. |
| def test_display_single_boot_two_datasets(self): |
| dataset_dirs = [self.ExampleDataDir(mean=1000, single_boot=True), |
| self.ExampleDataDir(mean=2000, single_boot=True, |
| drop_one=True)] |
| stdout = io.StringIO() |
| perfcompare.Main(['compare_perf'] + dataset_dirs, stdout) |
| output = stdout.getvalue() |
| GOLDEN.AssertCaseEq('display_single_boot_two_datasets', output) |
| |
| def test_factor_range_formatting(self): |
| # Construct an interval pair of the same type used in the |
| # software-under-test, checking that the interval is well-formed. |
| def Interval(min_val, max_val): |
| assert min_val <= max_val |
| return (numpy.float64(min_val), numpy.float64(max_val)) |
| |
| # Check that the values are of the same type as in the |
| # software-under-test. |
| interval_test = Interval(10, 20) |
| interval_real = perfcompare.Stats([1, 2, 3], 'some_unit').interval |
| self.assertEqual(type(interval_test[0]), type(interval_real[0])) |
| self.assertEqual(type(interval_test[1]), type(interval_real[1])) |
| |
| def Format(interval_before, interval_after): |
| return perfcompare.FormatFactorRange(Interval(*interval_before), |
| Interval(*interval_after)) |
| |
| self.assertEqual(Format((1, 2), (3, 4)), '1.500-4.000') |
| # Test zero "min" values. |
| self.assertEqual(Format((0, 2), (3, 4)), '1.500-inf') |
| self.assertEqual(Format((1, 2), (0, 4)), '0.000-4.000') |
| # Test zero "min" and "max" values. |
| self.assertEqual(Format((0, 0), (3, 4)), 'inf-inf') |
| self.assertEqual(Format((1, 2), (0, 0)), '0.000-0.000') |
| # Test zero "max" values, with negative "min". |
| self.assertEqual(Format((-1, 0), (3, 4)), 'ci_too_wide') |
| self.assertEqual(Format((1, 2), (-3, 0)), 'ci_too_wide') |
| # All values zero. |
| self.assertEqual(Format((0, 0), (0, 0)), 'no_change') |
| |
| def test_mismatch_rate(self): |
| self.assertEqual(perfcompare.MismatchRate([(0,1), (2,3)]), 1) |
| self.assertEqual(perfcompare.MismatchRate([(0,2), (1,3)]), 0) |
| self.assertEqual(perfcompare.MismatchRate([(0,2), (1,3), (4,5)]), 2./3) |
| |
| def test_validate_perfcompare(self): |
| def MakeExampleDirs(**kwargs): |
| by_boot_dir = os.path.join(self.ExampleDataDir(**kwargs), 'by_boot') |
| return [os.path.join(by_boot_dir, name) |
| for name in sorted(os.listdir(by_boot_dir))] |
| |
| # This is an example input dataset that gives a high mismatch rate, |
| # because the data is drawn from two very different distributions. |
| results_dirs = (MakeExampleDirs(mean=100, stddev=10) + |
| MakeExampleDirs(mean=200, stddev=10)) |
| stdout = io.StringIO() |
| perfcompare.Main(['validate_perfcompare', '--group_size=5'] |
| + results_dirs, stdout) |
| output = stdout.getvalue() |
| GOLDEN.AssertCaseEq('validate_perfcompare', output) |
| |
| |
| class RunLocalTest(TempDirTestCase): |
| |
| # Test basic operation of the "run_local" subcommand. |
| def test_run_local(self): |
| # Destination directory for the full multiboot dataset. Use a |
| # destination path that does not exist yet. |
| dest_dir = os.path.join(self.MakeTempDir(), 'new_dir') |
| |
| # Destination pathnames for process dataset files. |
| iter_temp_dir = self.MakeTempDir() |
| iter_temp_file = os.path.join(iter_temp_dir, 'result.fuchsiaperf.json') |
| iter_temp_glob = os.path.join(iter_temp_dir, '*.fuchsiaperf.json') |
| |
| data = GenerateData(mean=1000, |
| stddev_across_boots=10, |
| stddev_across_processes=10, |
| stddev_across_iters=10) |
| commands = [] |
| # Dummy version of subprocess.check_call() for testing. |
| def DummyRunCmd(cmd, shell=False): |
| self.assertEqual(shell, True) |
| commands.append(cmd) |
| if cmd == 'set -o errexit -o nounset; my_iter_cmd': |
| WriteJsonFile(iter_temp_file, |
| [{'label': 'MyTest', |
| 'test_suite': 'example_suite', |
| 'unit': 'nanoseconds', |
| 'values': data.pop(0)[0]}]) |
| |
| stdout = io.StringIO() |
| perfcompare.Main(['run_local', |
| '--boots=4', |
| '--iter_file', iter_temp_glob, |
| '--iter_cmd', 'my_iter_cmd', |
| '--reboot_cmd', 'my_reboot_cmd', |
| '--dest', dest_dir], |
| stdout, run_cmd=DummyRunCmd) |
| self.assertEqual(commands, |
| ['set -o errexit -o nounset; my_reboot_cmd', |
| 'set -o errexit -o nounset; my_iter_cmd'] * 4) |
| GOLDEN.AssertCaseEq('run_local', stdout.getvalue()) |
| |
| # "run_local" should give an error if the temporary files specified by |
| # --iter_file already exist. |
| def test_error_if_dest_files_already_exist(self): |
| dest_dir = os.path.join(self.MakeTempDir(), 'new_dir') |
| iter_temp_file = os.path.join( |
| self.MakeTempDir(), 'result.fuchsiaperf.json') |
| WriteJsonFile(iter_temp_file, []) |
| args = ['run_local', |
| '--boots=4', |
| '--iter_file', iter_temp_file, |
| '--iter_cmd', 'my_iter_cmd', |
| '--reboot_cmd', 'my_reboot_cmd', |
| '--dest', dest_dir] |
| self.assertRaises(AssertionError, |
| lambda: perfcompare.Main(args, sys.stdout)) |
| |
| # Check that error-checking is enabled in the shell commands that |
| # run_local runs. |
| def test_errexit_error_checking_in_shell_commands(self): |
| iter_temp_file = os.path.join( |
| self.MakeTempDir(), 'result.fuchsiaperf.json') |
| stdout = io.StringIO() |
| |
| def get_args(): |
| dest_dir = os.path.join(self.MakeTempDir(), 'new_dir') |
| return ['run_local', |
| '--boots=4', |
| '--iter_file', iter_temp_file, |
| '--dest', dest_dir] |
| |
| perfcompare.Main( |
| get_args() + ['--iter_cmd', 'true', '--reboot_cmd', 'true'], |
| stdout) |
| # Check that the failure of the "false" command gets caught. |
| self.assertRaises( |
| subprocess.CalledProcessError, |
| lambda: perfcompare.Main( |
| get_args() + ['--iter_cmd', 'false; true', |
| '--reboot_cmd', 'true'], |
| stdout)) |
| self.assertRaises( |
| subprocess.CalledProcessError, |
| lambda: perfcompare.Main( |
| get_args() + ['--iter_cmd', 'true', |
| '--reboot_cmd', 'false; true'], |
| stdout)) |
| |
| |
| if __name__ == '__main__': |
| TestMain() |
