| # Copyright 2019 The Fuchsia Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| import csv | |
| import importlib.resources | |
| import io | |
| import json | |
| import os | |
| import pathlib | |
| import re | |
| import statistics | |
| import subprocess | |
| import sys | |
| import tarfile | |
| import tempfile | |
| import threading | |
| import unittest | |
| import unittest.mock | |
| from typing import Any, Callable, Dict, List, Optional, Tuple, Union | |
| import local_workflow | |
| import perfcompare | |
| # The following allows perfcompare_test.py to be run either: | |
| # | |
| # 1. Using Fuchsia's GN plumbing for Python code, where test data must | |
| # be imported through a special module using importlib. This case | |
| # allows the test to be run on Fuchsia Infra. | |
| # 2. By running perfcompare_test.py directly through | |
| # fuchsia-vendored-python or python3. This case allows the golden file | |
| # to be updated, by passing "--generate". | |
| try: | |
| import perfcompare_test_data | |
| except ModuleNotFoundError: | |
| RESOURCES_DIR = pathlib.Path(__file__).parent | |
| else: | |
| RESOURCES_DIR = importlib.resources.files(perfcompare_test_data) | |
| Behavior = Union[ | |
| str, | |
| Callable[[List[str]], subprocess.CompletedProcess], | |
| Exception, | |
| ] | |
| # Test case helper class for creating temporary directories that will | |
| # be cleaned up when the test finishes. | |
| class TempDirTestCase(unittest.TestCase): | |
| def MakeTempDir(self): | |
| return self.enterContext( | |
| tempfile.TemporaryDirectory( | |
| prefix="tmp_unittest_%s_" % self.__class__.__name__ | |
| ) | |
| ) | |
| def WriteJsonFile(filename, json_data): | |
| with open(filename, "w") as fh: | |
| json.dump(json_data, fh) | |
| def ReadGoldenFile(file_obj): | |
| with file_obj.open("r") as fh: | |
| data = fh.read() | |
| matches = list(re.finditer("\n\n### (.*)\n", data, re.M)) | |
| starts = [m.end() for m in matches] | |
| ends = [m.start() for m in matches[1:]] + [len(data)] | |
| for m, start, end in zip(matches, starts, ends): | |
| yield m.group(1), data[start:end] | |
| # Helper for checking against test expectations in a golden file. | |
| # This provides an implementation of AssertCaseEq() that compares | |
| # results against the golden file. | |
| class GoldenDataInput(object): | |
| def __init__(self, filename): | |
| self._cases = dict(ReadGoldenFile(filename)) | |
| def AssertCaseEq(self, name, actual): | |
| expected = self._cases[name] | |
| if expected != actual: | |
| raise AssertionError('"%s" != "%s"' % (actual, expected)) | |
| # This provides an implementation of AssertCaseEq() that updates the | |
| # golden file with new expectations generated by the tests. | |
| class GoldenDataOutput(object): | |
| def __init__(self): | |
| self._cases = {} | |
| def AssertCaseEq(self, name, actual): | |
| assert name not in self._cases, name | |
| self._cases[name] = actual | |
| def WriteFile(self, file_obj): | |
| with file_obj.open("w") as fh: | |
| for name, data in sorted(self._cases.items()): | |
| fh.write("\n\n### %s\n%s" % (name, data)) | |
| GOLDEN_FILE = RESOURCES_DIR.joinpath("perfcompare_test_output.txt") | |
| GOLDEN = GoldenDataInput(GOLDEN_FILE) | |
| def TestMain(): | |
| global GOLDEN | |
| if "--generate" in sys.argv: | |
| sys.argv.pop(sys.argv.index("--generate")) | |
| GOLDEN = GoldenDataOutput() | |
| try: | |
| unittest.main() | |
| finally: | |
| GOLDEN.WriteFile(GOLDEN_FILE) | |
| else: | |
| unittest.main() | |
| # Test data from a normal distribution, generated using the following code: | |
| # ', '.join('%.4f' % random.gauss(0, 1) for _ in xrange(100)) | |
| TEST_VALUES = [ | |
| 0.4171, | |
| 2.1056, | |
| -0.0223, | |
| -1.6592, | |
| 0.4766, | |
| -0.6405, | |
| 0.3488, | |
| 1.5729, | |
| 2.0654, | |
| -0.1324, | |
| -0.8648, | |
| -0.2793, | |
| -0.7966, | |
| 0.2851, | |
| -0.9374, | |
| -2.0275, | |
| 0.8222, | |
| -0.2396, | |
| -0.6982, | |
| 0.9067, | |
| 0.9416, | |
| -2.2870, | |
| -0.1868, | |
| 1.0700, | |
| -1.2531, | |
| 0.8455, | |
| 1.4755, | |
| 0.2979, | |
| 0.3441, | |
| 0.6694, | |
| -0.1808, | |
| -0.9038, | |
| 0.8267, | |
| -0.4320, | |
| -0.7166, | |
| 0.3757, | |
| -0.5135, | |
| -0.9497, | |
| 2.0372, | |
| -0.3364, | |
| 0.3879, | |
| -0.2970, | |
| 1.3872, | |
| 0.6538, | |
| 1.0674, | |
| 1.2349, | |
| -0.6873, | |
| -0.1807, | |
| 0.6867, | |
| -0.1150, | |
| -1.0526, | |
| -0.6853, | |
| -0.5858, | |
| -1.8460, | |
| 1.6041, | |
| -1.1638, | |
| 0.5459, | |
| -1.6476, | |
| -0.8711, | |
| -0.9001, | |
| 0.0788, | |
| -0.8170, | |
| 0.2439, | |
| 0.0129, | |
| -0.8674, | |
| -1.1076, | |
| -0.0074, | |
| -0.6230, | |
| -0.4761, | |
| -2.2526, | |
| 0.4906, | |
| -0.5001, | |
| -0.2050, | |
| 0.7623, | |
| -0.5511, | |
| -0.2837, | |
| -0.8797, | |
| -0.5374, | |
| -1.2910, | |
| 0.9551, | |
| 0.4483, | |
| -0.6352, | |
| -0.3334, | |
| -0.5105, | |
| 0.1073, | |
| 2.9131, | |
| -0.4941, | |
| -0.2808, | |
| -0.2517, | |
| -1.9961, | |
| 0.9214, | |
| -0.6325, | |
| -1.1895, | |
| 0.8118, | |
| 1.5424, | |
| 0.5601, | |
| -1.0322, | |
| 0.7135, | |
| -0.2780, | |
| -0.1128, | |
| ] | |
| def GenerateTestData(mean, stddev): | |
| return [x * stddev + mean for x in TEST_VALUES] | |
| # This is an example of a slow running time value for an initial run of a | |
| # test. This should be skipped by the software under test. | |
| SLOW_INITIAL_RUN = [1e6] | |
| # Returns the output of the "compare_perf" subcommand when run on the | |
| # given directories. | |
| def ComparePerf(dir_paths, expected_sample_size=100): | |
| stdout = io.StringIO() | |
| perfcompare.Main( | |
| [ | |
| "compare_perf", | |
| "--expected_sample_size=%d" % expected_sample_size, | |
| ] | |
| + dir_paths, | |
| stdout, | |
| ) | |
| return stdout.getvalue() | |
| class FormatConfidenceIntervalTest(unittest.TestCase): | |
| def test_confidence_interval_formatting(self): | |
| Format = perfcompare.FormatConfidenceInterval | |
| self.assertEqual(Format(12345.6789, 2222), "12346 +/- 2222") | |
| self.assertEqual(Format(12345.6789, 0.02222), "12345.679 +/- 0.022") | |
| self.assertEqual(Format(12345.6789, 0.07777), "12345.679 +/- 0.078") | |
| self.assertEqual(Format(12345.6789, 0.09911), "12345.679 +/- 0.099") | |
| # Corner case: rounding 0.09950 to 2 significant figures produces | |
| # 0.100, which looks like 3 significant figures rather than 2. | |
| self.assertEqual(Format(12345.6789, 0.09950), "12345.679 +/- 0.100") | |
| self.assertEqual(Format(12345.6789, 2e-5), "12345.678900 +/- 0.000020") | |
| # Corner case: the offset is a power of 10. | |
| self.assertEqual(Format(12345.6789, 0.1), "12345.68 +/- 0.10") | |
| self.assertEqual(Format(12345.6789, 0.01), "12345.679 +/- 0.010") | |
| # Corner case: zero offset. | |
| self.assertEqual(Format(12345.6789, 0), "12345.7 +/- 0") | |
| # Corner case: negative offset. This does not make sense for a | |
| # confidence interval and should not happen, but let's ensure it | |
| # gets formatted anyway in case that it useful for debugging. | |
| self.assertEqual(Format(12345.6789, -1), "12345.7 +/- -1") | |
| # Corner cases: infinity and NaN. | |
| self.assertEqual(Format(12345.6789, float("inf")), "12345.7 +/- inf") | |
| self.assertEqual(Format(12345.6789, -float("inf")), "12345.7 +/- -inf") | |
| self.assertEqual(Format(12345.6789, float("nan")), "12345.7 +/- nan") | |
| self.assertEqual(Format(float("inf"), 0.1234), "inf +/- 0.12") | |
| self.assertEqual(Format(-float("inf"), 0.1234), "-inf +/- 0.12") | |
| self.assertEqual(Format(float("nan"), 0.1234), "nan +/- 0.12") | |
| # Generate some example perf test data, allowing variation at each level of | |
| # the sampling process (per boot, per process, and per iteration within | |
| # each process). This follows a random effects model. Returns a list of | |
| # lists of lists of values. | |
| def GenerateData( | |
| mean=1000, | |
| stddev_across_boots=0, | |
| stddev_across_processes=0, | |
| stddev_across_iters=0, | |
| ): | |
| it = iter(TEST_VALUES) | |
| def GenerateValues(mean, stddev, count): | |
| return [next(it) * stddev + mean for _ in range(count)] | |
| # This reads 4**3 + 4**2 + 4 = 84 values from TEST_VALUES, so it does | |
| # not exceed the number of values in TEST_VALUES. | |
| return [ | |
| [ | |
| SLOW_INITIAL_RUN | |
| + GenerateValues(mean_within_process, stddev_across_iters, 4) | |
| for mean_within_process in GenerateValues( | |
| mean_within_boot, stddev_across_processes, 4 | |
| ) | |
| ] | |
| for mean_within_boot in GenerateValues(mean, stddev_across_boots, 4) | |
| ] | |
| class StatisticsTest(TempDirTestCase): | |
| def ResultsDictForValues(self, run_values): | |
| return { | |
| "label": "ExampleTest", | |
| "test_suite": "example_suite", | |
| "unit": "nanoseconds", | |
| "values": run_values, | |
| } | |
| # Given data in the format returned by GenerateData(), writes this data | |
| # to a temporary directory. | |
| def DirOfData(self, data): | |
| dir_path = self.MakeTempDir() | |
| os.mkdir(os.path.join(dir_path, "by_boot")) | |
| for boot_idx, results_for_boot in enumerate(data): | |
| test_dir = os.path.join( | |
| dir_path, | |
| "by_boot", | |
| "boot%06d" % boot_idx, | |
| "test-name", | |
| "subdir", | |
| ) | |
| os.makedirs(test_dir) | |
| for process_idx, run_values in enumerate(results_for_boot): | |
| dest_file = os.path.join( | |
| test_dir, | |
| "example_process%06d.fuchsiaperf.json" % process_idx, | |
| ) | |
| WriteJsonFile( | |
| dest_file, [self.ResultsDictForValues(run_values)] | |
| ) | |
| return dir_path | |
| # Sanity-check that DirOfData() writes data in the correct format by | |
| # reading back some simple test data. | |
| def test_readback_of_data(self): | |
| data = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] | |
| dataset = perfcompare.MultiBootDataset(self.DirOfData(data)) | |
| boot_datasets = list(dataset.GetBootDatasets()) | |
| self.assertEqual(len(boot_datasets), 2) | |
| self.assertEqual( | |
| list(boot_datasets[0].GetProcessDatasets()), | |
| [ | |
| [self.ResultsDictForValues([1, 2])], | |
| [self.ResultsDictForValues([3, 4])], | |
| ], | |
| ) | |
| self.assertEqual( | |
| list(boot_datasets[1].GetProcessDatasets()), | |
| [ | |
| [self.ResultsDictForValues([5, 6])], | |
| [self.ResultsDictForValues([7, 8])], | |
| ], | |
| ) | |
| def TarFileOfDir(self, dir_path, write_mode): | |
| tar_filename = os.path.join(self.MakeTempDir(), "out.tar") | |
| with tarfile.open(tar_filename, write_mode) as tar: | |
| for name in os.listdir(dir_path): | |
| tar.add(os.path.join(dir_path, name), arcname=name) | |
| return tar_filename | |
| def test_readback_of_data_from_tar_file(self): | |
| data = [[[1, 2], [3, 4]]] | |
| dir_path = self.DirOfData(data) | |
| self.assertEqual(len(os.listdir(os.path.join(dir_path, "by_boot"))), 1) | |
| # Test the uncompressed and gzipped cases. | |
| for write_mode in ("w", "w:gz"): | |
| tar_filename = self.TarFileOfDir( | |
| os.path.join(dir_path, "by_boot", "boot000000"), write_mode | |
| ) | |
| boot_dataset = perfcompare.SingleBootDataset(tar_filename) | |
| self.assertEqual( | |
| list(boot_dataset.GetProcessDatasets()), | |
| [ | |
| [self.ResultsDictForValues([1, 2])], | |
| [self.ResultsDictForValues([3, 4])], | |
| ], | |
| ) | |
| def CheckConfidenceInterval(self, data, interval_string): | |
| dir_path = self.DirOfData(data) | |
| test_name = "example_suite: ExampleTest" | |
| stats = perfcompare.StatsFromMultiBootDataset( | |
| perfcompare.MultiBootDataset(dir_path) | |
| )[test_name] | |
| self.assertEqual(stats.FormatConfidenceInterval(), interval_string) | |
| # Test the CIs produced with variation at different levels of the | |
| # multi-level sampling process. | |
| def test_confidence_intervals(self): | |
| self.CheckConfidenceInterval(GenerateData(), "1000 +/- 0 ns") | |
| self.CheckConfidenceInterval( | |
| GenerateData(stddev_across_boots=100), "1021 +/- 452 ns" | |
| ) | |
| self.CheckConfidenceInterval( | |
| GenerateData(stddev_across_processes=100), "1012 +/- 151 ns" | |
| ) | |
| self.CheckConfidenceInterval( | |
| GenerateData(stddev_across_iters=100), "981 +/- 74 ns" | |
| ) | |
| # Test the case where just a single value is produced per process run. | |
| def test_confidence_interval_with_single_value_per_process(self): | |
| self.CheckConfidenceInterval([[[100]], [[101]]], "100 +/- 32 ns") | |
| # If the "before" and "after" results have identical confidence | |
| # intervals, that should be treated as "no difference", including when | |
| # the CIs are zero-width (as tested here). | |
| def test_comparing_equal_zero_width_confidence_intervals(self): | |
| dir_path = self.DirOfData([[[200]], [[200]]]) | |
| output = ComparePerf([dir_path, dir_path], expected_sample_size=2) | |
| GOLDEN.AssertCaseEq("comparison_no_change_zero_width_ci", output) | |
| def test_sample_size_check_larger(self): | |
| dir1_path = self.DirOfData([[[10]]] * 4) | |
| dir2_path = self.DirOfData([[[10]]] * 5) | |
| # Check that we get a descriptive error if the actual sample | |
| # sizes do not match the "--expected_sample_size" argument. | |
| error = ( | |
| "^The following metrics had an unexpected sample size" | |
| r" \(expected <=3\):\n" | |
| r" example_suite: ExampleTest \(got 4\)\n" | |
| r" example_suite: ExampleTest \(got 5\)$" | |
| ) | |
| with self.assertRaisesRegex(AssertionError, error): | |
| ComparePerf([dir1_path, dir2_path], expected_sample_size=3) | |
| # We should get a similar error if we pass only one directory. | |
| error = ( | |
| "^The following metrics had an unexpected sample size" | |
| r" \(expected <=3\):\n" | |
| r" example_suite: ExampleTest \(got 4\)$" | |
| ) | |
| with self.assertRaisesRegex(AssertionError, error): | |
| ComparePerf([dir1_path], expected_sample_size=3) | |
| # There should be no check for the sample sizes if the | |
| # "--expected_sample_size" argument is not given. | |
| stdout = io.StringIO() | |
| perfcompare.Main(["compare_perf", dir1_path, dir2_path], stdout) | |
| def test_sample_size_check_smaller(self): | |
| # We shouldn't get an error if the actual sample sizes are | |
| # smaller than expected. | |
| dir1_path = self.DirOfData([[[10]]] * 3) | |
| dir2_path = self.DirOfData([[[10]]] * 4) | |
| ComparePerf([dir1_path, dir2_path], expected_sample_size=5) | |
| class PerfCompareTest(TempDirTestCase): | |
| def AddIgnoredFiles(self, dest_dir): | |
| # Include a summary.json file to check that we skip reading it. | |
| with open(os.path.join(dest_dir, "summary.json"), "w") as fh: | |
| fh.write("dummy_data") | |
| # Include a *.catapult_json file to check that we skip reading these. | |
| with open(os.path.join(dest_dir, "foo.catapult_json"), "w") as fh: | |
| fh.write("dummy_data") | |
| def WriteExampleDataDir( | |
| self, | |
| dir_path, | |
| mean=1000, | |
| stddev=100, | |
| drop_one=False, | |
| single_boot=False, | |
| unit="nanoseconds", | |
| ): | |
| results = [("ClockGetTimeExample", GenerateTestData(mean, stddev))] | |
| if not drop_one: | |
| results.append(("SecondExample", GenerateTestData(2000, 300))) | |
| if single_boot: | |
| for test_name, values in results: | |
| dest_dir = os.path.join(dir_path, "by_boot", "boot0") | |
| dest_file = os.path.join( | |
| dest_dir, "%s.fuchsiaperf.json" % test_name | |
| ) | |
| if not os.path.exists(dest_dir): | |
| os.makedirs(dest_dir) | |
| self.AddIgnoredFiles(dest_dir) | |
| WriteJsonFile( | |
| dest_file, | |
| [ | |
| { | |
| "label": test_name, | |
| "test_suite": "fuchsia.example", | |
| "unit": unit, | |
| "values": SLOW_INITIAL_RUN + values, | |
| } | |
| ], | |
| ) | |
| else: | |
| for test_name, values in results: | |
| for idx, value in enumerate(values): | |
| dest_dir = os.path.join( | |
| dir_path, "by_boot", "boot%06d" % idx | |
| ) | |
| dest_file = os.path.join( | |
| dest_dir, "%s.fuchsiaperf.json" % test_name | |
| ) | |
| if not os.path.exists(dest_dir): | |
| os.makedirs(dest_dir) | |
| self.AddIgnoredFiles(dest_dir) | |
| WriteJsonFile( | |
| dest_file, | |
| [ | |
| { | |
| "label": test_name, | |
| "test_suite": "fuchsia.example", | |
| "unit": unit, | |
| "values": SLOW_INITIAL_RUN + [value], | |
| } | |
| ], | |
| ) | |
| def ExampleDataDir(self, **kwargs): | |
| dir_path = self.MakeTempDir() | |
| self.WriteExampleDataDir(dir_path, **kwargs) | |
| return dir_path | |
| def test_reading_results_from_dir(self): | |
| dir_path = self.ExampleDataDir() | |
| results = perfcompare.StatsFromMultiBootDataset( | |
| perfcompare.MultiBootDataset(dir_path) | |
| ) | |
| test_name = "fuchsia.example: ClockGetTimeExample" | |
| self.assertEqual( | |
| results[test_name].FormatConfidenceInterval(), "992 +/- 26 ns" | |
| ) | |
| def test_mean_and_stddev(self): | |
| values = [10, 5, 15] | |
| mean_val, stddev_val = perfcompare.MeanAndStddev(values) | |
| self.assertEqual(mean_val, 10.0) | |
| self.assertEqual(perfcompare.Mean(values), 10.0) | |
| self.assertEqual(stddev_val, 5.0) | |
| # Single-value sample. | |
| self.assertEqual(perfcompare.MeanAndStddev([123]), (123.0, None)) | |
| # Check error cases. | |
| self.assertRaises(AssertionError, lambda: perfcompare.Mean([])) | |
| self.assertRaises(AssertionError, lambda: perfcompare.MeanAndStddev([])) | |
| # Check that data written using the golden file helper reads back | |
| # the same. | |
| def test_golden_file_write_and_read(self): | |
| temp_file = pathlib.Path(os.path.join(self.MakeTempDir(), "file")) | |
| writer = GoldenDataOutput() | |
| writer.AssertCaseEq("a_key", "a_value") | |
| writer.AssertCaseEq("b_key", "line 1\n" "line 2\n") | |
| writer.WriteFile(temp_file) | |
| reader = GoldenDataInput(temp_file) | |
| reader.AssertCaseEq("a_key", "a_value") | |
| reader.AssertCaseEq("b_key", "line 1\n" "line 2\n") | |
| self.assertRaises( | |
| AssertionError, lambda: reader.AssertCaseEq("a_key", "other_value") | |
| ) | |
| def test_comparison_no_change(self): | |
| before_dir = self.ExampleDataDir() | |
| after_dir = self.ExampleDataDir() | |
| output = ComparePerf([before_dir, after_dir]) | |
| GOLDEN.AssertCaseEq("comparison_no_change", output) | |
| # Test a regression that is large enough to be flagged. | |
| def test_comparison_regression(self): | |
| before_dir = self.ExampleDataDir(mean=1500, stddev=100) | |
| after_dir = self.ExampleDataDir(mean=1600, stddev=100) | |
| output = ComparePerf([before_dir, after_dir]) | |
| GOLDEN.AssertCaseEq("comparison_regression", output) | |
| # Test an improvement that is large enough to be flagged. | |
| def test_comparison_improvement(self): | |
| before_dir = self.ExampleDataDir(mean=1500, stddev=100) | |
| after_dir = self.ExampleDataDir(mean=1400, stddev=100) | |
| output = ComparePerf([before_dir, after_dir]) | |
| GOLDEN.AssertCaseEq("comparison_improvement", output) | |
| # Test an improvement that is not large enough to be flagged. | |
| def test_comparison_improvement_small(self): | |
| before_dir = self.ExampleDataDir(mean=1500, stddev=100) | |
| after_dir = self.ExampleDataDir(mean=1450, stddev=100) | |
| output = ComparePerf([before_dir, after_dir]) | |
| GOLDEN.AssertCaseEq("comparison_improvement_small", output) | |
| # Test a decrease that is a regression (in contrast to the default | |
| # where decreases are improvements). | |
| def test_comparison_regression_biggerisbetter(self): | |
| unit = "bytes/second" | |
| before_dir = self.ExampleDataDir(mean=1500, stddev=100, unit=unit) | |
| after_dir = self.ExampleDataDir(mean=1400, stddev=100, unit=unit) | |
| output = ComparePerf([before_dir, after_dir]) | |
| GOLDEN.AssertCaseEq("comparison_regression_biggerisbetter", output) | |
| # Test an increase that is an improvement (in contrast to the default | |
| # where increases are regressions). | |
| def test_comparison_improvement_biggerisbetter(self): | |
| unit = "bytes/second" | |
| before_dir = self.ExampleDataDir(mean=1400, stddev=100, unit=unit) | |
| after_dir = self.ExampleDataDir(mean=1500, stddev=100, unit=unit) | |
| output = ComparePerf([before_dir, after_dir]) | |
| GOLDEN.AssertCaseEq("comparison_improvement_biggerisbetter", output) | |
| def test_adding_test(self): | |
| before_dir = self.ExampleDataDir(drop_one=True) | |
| after_dir = self.ExampleDataDir() | |
| output = ComparePerf([before_dir, after_dir]) | |
| GOLDEN.AssertCaseEq("adding_test", output) | |
| def test_removing_test(self): | |
| before_dir = self.ExampleDataDir() | |
| after_dir = self.ExampleDataDir(drop_one=True) | |
| output = ComparePerf([before_dir, after_dir]) | |
| GOLDEN.AssertCaseEq("removing_test", output) | |
| def test_display_single_dataset(self): | |
| dataset_dir = self.ExampleDataDir() | |
| output = ComparePerf([dataset_dir]) | |
| GOLDEN.AssertCaseEq("display_single_dataset", output) | |
| def test_display_three_datasets(self): | |
| dataset_dirs = [ | |
| self.ExampleDataDir(mean=1000), | |
| self.ExampleDataDir(mean=2000, drop_one=True), | |
| self.ExampleDataDir(mean=3000), | |
| ] | |
| output = ComparePerf(dataset_dirs) | |
| GOLDEN.AssertCaseEq("display_three_datasets", output) | |
| # Test printing a table of point estimates. | |
| def test_display_single_boot_single_dataset(self): | |
| dataset_dir = self.ExampleDataDir(single_boot=True) | |
| output = ComparePerf([dataset_dir], expected_sample_size=1) | |
| GOLDEN.AssertCaseEq("display_single_boot_single_dataset", output) | |
| # Test printing a table of point estimates. | |
| def test_display_single_boot_two_datasets(self): | |
| dataset_dirs = [ | |
| self.ExampleDataDir(mean=1000, single_boot=True), | |
| self.ExampleDataDir(mean=2000, single_boot=True, drop_one=True), | |
| ] | |
| output = ComparePerf(dataset_dirs, expected_sample_size=1) | |
| GOLDEN.AssertCaseEq("display_single_boot_two_datasets", output) | |
| def test_factor_range_formatting(self): | |
| # Construct an interval pair of the same type used in the | |
| # software-under-test, in order to ensure that this test case is | |
| # realistic. (Earlier versions used numpy.float64.) Also check | |
| # that the interval is well-formed. | |
| def Interval(min_val, max_val): | |
| assert min_val <= max_val | |
| return (float(min_val), float(max_val)) | |
| # Check that the values are of the same type as in the | |
| # software-under-test. | |
| interval_test = Interval(10, 20) | |
| interval_real = perfcompare.Stats([1, 2, 3], "some_unit").interval | |
| self.assertEqual(type(interval_test[0]), type(interval_real[0])) | |
| self.assertEqual(type(interval_test[1]), type(interval_real[1])) | |
| def Format(interval_before, interval_after): | |
| return perfcompare.FormatFactorRange( | |
| Interval(*interval_before), Interval(*interval_after) | |
| ) | |
| self.assertEqual(Format((1, 2), (3, 4)), "1.500-4.000") | |
| # Test zero "min" values. | |
| self.assertEqual(Format((0, 2), (3, 4)), "1.500-inf") | |
| self.assertEqual(Format((1, 2), (0, 4)), "0.000-4.000") | |
| # Test zero "min" and "max" values. | |
| self.assertEqual(Format((0, 0), (3, 4)), "inf-inf") | |
| self.assertEqual(Format((1, 2), (0, 0)), "0.000-0.000") | |
| # Test zero "max" values, with negative "min". | |
| self.assertEqual(Format((-1, 0), (3, 4)), "ci_too_wide") | |
| self.assertEqual(Format((1, 2), (-3, 0)), "ci_too_wide") | |
| # All values zero. | |
| self.assertEqual(Format((0, 0), (0, 0)), "no_change") | |
| def test_mismatch_rate(self): | |
| self.assertEqual(perfcompare.MismatchRate([(0, 1), (2, 3)]), 1) | |
| self.assertEqual(perfcompare.MismatchRate([(0, 2), (1, 3)]), 0) | |
| self.assertEqual( | |
| perfcompare.MismatchRate([(0, 2), (1, 3), (4, 5)]), 2.0 / 3 | |
| ) | |
| def test_validate_perfcompare(self): | |
| def MakeExampleDirs(**kwargs): | |
| by_boot_dir = os.path.join(self.ExampleDataDir(**kwargs), "by_boot") | |
| return [ | |
| os.path.join(by_boot_dir, name) | |
| for name in sorted(os.listdir(by_boot_dir)) | |
| ] | |
| # This is an example input dataset that gives a high mismatch rate, | |
| # because the data is drawn from two very different distributions. | |
| results_dirs = MakeExampleDirs(mean=100, stddev=10) + MakeExampleDirs( | |
| mean=200, stddev=10 | |
| ) | |
| stdout = io.StringIO() | |
| perfcompare.Main( | |
| ["validate_perfcompare", "--group_size=5"] + results_dirs, stdout | |
| ) | |
| output = stdout.getvalue() | |
| GOLDEN.AssertCaseEq("validate_perfcompare", output) | |
| def test_biggerisbetter_handling(self): | |
| self.assertFalse(perfcompare.UnitBiggerIsBetter("ns")) | |
| self.assertTrue(perfcompare.UnitBiggerIsBetter("bytes/second")) | |
| self.assertTrue(perfcompare.UnitBiggerIsBetter("frames/second")) | |
| self.assertTrue(perfcompare.UnitBiggerIsBetter("count_biggerIsBetter")) | |
| self.assertFalse( | |
| perfcompare.UnitBiggerIsBetter("count_smallerIsBetter") | |
| ) | |
| class RunLocalTest(TempDirTestCase): | |
| # Test basic operation of the "run_local" subcommand. | |
| def test_run_local(self): | |
| # Destination directory for the full multiboot dataset. Use a | |
| # destination path that does not exist yet. | |
| dest_dir = os.path.join(self.MakeTempDir(), "new_dir") | |
| # Destination pathnames for process dataset files. | |
| iter_temp_dir = self.MakeTempDir() | |
| iter_temp_file = os.path.join(iter_temp_dir, "result.fuchsiaperf.json") | |
| iter_temp_glob = os.path.join(iter_temp_dir, "*.fuchsiaperf.json") | |
| data = GenerateData( | |
| mean=1000, | |
| stddev_across_boots=10, | |
| stddev_across_processes=10, | |
| stddev_across_iters=10, | |
| ) | |
| commands = [] | |
| # Dummy version of subprocess.check_call() for testing. | |
| def DummyRunCmd(cmd, shell=False): | |
| self.assertEqual(shell, True) | |
| commands.append(cmd) | |
| if cmd == "set -o errexit -o nounset; my_iter_cmd": | |
| WriteJsonFile( | |
| iter_temp_file, | |
| [ | |
| { | |
| "label": "MyTest", | |
| "test_suite": "example_suite", | |
| "unit": "nanoseconds", | |
| "values": data.pop(0)[0], | |
| } | |
| ], | |
| ) | |
| stdout = io.StringIO() | |
| perfcompare.Main( | |
| [ | |
| "run_local", | |
| "--boots=4", | |
| "--iter_file", | |
| iter_temp_glob, | |
| "--iter_cmd", | |
| "my_iter_cmd", | |
| "--reboot_cmd", | |
| "my_reboot_cmd", | |
| "--dest", | |
| dest_dir, | |
| ], | |
| stdout, | |
| run_cmd=DummyRunCmd, | |
| ) | |
| self.assertEqual( | |
| commands, | |
| [ | |
| "set -o errexit -o nounset; my_reboot_cmd", | |
| "set -o errexit -o nounset; my_iter_cmd", | |
| ] | |
| * 4, | |
| ) | |
| GOLDEN.AssertCaseEq("run_local", stdout.getvalue()) | |
| # "run_local" should give an error if the temporary files specified by | |
| # --iter_file already exist. | |
| def test_error_if_dest_files_already_exist(self): | |
| dest_dir = os.path.join(self.MakeTempDir(), "new_dir") | |
| iter_temp_file = os.path.join( | |
| self.MakeTempDir(), "result.fuchsiaperf.json" | |
| ) | |
| WriteJsonFile(iter_temp_file, []) | |
| args = [ | |
| "run_local", | |
| "--boots=4", | |
| "--iter_file", | |
| iter_temp_file, | |
| "--iter_cmd", | |
| "my_iter_cmd", | |
| "--reboot_cmd", | |
| "my_reboot_cmd", | |
| "--dest", | |
| dest_dir, | |
| ] | |
| self.assertRaises( | |
| AssertionError, lambda: perfcompare.Main(args, sys.stdout) | |
| ) | |
| # Check that error-checking is enabled in the shell commands that | |
| # run_local runs. | |
| def test_errexit_error_checking_in_shell_commands(self): | |
| iter_temp_file = os.path.join( | |
| self.MakeTempDir(), "result.fuchsiaperf.json" | |
| ) | |
| stdout = io.StringIO() | |
| def get_args(): | |
| dest_dir = os.path.join(self.MakeTempDir(), "new_dir") | |
| return [ | |
| "run_local", | |
| "--boots=4", | |
| "--iter_file", | |
| iter_temp_file, | |
| "--dest", | |
| dest_dir, | |
| ] | |
| perfcompare.Main( | |
| get_args() + ["--iter_cmd", "true", "--reboot_cmd", "true"], stdout | |
| ) | |
| # Check that the failure of the "false" command gets caught. | |
| self.assertRaises( | |
| subprocess.CalledProcessError, | |
| lambda: perfcompare.Main( | |
| get_args() | |
| + ["--iter_cmd", "false; true", "--reboot_cmd", "true"], | |
| stdout, | |
| ), | |
| ) | |
| self.assertRaises( | |
| subprocess.CalledProcessError, | |
| lambda: perfcompare.Main( | |
| get_args() | |
| + ["--iter_cmd", "true", "--reboot_cmd", "false; true"], | |
| stdout, | |
| ), | |
| ) | |
| class LocalWorkflowTest(TempDirTestCase): | |
| def setUp(self): | |
| super().setUp() | |
| self.state_file = os.path.join(self.MakeTempDir(), "state.json") | |
| state_data = { | |
| "2026-04-09": { | |
| "launched": { | |
| "builder1": {"build_id": "1001"}, | |
| } | |
| }, | |
| "2026-04-10": { | |
| "launched": { | |
| "builder1": {"build_id": "1002"}, | |
| "builder2": {"build_id": "1003"}, | |
| } | |
| }, | |
| } | |
| with open(self.state_file, "w") as f: | |
| json.dump(state_data, f) | |
| # Patch the code for finding the "bb" and "cas" tools so that the | |
| # test does not depend on the FUCHSIA_DIR env var being set and the | |
| # tools being present. | |
| self.enterContext( | |
| unittest.mock.patch( | |
| "local_workflow._find_bb_tool", lambda: "bb_for_testing" | |
| ) | |
| ) | |
| self.enterContext( | |
| unittest.mock.patch( | |
| "local_workflow._find_cas_tool", lambda: "cas_for_testing" | |
| ) | |
| ) | |
| @staticmethod | |
| def _make_cas_download_fake( | |
| content: str, | |
| ) -> Callable[[List[str]], subprocess.CompletedProcess]: | |
| def handler(cmd: List[str]) -> subprocess.CompletedProcess: | |
| dir_index = cmd.index("-dir") | |
| target_dir = cmd[dir_index + 1] | |
| os.makedirs(target_dir, exist_ok=True) | |
| with open(os.path.join(target_dir, "metrics.json"), "w") as f: | |
| f.write(content) | |
| return subprocess.CompletedProcess(cmd, 0, stdout="download ok") | |
| return handler | |
| @staticmethod | |
| def _make_bb_cas_fake( | |
| mapping: Dict[Tuple[str, ...], Behavior] | |
| ) -> Callable[[List[str]], subprocess.CompletedProcess]: | |
| lock = threading.Lock() | |
| def side_effect( | |
| cmd: List[str], **kwargs: Any | |
| ) -> subprocess.CompletedProcess: | |
| for patterns, behavior in mapping.items(): | |
| if all(p in cmd for p in patterns): | |
| if callable(behavior): | |
| # The code under test runs `cas` commands from | |
| # multiple threads. | |
| with lock: | |
| return behavior(cmd) | |
| elif isinstance(behavior, Exception): | |
| raise behavior | |
| elif isinstance(behavior, str): | |
| return subprocess.CompletedProcess( | |
| cmd, 0, stdout=behavior, stderr="" | |
| ) | |
| else: | |
| raise ValueError( | |
| f"Unexpected behavior type: {type(behavior)}" | |
| ) | |
| raise RuntimeError(f"No match found for command: {cmd}") | |
| return side_effect | |
| @staticmethod | |
| def _build_info( | |
| properties: Optional[Dict[str, Any]] = None, | |
| steps: Optional[List[Any]] = None, | |
| ) -> str: | |
| """Returns `bb get -fields output.properties,steps -json` output.""" | |
| return json.dumps( | |
| {"output": {"properties": properties or {}}, "steps": steps or []} | |
| ) | |
| @staticmethod | |
| def _task_step(instance: str, digest: str) -> Dict[str, str]: | |
| """Returns a build step that links to a task's CAS outputs.""" | |
| return { | |
| "summaryMarkdown": ( | |
| "* [task outputs](https://cas-viewer.appspot.com/" | |
| f"{instance}/blobs/{digest}/tree)" | |
| ) | |
| } | |
| @staticmethod | |
| def _make_cas_tree_fake( | |
| trees: Dict[str, Dict[str, str]] | |
| ) -> Callable[[List[str]], subprocess.CompletedProcess]: | |
| """Fakes `cas tree`, given {root digest: {path: file digest}}.""" | |
| def handler(cmd: List[str]) -> subprocess.CompletedProcess: | |
| digest = cmd[cmd.index("-digest") + 1] | |
| files = { | |
| path: {"digest": file_digest, "size": 1} | |
| for path, file_digest in trees[digest].items() | |
| } | |
| return subprocess.CompletedProcess( | |
| cmd, 0, stdout=json.dumps({"digest": digest, "files": files}) | |
| ) | |
| return handler | |
| @staticmethod | |
| def _make_cas_file_download_fake() -> ( | |
| Callable[[List[str]], subprocess.CompletedProcess] | |
| ): | |
| """Fakes `cas download -file` by writing '<instance>:<digest>'.""" | |
| def handler(cmd: List[str]) -> subprocess.CompletedProcess: | |
| instance = cmd[cmd.index("-cas-instance") + 1] | |
| digest = cmd[cmd.index("-digest") + 1] | |
| dest = cmd[cmd.index("-file") + 1] | |
| with open(dest, "w") as f: | |
| f.write(f"{instance}:{digest}") | |
| return subprocess.CompletedProcess(cmd, 0, stdout="") | |
| return handler | |
| @staticmethod | |
| def _read_dir_tree(root_dir: str) -> Dict[str, str]: | |
| """Returns {relative_path: file_contents} for all files under root_dir.""" | |
| files = {} | |
| for dirpath, _, filenames in os.walk(root_dir): | |
| for filename in filenames: | |
| full_path = os.path.join(dirpath, filename) | |
| rel_path = os.path.relpath(full_path, root_dir) | |
| with open(full_path) as f: | |
| files[rel_path] = f.read() | |
| return files | |
| def test_download_metrics(self): | |
| # Use unittest.TestCase.subTest rather than third_party/parameterized | |
| # so perfcompare_test.py remains runnable directly via python3 (e.g. | |
| # with --generate to update golden files) without requiring non-stdlib | |
| # packages to be installed or packaged via GN. | |
| for build_id in ("1000", "b1000"): | |
| with ( | |
| self.subTest(build_id=build_id), | |
| unittest.mock.patch("subprocess.run") as mock_run, | |
| ): | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "-fields"): self._build_info( | |
| { | |
| "cas_instance": "chromium_swarm", | |
| "perf_dataset_digest": "123abc/82", | |
| } | |
| ), | |
| ("download", "-dir"): self._make_cas_download_fake( | |
| '{"metric": 1.0}' | |
| ), | |
| } | |
| ) | |
| out_dir = self.MakeTempDir() | |
| stdout = io.StringIO() | |
| perfcompare.Main( | |
| ["download_metrics", build_id, out_dir], stdout | |
| ) | |
| self.assertEqual( | |
| self._read_dir_tree(out_dir), | |
| {"metrics.json": '{"metric": 1.0}'}, | |
| ) | |
| output = stdout.getvalue() | |
| self.assertIn("Fetching build info for 1000 using bb", output) | |
| self.assertIn("Downloading from cas to", output) | |
| bb_get_cmd = [ | |
| call[0][0] | |
| for call in mock_run.call_args_list | |
| if "get" in call[0][0] | |
| ][0] | |
| self.assertEqual(bb_get_cmd[-1], "1000") | |
| def test_normalize_build_id(self): | |
| self.assertEqual(local_workflow._normalize_build_id("1000"), "1000") | |
| self.assertEqual(local_workflow._normalize_build_id("b1000"), "1000") | |
| self.assertEqual(local_workflow._normalize_build_id("B1000"), "1000") | |
| self.assertEqual(local_workflow._normalize_build_id(" b1000 "), "1000") | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_metrics_perfcompare_digest(self, mock_run): | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "-fields"): self._build_info( | |
| { | |
| "cas_instance": "chromium_swarm", | |
| "perfcompare_dataset_digest": "123abc/82", | |
| } | |
| ), | |
| ("download", "-dir"): self._make_cas_download_fake( | |
| '{"metric": 1.0}' | |
| ), | |
| } | |
| ) | |
| out_dir = self.MakeTempDir() | |
| stdout = io.StringIO() | |
| perfcompare.Main(["download_metrics", "1000", out_dir], stdout) | |
| self.assertEqual( | |
| self._read_dir_tree(out_dir), {"metrics.json": '{"metric": 1.0}'} | |
| ) | |
| output = stdout.getvalue() | |
| self.assertIn("Fetching build info for 1000 using bb", output) | |
| self.assertIn("Downloading from cas to", output) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_metrics_missing_properties(self, mock_run): | |
| # A dataset digest without a cas_instance should fail rather than | |
| # guess an instance. | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "-fields"): self._build_info( | |
| {"perf_dataset_digest": "123abc/82"} | |
| ), | |
| } | |
| ) | |
| with self.assertRaises(RuntimeError) as cm: | |
| perfcompare.Main( | |
| ["download_metrics", "1000", self.MakeTempDir()], io.StringIO() | |
| ) | |
| self.assertIn("Download failed for build 1000", str(cm.exception)) | |
| self.assertIsInstance(cm.exception.__cause__, KeyError) | |
| self.assertIn("cas_instance", str(cm.exception.__cause__)) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_metrics_bb_get_fails(self, mock_run): | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "-fields"): subprocess.CalledProcessError( | |
| 1, ["bb", "get"], stderr="bb error" | |
| ), | |
| } | |
| ) | |
| with self.assertRaises(RuntimeError) as cm: | |
| perfcompare.Main( | |
| ["download_metrics", "1000", self.MakeTempDir()], io.StringIO() | |
| ) | |
| self.assertIn("Download failed for build 1000", str(cm.exception)) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_metrics_cas_download_fails(self, mock_run): | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "-fields"): self._build_info( | |
| { | |
| "cas_instance": "chromium_swarm", | |
| "perf_dataset_digest": "123abc/82", | |
| } | |
| ), | |
| ("download",): subprocess.CalledProcessError( | |
| 1, ["cas", "download"], stderr="cas error" | |
| ), | |
| } | |
| ) | |
| with self.assertRaises(RuntimeError) as cm: | |
| perfcompare.Main( | |
| ["download_metrics", "1000", self.MakeTempDir()], | |
| io.StringIO(), | |
| ) | |
| self.assertIn("Download failed for build 1000", str(cm.exception)) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_metrics_fallback_success(self, mock_run): | |
| # Set up a build with no perf_dataset_digest, a failed test in | |
| # output.properties, and three test task CAS trees in the build steps. | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "-fields"): self._build_info( | |
| { | |
| "failed_test_names": [ | |
| "fuchsia-pkg://fuchsia.com/failing#meta/failing.cm" | |
| ] | |
| }, | |
| [ | |
| {"name": "no summary"}, | |
| self._task_step("projects/p1/instances/i1", "aaaa/1"), | |
| self._task_step("projects/p1/instances/i1", "bbbb/2"), | |
| # A repeated link to the first task must not produce a | |
| # second boot directory. | |
| self._task_step("projects/p1/instances/i1", "aaaa/1"), | |
| self._task_step("projects/p2/instances/i2", "cccc/3"), | |
| ], | |
| ), | |
| ("tree",): self._make_cas_tree_fake( | |
| { | |
| "aaaa/1": { | |
| "out/test1/0/a2.fuchsiaperf.json": "a2/10", | |
| "out/test1/0/a1.fuchsiaperf.json": "a1/10", | |
| "out/test1/0/trace.fxt": "a3/1000000000", | |
| "out/summary.json": "a4/10", | |
| # Results from the failed test are excluded. | |
| # testrunner removes colons from the test name | |
| # and then URL-escapes it to name the directory. | |
| "out/fuchsia-pkg%2F%2Ffuchsia.com%2Ffailing%23meta%2Ffailing.cm/0/a5.fuchsiaperf.json": "a5/10", | |
| }, | |
| # A task without perf results is skipped so that | |
| # boot directories are numbered contiguously. | |
| "bbbb/2": {"out/test2/0/log.txt": "b1/10"}, | |
| "cccc/3": {"out/test3/0/c1.fuchsiaperf.json": "c1/10"}, | |
| } | |
| ), | |
| ("download", "-file"): self._make_cas_file_download_fake(), | |
| } | |
| ) | |
| out_dir = self.MakeTempDir() | |
| stdout = io.StringIO() | |
| perfcompare.Main(["download_metrics", "1000", out_dir], stdout) | |
| # Check that only *.fuchsiaperf.json files from passing tests were | |
| # downloaded, with the "out/" prefix stripped and grouped into | |
| # bootNNNNNN directories. | |
| self.assertEqual( | |
| self._read_dir_tree(out_dir), | |
| { | |
| os.path.join( | |
| "by_boot", "boot000000", "test1", "0", "a1.fuchsiaperf.json" | |
| ): "projects/p1/instances/i1:a1/10", | |
| os.path.join( | |
| "by_boot", "boot000000", "test1", "0", "a2.fuchsiaperf.json" | |
| ): "projects/p1/instances/i1:a2/10", | |
| os.path.join( | |
| "by_boot", "boot000001", "test3", "0", "c1.fuchsiaperf.json" | |
| ): "projects/p2/instances/i2:c1/10", | |
| }, | |
| ) | |
| output = stdout.getvalue() | |
| self.assertIn( | |
| "Excluding results from failed tests:\n" | |
| " fuchsia-pkg://fuchsia.com/failing#meta/failing.cm\n", | |
| output, | |
| ) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_metrics_fallback_no_task_outputs(self, mock_run): | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "-fields"): self._build_info( | |
| steps=[{"summaryMarkdown": "No CAS viewer URLs here"}] | |
| ), | |
| } | |
| ) | |
| with self.assertRaises(RuntimeError) as cm: | |
| perfcompare.Main( | |
| ["download_metrics", "1000", self.MakeTempDir()], io.StringIO() | |
| ) | |
| self.assertIn("Download failed for build 1000", str(cm.exception)) | |
| self.assertIn( | |
| "No task outputs found in build steps for 1000", | |
| str(cm.exception.__cause__), | |
| ) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_metrics_fallback_no_perf_results(self, mock_run): | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "-fields"): self._build_info( | |
| steps=[self._task_step("projects/p/instances/i", "aaaa/1")] | |
| ), | |
| ("tree",): self._make_cas_tree_fake( | |
| {"aaaa/1": {"out/log.txt": "a1/10"}} | |
| ), | |
| } | |
| ) | |
| out_dir = self.MakeTempDir() | |
| with self.assertRaises(RuntimeError) as cm: | |
| perfcompare.Main( | |
| ["download_metrics", "1000", out_dir], io.StringIO() | |
| ) | |
| self.assertIn( | |
| "No .fuchsiaperf.json files found in task outputs for build 1000", | |
| str(cm.exception.__cause__), | |
| ) | |
| self.assertEqual(os.listdir(out_dir), []) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_metrics_fallback_by_boot_already_exists(self, mock_run): | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "-fields"): self._build_info( | |
| steps=[self._task_step("projects/p/instances/i", "aaaa/1")] | |
| ), | |
| } | |
| ) | |
| out_dir = self.MakeTempDir() | |
| old_file = os.path.join(out_dir, "by_boot", "boot000000", "old.txt") | |
| os.makedirs(os.path.dirname(old_file)) | |
| with open(old_file, "w") as f: | |
| f.write("old data") | |
| with self.assertRaises(RuntimeError) as cm: | |
| perfcompare.Main( | |
| ["download_metrics", "1000", out_dir], io.StringIO() | |
| ) | |
| self.assertIn("already exists", str(cm.exception.__cause__)) | |
| # Check that the existing directory was left untouched. | |
| self.assertEqual( | |
| self._read_dir_tree(out_dir), | |
| {os.path.join("by_boot", "boot000000", "old.txt"): "old data"}, | |
| ) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_metrics_fallback_cas_download_fails(self, mock_run): | |
| def failing_download(cmd: List[str]) -> subprocess.CompletedProcess: | |
| dest = cmd[cmd.index("-file") + 1] | |
| if cmd[cmd.index("-digest") + 1] == "a2/10": | |
| raise subprocess.CalledProcessError(1, cmd) | |
| with open(dest, "w") as f: | |
| f.write("partial") | |
| return subprocess.CompletedProcess(cmd, 0, stdout="") | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "-fields"): self._build_info( | |
| steps=[self._task_step("projects/p/instances/i", "aaaa/1")] | |
| ), | |
| ("tree",): self._make_cas_tree_fake( | |
| { | |
| "aaaa/1": { | |
| "out/a1.fuchsiaperf.json": "a1/10", | |
| "out/a2.fuchsiaperf.json": "a2/10", | |
| } | |
| } | |
| ), | |
| ("download", "-file"): failing_download, | |
| } | |
| ) | |
| out_dir = self.MakeTempDir() | |
| with self.assertRaises(RuntimeError) as cm: | |
| perfcompare.Main( | |
| ["download_metrics", "1000", out_dir], io.StringIO() | |
| ) | |
| self.assertIn("Download failed for build 1000", str(cm.exception)) | |
| # Check that neither a partial by_boot directory nor the staging | |
| # directory is left behind. | |
| self.assertEqual(os.listdir(out_dir), []) | |
| def test_extract_task_outputs_from_steps(self): | |
| steps = [ | |
| {"summaryMarkdown": "No url here"}, | |
| {"otherField": "No summaryMarkdown"}, | |
| { | |
| "summaryMarkdown": ( | |
| "* [task outputs](https://cas-viewer.appspot.com/projects/p2/instances/i2/blobs/5678/20/tree)\n" | |
| "* [task outputs](https://cas-viewer.api.luci.app/projects/p1/instances/i1/blobs/ABCD12/10/tree)" | |
| ) | |
| }, | |
| { | |
| # The same task, linked from a different host. | |
| "summaryMarkdown": ( | |
| "* [task outputs](https://cas-viewer.api.luci.app/projects/p1/instances/i1/blobs/ABCD12/10/tree)" | |
| ) | |
| }, | |
| { | |
| # The same digest in a different instance is kept. | |
| "summaryMarkdown": ( | |
| "* [task outputs](https://cas-viewer.appspot.com/projects/p2/instances/i2/blobs/ABCD12/10/tree)" | |
| ) | |
| }, | |
| { | |
| # Links to other CAS uploads, such as the aggregated perf | |
| # dataset, are ignored. | |
| "summaryMarkdown": ( | |
| "* [CAS UI](https://cas-viewer.appspot.com/projects/p3/instances/i3/blobs/EEEE/82/tree)\n" | |
| "* [task cas outputs: shard](https://cas-viewer.appspot.com/projects/p3/instances/i3/blobs/FFFF/10/tree)" | |
| ) | |
| }, | |
| ] | |
| self.assertEqual( | |
| local_workflow._extract_task_outputs_from_steps(steps), | |
| [ | |
| ("projects/p1/instances/i1", "ABCD12/10"), | |
| ("projects/p2/instances/i2", "5678/20"), | |
| ("projects/p2/instances/i2", "ABCD12/10"), | |
| ], | |
| ) | |
| self.assertEqual( | |
| local_workflow._extract_task_outputs_from_steps([]), [] | |
| ) | |
| def test_parallel_map(self): | |
| self.assertEqual( | |
| local_workflow._parallel_map(lambda x: x * x, range(20)), | |
| [x * x for x in range(20)], | |
| ) | |
| def fail_on_3(x: int) -> int: | |
| if x == 3: | |
| raise ValueError("boom") | |
| return x | |
| with self.assertRaisesRegex(ValueError, "boom"): | |
| local_workflow._parallel_map(fail_on_3, range(10)) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_baseline_success(self, mock_run): | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| # Order matters: check for specific 'get' with '-fields' | |
| # before general 'get'. | |
| ("get", "-fields"): self._build_info( | |
| { | |
| "cas_instance": "chromium_swarm", | |
| "perf_dataset_digest": "123abc/82", | |
| } | |
| ), | |
| ("get",): json.dumps({"status": "SUCCESS"}), | |
| ("download",): self._make_cas_download_fake('{"metric": 1.0}'), | |
| } | |
| ) | |
| out_dir = self.MakeTempDir() | |
| stdout = io.StringIO() | |
| perfcompare.Main( | |
| [ | |
| "download_baseline", | |
| "--state_file", | |
| self.state_file, | |
| "builder2", | |
| out_dir, | |
| ], | |
| stdout, | |
| ) | |
| expected_file = os.path.join(out_dir, "metrics.json") | |
| self.assertTrue(os.path.exists(expected_file)) | |
| with open(expected_file) as f: | |
| self.assertEqual(f.read(), '{"metric": 1.0}') | |
| output = stdout.getvalue() | |
| self.assertIn("1003 (SUCCESS) - Triggering download", output) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_baseline_fallback(self, mock_run): | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get", "1002"): json.dumps({"status": "FAILURE"}), | |
| # Order matters: check for specific 'get' with '-fields' | |
| # before general 'get'. | |
| ("get", "1001", "-fields"): self._build_info( | |
| { | |
| "cas_instance": "chromium_swarm", | |
| "perf_dataset_digest": "123abc/82", | |
| } | |
| ), | |
| ("get", "1001"): json.dumps({"status": "SUCCESS"}), | |
| ("download",): self._make_cas_download_fake('{"metric": 1.0}'), | |
| } | |
| ) | |
| out_dir = self.MakeTempDir() | |
| stdout = io.StringIO() | |
| perfcompare.Main( | |
| [ | |
| "download_baseline", | |
| "--state_file", | |
| self.state_file, | |
| "builder1", | |
| out_dir, | |
| ], | |
| stdout, | |
| ) | |
| expected_file = os.path.join(out_dir, "metrics.json") | |
| self.assertTrue(os.path.exists(expected_file)) | |
| with open(expected_file) as f: | |
| self.assertEqual(f.read(), '{"metric": 1.0}') | |
| output = stdout.getvalue() | |
| self.assertIn("1002 (FAILURE) - Skipping", output) | |
| self.assertIn("1001 (SUCCESS) - Triggering download", output) | |
| @unittest.mock.patch("subprocess.run") | |
| def test_download_baseline_no_successful_build(self, mock_run): | |
| mock_run.side_effect = self._make_bb_cas_fake( | |
| { | |
| ("auth-info",): "auth ok", | |
| ("get",): json.dumps({"status": "FAILURE"}), | |
| } | |
| ) | |
| out_dir = self.MakeTempDir() | |
| stdout = io.StringIO() | |
| perfcompare.Main( | |
| [ | |
| "download_baseline", | |
| "--state_file", | |
| self.state_file, | |
| "builder1", | |
| out_dir, | |
| ], | |
| stdout, | |
| ) | |
| output = stdout.getvalue() | |
| self.assertIn("1002 (FAILURE) - Skipping", output) | |
| self.assertIn("1001 (FAILURE) - Skipping", output) | |
| self.assertIn("No successful sample build IDs found", output) | |
| def test_extract_metrics(self): | |
| data_dir = self.MakeTempDir() | |
| WriteJsonFile( | |
| os.path.join(data_dir, "test.fuchsiaperf.json"), | |
| [ | |
| { | |
| "test_suite": "fuchsia.test", | |
| "label": "metric1", | |
| "values": [10.5], | |
| }, | |
| { | |
| "test_suite": "fuchsia.test", | |
| "label": "metric2", | |
| "values": [20.0], | |
| }, | |
| { | |
| "test_suite": "other.suite", | |
| "label": "metric3", | |
| "values": [30.0], | |
| }, | |
| ], | |
| ) | |
| stdout = io.StringIO() | |
| perfcompare.Main(["extract_metrics", "fuchsia", data_dir], stdout) | |
| output = stdout.getvalue() | |
| self.assertIn("fuchsia.test,metric1,10.5", output) | |
| self.assertIn("fuchsia.test,metric2,20.0", output) | |
| self.assertNotIn("other.suite", output) | |
| def test_extract_metrics_aggregation(self): | |
| data_dir = self.MakeTempDir() | |
| WriteJsonFile( | |
| os.path.join(data_dir, "test1.fuchsiaperf.json"), | |
| [ | |
| { | |
| "test_suite": "fuchsia.test", | |
| "label": "metric1", | |
| "values": [10.5], | |
| } | |
| ], | |
| ) | |
| WriteJsonFile( | |
| os.path.join(data_dir, "test2.fuchsiaperf.json"), | |
| [ | |
| { | |
| "test_suite": "fuchsia.test", | |
| "label": "metric1", | |
| "values": [15.0], | |
| } | |
| ], | |
| ) | |
| stdout = io.StringIO() | |
| perfcompare.Main(["extract_metrics", "fuchsia", data_dir], stdout) | |
| stdout.seek(0) | |
| reader = csv.reader(stdout) | |
| rows = list(reader) | |
| self.assertEqual(rows, [["fuchsia.test", "metric1", "10.5", "15.0"]]) | |
| def test_extract_metrics_empty_values(self): | |
| data_dir = self.MakeTempDir() | |
| WriteJsonFile( | |
| os.path.join(data_dir, "test.fuchsiaperf.json"), | |
| [ | |
| { | |
| "test_suite": "fuchsia.test", | |
| "label": "metric1", | |
| "values": [], | |
| } | |
| ], | |
| ) | |
| with self.assertRaises(statistics.StatisticsError): | |
| perfcompare.Main( | |
| ["extract_metrics", "fuchsia", data_dir], io.StringIO() | |
| ) | |
| def test_extract_metrics_malformed_json(self): | |
| data_dir = self.MakeTempDir() | |
| file_path = os.path.join(data_dir, "bad.fuchsiaperf.json") | |
| with open(file_path, "w") as f: | |
| f.write("{invalid}") | |
| with self.assertRaises(RuntimeError) as cm: | |
| perfcompare.Main( | |
| ["extract_metrics", "fuchsia", data_dir], io.StringIO() | |
| ) | |
| self.assertIn("Malformed JSON", str(cm.exception)) | |
| if __name__ == "__main__": | |
| TestMain() |