[benchmarks] Factor out duplication in use of ssh.run() + processResults()
Introduce runTestCommand() function for the common code.
Also add a cleanup step that removes the performance results file from
the Fuchsia device under test.
Disable the "cascade_invocations" Dart lint warning/error because it
complains about the use of _tests.add(), and cascading these would be
less readable.
Bug: 68471
Test: fx set terminal.x64 --with src/tests/end_to_end/perf:tests
+ run the individual tests such as this one:
fx test --e2e fidlc_microbenchmarks_test
Change-Id: Iaeacd7a263eac077bd3b6319493c03ea96a7a82c
Reviewed-on: https://fuchsia-review.googlesource.com/c/fuchsia/+/514062
Commit-Queue: Mark Seaborn <mseaborn@google.com>
Reviewed-by: Nathan Rogers <nathanrogers@google.com>
diff --git a/src/tests/end_to_end/perf/analysis_options.yaml b/src/tests/end_to_end/perf/analysis_options.yaml
index c288793..58e32da 100644
--- a/src/tests/end_to_end/perf/analysis_options.yaml
+++ b/src/tests/end_to_end/perf/analysis_options.yaml
@@ -3,3 +3,7 @@
# found in the LICENSE file.
include: ../../../../topaz/tools/analysis_options.yaml
+
+linter:
+ rules:
+ cascade_invocations: false
diff --git a/src/tests/end_to_end/perf/test/archivist_benchmarks_test.dart b/src/tests/end_to_end/perf/test/archivist_benchmarks_test.dart
index 1a532ad..f4f1d32 100644
--- a/src/tests/end_to_end/perf/test/archivist_benchmarks_test.dart
+++ b/src/tests/end_to_end/perf/test/archivist_benchmarks_test.dart
@@ -6,25 +6,12 @@
import 'helpers.dart';
-String tmpPerfResultsJson(String benchmarkBinary) {
- return '/tmp/perf_results_$benchmarkBinary.json';
-}
-
-void runBenchmark(String benchmarkBinary, String resultsFile) {
- final command = '/bin/$benchmarkBinary $resultsFile';
- test(benchmarkBinary, () async {
- final helper = await PerfTestHelper.make();
- var result = await helper.sl4fDriver.ssh.run(command);
- expect(result.exitCode, equals(0));
- await helper.processResults(resultsFile);
- result = await helper.sl4fDriver.ssh.run('rm $resultsFile');
- expect(result.exitCode, equals(0));
- }, timeout: Timeout.none);
-}
-
void main() {
enableLoggingOutput();
- runBenchmark(
- 'archivist_benchmarks', tmpPerfResultsJson('archivist_benchmarks'));
+ test('archivist_benchmarks', () async {
+ final helper = await PerfTestHelper.make();
+ await helper.runTestCommand(
+ (resultsFile) => '/bin/archivist_benchmarks $resultsFile');
+ }, timeout: Timeout.none);
}
diff --git a/src/tests/end_to_end/perf/test/audio_mixer_profiler_test.dart b/src/tests/end_to_end/perf/test/audio_mixer_profiler_test.dart
index fa55398..2706b37 100644
--- a/src/tests/end_to_end/perf/test/audio_mixer_profiler_test.dart
+++ b/src/tests/end_to_end/perf/test/audio_mixer_profiler_test.dart
@@ -11,10 +11,7 @@
test('fuchsia.audio.mixer', () async {
final helper = await PerfTestHelper.make();
- const resultsFile = '/tmp/perf_results.json';
- final result = await helper.sl4fDriver.ssh
- .run('/bin/audio_mixer_profiler --perftest-json=$resultsFile');
- expect(result.exitCode, equals(0));
- await helper.processResults(resultsFile);
+ await helper.runTestCommand((resultsFile) =>
+ '/bin/audio_mixer_profiler --perftest-json=$resultsFile');
}, timeout: Timeout.none);
}
diff --git a/src/tests/end_to_end/perf/test/diagnostics_benchmarks_test.dart b/src/tests/end_to_end/perf/test/diagnostics_benchmarks_test.dart
index ffb3253..09d6b26 100644
--- a/src/tests/end_to_end/perf/test/diagnostics_benchmarks_test.dart
+++ b/src/tests/end_to_end/perf/test/diagnostics_benchmarks_test.dart
@@ -6,25 +6,12 @@
import 'helpers.dart';
-String tmpPerfResultsJson(String benchmarkBinary) {
- return '/tmp/perf_results_$benchmarkBinary.json';
-}
-
-void runBenchmark(String benchmarkBinary, String resultsFile) {
- final command = '/bin/$benchmarkBinary $resultsFile';
- test(benchmarkBinary, () async {
- final helper = await PerfTestHelper.make();
- var result = await helper.sl4fDriver.ssh.run(command);
- expect(result.exitCode, equals(0));
- await helper.processResults(resultsFile);
- result = await helper.sl4fDriver.ssh.run('rm $resultsFile');
- expect(result.exitCode, equals(0));
- }, timeout: Timeout.none);
-}
-
void main() {
enableLoggingOutput();
- runBenchmark(
- 'selectors_benchmarks', tmpPerfResultsJson('selectors_benchmarks'));
+ test('selectors_benchmarks', () async {
+ final helper = await PerfTestHelper.make();
+ await helper.runTestCommand(
+ (resultsFile) => '/bin/selectors_benchmarks $resultsFile');
+ }, timeout: Timeout.none);
}
diff --git a/src/tests/end_to_end/perf/test/fidl_microbenchmarks_test.dart b/src/tests/end_to_end/perf/test/fidl_microbenchmarks_test.dart
index 8b0324f..cac29b3 100644
--- a/src/tests/end_to_end/perf/test/fidl_microbenchmarks_test.dart
+++ b/src/tests/end_to_end/perf/test/fidl_microbenchmarks_test.dart
@@ -16,19 +16,6 @@
return '/tmp/perf_results_$benchmarkBinary.json';
}
-void runFidlBenchmark(String benchmarkBinary, String args) {
- final resultsFile = tmpPerfResultsJson(benchmarkBinary);
- final path = '/bin/$benchmarkBinary $args';
- _tests.add(() {
- test(benchmarkBinary, () async {
- final helper = await PerfTestHelper.make();
- final result = await helper.sl4fDriver.ssh.run(path);
- expect(result.exitCode, equals(0));
- await helper.processResults(resultsFile);
- }, timeout: Timeout.none);
- });
-}
-
// Runs a benchmark that uses the C++ perftest runner.
// It is believed that benchmarks converge to different means in different
// process runs (and reboots). Since each of these benchmarks are currently
@@ -57,17 +44,29 @@
void main(List<String> args) {
enableLoggingOutput();
- runFidlBenchmark('go_fidl_microbenchmarks',
- '--out_file ${tmpPerfResultsJson('go_fidl_microbenchmarks')}');
runPerftestFidlBenchmark('hlcpp_fidl_microbenchmarks');
runPerftestFidlBenchmark('lib_fidl_microbenchmarks');
runPerftestFidlBenchmark('llcpp_fidl_microbenchmarks');
runPerftestFidlBenchmark('walker_fidl_microbenchmarks');
- runFidlBenchmark('rust_fidl_microbenchmarks',
- tmpPerfResultsJson('rust_fidl_microbenchmarks'));
runPerftestFidlBenchmark('reference_fidl_microbenchmarks');
_tests.add(() {
+ test('go_fidl_microbenchmarks', () async {
+ final helper = await PerfTestHelper.make();
+ await helper.runTestCommand((resultsFile) =>
+ '/bin/go_fidl_microbenchmarks --out_file $resultsFile');
+ }, timeout: Timeout.none);
+ });
+
+ _tests.add(() {
+ test('rust_fidl_microbenchmarks', () async {
+ final helper = await PerfTestHelper.make();
+ await helper.runTestCommand(
+ (resultsFile) => '/bin/rust_fidl_microbenchmarks $resultsFile');
+ }, timeout: Timeout.none);
+ });
+
+ _tests.add(() {
test('dart_fidl_microbenchmarks', () async {
final helper = await PerfTestHelper.make();
const resultsFile =
diff --git a/src/tests/end_to_end/perf/test/fidlc_microbenchmarks_test.dart b/src/tests/end_to_end/perf/test/fidlc_microbenchmarks_test.dart
index e7fe3af..38daf55 100644
--- a/src/tests/end_to_end/perf/test/fidlc_microbenchmarks_test.dart
+++ b/src/tests/end_to_end/perf/test/fidlc_microbenchmarks_test.dart
@@ -11,10 +11,7 @@
test('/bin/fidlc_microbenchmarks', () async {
final helper = await PerfTestHelper.make();
- const resultsFile = '/tmp/perf_results_fidlc_microbenchmarks';
- final result = await helper.sl4fDriver.ssh
- .run('/bin/fidlc_microbenchmarks -p --quiet --out $resultsFile');
- expect(result.exitCode, equals(0));
- await helper.processResults(resultsFile);
+ await helper.runTestCommand((resultsFile) =>
+ '/bin/fidlc_microbenchmarks -p --quiet --out $resultsFile');
}, timeout: Timeout.none);
}
diff --git a/src/tests/end_to_end/perf/test/helpers.dart b/src/tests/end_to_end/perf/test/helpers.dart
index be37aae..ba5e342 100644
--- a/src/tests/end_to_end/perf/test/helpers.dart
+++ b/src/tests/end_to_end/perf/test/helpers.dart
@@ -95,6 +95,35 @@
await performance.convertResults('runtime_deps/catapult_converter',
jsonSummaryFile, Platform.environment);
}
+
+ // Runs a command over SSH and publishes its output as performance
+ // test results.
+ //
+ // The command to run is specified via a function that takes a
+ // filename as an argument and returns a shell command string. The
+ // filename is for the results file that the command will write its
+ // results to, in fuchsiaperf.json format.
+ Future<void> runTestCommand(
+ String Function(String resultsFilename) getCommand) async {
+ // Make a filename that is very likely to be unique. Using a
+ // unique filename should not be strictly necessary, but it should
+ // avoid potential problems. We do not expect performance tests
+ // to be run concurrently on the Infra builders, but it may be
+ // useful to do so locally for development purposes when we don't
+ // care about the performance results.
+ final timestamp = DateTime.now().microsecondsSinceEpoch;
+ final resultsFile = '/tmp/perf_results_$timestamp.fuchsiaperf.json';
+ final command = getCommand(resultsFile);
+ final result = await sl4fDriver.ssh.run(command);
+ expect(result.exitCode, equals(0));
+ try {
+ await processResults(resultsFile);
+ } finally {
+ // Clean up: remove the temporary file.
+ final result = await sl4fDriver.ssh.run('rm -f $resultsFile');
+ expect(result.exitCode, equals(0));
+ }
+ }
}
final _log = Logger('TouchInputLatencyMetricsProcessor');
diff --git a/src/tests/end_to_end/perf/test/kernel_boot_stats_test.dart b/src/tests/end_to_end/perf/test/kernel_boot_stats_test.dart
index 5f80cb0..a6ea7e3 100644
--- a/src/tests/end_to_end/perf/test/kernel_boot_stats_test.dart
+++ b/src/tests/end_to_end/perf/test/kernel_boot_stats_test.dart
@@ -11,10 +11,7 @@
test('fuchsia.kernel.boot', () async {
final helper = await PerfTestHelper.make();
- const resultsFile = '/tmp/perf_results.json';
- final result =
- await helper.sl4fDriver.ssh.run('/bin/kernel-boot-stats $resultsFile');
- expect(result.exitCode, equals(0));
- await helper.processResults(resultsFile);
+ await helper
+ .runTestCommand((resultsFile) => '/bin/kernel-boot-stats $resultsFile');
}, timeout: Timeout.none);
}
