| # Copyright 2018 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for comparing performance between two revisions of Fuchsia.""" |
| |
| from recipe_engine.config import Enum, List, Single |
| from recipe_engine.recipe_api import Property |
| |
| TARGETS = ['arm64', 'x64'] |
| |
| BUILD_TYPES = ['debug', 'release', 'thinlto', 'lto'] |
| |
| DEVICES = [ |
| 'QEMU', |
| 'Intel NUC Kit NUC6i3SYK', |
| 'Intel NUC Kit NUC7i5DNHE', |
| 'Khadas Vim2 Max', |
| ] |
| |
| DEPS = [ |
| 'fuchsia/build', |
| 'fuchsia/checkout', |
| 'fuchsia/fuchsia', |
| 'fuchsia/testing', |
| 'fuchsia/testing_requests', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/python', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/step', |
| 'recipe_engine/time', |
| ] |
| |
| PROPERTIES = { |
| 'project': |
| Property(kind=str, help='Jiri remote manifest project', default=None), |
| 'manifest': |
| Property(kind=str, help='Jiri manifest to use'), |
| 'remote': |
| Property(kind=str, help='Remote manifest repository'), |
| 'target': |
| Property(kind=Enum(*TARGETS), help='Target to build'), |
| 'build_type': |
| Property( |
| kind=Enum(*BUILD_TYPES), help='The build type', default='debug'), |
| 'packages': |
| Property(kind=List(basestring), help='Packages to build', default=[]), |
| 'variants': |
| Property( |
| kind=List(basestring), |
| help='--variant arguments to GN in `select_variant`', |
| default=[]), |
| 'gn_args': |
| Property( |
| kind=List(basestring), help='Extra args to pass to GN', default=[]), |
| 'ninja_targets': |
| Property( |
| kind=List(basestring), |
| help='Extra target args to pass to ninja', |
| default=[]), |
| 'board': |
| Property(kind=str, help='Board to build', default=None), |
| 'product': |
| Property(kind=str, help='Product to build', default=None), |
| 'test_pool': |
| Property( |
| kind=str, |
| help='Swarming pool from which a test task will be drawn', |
| default='fuchsia.tests'), |
| 'device_type': |
| Property( |
| kind=str, |
| help='The type of device to execute tests on, if the value is' |
| ' not QEMU it will be passed to Swarming as the device_type' |
| ' dimension', |
| default='QEMU'), |
| 'pave': |
| Property( |
| kind=bool, |
| help='Whether to pave images the device for testing. (Ignored if' |
| ' device_type == QEMU)', |
| default=True), |
| 'test_timeout_secs': |
| Property( |
| kind=Single((int, float)), |
| help='How long to wait until timing out on tests', |
| default=40 * 60), |
| # This property is not intended to be set by a config. It is just here |
| # so that a test case can pass a smaller number in order to reduce the |
| # size of the generated test expectations. |
| 'boots_per_revision': |
| Property( |
| kind=int, |
| help='Number of boots of Fuchsia to run performance tests on,' |
| ' in order to deal with cross-boot variation of performance', |
| default=5), |
| 'debug_symbol_gcs_bucket': |
| Property( |
| kind=str, |
| help='GCS bucket to upload to and read debug symbols from'), |
| } |
| |
| |
| def git_rev_parse(api, desc, git_dir, rev): |
| with api.context(cwd=git_dir): |
| result = api.step( |
| desc, ['git', 'rev-parse', rev], stdout=api.raw_io.output()) |
| return result.stdout.strip() |
| |
| |
| def git_checkout(api, git_dir, rev): |
| with api.context(cwd=git_dir): |
| api.step('git checkout', ['git', 'checkout', rev]) |
| |
| |
| # Run "git log" on the repo of the change being tested, to list the most |
| # recent commits there. This is to help developers debug the bot run and |
| # understand what it is testing. The change will often be rebased onto |
| # tip-of-tree; the step records what the change was rebased onto. |
| def git_log(api, git_dir): |
| with api.context(cwd=git_dir): |
| api.step('git log for debugging', ['git', 'log', '--max-count=10']) |
| |
| |
| def RunSteps(api, project, manifest, remote, target, build_type, packages, |
| variants, gn_args, ninja_targets, test_pool, device_type, pave, |
| board, product, test_timeout_secs, boots_per_revision, |
| debug_symbol_gcs_bucket): |
| checkout_root = api.path['start_dir'].join('fuchsia') |
| checkout = api.checkout.fuchsia_with_options( |
| path=checkout_root, |
| build=api.buildbucket.build, |
| manifest=manifest, |
| remote=remote, |
| project=project, |
| ) |
| |
| snapshot_target = 'obj/build/images/system.snapshot' |
| ninja_targets = list(ninja_targets) |
| ninja_targets.append(snapshot_target) |
| |
| test_cmds = [ |
| ' '.join([ |
| '/pkgfs/packages/fuchsia_benchmarks/0/bin/benchmarks_perfcompare.sh', |
| api.testing_requests.results_dir_on_target |
| ]) |
| ] |
| |
| # This builds Fuchsia at the given revision and kicks off a swarming task |
| # to run the tests. It returns a function that, when called, will wait |
| # for the swarming job to complete and return the test results. This |
| # division into two parts allows swarming tasks to run concurrently. |
| def test_version(before_or_after, git_rev): |
| with api.step.nest('build and launch tests for "%s" revision' % |
| before_or_after): |
| git_checkout(api, checkout.root_dir, git_rev) |
| build_dir = checkout.root_dir.join('out') |
| build = api.build.with_options( |
| build_dir=build_dir, |
| checkout=checkout, |
| target=target, |
| build_type=build_type, |
| packages=packages, |
| variants=variants, |
| gn_args=gn_args, |
| ninja_targets=ninja_targets, |
| board=board, |
| product=product, |
| ) |
| if debug_symbol_gcs_bucket: |
| build.upload_debug_symbols( |
| debug_symbol_gcs_bucket=debug_symbol_gcs_bucket) |
| # Make a copy of the binary size data. |
| snapshot_file = build.fuchsia_build_dir.join(snapshot_target) |
| copied_file = checkout.root_dir.join('snapshot_%s' % before_or_after) |
| api.step('copy binary size data', ['cp', snapshot_file, copied_file]) |
| # Copy the results data into the logs. |
| api.step('binary size data', ['cat', snapshot_file]) |
| |
| build_artifacts = build.get_artifacts() |
| shard_requests = api.testing_requests.deprecated_shard_requests( |
| build_artifacts, test_cmds, device_type, test_pool, test_timeout_secs, |
| pave) |
| orchestration_inputs = api.build.TestOrchestrationInputs( |
| build_artifacts.llvm_symbolizer, build_artifacts.minfs, |
| build_artifacts.symbolize_tool, shard_requests, |
| build_artifacts.tests_file) |
| collect_funcs = [] |
| for boot_idx in xrange(boots_per_revision): |
| with api.step.nest('boot %d of %d' % |
| (boot_idx + 1, boots_per_revision)): |
| collect_funcs.append( |
| api.testing.deprecated_test_async( |
| debug_symbol_gcs_bucket, |
| device_type, |
| orchestration_inputs, |
| overwrite_summary=False)) |
| |
| def finish_func(): |
| with api.step.nest('collect results for "%s" revision' % before_or_after): |
| results_dir = checkout.root_dir.join('perf_results_%s' % |
| before_or_after) |
| by_boot_dir = results_dir.join('by_boot') |
| api.step('make results directory', ['mkdir', '-p', by_boot_dir]) |
| for boot_idx, collect_func in enumerate(collect_funcs): |
| with api.step.nest('boot %d of %d' % |
| (boot_idx + 1, boots_per_revision)): |
| test_results = collect_func() |
| # Make a copy of the perf test results directory. |
| api.step('copy perf test results', [ |
| 'cp', '-r', test_results.results_dir, |
| by_boot_dir.join('boot%06d' % boot_idx) |
| ]) |
| |
| return { |
| 'binary_size_data_file': copied_file, |
| 'perf_test_results_dir': results_dir |
| } |
| |
| return finish_func |
| |
| rev_before = git_rev_parse(api, 'get "before" revision', checkout.root_dir, |
| 'HEAD^') |
| rev_after = git_rev_parse(api, 'get "after" revision', checkout.root_dir, |
| 'HEAD') |
| git_log(api, checkout.root_dir) |
| # Build the "before" and "after" revisions and launch swarming tasks to |
| # test them. Building will happen sequentially, but the swarming tasks |
| # will run concurrently. |
| finish_before = test_version('before', rev_before) |
| finish_after = test_version('after', rev_after) |
| results_before = finish_before() |
| results_after = finish_after() |
| # Runs with the checkout in the "after" state. |
| perfcompare_tool = checkout.root_dir.join( |
| 'garnet/bin/perfcompare/perfcompare.py') |
| api.python('compare binary sizes before and after', perfcompare_tool, [ |
| 'compare_sizes', results_before['binary_size_data_file'], |
| results_after['binary_size_data_file'] |
| ]) |
| # Output the perf results dataset as a single file so that it can be |
| # easily downloaded. |
| api.python('generate raw_perf_dataset.json', perfcompare_tool, [ |
| 'make_combined_perf_dataset_file', |
| results_before['perf_test_results_dir'], |
| results_after['perf_test_results_dir'] |
| ]) |
| api.python('compare perf test results before and after', perfcompare_tool, [ |
| 'compare_perf', results_before['perf_test_results_dir'], |
| results_after['perf_test_results_dir'] |
| ]) |
| |
| |
| def GenTests(api): |
| # Since the recipe above runs Fuchsia twice (for the "before" and "after" |
| # builds), adjust the dummy output to contain two sets of results. |
| def testing_steps(): |
| steps = [] |
| task_result_step = api.testing.task_step_data( |
| [ |
| api.fuchsia.m.swarming.task_result( |
| # Include serial.txt to cover the processing of the serial log, |
| # since that codepath is used in production and has broken in |
| # the past. |
| id='1', |
| name='test', |
| outputs=['output.fs', 'serial.txt']), |
| ], |
| enable_retries=False, |
| ) |
| steps.append(task_result_step) |
| for rev in ('before', 'after'): |
| for boot in (1, 2): |
| base_name = 'collect results for "%s" revision.boot %d of 2' % (rev, |
| boot) |
| task_result_step.step_data[ |
| base_name + '.collect'] = task_result_step.step_data['collect'] |
| # Include summary.json in the mocked contents of the output archive to |
| # get better coverage of codepaths in the testing recipe module used by |
| # this recipe. |
| summary_data = { |
| 'tests': [{ |
| 'name': 'perfcompare_benchmark.catapult_json', |
| 'result': 'SUCCESS', |
| }] |
| } |
| steps.append( |
| api.step_data( |
| base_name + '.extract results', |
| api.raw_io.output_dir({ |
| 'summary.json': api.json.dumps(summary_data), |
| }), |
| )) |
| del task_result_step.step_data['collect'] |
| return steps |
| |
| yield api.fuchsia.test( |
| 'successful_run', |
| # Pass a smaller value than the default for boots_per_revision to |
| # reduce the size of the test expectations output, but use a number |
| # >1 in order to test multiple boots. |
| properties=dict( |
| run_tests=True, |
| boots_per_revision=2, |
| debug_symbol_gcs_bucket='debug-symbols', |
| ), |
| clear_default_steps=True, |
| steps=testing_steps(), |
| ) |
