| # Copyright 2019 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """ fuchsia.py - Builds and optionally tests Fuchsia. |
| |
| # Execution overview |
| |
| ## Configuration |
| |
| This recipe uses a protocol buffer message called a spec for most of its |
| configuration. The only PROPERTIES are those required to acquire the spec. |
| The recipe fetches the spec from the git repo |spec_remote|. It determines |
| the correct revision to use from the BuildBucket build input to ensure it |
| retrieves the correct config for a pending change vs a committed change. |
| |
| ## Checkout + Build |
| |
| This recipe triggers a child build which runs the fuchsia/build recipe. |
| That recipe checks out the source code and builds it. This recipe |
| retrieves the data required to orchestrate tests via Isolate. |
| |
| ## Test |
| |
| If configured to run tests, this recipe uses the test orchestration data to run tests. |
| That logic is in the testing recipe module. Under the hood, that module |
| triggers Swarming tasks that do the actual testing, waits for them, and |
| reports the results. |
| """ |
| |
| from google.protobuf import json_format |
| from google.protobuf import text_format |
| |
| from recipe_engine.recipe_api import Property |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2 |
| from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2 |
| from PB.go.chromium.org.luci.buildbucket.proto import rpc as rpc_pb2 |
| from PB.infra.fuchsia import Fuchsia |
| from PB.recipe_modules.recipe_engine.led.properties import InputProperties as LedInputProperties |
| |
| DEPS = [ |
| 'fuchsia/artifacts', |
| 'fuchsia/build', |
| 'fuchsia/build_input_resolver', |
| 'fuchsia/buildbucket_util', |
| 'fuchsia/checkout', |
| 'fuchsia/fuchsia', |
| 'fuchsia/gitiles', |
| 'fuchsia/spec', |
| 'fuchsia/testing', |
| 'fuchsia/testing_requests', |
| 'fuchsia/testsharder', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/file', |
| 'recipe_engine/isolated', |
| 'recipe_engine/json', |
| 'recipe_engine/led', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/step', |
| 'recipe_engine/swarming', |
| ] |
| |
| PROPERTIES = { |
| 'child_build_id': |
| Property( |
| kind=str, |
| help=('The buildbucket ID of the child build. If set, ' |
| 'will use this build instead of launching a new one.'), |
| default=None), |
| 'spec_remote': |
| Property( |
| kind=str, |
| help='URL of the specs git repository', |
| default='http://fuchsia.googlesource.com/integration'), |
| } |
| |
| |
| def RunSteps(api, child_build_id, spec_remote): |
| # At some stage BuildBucket stores properties as google.protobuf.Value, |
| # which converts all numbers (including ints) to floats, which is |
| # lossy, so we have to use a string properties and convert to int |
| # internally. |
| child_build_id = int(child_build_id) if child_build_id else None |
| |
| spec, spec_revision = api.fuchsia.setup_with_spec(spec_remote) |
| |
| orchestrator_id = api.buildbucket_util.id |
| if not spec.build.run_tests: |
| raise api.step.InfraFailure( |
| 'if not running tests, use the fuchsia/build recipe directly') |
| |
| with api.step.nest('build') as presentation: |
| child_build = run_build_steps(api, presentation, child_build_id, |
| spec_revision, orchestrator_id) |
| child_props = json_format.MessageToDict(child_build.output.properties) |
| orchestration_inputs = collect_test_orchestration_inputs(api, child_props) |
| # Copy to our own properties so the results uploader in google3 can find |
| # it without knowing about the child. |
| rev_count_prop = api.checkout.REVISION_COUNT_PROPERTY |
| if rev_count_prop in child_props: |
| presentation.properties[rev_count_prop] = child_props[rev_count_prop] |
| |
| # Configure context of uploaded artifacts for test task construction. |
| api.artifacts.gcs_bucket = spec.artifact_gcs_bucket |
| api.artifacts.uuid = orchestrator_id |
| |
| run_test_steps(api, orchestration_inputs, spec) |
| |
| |
| def run_build_steps(api, presentation, child_build_id, spec_revision, |
| orchestrator_id): |
| builder_name = '{}-subbuild'.format(api.buildbucket.build.builder.builder) |
| if child_build_id: |
| # Text is meant to avoid confusion. |
| presentation.step_text = 'Reusing child build instead of triggering' |
| output_build = api.buildbucket.get(child_build_id) |
| build_url = 'https://ci.chromium.org/b/%d' % child_build_id |
| else: |
| parent_properties = api.properties.thaw() |
| # These are reserved by kitchen and swarming. See |
| # https://chromium.googlesource.com/infra/infra/+/2c2389a00fcdb93d90a628f941814f2abd34428e/go/src/infra/tools/kitchen/cook.go#266 |
| # and https://chromium.googlesource.com/infra/infra/+/7fcd559afa7a866a5ad039019e6ef6a91922e09c/appengine/cr-buildbucket/validation.py#36. |
| # We also should not override the 'recipe' of the child builder. |
| reject_keys = { |
| '$recipe_engine/path', '$recipe_engine/step', 'bot_id', 'path_config', |
| 'buildbucket', '$recipe_engine/buildbucket', 'buildername', 'branch', |
| 'repository', '$recipe_engine/runtime', 'recipe' |
| } |
| properties = { |
| key: val |
| for key, val in parent_properties.items() |
| if key and key not in reject_keys |
| } |
| properties.update({ |
| 'spec_revision': spec_revision, |
| 'parent_id': orchestrator_id, |
| }) |
| # If this task was launched by led, we launch the child with led as well. |
| # This lets us ensure that the parent and child use the same version of |
| # the recipes code. That is a requirement for testing this recipe, as well as |
| # for avoiding the need to do soft transitions when updating the interface |
| # between the parent and child recipes. |
| if api.led.launched_by_led: |
| output_build, build_url = build_with_led(api, builder_name, properties, |
| presentation) |
| else: |
| output_build, build_url = build_with_buildbucket(api, builder_name, |
| properties) |
| presentation.links[builder_name] = build_url |
| if output_build.status == common_pb2.INFRA_FAILURE: |
| raise api.step.InfraFailure('[build](%s) raised infra failure' % build_url) |
| elif output_build.status != common_pb2.SUCCESS: |
| raise api.step.StepFailure('[build](%s) failed' % build_url) |
| return output_build |
| |
| |
| def build_with_led(api, builder_name, properties, presentation): |
| parent = api.buildbucket.build.builder |
| led_data = api.led( |
| 'get-builder', |
| 'luci.%s.%s:%s' % (parent.project, parent.bucket, builder_name)) |
| edit_args = [] |
| for k, v in properties.items(): |
| edit_args.extend(['-p', '%s=%s' % (k, api.json.dumps(v))]) |
| led_data = led_data.then('edit', *edit_args) |
| bb_input = api.buildbucket.build_input |
| if bb_input.gerrit_changes: |
| gerrit_change = bb_input.gerrit_changes[0] |
| led_data = led_data.then( |
| 'edit-cr-cl', 'https://%s/c/%s/+/%d' % |
| (gerrit_change.host, gerrit_change.project, gerrit_change.change)) |
| led_data = api.led.inject_input_recipes(led_data) |
| launch_res = led_data.then('launch') |
| task_id = launch_res.result['swarming']['task_id'] |
| swarming_result = api.swarming.collect( |
| 'collect', [task_id], output_dir=api.path['cleanup'])[0] |
| # Led launch ensures this file is present in the task root dir. |
| build_proto_path = swarming_result.output_dir.join('build.proto.json') |
| build_proto_json = api.file.read_text('read build.proto.json', |
| build_proto_path) |
| build_proto = build_pb2.Build() |
| presentation.logs['build.proto.json'] = build_proto_json.splitlines() |
| json_format.Parse(build_proto_json, build_proto) |
| |
| url = 'https://ci.chromium.org/swarming/task/%s?server=%s' % ( |
| task_id, launch_res.result['swarming']['host_name']) |
| return build_proto, url |
| |
| |
| def build_with_buildbucket(api, builder_name, properties): |
| req = api.buildbucket.schedule_request( |
| builder=builder_name, |
| properties=properties, |
| swarming_parent_run_id=api.swarming.task_id, |
| priority=None, # Leave unset to avoid overriding priority from configs. |
| ) |
| build_id = api.buildbucket.schedule([req], step_name='schedule')[0].id |
| # Consumed by rerun recipe. |
| with api.step.nest('child build id') as presentation: |
| # Passing build IDs directly as properties doesn't work because all |
| # numbers get cast to floats, which is lossy, so we convert to str. |
| presentation.properties['child_build_id'] = str(build_id) |
| # As of 2019-11-18, timeout defaults to something too short. |
| # We never want this step to time out. We'd rather the whole build time out. |
| builds_dict = api.buildbucket.collect_builds([build_id], |
| timeout=24 * 60 * 60, |
| step_name='collect') |
| return builds_dict[build_id], 'https://ci.chromium.org/b/%s' % build_id |
| |
| |
| def run_test_steps(api, orchestration_inputs, spec): |
| tryjob = api.buildbucket_util.is_tryjob |
| |
| if spec.test.test_in_shards: |
| testing_tasks = api.testing.test_in_shards( |
| collect_timeout_secs=spec.test.collect_timeout_secs, |
| debug_symbol_url=api.artifacts.debug_symbol_url(), |
| orchestration_inputs=orchestration_inputs, |
| max_attempts=spec.test.max_attempts, |
| rerun_budget_secs=spec.test.rerun_budget_secs) |
| else: |
| testing_tasks = [ |
| api.testing.deprecated_test( |
| api.artifacts.debug_symbol_url(), |
| spec.test.device_type, |
| orchestration_inputs, |
| max_attempts=spec.test.max_attempts, |
| rerun_budget_secs=spec.test.rerun_budget_secs) |
| ] |
| |
| all_results = [] |
| final_results = [] |
| for t in testing_tasks: |
| for attempt in t.attempts: |
| if attempt.test_results: |
| all_results.append(attempt.test_results) |
| if attempt.index == len(t.attempts) - 1: |
| final_results.append(attempt.test_results) |
| |
| # Upload test results |
| if spec.test.upload_results: |
| assert spec.gcs_bucket, 'gcs_bucket must be set if test.upload_results is' |
| with api.step.nest('upload test results') as presentation: |
| link = 'go/fuchsia-result-store/bid:%s' % api.buildbucket_util.id |
| presentation.links[link] = link.replace('go/', 'https://goto.google.com/') |
| swarming_task_ids = [] |
| # All test results, including non-final attempts that were retried. |
| # We care so we can analyze flakiness. |
| for test_results in all_results: |
| test_results.upload_results( |
| gcs_bucket=spec.gcs_bucket, |
| upload_to_catapult=(not tryjob and spec.test.upload_to_catapult)) |
| swarming_task_ids.append(test_results.swarming_task_id) |
| # Consumed by the google3 results uploader. |
| presentation.properties['test-swarming-task-ids'] = swarming_task_ids |
| |
| if 'profile' in spec.build.variants: |
| api.testing.process_coverage( |
| covargs_path=orchestration_inputs.covargs, |
| test_results=[ |
| # TODO(fxb/27336): coverage is only supported for tests on Fuchsia. |
| result for result in all_results if result.from_fuchsia |
| ], |
| debug_symbol_url=api.artifacts.debug_symbol_url(), |
| llvm_profdata=orchestration_inputs.llvm_profdata, |
| llvm_cov=orchestration_inputs.llvm_cov, |
| gcs_bucket=spec.gcs_bucket, |
| ) |
| |
| # Raise test failures |
| with api.step.defer_results(): |
| api.testing.raise_failures() |
| # Only for the final attempt of each testing task, otherwise this |
| # defeats the point of retries. |
| for test_results in final_results: |
| test_results.raise_failures() |
| |
| |
| def collect_test_orchestration_inputs(api, build_props): |
| """Downloads isolated orchestration inputs from a build. |
| |
| Args: |
| build_props (dict): The properties of the build that produced the test |
| orchestration inputs. |
| |
| Returns: |
| FuchsiaBuildApi.TestOrchestrationInputs |
| |
| Raises: |
| A StepFailure if the required HASH_PROPERTY is not found. |
| """ |
| prop_name = api.build.TestOrchestrationInputs.HASH_PROPERTY |
| orchestration_inputs_hash = build_props.get(prop_name) |
| if not orchestration_inputs_hash: |
| raise api.step.StepFailure('no `%s` property found' % prop_name) |
| return api.build.TestOrchestrationInputs.download(api, |
| orchestration_inputs_hash) |
| |
| |
| def GenTests(api): |
| |
| def ci_build_message(api, output_props=None, **kwargs): |
| """Generates a Buildbucket Build message. |
| |
| Args: |
| output_props (Dict): output properties to set on the build. |
| kwargs: Forwarded to BuildbucketApi.ci_build_message. |
| |
| See BuildBucketTestApi.ci_build_message for full parameter documentation. |
| """ |
| msg = api.buildbucket.ci_build_message(**kwargs) |
| msg.output.properties.update(output_props if output_props else {}) |
| return msg |
| |
| def child_build_steps(api, build): |
| """Generates step data to schedule and collect from a child build |
| |
| Args: |
| build (build_pb2.Build): The build to schedule and collect from. |
| """ |
| mock_schedule_data = api.buildbucket.simulated_schedule_output( |
| step_name='build.schedule', |
| batch_response=rpc_pb2.BatchResponse( |
| responses=[dict(schedule_build=dict(id=build.id))],), |
| ) |
| mock_collect_data = api.buildbucket.simulated_collect_output( |
| step_name='build.collect', |
| builds=[build], |
| ) |
| return mock_schedule_data + mock_collect_data |
| |
| def child_led_steps(api, build, tryjob=False): |
| """Generates step data to schedule and collect from a child build |
| |
| Args: |
| build (build_pb2.Build): The build to schedule and collect from. |
| """ |
| props = { |
| 'gcs_bucket': api.fuchsia.DEFAULT_GCS_BUCKET, |
| 'parent_id': build.id |
| } |
| change_prop = { |
| '$recipe_engine/buildbucket': { |
| 'build': { |
| 'input': { |
| 'gerritChanges': { |
| 'change': '12345', |
| 'host': 'fuchsia-review.googlesource.com', |
| 'project': 'fuchsia' |
| } |
| } |
| } |
| } |
| } |
| led_data = ( |
| api.led.get_builder(api, 'build.led get-builder') |
| .edit_properties('build.led edit', **props) |
| ) # yapf: disable |
| if tryjob: |
| led_data.edit_properties('build.led edit-cr-cl', **change_prop) |
| led_data.edit_input_recipes( |
| 'build.led edit (2)', |
| isolated_hash='new hash').launch('build.led launch') |
| return led_data.step_data + api.step_data( |
| 'build.read build.proto.json', |
| api.file.read_text(json_format.MessageToJson(build))) |
| |
| def download_step_data(legacy_qemu): |
| task_request_jsonish = api.testing.task_request_jsonish( |
| legacy_qemu=legacy_qemu) |
| return api.testing.task_requests_step_data( |
| [task_request_jsonish], |
| 'build.download test orchestration inputs.load task requests', |
| ) |
| |
| def test_step_data(test_in_shards=True): |
| if test_in_shards: |
| shard_name = 'QEMU' |
| legacy_qemu = False |
| outputs = ['out/path/to/output/file'] |
| else: |
| shard_name = None |
| legacy_qemu = True |
| outputs = ['output.fs'] |
| return download_step_data(legacy_qemu) + ( |
| api.testing.task_retry_step_data([ |
| api.swarming.task_result( |
| id='610', |
| name='QEMU', |
| outputs=outputs, |
| ), |
| ]) + api.testing.test_step_data( |
| shard_name=shard_name, legacy_qemu=legacy_qemu)) |
| |
| def spec_data(use_snapshot=False, |
| variants=(), |
| device_type='QEMU', |
| run_tests=True, |
| test_in_shards=True, |
| gcs_bucket=None): |
| test_spec = None |
| if run_tests: |
| test_spec = Fuchsia.Test( |
| device_type=device_type, |
| max_shard_size=0, |
| timeout_secs=30 * 60, |
| pool='fuchsia.tests', |
| test_in_shards=test_in_shards, |
| swarming_expiration_timeout_secs=10 * 60, |
| swarming_io_timeout_secs=5 * 60, |
| upload_results=bool(gcs_bucket), |
| use_runtests=True, |
| ) |
| |
| return api.spec.spec_loaded_ok( |
| step_name='load spec.build_init', |
| message=Fuchsia( |
| checkout=Fuchsia.Checkout( |
| manifest='manifest', |
| remote='remote', |
| upload_results=bool(gcs_bucket), |
| use_snapshot=use_snapshot, |
| ), |
| build=Fuchsia.Build( |
| variants=variants, |
| build_type='debug', |
| run_tests=run_tests, |
| board='boards/x64.gni', |
| product='products/core.gni', |
| target='x64', |
| include_breakpad_symbols=False, |
| upload_results=bool(gcs_bucket), |
| ), |
| test=test_spec, |
| gcs_bucket=gcs_bucket, |
| artifact_gcs_bucket=gcs_bucket, |
| ), |
| ) |
| |
| ci_build = api.buildbucket.ci_build( |
| project='fuchsia', git_repo='https://fuchsia.googlesource.com/fuchsia') |
| |
| yield api.fuchsia.test( |
| 'successful_build_and_test', |
| steps=[ |
| child_build_steps( |
| api=api, |
| build=ci_build_message( |
| api=api, |
| output_props={ |
| 'integration-revision-count': 1, |
| 'test_orchestration_inputs_hash': 'abc', |
| }, |
| status='SUCCESS', |
| ), |
| ), |
| api.override_step_data( |
| 'launch/collect.0.collect', |
| api.swarming.collect([ |
| api.swarming.task_result( |
| id='610', |
| name='QEMU', |
| outputs=['out/path/to/output/file'], |
| ), |
| ])), |
| ]) + spec_data( |
| gcs_bucket='gcs-bucket', variants=('profile',)) + test_step_data() |
| |
| child_build = ci_build_message( |
| api=api, |
| output_props={ |
| 'integration-revision-count': 1, |
| 'test_orchestration_inputs_hash': 'abc', |
| }, |
| status='SUCCESS', |
| ) |
| yield api.fuchsia.test( |
| 'child_build_provided__test_not_in_shards', |
| steps=[ |
| api.buildbucket.simulated_get( |
| child_build, step_name='build.buildbucket.get'), |
| ]) + spec_data( |
| gcs_bucket='gcs-bucket', test_in_shards=False) + test_step_data( |
| test_in_shards=False) + api.properties( |
| child_build_id=str(child_build.id)) |
| |
| yield api.fuchsia.test( |
| 'build_only_failed', |
| status='infra_failure', |
| ) + spec_data(run_tests=False) |
| |
| yield api.fuchsia.test( |
| 'build_failed', |
| status='failure', |
| steps=[ |
| child_build_steps( |
| api=api, |
| build=ci_build_message( |
| api=api, |
| output_props={ |
| 'integration-revision-count': 1, |
| 'test_orchestration_inputs_hash': 'abc', |
| }, |
| status='FAILURE', |
| ), |
| ), |
| ]) + spec_data( |
| gcs_bucket='gcs-bucket', variants=('profile',)) |
| |
| yield api.fuchsia.test( |
| 'build_infra_failure', |
| status='infra_failure', |
| steps=[ |
| child_build_steps( |
| api=api, |
| build=ci_build_message( |
| api=api, |
| output_props={ |
| 'integration-revision-count': 1, |
| 'test_orchestration_inputs_hash': 'abc', |
| }, |
| status='INFRA_FAILURE', |
| ), |
| ), |
| ]) + spec_data( |
| gcs_bucket='gcs-bucket', variants=('profile',)) |
| |
| yield api.fuchsia.test( |
| 'build_with_led', |
| status='failure', |
| properties={ |
| '$recipe_engine/led': |
| LedInputProperties( |
| led_run_id='led/user_example.com/abc123', |
| isolated_input=LedInputProperties.IsolatedInput( |
| hash='abc123', |
| namespace='default-gzip', |
| server='isolateserver.appspot.com')), |
| }, |
| steps=[ |
| child_led_steps( |
| api=api, |
| build=ci_build_message( |
| api=api, |
| output_props={'test_orchestration_inputs_hash': 'abc'}, |
| status='FAILURE', |
| ), |
| ), |
| ]) + spec_data() |
| |
| yield api.fuchsia.test( |
| 'build_with_led_tryjob', |
| status='failure', |
| properties={ |
| '$recipe_engine/led': |
| LedInputProperties( |
| led_run_id='led/user_example.com/abc123', |
| isolated_input=LedInputProperties.IsolatedInput( |
| hash='abc123', |
| namespace='default-gzip', |
| server='isolateserver.appspot.com')), |
| }, |
| tryjob=True, |
| steps=[ |
| child_led_steps( |
| api=api, |
| tryjob=True, |
| build=ci_build_message( |
| api=api, |
| output_props={'test_orchestration_inputs_hash': 'abc'}, |
| status='FAILURE', |
| ), |
| ), |
| ]) + api.build_input_resolver.set_gerrit_branch() + api.gitiles.refs( |
| 'refs', [ |
| 'refs/heads/master', |
| 'deadbeef', |
| ]) + spec_data() |
| |
| yield api.fuchsia.test( |
| 'build_passed_but_hash_is_missing', |
| status='failure', |
| steps=[ |
| child_build_steps( |
| api=api, |
| build=ci_build_message( |
| api=api, |
| status='SUCCESS', |
| ), |
| ) |
| ]) + spec_data() |
| |
| yield (api.test('spec_parse_error') + ci_build + |
| api.spec.spec_parse_error(step_name='load spec.build_init')) |
