| # Copyright 2017 The Fuchsia Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Recipe for building Fuchsia and running tests.""" |
| |
| from google.protobuf import text_format as textpb |
| |
| from recipe_engine.config import Enum, List, Set, Single |
| from recipe_engine.recipe_api import Property |
| |
| from PB.infra.fuchsia import Fuchsia |
| |
| TARGETS = ['arm64', 'x64'] |
| |
| BUILD_TYPES = ['debug', 'release', 'thinlto', 'lto'] |
| |
| BUILD_ARTIFACT_HASH_PROPERTY = 'build_artifact_hash' |
| |
| TEST_MULTIPLIER_KEY = 'MULTIPLY' |
| |
| DEPS = [ |
| 'fuchsia/artifacts', |
| 'fuchsia/build', |
| 'fuchsia/build_input_resolver', |
| 'fuchsia/buildbucket_util', |
| 'fuchsia/checkout', |
| 'fuchsia/experimental', |
| 'fuchsia/fuchsia', |
| 'fuchsia/git', |
| 'fuchsia/gitiles', |
| 'fuchsia/gsutil', |
| 'fuchsia/hash', |
| 'fuchsia/jiri', |
| 'fuchsia/jsonutil', |
| 'fuchsia/tar', |
| 'fuchsia/testsharder', |
| 'fuchsia/testing', |
| 'fuchsia/testing_requests', |
| 'fuchsia/spec', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/isolated', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/platform', |
| 'recipe_engine/properties', |
| 'recipe_engine/python', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/step', |
| 'recipe_engine/swarming', |
| 'recipe_engine/url', |
| ] |
| |
| PROPERTIES = { |
| 'spec_remote': |
| Property( |
| kind=str, |
| help='URL of the specs git repository', |
| default='http://fuchsia.googlesource.com/integration'), |
| } |
| |
| |
| def RunSteps(api, spec_remote): |
| tryjob = api.buildbucket_util.is_tryjob |
| |
| # Resolve the build input to always contain a Gitiles commit. |
| bb_build = api.buildbucket.build |
| api.build_input_resolver.resolve( |
| bb_build.input, |
| default_project_url='https://fuchsia.googlesource.com/fuchsia') |
| bb_input = bb_build.input |
| with api.step.nest('got_revision') as presentation: |
| # Read by the binary-size Gerrit plugin |
| presentation.properties['got_revision'] = bb_input.gitiles_commit.id |
| |
| with api.step.nest('load spec') as bootstrap_step: |
| try: |
| bootstrap_step.presentation.step_text = 'loading spec' |
| spec = api.spec.load_from_build( |
| build=bb_build, |
| spec_remote=spec_remote, |
| Type=Fuchsia, |
| ) |
| except api.spec.ParseError as e: |
| raise api.step.StepFailure('failed to parse spec: %s' % str(e)) |
| |
| # Log the textproto to logdog so we can programmatically scrape spec files |
| # to check into the tree. |
| bootstrap_step.presentation.logs['textproto'] = textpb.MessageToString( |
| spec).split('\n') |
| set_output_properties_from_spec(bootstrap_step, spec) |
| |
| # Handle illegal settings around secrets. |
| if spec.test.requires_secrets: |
| if tryjob or spec.test.device_type != 'QEMU': # pragma: no cover |
| raise api.step.InfraFailure( |
| 'the secrets pipeline is only supported in CI and on QEMU') |
| |
| checkout_root = api.path['start_dir'].join('fuchsia') |
| if spec.checkout.use_snapshot: |
| if bb_input.gerrit_changes: |
| assert len(bb_input.gerrit_changes) == 1 |
| checkout = api.checkout.from_patchset_snapshot( |
| path=checkout_root, |
| gerrit_change=bb_input.gerrit_changes[0], |
| attributes=spec.checkout.attributes) |
| else: |
| checkout = api.checkout.from_commit_snapshot( |
| path=checkout_root, |
| gitiles_commit=bb_input.gitiles_commit, |
| attributes=spec.checkout.attributes) |
| |
| else: |
| assert spec.checkout.manifest |
| assert spec.checkout.remote |
| checkout = api.checkout.fuchsia_with_options( |
| path=checkout_root, |
| build=bb_build, |
| manifest=spec.checkout.manifest, |
| remote=spec.checkout.remote, |
| project=spec.checkout.project, |
| rebase_revision=bb_input.gitiles_commit.id, |
| is_release_version=spec.checkout.is_release_version, |
| attributes=spec.checkout.attributes, |
| ) |
| |
| if spec.checkout.upload_results: |
| assert spec.gcs_bucket, ( |
| 'gcs_bucket must be set if checkout.upload_results is') |
| checkout.upload_results(spec.gcs_bucket) |
| |
| # We can only derive repo from buildbucket when triggered in CI or CQ; |
| # validate the checkout then. |
| # Moreover, skip validation if checking out a snapshot, as that may have been |
| # triggered by a project not corresponding to a repo in the checkout. |
| triggered_by_ci_cq = ( |
| len(bb_input.gerrit_changes) > 0 or bb_input.gitiles_commit.project) |
| if triggered_by_ci_cq and not spec.checkout.use_snapshot: |
| if bb_input.gerrit_changes: |
| project = bb_input.gerrit_changes[0].project |
| else: |
| project = bb_input.gitiles_commit.project |
| with api.step.nest('validate checkout'), api.context(cwd=checkout.root_dir): |
| repo_path = api.jiri.project(projects=[project]).json.output[0]['path'] |
| api.python( |
| 'validate FIDL namespaces', |
| checkout.root_dir.join('scripts', 'style', |
| 'verify-fidl-libraries.py'), |
| args=['--repo', repo_path], |
| ) |
| |
| # Check for properly formatted JSON in commit message before build |
| multipliers_path = None |
| if bb_input.gerrit_changes and spec.test.test_in_shards: |
| with api.step.nest('test multipliers'): |
| # Get project dir for gerrit change from source manifest |
| gerrit_change = bb_input.gerrit_changes[0] |
| project_dir = checkout.root_dir |
| repo_url = 'https://%s/%s' % (gerrit_change.host.replace( |
| '-review', ''), gerrit_change.project) |
| dirs = checkout.source_manifest['directories'] |
| for d in dirs: |
| if dirs[d]['git_checkout']['repo_url'] == repo_url: |
| if d != '.': |
| project_dir = checkout.root_dir.join(d) |
| break |
| |
| with api.context(cwd=project_dir): |
| commit_msg = api.git.get_commit_message(name='get commit msg') |
| multipliers = api.jsonutil.extract_from_text( |
| 'extract', |
| text=commit_msg, |
| key=TEST_MULTIPLIER_KEY, |
| default_contents=[]) |
| multipliers_path = api.path.mkstemp() |
| api.file.write_json('write', multipliers_path, multipliers, indent=2) |
| |
| collect_build_metrics = spec.build.upload_results |
| archives_to_build = [] |
| # If we are not building images, then we need not build package-related |
| # targets. |
| if not spec.build.exclude_images: |
| archives_to_build.extend(['archive', 'package-archive']) |
| if spec.build.include_breakpad_symbols: |
| archives_to_build.append('breakpad-symbol-archive') |
| if spec.build.include_symbol_archive: |
| archives_to_build.append('symbol-archive') |
| |
| build_dir = checkout.root_dir.join('out') |
| |
| # Add infratools to universe packages and ninja_targets to ensure they are built. |
| spec.build.universe_packages.append('//bundles:infratools') |
| spec.build.ninja_targets.append('bundles:infratools') |
| |
| build = api.build.with_options( |
| build_dir=build_dir, |
| checkout=checkout, |
| target=str(spec.build.target), |
| build_type=str(spec.build.build_type), |
| packages=spec.build.packages, |
| universe_packages=spec.build.universe_packages, |
| variants=spec.build.variants, |
| gn_args=spec.build.gn_args, |
| ninja_targets=list(spec.build.ninja_targets), |
| board=spec.build.board, |
| product=spec.build.product, |
| collect_build_metrics=collect_build_metrics, |
| build_images=not spec.build.exclude_images, |
| archives_to_build=tuple(archives_to_build), |
| gcs_bucket=spec.gcs_bucket, |
| ) |
| |
| check_sizes_result = api.step( |
| 'check sizes', [ |
| build.tool('size_checker'), '--build-dir', build.fuchsia_build_dir, |
| '--sizes-json-out', |
| api.json.output() |
| ], |
| step_test_data=lambda: api.json.test_api.output({'some-file': 123})) |
| |
| # This property is read by the binary-size Gerrit plugin. |
| if check_sizes_result.json.output: |
| check_sizes_result.presentation.properties[ |
| 'binary_sizes'] = check_sizes_result.json.output |
| |
| if spec.debug_symbol_gcs_bucket: |
| build.upload_debug_symbols( |
| debug_symbol_gcs_bucket=spec.debug_symbol_gcs_bucket, |
| gcs_bucket=spec.gcs_bucket if spec.build.upload_results else None, |
| ) |
| |
| if spec.build.upload_results: |
| assert spec.gcs_bucket, 'gcs_bucket must be set if build.upload_results is' |
| build.upload_results(gcs_bucket=spec.gcs_bucket) |
| |
| shards = [] |
| if spec.test.test_in_shards: |
| shards = api.testsharder.execute( |
| 'create test shards', |
| testsharder_path=build.tool('testsharder'), |
| build_dir=build.fuchsia_build_dir, |
| max_shard_size=spec.test.max_shard_size, |
| multipliers=multipliers_path, |
| tags=spec.build.environment_tags, |
| ) |
| |
| build_artifacts = build.get_artifacts(shards) |
| |
| api.artifacts.gcs_bucket = spec.artifact_gcs_bucket |
| api.artifacts.uuid = api.buildbucket_util.id |
| |
| orchestration_inputs = None |
| if spec.build.run_tests: |
| if spec.test.test_in_shards: |
| shard_requests = api.testing_requests.shard_requests( |
| build_artifacts, |
| bb_build, |
| spec.test.per_test_timeout_secs, |
| spec.test.pool, |
| spec.test.swarming_expiration_timeout_secs, |
| spec.test.swarming_io_timeout_secs, |
| spec.test.use_runtests, |
| timeout_secs=spec.test.timeout_secs) |
| else: |
| shard_requests = api.testing_requests.deprecated_shard_requests( |
| build_artifacts, |
| api.testing_requests.deprecated_test_cmds(spec.test), |
| spec.test.device_type, |
| spec.test.pool, |
| spec.test.timeout_secs, |
| spec.test.pave, |
| requires_secrets=spec.test.requires_secrets, |
| swarming_expiration_timeout_secs=spec.test |
| .swarming_expiration_timeout_secs, |
| swarming_io_timeout_secs=spec.test.swarming_io_timeout_secs, |
| ) |
| |
| orchestration_inputs = api.build.TestOrchestrationInputs( |
| build_artifacts.llvm_symbolizer, build_artifacts.minfs, |
| build_artifacts.symbolize_tool, shard_requests, |
| build_artifacts.tests_file) |
| |
| # Must be done after testing.shard_requests() is called, because that |
| # modifies the filesystem images. TODO(garymm,joshuaseaton): once legacy_qemu |
| # code paths are removed, remove this comment as it will become false. |
| api.artifacts.upload('upload artifacts', build=build) |
| |
| all_results = [] |
| if spec.build.run_tests: |
| if spec.test.test_in_shards: |
| all_results = api.testing.test_in_shards( |
| collect_timeout_secs=spec.test.collect_timeout_secs, |
| debug_symbol_gcs_bucket=spec.debug_symbol_gcs_bucket, |
| orchestration_inputs=orchestration_inputs, |
| max_attempts=spec.test.max_attempts) |
| else: |
| all_results = [ |
| api.testing.deprecated_test( |
| spec.debug_symbol_gcs_bucket, |
| spec.test.device_type, |
| orchestration_inputs, |
| max_attempts=spec.test.max_attempts) |
| ] |
| |
| if spec.test.upload_results: |
| assert spec.gcs_bucket, 'gcs_bucket must be set if test.upload_results is' |
| for test_results in all_results: |
| test_results.upload_results( |
| gcs_bucket=spec.gcs_bucket, |
| # Mixing data from and pre-submit and post-submit in catapult results |
| # in a confusing UI, so always exclude pre-submit (tryjob). We set |
| # upload_to_catapult to True for some tryjobs because it makes |
| # generating our configs easier. |
| upload_to_catapult=(not tryjob and spec.test.upload_to_catapult), |
| ) |
| |
| if 'profile' in build_artifacts.variants: |
| api.testing.process_coverage( |
| covargs_path=build_artifacts.covargs, |
| test_results=[ |
| # TODO(fxb/27336): coverage is only supported for tests on Fuchsia. |
| result for result in all_results if result.from_fuchsia |
| ], |
| ids_txt=build_artifacts.ids, |
| llvm_profdata=build_artifacts.llvm_profdata, |
| llvm_cov=build_artifacts.llvm_cov, |
| gcs_bucket=spec.gcs_bucket, |
| ) |
| |
| # Raise test failures |
| with api.step.defer_results(): |
| api.testing.raise_failures() |
| for test_results in all_results: |
| test_results.raise_failures() |
| |
| if spec.build.enforce_size_limits: |
| build.check_filesystem_sizes() |
| |
| |
| def set_output_properties_from_spec(step, spec): |
| """Sets certain properties from the spec as output properties on this recipe. |
| |
| Some builders read data from these output properties, and fail if they are not |
| present. This is to prevent them from failing. |
| """ |
| step.presentation.properties['gcs_bucket'] = spec.gcs_bucket |
| |
| |
| def GenTests(api): |
| ci_build = api.buildbucket.ci_build( |
| project='fuchsia', |
| git_repo='https://fuchsia.googlesource.com/fuchsia', |
| ) |
| |
| def try_build(project='fuchsia'): |
| return api.buildbucket.try_build( |
| project=project, |
| git_repo='https://fuchsia.googlesource.com/%s' % project, |
| ) |
| |
| # Mock gitiles refs fetched when the buildbucket build input is an unresolved |
| # Gerrit change or Gitiles commit. |
| default_gitiles_refs_steps = api.gitiles.refs('refs', [ |
| 'refs/heads/master', |
| 'deadbeef', |
| ]) |
| |
| def source_manifest_for_cq(use_snapshot=False): |
| step_name = 'checkout.jiri source-manifest' |
| return api.step_data( |
| step_name, |
| api.json.output( |
| { |
| 'directories': { |
| '.': { |
| 'git_checkout': { |
| 'repo_url': |
| 'https://fuchsia.googlesource.com/fuchsia', |
| 'revision': |
| '4c2b0da3c06341db5cebe4d02c78c93c3b2bd78b', |
| } |
| }, |
| 'garnet': { |
| 'git_checkout': { |
| 'repo_url': |
| 'https://fuchsia.googlesource.com/garnet', |
| 'revision': |
| '4c2b0da3c06341db5cebe4d02c78c93c3b2bd78b' |
| } |
| } |
| } |
| }, |
| name='source manifest')) |
| |
| def test_step_data(deprecated=False): |
| if deprecated: |
| return api.testing.task_step_data([ |
| api.swarming.task_result( |
| id='610', |
| name='QEMU', |
| outputs=['out.tar'], |
| ), |
| ]) |
| |
| return ( |
| api.testing.task_retry_step_data( |
| [ |
| api.swarming.task_result( |
| id='610', |
| name='QEMU', |
| outputs=['out.tar'], |
| ), |
| ], |
| iteration=0) + |
| api.testing_requests.shards_step_data( |
| step_name='create test shards', |
| shards=[ |
| api.testsharder.shard( |
| name='QEMU', |
| tests=api.testing_requests.default_tests(), |
| dimensions=dict(device_type='QEMU'), |
| ), |
| ]) + |
| api.testing.test_step_data(shard_name='QEMU')) # yapf: disable |
| |
| def spec_data(use_snapshot=False, |
| variants=(), |
| build_target='x64', |
| build_type='debug', |
| include_symbol_archive=False, |
| include_breakpad_symbols=False, |
| enforce_size_limits=False, |
| max_shard_size=0, |
| run_tests=True, |
| test_in_shards=True, |
| requires_secrets=False, |
| gcs_bucket=None, |
| debug_symbol_gcs_bucket='debug-symbols'): |
| test_spec = None |
| if run_tests: |
| test_spec = Fuchsia.Test( |
| max_shard_size=max_shard_size, |
| timeout_secs=30 * 60, |
| pool='fuchsia.tests', |
| test_in_shards=test_in_shards, |
| requires_secrets=requires_secrets, |
| # We should never expire a test task. This is currently 5 hours, but |
| # should be treated as infinite. |
| swarming_expiration_timeout_secs=18000, |
| swarming_io_timeout_secs=5 * 60, |
| upload_results=bool(gcs_bucket), |
| use_runtests='profile' in variants, |
| per_test_timeout_secs=5 * 60, |
| ) |
| if not test_in_shards: |
| test_spec.device_type = 'QEMU' |
| |
| step_data = api.spec.spec_loaded_ok( |
| step_name='load spec.build_init', |
| message=Fuchsia( |
| checkout=Fuchsia.Checkout( |
| manifest='manifest', |
| remote='remote', |
| upload_results=bool(gcs_bucket), |
| use_snapshot=use_snapshot, |
| ), |
| build=Fuchsia.Build( |
| variants=variants, |
| build_type=build_type, |
| run_tests=run_tests, |
| board='boards/x64.gni', |
| product='products/core.gni', |
| target=build_target, |
| include_symbol_archive=include_symbol_archive, |
| include_breakpad_symbols=include_breakpad_symbols, |
| enforce_size_limits=enforce_size_limits, |
| upload_results=bool(gcs_bucket), |
| ), |
| test=test_spec, |
| debug_symbol_gcs_bucket=debug_symbol_gcs_bucket, |
| gcs_bucket=gcs_bucket, |
| artifact_gcs_bucket='fuchsia-infra-artifacts', |
| ), |
| ) |
| if run_tests: |
| step_data += test_step_data(deprecated=not test_in_shards) |
| return step_data |
| |
| def test(name, tryjob=False, project='fuchsia', use_snapshot=False): |
| if tryjob: |
| return api.test(name) + try_build( |
| project) + default_gitiles_refs_steps + source_manifest_for_cq( |
| use_snapshot) |
| else: |
| return api.test(name) + ci_build |
| |
| # ParseError. Should result in a failure. |
| yield (test('spec_parse_error') + |
| api.spec.spec_parse_error(step_name='load spec.build_init')) |
| yield test('ci') + spec_data() |
| yield (test('cq', tryjob=True) + spec_data() + |
| api.build_input_resolver.set_gerrit_branch()) |
| |
| yield test('with_uploading') + spec_data(gcs_bucket='fuchsia-build') |
| |
| yield test('profile') + spec_data( |
| variants=['profile'], gcs_bucket='fuchsia-build') |
| |
| yield test('commit_snapshot') + spec_data(use_snapshot=True) |
| |
| yield ( |
| test('patchset_snapshot', |
| tryjob=True, |
| project='garnet', |
| use_snapshot=True) + |
| spec_data(use_snapshot=True) + |
| api.build_input_resolver.set_gerrit_branch()) # yapf: disable |
| |
| yield (test('release-buid_only') + spec_data( |
| build_type='release', run_tests=False, enforce_size_limits=True)) |
| |
| yield (test('with_archives') + |
| spec_data(include_symbol_archive=True, include_breakpad_symbols=True)) |
| |
| yield test('deprecated_test') + spec_data(test_in_shards=False) |
| |
| yield (test('requires_secrets') + |
| spec_data(requires_secrets=True, test_in_shards=False)) |
| |
| yield test('max_shard_size') + spec_data(max_shard_size=200) |
