# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building Fuchsia and isolating build artifacts."""

from google.protobuf import text_format

from PB.infra.fuchsia import Fuchsia
from recipe_engine.recipe_api import Property

TEST_MULTIPLIER_KEY = 'MULTIPLY'

# These represent the location of the isolated hash in the output of this
# recipe when building SDK archives. Must be kept in sync with sdk.py.
ISOLATE_STEP_NAME = 'isolate artifacts'
ISOLATED_OUTPUT_KEY = 'isolated_output_hash'

DEPS = [
    'fuchsia/artifacts',
    'fuchsia/build',
    'fuchsia/buildbucket_util',
    'fuchsia/build_input_resolver',
    'fuchsia/checkout',
    'fuchsia/fuchsia',
    'fuchsia/git',
    'fuchsia/gitiles',
    'fuchsia/jiri',
    'fuchsia/jsonutil',
    'fuchsia/spec',
    'fuchsia/status_check',
    'fuchsia/testing_requests',
    'fuchsia/testsharder',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/isolated',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/python',
    'recipe_engine/step',
]

PROPERTIES = {
    'parent_id':
        Property(
            # This is a string because led IDs are not integers.
            kind=str,
            help='Parent build\'s buildbucket or led id',
            default=None),
    'spec_remote':
        Property(
            kind=str,
            help='URL of the specs git repository',
            default='https://fuchsia.googlesource.com/integration'),
    'spec_revision':
        Property(
            kind=str,
            help='The revision of spec_remote to fetch',
            default='HEAD'),
}


def RunSteps(api, parent_id, spec_remote, spec_revision):
  spec, spec_revision = api.fuchsia.setup_with_spec(spec_remote, spec_revision)

  bb_input = api.buildbucket.build.input
  with api.step.nest('got_revision') as presentation:
    # Read by the binary-size Gerrit plugin
    presentation.properties['got_revision'] = bb_input.gitiles_commit.id

  # TODO(fxb/39958): Retrying all failed builds in Gerrit launches subbuilds as
  # well, which fail because they're not triggered by a parent build. Once
  # that's resolved we can get rid of this check.
  if spec.build.run_tests and not parent_id:
    raise api.python.infra_failing_step(
        'no parent_id', 'subbuilds can only be triggered by parent builds')

  checkout = api.checkout.from_spec(spec.checkout)

  # Load test multipliers before building so we can fail fast if the JSON is
  # malformed, which avoids wasting a build.
  multipliers_path = None
  if bb_input.gerrit_changes and spec.test.test_in_shards:
    multipliers_path = api.path.mkstemp()
    with api.step.nest('test multipliers'):
      write_multipliers(api, multipliers_path, bb_input, checkout)

  if spec.checkout.upload_results:
    assert spec.gcs_bucket, (
        'gcs_bucket must be set if checkout.upload_results is')
    checkout.upload_results(spec.gcs_bucket, namespace=parent_id)

  # We can only derive repo from buildbucket when triggered in CI or CQ;
  # validate the checkout then.
  if bb_input.gerrit_changes or bb_input.gitiles_commit.project:
    if bb_input.gerrit_changes:
      project = bb_input.gerrit_changes[0].project
    else:
      project = bb_input.gitiles_commit.project
    with api.step.nest('validate checkout'), api.context(cwd=checkout.root_dir):
      repo_path = api.jiri.project(projects=[project]).json.output[0]['path']
      api.python(
          'validate FIDL namespaces',
          checkout.root_dir.join('scripts', 'style',
                                 'verify-fidl-libraries.py'),
          args=['--repo', repo_path],
      )

  collect_build_metrics = False
  if spec.build.upload_results:
    assert spec.gcs_bucket, (
        'gcs_bucket must be set if checkout.upload_results is')
    collect_build_metrics = True
  # Build infratools.
  spec.build.universe_packages.append('//bundles:infratools')
  spec.build.ninja_targets.append('bundles:infratools')
  # If SDK subbuild, set SDK ID to parent ID.
  sdk_id = parent_id if spec.build.sdk_subbuild else None
  build = api.build.from_spec(
      spec.build,
      checkout,
      collect_build_metrics,
      pave=spec.test.pave,
      sdk_id=sdk_id,
      gcs_bucket=spec.gcs_bucket)

  check_sizes_result = api.step(
      'check sizes',
      [
          build.tool('size_checker'), '--build-dir', build.fuchsia_build_dir,
          '--sizes-json-out',
          api.json.output()
      ],
      step_test_data=lambda: api.json.test_api.output({'some-file': 123}),
      # If the size checks fail, we still want to write the output to
      # the output property below.
      ok_ret='any')

  # This property is read by the binary-size Gerrit plugin.
  if check_sizes_result.json.output:
    check_sizes_result.presentation.properties[
        'binary_sizes'] = check_sizes_result.json.output

  if check_sizes_result.exc_result.retcode:
    raise api.step.StepFailure('size checks failed')

  # In SDK subbuild mode, isolate SDK archive and ninja targets.
  if spec.build.sdk_subbuild:
    sdk_archive_path = build.fuchsia_build_dir.join('sdk', 'archive')
    isolated = api.isolated.isolated(sdk_archive_path)
    sdk_ninja_targets = [
        target for target in spec.build.ninja_targets
        if target.startswith('sdk/archive')
    ]
    for ninja_target in sdk_ninja_targets:
      isolated.add_file(
          path=build.fuchsia_build_dir.join(*ninja_target.split('/')))
    sdk_archive_isolated_hash = isolated.archive(ISOLATE_STEP_NAME)
    api.step.active_result.presentation.properties[
        ISOLATED_OUTPUT_KEY] = sdk_archive_isolated_hash

  # TODO(garymm): assert spec.gcs_bucket set if upload_results set.
  if spec.gcs_bucket and spec.build.upload_results:
    build.upload_results(
        gcs_bucket=spec.gcs_bucket,
        is_release_version=spec.checkout.is_release_version,
        namespace=parent_id,
    )

  if spec.build.enforce_size_limits:
    build.check_filesystem_sizes()

  # Must be set before testing_requests.task_requests() is called.
  api.artifacts.gcs_bucket = spec.artifact_gcs_bucket
  api.artifacts.uuid = parent_id or api.buildbucket_util.id

  if spec.build.run_tests:
    if parent_id.isdigit():
      # Use parent build so that testing task requests refer to
      # that build, which actually orchestrates testing.
      buildbucket_build = api.buildbucket.get(int(parent_id))
      # If it's a try build, the parent build will not have its gitiles_commit
      # populated (it's populated at runtime by `build_input_resolver`, but
      # that doesn't change the input values stored in Buildbucket). So we need
      # to populate it with the same commit that `build_input_resolver`
      # resolved for the subbuild.
      buildbucket_build.input.gitiles_commit.CopyFrom(bb_input.gitiles_commit)
    else:
      # When the parent was launched by led, it's not possible to retrieve
      # the parent build, so we fall back to using our own build.
      # This is technically incorrect and any tests that rely on having
      # correct buildbucket metadata may fail when run via led. Ideally
      # we wouldn't have any tests that knew about buildbucket, but
      # for now this is OK since none of those tests run in recipes CQ,
      # which uses led to test recipes changes.
      buildbucket_build = api.buildbucket.build

    if spec.test.test_in_shards:
      shards = api.testsharder.execute(
          'create test shards',
          testsharder_path=build.tool('testsharder'),
          build_dir=build.fuchsia_build_dir,
          max_shard_size=spec.test.max_shard_size,
          target_duration_secs=spec.test.target_shard_duration_secs,
          max_shards_per_env=spec.test.max_shards_per_env,
          multipliers=multipliers_path,
          tags=spec.build.environment_tags,
      )
      task_requests = api.testing_requests.task_requests(
          build,
          buildbucket_build,
          spec.test.per_test_timeout_secs,
          spec.test.pool,
          shards,
          spec.test.swarming_expiration_timeout_secs,
          spec.test.swarming_io_timeout_secs,
          spec.test.use_runtests,
          spec.test.timeout_secs,
          default_service_account=spec.test.default_service_account,
          pave=spec.test.pave,
          targets_serial=spec.test.targets_serial)
    else:
      task_requests = api.testing_requests.deprecated_task_requests(
          build,
          api.testing_requests.deprecated_test_cmds(spec.test),
          spec.test.device_type,
          spec.test.pool,
          spec.test.timeout_secs,
          spec.test.pave,
          swarming_expiration_timeout_secs=spec.test
          .swarming_expiration_timeout_secs,
          swarming_io_timeout_secs=spec.test.swarming_io_timeout_secs,
          default_service_account=spec.test.default_service_account,
      )

    orchestration_inputs = api.build.TestOrchestrationInputs.from_build_results(
        build, task_requests)
    orchestration_inputs_hash = orchestration_inputs.isolate(api)
    step_result = api.step('logging orchestration_inputs_hash', cmd=None)
    step_result.presentation.properties[
        orchestration_inputs.HASH_PROPERTY] = orchestration_inputs_hash

  # Must be done after testing_requests.task_requests() is called, because that
  # modifies the filesystem images. TODO(garymm,joshuaseaton): once legacy_qemu
  # code paths are removed, remove this comment as it will become false.
  if spec.artifact_gcs_bucket:
    api.artifacts.upload('upload artifacts', build)


def write_multipliers(api, multipliers_path, bb_input, checkout):
  # Get project dir for gerrit change from source manifest
  gerrit_change = bb_input.gerrit_changes[0]
  project_dir = checkout.root_dir
  repo_url = 'https://%s/%s' % (gerrit_change.host.replace(
      '-review', ''), gerrit_change.project)
  dirs = checkout.source_manifest['directories']
  for d in dirs:
    if dirs[d]['git_checkout']['repo_url'] == repo_url:
      if d != '.':
        project_dir = checkout.root_dir.join(d)
      break

  with api.context(cwd=project_dir):
    commit_msg = api.git.get_commit_message(name='get commit msg')
  multipliers = api.jsonutil.extract_from_text(
      'extract', text=commit_msg, key=TEST_MULTIPLIER_KEY, default_contents=[])
  api.file.write_json('write', multipliers_path, multipliers, indent=2)


def GenTests(api):

  def spec_data(use_snapshot=False,
                build_type='debug',
                ninja_targets=(),
                sdk_subbuild=False,
                variants=(),
                device_type='QEMU',
                enforce_size_limits=False,
                run_tests=True,
                test_in_shards=True,
                gcs_bucket=None,
                pave=True):
    test_spec = None
    if run_tests:
      test_spec = Fuchsia.Test(
          device_type=device_type,
          max_shard_size=0,
          target_shard_duration_secs=10 * 60,
          max_shards_per_env=8,
          timeout_secs=30 * 60,
          pool='fuchsia.tests',
          test_in_shards=test_in_shards,
          swarming_expiration_timeout_secs=10 * 60,
          swarming_io_timeout_secs=5 * 60,
          default_service_account='service_account',
          targets_serial=True,
          pave=pave,
      )

    spec = Fuchsia(
        checkout=Fuchsia.Checkout(
            manifest='minimal',
            project='integration',
            remote='https://fuchsia.googlesource.com/manifest',
            upload_results=bool(gcs_bucket),
            use_snapshot=use_snapshot,
        ),
        build=Fuchsia.Build(
            variants=variants,
            build_type=build_type,
            run_tests=run_tests,
            ninja_targets=ninja_targets,
            sdk_subbuild=sdk_subbuild,
            board='boards/x64.gni',
            product='products/core.gni',
            target='x64',
            include_breakpad_symbols=False,
            enforce_size_limits=enforce_size_limits,
            upload_results=bool(gcs_bucket),
        ),
        test=test_spec,
        gcs_bucket=gcs_bucket,
        artifact_gcs_bucket='fuchsia-infra-artifacts',
    )
    return api.spec.spec_loaded_ok(
        step_name='load spec.build_init', message=spec)

  default_gitiles_refs_steps = api.gitiles.refs('refs', [
      'refs/heads/master',
      'deadbeef',
  ])

  spec_remote = 'https://fuchsia.googlesource.com/integration'

  properties = {
      # We rely on the buildbucket test API using this same
      # ID for ci_build_message and the builds returned by get().
      'parent_id': str(api.buildbucket.ci_build_message().id),
      'spec_remote': spec_remote,
  }

  #yapf: disable
  yield (
    api.checkout.test('default', tryjob=False) +
    api.build.test('default') +
    spec_data(gcs_bucket='fuchsia-infra', run_tests=True) +
    api.properties(**properties)
  )

  yield (
    api.checkout.test('non_numeric_parent_id', tryjob=False) +
    api.build.test('default') +
    spec_data(gcs_bucket='fuchsia-infra', run_tests=True) +
    api.properties(parent_id='not-a-number')
  )

  yield (
    api.status_check.test('subbuild_no_parent_id', status='infra_failure') +
    spec_data(run_tests=True) +
    api.buildbucket.try_build() +
    api.build_input_resolver.set_gerrit_branch('master') +
    default_gitiles_refs_steps +
    api.properties(parent_id='')
  )

  yield (
    api.checkout.test('default_cq', tryjob=True) +
    api.build.test('default_cq', tryjob=True) +
    spec_data(run_tests=True) +
    api.buildbucket.try_build(
      # Values chosen to match the test data in jiri/test_api.py
      # example_source_manifest() so that we trigger the test multipliers
      # code path.
      project='manifest',
      git_repo='https://fuchsia.googlesource.com/manifest') +
    api.build_input_resolver.set_gerrit_branch('master') +
    default_gitiles_refs_steps +
    api.properties(**properties)
  )

  yield (
    api.checkout.test('build_type_release_not_run_tests', tryjob=False) +
    api.build.test('default', create_shards=False) +
    spec_data(build_type='release', gcs_bucket='fuchsia-infra',
              run_tests=False, enforce_size_limits=True) +
    api.properties(**properties)
  )

  # yapf: enable

  yield (api.checkout.test('sdk', tryjob=False) +
         api.build.test('sdk', tryjob=False, create_shards=False) + spec_data(
             run_tests=False,
             ninja_targets=[
                 'sdk/archive/core.tar.gz', 'sdk/archive/fuchsia_dart.tar.gz'
             ],
             sdk_subbuild=True) + api.properties(parent_id='sdk-id'))

  yield (api.checkout.test('not_test_in_shards') +
         api.build.test('', create_shards=False) +
         spec_data(run_tests=True, test_in_shards=False, pave=True) +
         api.properties(**properties))

  yield (
      api.checkout.test('check_sizes_fails', tryjob=False, status='failure') +
      api.build.test(
          'check_sizes_fails', create_shards=False, status='failure') +
      spec_data(gcs_bucket='fuchsia-infra') + api.properties(**properties) +
      api.override_step_data('check sizes', retcode=1))