# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building Fuchsia and isolating build artifacts."""

from google.protobuf import text_format

from PB.infra.fuchsia import Fuchsia
from recipe_engine.recipe_api import Property

TEST_MULTIPLIER_KEY = 'MULTIPLY'

# These represent the location of the isolated hash in the output of this
# recipe when building SDK archives. Must be kept in sync with sdk.py.
ISOLATE_STEP_NAME = 'isolate artifacts'
ISOLATED_OUTPUT_KEY = 'isolated_output_hash'

DEPS = [
    'fuchsia/artifacts',
    'fuchsia/build',
    'fuchsia/buildbucket_util',
    'fuchsia/build_input_resolver',
    'fuchsia/checkout',
    'fuchsia/git',
    'fuchsia/gitiles',
    'fuchsia/jiri',
    'fuchsia/jsonutil',
    'fuchsia/spec',
    'fuchsia/testing_requests',
    'fuchsia/testsharder',
    'fuchsia/upload_debug_symbols',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/isolated',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/python',
    'recipe_engine/step',
]

PROPERTIES = {
    'parent_id':
        Property(
            # This is a string because led IDs are not integers.
            kind=str,
            help='Parent build\'s buildbucket or led id',
            default=None),
    'spec_remote':
        Property(
            kind=str,
            help='URL of the specs git repository',
            default='https://fuchsia.googlesource.com/integration'),
    'spec_revision':
        Property(
            kind=str,
            help='The revision of spec_remote to fetch',
            default='HEAD'),
}


def RunSteps(api, parent_id, spec_remote, spec_revision):
  # Resolve the build input to always contain a Gitiles commit.
  bb_build = api.buildbucket.build
  api.build_input_resolver.resolve(
      bb_build.input,
      default_project_url='https://fuchsia.googlesource.com/fuchsia')
  commit_remote = 'https://%s/%s' % (bb_build.input.gitiles_commit.host,
                                     bb_build.input.gitiles_commit.project)
  if commit_remote == spec_remote:
    # If there was no parent build, then spec_revision may not have been
    # resolved, so use the revision that we resolved immediately above.
    if spec_revision == 'HEAD':
      spec_revision = bb_build.input.gitiles_commit.id
    # Otherwise the spec_revision was specified by the parent, and we want all
    # accesses to that remote to use that revision. In this case,
    # build_input_resolver.resolve() is unnecessary since we overwrite the
    # revision, but we only use it to pre-populate the gitiles_commit of the
    # build input and to keep the conditional logic simpler.
    else:
      bb_build.input.gitiles_commit.id = spec_revision

  with api.step.nest('got_revision') as presentation:
    # Read by the binary-size Gerrit plugin
    presentation.properties['got_revision'] = bb_build.input.gitiles_commit.id

  with api.step.nest('load spec') as presentation:
    presentation.step_text = 'loading spec'
    try:
      spec = api.spec.load_from_build(
          build=bb_build,
          spec_remote=spec_remote,
          Type=Fuchsia,
          spec_revision=spec_revision,
      )
    except api.spec.ParseError as e:
      raise api.step.StepFailure('failed to parse spec: %s' % str(e))

    presentation.logs['textproto'] = text_format.MessageToString(spec).split(
        '\n')
    # The artifacts tool relies on this output property.
    # This design has been fragile. Please don't add more dependencies on any
    # additional output properties.
    presentation.properties['gcs_bucket'] = spec.gcs_bucket

  checkout = api.checkout.from_spec(spec.checkout)

  if spec.checkout.upload_results:
    assert spec.gcs_bucket, (
        'gcs_bucket must be set if checkout.upload_results is')
    checkout.upload_results(spec.gcs_bucket, namespace=parent_id)

  # We can only derive repo from buildbucket when triggered in CI or CQ;
  # validate the checkout then.
  bb_input = api.buildbucket.build_input
  if bb_input.gerrit_changes or bb_input.gitiles_commit.project:
    if bb_input.gerrit_changes:
      project = bb_input.gerrit_changes[0].project
    else:
      project = bb_input.gitiles_commit.project
    with api.step.nest('validate checkout'), api.context(cwd=checkout.root_dir):
      repo_path = api.jiri.project(projects=[project]).json.output[0]['path']
      api.python(
          'validate FIDL namespaces',
          checkout.root_dir.join('scripts', 'style',
                                 'verify-fidl-libraries.py'),
          args=['--repo', repo_path],
      )

  collect_build_metrics = False
  if spec.build.upload_results:
    assert spec.gcs_bucket, (
        'gcs_bucket must be set if checkout.upload_results is')
    collect_build_metrics = True
  # Build infratools.
  spec.build.universe_packages.append('//bundles:infratools')
  spec.build.ninja_targets.append('bundles:infratools')
  # If SDK subbuild, set SDK ID to parent ID.
  sdk_id = parent_id if spec.build.sdk_subbuild else None
  build = api.build.from_spec(
      spec.build,
      checkout,
      collect_build_metrics,
      sdk_id=sdk_id,
      gcs_bucket=spec.gcs_bucket)

  check_sizes_result = api.step(
      'check sizes', [
          build.tool('size_checker'), '--build-dir', build.fuchsia_build_dir,
          '--sizes-json-out',
          api.json.output()
      ],
      step_test_data=lambda: api.json.test_api.output({'some-file': 123}))

  # This property is read by the binary-size Gerrit plugin.
  if check_sizes_result.json.output:
    check_sizes_result.presentation.properties[
        'binary_sizes'] = check_sizes_result.json.output

  # In SDK subbuild mode, isolate SDK archive and ninja targets.
  if spec.build.sdk_subbuild:
    sdk_archive_path = build.fuchsia_build_dir.join('sdk', 'archive')
    isolated = api.isolated.isolated(sdk_archive_path)
    sdk_ninja_targets = [
        target for target in spec.build.ninja_targets
        if target.startswith('sdk/archive')
    ]
    for ninja_target in sdk_ninja_targets:
      isolated.add_file(
          path=build.fuchsia_build_dir.join(*ninja_target.split('/')))
    sdk_archive_isolated_hash = isolated.archive(ISOLATE_STEP_NAME)
    api.step.active_result.presentation.properties[
        ISOLATED_OUTPUT_KEY] = sdk_archive_isolated_hash

  if spec.debug_symbol_gcs_bucket:
    build.upload_debug_symbols(
        debug_symbol_gcs_bucket=spec.debug_symbol_gcs_bucket,
        gcs_bucket=spec.gcs_bucket if spec.build.upload_results else None,
        namespace=parent_id,
    )

  # TODO(garymm): assert spec.gcs_bucket set if upload_results set.
  if spec.gcs_bucket and spec.build.upload_results:
    build.upload_results(
        gcs_bucket=spec.gcs_bucket,
        namespace=parent_id,
    )

  # Check for properly formatted JSON in commit message before build
  multipliers_path = None
  if bb_input.gerrit_changes and spec.test.test_in_shards:
    with api.step.nest('test multipliers'):
      # Get project dir for gerrit change from source manifest
      gerrit_change = bb_input.gerrit_changes[0]
      project_dir = checkout.root_dir
      repo_url = 'https://%s/%s' % (gerrit_change.host.replace(
          '-review', ''), gerrit_change.project)
      dirs = checkout.source_manifest['directories']
      for d in dirs:
        if dirs[d]['git_checkout']['repo_url'] == repo_url:
          if d != '.':
            project_dir = checkout.root_dir.join(d)
          break

      with api.context(cwd=project_dir):
        commit_msg = api.git.get_commit_message(name='get commit msg')
      multipliers = api.jsonutil.extract_from_text(
          'extract',
          text=commit_msg,
          key=TEST_MULTIPLIER_KEY,
          default_contents=[])
      multipliers_path = api.path.mkstemp()
      api.file.write_json('write', multipliers_path, multipliers, indent=2)

  if spec.build.enforce_size_limits:
    build.check_filesystem_sizes()

  # Run the testsharder to collect test specifications and shard them.
  # Always create shards regardless test_in_shards in true or not.
  shards = api.testsharder.execute(
      'create test shards',
      testsharder_path=build.tool('testsharder'),
      build_dir=build.fuchsia_build_dir,
      max_shard_size=spec.test.max_shard_size,
      multipliers=multipliers_path,
      tags=spec.build.environment_tags,
  )

  # Must be set before testing.shard_requests() is called.
  api.artifacts.gcs_bucket = spec.artifact_gcs_bucket
  api.artifacts.uuid = parent_id or api.buildbucket_util.id

  if spec.build.run_tests:
    if parent_id.isdigit():
      # Use parent build so that testing task requests refer to
      # that build, which actually orchestrates testing.
      buildbucket_build = api.buildbucket.get(int(parent_id))
    else:
      # When the parent was launched by led, it's not possible to retrieve
      # the parent build, so we fall back to using our own build.
      # This is technically incorrect and any tests that rely on having
      # correct buildbucket metadata may fail when run via led. Ideally
      # we wouldn't have any tests that knew about buildbucket, but
      # for now this is OK since none of those tests run in recipes CQ,
      # which uses led to test recipes changes.
      buildbucket_build = bb_build

    build_artifacts = build.get_artifacts(shards)
    if spec.test.test_in_shards:
      shard_requests = api.testing_requests.shard_requests(
          build_artifacts,
          buildbucket_build,
          spec.test.per_test_timeout_secs,
          spec.test.pool,
          spec.test.swarming_expiration_timeout_secs,
          spec.test.swarming_io_timeout_secs,
          spec.test.use_runtests,
          timeout_secs=spec.test.timeout_secs)
    else:
      shard_requests = api.testing_requests.deprecated_shard_requests(
          build_artifacts,
          api.testing_requests.deprecated_test_cmds(spec.test),
          spec.test.device_type,
          spec.test.pool,
          spec.test.timeout_secs,
          spec.test.pave,
          requires_secrets=spec.test.requires_secrets,
          swarming_expiration_timeout_secs=spec.test
          .swarming_expiration_timeout_secs,
          swarming_io_timeout_secs=spec.test.swarming_io_timeout_secs,
      )

    orchestration_inputs = api.build.TestOrchestrationInputs(
        build_artifacts.llvm_symbolizer, build_artifacts.minfs,
        build_artifacts.symbolize_tool, shard_requests,
        build_artifacts.tests_file)
    orchestration_inputs_hash = orchestration_inputs.isolate(api)
    step_result = api.step('logging orchestration_inputs_hash', cmd=None)
    step_result.presentation.properties[
        orchestration_inputs.HASH_PROPERTY] = orchestration_inputs_hash

  # Must be done after testing.shard_requests() is called, because that
  # modifies the filesystem images. TODO(garymm,joshuaseaton): once legacy_qemu
  # code paths are removed, remove this comment as it will become false.
  api.artifacts.upload('upload artifacts', build)


def GenTests(api):

  def spec_data(use_snapshot=False,
                build_type='debug',
                ninja_targets=(),
                sdk_subbuild=False,
                variants=(),
                device_type='QEMU',
                enforce_size_limits=False,
                run_tests=True,
                test_in_shards=True,
                requires_secrets=False,
                gcs_bucket=None,
                debug_symbol_gcs_bucket='debug-symbols'):
    test_spec = None
    if run_tests:
      test_spec = Fuchsia.Test(
          device_type=device_type,
          max_shard_size=0,
          timeout_secs=30 * 60,
          pool='fuchsia.tests',
          test_in_shards=test_in_shards,
          requires_secrets=requires_secrets,
          swarming_expiration_timeout_secs=10 * 60,
          swarming_io_timeout_secs=5 * 60,
      )

    spec = Fuchsia(
        checkout=Fuchsia.Checkout(
            manifest='minimal',
            project='integration',
            remote='https://fuchsia.googlesource.com/manifest',
            upload_results=bool(gcs_bucket),
            use_snapshot=use_snapshot,
        ),
        build=Fuchsia.Build(
            variants=variants,
            build_type=build_type,
            run_tests=run_tests,
            ninja_targets=ninja_targets,
            sdk_subbuild=sdk_subbuild,
            board='boards/x64.gni',
            product='products/core.gni',
            target='x64',
            include_symbol_archive=False,
            include_breakpad_symbols=False,
            enforce_size_limits=enforce_size_limits,
            upload_results=bool(gcs_bucket),
        ),
        test=test_spec,
        debug_symbol_gcs_bucket=debug_symbol_gcs_bucket,
        gcs_bucket=gcs_bucket,
        artifact_gcs_bucket='fuchsia-infra-artifacts',
    )
    return api.spec.spec_loaded_ok(
        step_name='load spec.build_init', message=spec)

  default_gitiles_refs_steps = api.gitiles.refs('refs', [
      'refs/heads/master',
      'deadbeef',
  ])

  spec_remote = 'https://fuchsia.googlesource.com/integration'

  properties = {
      # We rely on the buildbucket test API using this same
      # ID for ci_build_message and the builds returned by get().
      'parent_id': str(api.buildbucket.ci_build_message().id),
      'spec_remote': spec_remote,
  }

  #yapf: disable
  yield (
    api.checkout.test('default', tryjob=False) +
    api.build.test('default') +
    spec_data(gcs_bucket='fuchsia-infra', run_tests=True) +
    api.properties(**properties)
  )

  yield (
    api.checkout.test('non-numeric-parent-id', tryjob=False) +
    api.build.test('default') +
    spec_data(gcs_bucket='fuchsia-infra', run_tests=True) +
    api.properties(parent_id='not-a-number')
  )

  yield (
    api.checkout.test('default_cq', tryjob=True) +
    api.build.test('default_cq', tryjob=True) +
    spec_data(run_tests=True) +
    api.buildbucket.try_build(
      # Values chosen to match the test data in jiri/test_api.py
      # example_source_manifest() so that we trigger the test multipliers
      # code path.
      project='manifest',
      git_repo='https://fuchsia.googlesource.com/manifest') +
    api.build_input_resolver.set_gerrit_branch('master') +
    default_gitiles_refs_steps +
    api.properties(**properties)
  )

  yield (
    api.checkout.test('spec_remote_cq', tryjob=True) +
    api.build.test('spec_remote_cq', tryjob=True) +
    spec_data(run_tests=True) +
    api.buildbucket.try_build(git_repo=spec_remote) +
    api.build_input_resolver.set_gerrit_branch('master') +
    default_gitiles_refs_steps +
    api.properties(**properties)
  )

  yield (
    api.checkout.test('spec_remote_cq_with_spec_revision', tryjob=True) +
    api.build.test('spec_remote_cq', tryjob=True) +
    spec_data(run_tests=True) +
    api.buildbucket.try_build(git_repo=spec_remote) +
    api.build_input_resolver.set_gerrit_branch('master') +
    default_gitiles_refs_steps +
    api.properties(spec_revision='deadbeef2', **properties)
  )

  yield (
    api.checkout.test('build_type_release_not_run_tests', tryjob=False) +
    api.build.test('default') +
    spec_data(build_type='release', gcs_bucket='fuchsia-infra',
              run_tests=False, enforce_size_limits=True) +
    api.properties(**properties)
  )

  yield (api.checkout.test('spec_parse_error', tryjob=False, status='failure') +
         api.spec.spec_parse_error(step_name='load spec.build_init'))
  # yapf: enable

  yield (api.checkout.test('sdk', tryjob=False) +
         api.build.test('sdk', tryjob=False) + spec_data(
             run_tests=False,
             ninja_targets=[
                 'sdk/archive/core.tar.gz', 'sdk/archive/fuchsia_dart.tar.gz'
             ],
             sdk_subbuild=True) + api.properties(parent_id='sdk-id'))

  yield (api.checkout.test('not_test_in_shards') + api.build.test('') +
         spec_data(run_tests=True, test_in_shards=False) +
         api.properties(**properties))