# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Recipe for building Fuchsia SDKs."""

from recipe_engine.config import List
from recipe_engine.recipe_api import Property

from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2

DEPS = [
    'fuchsia/build',
    'fuchsia/buildbucket_util',
    'fuchsia/checkout',
    'fuchsia/fuchsia',
    'fuchsia/git',
    'fuchsia/gsutil',
    'fuchsia/hash',
    'fuchsia/jiri',
    'fuchsia/macos_sdk',
    'fuchsia/release',
    'fuchsia/tar',
    'fuchsia/sso',
    'fuchsia/upload',
    'recipe_engine/buildbucket',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/isolated',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/python',
    'recipe_engine/raw_io',
    'recipe_engine/step',
    'recipe_engine/swarming',
    'recipe_engine/time',
]

BUILD_TYPE = 'release'

# These represent the location of the isolated hash in the output of the
# sdk subbuild.  They need to be kept in sync with sdk_subbuild.py
ISOLATED_OUTPUT_KEY = 'isolated_output_hash'

TARGETS = ['arm64', 'x64']

PROPERTIES = {
    'manifest':
        Property(kind=str, help='Jiri manifest to use'),
    'remote':
        Property(kind=str, help='Remote manifest repository'),
    'checkout_snapshot':
        Property(
            kind=bool,
            help='Whether or not to checkout from a Jiri snapshot.'
            ' Snapshot is expected to be found at the location specified by'
            ' BuildBucket input.',
            default=False),
    'gcs_bucket':
        Property(
            kind=str,
            help='GCS bucket for uploading the sdk.',
            default='fuchsia'),
    'sdk_name':
        Property(
            kind=str,
            help='The name of the sdk to build.',
            default='core',
        ),
    'cipd_root':
        Property(
            kind=str,
            help='The CIPD package root for upload.',
            default='fuchsia'),
    'subbuilders':
        Property(
            kind=List(basestring),
            help='SDK subbuilders used to generate SDK archives.',
            # TODO(fxb/40777): Make this required. This is optional for a
            # soft-transition away from sdk_subbuild.py to fuchsia/build.py.
            # If this property is set, use these fuchsia/build.py subbuilders
            # instead of the default sdk_subbuild.py.
            default=()),
    'companion_images':
        Property(
            kind=List(dict),
            help='The Fuchsia images built with a version of the SDK.'
            ' Each object should have 3 properties:'
            '   name: name of the image when referred to in the SDK'
            '   builder: name of the builder generating the image'
            '   bucket: GCS bucket where the images should be installed',
            default=[]),
    'publish_branch':
        Property(
            kind=str,
            help='Publish if input is release version reachable on this branch.',
            default=''),
    'sdk_id':
        Property(kind=str, help='SDK version id', default=None),
}


def fetch_sdk_artifacts(api, checkout_root, builds):
  with api.step.nest('fetch sdk artifacts'):
    isolated_output_path = checkout_root.join('archives')
    for build in builds.values():
      with api.step.nest(build.builder.builder):
        isolated_output = build.output.properties[ISOLATED_OUTPUT_KEY].strip(
            '"')
        output_dir = isolated_output_path.join(build.builder.builder)
        api.file.ensure_directory('makedirs', dest=output_dir)
        api.isolated.download(
            step_name='download %s archive' % build.builder.builder,
            isolated_hash=isolated_output,
            output_dir=output_dir)


def relocate_artifacts(api, builds, companion_images, sdk_id):

  def relocate_artifact(api, src_bucket, dst_bucket, src_path, dst_path):
    api.gsutil.copy(
        src_bucket=src_bucket,
        src=src_path,
        dst_bucket=dst_bucket,
        dst=dst_path,
        link_name='gsutil.copy',
        unauthenticated_url=False)

  with api.step.nest('relocate sdk image(s)'):
    for build in builds.values():
      image_name, image_bucket = next(
          (image['name'], image['bucket'])
          for image in companion_images
          if image['builder'] == build.builder.builder)
      with api.step.nest(image_name):
        build_gcs_bucket = build.output.properties['gcs_bucket'].strip('"')
        # build-archive.tgz
        relocate_artifact(
            api=api,
            src_bucket=build_gcs_bucket,
            dst_bucket=image_bucket,
            src_path='builds/%s/build-archive.tgz' % build.id,
            dst_path='development/%s/images/%s.tgz' % (sdk_id, image_name),
        )
        # packages.tar.gz
        relocate_artifact(
            api=api,
            src_bucket=build_gcs_bucket,
            dst_bucket=image_bucket,
            src_path='builds/%s/packages.tar.gz' % build.id,
            dst_path='development/%s/packages/%s.tar.gz' % (sdk_id, image_name),
        )


def RunSteps(api, manifest, remote, checkout_snapshot, gcs_bucket, sdk_name,
             cipd_root, subbuilders, companion_images, publish_branch, sdk_id):
  build = api.buildbucket.build

  revision = build.input.gitiles_commit.id

  sdk_builds = []

  if not sdk_id:
    sdk_id = str(build.id)
  subbuild_properties = {'sdk_id': sdk_id, 'parent_id': sdk_id}

  sdk_schedule_reqs = []

  # TODO (fxb/40777): Make this unconditionally use subbuilders.
  if not subbuilders:
    builder_names = [
        'sdk-%s-subbuild-%s-%s' % (sdk_name, target, api.platform.name)
        for target in TARGETS
    ]
  else:
    builder_names = subbuilders

  for builder_name in builder_names:
    sdk_schedule_reqs.append(
        api.buildbucket.schedule_request(
            builder=builder_name,
            properties=subbuild_properties,
            # Leave unset to avoid overriding priority from configs.
            priority=None,
            # TODO(olivernewman): Swarming currently breaks if many builds are
            # launched simultaneously with the same swarming_parent_run_id. Set
            # the swarming_parent_run_id param here after that bug is fixed.
        ))

  sdk_builds.extend(
      api.buildbucket.schedule(
          schedule_build_requests=sdk_schedule_reqs,
          step_name='schedule sdk builds'))

  checkout_root = api.path['start_dir'].join('fuchsia')
  if checkout_snapshot:
    if api.buildbucket_util.is_tryjob:
      assert len(build.input.gerrit_changes) == 1
      checkout = api.checkout.from_patchset_snapshot(
          path=checkout_root, gerrit_change=build.input.gerrit_changes[0])
    else:
      checkout = api.checkout.from_commit_snapshot(
          path=checkout_root, gitiles_commit=build.input.gitiles_commit)
  else:
    assert manifest
    assert remote
    checkout = api.checkout.fuchsia_with_options(
        path=checkout_root,
        build=build,
        manifest=manifest,
        remote=remote,
    )

  # Only publish SDK and companion images if the input release version
  # is reachable on the publish branch.
  publish = False
  if revision and publish_branch:
    integration_repo = checkout.root_dir.join('integration')
    try:
      release_version = api.release.ref_to_release_version(
          ref=revision, repo_path=integration_repo)
      branch_head = api.git.get_remote_branch_head(
          api.sso.sso_to_https(remote), publish_branch)
      if release_version in api.release.get_release_versions(
          ref=branch_head, repo_path=integration_repo):
        publish = True
    # If the incoming revision was not a release version, continue
    # without publishing.
    except api.step.StepFailure:
      pass

  image_builds = []
  image_properties = {'sdk_id': sdk_id}

  if publish and companion_images:
    image_schedule_reqs = []
    for image_builder in [i['builder'] for i in companion_images]:
      image_schedule_reqs.append(
          api.buildbucket.schedule_request(
              builder=image_builder,
              properties=image_properties,
              # Leave unset to avoid overriding priority from configs.
              priority=None,
              # TODO(olivernewman): Swarming currently breaks if many builds
              # are launched simultaneously with the same
              # swarming_parent_run_id. Set the swarming_parent_run_id param
              # here after that bug is fixed.
          ))

    image_builds.extend(
        api.buildbucket.schedule(
            schedule_build_requests=image_schedule_reqs,
            step_name='schedule image builds'))

  sdk_builds = api.buildbucket.collect_builds(
      build_ids=[sdk_build.id for sdk_build in sdk_builds],
      step_name='collect sdk build results')

  # Display SDK builds status and show failures on UI, if any.
  api.buildbucket_util.display_builds(
      step_name='display sdk builds',
      builds=sdk_builds.values(),
      raise_on_failure=True,
  )

  # Download the individual SDK archives
  fetch_sdk_artifacts(api, checkout.root_dir, sdk_builds)

  # Merge the SDK archives for each target into a single archive.
  merge_path = checkout.root_dir.join('scripts', 'sdk', 'merger', 'merge.py')
  full_archive_path = api.path['cleanup'].join('merged_sdk_archive.tar.gz')

  sdk_archives_dir = checkout.root_dir.join('archives')
  sdk_archives_paths = api.file.glob_paths(
      'get_sdk_paths',
      sdk_archives_dir,
      '*/*.tar.gz',
      test_data=[
          '%s/core.tar.gz' % builder_name for builder_name in builder_names
      ])
  api.file.move('create merged_sdk_archive.tar.gz', sdk_archives_paths[0],
                full_archive_path)

  for sdk_archives_path in sdk_archives_paths[1:]:
    api.python(
        'merge %s' % sdk_archives_path,
        merge_path,
        args=[
            '--first-archive',
            sdk_archives_path,
            '--second-archive',
            full_archive_path,
            '--output-archive',
            full_archive_path,
        ])

  # Generate a Bazel workspace along with its tests.
  # These tests are being run for every SDK flavor.
  generate_bazel_path = checkout.root_dir.join('scripts', 'sdk', 'bazel',
                                               'generate.py')
  sdk_dir = api.path['cleanup'].join('sdk-bazel')
  test_workspace_dir = api.path['cleanup'].join('tests')

  api.python(
      'create bazel sdk',
      generate_bazel_path,
      args=[
          '--archive',
          full_archive_path,
          '--output',
          sdk_dir,
          '--tests',
          test_workspace_dir,
      ],
  )

  with api.step.nest('test sdk'):
    # Generate tool_path.json to access bazel tool.
    gn_results = api.build.gen(
        checkout_root=checkout.root_dir,
        fuchsia_build_dir=checkout.root_dir.join('out', 'default'),
        target='x64',
        build_type='debug',
        product='products/bringup.gni',
    )
    bazel_path = gn_results.tool('bazel')
    bazel_user_root_path = api.path['cleanup'].join('bazel')

    with api.macos_sdk():
      api.python(
          'run bazel tests',
          test_workspace_dir.join('run.py'),
          args=[
              '--output_user_root',
              bazel_user_root_path,
              '--bazel',
              bazel_path,
          ],
      )

  # Generate a GN workspace along with its tests.
  # These tests are being run for every SDK flavor.
    generate_gn_path = checkout.root_dir.join('scripts', 'sdk', 'gn',
                                              'generate.py')
    sdk_dir = api.path['cleanup'].join('sdk-gn')
    test_workspace_dir = api.path['cleanup'].join('tests')

    api.python(
        'create gn sdk',
        generate_gn_path,
        args=[
            '--archive',
            full_archive_path,
            '--output',
            sdk_dir,
            '--tests',
            test_workspace_dir,
        ],
    )
    api.python('run gn tests', test_workspace_dir.join('run.py'))

  if image_builds:
    image_builds = api.buildbucket.collect_builds(
        build_ids=[image_build.id for image_build in image_builds],
        step_name='collect image build results')

    # Display SDK builds status and show failures on UI, if any.
    api.buildbucket_util.display_builds(
        step_name='display image builds',
        builds=image_builds.values(),
        raise_on_failure=True,
    )

  # Publish the core SDK.
  if not publish:
    return

  if companion_images:
    # Relocate the image build outputs into a well-known location based on
    # sdk_id.
    relocate_artifacts(api, image_builds, companion_images, build.id)

  gcs_archive_path = 'sdk/%s/%s-amd64' % (sdk_name, api.platform.name)
  new_gcs_archive_path = 'development/%s/sdk/%s-amd64/%s.tar.gz' % (
      sdk_id, api.platform.name, sdk_name)
  cipd_pkg_name = '%s/sdk/%s/${platform}' % (cipd_root, sdk_name)

  upload_raw_sdk(
      api,
      sdk_name='raw-core-sdk',
      gcs_archive_bucket=gcs_bucket,
      gcs_archive_path=gcs_archive_path,
      new_gcs_archive_path=new_gcs_archive_path,
      cipd_pkg_name=cipd_pkg_name,
      archive_path=full_archive_path,
      revision=revision,
      update_latest=True,
      sdk_id=sdk_id,
      build_id=str(build.id),
      checkout_root=checkout.root_dir)


def upload_raw_sdk(api, sdk_name, gcs_archive_bucket, gcs_archive_path,
                   new_gcs_archive_path, cipd_pkg_name, archive_path, revision,
                   update_latest, sdk_id, build_id, checkout_root):
  sdk_dir = api.path['cleanup'].join(sdk_name)

  # Extract the archive to a directory for CIPD processing.
  with api.step.nest('extract ' + sdk_name):
    api.file.ensure_directory('create sdk dir', sdk_dir)
    api.tar.extract(
        step_name='unpack sdk archive',
        path=archive_path,
        directory=sdk_dir,
    )

  if revision:
    with api.step.nest('upload ' + sdk_name):
      # Upload the SDK to GCS and CIPD. Only upload sdk_id when update_latest is
      # true.
      upload_archive(
          api,
          gcs_archive_bucket=gcs_archive_bucket,
          gcs_archive_path=gcs_archive_path,
          new_gcs_archive_path=new_gcs_archive_path,
          cipd_pkg_name=cipd_pkg_name,
          sdk=archive_path,
          out_dir=sdk_dir,
          revision=revision,
          update_latest=update_latest,
          sdk_id=sdk_id,
          build_id=build_id,
          checkout_root=checkout_root,
      )


def upload_archive(api, gcs_archive_bucket, gcs_archive_path,
                   new_gcs_archive_path, cipd_pkg_name, sdk, out_dir, revision,
                   update_latest, sdk_id, build_id, checkout_root):
  api.gsutil.upload(
      bucket=gcs_archive_bucket,
      src=sdk,
      dst='%s/%s' % (gcs_archive_path, sdk_id),
      link_name='archive',
      name='upload fuchsia-sdk %s' % sdk_id,
      # Publicly available.
      unauthenticated_url=True)

  api.gsutil.upload(
      bucket=gcs_archive_bucket,
      src=sdk,
      dst=new_gcs_archive_path,
      link_name='archive',
      name='upload new fuchsia-sdk %s' % sdk_id,
      # Publicly available.
      unauthenticated_url=True)

  # Only upload a snapshot if this build was invoked by the scheduler.  If the
  # sdk_id equals the build_id then the sdk_id was not passed as a property by
  # another recipe.
  if sdk_id == build_id:
    # Note that this will upload the snapshot to a location different from the
    # path that api.fuchsia copied it to. This uses a path based on the hash of
    # the SDK artifact, not based on the hash of the snapshot itself. Clients can
    # use this to find the snapshot used to build a specific SDK artifact.
    snapshot_file = api.path['cleanup'].join('jiri.snapshot')
    with api.context(cwd=checkout_root):
      api.jiri.snapshot(snapshot_file)
    api.gsutil.upload(
        bucket='fuchsia-snapshots',
        src=snapshot_file,
        dst=sdk_id,
        link_name='jiri.snapshot',
        name='upload jiri.snapshot')

  if update_latest:
    # Record the sdk_id of the most recently uploaded archive for downstream autorollers.
    sdk_id_path = api.path['cleanup'].join('sdk_id')
    api.file.write_text('write sdk_id', sdk_id_path, sdk_id)
    api.gsutil.upload(
        bucket=gcs_archive_bucket,
        src=sdk_id_path,
        dst='%s/LATEST_ARCHIVE' % gcs_archive_path,
        link_name='LATEST_ARCHIVE',
        name='upload latest sdk_id')
    api.gsutil.upload(
        bucket=gcs_archive_bucket,
        src=sdk_id_path,
        dst='development/LATEST_%s' % api.platform.name.upper(),
        link_name='LATEST',
        name='upload new latest sdk_id')

  # Upload the SDK to CIPD as well.
  api.upload.cipd_package(
      cipd_pkg_name,
      out_dir, [api.upload.DirectoryPath(out_dir)], {'git_revision': revision},
      repository=None,
      extra_tags={'jiri_snapshot': sdk_id})


def GenTests(api):

  def add_hash_property(build):
    build.output.properties[ISOLATED_OUTPUT_KEY] = '###HASH###'
    return build

  revision = api.jiri.example_revision

  # TODO(fxb/40777): Make this a default property.
  subbuilders_properties = api.properties(subbuilders=[
      'core.arm64-linux-sdk-subbuild',
      'core.x64-linux-sdk-subbuild',
  ])

  topaz_properties = api.properties(
      project='integration',
      manifest='fuchsia/topaz/topaz',
      remote='https://fuchsia.googlesource.com/integration',
  )

  topaz_local_ci = topaz_properties + api.buildbucket.ci_build(
      git_repo='https://fuchsia.googlesource.com/topaz',
      revision=revision,
  ) + api.properties(revision=revision)

  topaz_global_ci = topaz_properties + api.buildbucket.ci_build(
      git_repo='https://fuchsia.googlesource.com/topaz',
      revision=revision,
      bucket='###global-integration-bucket###') + api.properties(
          revision=revision)

  topaz_release_ci = topaz_properties + api.buildbucket.ci_build(
      git_repo='https://fuchsia.googlesource.com/topaz',
      git_ref='refs/heads/release',
      revision=revision,
      bucket='###global-integration-bucket###') + api.properties(
          revision=revision)

  topaz_local_cq = topaz_properties + api.buildbucket.try_build()

  ci_subbuilds = api.buildbucket.simulated_collect_output(
      builds=[
          add_hash_property(api.buildbucket.ci_build_message(status='SUCCESS')),
          add_hash_property(api.buildbucket.ci_build_message(status='SUCCESS'))
      ],
      step_name='collect sdk build results')
  ci_subbuilds_infra_failure = api.buildbucket.simulated_collect_output(
      builds=[
          add_hash_property(
              api.buildbucket.ci_build_message(status='INFRA_FAILURE')),
          add_hash_property(
              api.buildbucket.ci_build_message(status='INFRA_FAILURE'))
      ],
      step_name='collect sdk build results')
  ci_subbuilds_failure = api.buildbucket.simulated_collect_output(
      builds=[
          add_hash_property(api.buildbucket.ci_build_message(status='FAILURE')),
          add_hash_property(api.buildbucket.ci_build_message(status='FAILURE'))
      ],
      step_name='collect sdk build results')
  cq_subbuilds = api.buildbucket.simulated_collect_output(
      builds=[
          add_hash_property(
              api.buildbucket.try_build_message(status='SUCCESS')),
          add_hash_property(
              api.buildbucket.try_build_message(status='SUCCESS'))
      ],
      step_name='collect sdk build results')
  ci_subbuilds_with_images = api.buildbucket.simulated_collect_output(
      builds=[
          add_hash_property(api.buildbucket.ci_build_message(status='SUCCESS')),
          add_hash_property(api.buildbucket.ci_build_message(status='SUCCESS')),
          add_hash_property(api.buildbucket.ci_build_message(status='SUCCESS')),
          add_hash_property(api.buildbucket.ci_build_message(status='SUCCESS'))
      ],
      step_name='collect sdk build results')

  image_build = api.buildbucket.ci_build_message(
      builder='###SDK_IMAGE_BUILDER###', status='SUCCESS', build_id=123456789)
  image_build.output.properties['gcs_bucket'] = '###BUCKET###'
  other_image_build = api.buildbucket.ci_build_message(
      builder='###OTHER_SDK_IMAGE_BUILDER###',
      status='SUCCESS',
      build_id=9876543210)
  other_image_build.output.properties['gcs_bucket'] = '###BUCKET###'

  ci_image_builds = api.buildbucket.simulated_collect_output(
      builds=[image_build, other_image_build],
      step_name='collect image build results')
  ci_image_builds_failure = api.buildbucket.simulated_collect_output(
      builds=[
          api.buildbucket.ci_build_message(
              builder='###SDK_IMAGE_BUILDER###',
              status='FAILURE',
              build_id=123456789),
          other_image_build,
      ],
      step_name='collect image build results')

  tags = [
      'releases/0.20191019.0.1',
      'releases/0.20191018.0.1',
  ]
  describe = api.step_data('git describe',
                           api.raw_io.stream_output('releases/0.20191018.0.1'))
  failed_describe = api.step_data('git describe', retcode=1)
  publish_branch = 'refs/heads/master'
  release_versions = api.step_data(
      'get release versions on h3ll0.git --no-pager',
      api.raw_io.stream_output('\n'.join(tags)))

  companion_images = [{
      'name': '###SDK_IMAGE###',
      'builder': '###SDK_IMAGE_BUILDER###',
      'bucket': '###DEV_BUCKET###',
  }, {
      'name': '###OTHER_SDK_IMAGE###',
      'builder': '###OTHER_SDK_IMAGE_BUILDER###',
      'bucket': '###OTHER_DEV_BUCKET###',
  }]

  yield (api.test('local_ci') + topaz_local_ci + ci_subbuilds)
  yield (api.test('local_cq') + topaz_local_cq + cq_subbuilds)
  yield (api.test('snapshot_ci') + topaz_local_ci +
         api.properties(checkout_snapshot=True) + ci_subbuilds)
  yield (api.test('snapshot_cq') + topaz_local_cq +
         api.properties(checkout_snapshot=True) + cq_subbuilds)
  yield (api.test('local_ci_mac') + topaz_local_ci + api.platform.name('mac') +
         ci_subbuilds)
  yield (api.test('global_ci') + topaz_global_ci + ci_subbuilds +
         api.properties(
             companion_images=companion_images,
             publish_branch=publish_branch,
         ) + failed_describe + subbuilders_properties)
  yield (api.test('release_ci') + topaz_release_ci + ci_subbuilds +
         ci_image_builds + api.properties(
             companion_images=companion_images,
             publish_branch=publish_branch,
         ) + describe + subbuilders_properties + release_versions)
  yield (api.test('local_ci_build_failure') + topaz_global_ci +
         ci_subbuilds_failure)
  yield (api.test('local_ci_infra_failure') + topaz_global_ci +
         ci_subbuilds_infra_failure)
  yield (api.test('release_ci_image_failure') + topaz_global_ci + ci_subbuilds +
         ci_image_builds_failure + api.properties(
             companion_images=companion_images, publish_branch=publish_branch) +
         describe + release_versions)
  yield (api.test('release_ci_new_upload') + topaz_release_ci + api.step_data(
      'upload raw-core-sdk.cipd.cipd search fuchsia/sdk/core/${platform} ' +
      'git_revision:%s' % revision, api.json.output({'result': []})) +
         ci_subbuilds + ci_image_builds + api.properties(
             companion_images=companion_images,
             publish_branch=publish_branch,
         ) + describe + release_versions)