# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from contextlib import contextmanager
from recipe_engine import recipe_api

import copy
import os
import re

from .build_artifacts import ShardTaskRequest
from .build_artifacts import TestOrchestrationInputs
from .fvm import FvmImage

# Host architecture -> number of bits -> host platform name.
# Add to this dictionary as we support building on more devices.
HOST_PLATFORMS = {
    'intel': {
        64: 'x64',
    },
}

# List of available targets.
TARGETS = ['x64', 'arm64']

# List of available build types.
BUILD_TYPES = ['debug', 'release', 'thinlto', 'lto']

# List of canonical names for archives that the build can produce.
ARCHIVES = [
    'archive',  # Images and scripts for paving/netbooting.
    'package-archive',  # Package metadata, blobs, and tools.
    'symbol-archive',  # Metadata for symbolization.
    'breakpad-symbol-archive',  # Breakpad symbols.
]

# Canonical name for fvm.blk.
STORAGE_FULL = 'storage-full'
# Canonical name for fvm.sparse.blk.
STORAGE_SPARSE = 'storage-sparse'

# File in build out dir in which the sizes of filesystems are recorded.
FILESYSTEM_SIZES_FILENAME = 'filesystem_sizes.json'

# Manifests produced by the build.
IMAGES_JSON = 'images.json'
TESTS_JSON = 'tests.json'
ZIRCON_JSON = 'zircon.json'
TOOL_PATHS_JSON = 'tool_paths.json'

# Name of BigQuery project and table for uploading artifacts.
BIGQUERY_PROJECT = 'fuchsia-infra'
BIGQUERY_ARTIFACTS_DATASET = 'artifacts'

# Name of the build results manifest to isolate.
BUILD_RESULTS_JSON = 'build_results.json'

# The private and authorized SSH keys pulled down in the checkout, relative to
# the fuchsia root.
CHECKOUT_AUTHORIZED_KEY = '.ssh/authorized_keys'
CHECKOUT_PRIVATE_KEY = '.ssh/pkey'


class FuchsiaBuildResults(object):
  """Represents a completed build of Fuchsia."""

  # TODO(garymm): Use attr.ib

  def __init__(self,
               api,
               checkout_root,
               target,
               variants,
               build_type,
               fuchsia_build_dir,
               zircon_build_dir,
               board,
               product,
               gn_results,
               images=None,
               archives=None):
    assert target in TARGETS
    self._api = api
    self._checkout_root = checkout_root
    self._fuchsia_build_dir = fuchsia_build_dir
    self._zircon_build_dir = zircon_build_dir
    self._target = target
    self._variants = variants
    self._build_type = build_type
    self._filesystem_sizes = None
    self._images = images if images else {}
    self._archives = archives if archives else {}
    self._board = board
    self._product = product
    self._gn_results = gn_results
    self._storage_full = None

    self.authorized_key = self.checkout_root.join(CHECKOUT_AUTHORIZED_KEY)
    self.llvm_symbolizer = self.tool('llvm-symbolizer')
    self.minfs = self.tool('minfs')
    self.private_key = self.checkout_root.join(CHECKOUT_PRIVATE_KEY)
    self.zbi = self.tool('zbi')

  def __deepcopy__(self, memodict):
    # Shallow copy first.
    new = copy.copy(self)
    # Only images needs to be a real deep copy.
    new._images = copy.deepcopy(new._images, memodict)
    return new

  @property
  def target(self):
    """The build target for this build."""
    return self._target

  @property
  def variants(self):
    """The variants for this build."""
    return self._variants

  @property
  def build_type(self):
    """The build type for this build."""
    return self._build_type

  @property
  def zircon_build_dir(self):
    """The directory where Zircon build artifacts may be found."""
    return self._zircon_build_dir

  @property
  def fuchsia_build_dir(self):
    """The directory where Fuchsia build artifacts may be found."""
    return self._fuchsia_build_dir

  @property
  def tests_file(self):
    return self._fuchsia_build_dir.join(TESTS_JSON)

  @property
  def ids(self):
    return self._fuchsia_build_dir.join('ids.txt')

  @property
  def secret_specs(self):
    return self._fuchsia_build_dir.join('secret_specs')

  @property
  def images(self):
    """Mapping between the canonical name of an image produced by the Fuchsia
       build to its path relative to the fuchsia build directory."""
    return self._images

  @property
  def archives(self):
    """Mapping between the canonical name of an archive produced by the Fuchsia
       build to the absolute path to that archive on the local disk."""
    return self._archives

  @property
  def checkout_root(self):
    """The Fuchsia checkout root."""
    return self._checkout_root

  @property
  def board(self):
    """The board for this build."""
    return self._board

  @property
  def product(self):
    """The product for this build."""
    return self._product

  @property
  def storage_full(self):
    """The storage-full image (fvm.blk) produced by the build."""
    if self._storage_full:
      return self._storage_full
    elif STORAGE_FULL not in self.images:
      return None
    path = self.fuchsia_build_dir.join(self.images[STORAGE_FULL]['path'])
    self._storage_full = FvmImage(self._api, path, self.tool('fvm'))
    return self._storage_full

  @property
  def filesystem_sizes(self):
    """The filesystem_sizes data produced by the build."""
    if self._filesystem_sizes is None:
      filesystem_sizes_file = self.fuchsia_build_dir.join(
          FILESYSTEM_SIZES_FILENAME)
      # Not all supported build configurations actually produce the file.
      if self._api.path.exists(filesystem_sizes_file):
        self._filesystem_sizes = self._api.file.read_json(
            'read filesystem sizes', filesystem_sizes_file)
      else:
        self._filesystem_sizes = []
    return self._filesystem_sizes

  def tool(self, name, cpu='x64', os=None):
    """The path to a tool of the given name and cpu."""
    return self._gn_results.tool(name, cpu, os)

  def check_filesystem_sizes(self):
    """Checks that filesystem sizes are less than the maximum allowed

    Raises:
      StepFailure if the size exceeds the maximum.
    """
    with self._api.step.nest('check filesystem sizes'):
      error_messages = []
      for fs_dict in self.filesystem_sizes:
        fs_name = fs_dict['name']
        with self._api.step.nest(fs_name) as size_step:
          size = fs_dict['value']
          limit = fs_dict['limit']
          size_step.presentation.step_text = 'size, limit = %d, %d' % (
              size,
              limit,
          )
          if size > limit > 0:
            size_step.presentation.status = self._api.step.FAILURE
            error_message = '%s too large; was %d, but must be < %d' % (
                fs_name, size, limit)
            if 'debug_instructions' in fs_dict:
              error_message = '%s. %s' % (error_message,
                                          fs_dict['debug_instructions'])
            error_messages.append(error_message)
      if error_messages:
        raise self._api.step.StepFailure('. '.join(error_messages))

  def upload_debug_symbols(self,
                           debug_symbol_gcs_bucket,
                           gcs_bucket=None,
                           namespace=None):
    """Uploads debug symbols to Google Cloud Storage.

    Args:
      debug_symbol_gcs_bucket (str): GCS bucket name to upload debug symbols to.
      gcs_bucket (str|None): GCS bucket to upload debug-binaries.txt record.
        If None, do not create or upload the record.
      namespace (str|None): A unique namespace for the GCS debug-binaries
        record upload location. If None, the current build's ID is used.
    """
    assert debug_symbol_gcs_bucket
    path = self._api.path['cleanup'].join(
        'debug-binaries.txt') if gcs_bucket else None
    self._api.upload_debug_symbols(
        step_name='upload debug symbols',
        upload_debug_symbols_path=self.tool('upload_debug_symbols'),
        bucket=debug_symbol_gcs_bucket,
        # Recursively walk checkout root to discover all debug symbols.
        # NOTE: A subset of debug symbols are expected to be under the build
        # directory, which is currently always under checkout root.
        build_id_dirs=[self.checkout_root],
        record=path,
    )
    if path:
      self._api.upload.file_to_gcs(
          source=path,
          bucket=gcs_bucket,
          subpath=self._api.path.basename(path),
          namespace=namespace,
      )

  def upload_results(self, gcs_bucket, namespace=None):
    """Uploads artifacts from the build to Google Cloud Storage.

    Args:
      gcs_bucket (str): GCS bucket name to upload build results to.
      namespace (str|None): A unique namespace for the GCS upload location;
        if None, the current build's ID is used.
    """
    assert gcs_bucket
    with self._api.step.nest('upload build results'):
      self._api.build._upload_build_results(self, gcs_bucket, namespace)
    self._api.build._upload_package_snapshot(self, gcs_bucket, namespace)


class GNResults(object):
  """GNResults is represents the result of a `gn gen` invocation in the fuchsia
  build.

  It exposes the API of the build, which defines how one can invoke ninja.
  """
  _COMPDB_FILENAME = 'compile_commands.json'

  def __init__(self, api, fuchsia_build_dir, compdb_exported=False):
    self._api = api
    self._fuchsia_build_dir = fuchsia_build_dir

    zircon_ninja_instructions = api.json.read(
        'read zircon ninja instructions',
        fuchsia_build_dir.join(ZIRCON_JSON),
        step_test_data=api.build.test_api.mock_zircon_instructions,
    ).json.output

    self._zircon_build_dir = api.path.abs_to_path(
        api.path.realpath(
            fuchsia_build_dir.join(zircon_ninja_instructions['dir']),),)
    self._canonical_zircon_ninja_targets = zircon_ninja_instructions['targets']

    self._image_manifest = None
    self._archives = None
    self._test_manifest = None
    self._tools = None
    self._generated_sources = None
    self._zircon_generated_sources = None
    self._fuchsia_compdb = None
    self._zircon_compdb = None
    self._zbi_tests = None

    if compdb_exported:
      self._fuchsia_compdb = self._fuchsia_build_dir.join(self._COMPDB_FILENAME)
      api.path.mock_add_paths(self._fuchsia_compdb)
      self._zircon_compdb = self._zircon_build_dir.join(self._COMPDB_FILENAME)
      api.path.mock_add_paths(self._zircon_compdb)

  @property
  def fuchsia_build_dir(self):
    """Returns the fuchsia build directory (Path)."""
    return self._fuchsia_build_dir

  @property
  def zircon_build_dir(self):
    """Returns the associated zircon build directory (Path)."""
    return self._zircon_build_dir

  @property
  def canonical_zircon_ninja_targets(self):
    """Returns the canonical zircon ninja targets (list(string)) that the
    fuchsia build is informed of."""
    return self._canonical_zircon_ninja_targets

  @property
  def image_manifest(self):
    """Returns the manifest of images (dict) in the GN graph.

    TODO(BLD-253): Point to the schema once there is one.
    """
    if not self._image_manifest:
      self._image_manifest = self._api.json.read(
          'read image manifest',
          self.fuchsia_build_dir.join(IMAGES_JSON),
          step_test_data=self._api.build.test_api.mock_image_manifest,
      ).json.output

    return self._image_manifest

  @property
  def archives(self):
    """Returns the archives (dict[str]str) in the GN graph.

    Maps archive name to path relative to the fuchsia build directory.

    TODO(IN-882): Make this backed by its own build API module.
    """
    if not self._archives:
      self._archives = {}
      for image in self.image_manifest:
        if image['type'] == 'tgz':
          self._archives[image['name']] = image['path']
    return self._archives

  @property
  def test_manifest(self):
    """Returns the manifest of tests (dict) in the GN graph.

    TODO(BLD-253): Point to the schema once there is one.
    """
    if not self._test_manifest:
      self._test_manifest = self._api.json.read(
          'read test spec manifest',
          self.fuchsia_build_dir.join(TESTS_JSON),
          step_test_data=self._api.build.test_api.mock_test_spec_manifest,
      ).json.output
    return self._test_manifest

  @property
  def generated_sources(self):
    """Returns the generated source files (list(str)) from the fuchsia build.

    The returned paths are relative to the fuchsia build directory.
    """
    if not self._generated_sources:
      self._generated_sources = self._api.file.read_raw(
          'read generated sources',
          self.fuchsia_build_dir.join('all_fidl_json.txt'),
          '//generated_fidl.json',
      ).splitlines()
    return self._generated_sources

  @property
  def zircon_generated_sources(self):
    """Returns the generated source files (list(str)) from the zircon build.

    The returned paths are relative to the zircon build directory.
    """
    if not self._zircon_generated_sources:
      self._zircon_generated_sources = self._api.json.read(
          'read zircon generated sources',
          self.zircon_build_dir.join('generated_sources.json'),
          step_test_data=lambda: self._api.json.test_api.output(
              ['//generated_header.h']),
      ).json.output
    return self._zircon_generated_sources

  def tool(self, name, cpu='x64', os=None):
    """Returns the path to the specified tool provided from the tool_paths
    manifest.
    """
    os = os or self._api.platform.name
    if not self._tools:
      self._tools = {}
      tool_paths_manifest = self._api.json.read(
          'read tool_paths manifest',
          self._fuchsia_build_dir.join(TOOL_PATHS_JSON),
          step_test_data=self._api.build.test_api.mock_tool_paths_manifest
      ).json.output
      for tool in tool_paths_manifest:
        key = (tool['name'], tool['cpu'], tool['os'])
        assert key not in self._tools, (
            'only one tool with (name, cpu, os) == (%s, %s, %s) is allowed' %
            key)
        tool_path = self._api.path.abs_to_path(
            self._api.path.realpath(self._fuchsia_build_dir.join(tool['path'])))
        self._tools[key] = tool_path
    return self._tools.get((name, cpu, os), None)

  def filtered_compdb(self, filters):
    """The path to a merged compilation database, filtered via the passed filters."""
    with self._api.step.nest('merge compdbs'):
      compdb = []
      compdb += self._api.json.read(
          'read zircon compdb',
          self._zircon_compdb,
          stdout=self._api.json.output(),
          step_test_data=lambda: self._api.json.test_api.output([{
              'directory': '[START_DIR]/out/default.zircon',
              'file': '../../zircon.cpp',
              'command': 'clang++ zircon.cpp',
          }]),
      ).json.output

      compdb += self._api.json.read(
          'read compdb',
          self._fuchsia_compdb,
          stdout=self._api.json.output(),
          step_test_data=lambda: self._api.json.test_api.output([{
              'directory': '[START_DIR]/out/default',
              'file': '../../foo.cpp',
              'command': 'clang++ foo.cpp',
          }, {
              'directory': '[START_DIR]/out/default',
              'file': '../../third_party/foo.cpp',
              'command': 'clang++ third_party/foo.cpp',
          }, {
              'directory': '[START_DIR]/out/default',
              'file': '../../out/default/foo.cpp',
              'command': 'clang++ foo.cpp',
          }]),
      ).json.output

    def keep_in_compdb(entry):
      # Filenames are relative to the build directory, and the build directory is absolute.
      build_dir = self._api.path.abs_to_path(entry['directory'])
      full_path = self._api.path.abs_to_path(
          self._api.path.realpath(
              self._api.path.join(build_dir, entry['file'])))
      if build_dir.is_parent_of(full_path):
        return False

      segments = entry['file'].split(os.sep)
      if any(bad_segments in segments for bad_segments in filters):
        return False
      return True

    compdb_filtered = [entry for entry in compdb if keep_in_compdb(entry)]
    compdb_path = self._api.path['cleanup'].join('compile_commands.json')
    self._api.file.write_json('write merged compdb', compdb_path,
                              compdb_filtered)
    return compdb_path

  @property
  def zbi_tests(self):
    """Returns the ZBI tests from the Fuchsia build directory."""
    if not self._zbi_tests:
      zbi_tests = self._api.json.read(
          'read zbi test manifest',
          self.fuchsia_build_dir.join('zbi_tests.json'),
          step_test_data=lambda: self._api.json.test_api.output([]),
      ).json.output
      self._zbi_tests = {zbi_test['name']: zbi_test for zbi_test in zbi_tests}
    return self._zbi_tests


class ToolchainContext(object):
  """A ToolchainContext provides specifications for building with a specific
  toolchain."""

  def __init__(self, api):

    def cipd(package, instance, root_dir):
      pkgs = api.cipd.EnsureFile()
      pkgs.add_package('fuchsia/%s/${platform}' % package, instance)
      api.cipd.ensure(root_dir, pkgs)

    def isolated(instance, root_dir):
      api.isolated.download(
          'download', isolated_hash=instance, output_dir=root_dir)

    self._gn_args = []

    use_goma = True

    if api.build._clang_toolchain:
      use_goma = False
      with api.step.nest('clang_toolchain'), api.context(infra_steps=True):
        clang_instance = api.build._clang_toolchain['instance']
        clang_type = api.build._clang_toolchain['type']
        clang_dir = api.path.mkdtemp('clang')

        if clang_type == 'cipd':
          cipd('third_party/clang', clang_instance, clang_dir)
        elif clang_type == 'isolated':
          isolated(clang_instance, clang_dir)
        else:  # pragma: no cover
          raise KeyError('clang_toolchain type "%s" not recognized' %
                         clang_type)

        self._gn_args += ['clang_prefix="%s"' % clang_dir.join('bin')]

    if api.build._gcc_toolchain:
      use_goma = False
      with api.step.nest('gcc_toolchain'), api.context(infra_steps=True):
        gcc_instance = api.build._gcc_toolchain['instance']
        gcc_type = api.build._gcc_toolchain['type']
        gcc_dir = api.path.mkdtemp('gcc')

        if gcc_type == 'cipd':
          cipd('third_party/gcc', gcc_instance, gcc_dir)
        elif gcc_type == 'isolated':
          isolated(gcc_instance, gcc_dir)
        else:  # pragma: no cover
          raise KeyError('gcc_toolchain type "%s" not recognized' % gcc_type)

        self._gn_args += [
            '''
if (!defined(zircon_extra_args) { zircon_extra_args = {} }
zircon_extra_args.gcc_tool_dir = "%s"
''' % gcc_dir.join('bin')
        ]

    if not use_goma:

      @contextmanager
      def null_context():
        yield

      self._context = null_context
      self._ninja_jobs = api.platform.cpu_count
    else:
      api.goma.ensure()
      # goma sometimes takes a while to start, so bump the timeout from the
      # default of 20 seconds (it uses GOMA_PING_TIMEOUT_SEC + 20 as the actual
      # timeout).
      goma_env = {'GOMA_PING_TIMEOUT_SEC': '100'}
      self._context = lambda: api.goma.build_with_goma(env=goma_env)
      self._ninja_jobs = api.goma.jobs
      self._gn_args += ['use_goma=true', 'goma_dir="%s"' % api.goma.goma_dir]

    if api.build._rust_toolchain:
      with api.step.nest('rust_toolchain'), api.context(infra_steps=True):
        rust_instance = api.build._rust_toolchain['instance']
        rust_type = api.build._rust_toolchain['type']
        rust_dir = api.path.mkdtemp('rust')

        if rust_type == 'cipd':
          cipd('rust', rust_instance, rust_dir)
        elif rust_type == 'isolated':
          isolated(rust_instance, rust_dir)
        else:  # pragma: no cover
          raise KeyError('rust_toolchain type "%s" not recognized' % rust_type)

        self._gn_args += ['rustc_prefix="%s"' % rust_dir.join('bin')]

  def __call__(self):
    """Returns a context for building with the associated toolchain."""
    return self._context()

  @property
  def ninja_jobs(self):
    """Returns the number (int) of recommended ninja jobs."""
    return self._ninja_jobs

  @property
  def gn_args(self):
    """Returns GN args list(str) contributed by the toolchain."""
    return copy.copy(self._gn_args)


class FuchsiaBuildApi(recipe_api.RecipeApi):
  """APIs for building Fuchsia."""

  FuchsiaBuildResults = FuchsiaBuildResults
  GNResults = GNResults
  TestOrchestrationInputs = TestOrchestrationInputs
  ShardTaskRequest = ShardTaskRequest

  def __init__(self, clang_toolchain, gcc_toolchain, rust_toolchain, *args,
               **kwargs):
    super(FuchsiaBuildApi, self).__init__(*args, **kwargs)
    # TODO(fxb/35063): Remove most of these properties, use a spec msg.
    self._clang_toolchain = clang_toolchain
    self._gcc_toolchain = gcc_toolchain
    self._rust_toolchain = rust_toolchain
    self.__toolchain_context = None

  @property
  def _toolchain_context(self):
    if not self.__toolchain_context:
      self.__toolchain_context = ToolchainContext(self.m)
    return self.__toolchain_context

  def _prebuilt_path(self, checkout_root, *path):
    """Returns the Path to the host-platform subdir under the given subdirs."""
    path = list(path)
    path.append('{os}-{arch}'.format(
        os=self.m.platform.name,
        arch={'intel': 'x64'}[self.m.platform.arch],
    ))
    return checkout_root.join('prebuilt', *path)

  def from_spec(self,
                build_spec,
                checkout,
                collect_build_metrics,
                sdk_id=None,
                gcs_bucket=None):
    """Builds Fuchsia from a Jiri checkout

    Args:
      build_spec (fuchsia_pb2.Fuchsia.Build): The input build spec.
      checkout (CheckoutApi.CheckoutResults): The Fuchsia checkout.
      collect_build_metrics (bool): Whether to upload build metrics to cloud
        storage.
      sdk_id (str): If specified, set sdk_id in GN.
      gcs_bucket (str|None): The GCS bucket to upload results to, if set.

    Returns:
      A FuchsiaBuildResults, representing the build.
    """
    archives_to_build = []
    # If we are not building images, then we need not build package-related
    # targets.
    if not build_spec.exclude_images:
      archives_to_build.extend(['archive', 'package-archive'])
    if build_spec.include_breakpad_symbols:
      archives_to_build.append('breakpad-symbol-archive')
    if build_spec.include_symbol_archive:
      archives_to_build.append('symbol-archive')

    build_dir = checkout.root_dir.join('out')

    return self.with_options(
        build_dir=build_dir,
        checkout=checkout,
        target=build_spec.target,
        build_type=build_spec.build_type,
        packages=build_spec.packages,
        universe_packages=build_spec.universe_packages,
        variants=build_spec.variants,
        gn_args=build_spec.gn_args,
        sdk_id=sdk_id,
        ninja_targets=list(build_spec.ninja_targets),
        board=build_spec.board,
        product=build_spec.product,
        collect_build_metrics=collect_build_metrics,
        build_images=not build_spec.exclude_images,
        archives_to_build=tuple(archives_to_build),
        gcs_bucket=gcs_bucket,
    )

  def with_options(self,
                   build_dir,
                   checkout,
                   target,
                   build_type,
                   packages,
                   universe_packages=(),
                   variants=(),
                   gn_args=(),
                   sdk_id=None,
                   ninja_targets=(),
                   board=None,
                   product=None,
                   collect_build_metrics=False,
                   build_images=True,
                   archives_to_build=(),
                   gcs_bucket=None):
    """Builds Fuchsia from a Jiri checkout.

    Args:
      build_dir (Path): The directory to output build results into.
      checkout (CheckoutResult): The Fuchsia checkout result.
      target (str): The build target, see TARGETS for allowed targets.
      build_type (str): One of the build types in BUILD_TYPES
      packages (sequence[str]): A sequence of packages to pass to GN to build
      universe_packages (sequence[str]): A sequence of packages to pass to GN
        to build and add to the set of 'universe" packages
      variants (sequence[str]): A sequence of build variant selectors to pass
        to GN in `select_variant`
      gn_args (sequence[str]): Additional arguments to pass to GN
      ninja_targets (sequence[str]): Additional target args to pass to ninja
      sdk_id (str): If specified, set sdk_id in GN.
      board (str): A board to pass to GN to build
      product (str): A product to pass to GN to build
      collect_build_metrics (bool): Whether to collect build metrics.
      build_images (bool): Whether to build the basic images needed to boot
        and test on fuchsia.
      archives_to_build (seq(str)): A tuple of canonical names of archives to
        build.
      gcs_bucket (str): GCS bucket name to upload build results to.

    Returns:
      A FuchsiaBuildResults, representing the recently completed build.
    """
    assert target in TARGETS
    assert build_type in BUILD_TYPES
    assert all(archive in ARCHIVES for archive in archives_to_build)

    # We need to modify the arguments when setting up toolchain so make a copy.
    gn_args = copy.copy(gn_args) if gn_args else []
    if checkout.release_version:
      gn_args.append('build_info_version="{version}"'.format(
          version=checkout.release_version))

    if product:
      product_base = os.path.splitext(os.path.basename(product))[0]
      gn_args.append('build_info_product="%s"' % product_base)
    if board:
      board_base = os.path.splitext(os.path.basename(board))[0]
      gn_args.append('build_info_board="%s"' % board_base)

    if sdk_id:
      gn_args.append('sdk_id="%s"' % sdk_id)

    with self.m.step.nest('build'):
      return self._with_options(
          checkout_root=checkout.root_dir,
          fuchsia_build_dir=build_dir.join('default'),
          build_type=build_type,
          target=target,
          packages=packages,
          universe_packages=universe_packages,
          variants=variants,
          gn_args=gn_args,
          ninja_targets=list(ninja_targets),
          board=board,
          product=product,
          collect_build_metrics=collect_build_metrics,
          build_images=build_images,
          archives_to_build=archives_to_build,
          gcs_bucket=gcs_bucket,
      )

  def _run_ninja(self, step_name, build_dir, ninja_targets, jobs, build_id,
                 gcs_bucket):
    try:
      self.m.ninja(
          step_name=step_name,
          build_dir=build_dir,
          targets=ninja_targets,
          job_count=jobs,
      )
    except self.m.step.StepFailure:
      crashreports_dir = build_dir.join('clang-crashreports')
      self.m.path.mock_add_paths(crashreports_dir)
      if gcs_bucket and self.m.path.exists(crashreports_dir):
        with self.m.step.nest('clang-crashreports'):
          with self.m.context(infra_steps=True):
            temp = self.m.path.mkdtemp('reproducers')
            reproducers = self.m.file.glob_paths(
                'find reproducers',
                crashreports_dir,
                '*.sh',
                test_data=(crashreports_dir.join('foo.sh'),))
            for reproducer in reproducers:
              base = self.m.path.splitext(self.m.path.basename(reproducer))[0]
              files = self.m.file.glob_paths(
                  'find %s files' % base,
                  crashreports_dir,
                  base + '.*',
                  test_data=(crashreports_dir.join('foo.sh'),
                             crashreports_dir.join('foo.cpp')))
              tgz_basename = '%s.tar.gz' % base
              tgz_path = temp.join(tgz_basename)
              archive = self.m.tar.create(tgz_path, compression='gzip')
              for f in files:
                archive.add(f, crashreports_dir)
              archive.tar('create %s' % tgz_basename)
              self.m.upload.file_to_gcs(
                  source=tgz_path,
                  bucket=gcs_bucket,
                  subpath=tgz_basename,
                  namespace=build_id,
              )
      raise

  def gen(self,
          checkout_root,
          fuchsia_build_dir,
          target,
          build_type,
          product,
          board=None,
          packages=(),
          universe_packages=(),
          variants=(),
          args=(),
          record_tracelogs=False,
          export_compdb=False):
    """Sets up and calls `gn gen`.

    Args:
      checkout_root (str): The fuchsia checkout root.
      target (str): The build target, see TARGETS for allowed targets.
      build_type (str): One of the build types in BUILD_TYPES.
      product (str): A product to pass to GN to build.
      board (str|None): A board to pass to GN to build.
      packages (sequence[str]): A sequence of packages to pass to GN to build.
      universe_packages (sequence[str]): A sequence of packages to pass to GN
        to build and add to the set of 'universe" packages.
      variants (sequence[str]): A sequence of build variant selectors to pass
        to GN in `select_variant`.
      gn_args (sequence[str]): Additional arguments to pass to GN.
      record_tracelogs (bool): Whether to record tracelogs.
      export_compdb (bool): Whether to generate a compilation database.

    Returns:
      A GNResults object.
    """
    # Set the path to GN executable since it is not installed from CIPD.
    self.m.gn.set_path(
        self._prebuilt_path(checkout_root, 'third_party', 'gn').join('gn'))

    args = list(args)
    args.extend(self._toolchain_context.gn_args)
    args.extend([
        'target_cpu="%s"' % target,
        'is_debug=%s' % ('true' if build_type == 'debug' else 'false'),
    ])

    if board:
      args.append('import("//%s") ' % board)
    if product:
      args.append('import("//%s") ' % product)
    if packages:
      base_package_labels_format = 'base_package_labels=[%s]'
      # if product is set, append to base_package_labels.
      if product:
        base_package_labels_format = 'base_package_labels+=[%s]'

      args.append(base_package_labels_format %
                  ','.join('"%s"' % pkg for pkg in packages))

    if universe_packages:
      universe_package_labels_format = 'universe_package_labels=[%s]'
      # if product is set, append to universe_package_labels.
      if product:
        universe_package_labels_format = 'universe_package_labels+=[%s]'

      args.append(universe_package_labels_format %
                  ','.join('"%s"' % pkg for pkg in universe_packages))

    if variants:
      # Apply double-quotes to `select_variant_shortcut`s (a string;
      # e.g host-asan), but do not for `selector`s (a scope;
      # e.g {variant="asan-fuzzer" target_type=["fuzzed_executable"]})
      formatted_variants = [
          v if re.match('^{.*}$', v) else '"%s"' % v for v in variants
      ]
      args.append('select_variant=[%s]' % ','.join(formatted_variants))

      if 'thinlto' in variants:
        args.append('thinlto_cache_dir="%s"' %
                    self.m.path['cache'].join('thinlto'))

    if record_tracelogs:
      args.append("zircon_tracelog=\"%s\"" %
                  str(self.m.path['cleanup'].join('zircon_gn_trace.json')))

    def sorted_with_imports_first(arglist):
      """Sorts arguments while ensuring that imports come first.

      The `--args` passed to gen are sorted for a deterministic ordering with
      imports coming first, as otherwise they might blindly redefine variables
      set or modified by other arguments). This will ease reviews, as a
      re-ordering of build logic will yield the same GN invocation.
      """
      import_args, normal_args = [], []
      for arg in arglist:
        if arg.startswith('import('):
          import_args.append(arg)
        else:
          normal_args.append(arg)
      return sorted(import_args) + sorted(normal_args)

    gen_options = [
        fuchsia_build_dir,
        '--check',
        '--args=%s' % ' '.join(sorted_with_imports_first(args)),
    ]

    if export_compdb:
      gen_options.append('--export-compile-commands')

    if record_tracelogs:
      gen_options.append('--tracelog=%s' %
                         str(self.m.path['cleanup'].join('gn_trace.json')))

    with self.m.macos_sdk(), self.m.context(cwd=checkout_root):
      self.m.gn('gen', *gen_options)
      return self.GNResults(self.m, fuchsia_build_dir, export_compdb)

  def ninja(self,
            checkout_root,
            gn_results,
            targets=(),
            zircon_targets=(),
            build_canonical_zircon_targets=True,
            build_images=False,
            image_filter=None,
            archives_to_build=(),
            build_host_tests=False,
            build_generated_sources=False,
            build_zbi_tests=False,
            gcs_bucket=None,
            build_fuchsia=True,
            build_zircon=True):
    """A high-level ninja abstraction that consumes GN build APIs - exposed
    through GNResults - in determining what to run.

    Args:
      checkout_root (str): The fuchsia checkout root.
      gn_results (GNResults): GN gen results.
      targets (seq(str)): Fuchsia ninja targets.
      zircon_targets (seq(str)): Zircon ninja targets.
      build_canonical_zircon_targets (bool): Whether to build the zircon ninja
        targets given in the fuchsia build 'instructions' of zircon.json.
      build_images (bool): Whether to build images within the GN graph.
      image_filter (lamda): A bool-valued map on the contents of
        gn_results.image_manifest. If build_images is true, then only the images
        for which this function is true will be built.
      archives_to_build (seq(str)): A list of archives in the GN graph to build.
      build_host_tests (bool): Whether to build host test executables in the
        GN graph.
      build_generated_sources (bool): Whether to build generated sources.
      build_zbi_tests (bool): Whether to build the zbi tests in the GN graph.
      gcs_bucket (str|None): A GCS bucket to upload crash reports to.
      build_fuchsia (bool): Don't run Ninja on Fuchsia, only Zircon. Useful when
        targets = () indicates no targets should be built for Fuchsia, rather
        than indicating all targets should be built.
      build_zircon (bool): Don't run Ninja on Zircon, only Fuchsia. Useful when
        targets = () indicates no targets should be built for Zircon, rather
        than indicating all targets should be built.
    """
    # Set the path to Ninja executable since it is not installed from CIPD.
    self.m.ninja.set_path(gn_results.tool('ninja'))

    targets = list(targets)
    zircon_targets = list(zircon_targets)

    def append_target(path):
      abspath = self.m.path.abs_to_path(
          self.m.path.realpath(gn_results.fuchsia_build_dir.join(path)))
      if gn_results.zircon_build_dir.is_parent_of(abspath):
        zircon_targets.append(
            os.path.relpath(
                str(abspath),
                str(gn_results.zircon_build_dir),
            ))
      else:
        targets.append(path)

    if build_images:
      image_filter = image_filter or (lambda image: True)
      filtered_image_manifest = filter(image_filter, gn_results.image_manifest)
      for image in filtered_image_manifest:
        append_target(image['path'])

    if archives_to_build:
      targets.extend([
          archive for name, archive in gn_results.archives.iteritems()
          if name in archives_to_build
      ])

    if build_host_tests:
      for test_spec in gn_results.test_manifest:
        test = self.m.testsharder.Test.from_jsonish(test_spec['test'])
        if test.os != 'fuchsia':
          targets.append(test.path)

    if build_generated_sources:
      zircon_targets.extend(gn_results.zircon_generated_sources)
      targets.extend(gn_results.generated_sources)

    if build_zbi_tests:
      for zbi_test in gn_results.zbi_tests.itervalues():
        append_target(zbi_test['path'])

    if build_canonical_zircon_targets:
      zircon_targets.extend(gn_results.canonical_zircon_ninja_targets)

    with self.m.macos_sdk(), self.m.step.nest('ninja'):
      with self._toolchain_context():
        jobs = self._toolchain_context.ninja_jobs
        if build_zircon:
          self._run_ninja('zircon', gn_results.zircon_build_dir, zircon_targets,
                          jobs, self.m.buildbucket.build.id, gcs_bucket)
        if build_fuchsia:
          self._run_ninja('fuchsia', gn_results.fuchsia_build_dir, targets,
                          jobs, self.m.buildbucket.build.id, gcs_bucket)

  def _with_options(self, checkout_root, fuchsia_build_dir, build_type, target,
                    packages, universe_packages, variants, gn_args,
                    ninja_targets, board, product, collect_build_metrics,
                    build_images, archives_to_build, gcs_bucket):
    with self.m.step.nest('build fuchsia'):
      gn_results = self.gen(
          checkout_root=checkout_root,
          fuchsia_build_dir=fuchsia_build_dir,
          target=target,
          build_type=build_type,
          packages=packages,
          universe_packages=universe_packages,
          variants=variants,
          args=gn_args,
          board=board,
          product=product,
          record_tracelogs=collect_build_metrics)

      def image_filter(image):
        name = image['name']
        # If an image is used in paving or netbooting, its manifest entry will
        # specify what flags to pass to the bootserver when doing so;
        # conversely, if these arguments are present, we know the image is
        # needed for this.
        used_for_testing = (
            image.get('bootserver_pave', []) or  # used to pave
            image.get('bootserver_netboot', []) or  # used to netboot
            name in ['qemu-kernel', STORAGE_FULL]  # used for qemu
        )
        return used_for_testing

      images = {}
      ninja_targets = copy.copy(ninja_targets)

      if build_images:
        images = {
            image['name']: image
            for image in gn_results.image_manifest
            if image_filter(image)
        }
        # ids.txt is needed for symbolization.
        ninja_targets.append('ids.txt')
        # Needed for size checks and tracking.
        ninja_targets.append('build/images:record_filesystem_sizes')
        ninja_targets.append('build/images:system_snapshot')

      archives = {
          name: self.m.path.abspath(fuchsia_build_dir.join(archive))
          for name, archive in gn_results.archives.iteritems()
          if name in archives_to_build
      }
      # TODO(IN-882): Once the following two archives are added to images.json,
      # the following logic will happen automatically in the above for-loop.
      if 'package-archive' in archives_to_build:
        ninja_targets.append('build/gn:package_archive')
        archives['package-archive'] = fuchsia_build_dir.join('packages.tar.gz')

      if 'breakpad-symbol-archive' in archives_to_build:
        ninja_targets.append('build/gn:breakpad_symbols')
        archives['breakpad-symbols-archive'] = (
            fuchsia_build_dir.join('breakpad_symbols',
                                   'breakpad_symbols.tar.gz'))

      self.ninja(
          checkout_root=checkout_root,
          gn_results=gn_results,
          gcs_bucket=gcs_bucket,
          targets=ninja_targets,
          build_images=build_images,
          image_filter=image_filter,
          archives_to_build=archives_to_build,
          build_host_tests=True,
      )

      results = self.FuchsiaBuildResults(
          api=self.m,
          target=target,
          variants=variants,
          build_type=build_type,
          fuchsia_build_dir=fuchsia_build_dir,
          zircon_build_dir=gn_results.zircon_build_dir,
          checkout_root=checkout_root,
          board=board,
          product=product,
          gn_results=gn_results,
          images=images,
          archives=archives,
      )

      # The build produces an fvm.blk that is just big enough to fit its inputs.
      # In a small number of cases, there is not enough padding for tests to run
      # (on QEMU) and we see failures as a result. We guard against this by
      # doubling the amount of free space.
      # Note that this image is only used by QEMU and that no host side disk
      # blocks are allocated right away, so the operation is cheap regardless of
      # the size we extend to.
      if results.storage_full:
        results.storage_full.extend(2)
      return results

  def _upload_build_results(self, build_results, gcs_bucket, build_id):
    assert gcs_bucket
    # Upload archives.
    for path in build_results.archives.itervalues():
      self.m.upload.file_to_gcs(
          source=path,
          bucket=gcs_bucket,
          subpath=self.m.path.basename(path),
          namespace=build_id,
      )
    # Upload build metrics.
    self._upload_tracing_data(build_results, gcs_bucket, build_id)
    self._upload_fileystem_sizes(build_results)
    self._upload_blobstats_output(build_results, gcs_bucket, build_id)

  def _upload_package_snapshot(self, build_results, gcs_bucket, build_id):
    assert gcs_bucket
    snapshot_path = build_results.fuchsia_build_dir.join(
        'obj', 'build', 'images', 'system.snapshot')
    if not self.m.path.exists(snapshot_path):
      return

    # Upload a new table row for the system snapshot data generated during this build
    snapshot = self.m.file.read_raw('read package snapshot file', snapshot_path)
    build_packages_entry = {
        'build_id': self.m.buildbucket.build.id,
        'snapshot': snapshot,
    }

    basename = 'system.snapshot.json'
    build_packages_entry_file = self.m.path['tmp_base'].join(basename)

    self.m.step(
        'write build_packages_entry_file',
        ['cat', self.m.json.input(build_packages_entry)],
        stdout=self.m.raw_io.output(leak_to=build_packages_entry_file))

    self.m.upload.file_to_gcs(
        source=build_packages_entry_file,
        bucket=gcs_bucket,
        subpath=basename,
        namespace=build_id,
    )

    # Upload a new table row describing this particular build. Other tables' rows
    # are linked into this table using the build id as a foreign key.
    builds_entry = {
        'bucket': self.m.buildbucket.bucket_v1,
        'builder': self.m.buildbucket.builder_name,
        'build_id': self.m.buildbucket.build.id,
        'gitiles_commit': [self.m.buildbucket.gitiles_commit.id],
        'datetime': str(self.m.buildbucket.build.create_time.ToDatetime()),
        'start_time': str(self.m.buildbucket.build.start_time.ToDatetime()),
        'repo': self.m.buildbucket.build_input.gitiles_commit.project,
        'arch': build_results.target,
        'product': build_results.product,
        'board': build_results.board,
        'channel': [''],
    }

    self.m.bqupload.insert(
        step_name='add table row: %s/%s/builds_beta' %
        (BIGQUERY_PROJECT, BIGQUERY_ARTIFACTS_DATASET),
        project=BIGQUERY_PROJECT,
        dataset=BIGQUERY_ARTIFACTS_DATASET,
        table='builds_beta',
        data_file=self.m.json.input(builds_entry),
    )

  def _upload_fileystem_sizes(self, build_results):
    """Uploads filesystem sizes to BigQuery.

    The upload also includes metadata about this build so that the data can
    be used to create a self-contained BigQuery table. See INTK-709 and
    INTK-1153 for context.
    """
    if not build_results.filesystem_sizes:
      return

    metadata = {
        'builder_name':
            self.m.buildbucket.builder_name,
        # This field is set to a string in the BQ table schema because it's just
        # an opaque ID. The conversion from int to string that happens on the
        # BigQuery side is not what we want, so convert here.
        'build_id':
            str(self.m.buildbucket.build.id),
        'build_create_time_seconds':
            self.m.buildbucket.build.create_time.seconds,
        'gitiles_commit_host':
            self.m.buildbucket.gitiles_commit.host,
        'gitiles_commit_id':
            self.m.buildbucket.gitiles_commit.id,
        'gitiles_commit_project':
            self.m.buildbucket.gitiles_commit.project,
    }
    size_data = []
    for fs_dict in build_results.filesystem_sizes:
      datum = metadata.copy()
      datum['image_name'] = fs_dict['name']
      datum['image_size'] = fs_dict['value']
      size_data.append(datum)
    size_path = self.m.path['cleanup'].join('filesystem_sizes_bq.json')
    bq_formatted_str = '\n'.join(self.m.json.dumps(d) for d in size_data)
    with self.m.step.nest('upload filesystem sizes'):
      self.m.file.write_raw('write BQ formatted file', size_path,
                            bq_formatted_str)
      self.m.bqupload.insert('bqupload', 'fuchsia-infra', 'fuchsia_build',
                             'filesystem_sizes', size_path)

  def _upload_blobstats_output(self, build_results, gcs_bucket, build_id):
    """Runs the blobstats command and uploads the output files to GCS."""
    dir_name = 'blobstats'
    blobstats_output_dir = self.m.path['cleanup'].join(dir_name)
    if self._run_blobstats(build_results, blobstats_output_dir):
      # If blobstats failed, it's probably because the build intentionally
      # didn't produce the input files that blobstats requires. Blobstats is
      # generally just a nice-to-have anyway, so either way it's probably okay
      # to silently continue without uploading results if blobstats fails.
      self.m.upload.directory_to_gcs(
          source=blobstats_output_dir,
          bucket=gcs_bucket,
          subpath=dir_name,
          namespace=build_id,
      )

  def _run_blobstats(self, build_results, output_dir):
    """Runs the blobstats script for this build.

    Args:
      build_results (FuchsiaBuildResults): The results of the build on which
        blobstats will be run.
      output_dir (Path): The directory that the blobstats output files will be
        written to.

    Returns:
      A boolean indicating whether blobstats completed successfully.
    """
    root_dir = build_results.checkout_root

    dart_binary_path = build_results.tool('dart')
    blobstats_src_dir = root_dir.join('scripts', 'blobstats')
    command = [
        dart_binary_path,
        '--packages=%s' % blobstats_src_dir.join('blobstats.packages'),
        blobstats_src_dir.join('blobstats.dart'),
        '--output=%s' % output_dir,
    ]
    with self.m.context(cwd=build_results.fuchsia_build_dir):
      result = self.m.step('blobstats', command, ok_ret='any')
    return result.retcode == 0

  def _upload_tracing_data(self, build_results, gcs_bucket, build_id):
    """Uploads GN and ninja tracing results for this build to GCS"""
    # Only upload if the bucket is specified.
    gn_data = self._extract_gn_tracing_data(build_results.checkout_root,
                                            'gn_trace')
    zircon_gn_data = self._extract_gn_tracing_data(build_results.checkout_root,
                                                   'zircon_gn_trace')
    ninjatrace_tool = build_results.tool('ninjatrace')
    ninja_data = self._extract_ninja_tracing_data(
        ninjatrace_tool, build_results.fuchsia_build_dir,
        build_results.checkout_root, 'ninja_trace')
    zircon_ninja_data = self._extract_ninja_tracing_data(
        ninjatrace_tool, build_results.zircon_build_dir,
        build_results.checkout_root, 'zircon_ninja_trace')

    for data in [gn_data, zircon_gn_data, ninja_data, zircon_ninja_data]:
      self.m.upload.file_to_gcs(
          source=data,
          bucket=gcs_bucket,
          subpath=self.m.path.basename(data),
          namespace=build_id,
      )

  def _extract_ninja_tracing_data(self, ninjatrace_tool, build_dir,
                                  checkout_root, filename):
    """Extracts the tracing data from the .ninja_log.

    Args:
      cipd_dir (Path): The CIPD directory.
      build_dir (Path): The build directory.
      checkout_root (Path): The checkout directory.
      filename (str): The name of the output file (no extension).

    Returns:
      A Path to the file containing the gn tracing data in Chromium's
      about:tracing html format.
    """
    trace = self.m.path['cleanup'].join('%s.json' % filename)
    html = self.m.path['cleanup'].join('%s.html' % filename)
    self.m.step(
        'ninja tracing', [
            ninjatrace_tool,
            '-filename',
            build_dir.join('.ninja_log'),
            '-trace-json',
            trace,
        ],
        stdout=self.m.raw_io.output(leak_to=trace))
    return self._trace2html('ninja trace2html', trace, html, checkout_root)

  def _extract_gn_tracing_data(self, checkout_root, filename):
    """Extracts the tracing data from this GN run.

    Args:
      checkout_root (Path): The checkout directory.
      filename (str): The name of the output file (no extension).

    Returns:
      A Path to the file containing the gn tracing data in Chromium's
      about:tracing html format.
    """
    return self._trace2html('gn trace2html',
                            self.m.path['cleanup'].join('%s.json' % filename),
                            self.m.path['cleanup'].join('%s.html' % filename),
                            checkout_root)

  def _trace2html(self, name, trace, html, checkout_root):
    """Converts an about:tracing file to HTML using the trace2html tool"""

    # Catapult is imported in manifest/garnet, so we abort if it wasn't included
    # in this checkout.
    # if self.m.path['third_party'].join('catapult') not in self.m.path:
    #     return
    self.m.python(
        name=name,
        script=checkout_root.join('third_party', 'catapult', 'tracing', 'bin',
                                  'trace2html'),
        args=['--output', html, trace])
    return html