tools/devshell/build

#!/bin/bash
# Copyright 2017 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

#### CATEGORY=Build
### Run Ninja to build Fuchsia

## usage: fx build [options] [LABELS_OR_TARGETS]... [-- NINJA_OPTIONS...]
##
## Builds a set of GN or Ninja targets after performing a few correctness checks for
## the Fuchsia build. Ninja-specific options must be provided after a -- specifier.
##
## LABELS_OR_TARGETS is a list of Ninja target path, GN labels (beginning with //)
## or `--toolchain=NAME` options or related aliases (see below). For example, the
## following lines are equivalent ways to ask for the host `zbi` tool to be built:
##
##    fx build --host //zircon/tools/zbi
##    fx build --host //zircon/tools/zbi:zbi
##    fx build --toolchain=//build/toolchain:host_x64 //zircon/tools/zbi
##    fx build '//zircon/tools/zbi(//build/toolchain:host_x64)'
##    fx build host_x64/zbi
##
## The first line above is favored. Using the last line will print a warning
## instructing the user which (optional) toolchain option + label to use instead.
##
## optional arguments:
##   -h, --help               Print this message. Use '-- --help' to print
##                            Ninja-specific help.
##
##   --no-checks              Disable consistency checks (for fx development only).
##
##   --log LOGFILE            Print debug information to LOGFILE. Please attach
##                            the resulting file when reporting bugs.
##
##   --fint-params-path PATH  Path to a fint params file used by an infra
##                            builder. This is useful for reproducing the exact
##                            set of targets built by the infrastructure.
##                            All other options listed below are ignored in this mode.
##
##   --toolchain=TOOLCHAIN    Specify the current GN toolchain suffix to use for
##                            GN labels that appear after this option.
##
##   -j <count>               Specify an explicit max job count for both Ninja
##                            and Bazel.
##
##   --skip-auth-check        Bypass pre-flight build service authentication
##                            checks.
##
##   --verbose                Print extra information, such as the list of
##                            Ninja targets corresponding to the input GN labels
##                            to build. This does *not* enable Ninja --verbose flag,
##                            use '-- --verbose' to do that.
##
##   --host                   Alias for --toolchain=host
##   --default                Alias for --toolchain=default
##   --fuchsia                Alias for --toolchain=fuchsia
##   --fidl                   Alias for --toolchain=fidl
##
##   -- [ARGS...]             Pass all following arguments directly to Ninja.
##
##   --help-toolchains        Print list of valid TOOLCHAIN values.
##
## Run `fx build -h` to see Ninja argument details.

declare -r start_time="${EPOCHREALTIME/./}"

# Note: Do not use $(dirname "${BASH_SOURCE[0]}") to locate the script's
# directory to save about 7ms of startup time. See https://fxbug.dev/42085680
# The fallback is only needed when invoking directly with bash as in:
# `(cd tools/devshell && bash build)`
_script_dir="${BASH_SOURCE[0]%/*}"; if [[ "${_script_dir}" == "${BASH_SOURCE[0]}" ]]; then _script_dir=.; fi
# shellcheck source=/dev/null
source "${_script_dir}/lib/vars.sh" || exit 1
# shellcheck source=/dev/null
source "${_script_dir}/lib/build_api_client.sh" || exit $?
fx-config-read

# shellcheck source=/dev/null
source "${_script_dir}/lib/metrics.sh" || exit $?

shopt -s extglob

function print-toolchain-help {
  cat <<EOF
Possible --toolchain aliases

  host     Host toolchain    (//build/toolchain:host_$HOST_CPU)
  default  Default toolchain (//build/toolchain/fuchsia:\$TARGET_CPU)
  fuchsia  Default toolchain as well.
  fidl     FIDL toolchain    (//build/fidl:fidling)

Apart from that, the --toolchain value must be a valid GN toolchain
label, such as '//some/toolchain:target'.
EOF
}

function count-ninja-log-lines {
  NINJA_LOG_FILE="${FUCHSIA_BUILD_DIR}/.ninja_log"
  if [[ -f "${NINJA_LOG_FILE}" ]]; then
    wc -l <"${NINJA_LOG_FILE}" 2>/dev/null
  else
    echo -1
  fi
}

function main {
  local switches=() fuchsia_targets=() ninja_switches=()
  local log_file is_logging fint_params_path
  local no_checks
  local verbose
  local skip_auth_check=0
  local concurrency=0

  # When tracing, increase verbosity.
  if [[ -o xtrace ]]; then
    verbose="true"
  fi

  is_logging=false
  while [[ $# -gt 0 ]]; do
    case "$1" in
      --log)
        switches+=("$1")
        if [[ $# -lt 2 ]]; then
          fx-command-help
          return 1
        fi
        log_file="$2"
        if [[ -f "${log_file}" ]]; then
          fx-error "File \"${log_file}\" exists."
          return 1
        fi
       # if ! touch "${log_file}"; then
       #   fx-error "Cannot create logfile \"${log_file}\""
       #   return 1
       # fi
        is_logging=true
        shift
        ;;
      --fint-params-path)
        switches+=("$1")
        if [[ $# -lt 2 ]]; then
          fx-command-help
          return 1
        fi
        fint_params_path="$2"
        shift
        ;;
      # Use -- as a separator for command-line options for Ninja.
      # Anything that follows that goes directly into ninja_switches and the loop exits.
      --)
        shift
        ninja_switches+=("$@")
        break
        ;;

      --no-checks)
        no_checks=true
        ;;

      --skip-auth-check)
        skip_auth_check=1
        ;;

      --verbose)
        verbose=true
        ;;

      # As a special case, recognize `-j<count>` and `-j <count>` as valid
      # options. For now pass them directly to fx-run-ninja, which will eventually
      # pass them to Bazel. See https://fxbug.dev/351623259
      -j)
        concurrency="$2"
        ninja_switches+=("$1" "$2")
        shift
        ;;

      -j*)
        concurrency="${1#-j}"
        ninja_switches+=("$1")
        ;;

      -h|--help)
        fx-print-command-help "${BASH_SOURCE[0]}"
        exit 1
        ;;

      --help-toolchains)
        print-toolchain-help
        exit 1
        ;;

      # The following options are used to specify GN toolchain for future
      # labels that appear on the command line.
      --host|--fidl|--default|--fuchsia|--toolchain=*)
        fuchsia_targets+=("$1")
        ;;

      -*)
        fx-error "Unsupported option $1, see 'fx help build' for details."
        exit 1
        ;;
      *)
        fuchsia_targets+=("$1")
        ;;
    esac
    shift
  done

  # Set no_checks=true when running a Ninja tool or printing the Ninja help.
  for arg in "${ninja_switches[@]}"; do
    case "${arg}" in
      -h|--help|-t*)
        no_checks=true
        break
    esac
  done

  if [[ "${HOST_OS}" == "mac" ]]; then
    fx-warn "Building Fuchsia on macOS is not officially supported."
    fx-warn "Please see http://go/local-platform-support-prd for details."
  fi

  if [[ -n "${fint_params_path}" ]]; then
    if [[ ${#ninja_switches[@]} -gt 0 ]]; then
      fx-error "It's invalid to specify extra Ninja flags along with --fint-params-path."
      exit 1
    elif [[ ${#fuchsia_targets[@]} -gt 0 ]]; then
      fx-error "It's invalid to specify targets along with --fint-params-path."
      exit 1
    fi
  fi

  if [[ "${is_logging}" = true ]]; then
    # log file header with relevant environment information
    {
      TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
      echo "Build initiated at ${TIMESTAMP}"
      echo
      echo "------ GIT QUICK INFO ------"
      echo "$ git status"
      git --no-optional-locks --git-dir="${FUCHSIA_DIR}/.git" status
      echo
      echo "$ git rev-parse JIRI_HEAD"
      git --no-optional-locks --git-dir="${FUCHSIA_DIR}/.git" rev-parse JIRI_HEAD
      echo
      echo "------ CONTENTS OF args.gn ------"
      echo "$ cat ${FUCHSIA_BUILD_DIR}/args.gn"
      echo
      cat "${FUCHSIA_BUILD_DIR}/args.gn"
      echo
    } >> "${log_file}" 2>&1

    # tee stdout and stderr to log_file
    exec > >(tee -a "${log_file}") 2>&1
  fi

  # A list of reasons to run `fx-gen` when needed.
  local need_fx_gen_for=()

  if [[ -z "${no_checks}" ]]; then
    # A change to any of these might mean things are now done differently enough
    # that ninja's automatic re-gen rule might not be triggered or might not work
    # properly if it is triggered.  So preemptively force a re-gen if that seems
    # like a plausible possibility.
    local -r landmines=("$PREBUILT_GN"
                        "$FUCHSIA_DIR/tools/devshell/build"
                        "$FUCHSIA_DIR/tools/devshell/lib/vars.sh"
                        "${FUCHSIA_BUILD_DIR}/args.gn"
                      )
    local mine
    for mine in "${landmines[@]}"; do
      if [[ "$mine" -nt "${FUCHSIA_BUILD_DIR}/build.ninja" ]]; then
        need_fx_gen_for+=("$mine changed")
      fi
    done

    if [[ ! -d "$FUCHSIA_BUILD_DIR" ]]; then
      need_fx_gen_for+=("$FUCHSIA_BUILD_DIR missing")
    fi

    # We have a build directory, execute force clean checker, usually a no-op
    local _verbose_opt=()
    if [[ -n "$verbose" ]]; then
      _verbose_opt=(--verbose)
    fi
    local -r force_clean_status_file="$(mktemp --tmpdir force_clean_status.XXXXXX)"
    "$FUCHSIA_DIR/scripts/fuchsia-vendored-python" -S \
      "$FUCHSIA_DIR/build/force_clean/force_clean_if_needed.py" \
      "${_verbose_opt[@]}" \
      --gn-bin "$PREBUILT_GN" \
      --checkout-dir "$FUCHSIA_DIR" \
      --build-dir "$FUCHSIA_BUILD_DIR" \
      --output-status="${force_clean_status_file}" || return "$?"

    local force_clean_status
    force_clean_status="$(< "${force_clean_status_file}")"
    rm "${force_clean_status_file}"

    if [[ "${is_logging}" == true ]]; then
      echo "force_clean status: ${force_clean_status}"
    fi
    case "${force_clean_status}" in
      clean:*)
        need_fx_gen_for+=("${force_clean_status}")
        ;;
    esac

    if [[ "${#need_fx_gen_for[@]}" != 0 ]]; then
      if [[ "${is_logging}" == true ]]; then
        echo -e "\\n------ RUNNING gn gen ------"
      fi
      if [[ "${#need_fx_gen_for[@]}" == 1 ]]; then
        # A single reason goes on a single line.
        echo >&2 "Re-running 'fx gen' first (${need_fx_gen_for[*]})"
      else
        # For multiple reasons, print one per line before the header.
        printf >&2 "Re-running 'fx gen' first:\n\n"
        for reason in "${need_fx_gen_for[@]}"; do
          printf >&2 "  %s\n" "$reason"
        done
      fi
      fx-gen || return "$?"
    fi
  fi

  local status

  if [[ "${is_logging}" = true ]]; then
    local tool="ninja"
    if [ -n "${fint_params_path}" ]; then
      tool="fint build"
    fi
    echo -e "\\n------ RUNNING ${tool} ------"
  fi

  # A flag indicating whether a Ninja tool is invoked, instead of a build action.
  local run_ninja_build=true
  for opt in "${ninja_switches[@]}"; do
    case "${opt}" in
      -n|--dry-run|-t*)
        run_ninja_build=
        ;;
    esac
  done


  # LINT.IfChange
  # A file that is only created when the build succeeds.
  local last_build_success_stamp="${FUCHSIA_BUILD_DIR}/last_ninja_build_success.stamp"
  # LINT.ThenChange(//tools/integration/fint/build.go)

  if [[ -n "${run_ninja_build}" ]]; then
    rm -f "${last_build_success_stamp}"
  fi

  local -i target_count=0
  local -i is_clean_build=0
  local -i ninja_log_line_count_before
  ninja_log_line_count_before="$(count-ninja-log-lines)"
  if (( ninja_log_line_count_before < 0 )); then
    is_clean_build=1
    ninja_log_line_count_before=0
  fi

  if [ -n "${fint_params_path}" ]; then
    readonly fint="${FX_CACHE_DIR}/fint"
    "$FUCHSIA_DIR/tools/integration/bootstrap.sh" -o "$fint" || exit $?

    local rbe_wrapper=()
    if fx-rbe-enabled
    then
      rbe_wrapper=("${RBE_WRAPPER[@]}" -- )
    fi

    # It's not ideal that we resort to constructing the textproto file as a
    # string, but it's easier than writing a Go tool solely for the purpose of
    # constructing a protobuf with a couple top-level string fields set.
    if (( concurrency == 0 )); then
      concurrency="$(fx-choose-build-concurrency)"
      # macOS in particular has a low default for number of open file descriptors
      # per process, which is prohibitive for higher job counts. Here we raise
      # the number of allowed file descriptors per process if it appears to be
      # low in order to avoid failures due to the limit. See `getrlimit(2)` for
      # more information.
      local min_limit=$((concurrency * 2))
      if [[ $(ulimit -n) -lt "${min_limit}" ]]; then
        ulimit -n "${min_limit}"
      fi
    fi
    "${rbe_wrapper[@]}" "$fint" -log-level=error build -static="${fint_params_path}" -context=<(echo "
checkout_dir: \"${FUCHSIA_DIR}\"
build_dir: \"${FUCHSIA_BUILD_DIR}\"
job_count: "$concurrency"
")
  else
    if [[ "${#fuchsia_targets[@]}" -gt 0 ]]; then
      # A common mistake is to place a toolchain option after the GN label, thinking it
      # applies to all labels. For example `fx build //src:foo --host` instead of
      # `fx build --host //src:foo`.
      #
      # The latter builds //src:foo for the host, but the former builds //src:foo for Fuchsia,
      # then ignores the --host option. This results in confusing build error messages
      # (see https://fxbug.dev/328421720), so detect this here and make trailing toolchain
      # options an error with an explicit message.
      local last_index=$((${#fuchsia_targets[@]} - 1))
      local last_fuchsia_target="${fuchsia_targets[${last_index}]}"
      if [[ "${last_fuchsia_target}" =~ --.* ]]; then
        fx-error "Toolchain option ${last_fuchsia_target} must be followed by at least one GN label"
        exit 1
      fi
      # TODO(https://fxbug.dev/328316506): Remove --allow-unknown once GN is fixed.
      local -a gn_labels  # define variable to quiet shellcheck
      fx-command-stdout-to-array gn_labels fx-build-api-client fx_build_args_to_labels --allow-unknown --args "${fuchsia_targets[@]}"

      # Convert GN labels to Ninja targets now.
      # TODO(https://fxbug.dev/328316506): Remove --allow-unknown once GN is fixed.
      local -a ninja_targets  # define variable to quiet shellcheck
      fx-command-stdout-to-array ninja_targets fx-build-api-client gn_label_to_ninja_paths --allow-unknown "${gn_labels[@]}"

      # For invalid labels, the above command will print errors to stderr directly and not list them in the result.
      # If ninja_targets is empty, we can stop here.
      if [[ -z "${ninja_targets[*]}" ]]; then
        exit 1
      fi

      if [[ -n "${run_ninja_build}" && -n "${verbose}" ]]; then
        echo "Building Ninja target(s): ${ninja_targets[*]}"
      fi
    else
      ninja_targets=("${fuchsia_targets[@]}")
    fi

    if [[ -n "${run_ninja_build}" ]]; then
      # Write the list of Ninja targets to the build directory. See
      # `//build/api/client last_ninja_artifacts` command.
      #
      # Only update the file when its content changes, to avoid the cache
      # created by `//build/api/client` to be regenerated when not needed.
      local ninja_targets_list="${ninja_targets[*]}"
      # LINT.IfChange
      local ninja_targets_list_file="${FUCHSIA_BUILD_DIR}/last_ninja_build_targets.txt"
      # LINT.ThenChange(//tools/integration/fint/build.go)
      if [[ ! -f "${ninja_targets_list_file}" || "$(<"${ninja_targets_list_file}")" != "${ninja_targets_list}" ]]; then
        printf "%s" "${ninja_targets_list}" > "${ninja_targets_list_file}"
      fi

      # If enable_jobserver is set in args.gn, add a --jobserver option to
      # the Ninja command-line.
      local enable_jobserver
      enable_jobserver="$("$PREBUILT_JQ" .enable_jobserver "${FUCHSIA_BUILD_DIR}/args.json")"
      if [[ "$enable_jobserver" == "true" ]]; then
        if [[ -n "${verbose}" ]]; then
          echo "Ninja jobserver mode is enabled."
        fi
        ninja_switches+=(--jobserver)
      fi

      # When using any remote build services, do a quick authentication
      # check before starting the build.
      if [[ "$skip_auth_check" == 0 ]]; then
        fx-command-run rbe preflight
      fi
    fi

    (fx-run-ninja "${is_logging}" "$PREBUILT_NINJA" -C "${FUCHSIA_BUILD_DIR}" \
      "${ninja_switches[@]}" "${ninja_targets[@]}")

  fi

  status=$?
  local -i ninja_log_line_count_after
  ninja_log_line_count_after="$(count-ninja-log-lines)"
  target_count="$(( ninja_log_line_count_after - ninja_log_line_count_before ))"

  if [[ "${status}" == 0 && -n "${run_ninja_build}" ]]; then
    touch "${last_build_success_stamp}"
  fi

  declare -r end_time="${EPOCHREALTIME/./}"
  track-build-event "${start_time}" "${end_time}" "${status}" "${ninja_switches[*]}" "${switches[*]}" "${fuchsia_targets[*]}" "${FUCHSIA_BUILD_DIR}" "${target_count}" "${is_clean_build}"&
  exit-with-message
}

function exit-with-message {
  if [[ "${is_logging}" = true ]]; then
    fx-warn "Debug log saved to ${log_file}. Please attach this file when reporting a bug"
  elif [[ "${status}" -ne 0 ]]; then
    echo >&2 "Hint: run \`fx build\` with the option \`--log LOGFILE\` to generate a debug log if you are reporting a bug."
  fi
  exit "${status}"
}

main "$@"