tools/devshell/serve-updates

#!/bin/bash
# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

#### CATEGORY=Software delivery
### start the update server and attach to a running fuchsia device
## usage: fx serve-updates [-v] [-l host[:port]] [-c version] [--no-auto-config] [--name NAME]
##
##   -l host:port       host and port that "pm serve" will listen on
##   -c version         configuration format version for the served config.json
##                      that "pm" will serve. Valid choices: 1 or 2.
##                      Choosing `1` will serve a file which can be processed by
##                      `pkgctl repo add ... -f 1`.
##                      Choosing `2` will serve a file which can be processed by
##                      `pkgctl repo ...` without the `-f 1` switch.
##   --no-auto-config   do not configure this host as a package server on the device
##   --name NAME        Name the generated update source config NAME.
##   --[no-]persist     enable or disable persistence of repository metadata. Disabled
##                      by default.
##   -v                 verbose mode, shows info and debug messages from "pm"
##   -C|--clean         clean the package repository first. This flag is only
##                      valid if the incremental package publishing is enabled.
##
## This command supports incremental package publishing. If enabled, it will
## auto-publish packages as they are created or modified.
##
## To enable incremental package serving, run "fx --enable=incremental serve-updates ..."

source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"/lib/vars.sh || exit $?
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"/lib/updates.sh || exit $?
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"/lib/fx-optional-features.sh || exit $?
fx-config-read

function usage {
  fx-command-help serve-updates
}

fx-standard-switches "$@"
set -- "${FX_ARGV[@]}"

port=""
serve_flags=()
verbose=false
auto_config=true
clean_first=false
config_format="2"
source_name=""
persist="--no-persist"
while (($#)); do
  case "$1" in
    -l)
      shift
      port="${1##*:}"
      serve_flags+=(-l "$1")
      ;;
    -c)
      config_format=("$2")
      shift
      ;;
    --name)
      source_name="$2"
      shift
      ;;
    --no-persist)
      ;;
    --persist)
      persist="--persist"
      ;;
    --no-auto-config)
      auto_config=false
      ;;
    -v|--verbose)
      verbose=true
      ;;
    -C|--clean)
      clean_first=true
      ;;
    *)
      echo "Unrecognized option: $1"
      usage
      exit 1
      ;;
  esac
  shift
done
serve_flags+=("-c" "${config_format}")

if fx-is-bringup; then
  fx-error "$0 is not supported in the bringup build configuration, as there are no package features in bringup."
  exit 1
fi

if [[ "${verbose}" != true ]]; then
  serve_flags+=("-q")
fi

# Determine which server we are using.
server_mode="$(package-server-mode)"
err=$?
if [[ "${err}" -ne 0 ]]; then
  exit 1
fi

pm_pid=
ffx_server_started=
cleanup() {
  if [[ "${server_mode}" == "pm" ]]; then
    if [[ -n "${pm_pid}" ]]; then
      if kill -0 "${pm_pid}" 2> /dev/null; then
        kill -TERM "${pm_pid}" 2> /dev/null
        wait "${pm_pid}" 2> /dev/null
      fi
    fi
  else
    if [[ ! -z "${ffx_server_started}" ]]; then
      ffx-stop-server
    fi
  fi
}
trap cleanup EXIT

log() {
  # This format matches bootserver so that `fx serve` ui is easier to read.
  echo "$(date '+%Y-%m-%d %H:%M:%S') [serve-updates] $@"
}

# In any multi-homing scenario (two target interfaces, or two host interfaces),
# the resolve process can return different results at different times. As such
# we pin the address here, and let the ssh check loop below perform a clear
# whenever it is in the "discovery" state. This stabilizes the "connection"
# under multi-homed conditions.
serve_updates_target_addr=""
target-addr() {
  if [[ -z "${serve_updates_target_addr}" ]]; then
    serve_updates_target_addr="$(get-device-addr-resource)"

    if [[ -n "${serve_updates_target_addr}" && -n "$(get-fuchsia-device-port)" ]]; then
      serve_updates_target_addr="${serve_updates_target_addr}:$(get-fuchsia-device-port)"
    fi
  fi

  if [[ -n "${serve_updates_target_addr}" ]]; then
    echo "${serve_updates_target_addr}"
    return 0
  fi

  return 1
}

clear-target-addr() {
  serve_updates_target_addr=""
}

with-pinned-target() {
  # Ensure we pin in the running shell before making a subshell. We also want
  # to make sure we get an address at all.
  if ! target-addr > /dev/null; then
    return 1
  fi

  (
    export FUCHSIA_DEVICE_NAME="$(target-addr)"
    "$@"
  )
}

repo_dir="${FUCHSIA_BUILD_DIR}/amber-files"

if is_feature_enabled "incremental"; then
  # FIXME(http://fxbug.dev/79636): ffx doesn't yet support incremental package building.
  server_mode="pm"

  fx-info "Incremental package auto-publishing is enabled."
  serve_flags+=( -p "all_package_manifests.list" )

  # macOS in particular has a low default for number of open file descriptors
  # per process, which is prohibitive for higher job counts. Here we raise
  # the number of allowed file descriptors per process if it appears to be
  # low in order to avoid failures due to the limit. See `getrlimit(2)` for
  # more information.
  if [[ $(ulimit -n) -lt 1000 ]]; then
    ulimit -n 32768
  fi

  if $clean_first; then
    $verbose && echo -n >&2 "Cleaning the package repository..."
    if [[ -d "${repo_dir}" ]]; then
      rm -Rf "${repo_dir}"
    fi
    $verbose && echo >&2 "done"
  fi
  $verbose && echo -n >&2 "Preparing the package repository..."
  # only show output of prepare_publish if it fails
  out="$(fx-command-run build build/images/updates:prepare_publish 2>&1)"
  if [[ $? -ne 0 ]]; then
    echo >&2 "${out}"
    exit $?
  fi
  $verbose && echo >&2 "done"
else
  if $clean_first; then
    fx-error "Flag '-C' or '--clean' can only be used if the incremental feature is enabled"
    exit 1
  fi
fi

if [[ "${server_mode}" = "pm" ]]; then
  # Default the port to 8083 if it is unset.
  if [[ -z "${port}" ]]; then
    port="8083"
  fi

  pm_serve_args=( serve -vt -repo "${repo_dir}" "${serve_flags[@]}" )

  # the manifest list file has relative paths and "pm serve" looks for them
  # from PWD, so pm command needs to be executed in "$FUCHSIA_BUILD_DIR"
  cd "${FUCHSIA_BUILD_DIR}"
else
  # FIXME(http://fxbug.dev/79636): ffx doesn't yet support incremental package building.
  if is_feature_enabled "incremental"; then
    fx-error "The feature 'incremental' is not supported yet by ffx."
    fx-error "To use the 'incremental' feature, switch to the legacy pm"
    fx-error "package server with:"
    fx-error ""
    fx-error "$ ffx config set repository.server.mode pm"
    fx-error "$ ffx doctor --restart-daemon"
    fx-error ""
    fx-error "See http://fxbug.dev/79636 for more details."
    exit 1
  fi
fi

# Error out if we can't start a package server.
check-if-we-can-start-package-server "${server_mode}" "" "${port}"
err=$?
if [[ "${err}" -ne 0 ]]; then
  exit 1
fi

if [[ "${auto_config}" == false ]]; then
  log "Flag '--no-auto-config' used, automatic device configuration disabled."
  log "Use 'fx -d DEVICE add-update-source --port ${port}' to reconfigure devices as needed, since it is not persistent accross reboots."
  log "Serving packages on port ${port}..."

  if [[ "${server_mode}" = "pm" ]]; then
    fx-command-exec host-tool pm "${pm_serve_args[@]}"
  else
    log "Repository is being served in the background by ffx"
    exit 0
  fi
fi

if [[ "${server_mode}" = "pm" ]]; then
  fx-command-exec host-tool pm "${pm_serve_args[@]}" &
  pm_pid=$!

  # Allow a little slack for pm serve to startup, that way the first kill -0 will catch a dead server
  sleep 0.1
  if ! kill -0 "${pm_pid}" 2> /dev/null; then
    wait "${pm_pid}"
    err=$?
    fx-error "Server died"
    fx-error "If the server exited with 'address already in use', pm may be colliding with"
    fx-error "another 'pm serve', or the ffx repository server."
    fx-error ""
    fx-error "To disable the ffx repository server, run:"
    fx-error ""
    fx-error "$ ffx config set repository.server.mode pm"
    fx-error "$ ffx doctor --restart-daemon"
    wait
    exit $err
  fi
else
  ffx-start-server
  err=$?
  if [[ "${err}" -ne 0 ]]; then
    exit "${err}"
  fi

  ffx_server_started=1

  if [[ "${verbose}" == true ]]; then
    # FIXME(http://fxbug.dev/78891): `ffx daemon log` doesn't have a mechanism for
    # filtering based off of a specific tag, so we'll grep for the lines that
    # denote traffic to the web server.
    fx-command-run ffx daemon log --follow --line-count 0 | grep ' \(pkg::repository::server\)\|\(ffx_daemon_service_repo\): ' &
    pm_pid=$!
  fi
fi

# Check if device is set via fx set-device before the discovery loop is started
device_name="$(get-device-name 2>/dev/null)"
if [[ -z "${device_name}" ]]; then
  fx-warn ""
  fx-warn "No default device set. fx serve-updates will get stuck on \"Discovery...\""
  fx-warn "Run 'fx list-devices' and then 'fx set-device'"
  fx-warn "\n"
else
  fx-info ""
  fx-info "Default device set: ${device_name}"
  fx-info "\n"
fi

log "Discovery..."

# State is used to prevent too much output
state="discover"

# Keep track if we reported any discovery failures
reported_discovery_failed=0
while true; do
  if [[ -n "${pm_pid}" ]]; then
    if ! kill -0 "${pm_pid}" 2> /dev/null; then
      log "Server died, exiting"
      pm_pid=
      exit 1
    fi
  fi

  if [[ "${state}" == "discover" ]]; then
    # While we're still trying to connect to the device, clear the target
    # address state so we re-resolve.
    clear-target-addr
    with-pinned-target fx-command-run shell exit 2>/dev/null
    ping_result=$?

    if [[ "${ping_result}" == 0 ]]; then
      log "Device up"
      reported_discovery_failed=0
      state="config"
    elif [[ "${reported_discovery_failed}" == 0 ]]; then
      reported_discovery_failed=1

      # Log if we don't know the device's address yet, or if we don't have a device configured.
      device_addr="$(get-fuchsia-device-addr 2>/dev/null)"
      if [[ -z "${device_addr}" ]]; then
        device_name="$(get-device-name 2>/dev/null)"
        if [[ -n "${device_name}" ]]; then
          log "Trying to resolve '${device_name}'..."
        else
          log "Either no device or multiple devices are discoverable. Run 'fx list-devices' and if necessary run 'fx set-device'."
        fi
      else
        log "Trying to connect to ${device_addr}..."
      fi
    fi
  else
    with-pinned-target fx-command-run shell -O check > /dev/null 2>&1
    ping_result=$?
  fi

  if [[ "${state}" == "config" ]]; then
    log "Registering devhost as update source"

    add_update_source_args=( "${persist}" --port "${port}" --server-mode "${server_mode}" )
    if [[ ! -z "${source_name}" ]]; then
      add_update_source_args+=( --name "${source_name}" )
    fi

    if with-pinned-target fx-command-run add-update-source "${add_update_source_args[@]}"; then
      log "Ready to push packages!"
      # Log the uptime so that it is clear(er) from fx serve if the device rebooted.
      # The tr is present as the output sometimes contains spurious control characters.
      clock=$(with-pinned-target fx-command-run shell clock --monotonic)
      if [[ -n ${clock} ]]; then
        uptime=$(echo $clock | tr '[[:cntrl:]]' ' ' )
        log "Target uptime: $(( $uptime / 1000000000 ))"
      fi
      state="ready"
    else
      log "Device lost while configuring update source"
      state="discover"
    fi
  fi

  if [[ "${state}" == "ready" ]]; then
    if [[ "${ping_result}" != 0 ]]; then
      log "Device lost"
      state="discover"
    else
      sleep 1
    fi
  fi
done

# See EXIT trap above for cleanup that occurs