// Copyright 2018 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT

#include <lib/cmdline.h>
#include <lib/counters.h>
#include <trace.h>

#include <kernel/thread.h>
#include <lk/init.h>
#include <object/pager_dispatcher.h>
#include <object/thread_dispatcher.h>
#include <vm/page_source.h>

#define LOCAL_TRACE 0

KCOUNTER(dispatcher_pager_create_count, "dispatcher.pager.create")
KCOUNTER(dispatcher_pager_destroy_count, "dispatcher.pager.destroy")
KCOUNTER(dispatcher_pager_overtime_wait_count, "dispatcher.pager.overtime_waits")
KCOUNTER(dispatcher_pager_total_request_count, "dispatcher.pager.total_requests")
KCOUNTER(dispatcher_pager_succeeded_request_count, "dispatcher.pager.succeeded_requests")
KCOUNTER(dispatcher_pager_failed_request_count, "dispatcher.pager.failed_requests")
KCOUNTER(dispatcher_pager_timed_out_request_count, "dispatcher.pager.timed_out_requests")

// Log warnings every |pager_overtime_wait_seconds| a thread is blocked waiting on a page
// request. If the thread has been waiting for |pager_overtime_timeout_seconds|, return an
// error instead of waiting indefinitely.
static constexpr uint64_t kDefaultPagerOvertimeWaitSeconds = 20;
static constexpr uint64_t kDefaultPagerOvertimeTimeoutSeconds = 300;
static uint64_t pager_overtime_wait_seconds = kDefaultPagerOvertimeWaitSeconds;
static uint64_t pager_overtime_timeout_seconds = kDefaultPagerOvertimeTimeoutSeconds;

zx_status_t PagerDispatcher::Create(KernelHandle<PagerDispatcher>* handle, zx_rights_t* rights) {
  fbl::AllocChecker ac;
  KernelHandle new_handle(fbl::AdoptRef(new (&ac) PagerDispatcher()));
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }

  *rights = default_rights();
  *handle = ktl::move(new_handle);
  return ZX_OK;
}

PagerDispatcher::PagerDispatcher() : SoloDispatcher() {
  kcounter_add(dispatcher_pager_create_count, 1);
}

PagerDispatcher::~PagerDispatcher() {
  DEBUG_ASSERT(srcs_.is_empty());
  kcounter_add(dispatcher_pager_destroy_count, 1);
}

zx_status_t PagerDispatcher::CreateSource(fbl::RefPtr<PortDispatcher> port, uint64_t key,
                                          fbl::RefPtr<PageSource>* src_out) {
  fbl::AllocChecker ac;
  auto src = fbl::AdoptRef(new (&ac) PagerSource(this, ktl::move(port), key));
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }

  Guard<Mutex> guard{&list_mtx_};
  srcs_.push_front(src);
  *src_out = ktl::move(src);
  return ZX_OK;
}

fbl::RefPtr<PagerSource> PagerDispatcher::ReleaseSource(PagerSource* src) {
  Guard<Mutex> guard{&list_mtx_};
  return src->InContainer() ? srcs_.erase(*src) : nullptr;
}

void PagerDispatcher::on_zero_handles() {
  Guard<Mutex> guard{&list_mtx_};
  while (!srcs_.is_empty()) {
    fbl::RefPtr<PagerSource> src = srcs_.pop_front();

    // Call unlocked to prevent a double-lock if PagerDispatcher::ReleaseSource is called,
    // and to preserve the lock order that PagerSource locks are acquired before the
    // list lock.
    guard.CallUnlocked([&src]() mutable {
      src->Close();
      src->OnDispatcherClosed();
    });
  }
}

zx_status_t PagerDispatcher::RangeOp(uint32_t op, fbl::RefPtr<VmObject> vmo, uint64_t offset,
                                     uint64_t length, uint64_t data) {
  switch (op) {
    case ZX_PAGER_OP_FAIL: {
      auto signed_data = static_cast<int64_t>(data);
      if (signed_data < INT32_MIN || signed_data > INT32_MAX) {
        return ZX_ERR_INVALID_ARGS;
      }
      auto error_status = static_cast<zx_status_t>(data);
      if (!PageSource::IsValidFailureCode(error_status)) {
        return ZX_ERR_INVALID_ARGS;
      }
      return vmo->FailPageRequests(offset, length, error_status);
    }
    default:
      return ZX_ERR_NOT_SUPPORTED;
  }
}

PagerSource::PagerSource(PagerDispatcher* dispatcher, fbl::RefPtr<PortDispatcher> port,
                         uint64_t key)
    : PageSource(), pager_(dispatcher), port_(ktl::move(port)), key_(key) {
  LTRACEF("%p key %lx\n", this, key_);
}

PagerSource::~PagerSource() {
  LTRACEF("%p\n", this);
  DEBUG_ASSERT(closed_);
  DEBUG_ASSERT(!complete_pending_);
}

void PagerSource::GetPageAsync(page_request_t* request) {
  Guard<Mutex> guard{&mtx_};
  ASSERT(!closed_);

  QueueMessageLocked(request);
}

void PagerSource::QueueMessageLocked(page_request_t* request) {
  if (packet_busy_) {
    list_add_tail(&pending_requests_, &request->provider_node);
    return;
  }

  packet_busy_ = true;
  active_request_ = request;

  uint64_t offset, length;
  uint16_t cmd;
  if (request != &complete_request_) {
    cmd = ZX_PAGER_VMO_READ;
    offset = request->offset;
    length = request->length;

    // The vm subsystem should guarantee this
    uint64_t unused;
    DEBUG_ASSERT(!add_overflow(offset, length, &unused));

    // Trace flow events require an enclosing duration.
    VM_KTRACE_DURATION(1, "page_request_queue", offset, length);
    VM_KTRACE_FLOW_BEGIN(1, "page_request_queue", reinterpret_cast<uintptr_t>(&packet_));
  } else {
    offset = length = 0;
    cmd = ZX_PAGER_VMO_COMPLETE;
  }

  zx_port_packet_t packet = {};
  packet.key = key_;
  packet.type = ZX_PKT_TYPE_PAGE_REQUEST;
  packet.page_request.command = cmd;
  packet.page_request.offset = offset;
  packet.page_request.length = length;

  packet_.packet = packet;

  // We can treat ZX_ERR_BAD_HANDLE as if the packet was queued
  // but the pager service never responds.
  // TODO: Bypass the port's max queued packet count to prevent ZX_ERR_SHOULD_WAIT
  ASSERT(port_->Queue(&packet_, ZX_SIGNAL_NONE) != ZX_ERR_SHOULD_WAIT);
}

void PagerSource::ClearAsyncRequest(page_request_t* request) {
  Guard<Mutex> guard{&mtx_};
  ASSERT(!closed_);

  if (request == active_request_) {
    if (request != &complete_request_) {
      // Trace flow events require an enclosing duration.
      VM_KTRACE_DURATION(1, "page_request_queue", active_request_->offset, active_request_->length);
      VM_KTRACE_FLOW_END(1, "page_request_queue", reinterpret_cast<uintptr_t>(&packet_));
    }
    // Condition on whether or not we actually cancel the packet, to make sure
    // we don't race with a call to PagerSource::Free.
    if (port_->CancelQueued(&packet_)) {
      OnPacketFreedLocked();
    }
  } else if (list_in_list(&request->provider_node)) {
    list_delete(&request->provider_node);
  }
}

void PagerSource::SwapRequest(page_request_t* old, page_request_t* new_req) {
  Guard<Mutex> guard{&mtx_};
  ASSERT(!closed_);

  if (list_in_list(&old->provider_node)) {
    list_replace_node(&old->provider_node, &new_req->provider_node);
  } else if (old == active_request_) {
    active_request_ = new_req;
  }
}

void PagerSource::OnDetach() {
  Guard<Mutex> guard{&mtx_};
  ASSERT(!closed_);

  complete_pending_ = true;
  QueueMessageLocked(&complete_request_);
}

void PagerSource::OnClose() {
  fbl::RefPtr<PagerSource> self;

  Guard<Mutex> guard{&mtx_};
  ASSERT(!closed_);

  closed_ = true;
  if (!complete_pending_) {
    // We know PagerDispatcher::on_zero_handles hasn't been invoked, since that would
    // have already closed this pager source.
    self = pager_->ReleaseSource(this);
  }  // else this is released in PagerSource::Free
}

void PagerSource::OnDispatcherClosed() {
  // The pager dispatcher's reference to this object is the only one we completely control. Now
  // that it's gone, we need to make sure that port_ doesn't end up with an invalid pointer
  // to packet_ if all external RefPtrs to this object go away.
  Guard<Mutex> guard{&mtx_};

  if (complete_pending_) {
    if (port_->CancelQueued(&packet_)) {
      // We successfully cancelled the message, so we don't have to worry about
      // PagerSource::Free being called.
      complete_pending_ = false;
    } else {
      // If we failed to cancel the message, then there is a pending call to
      // PagerSource::Free. We need to make sure the object isn't deleted too early,
      // so have it keep a reference to itself, which PagerSource::Free will then
      // clean up.
      self_ref_ = fbl::RefPtr(this);
    }
  } else {
    // Either the complete message had already been dispatched when this object was closed or
    // PagerSource::Free was called between this object being closed and this method taking the
    // lock. In either case, the port no longer has a reference and cleanup is already done.
  }
}

void PagerSource::Free(PortPacket* packet) {
  fbl::RefPtr<PagerSource> self;

  Guard<Mutex> guard{&mtx_};
  if (active_request_ != &complete_request_) {
    // Trace flow events require an enclosing duration.
    VM_KTRACE_DURATION(1, "page_request_queue", active_request_->offset, active_request_->length);
    VM_KTRACE_FLOW_END(1, "page_request_queue", reinterpret_cast<uintptr_t>(packet));
    OnPacketFreedLocked();
  } else {
    complete_pending_ = false;
    if (closed_) {
      // If the source is closed, we need to do delayed cleanup. If the dispatcher
      // has already been torn down, then there's a self-reference we need to clean
      // up. Otherwise, clean up the dispatcher's reference to us.
      self = ktl::move(self_ref_);
      if (!self) {
        self = pager_->ReleaseSource(this);
      }
    }
  }
}

void PagerSource::OnPacketFreedLocked() {
  packet_busy_ = false;
  active_request_ = nullptr;
  if (!list_is_empty(&pending_requests_)) {
    QueueMessageLocked(list_remove_head_type(&pending_requests_, page_request, provider_node));
  }
}

zx_status_t PagerSource::WaitOnEvent(Event* event) {
  ThreadDispatcher::AutoBlocked by(ThreadDispatcher::Blocked::PAGER);
  kcounter_add(dispatcher_pager_total_request_count, 1);
  uint32_t waited = 0;
  // declare a lambda to calculate our deadline to avoid an excessively large statement in our
  // loop condition.
  auto make_deadline = []() {
    if (pager_overtime_wait_seconds == 0) {
      return Deadline::infinite();
    } else {
      return Deadline::after(ZX_SEC(pager_overtime_wait_seconds));
    }
  };
  zx_status_t result;
  while ((result = event->Wait(make_deadline())) == ZX_ERR_TIMED_OUT) {
    waited++;
    // We might trigger this loop multiple times as we exceed multiples of the overtime counter, but
    // we only want to count each unique overtime event in the kcounter.
    if (waited == 1) {
      dispatcher_pager_overtime_wait_count.Add(1);
    }

    // Error out if we've been waiting for longer than the specified timeout, to allow the rest of
    // the system to make progress (if possible).
    if (pager_overtime_timeout_seconds > 0 &&
        waited * pager_overtime_wait_seconds >= pager_overtime_timeout_seconds) {
      printf("ERROR Pager source %p has been blocked for %" PRIu64
             " seconds. Page request timed out.\n",
             this, pager_overtime_timeout_seconds);
      dump_thread(Thread::Current::Get(), false);
      kcounter_add(dispatcher_pager_timed_out_request_count, 1);
      return ZX_ERR_TIMED_OUT;
    }

    // Determine whether we have any requests that have not yet been received off of the port.
    bool active;
    {
      Guard<Mutex> guard{&mtx_};
      active = !!active_request_;
    }
    printf("WARNING pager source %p has been blocked for %" PRIu64
           " seconds with%s message waiting on port.\n",
           this, waited * pager_overtime_wait_seconds, active ? "" : " no");
    // Dump out the rest of the state of the oustanding requests.
    Dump();
  }

  if (result == ZX_OK) {
    kcounter_add(dispatcher_pager_succeeded_request_count, 1);
  } else {
    // Only counts failures that are *not* pager timeouts. Timeouts are tracked with
    // dispatcher_pager_timed_out_request_count, which is updated above when we
    // return early with ZX_ERR_TIMED_OUT.
    kcounter_add(dispatcher_pager_failed_request_count, 1);
  }

  return result;
}

static void pager_init_func(uint level) {
  pager_overtime_wait_seconds = gCmdline.GetUInt64("kernel.userpager.overtime_wait_seconds",
                                                   kDefaultPagerOvertimeWaitSeconds);
  pager_overtime_timeout_seconds = gCmdline.GetUInt64("kernel.userpager.overtime_timeout_seconds",
                                                      kDefaultPagerOvertimeTimeoutSeconds);
}

LK_INIT_HOOK(pager_init, &pager_init_func, LK_INIT_LEVEL_LAST)