// Copyright 2020 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/devices/usb/drivers/xhci/xhci-event-ring.h"

#include <lib/async-loop/cpp/loop.h>
#include <lib/async/cpp/executor.h>
#include <lib/fit/defer.h>
#include <lib/fpromise/promise.h>
#include <lib/zx/clock.h>
#include <zircon/errors.h>

#include <optional>

#include "src/devices/usb/drivers/xhci/usb-xhci.h"
#include "src/devices/usb/drivers/xhci/xhci-enumeration.h"

namespace usb_xhci {

// The minimum required number of event ring segment table entries.
static constexpr uint16_t kMinERSTEntries = 16;

// The target number of TRBs we would like to have in our event ring, available
// to the HC, at startup.
static constexpr size_t kTargetEventRingTRBs = 2048;

// The number of TRBs we should update ERDP Reg after. According to note in 4.9.4, we should process
// as many Events as possible before writing to ERDP. But practically, we need to limit the maximum
// number of TRBs before writing. Otherwise, the lag between consuming an ED and moving the ERDP
// forward may cause the ring to become full with consumed EDs which have yet to be relinquished.
static constexpr size_t kUpdateERDPAfterTRBs = 128;

zx_status_t EventRingSegmentTable::Init(size_t page_size, const zx::bti& bti, bool is_32bit,
                                        uint32_t erst_max, ERSTSZ erst_size,
                                        const dma_buffer::BufferFactory& factory,
                                        fdf::MmioBuffer* mmio) {
  erst_size_ = erst_size;
  bti_ = &bti;
  page_size_ = page_size;
  is_32bit_ = is_32bit;
  mmio_.emplace(mmio->View(0));
  zx_status_t status = factory.CreatePaged(bti, page_size_, false, &erst_);
  if (status != ZX_OK) {
    return status;
  }
  if (is_32bit && (erst_->phys()[0] >= UINT32_MAX)) {
    return ZX_ERR_NO_MEMORY;
  }

  count_ = page_size / sizeof(ERSTEntry);
  if (count_ > erst_max) {
    count_ = erst_max;
  }
  entries_ = static_cast<ERSTEntry*>(erst_->virt());
  return ZX_OK;
}

zx_status_t EventRingSegmentTable::AddSegment(zx_paddr_t paddr) {
  if (offset_ >= count_) {
    if (offset_ > count_) {
      return ZX_ERR_BAD_STATE;
    }
    return ZX_ERR_NO_MEMORY;
  }
  ERSTEntry entry;
  entry.address_low = static_cast<uint32_t>(paddr & UINT32_MAX);
  entry.address_high = static_cast<uint32_t>(paddr >> 32);
  entry.size = static_cast<uint16_t>(page_size_ / kMinERSTEntries);
  entries_[offset_] = entry;
  hw_mb();
  offset_++;
  erst_size_.set_TableSize(offset_).WriteTo(&mmio_.value());
  erst_pressure_++;
  return ZX_OK;
}

zx_status_t EventRing::Init(size_t page_size, const zx::bti& bti, fdf::MmioBuffer* buffer,
                            bool is_32bit, uint32_t erst_max, ERSTSZ erst_size, ERDP erdp_reg,
                            IMAN iman_reg, uint8_t cap_length, HCSPARAMS1 hcs_params_1,
                            CommandRing* command_ring, DoorbellOffset doorbell_offset, UsbXhci* hci,
                            HCCPARAMS1 hcc_params_1, uint64_t* dcbaa, uint16_t interrupter,
                            inspect::Node* interrupter_node) {
  fbl::AutoLock l(&segment_mutex_);
  erdp_reg_ = erdp_reg;
  hcs_params_1_ = hcs_params_1;
  mmio_ = buffer;
  bti_ = &bti;
  page_size_ = page_size;
  is_32bit_ = is_32bit;
  mmio_ = buffer;
  iman_reg_ = iman_reg;
  cap_length_ = cap_length;
  command_ring_ = command_ring;
  doorbell_offset_ = doorbell_offset;
  hci_ = hci;
  hcc_params_1_ = hcc_params_1;
  dcbaa_ = dcbaa;
  interrupter_ = interrupter;

  if (interrupter_node != nullptr) {
    total_event_trbs_ = interrupter_node->CreateUint("Total Event TRBs", 0);
    max_single_irq_event_trbs_ = interrupter_node->CreateUint("Max single IRQ event TRBs", 0);
    events_ = interrupter_node->CreateChild("Events");
  }

  zx_status_t status =
      segments_.Init(page_size, bti, is_32bit, erst_max, erst_size, hci->buffer_factory(), mmio_);
  if (status != ZX_OK) {
    return status;
  }

  // Attempt to grow our event ring to the desired size.
  do {
    status = AddSegment(true);
    if (status != ZX_OK) {
      zxlogf(WARNING,
             "Event ring failed to add a segment during initialization. "
             "The EventRing currently has space for %zu TRBs (status %d)",
             segments_.TrbCount(), status);

      if (segments_.TrbCount() == 0) {
        return status;
      }
      break;
    }
  } while (segments_.CanGrow() && (segments_.TrbCount() < kTargetEventRingTRBs));

  return ZX_OK;
}

void EventRing::RemovePressure() {
  fbl::AutoLock l(&segment_mutex_);
  segments_.RemovePressure();
}

size_t EventRing::GetPressure() {
  fbl::AutoLock l(&segment_mutex_);
  return segments_.Pressure();
}

zx_status_t EventRing::AddSegmentIfNone() {
  if (!erdp_phys_) {
    return AddSegment();
  }
  return ZX_OK;
}

zx_status_t EventRing::AddSegment(bool initialization) {
  if (segments_.Pressure() < segments_.SegmentCount()) {
    segments_.AddPressure();
    return ZX_OK;
  }
  std::unique_ptr<dma_buffer::ContiguousBuffer> buffer;
  {
    std::unique_ptr<dma_buffer::ContiguousBuffer> buffer_tmp;
    zx_status_t status = hci_->buffer_factory().CreateContiguous(
        *bti_, page_size_,
        static_cast<uint32_t>(page_size_ == zx_system_get_page_size() ? 0 : page_size_ >> 12),
        &buffer_tmp);
    if (status != ZX_OK) {
      return status;
    }
    buffer = std::move(buffer_tmp);
  }
  if (is_32bit_ && (buffer->phys() >= UINT32_MAX)) {
    return ZX_ERR_NO_MEMORY;
  }
  zx_status_t status = segments_.AddSegment(buffer->phys());
  if (status != ZX_OK) {
    return status;
  }
  bool needs_iterator = false;
  if (!erdp_phys_) {
    erdp_phys_ = buffer->phys();
    erdp_virt_ = static_cast<TRB*>(buffer->virt());
    needs_iterator = true;
  }
  // Segments added to event ring on initialization (before event ring is used) are not considered
  // "new segments"
  buffers_.push_back(std::make_unique<SegmentBuf>(std::move(buffer), !initialization));
  if (needs_iterator) {
    buffers_it_ = buffers_.begin();
  }
  return ZX_OK;
}

fpromise::promise<void, zx_status_t> EventRing::HandlePortStatusChangeEvent(uint8_t port_id) {
  auto sc = PORTSC::Get(cap_length_, port_id).ReadFrom(mmio_);
  std::optional<fpromise::promise<void, zx_status_t>> pending_enumeration;
  // Read status bits
  bool needs_enum = false;

  // xHCI doesn't provide a way of retrieving the port speed prior to a device being fully
  // online (without using ACPI or another out-of-band mechanism).
  // In order to correctly enumerate devices, we use heuristics to try and determine
  // whether or not a port is 2.0 or 3.0.
  if (sc.CCS()) {
    // Wait for the port to exit polling state, if applicable.
    // Only 2.0 ports should go into a polling state, so if we get here,
    // we can be sure that it's a 2.0 port. Some controllers may skip this step though....
    if ((sc.PLS() == PORTSC::Polling) || (hci_->GetPortState()[port_id - 1].is_connected &&
                                          !hci_->GetPortState()[port_id - 1].is_USB3)) {
      // USB 2.0 port connect
      if (!hci_->GetPortState()[port_id - 1].is_connected) {
        // USB 2.0 requires a port reset to advance to U0
        Usb2DeviceAttach(port_id);
        needs_enum = true;
        zxlogf(DEBUG, "Port %d is a USB 2 device and will be enumerated.", port_id);
      }
    } else {
      // USB 3.0 port connect, since we got a connect status bit set,
      // and were not polling.
      if (!hci_->GetPortState()[port_id - 1].is_connected) {
        Usb3DeviceAttach(port_id);
        needs_enum = true;
        zxlogf(DEBUG, "Port %d is a USB 3 device and will be enumerated.", port_id);
      }
      if ((sc.PLS() == PORTSC::U0) && (sc.PED()) && (!sc.PR()) &&
          !hci_->GetPortState()[port_id - 1].link_active) {
        // Set the link active bit here to prevent us from onlining the same device twice.
        hci_->GetPortState()[port_id - 1].link_active = true;
        needs_enum = false;
        pending_enumeration = LinkUp(port_id);
      }
    }

    // Link could be active from connect status change above.
    // To prevent enumerating a device twice, we ensure that the link wasn't previously active
    // before enumerating.
    if ((sc.PLS() == PORTSC::U0) && sc.CCS() && !(hci_->GetPortState()[port_id - 1].link_active)) {
      if (!hci_->GetPortState()[port_id - 1].is_connected) {
        // Spontaneous initialization of USB 3.0 port without going through
        // CSC event. We know this is USB 3.0 since this cannot possibly happen
        // with a 2.0 port.
        hci_->GetPortState()[port_id - 1].is_USB3 = true;
        hci_->GetPortState()[port_id - 1].is_connected = true;
      }
      hci_->GetPortState()[port_id - 1].link_active = true;
      if (!hci_->GetPortState()[port_id - 1].is_USB3) {
        // USB 2.0 specification section 9.2.6.3
        // states that we must wait 10 milliseconds.
        needs_enum = false;
        pending_enumeration = hci_->Timeout(interrupter_, zx::deadline_after(zx::msec(10)))
                                  .and_then([=]() { return LinkUp(static_cast<uint8_t>(port_id)); })
                                  .box();
      } else {
        needs_enum = false;
        pending_enumeration = LinkUp(static_cast<uint8_t>(port_id));
      }
    }

  } else {
    // For hubs, we need to take the device offline from the bus's standpoint before tearing down
    // the hub. This means that the slot has to be kept alive until the hub driver is removed.
    hci_->GetPortState()[port_id - 1].retry = false;
    hci_->GetPortState()[port_id - 1].link_active = false;
    hci_->GetPortState()[port_id - 1].is_connected = false;
    hci_->GetPortState()[port_id - 1].is_USB3 = false;
    if (hci_->GetPortState()[port_id - 1].slot_id) {
      ScheduleTask(hci_->DeviceOffline(hci_->GetPortState()[port_id - 1].slot_id).box());
    }
  }

  // Update registers if not init
  if (sc.OCC()) {
    bool overcurrent = sc.OCA();
    PORTSC::Get(cap_length_, port_id)
        .FromValue(0)
        .set_CCS(sc.CCS())
        .set_PortSpeed(sc.PortSpeed())
        .set_PIC(sc.PIC())
        .set_PLS(sc.PLS())
        .set_PP(sc.PP())
        .set_OCC(1)
        .WriteTo(mmio_);
    if (overcurrent) {
      zxlogf(ERROR, "Port %i has overcurrent active.", static_cast<int>(port_id));
    } else {
      zxlogf(ERROR, "Overcurrent event on port %i cleared.", static_cast<int>(port_id));
    }
  }
  if (sc.CSC()) {
    // Connect status change
    hci_->GetPortState()[port_id - 1].retry = false;
    PORTSC::Get(cap_length_, port_id)
        .FromValue(0)
        .set_CCS(sc.CCS())
        .set_PLC(sc.PLC())
        .set_PortSpeed(sc.PortSpeed())
        .set_PIC(sc.PIC())
        .set_PLS(sc.PLS())
        .set_PP(sc.PP())
        .set_CSC(sc.CSC())
        .WriteTo(mmio_);
  }
  if (sc.PEC()) {
    return fpromise::make_error_promise<zx_status_t>(ZX_ERR_BAD_STATE);
  }
  if (sc.PRC() || sc.WRC()) {
    PORTSC::Get(cap_length_, port_id)
        .FromValue(0)
        .set_CCS(sc.CCS())
        .set_PortSpeed(sc.PortSpeed())
        .set_PIC(sc.PIC())
        .set_PLS(sc.PLS())
        .set_PP(sc.PP())
        .set_PRC(sc.PRC())
        .set_WRC(sc.WRC())
        .WriteTo(mmio_);
  }
  if (pending_enumeration.has_value()) {
    return *std::move(pending_enumeration);
  }
  if (needs_enum) {
    return WaitForPortStatusChange(port_id)
        .and_then([=]() {
          // Retry enumeration
          SchedulePortStatusChange(port_id, true);
          return fpromise::ok();
        })
        .box();
  }
  return fpromise::make_result_promise<void, zx_status_t>(fpromise::ok());
}

fpromise::promise<void, zx_status_t> EventRing::WaitForPortStatusChange(uint8_t port_id) {
  fpromise::bridge<TRB*, zx_status_t> bridge;
  auto context = hci_->GetCommandRing()->AllocateContext();
  context->completer = std::move(bridge.completer);
  hci_->GetPortState()[port_id - 1].wait_for_port_status_change_ = std::move(context);
  return bridge.consumer.promise().discard_value();
}

void EventRing::CallPortStatusChanged(fbl::RefPtr<PortStatusChangeState> state) {
  if (state->port_index < state->port_count) {
    ScheduleTask(HandlePortStatusChangeEvent(static_cast<uint8_t>(state->port_index))
                     .then([=](fpromise::result<void, zx_status_t>& result)
                               -> fpromise::result<void, zx_status_t> {
                       if (result.is_error()) {
                         if (result.error() == ZX_ERR_BAD_STATE) {
                           return fpromise::error(ZX_ERR_BAD_STATE);
                         }
                       }
                       state->port_index++;
                       CallPortStatusChanged(state);
                       return fpromise::ok();
                     })
                     .box());
  } else {
    if (enumeration_queue_.is_empty()) {
      enumerating_ = false;
    } else {
      enumerating_ = true;
      auto enum_task = enumeration_queue_.pop_front();
      ScheduleTask(HandlePortStatusChangeEvent(enum_task->port_number)
                       .then([this, state, task = std::move(enum_task)](
                                 fpromise::result<void, zx_status_t>& result) mutable
                             -> fpromise::result<void, zx_status_t> {
                         if (result.is_error()) {
                           if (result.error() == ZX_ERR_BAD_STATE) {
                             return fpromise::error(ZX_ERR_BAD_STATE);
                           }
                           task->completer->complete_error(result.error());
                         } else {
                           task->completer->complete_ok(nullptr);
                         }
                         state->port_index = state->port_count;
                         CallPortStatusChanged(state);
                         return result;
                       }));
    }
  }
}

void EventRing::SchedulePortStatusChange(uint8_t port_id, bool preempt) {
  auto ctx = hci_->GetCommandRing()->AllocateContext();
  ctx->port_number = port_id;
  fpromise::bridge<TRB*, zx_status_t> bridge;
  ctx->completer = std::move(bridge.completer);
  ScheduleTask(bridge.consumer.promise()
                   .then([=](fpromise::result<TRB*, zx_status_t>& result) { return result; })
                   .box());
  if (preempt) {
    enumeration_queue_.push_front(std::move(ctx));
  } else {
    enumeration_queue_.push_back(std::move(ctx));
  }
  if (!enumerating_) {
    auto state = fbl::MakeRefCounted<PortStatusChangeState>(0, 0);
    CallPortStatusChanged(std::move(state));
  }
}

zx_status_t EventRing::Ring0Bringup() {
  hci_->WaitForBringup();
  enumerating_ = false;
  return ZX_OK;
}

void EventRing::ScheduleTask(fpromise::promise<void, zx_status_t> promise) {
  auto continuation = promise.or_else([=](const zx_status_t& status) {
    // ZX_ERR_BAD_STATE is a special value that we use to signal
    // a fatal error in xHCI. When this occurs, we should immediately
    // attempt to shutdown the controller. This error cannot be recovered from.
    if (status == ZX_ERR_BAD_STATE) {
      zxlogf(ERROR, "Scheduled task returned a fatal error, shutting down");
      hci_->Shutdown(status);
    }
  });
  executor_.schedule_task(std::move(continuation));
}

void EventRing::RunUntilIdle() { executor_.run_until_idle(); }

bool EventRing::StallWorkaroundForDefectiveHubs(std::unique_ptr<TRBContext>& context) {
  ZX_ASSERT(std::holds_alternative<Request>(*context->request));
  // Workaround for full-speed hub issue in Gateway keyboard
  auto request = std::get<Request>(*context->request).request();
  if ((request->header.ep_address == 0) && (request->setup.b_request == USB_REQ_GET_DESCRIPTOR) &&
      (request->setup.w_index == 0) && (request->setup.w_value == (USB_DT_DEVICE_QUALIFIER << 8))) {
    usb_device_qualifier_descriptor_t* desc;
    if ((std::get<Request>(*context->request).Mmap(reinterpret_cast<void**>(&desc)) == ZX_OK) &&
        (request->header.length >= sizeof(desc))) {
      desc->b_device_protocol =
          0;  // Don't support multi-TT unless we're sure the device supports it.
      ScheduleTask(hci_->UsbHciResetEndpointAsync(request->header.device_id, 0)
                       .and_then([ctx = std::move(context)]() {
                         std::get<Request>(*ctx->request).Complete(ZX_OK, sizeof(*desc));
                         return fpromise::ok();
                       }));

      return true;
    }
  }
  return false;
}

void EventRing::AdvanceErdp() {
  enum {
    STOP = 0,
    INCREMENT = 1,
    NEXT = 2,
    WRAP_AROUND = 3,
  } action = STOP;

  reevaluate_ = false;
  fbl::AutoLock l(&segment_mutex_);
  if (unlikely((reinterpret_cast<size_t>(erdp_virt_ + 1) / 4096) !=
               (reinterpret_cast<size_t>(erdp_virt_) / 4096))) {
    // Page transition -- next buffer
    auto next_buffer = buffers_it_;
    next_buffer++;
    ZX_DEBUG_ASSERT(next_buffer != buffers_it_);
    if (unlikely(next_buffer == buffers_.end())) {
      // Last buffer: wrap around
      action = WRAP_AROUND;
      buffers_it_->new_segment = false;
    } else if (unlikely(next_buffer->new_segment)) {
      // New segment. Check for valid Completion Code to see if HW is using the new segment yet.
      if (static_cast<CommandCompletionEvent*>(next_buffer->buf->virt())->CompletionCode() ==
          CommandCompletionEvent::Invalid) {
        // Invalid completion code. New segment not in use yet, so we need to check if HW has
        // wrapped around to the first segment in the ring. We need to do this because adding a new
        // segment can race with the position of HW's enqueue pointer, it might have already started
        // writing to the first segment before seeing that there is a new segment available.
        if (Control::FromTRB(reinterpret_cast<TRB*>(buffers_.front().buf->virt())).Cycle() ==
            ccs_) {
          // HW hasn't started using the first segment, or the new segment, yet so we can't tell
          // which direction the enqueue pointer will move next. Set reevaluate_ and return an
          // invalid TRB to wait for the next interrupt and try again.
          reevaluate_ = true;
          return;
        }
        // HW has wrapped around to use the first segment.
        action = WRAP_AROUND;
      } else {
        // Valid completion code. New segment already in use.
        action = NEXT;
      }
    } else {
      // Not new segment.
      action = NEXT;
    }
    buffers_it_->new_segment = false;
  } else {
    action = INCREMENT;
  }

  switch (action) {
    case INCREMENT: {
      // Increment within segment
      erdp_virt_++;
      erdp_phys_ += sizeof(TRB);
    } break;
    case NEXT: {
      // Next segment
      buffers_it_++;
      erdp_virt_ = reinterpret_cast<TRB*>((*buffers_it_).buf->virt());
      erdp_phys_ = (*buffers_it_).buf->phys();
      segment_index_ = (segment_index_ + 1) & 0b111;
    } break;
    case WRAP_AROUND: {
      // Wrap around to first segment
      ccs_ = !ccs_;
      buffers_it_ = buffers_.begin();
      erdp_virt_ = reinterpret_cast<TRB*>((*buffers_it_).buf->virt());
      erdp_phys_ = (*buffers_it_).buf->phys();
      segment_index_ = 0;
    } break;
    default: {
      zxlogf(ERROR, "This should not happen.");
    } break;
  }
}

std::optional<Control> EventRing::CurrentErdp() {
  if (reevaluate_) {
    return std::nullopt;
  }

  {
    fbl::AutoLock l(&segment_mutex_);
    if (unlikely(buffers_it_->new_segment)) {
      // On the first pass through a new segment the cycle bit is invalid and software should use
      // the completion code to check if the event TRB is valid. Section 4.9.4.1.
      if (static_cast<CommandCompletionEvent*>(erdp_virt_)->CompletionCode() ==
          CommandCompletionEvent::Invalid) {
        return std::nullopt;
      }

      // Barrier to ensure that the read of erdp_virt_ is ordered after the above validity check.
      hw_rmb();
      return Control::FromTRB(erdp_virt_);
    }
  }

  auto control = Control::FromTRB(erdp_virt_);
  if (control.Cycle() != ccs_) {
    return std::nullopt;
  }

  // Barrier to ensure that all subsequent reads from erdp_virt_ are after the above cycle bit
  // check.
  hw_rmb();

  return control;
}

zx_paddr_t EventRing::UpdateErdpReg(zx_paddr_t last_phys, size_t processed_trb_count) {
  if (last_phys != erdp_phys_) {
    if (async_id_.has_value()) {
      TRACE_ASYNC_END("UsbXhci", "EventRing::UpdateErdpReg", async_id_.value(),
                      "processed_trb_count", processed_trb_count);
    }
    executor_.run_until_idle();
    {
      fbl::AutoLock l(&segment_mutex_);
      erdp_reg_ =
          erdp_reg_.set_Pointer(erdp_phys_).set_DESI(segment_index_).set_EHB(1).WriteTo(mmio_);
    }
    last_phys = erdp_phys_;
    async_id_.emplace(TRACE_NONCE());
    TRACE_ASYNC_BEGIN("UsbXhci", "EventRing::UpdateErdpReg", async_id_.value(), erdp_phys_);
  }
  return last_phys;
}

zx_status_t EventRing::HandleIRQ() {
  iman_reg_.set_IP(1).set_IE(1).WriteTo(mmio_);
  bool avoid_yield = false;
  zx_paddr_t last_phys = 0;
  uint64_t processed_trbs{0};

  auto update_inspect_data = fit::defer([&processed_trbs, this]() {
    total_event_trbs_.Add(processed_trbs);
    if (max_single_irq_event_trbs_value_ < processed_trbs) {
      max_single_irq_event_trbs_value_ = processed_trbs;
      max_single_irq_event_trbs_.Set(max_single_irq_event_trbs_value_);
    }
  });

  // avoid_yield is used to indicate that we are in "realtime mode". When in this mode, we should
  // avoid yielding our timeslice to the scheduler if at all possible, because yielding could result
  // in us getting behind on our deadlines. Currently; we only ever need this on systems that don't
  // support cache coherency where we may have to go through the loop several times due to stale
  // values in the cache (after invalidating of course). On systems with a coherent cache this isn't
  // necessary. Additionally; if we had a guarantee from the scheduler that we would be woken up in
  // <125 microseconds (length of USB frame), we could safely yield after flushing our caches and
  // wouldn't need this loop.
  do {
    avoid_yield = false;

    if (reevaluate_) {
      AdvanceErdp();
    }

    std::optional<Control> control = CurrentErdp();
    while (control.has_value()) {
      ++processed_trbs;
      switch (control->Type()) {
        case Control::PortStatusChangeEvent:
          if (!port_status_change_event_) {
            port_status_change_event_ = events_.CreateUint("PortStatusChangeEvent", 0);
          }
          port_status_change_event_->Add(1);
          HandlePortStatusChangeInterrupt();
          break;
        case Control::CommandCompletionEvent: {
          zx_status_t status = HandleCommandCompletionInterrupt();
          if (status != ZX_OK) {
            return status;
          }
        } break;
        case Control::TransferEvent:
          HandleTransferInterrupt();
          break;
        case Control::MFIndexWrapEvent: {
          if (!mf_index_wrap_event_) {
            mf_index_wrap_event_ = events_.CreateUint("MFIndexWrapEvent", 0);
          }
          mf_index_wrap_event_->Add(1);
          hci_->MfIndexWrapped();
        } break;
        case Control::HostControllerEvent:
          if (!host_controller_event_) {
            host_controller_event_ = events_.CreateUint("HostControllerEvent", 0);
          }
          host_controller_event_->Add(1);
          // NOTE: We can't really do anything here. This typically indicates some kind of error
          // condition.
          zxlogf(DEBUG, "Host controller event: %u",
                 static_cast<CommandCompletionEvent*>(erdp_virt_)->CompletionCode());
          break;
        default:
          if (!unhandled_events_) {
            unhandled_events_ = events_.CreateLinearUintHistogram("UnhandledEvents", 1, 1, 40);
          }
          unhandled_events_->Insert(control->Type());
          zxlogf(ERROR, "Unexpected transfer event: %u", control->Type());
          break;
      }

      AdvanceErdp();
      control = CurrentErdp();

      if (processed_trbs % kUpdateERDPAfterTRBs) {
        last_phys = UpdateErdpReg(last_phys, processed_trbs / kUpdateERDPAfterTRBs + 1);
      }
    }

    last_phys = UpdateErdpReg(last_phys, processed_trbs / kUpdateERDPAfterTRBs + 1);
    if (!hci_->HasCoherentState()) {
      // Check for stale value in cache
      InvalidatePageCache(erdp_virt_, ZX_CACHE_FLUSH_INVALIDATE | ZX_CACHE_FLUSH_DATA);
      if (CurrentErdp().has_value()) {
        avoid_yield = true;
      }
    }
  } while (avoid_yield);
  return ZX_OK;
}

void EventRing::HandlePortStatusChangeInterrupt() {
  // Section 4.3 -- USB device intialization
  // Section 6.4.2.3 (Port Status change TRB)
  auto change_event = static_cast<PortStatusChangeEvent*>(erdp_virt_);
  uint8_t port_id = static_cast<uint8_t>(change_event->PortID());
  auto event = std::move(hci_->GetPortState()[port_id - 1].wait_for_port_status_change_);
  // Resume interrupted wait
  if (event) {
    event->completer->complete_ok(nullptr);
  } else {
    SchedulePortStatusChange(port_id);
  }
}

zx_status_t EventRing::HandleCommandCompletionInterrupt() {
  auto completion_event = static_cast<CommandCompletionEvent*>(erdp_virt_);
  if (!command_completion_event_) {
    command_completion_event_ =
        events_.CreateLinearUintHistogram("CommandCompletionEvent", 1, 1, 40);
  }
  command_completion_event_->Insert(completion_event->CompletionCode());
  // We do not expect to receive command completion events with invalid TRB pointers, so we always
  // check the pointer against the command ring pending TRB queue. Section 6.4.2.2 states that not
  // all command completion events have valid TRB pointers, but it's not clear in what case that
  // would actually occur.
  TRB* trb = command_ring_->PhysToVirt(erdp_virt_->ptr);
  std::unique_ptr<TRBContext> context;
  zx_status_t status = command_ring_->CompleteTRB(trb, &context);
  if (status != ZX_OK) {
    zxlogf(ERROR, "command_ring_->CompleteTRB(): %s", zx_status_get_string(status));
    hci_->Shutdown(ZX_ERR_BAD_STATE);
    return ZX_ERR_BAD_STATE;
  }

  if (completion_event->CompletionCode() != CommandCompletionEvent::Success) {
    // Log all failing commands for now. As error handling is added to the command callbacks, this
    // can be reduced.
    zxlogf(WARNING, "Received command completion event with completion code: %u",
           completion_event->CompletionCode());
  }

  if (context->completer.has_value()) {
    context->completer.value().complete_ok(completion_event);
  }
  return ZX_OK;
}

void EventRing::HandleTransferInterrupt() {
  if (!transfer_event_) {
    transfer_event_ = events_.CreateLinearUintHistogram("TransferEvent", 1, 1, 40);
  }
  auto transfer_event = static_cast<TransferEvent*>(erdp_virt_);
  transfer_event_->Insert(transfer_event->CompletionCode());

  auto device_state = hci_->GetDeviceState()[transfer_event->SlotID() - 1];
  if (!device_state) {
    zxlogf(WARNING, "Device state invalid");
    return;
  }
  fbl::AutoLock l(&device_state->transaction_lock());
  if (!device_state->IsValid()) {
    zxlogf(WARNING, "Device state invalid");
    return;
  }

  Endpoint* ep;
  uint8_t endpoint_id = static_cast<uint8_t>(transfer_event->EndpointID() - 1);
  if (unlikely(endpoint_id == 0)) {
    ep = &device_state->GetEndpoint();
  } else {
    ep = &device_state->GetEndpoint(endpoint_id - 1);
  }

  if (transfer_event->CompletionCode() == CommandCompletionEvent::RingOverrun) {
    zxlogf(DEBUG, "Transfer ring overrrun on slot %u endpoint %u", transfer_event->SlotID(),
           transfer_event->EndpointID());
    return;
  }
  if (transfer_event->CompletionCode() == CommandCompletionEvent::RingUnderrun) {
    zxlogf(DEBUG, "Transfer ring underrun on slot %u endpoint %u", transfer_event->SlotID(),
           transfer_event->EndpointID());
    return;
  }

  if (transfer_event->CompletionCode() == CommandCompletionEvent::EndpointNotEnabledError) {
    zxlogf(WARNING, "Endpoint not enabled error for slot %u endpoint %u", transfer_event->SlotID(),
           transfer_event->EndpointID());
    return;
  }

  auto& ring = ep->transfer_ring();
  if (transfer_event->CompletionCode() == CommandCompletionEvent::MissedServiceError) {
    zxlogf(DEBUG, "Missed service error on slot %u endpoint %u", transfer_event->SlotID(),
           transfer_event->EndpointID());
    std::unique_ptr<TRBContext> context;
    ring.CompleteTRB(nullptr, &context);
    ep->RequestComplete(ZX_ERR_IO_MISSED_DEADLINE, 0, std::move(*context->request));

    // set resynchronize_ to wait for next successful transfer.
    resynchronize_ = true;
    return;
  }

  if (transfer_event->CompletionCode() == CommandCompletionEvent::StallError) {
    zxlogf(DEBUG, "Transfer ring stall on slot %u endpoint %u", transfer_event->SlotID(),
           transfer_event->EndpointID());
    ring.set_stall(true);
    auto completions = ring.TakePendingTRBs();

    // A stall error may not have an associated transfer TRB. Section 4.17.4.
    std::unique_ptr<TRBContext> context;
    if (erdp_virt_->ptr) {
      TRB* trb = ring.PhysToVirt(erdp_virt_->ptr);
      ring.CompleteTRB(trb, &context);
    }
    l.release();

    if (context) {
      if (completions.is_empty()) {
        bool handled = StallWorkaroundForDefectiveHubs(context);
        if (handled) {
          zxlogf(DEBUG, "Handled stall with workaround for defective hubs");
          return;
        }
      }
      ep->RequestComplete(ZX_ERR_IO_REFUSED, 0, std::move(*context->request));
    }
    while (true) {
      auto completion = completions.pop_front();
      if (!completion) {
        break;
      }
      ep->RequestComplete(ZX_ERR_IO_REFUSED, 0, std::move(*completion->request));
    }
    return;
  }

  // If there is no transfer TRB pointer here then it probably indicates an event type that we don't
  // handle. Section 4.17.4.
  if (unlikely(!erdp_virt_->ptr)) {
    zxlogf(ERROR, "Unhandled event (completion code %u) with no transfer TRB pointer.",
           transfer_event->CompletionCode());
    return;
  }

  TRB* trb = ring.PhysToVirt(erdp_virt_->ptr);

  if (transfer_event->CompletionCode() == CommandCompletionEvent::ShortPacket) {
    TRB* last_trb = trb;
    zx_status_t status = ring.HandleShortPacket(trb, transfer_event->TransferLength(), &last_trb);

    // If the TRB is not at the head of the pending list then we're out of sync with the device,
    // which indicates a bug in the driver.
    ZX_ASSERT(status == ZX_OK);

    if (trb != last_trb) {
      // We'll get a second event for this TRB, we have recorded TransferLength in the appropriate
      // TRBContext.
      return;
    }
  }

  fbl::DoublyLinkedList<std::unique_ptr<TRBContext>> resynchronize_completions;
  auto cleanup = fit::defer([&resynchronize_completions, &ep]() {
    while (true) {
      auto completion = resynchronize_completions.pop_front();
      if (!completion) {
        break;
      }
      ep->RequestComplete(ZX_ERR_IO_MISSED_DEADLINE, 0, std::move(*completion->request));
    }
  });
  if (resynchronize_) {
    resynchronize_ = false;
    // Resynchronize (4.11.2.5.2). Just find the next successful TRB.
    resynchronize_completions = ring.TakePendingTRBsUntil(trb);
  }

  // If the TRB is not at the head of the pending list then we're out of sync with the device, which
  // indicates a bug in the driver.
  std::unique_ptr<TRBContext> context;
  zx_status_t status = ring.CompleteTRB(trb, &context);

  // TODO(https://fxbug.dev/42059338): Once we reliably keep track of TRBs, this error handling should be
  // removed and replaced by: ZX_ASSERT(status == ZX_OK).
  if (status != ZX_OK) {
    zxlogf(ERROR, "Lost a TRB! Completion code is %u", transfer_event->CompletionCode());

    auto completions = ring.TakePendingTRBs();
    l.release();

    // CompleteTRB already popped the head off the list, so complete it first.
    if (context) {
      ep->RequestComplete(ZX_ERR_IO, 0, std::move(*context->request));
    }

    size_t index = 1;
    bool found = false;
    while (true) {
      auto completion = completions.pop_front();
      if (!completion) {
        break;
      }

      if (completion->trb == trb) {
        zxlogf(ERROR, "Current TRB was found at index %zd", index);
        found = true;
      }
      ep->RequestComplete(ZX_ERR_IO, 0, std::move(*completion->request));
      index++;
    }
    if (!found) {
      zxlogf(ERROR, "Current TRB was not found");
    }

    return;
  }

  l.release();

  if ((transfer_event->CompletionCode() != CommandCompletionEvent::Success) &&
      (transfer_event->CompletionCode() != CommandCompletionEvent::ShortPacket)) {
    // asix-88179 will stall the endpoint if we're sending data too fast. The driver expects us to
    // give it a ZX_ERR_IO_INVALID response when this happens.
    zxlogf(WARNING, "transfer_event->CompletionCode() == %u", transfer_event->CompletionCode());
    ep->RequestComplete(ZX_ERR_IO_INVALID, 0, std::move(*context->request));
    return;
  }

  size_t length;
  if (context->short_transfer_len.has_value()) {
    length = context->short_transfer_len.value();
  } else {
    length = std::holds_alternative<Request>(*context->request)
                 ? std::get<Request>(*context->request).request()->header.length
                 : std::get<usb::FidlRequest>(*context->request).length();
  }

  ep->RequestComplete(ZX_OK, length, std::move(*context->request));
}

fpromise::promise<void, zx_status_t> EventRing::LinkUp(uint8_t port_id) {
  // Port is in U0 state (link up)
  // Enumerate device
  zxlogf(DEBUG, "Event Link Up %d.", port_id);
  return EnumerateDevice(hci_, port_id, std::nullopt);
}

void EventRing::Usb2DeviceAttach(uint16_t port_id) {
  hci_->GetPortState()[port_id - 1].is_connected = true;
  hci_->GetPortState()[port_id - 1].is_USB3 = false;
  auto sc = PORTSC::Get(cap_length_, port_id).ReadFrom(mmio_);
  PORTSC::Get(cap_length_, port_id)
      .FromValue(0)
      .set_CCS(sc.CCS())
      .set_PortSpeed(sc.PortSpeed())
      .set_PIC(sc.PIC())
      .set_PLS(sc.PLS())
      .set_PP(sc.PP())
      .set_PR(1)
      .WriteTo(mmio_);
}

void EventRing::Usb3DeviceAttach(uint16_t port_id) {
  hci_->GetPortState()[port_id - 1].is_connected = true;
  hci_->GetPortState()[port_id - 1].is_USB3 = true;
}

}  // namespace usb_xhci