// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "scsi.h"

#include <inttypes.h>
#include <lib/scsi/scsilib.h>
#include <lib/scsi/scsilib_controller.h>
#include <netinet/in.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <sys/param.h>
#include <zircon/compiler.h>

#include <algorithm>
#include <utility>

#include <ddk/debug.h>
#include <fbl/algorithm.h>
#include <fbl/auto_call.h>
#include <fbl/auto_lock.h>
#include <pretty/hexdump.h>
#include <virtio/scsi.h>

#include "src/devices/bus/lib/virtio/trace.h"

#define LOCAL_TRACE 0

namespace virtio {

// Fill in req->lun with a single-level LUN structure representing target:lun.
void ScsiDevice::FillLUNStructure(struct virtio_scsi_req_cmd* req, uint8_t target, uint16_t lun) {
  req->lun[0] = 1;
  req->lun[1] = target;
  req->lun[2] = 0x40 | static_cast<uint8_t>(lun >> 8);
  req->lun[3] = static_cast<uint8_t>(lun) & 0xff;
}

ScsiDevice::scsi_io_slot* ScsiDevice::GetIO() {
  // For testing purposes, this condition can be triggered
  // by lowering MAX_IOS (to say 2). And running biotime
  // (with default IO concurrency).
  while (active_ios_ == MAX_IOS) {
    ioslot_cv_.Wait(&lock_);
  }
  active_ios_++;
  for (int i = 0; i < MAX_IOS; i++) {
    if (scsi_io_slot_table_[i].avail) {
      scsi_io_slot_table_[i].avail = false;
      return &scsi_io_slot_table_[i];
    }
  }
  ZX_DEBUG_ASSERT(false);  // Unexpected.
  return NULL;
}

void ScsiDevice::FreeIO(scsi_io_slot* io_slot) {
  io_slot->avail = true;
  active_ios_--;
  ioslot_cv_.Signal();
}

void ScsiDevice::IrqRingUpdate() {
  // Parse our descriptor chain and add back to the free queue.
  auto free_chain = [this](vring_used_elem* elem) TA_NO_THREAD_SAFETY_ANALYSIS {
    auto index = static_cast<uint16_t>(elem->id);
    vring_desc const* tail_desc;

    // Reclaim the entire descriptor chain.
    for (;;) {
      vring_desc const* desc = request_queue_.DescFromIndex(index);
      const bool has_next = desc->flags & VRING_DESC_F_NEXT;
      const auto next = desc->next;

      this->request_queue_.FreeDesc(index);
      if (!has_next) {
        tail_desc = desc;
        break;
      }
      index = next;
    }
    desc_cv_.Broadcast();
    // Search for the IO that just completed, using tail_desc.
    for (int i = 0; i < MAX_IOS; i++) {
      scsi_io_slot* io_slot = &scsi_io_slot_table_[i];

      if (io_slot->avail)
        continue;
      if (io_slot->tail_desc == tail_desc) {
        // Capture response before freeing iobuffer.
        zx_status_t status;
        if (io_slot->response->response || io_slot->response->status) {
          status = ZX_ERR_INTERNAL;
        } else {
          status = ZX_OK;
        }
        // If Read, copy data from iobuffer to the iovec.
        if (status == ZX_OK && io_slot->data_in.iov_len) {
          memcpy(io_slot->data_in.iov_base, io_slot->data_in_region, io_slot->data_in.iov_len);
        }
        void* cookie = io_slot->cookie;
        auto (*callback)(void* cookie, zx_status_t status) = io_slot->callback;
        FreeIO(io_slot);
        lock_.Release();
        callback(cookie, status);
        lock_.Acquire();
        return;
      }
    }
    ZX_DEBUG_ASSERT(false);  // Unexpected.
  };

  // Tell the ring to find free chains and hand it back to our lambda.
  fbl::AutoLock lock(&lock_);
  request_queue_.IrqRingUpdate(free_chain);
}

zx_status_t ScsiDevice::ExecuteCommandSync(uint8_t target, uint16_t lun, struct iovec cdb,
                                           struct iovec data_out, struct iovec data_in) {
  struct scsi_sync_callback_state {
    sync_completion_t completion;
    zx_status_t status;
  };
  scsi_sync_callback_state cookie;
  sync_completion_reset(&cookie.completion);
  auto callback = [](void* cookie, zx_status_t status) {
    auto* state = reinterpret_cast<scsi_sync_callback_state*>(cookie);
    state->status = status;
    sync_completion_signal(&state->completion);
  };
  ExecuteCommandAsync(target, lun, cdb, data_out, data_in, callback, &cookie);
  sync_completion_wait(&cookie.completion, ZX_TIME_INFINITE);
  return cookie.status;
}

zx_status_t ScsiDevice::ExecuteCommandAsync(uint8_t target, uint16_t lun, struct iovec cdb,
                                            struct iovec data_out, struct iovec data_in,
                                            void (*cb)(void*, zx_status_t), void* cookie) {
  // We do all of the error checking up front, so we don't need to fail the IO
  // after acquiring the IO slot and the descriptors.
  // If data_in fits within request_buffers_, all the regions of this request will fit.
  if ((sizeof(struct virtio_scsi_req_cmd) + data_out.iov_len + sizeof(struct virtio_scsi_resp_cmd) +
       data_in.iov_len) > request_buffers_size_) {
    return ZX_ERR_NO_MEMORY;
  }

  uint16_t descriptor_chain_length = 2;
  if (data_out.iov_len) {
    descriptor_chain_length++;
  }
  if (data_in.iov_len) {
    descriptor_chain_length++;
  }

  lock_.Acquire();
  // Get both the IO slot and the descriptors needed up front.
  auto io_slot = GetIO();
  uint16_t id = 0;
  auto request_desc = request_queue_.AllocDescChain(/*count=*/descriptor_chain_length, &id);
  // For testing purposes, this condition can be triggered by failing
  // AllocDescChain every N attempts. But we would have to Signal the cv
  // somewhere. A good place to do that is at the bottom of WorkerThread,
  // after the luns are probed, in a loop. If we do the signaling there,
  // we'd need to ensure error injection doesn't start until after luns are
  // probed.
  while (request_desc == nullptr) {
    // Drop the request buf, before blocking, waiting for descs to free up.
    FreeIO(io_slot);
    desc_cv_.Wait(&lock_);
    io_slot = GetIO();
    request_desc = request_queue_.AllocDescChain(/*count=*/descriptor_chain_length, &id);
  }

  auto* request_buffers = &io_slot->request_buffer;
  // virtio-scsi requests have a 'request' region, an optional data-out region, a 'response'
  // region, and an optional data-in region. Allocate and fill them and then execute the request.
  const auto request_offset = 0ull;
  const auto data_out_offset = request_offset + sizeof(struct virtio_scsi_req_cmd);
  const auto response_offset = data_out_offset + data_out.iov_len;
  const auto data_in_offset = response_offset + sizeof(struct virtio_scsi_resp_cmd);

  auto* const request_buffers_addr = reinterpret_cast<uint8_t*>(io_buffer_virt(request_buffers));
  auto* const request =
      reinterpret_cast<struct virtio_scsi_req_cmd*>(request_buffers_addr + request_offset);
  auto* const data_out_region = reinterpret_cast<uint8_t*>(request_buffers_addr + data_out_offset);
  auto* const response =
      reinterpret_cast<struct virtio_scsi_resp_cmd*>(request_buffers_addr + response_offset);
  auto* const data_in_region = reinterpret_cast<uint8_t*>(request_buffers_addr + data_in_offset);

  memset(request, 0, sizeof(*request));
  memset(response, 0, sizeof(*response));
  memcpy(&request->cdb, cdb.iov_base, cdb.iov_len);
  FillLUNStructure(request, /*target=*/target, /*lun=*/lun);
  request->id = scsi_transport_tag_++;

  vring_desc* tail_desc;
  request_desc->addr = io_buffer_phys(request_buffers) + request_offset;
  request_desc->len = sizeof(*request);
  request_desc->flags = VRING_DESC_F_NEXT;
  auto next_id = request_desc->next;

  if (data_out.iov_len) {
    memcpy(data_out_region, data_out.iov_base, data_out.iov_len);
    auto* data_out_desc = request_queue_.DescFromIndex(next_id);
    data_out_desc->addr = io_buffer_phys(request_buffers) + data_out_offset;
    data_out_desc->len = static_cast<uint32_t>(data_out.iov_len);
    data_out_desc->flags = VRING_DESC_F_NEXT;
    next_id = data_out_desc->next;
  }

  auto* response_desc = request_queue_.DescFromIndex(next_id);
  response_desc->addr = io_buffer_phys(request_buffers) + response_offset;
  response_desc->len = sizeof(*response);
  response_desc->flags = VRING_DESC_F_WRITE;

  if (data_in.iov_len) {
    response_desc->flags |= VRING_DESC_F_NEXT;
    auto* data_in_desc = request_queue_.DescFromIndex(response_desc->next);
    data_in_desc->addr = io_buffer_phys(request_buffers) + data_in_offset;
    data_in_desc->len = static_cast<uint32_t>(data_in.iov_len);
    data_in_desc->flags = VRING_DESC_F_WRITE;
    tail_desc = data_in_desc;
  } else {
    tail_desc = response_desc;
  }

  io_slot->tail_desc = tail_desc;
  io_slot->data_in = data_in;
  io_slot->data_in_region = data_in_region;
  io_slot->callback = cb;
  io_slot->cookie = cookie;
  io_slot->request_buffers = request_buffers;
  io_slot->response = response;

  request_queue_.SubmitChain(id);
  request_queue_.Kick();

  lock_.Release();
  return ZX_OK;
}

constexpr uint32_t SCSI_SECTOR_SIZE = 512;
constexpr uint32_t SCSI_MAX_XFER_SIZE = 1024;  // 512K clamp

// Read Block Limits VPD Page (0xB0), if supported and return the max xfer size
// (in blocks) supported by the target.
zx_status_t ScsiDevice::TargetMaxXferSize(uint8_t target, uint16_t lun,
                                          uint32_t& xfer_size_sectors) {
  scsi::InquiryCDB inquiry_cdb = {};
  scsi::VPDPageList vpd_pagelist = {};
  inquiry_cdb.opcode = scsi::Opcode::INQUIRY;
  // Query for all supported VPD pages.
  inquiry_cdb.reserved_and_evpd = 0x1;
  inquiry_cdb.page_code = 0x00;
  inquiry_cdb.allocation_length = ntohs(sizeof(vpd_pagelist));
  auto status = ExecuteCommandSync(/*target=*/target, /*lun=*/lun,
                                   /*cdb=*/{&inquiry_cdb, sizeof(inquiry_cdb)},
                                   /*data_out=*/{nullptr, 0},
                                   /*data_in=*/{&vpd_pagelist, sizeof(vpd_pagelist)});
  if (status != ZX_OK)
    return status;
  uint8_t i;
  for (i = 0; i < vpd_pagelist.page_length; i++) {
    if (vpd_pagelist.pages[i] == 0xB0)
      break;
  }
  if (i == vpd_pagelist.page_length)
    return ZX_ERR_NOT_SUPPORTED;
  // The Block Limits VPD page is supported, fetch it.
  scsi::VPDBlockLimits block_limits = {};
  inquiry_cdb.page_code = 0xB0;
  inquiry_cdb.allocation_length = ntohs(sizeof(block_limits));
  status = ExecuteCommandSync(/*target=*/target, /*lun=*/lun,
                              /*cdb=*/{&inquiry_cdb, sizeof(inquiry_cdb)},
                              /*data_out=*/{nullptr, 0},
                              /*data_in=*/{&block_limits, sizeof(block_limits)});
  if (status != ZX_OK)
    return status;
  xfer_size_sectors = block_limits.max_xfer_length_blocks;
  return ZX_OK;
}

zx_status_t ScsiDevice::WorkerThread() {
  uint16_t max_target;
  uint32_t max_lun;
  uint32_t max_sectors;  // controller's max sectors.
  {
    fbl::AutoLock lock(&lock_);
    // virtio-scsi has a 16-bit max_target field, but the encoding we use limits us to one byte
    // target identifiers.
    max_target = std::min(config_.max_target, static_cast<uint16_t>(UINT8_MAX - 1));
    max_lun = config_.max_lun;
    max_sectors = config_.max_sectors;
  }

  // Execute TEST UNIT READY on every possible target to find potential disks.
  // TODO(fxbug.dev/32170): For SCSI-3 targets, we could optimize this by using REPORT LUNS.
  //
  // virtio-scsi nominally supports multiple channels, but the device support is not
  // complete. The device encoding for targets in commands does not allow encoding the
  // channel number, so we do not attempt to scan beyond channel 0 here.
  //
  // QEMU and GCE disagree on the definition of the max_target and max_lun config fields;
  // QEMU's max_target/max_lun refer to the last valid whereas GCE's refers to the first
  // invalid target/lun. Use <= to handle both.
  //
  // TODO(fxbug.dev/32170): Move probe sequence to ScsiLib -- have it call down into LLDs to execute
  // commands.
  for (uint8_t target = 0u; target <= max_target; target++) {
    const uint32_t luns_on_this_target = CountLuns(this, target);
    if (luns_on_this_target == 0) {
      continue;
    }

    uint16_t luns_found = 0;
    uint32_t max_xfer_size_sectors = 0;
    for (uint16_t lun = 0u; lun <= max_lun; lun++) {
      scsi::TestUnitReadyCDB cdb = {};
      cdb.opcode = scsi::Opcode::TEST_UNIT_READY;

      auto status = ExecuteCommandSync(
          /*target=*/target,
          /*lun=*/lun, {&cdb, sizeof(cdb)}, {}, {});
      if ((status == ZX_OK) && (max_xfer_size_sectors == 0)) {
        // If we haven't queried the VPD pages for the target's xfer size
        // yet, do it now. We only query this once per target.
        status = TargetMaxXferSize(target, lun, max_xfer_size_sectors);
        if (status == ZX_OK) {
          // smaller of controller and target max_xfer_sizes
          max_xfer_size_sectors = std::min(max_xfer_size_sectors, max_sectors);
          // and the 512K clamp
          max_xfer_size_sectors = std::min(max_xfer_size_sectors, SCSI_MAX_XFER_SIZE);
        } else {
          max_xfer_size_sectors = std::min(max_sectors, SCSI_MAX_XFER_SIZE);
        }
        zxlogf(INFO, "Virtio SCSI %u:%u Max Xfer Size %ukb", target, lun,
               max_xfer_size_sectors * 2);
        scsi::Disk::Create(this, device_, /*target=*/target, /*lun=*/lun, max_xfer_size_sectors);
        luns_found++;
      }
      // If we've found all the LUNs present on this target, move on.
      // Subtle detail - LUN 0 may respond to TEST UNIT READY even if it is not a valid LUN
      // and there is a valid LUN elsewhere on the target. Test for one more LUN than we
      // expect to work around that.
      if (luns_found > luns_on_this_target) {
        break;
      }
    }
  }
  return ZX_OK;
}

zx_status_t ScsiDevice::Init() {
  LTRACE_ENTRY;

  virtio::Device::DeviceReset();
  virtio::Device::ReadDeviceConfig<uint32_t>(offsetof(virtio_scsi_config, num_queues),
                                             &config_.num_queues);
  virtio::Device::ReadDeviceConfig<uint32_t>(offsetof(virtio_scsi_config, seg_max),
                                             &config_.seg_max);
  virtio::Device::ReadDeviceConfig<uint32_t>(offsetof(virtio_scsi_config, max_sectors),
                                             &config_.max_sectors);
  virtio::Device::ReadDeviceConfig<uint32_t>(offsetof(virtio_scsi_config, cmd_per_lun),
                                             &config_.cmd_per_lun);
  virtio::Device::ReadDeviceConfig<uint32_t>(offsetof(virtio_scsi_config, event_info_size),
                                             &config_.event_info_size);
  virtio::Device::ReadDeviceConfig<uint32_t>(offsetof(virtio_scsi_config, sense_size),
                                             &config_.sense_size);
  virtio::Device::ReadDeviceConfig<uint32_t>(offsetof(virtio_scsi_config, cdb_size),
                                             &config_.cdb_size);
  virtio::Device::ReadDeviceConfig<uint16_t>(offsetof(virtio_scsi_config, max_channel),
                                             &config_.max_channel);
  virtio::Device::ReadDeviceConfig<uint16_t>(offsetof(virtio_scsi_config, max_target),
                                             &config_.max_target);
  virtio::Device::ReadDeviceConfig<uint32_t>(offsetof(virtio_scsi_config, max_lun),
                                             &config_.max_lun);

  // Validate config.
  {
    fbl::AutoLock lock(&lock_);
    if (config_.max_channel > 1) {
      zxlogf(WARNING, "config_.max_channel %d not expected.", config_.max_channel);
    }
  }

  virtio::Device::DriverStatusAck();

  if (!bti().is_valid()) {
    zxlogf(ERROR, "invalid bti handle");
    return ZX_ERR_BAD_HANDLE;
  }
  {
    fbl::AutoLock lock(&lock_);
    auto err = control_ring_.Init(/*index=*/Queue::CONTROL);
    if (err) {
      zxlogf(ERROR, "failed to allocate control queue");
      return err;
    }

    err = request_queue_.Init(/*index=*/Queue::REQUEST);
    if (err) {
      zxlogf(ERROR, "failed to allocate request queue");
      return err;
    }
    request_buffers_size_ =
        (SCSI_SECTOR_SIZE * std::min(config_.max_sectors, SCSI_MAX_XFER_SIZE)) +
        (sizeof(struct virtio_scsi_req_cmd) + sizeof(struct virtio_scsi_resp_cmd));
    for (int i = 0; i < MAX_IOS; i++) {
      auto status = io_buffer_init(&scsi_io_slot_table_[i].request_buffer, bti().get(),
                                   /*size=*/request_buffers_size_, IO_BUFFER_RW | IO_BUFFER_CONTIG);
      if (status) {
        zxlogf(ERROR, "failed to allocate queue working memory");
        return status;
      }
      scsi_io_slot_table_[i].avail = true;
    }
    active_ios_ = 0;
    scsi_transport_tag_ = 0;
  }
  virtio::Device::StartIrqThread();
  virtio::Device::DriverStatusOk();

  // Synchronize against Unbind()/Release() before the worker thread is running.
  fbl::AutoLock lock(&lock_);
  auto status = DdkAdd("virtio-scsi");
  device_ = zxdev();
  if (status != ZX_OK) {
    zxlogf(ERROR, "failed to run DdkAdd");
    device_ = nullptr;
    return status;
  }

  auto td = [](void* ctx) {
    ScsiDevice* const device = static_cast<ScsiDevice*>(ctx);
    return device->WorkerThread();
  };
  int ret = thrd_create_with_name(&worker_thread_, td, this, "virtio-scsi-worker");
  if (ret != thrd_success) {
    return ZX_ERR_INTERNAL;
  }

  return status;
}

void ScsiDevice::DdkUnbind(ddk::UnbindTxn txn) { virtio::Device::Unbind(std::move(txn)); }

void ScsiDevice::DdkRelease() {
  {
    fbl::AutoLock lock(&lock_);
    worker_thread_should_exit_ = true;
    for (int i = 0; i < MAX_IOS; i++) {
      io_buffer_release(&scsi_io_slot_table_[i].request_buffer);
    }
  }
  thrd_join(worker_thread_, nullptr);
  virtio::Device::Release();
}

}  // namespace virtio