blob: 361f4cc3eaebd2719fd2c57a31c63e4bf0c08e41 [file]
// Copyright 2026 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT
#include <lib/page/size.h>
#include <lib/uart/all.h>
#include <lib/uart/null.h>
#include <lib/uart/uart.h>
#include <lib/zbi-format/driver-config.h>
#include <lib/zircon-internal/thread_annotations.h>
#include <platform.h>
#include <stdio.h>
#include <cassert>
#include <type_traits>
#include <arch/arch_interrupt.h>
#include <dev/init.h>
#include <kernel/event.h>
#include <kernel/ffi.h>
#include <kernel/lockdep.h>
#include <kernel/spinlock.h>
#include <ktl/optional.h>
#include <ktl/type_traits.h>
#include <lockdep/guard.h>
#include <platform/debug.h>
#include <platform/uart.h>
extern "C" {
bool rust_uart_is_tx_irq_enabled();
void rust_uart_irq_provider_set_interrupts_enabled(uint32_t irq_num, bool enabled);
bool rust_uart_rx_queue_full();
void rust_uart_rx_queue_write_char(uint8_t c);
void rust_uart_driver_handoff_early(const uart::all::Driver* serial);
void rust_uart_driver_handoff_post_vm(const uart::all::Driver* serial);
void rust_uart_driver_handoff_late(const uart::all::Driver* serial);
enum class UartDriverKind : uint32_t {
kNull = 0,
kMmio = 1,
kPio = 2,
};
struct UartDriverInfo {
UartDriverKind kind;
bool has_irq;
uint32_t irq;
bool has_simple_cfg;
uint32_t cfg_flags;
size_t io_base;
size_t io_size;
};
static_assert(sizeof(UartDriverKind) == 4);
static_assert(alignof(UartDriverKind) == 4);
static_assert(sizeof(UartDriverInfo) == 40);
static_assert(alignof(UartDriverInfo) == 8);
} // extern "C"
namespace {
// No locks will be acquired.
struct NullLockPolicy {};
template <typename LockPolicy, typename Guard>
using GuardSelector =
std::conditional_t<std::is_same_v<LockPolicy, NullLockPolicy>, NullGuard, Guard>;
// Implements SyncPolicy as defined in <lib/uart/sync.h>
struct UartSyncPolicy {
// While tempting, do *not* use a templated `using` statement here. In some
// places in code, we will need to deduce (via argument passing) what
// `MemberOf` is. Attempting to do this when we declare lock as a using alias
// is not going to work in this situation. The deduction is going to execute
// against the underlying type, not the `using` statement. So the final type
// (in this case, a LockDep instance) would need to be a template with exactly
// one argument which is identical to MemberOf. In the case of LockDep locks,
// this is *not* the case.
//
// Declaring this as a single-argument templated struct with no members which
// inherits from the underlying LockDep type works around this issue. Guards
// and other things will auto-upcast to the proper underlying LockDep type,
// and the only constructor needed/supported is the default constructor, so we
// don't need to write a forwarding constructor to the underlying type.
template <typename MemberOf>
struct Lock : public DECLARE_SPINLOCK_WITH_TYPE(MemberOf, MonitoredSpinLock) {};
template <typename LockPolicy>
using Guard = GuardSelector<LockPolicy, Guard<MonitoredSpinLock, LockPolicy>>;
class Waiter {
public:
enum class Blocking {
// `Wait` is allowed to block callers, e.g. wait on an event.
kYes,
// `Wait` is not allowed to block callers, should spin instead.
kNo,
};
template <typename Guard, typename T>
void Wait(Guard& guard, T&& enable_tx_interrupt, Blocking blocking) TA_REQ(guard) {
if (blocking == Blocking::kYes && rust_uart_is_tx_irq_enabled()) {
enable_tx_interrupt();
guard.CallUnlocked([this]() { tx_fifo_not_full_.Wait(); });
} else {
// Drop the spinlock while spinning.
guard.CallUnlocked([]() { arch::Yield(); });
}
}
void Wake() { tx_fifo_not_full_.Signal(); }
private:
AutounsignalEvent tx_fifo_not_full_{true};
};
using DefaultLockPolicy = NullLockPolicy;
template <typename LockType>
static void AssertHeld(LockType& lock) TA_ASSERT(lock) TA_ASSERT(lock.lock()) {
lock.lock().AssertHeld();
}
};
class KernelIrqProvider {
public:
KernelIrqProvider() = default;
~KernelIrqProvider() = default;
// No copy, no move.
KernelIrqProvider(const KernelIrqProvider&) = delete;
KernelIrqProvider& operator=(const KernelIrqProvider&) = delete;
KernelIrqProvider(KernelIrqProvider&&) = delete;
KernelIrqProvider& operator=(KernelIrqProvider&&) = delete;
void Init(int irq_num) { irq_num_ = irq_num; }
void SetInterruptsEnabled(bool enabled) const {
rust_uart_irq_provider_set_interrupts_enabled(static_cast<uint32_t>(irq_num_), enabled);
}
private:
int irq_num_{0};
};
uart::all::KernelDriver<PlatformUartIoProvider, UartSyncPolicy, KernelIrqProvider> gUart;
size_t UnparseUart(char* buf, size_t len) {
StringFile file{{buf, len}};
gUart.Unparse(&file);
return file.as_string_view().size();
}
void GetUartDriverInfo(UartDriverInfo* out) {
gUart.Visit([out]<typename DriverType>(DriverType& driver) {
using uart_type = DriverType::uart_type;
using cfg_type = uart_type::config_type;
if constexpr (ktl::is_same_v<cfg_type, zbi_dcfg_simple_t>) {
out->has_simple_cfg = true;
out->cfg_flags = driver.config().flags;
} else {
out->has_simple_cfg = false;
out->cfg_flags = 0;
}
if constexpr (uart::MmioDriver<uart_type>) {
out->kind = UartDriverKind::kMmio;
ktl::optional<uint32_t> irq = PlatformUartGetIrqNumber(driver.config().irq);
out->has_irq = irq.has_value();
out->irq = irq.value_or(0);
uart::MmioRange aligned_mmio_range = driver.mmio_range().AlignedTo(kPageSize);
out->io_base = aligned_mmio_range.address;
out->io_size = aligned_mmio_range.size;
} else if constexpr (uart::PioDriver<uart_type>) {
out->kind = UartDriverKind::kPio;
ktl::optional<uint32_t> irq = PlatformUartGetIrqNumber(driver.config().irq);
out->has_irq = irq.has_value();
out->irq = irq.value_or(0);
uart::IoPortRange ioports = driver.ioports();
out->io_base = ioports.base;
out->io_size = ioports.count;
} else { // Only |uart::null::Driver| is expected to have a different configuration type.
constexpr auto kIsNullDriver = ktl::is_same_v<uart_type, uart::null::Driver>;
ZX_ASSERT_MSG(kIsNullDriver, "Unexpected UART Configuration.");
out->kind = UartDriverKind::kNull;
out->has_irq = false;
out->irq = 0;
out->io_base = 0;
out->io_size = 0;
}
});
}
void InitUartInterrupt(uint32_t uart_irq) {
gUart.Visit([uart_irq](auto& driver) { driver.InitInterrupt(uart_irq); });
}
void HandleUartInterrupt() {
static constexpr auto rx_irq_handler = [](auto& rx_interrupt) {
// This check needs to be performed under a lock, such that we prevent operation
// interleaving that would leave us in a blocked state.
//
// E.g.
// Assume a simple MT scenario with one reader R and one writer R:
//
// * W: Observes the buffer is full.
// * R: Reads a character. The buffer is now empty.
// * R: Unmasks RX.
// * W: Masks RX.
//
// At this point, we have an empty buffer and RX interrupts are masked -
// we're stuck! Thus, to avoid this, we acquire the spinlock before
// checking if the buffer is full, and release after (conditionally)
// masking RX interrupts. By pairing this with the acquisition of the
// same lock around unmasking RX interrupts, we prevent the writer above
// from being interrupted by a read-and-unmask.
uint8_t c;
{
Guard<MonitoredSpinLock, NoIrqSave> lock(&rx_interrupt.lock(), SOURCE_TAG);
if (rust_uart_rx_queue_full()) {
// disables RX interrupts.
rx_interrupt.DisableInterrupt();
return;
}
c = static_cast<uint8_t>(rx_interrupt.ReadChar());
}
rust_uart_rx_queue_write_char(c);
};
static constexpr auto tx_irq_handler = [](auto& tx_interrupt) {
// Mask the TX interrupt before signalling any blocked thread as there may
// be a race between masking TX here below and unmasking by the blocked
// thread.
{
Guard<MonitoredSpinLock, NoIrqSave> lock(&tx_interrupt.lock(), SOURCE_TAG);
tx_interrupt.DisableInterrupt();
}
// Do not signal the event while holding the sync capability, this could lead
// to invalid lock dependencies.
tx_interrupt.Notify();
};
gUart.Visit([](auto& driver) { driver.Interrupt(tx_irq_handler, rx_irq_handler); });
}
void EnableUartRxInterrupt() {
gUart.Visit([](auto& driver) { driver.template EnableRxInterrupt<IrqSave>(); });
}
bool IsUartTxDrained() {
bool drained = true;
gUart.Visit([&drained](auto& driver) { drained = driver.template TxDrained<IrqSave>(); });
return drained;
}
void PrepareUartForSuspend() {
gUart.Visit([](auto& driver) { driver.template PrepareForSuspend<IrqSave>(); });
}
void WakeupUartFromSuspend() {
gUart.Visit([](auto& driver) { driver.template WakeupFromSuspend<IrqSave>(); });
}
bool ReadUartChar(uint8_t* out_c) {
ktl::optional<char> read;
gUart.Visit([&](auto& driver) { read = driver.Read(); });
if (read) {
*out_c = static_cast<uint8_t>(*read);
return true;
}
return false;
}
void WriteUart(const char* str, size_t len, bool blocking) {
const auto mode =
blocking ? UartSyncPolicy::Waiter::Blocking::kYes : UartSyncPolicy::Waiter::Blocking::kNo;
gUart.Visit([str, len, mode](auto& driver) { driver.template Write<IrqSave>({str, len}, mode); });
}
void WriteUartPanicChar(char c) {
gUart.Visit([c](auto& driver) { driver.Write({&c, 1}, UartSyncPolicy::Waiter::Blocking::kNo); });
}
} // namespace
void UartDriverHandoffEarly(const uart::all::Driver& serial) {
rust_uart_driver_handoff_early(&serial);
}
void UartDriverHandoffPostVm(const uart::all::Driver& serial) {
rust_uart_driver_handoff_post_vm(&serial);
}
void UartDriverHandoffLate(const uart::all::Driver& serial) {
rust_uart_driver_handoff_late(&serial);
}
extern "C" {
// TODO(https://fxbug.dev/537458631): Remove the annotations once cross-language inlining works.
FFI_ALWAYS_INLINE bool cpp_uart_enabled() { return static_cast<bool>(gUart); }
FFI_ALWAYS_INLINE void cpp_uart_set_driver(const void* serial) {
gUart = *static_cast<const uart::all::Driver*>(serial);
}
FFI_ALWAYS_INLINE size_t cpp_uart_unparse(char* buf, size_t len) { return UnparseUart(buf, len); }
FFI_ALWAYS_INLINE void cpp_uart_get_driver_info(UartDriverInfo* out) { GetUartDriverInfo(out); }
FFI_ALWAYS_INLINE void cpp_uart_init_interrupt(uint32_t uart_irq) { InitUartInterrupt(uart_irq); }
FFI_ALWAYS_INLINE void cpp_uart_interrupt() { HandleUartInterrupt(); }
FFI_ALWAYS_INLINE void cpp_uart_enable_rx_interrupt() { EnableUartRxInterrupt(); }
FFI_ALWAYS_INLINE bool cpp_uart_tx_drained() { return IsUartTxDrained(); }
FFI_ALWAYS_INLINE void cpp_uart_prepare_for_suspend() { PrepareUartForSuspend(); }
FFI_ALWAYS_INLINE void cpp_uart_wakeup_from_suspend() { WakeupUartFromSuspend(); }
FFI_ALWAYS_INLINE bool cpp_uart_read(uint8_t* out_c) { return ReadUartChar(out_c); }
FFI_ALWAYS_INLINE void cpp_uart_write(const char* str, size_t len, bool blocking) {
WriteUart(str, len, blocking);
}
FFI_ALWAYS_INLINE void cpp_uart_pputc(char c) { WriteUartPanicChar(c); }
} // extern "C"