zircon/kernel/dev/timer/arm_generic/arm_generic_timer.cpp

// Copyright 2016 The Fuchsia Authors
// Copyright (c) 2013, Google Inc. All rights reserved.
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT

#include <dev/timer/arm_generic.h>

#include <arch/ops.h>
#include <assert.h>
#include <dev/interrupt.h>
#include <inttypes.h>
#include <lib/fixed_point.h>
#include <lk/init.h>
#include <pdev/driver.h>
#include <platform.h>
#include <platform/timer.h>
#include <trace.h>
#include <zircon/boot/driver-config.h>
#include <zircon/types.h>

#define LOCAL_TRACE 0

/* CNTFRQ AArch64 register */
#define TIMER_REG_CNTFRQ "cntfrq_el0"

/* CNTP AArch64 registers */
#define TIMER_REG_CNTP_CTL "cntp_ctl_el0"
#define TIMER_REG_CNTP_CVAL "cntp_cval_el0"
#define TIMER_REG_CNTP_TVAL "cntp_tval_el0"
#define TIMER_REG_CNTPCT "cntpct_el0"

/* CNTPS "AArch64" registers */
#define TIMER_REG_CNTPS_CTL "cntps_ctl_el1"
#define TIMER_REG_CNTPS_CVAL "cntps_cval_el1"
#define TIMER_REG_CNTPS_TVAL "cntps_tval_el1"

/* CNTV "AArch64" registers */
#define TIMER_REG_CNTV_CTL "cntv_ctl_el0"
#define TIMER_REG_CNTV_CVAL "cntv_cval_el0"
#define TIMER_REG_CNTV_TVAL "cntv_tval_el0"
#define TIMER_REG_CNTVCT "cntvct_el0"

static int timer_irq;

struct fp_32_64 cntpct_per_ns;
struct fp_32_64 ns_per_cntpct;

static uint64_t zx_time_to_cntpct(zx_time_t zx_time) {
    DEBUG_ASSERT(zx_time >= 0);
    return u64_mul_u64_fp32_64(zx_time, cntpct_per_ns);
}

zx_time_t cntpct_to_zx_time(uint64_t cntpct) {
    return u64_mul_u64_fp32_64(cntpct, ns_per_cntpct);
}

static uint32_t read_cntfrq(void) {
    uint32_t cntfrq;

    cntfrq = __arm_rsr(TIMER_REG_CNTFRQ);
    LTRACEF("cntfrq: 0x%08x, %u\n", cntfrq, cntfrq);
    return cntfrq;
}

static uint32_t read_cntp_ctl(void) {
    return __arm_rsr(TIMER_REG_CNTP_CTL);
}

static uint32_t read_cntv_ctl(void) {
    return __arm_rsr(TIMER_REG_CNTV_CTL);
}

static uint32_t read_cntps_ctl(void) {
    return __arm_rsr(TIMER_REG_CNTPS_CTL);
}

static void write_cntp_ctl(uint32_t val) {
    LTRACEF_LEVEL(3, "cntp_ctl: 0x%x %x\n", val, read_cntp_ctl());
    __arm_wsr(TIMER_REG_CNTP_CTL, val);
    __isb(ARM_MB_SY);
}

static void write_cntv_ctl(uint32_t val) {
    LTRACEF_LEVEL(3, "cntv_ctl: 0x%x %x\n", val, read_cntv_ctl());
    __arm_wsr(TIMER_REG_CNTV_CTL, val);
    __isb(ARM_MB_SY);
}

static void write_cntps_ctl(uint32_t val) {
    LTRACEF_LEVEL(3, "cntps_ctl: 0x%x %x\n", val, read_cntps_ctl());
    __arm_wsr(TIMER_REG_CNTPS_CTL, val);
    __isb(ARM_MB_SY);
}

static void write_cntp_cval(uint64_t val) {
    LTRACEF_LEVEL(3, "cntp_cval: 0x%016" PRIx64 ", %" PRIu64 "\n",
                  val, val);
    __arm_wsr64(TIMER_REG_CNTP_CVAL, val);
    __isb(ARM_MB_SY);
}

static void write_cntv_cval(uint64_t val) {
    LTRACEF_LEVEL(3, "cntv_cval: 0x%016" PRIx64 ", %" PRIu64 "\n",
                  val, val);
    __arm_wsr64(TIMER_REG_CNTV_CVAL, val);
    __isb(ARM_MB_SY);
}

static void write_cntps_cval(uint64_t val) {
    LTRACEF_LEVEL(3, "cntps_cval: 0x%016" PRIx64 ", %" PRIu64 "\n",
                  val, val);
    __arm_wsr64(TIMER_REG_CNTPS_CVAL, val);
    __isb(ARM_MB_SY);
}

static void write_cntp_tval(int32_t val) {
    LTRACEF_LEVEL(3, "cntp_tval: %d\n", val);
    __arm_wsr(TIMER_REG_CNTP_TVAL, val);
    __isb(ARM_MB_SY);
}

static void write_cntv_tval(int32_t val) {
    LTRACEF_LEVEL(3, "cntv_tval: %d\n", val);
    __arm_wsr(TIMER_REG_CNTV_TVAL, val);
    __isb(ARM_MB_SY);
}

static void write_cntps_tval(int32_t val) {
    LTRACEF_LEVEL(3, "cntps_tval: %d\n", val);
    __arm_wsr(TIMER_REG_CNTPS_TVAL, val);
    __isb(ARM_MB_SY);
}

static uint64_t read_cntpct(void) {
    return __arm_rsr64(TIMER_REG_CNTPCT);
}

static uint64_t read_cntvct(void) {
    return __arm_rsr64(TIMER_REG_CNTVCT);
}

struct timer_reg_procs {
    void (*write_ctl)(uint32_t val);
    void (*write_cval)(uint64_t val);
    void (*write_tval)(int32_t val);
    uint64_t (*read_ct)(void);
};

__UNUSED static const struct timer_reg_procs cntp_procs = {
    .write_ctl = write_cntp_ctl,
    .write_cval = write_cntp_cval,
    .write_tval = write_cntp_tval,
    .read_ct = read_cntpct,
};

__UNUSED static const struct timer_reg_procs cntv_procs = {
    .write_ctl = write_cntv_ctl,
    .write_cval = write_cntv_cval,
    .write_tval = write_cntv_tval,
    .read_ct = read_cntvct,
};

__UNUSED static const struct timer_reg_procs cntps_procs = {
    .write_ctl = write_cntps_ctl,
    .write_cval = write_cntps_cval,
    .write_tval = write_cntps_tval,
    .read_ct = read_cntpct,
};

#if (TIMER_ARM_GENERIC_SELECTED_CNTV)
static const struct timer_reg_procs* reg_procs = &cntv_procs;
#else
static const struct timer_reg_procs* reg_procs = &cntp_procs;
#endif

static inline void write_ctl(uint32_t val) {
    reg_procs->write_ctl(val);
}

static inline void write_cval(uint64_t val) {
    reg_procs->write_cval(val);
}

static inline void write_tval(uint32_t val) {
    reg_procs->write_tval(val);
}

static uint64_t read_ct(void) {
    uint64_t cntpct = reg_procs->read_ct();
    LTRACEF_LEVEL(3, "cntpct: 0x%016" PRIx64 ", %" PRIu64 "\n",
                  cntpct, cntpct);
    return cntpct;
}

static interrupt_eoi platform_tick(void* arg) {
    write_ctl(0);
    timer_tick(current_time());
    return IRQ_EOI_DEACTIVATE;
}

zx_status_t platform_set_oneshot_timer(zx_time_t deadline) {
    DEBUG_ASSERT(arch_ints_disabled());

    if (deadline < 0) {
        deadline = 0;
    }

    // Add one to the deadline, since with very high probability the deadline
    // straddles a counter tick.
    const uint64_t cntpct_deadline = zx_time_to_cntpct(deadline) + 1;

    // Even if the deadline has already passed, the ARMv8-A timer will fire the
    // interrupt.
    write_cval(cntpct_deadline);
    write_ctl(1);

    return 0;
}

void platform_stop_timer(void) {
    write_ctl(0);
}

void platform_shutdown_timer(void) {
    DEBUG_ASSERT(arch_ints_disabled());
    mask_interrupt(timer_irq);
}

zx_time_t current_time(void) {
    return cntpct_to_zx_time(read_ct());
}

zx_ticks_t current_ticks(void) {
    return read_ct();
}

zx_ticks_t ticks_per_second(void) {
    return u64_mul_u32_fp32_64(1000 * 1000 * 1000, cntpct_per_ns);
}

static uint64_t abs_int64(int64_t a) {
    return (a > 0) ? a : -a;
}

static void test_time_conversion_check_result(uint64_t a, uint64_t b, uint64_t limit) {
    if (a != b) {
        uint64_t diff = abs_int64(a - b);
        if (diff <= limit)
            LTRACEF("ROUNDED by %" PRIu64 " (up to %" PRIu64 " allowed)\n", diff, limit);
        else
            TRACEF("FAIL, off by %" PRIu64 "\n", diff);
    }
}

static void test_zx_time_to_cntpct(uint32_t cntfrq, zx_time_t zx_time) {
    uint64_t cntpct = zx_time_to_cntpct(zx_time);
    const uint64_t nanos_per_sec = ZX_SEC(1);
    uint64_t expected_cntpct = ((uint64_t)cntfrq * zx_time + nanos_per_sec / 2) / nanos_per_sec;

    test_time_conversion_check_result(cntpct, expected_cntpct, 1);
    LTRACEF_LEVEL(2, "zx_time_to_cntpct(%" PRIi64 "): got %" PRIu64
                     ", expect %" PRIu64 "\n",
                  zx_time, cntpct, expected_cntpct);
}

static void test_cntpct_to_zx_time(uint32_t cntfrq, uint64_t expected_s) {
    zx_time_t expected_zx_time = ZX_SEC(expected_s);
    uint64_t cntpct = (uint64_t)cntfrq * expected_s;
    zx_time_t zx_time = cntpct_to_zx_time(cntpct);

    test_time_conversion_check_result(zx_time, expected_zx_time, (1000 * 1000 + cntfrq - 1) / cntfrq);
    LTRACEF_LEVEL(2, "cntpct_to_zx_time(%" PRIu64
                     "): got %" PRIi64 ", expect %" PRIi64 "\n",
                  cntpct, zx_time, expected_zx_time);
}

static void test_time_conversions(uint32_t cntfrq) {
    test_zx_time_to_cntpct(cntfrq, 0);
    test_zx_time_to_cntpct(cntfrq, 1);
    test_zx_time_to_cntpct(cntfrq, 60 * 60 * 24);
    test_zx_time_to_cntpct(cntfrq, 60 * 60 * 24 * 365);
    test_zx_time_to_cntpct(cntfrq, 60 * 60 * 24 * (365 * 10 + 2));
    test_zx_time_to_cntpct(cntfrq, 60ULL * 60 * 24 * (365 * 100 + 2));
    test_zx_time_to_cntpct(cntfrq, 1ULL << 60);
    test_cntpct_to_zx_time(cntfrq, 0);
    test_cntpct_to_zx_time(cntfrq, 1);
    test_cntpct_to_zx_time(cntfrq, 60 * 60 * 24);
    test_cntpct_to_zx_time(cntfrq, 60 * 60 * 24 * 365);
    test_cntpct_to_zx_time(cntfrq, 60 * 60 * 24 * (365 * 10 + 2));
    test_cntpct_to_zx_time(cntfrq, 60ULL * 60 * 24 * (365 * 100 + 2));
}

static void arm_generic_timer_init_conversion_factors(uint32_t cntfrq) {
    fp_32_64_div_32_32(&cntpct_per_ns, cntfrq, ZX_SEC(1));
    fp_32_64_div_32_32(&ns_per_cntpct, ZX_SEC(1), cntfrq);
    dprintf(SPEW, "cntpct_per_ns: %08x.%08x%08x\n", cntpct_per_ns.l0, cntpct_per_ns.l32, cntpct_per_ns.l64);
    dprintf(SPEW, "ns_per_cntpct: %08x.%08x%08x\n", ns_per_cntpct.l0, ns_per_cntpct.l32, ns_per_cntpct.l64);
}

static void arm_generic_timer_init(uint32_t freq_override) {
    uint32_t cntfrq;

    if (freq_override == 0) {
        cntfrq = read_cntfrq();

        if (!cntfrq) {
            TRACEF("Failed to initialize timer, frequency is 0\n");
            return;
        }
    } else {
        cntfrq = freq_override;
    }

    dprintf(INFO, "arm generic timer freq %u Hz\n", cntfrq);

#if LOCAL_TRACE
    LTRACEF("Test min cntfrq\n");
    arm_generic_timer_init_conversion_factors(1);
    test_time_conversions(1);
    LTRACEF("Test max cntfrq\n");
    arm_generic_timer_init_conversion_factors(~0);
    test_time_conversions(~0);
    LTRACEF("Set actual cntfrq\n");
#endif
    arm_generic_timer_init_conversion_factors(cntfrq);
    test_time_conversions(cntfrq);

    LTRACEF("register irq %d on cpu %u\n", timer_irq, arch_curr_cpu_num());
    zx_status_t status = register_int_handler(timer_irq, &platform_tick, NULL);
    DEBUG_ASSERT(status == ZX_OK);
    unmask_interrupt(timer_irq);
}

static void arm_generic_timer_init_secondary_cpu(uint level) {
    LTRACEF("unmask irq %d on cpu %u\n", timer_irq, arch_curr_cpu_num());
    unmask_interrupt(timer_irq);
}

/* secondary cpu initialize the timer just before the kernel starts with interrupts enabled */
LK_INIT_HOOK_FLAGS(arm_generic_timer_init_secondary_cpu,
                   arm_generic_timer_init_secondary_cpu,
                   LK_INIT_LEVEL_THREADING - 1, LK_INIT_FLAG_SECONDARY_CPUS)

static void arm_generic_timer_resume_cpu(uint level) {
    /* Always trigger a timer interrupt on each cpu for now */
    write_tval(0);
    write_ctl(1);
}

LK_INIT_HOOK_FLAGS(arm_generic_timer_resume_cpu, arm_generic_timer_resume_cpu,
                   LK_INIT_LEVEL_PLATFORM, LK_INIT_FLAG_CPU_RESUME)

static void arm_generic_timer_pdev_init(const void* driver_data, uint32_t length) {
    ASSERT(length >= sizeof(dcfg_arm_generic_timer_driver_t));
    auto driver = static_cast<const dcfg_arm_generic_timer_driver_t*>(driver_data);
    uint32_t irq_phys = driver->irq_phys;
    uint32_t irq_virt = driver->irq_virt;
    uint32_t irq_sphys = driver->irq_sphys;

    if (irq_phys && irq_virt && arm64_get_boot_el() < 2) {
        // If we did not boot at EL2 or above, prefer the virtual timer.
        irq_phys = 0;
    }
    if (irq_phys) {
        timer_irq = irq_phys;
        reg_procs = &cntp_procs;
    } else if (irq_virt) {
        timer_irq = irq_virt;
        reg_procs = &cntv_procs;
    } else if (irq_sphys) {
        timer_irq = irq_sphys;
        reg_procs = &cntps_procs;
    } else {
        panic("no irqs set in arm_generic_timer_pdev_init\n");
    }
    smp_mb();

    arm_generic_timer_init(driver->freq_override);
}

LK_PDEV_INIT(arm_generic_timer_pdev_init, KDRV_ARM_GENERIC_TIMER, arm_generic_timer_pdev_init, LK_INIT_LEVEL_PLATFORM_EARLY)