| // Copyright 2016 The Fuchsia Authors |
| // Copyright (c) 2014 Travis Geiselbrecht |
| // |
| // Use of this source code is governed by a MIT-style |
| // license that can be found in the LICENSE file or at |
| // https://opensource.org/licenses/MIT |
| |
| #include <asm.h> |
| #include <arch/asm_macros.h> |
| #include <arch/arch_thread.h> |
| #include <arch/arm64.h> |
| #include <arch/arm64/exceptions.h> |
| #include <zircon/zx-syscall-numbers.h> |
| |
| .section .text.boot.vectab,"ax",@progbits |
| .align 12 |
| |
| #define DW_REG_lr 30 |
| #define DW_REG_sp 31 |
| // The "current mode exception link register", which for our purposes is elr_el1. |
| #define DW_REG_ELR_mode 33 |
| |
| #define lr x30 |
| #define elr1 DW_REG_ELR_mode |
| |
| // offset where sp,elr,spsr,lr,mdscr goes in the iframe |
| #define regsave_special_reg_offset (30 * 8) |
| |
| // offset where x20-x29 goes in the iframe |
| #define regsave_high_reg_offset (20 * 8) |
| |
| .macro regsave_long |
| // There are 5 regs @ regsave_special_reg_offset, plus one unused space = 6 to |
| // maintain 16-byte padding. |
| sub_from_sp (6*8) |
| push_regs x28, x29 |
| push_regs x26, x27 |
| push_regs x24, x25 |
| push_regs x22, x23 |
| push_regs x20, x21 |
| push_regs x18, x19 |
| push_regs x16, x17 |
| push_regs x14, x15 |
| push_regs x12, x13 |
| push_regs x10, x11 |
| push_regs x8, x9 |
| push_regs x6, x7 |
| push_regs x4, x5 |
| push_regs x2, x3 |
| push_regs x0, x1 |
| // Preserve x0-x7 for syscall arguments |
| mrs x9, sp_el0 |
| // x10 (containing elr_el1) is used in syscall handler |
| mrs x10, elr_el1 |
| mrs x11, spsr_el1 |
| mrs x12, mdscr_el1 |
| stp lr, x9, [sp, #regsave_special_reg_offset] |
| .cfi_rel_offset lr, (regsave_special_reg_offset) |
| .cfi_rel_offset sp, (regsave_special_reg_offset + 8) |
| stp x10, x11, [sp, #regsave_special_reg_offset + 16] |
| .cfi_rel_offset elr1, (regsave_special_reg_offset + 16) |
| str x12, [sp, #regsave_special_reg_offset + 32] |
| .endm |
| |
| .macro regsave_short |
| // Align the same as the long version above. The 10 extra words correspond to |
| // x20 through x29 which are not saved here. |
| sub_from_sp ((6*8) + (10*8)) |
| push_regs x18, x19 |
| push_regs x16, x17 |
| push_regs x14, x15 |
| push_regs x12, x13 |
| push_regs x10, x11 |
| push_regs x8, x9 |
| push_regs x6, x7 |
| push_regs x4, x5 |
| push_regs x2, x3 |
| push_regs x0, x1 |
| // Preserve x0-x7 to mirror regsave_long |
| mrs x9, sp_el0 |
| mrs x10, elr_el1 |
| mrs x11, spsr_el1 |
| stp lr, x9, [sp, #regsave_special_reg_offset] |
| .cfi_rel_offset lr, (regsave_special_reg_offset) |
| .cfi_rel_offset sp, (regsave_special_reg_offset + 8) |
| stp x10, x11, [sp, #regsave_special_reg_offset + 16] |
| .cfi_rel_offset elr1, (regsave_special_reg_offset + 16) |
| .endm |
| |
| // convert a short iframe to a long one by patching in the additional 10 words to save |
| .macro regsave_short_to_long |
| stp x20, x21, [sp, #regsave_high_reg_offset] |
| stp x22, x23, [sp, #regsave_high_reg_offset + 0x10] |
| stp x24, x25, [sp, #regsave_high_reg_offset + 0x20] |
| stp x26, x27, [sp, #regsave_high_reg_offset + 0x30] |
| stp x28, x29, [sp, #regsave_high_reg_offset + 0x40] |
| mrs x12, mdscr_el1 |
| str x12, [sp, #regsave_special_reg_offset + 32] |
| .endm |
| |
| // Once we pop the stack past the saved sp_el0, elr_el1 the userspace values |
| // are inaccessible. |
| .macro mark_lr_sp_inaccessible |
| // TODO(dje): gdb tries to use some value for these even if "undefined", |
| // as a workaround set their values to zero which will cause gdb to |
| // terminate the backtrace. Need to revisit, file gdb bug if necessary. |
| cfi_register_is_zero DW_REG_sp |
| cfi_register_is_zero DW_REG_ELR_mode |
| .endm |
| |
| .macro regrestore_long |
| // Preserve x0-x1 for syscall returns (eventually x0-x7) |
| ldp lr, x9, [sp, #regsave_special_reg_offset] |
| .cfi_same_value lr |
| ldp x10, x11, [sp, #regsave_special_reg_offset + 16] |
| ldr x12, [sp, #regsave_special_reg_offset + 32] |
| msr sp_el0, x9 |
| msr elr_el1, x10 |
| msr spsr_el1, x11 |
| msr mdscr_el1, x12 |
| pop_regs x0, x1 |
| pop_regs x2, x3 |
| pop_regs x4, x5 |
| pop_regs x6, x7 |
| pop_regs x8, x9 |
| pop_regs x10, x11 |
| pop_regs x12, x13 |
| pop_regs x14, x15 |
| pop_regs x16, x17 |
| pop_regs x18, x19 |
| pop_regs x20, x21 |
| pop_regs x22, x23 |
| pop_regs x24, x25 |
| pop_regs x26, x27 |
| pop_regs x28, x29 |
| add_to_sp (6*8) |
| mark_lr_sp_inaccessible |
| .endm |
| |
| .macro regrestore_short |
| // Preserve x0-x7 to mirror regrestore_long |
| ldp lr, x9, [sp, #regsave_special_reg_offset] |
| .cfi_same_value lr |
| ldp x10, x11, [sp, #regsave_special_reg_offset + 16] |
| msr sp_el0, x9 |
| msr elr_el1, x10 |
| msr spsr_el1, x11 |
| pop_regs x0, x1 |
| pop_regs x2, x3 |
| pop_regs x4, x5 |
| pop_regs x6, x7 |
| pop_regs x8, x9 |
| pop_regs x10, x11 |
| pop_regs x12, x13 |
| pop_regs x14, x15 |
| pop_regs x16, x17 |
| pop_regs x18, x19 |
| add_to_sp ((6*8) + (10*8)) |
| mark_lr_sp_inaccessible |
| .endm |
| |
| .macro start_isr_cfi |
| .cfi_startproc simple |
| .cfi_signal_frame |
| // The return address is in elr_el1, not lr. |
| .cfi_return_column elr1 |
| .cfi_def_cfa sp, 0 |
| .endm |
| |
| .macro start_isr_func |
| start_isr_cfi |
| ALL_CFI_SAME_VALUE |
| .cfi_undefined elr1 |
| .endm |
| |
| .macro start_helper_cfi |
| .cfi_startproc simple |
| .cfi_signal_frame |
| .cfi_def_cfa sp, (regsave_special_reg_offset + 4 * 8) |
| .endm |
| |
| // The CFA offset of integer register |regno| (regno = 0-29). |
| #define REG_CFA_OFFSET(regno) .cfi_offset x##regno, -((4 * 8) + ((30 - (regno)) * 8)) |
| |
| // Mark the locations of the registers based on the CFA so that the |
| // location doesn't change as the regs are popped. |
| .macro setup_short_helper_cfi |
| REG_CFA_OFFSET(0) |
| REG_CFA_OFFSET(1) |
| REG_CFA_OFFSET(2) |
| REG_CFA_OFFSET(3) |
| REG_CFA_OFFSET(4) |
| REG_CFA_OFFSET(5) |
| REG_CFA_OFFSET(6) |
| REG_CFA_OFFSET(7) |
| REG_CFA_OFFSET(8) |
| REG_CFA_OFFSET(9) |
| REG_CFA_OFFSET(10) |
| REG_CFA_OFFSET(11) |
| REG_CFA_OFFSET(12) |
| REG_CFA_OFFSET(13) |
| REG_CFA_OFFSET(14) |
| REG_CFA_OFFSET(15) |
| REG_CFA_OFFSET(16) |
| REG_CFA_OFFSET(17) |
| REG_CFA_OFFSET(18) |
| REG_CFA_OFFSET(19) |
| .cfi_offset sp, -(3 * 8) |
| .cfi_offset lr, -(4 * 8) |
| .endm |
| |
| .macro setup_long_helper_cfi |
| setup_short_helper_cfi |
| REG_CFA_OFFSET(20) |
| REG_CFA_OFFSET(21) |
| REG_CFA_OFFSET(22) |
| REG_CFA_OFFSET(23) |
| REG_CFA_OFFSET(24) |
| REG_CFA_OFFSET(25) |
| REG_CFA_OFFSET(26) |
| REG_CFA_OFFSET(27) |
| REG_CFA_OFFSET(28) |
| REG_CFA_OFFSET(29) |
| .endm |
| |
| .macro start_short_helper |
| start_helper_cfi |
| setup_short_helper_cfi |
| .endm |
| |
| .macro start_long_helper |
| start_helper_cfi |
| setup_long_helper_cfi |
| .endm |
| |
| .macro invalid_exception, which |
| start_isr_func |
| regsave_long |
| mov x1, #\which |
| mov x0, sp |
| bl arm64_invalid_exception |
| b . |
| .endm |
| |
| .macro irq_exception, exception_flags |
| start_isr_func |
| regsave_short |
| msr daifclr, #1 /* reenable fiqs once elr and spsr have been saved */ |
| mov x0, sp |
| mov x1, \exception_flags |
| bl arm64_irq |
| cbnz x0, arm64_finish_user_irq_wrapper /* anything extra to do? */ |
| msr daifset, #1 /* disable fiqs to protect elr and spsr restore */ |
| b arm64_exc_shared_restore_short |
| .endm |
| |
| .macro sync_exception, exception_flags, from_lower_el_64=0 |
| start_isr_func |
| regsave_long |
| mrs x9, esr_el1 |
| .if \from_lower_el_64 |
| // If this is a syscall, x0-x7 contain args and x16 contains syscall num. |
| // x10 contains elr_el1. |
| lsr x11, x9, #26 // shift esr right 26 bits to get ec |
| cmp x11, #0x15 // check for 64-bit syscall |
| beq arm64_syscall_dispatcher // and jump to syscall handler |
| .endif |
| // Prepare the default sync_exception args |
| mov x0, sp |
| mov x1, \exception_flags |
| mov w2, w9 |
| bl arm64_sync_exception |
| b arm64_exc_shared_restore_long |
| .endm |
| |
| FUNCTION_LABEL(arm64_el1_exception_base) |
| |
| /* exceptions from current EL, using SP0 */ |
| .org 0x000 |
| LOCAL_FUNCTION_LABEL(arm64_el1_sync_exc_current_el_SP0) |
| invalid_exception 0 |
| END_FUNCTION(arm64_el1_sync_exc_current_el_SP0) |
| |
| .org 0x080 |
| LOCAL_FUNCTION_LABEL(arm64_el1_irq_current_el_SP0) |
| invalid_exception 1 |
| END_FUNCTION(arm64_el1_irq_current_el_SP0) |
| |
| .org 0x100 |
| LOCAL_FUNCTION_LABEL(arm64_el1_fiq_current_el_SP0) |
| invalid_exception 2 |
| END_FUNCTION(arm64_el1_fiq_current_el_SP0) |
| |
| .org 0x180 |
| LOCAL_FUNCTION_LABEL(arm64_el1_err_exc_current_el_SP0) |
| invalid_exception 3 |
| END_FUNCTION(arm64_el1_err_exc_current_el_SP0) |
| |
| /* exceptions from current EL, using SPx */ |
| .org 0x200 |
| LOCAL_FUNCTION_LABEL(arm64_el1_sync_exc_current_el_SPx) |
| sync_exception #0 /* same EL, arm64 */ |
| END_FUNCTION(arm64_el1_sync_exc_current_el_SPx) |
| |
| .org 0x280 |
| LOCAL_FUNCTION_LABEL(arm64_el1_irq_current_el_SPx) |
| irq_exception #0 /* same EL, arm64 */ |
| END_FUNCTION(arm64_el1_irq_current_el_SPx) |
| |
| .org 0x300 |
| LOCAL_FUNCTION_LABEL(arm64_el1_fiq_current_el_SPx) |
| start_isr_func |
| regsave_short |
| mov x0, sp |
| bl platform_fiq |
| b arm64_exc_shared_restore_short |
| END_FUNCTION(arm64_el1_fiq_current_el_SPx) |
| |
| .org 0x380 |
| LOCAL_FUNCTION_LABEL(arm64_el1_err_exc_current_el_SPx) |
| invalid_exception 0x13 |
| END_FUNCTION(arm64_el1_err_exc_current_el_SPx) |
| |
| /* exceptions from lower EL, running arm64 */ |
| .org 0x400 |
| LOCAL_FUNCTION_LABEL(arm64_el1_sync_exc_lower_el_64) |
| sync_exception #(ARM64_EXCEPTION_FLAG_LOWER_EL), 1 |
| END_FUNCTION(arm64_el1_sync_exc_lower_el_64) |
| |
| .org 0x480 |
| LOCAL_FUNCTION_LABEL(arm64_el1_irq_lower_el_64) |
| irq_exception #(ARM64_EXCEPTION_FLAG_LOWER_EL) |
| END_FUNCTION(arm64_el1_irq_lower_el_64) |
| |
| .org 0x500 |
| LOCAL_FUNCTION_LABEL(arm64_el1_fiq_lower_el_64) |
| start_isr_func |
| regsave_short |
| mov x0, sp |
| bl platform_fiq |
| b arm64_exc_shared_restore_short |
| END_FUNCTION(arm64_el1_fiq_lower_el_64) |
| |
| .org 0x580 |
| LOCAL_FUNCTION_LABEL(arm64_el1_err_exc_lower_el_64) |
| invalid_exception 0x23 |
| END_FUNCTION(arm64_el1_err_exc_lower_el_64) |
| |
| /* exceptions from lower EL, running arm32 */ |
| .org 0x600 |
| LOCAL_FUNCTION_LABEL(arm64_el1_sync_exc_lower_el_32) |
| sync_exception #(ARM64_EXCEPTION_FLAG_LOWER_EL|ARM64_EXCEPTION_FLAG_ARM32) |
| END_FUNCTION(arm64_el1_sync_exc_lower_el_32) |
| |
| .org 0x680 |
| LOCAL_FUNCTION_LABEL(arm64_el1_irq_lower_el_32) |
| irq_exception #(ARM64_EXCEPTION_FLAG_LOWER_EL|ARM64_EXCEPTION_FLAG_ARM32) |
| END_FUNCTION(arm64_el1_irq_lower_el_32) |
| |
| .org 0x700 |
| LOCAL_FUNCTION_LABEL(arm64_el1_fiq_lower_el_32) |
| start_isr_func |
| regsave_short |
| mov x0, sp |
| bl platform_fiq |
| b arm64_exc_shared_restore_short |
| END_FUNCTION(arm64_el1_fiq_lower_el_32) |
| |
| .org 0x780 |
| LOCAL_FUNCTION_LABEL(arm64_el1_err_exc_lower_el_32) |
| invalid_exception 0x33 |
| END_FUNCTION(arm64_el1_err_exc_lower_el_32) |
| |
| /* If an IRQ happened in userspace, and either the thread was signaled or |
| needs to be rescheduled, then we end up here after arm64_irq returns. |
| Suspending the thread requires constructing a long iframe in order to |
| provide the values of all regs to any debugger that wishes to access |
| them, but we can't do that until arm64_irq returns as we rely on the |
| compiler to save/restore callee-saved regs. */ |
| LOCAL_FUNCTION_LABEL(arm64_finish_user_irq_wrapper) |
| start_short_helper |
| /* if we're only here because of a need to reschedule then we don't |
| need to construct a long iframe */ |
| cmp x0, #ARM64_IRQ_EXIT_RESCHEDULE |
| bne 1f |
| mov x1, #0 /* don't need an iframe, just pass NULL */ |
| bl arm64_finish_user_irq |
| msr daifset, #1 /* disable fiqs to protect elr and spsr restore */ |
| b arm64_exc_shared_restore_short |
| 1: |
| /* convert the short to a long frame */ |
| regsave_short_to_long |
| mov x1, sp |
| bl arm64_finish_user_irq |
| msr daifset, #1 /* disable fiqs to protect elr and spsr restore */ |
| b arm64_exc_shared_restore_long |
| END_FUNCTION(arm64_finish_user_irq_wrapper) |
| |
| LOCAL_FUNCTION_LABEL(arm64_exc_shared_restore_long) |
| start_long_helper |
| regrestore_long |
| eret |
| END_FUNCTION(arm64_exc_shared_restore_long) |
| |
| LOCAL_FUNCTION_LABEL(arm64_exc_shared_restore_short) |
| start_short_helper |
| regrestore_short |
| eret |
| END_FUNCTION(arm64_exc_shared_restore_short) |
| |
| // |
| // Syscall args are in x0-x7 already. |
| // pc is in x10 and needs to go in the next available register, |
| // or the stack if the regs are full. |
| // |
| .macro pre_args, nargs |
| .if \nargs == 8 |
| push_regs x10, x11 // push twice to maintain alignment |
| .else |
| mov x\nargs, x10 |
| .endif |
| .endm |
| |
| .macro post_args, nargs |
| .if \nargs == 8 |
| pop_regs x10, x11 |
| .endif |
| b .Lpost_syscall |
| .endm |
| |
| // |
| // Expected state prior to arm64_syscall_dispatcher branch... |
| // |
| // x0-x7 - contains syscall arguments |
| // x9 - contains esr_el1 |
| // x10 - contains elr_el1 |
| // x16 - contains syscall_num |
| // sp - points to base of frame (frame->r[0]) |
| // |
| // Expected state prior to unknown_syscall and wrapper_syscall... |
| // |
| // x0-x7 - contains syscall arguments |
| // x10 - contains userspace pc |
| // |
| LOCAL_FUNCTION_LABEL(arm64_syscall_dispatcher) |
| start_isr_func |
| // Restore per cpu pointer |
| mrs x11, tpidr_el1 |
| ldr x18, [x11, #CURRENT_PERCPU_PTR_OFFSET] |
| // Verify syscall number and call the unknown handler if bad. |
| cmp x16, #ZX_SYS_COUNT |
| bhs .Lunknown_syscall |
| // Jump to the right syscall wrapper. |
| // call_wrapper_table is 4096 byte aligned so just use adrp |
| adrp x12, call_wrapper_table |
| ldr x12, [x12, x16, LSL#3] |
| br x12 |
| .Lunknown_syscall: |
| mov x0, x16 |
| pre_args 1 |
| bl unknown_syscall |
| post_args 1 |
| .Lpost_syscall: |
| // Upon return from syscall, x0 = status, x1 = thread signalled |
| // Move the status to frame->r[0] for return to userspace. |
| str x0, [sp] |
| // Check for pending signals. If none, just return. |
| cbz x1, arm64_exc_shared_restore_long |
| mov x0, sp |
| bl arm64_thread_process_pending_signals |
| b arm64_exc_shared_restore_long |
| END_FUNCTION(arm64_syscall_dispatcher) |
| |
| // Adds a label for making the syscall and adds it to the jump table. |
| .macro syscall_dispatch nargs, syscall |
| .pushsection .text.syscall-dispatch,"ax",%progbits |
| LOCAL_FUNCTION(.Lcall_\syscall\()) |
| pre_args \nargs |
| bl wrapper_\syscall |
| post_args \nargs |
| END_FUNCTION(.Lcall_\syscall\()) |
| .popsection |
| .pushsection .rodata.syscall-table,"a",%progbits |
| .quad .Lcall_\syscall |
| .popsection |
| .endm |
| |
| // Adds the label for the jump table. |
| .macro start_syscall_dispatch |
| .pushsection .rodata.syscall-table,"a",%progbits |
| // align on 4096 byte boundary to save an instruction on table lookup |
| .balign 4096 |
| call_wrapper_table: |
| .popsection |
| .endm |
| |
| #include <zircon/syscall-kernel-branches.S> |
