| // Copyright 2016 The Fuchsia Authors |
| // Copyright (c) 2016 Travis Geiselbrecht |
| // |
| // Use of this source code is governed by a MIT-style |
| // license that can be found in the LICENSE file or at |
| // https://opensource.org/licenses/MIT |
| |
| #include <asm.h> |
| #include <arch/x86/mp.h> |
| #include <lib/code-patching/asm.h> |
| #include <arch/code-patches/case-id-asm.h> |
| #include <lib/syscalls/zx-syscall-numbers.h> |
| |
| #define DW_REG_rsp 0x7 |
| #define DW_REG_rip 0x10 |
| |
| // These macros ensure the stack pointer remains 16-byte aligned. |
| .macro pre_push n |
| .if \n % 2 == 1 |
| push_value $0 |
| .endif |
| .endm |
| |
| .macro post_pop n |
| add_to_sp ((\n + (\n % 2)) * 8) |
| .endm |
| |
| #define ZERO_COMMON_UNUSED_REGISTERS \ |
| xorl %eax, %eax; \ |
| xorl %ebx, %ebx; \ |
| xorl %ebp, %ebp; \ |
| xorq %r10, %r10; \ |
| xorq %r11, %r11; \ |
| xorq %r12, %r12; \ |
| xorq %r13, %r13; \ |
| xorq %r14, %r14; \ |
| xorq %r15, %r15 |
| |
| // Macros for preparing ABI conformant calls for syscall wrappers. |
| // Shuffles syscall arguments into x86-64 ABI locations before |
| // calling C++ syscall handlers. |
| // Zeros unused registers to constrain speculative execution in syscall |
| // handlers with user-passed register values. |
| .macro pre_args n |
| .if \n > 5 |
| // We use the stack for arguments 6, 7, and 8. |
| pre_push (\n - 5) |
| .endif |
| |
| .if \n == 0 |
| // syscall_0(rip) |
| // |
| // rip from rcx to rdi |
| mov %rcx, %rdi |
| xorl %ecx, %ecx |
| xorl %edx, %edx |
| xorl %esi, %esi |
| xorq %r8, %r8 |
| xorq %r9, %r9 |
| ZERO_COMMON_UNUSED_REGISTERS |
| .elseif \n == 1 |
| // syscall_1(arg_1, rip) |
| // |
| // arg_1 from rdi to rdi |
| // rip from rcx to rsi |
| mov %rcx, %rsi |
| xorl %ecx, %ecx |
| xorl %edx, %edx |
| xorq %r8, %r8 |
| xorq %r9, %r9 |
| ZERO_COMMON_UNUSED_REGISTERS |
| .elseif \n == 2 |
| // syscall_2(arg_1, arg_2, rip) |
| // |
| // arg_1 from rdi to rdi |
| // arg_2 from rsi to rsi |
| // rip from rcx to rdx |
| mov %rcx, %rdx |
| xorl %ecx, %ecx |
| xorq %r8, %r8 |
| xorq %r9, %r9 |
| ZERO_COMMON_UNUSED_REGISTERS |
| .elseif \n == 3 |
| // syscall_3(arg_1, arg_2, arg_3, rip) |
| // |
| // arg_1 from rdi to rdi |
| // arg_2 from rsi to rsi |
| // arg_3 from rdx to rdx |
| // rip from rcx to rcx |
| xorq %r8, %r8 |
| xorq %r9, %r9 |
| ZERO_COMMON_UNUSED_REGISTERS |
| .elseif \n == 4 |
| // syscall_4(arg_1, arg_2, arg_3, arg_4, rip) |
| // |
| // arg_1 from rdi to rdi |
| // arg_2 from rsi to rsi |
| // arg_3 from rdx to rdx |
| // arg_4 from r10 to rcx |
| // rip from rcx to r8 |
| mov %rcx, %r8 |
| mov %r10, %rcx |
| xorq %r9, %r9 |
| ZERO_COMMON_UNUSED_REGISTERS |
| .elseif \n == 5 |
| // syscall_5(arg_1, arg_2, arg_3, arg_4, arg_5, rip) |
| // |
| // arg_1 from rdi to rdi |
| // arg_2 from rsi to rsi |
| // arg_3 from rdx to rdx |
| // arg_4 from r10 to rcx |
| // arg_5 from r8 to r8 |
| // rip from rcx to r9 |
| mov %rcx, %r9 |
| mov %r10, %rcx |
| ZERO_COMMON_UNUSED_REGISTERS |
| .elseif \n == 6 |
| // syscall_6(arg_1, arg_2, arg_3, arg_4, arg_5, arg_6, rip) |
| // |
| // arg_1 from rdi to rdi |
| // arg_2 from rsi to rsi |
| // arg_3 from rdx to rdx |
| // arg_4 from r10 to rcx |
| // arg_5 from r8 to r8 |
| // arg_6 from r9 to r9 |
| // rip from rcx to (rsp) |
| push_value %rcx |
| mov %r10, %rcx |
| ZERO_COMMON_UNUSED_REGISTERS |
| .elseif \n == 7 |
| // syscall_7(arg_1, arg_2, arg_3, arg_4, arg_5, arg_6, arg_7, rip) |
| // |
| // arg_1 from rdi to rdi |
| // arg_2 from rsi to rsi |
| // arg_3 from rdx to rdx |
| // arg_4 from r10 to rcx |
| // arg_5 from r8 to r8 |
| // arg_6 from r9 to r9 |
| // arg_7 from r12 to (rsp) |
| // rip from rcx to 8(rsp) |
| push_value %rcx |
| push_value %r12 |
| mov %r10, %rcx |
| ZERO_COMMON_UNUSED_REGISTERS |
| .elseif \n == 8 |
| // syscall_8(arg_1, arg_2, arg_3, arg_4, arg_5, arg_6, arg_7, arg_8, rip) |
| // |
| // arg_1 from rdi to rdi |
| // arg_2 from rsi to rsi |
| // arg_3 from rdx to rdx |
| // arg_4 from r10 to rcx |
| // arg_5 from r8 to r8 |
| // arg_6 from r9 to r9 |
| // arg_7 from r12 to (%rsp) |
| // arg_8 from r13 to 8(%rsp) |
| // rip from rcx to 16(%rsp) |
| push_value %rcx |
| push_value %r13 |
| push_value %r12 |
| mov %r10, %rcx |
| ZERO_COMMON_UNUSED_REGISTERS |
| .endif |
| .endm |
| |
| .macro post_args n |
| .if \n > 5 |
| // We use the stack for arguments 6, 7, and 8. |
| post_pop (\n - 5) |
| .endif |
| |
| JMP_AND_SPECULATION_POSTFENCE(x86_syscall_cleanup_and_return) |
| .endm |
| |
| .macro cfi_outermost_frame |
| // TODO(dje): IWBN to use .cfi_undefined here, but gdb didn't properly |
| // handle initial attempts. Need to try again (or file gdb bug). |
| cfi_register_is_zero DW_REG_rsp |
| cfi_register_is_zero DW_REG_rip |
| .endm |
| |
| // Adds a label for making the syscall and adds it to the jump table. |
| .macro syscall_dispatch nargs, name |
| .pushsection .text.syscall-dispatch,"ax",%progbits |
| .balign 16 |
| LOCAL_FUNCTION(x86_syscall_call_\name) |
| // See x86_syscall for why this is here. |
| cfi_outermost_frame |
| pre_args \nargs |
| call wrapper_\name |
| post_args \nargs |
| END_FUNCTION(x86_syscall_call_\name) |
| .popsection |
| .pushsection .rodata.syscall-table,"a",%progbits |
| .quad x86_syscall_call_\name |
| .popsection |
| .endm |
| |
| // Adds the label for the jump table. |
| .macro start_syscall_dispatch |
| .pushsection .rodata.syscall-table,"a",%progbits |
| .balign 8 |
| .Lcall_wrapper_table: |
| .popsection |
| .endm |
| |
| .text |
| |
| // kernel side of the SYSCALL instruction |
| // state on entry: |
| // RCX holds user RIP |
| // R11 holds user RFLAGS |
| // RSP still holds user stack |
| // CS loaded with kernel CS from IA32_STAR |
| // SS loaded with kernel CS + 8 from IA32_STAR |
| |
| // args passed: |
| // rax - syscall # and return |
| // rbx - saved |
| // rcx - modified as part of syscall instruction |
| // rdx - arg 3 |
| // rsi - arg 2 |
| // rdi - arg 1 |
| // rbp - saved |
| // rsp - saved |
| // r8 - arg 5 |
| // r9 - arg 6 |
| // r10 - arg 4 |
| // r11 - modified as part of syscall instruction |
| // r12 - arg 7 |
| // r13 - arg 8 |
| // r14 - saved |
| // r15 - saved |
| // |
| .balign 16 |
| FUNCTION_LABEL(x86_syscall) |
| .cfi_startproc simple |
| // CFI tracking here doesn't (currently) try to support backtracing from |
| // kernel space to user space. This is left for later. For now just say |
| // %rsp and %rip of the previous frame are zero, mark all the other |
| // registers as undefined, and have all register push/pop just specify |
| // stack adjustments and not how to find the register's value. |
| cfi_outermost_frame |
| // The default for caller-saved regs is "undefined", but for completeness |
| // sake mark them all as undefined. |
| ALL_CFI_UNDEFINED |
| |
| // swap to the kernel GS register |
| swapgs |
| |
| // save the user stack pointer |
| mov %rsp, %gs:PERCPU_SAVED_USER_SP_OFFSET |
| |
| // load the kernel stack pointer |
| mov %gs:PERCPU_KERNEL_SP_OFFSET, %rsp |
| .cfi_def_cfa %rsp, 0 |
| |
| // Save all the general purpose registers in a syscall_regs_t |
| // struct on the kernel's stack. |
| // |
| // By saving (and later restoring) all of the registers rather than just |
| // the bare minimum, we ensure that kernel data is not inadvertently |
| // leaked back to user mode. |
| push_value %gs:PERCPU_SAVED_USER_SP_OFFSET // User stack |
| push_value %r11 // RFLAGS |
| push_value %rcx // RIP |
| push_value %r15 |
| push_value %r14 |
| push_value %r13 |
| push_value %r12 |
| push_value $0 // R11 was trashed by the syscall instruction. |
| push_value %r10 |
| push_value %r9 |
| push_value %r8 |
| push_value %rbp |
| push_value %rdi |
| push_value %rsi |
| push_value %rdx |
| push_value $0 // RCX was trashed by the syscall instruction. |
| push_value %rbx |
| push_value %rax |
| |
| // At this point: |
| // rsp points at a syscall_regs_t struct |
| // rsp is 16-byte aligned |
| // |
| // Any changes to the stack here need to be reflected in |
| // pre_push and post_pop macros above to maintain alignment. |
| |
| // check to see if we're in restricted mode |
| cmpl $0, %gs:PERCPU_IN_RESTRICTED_MODE |
| jne .Lrestricted_syscall |
| |
| // Bounds-check system call number and jump to handler. |
| xorq %r11, %r11 |
| cmp $ZX_SYS_COUNT, %rax |
| jae .Lunknown_syscall |
| // Spectre V1: If syscall number >= ZX_SYS_COUNT, replace it with zero. The test/branch above |
| // means this can only occur in wrong-path speculative executions. It's critical to the |
| // correctness of the mitigation that the following comparison is performed using cmov rather |
| // than a test / conditional branch. |
| cmovge %r11, %rax |
| leaq .Lcall_wrapper_table(%rip), %r11 |
| movq (%r11,%rax,8), %r11 |
| // Spectre V2: Use retpoline to invoke system call handler. |
| JMP_AND_SPECULATION_POSTFENCE(__x86_indirect_thunk_r11) |
| .Lunknown_syscall: |
| mov %rax, %rdi // move the syscall number into the 0 arg slot |
| mov %rcx, %rsi // pc into arg 1 |
| call unknown_syscall |
| JMP_AND_SPECULATION_POSTFENCE(x86_syscall_cleanup_and_return) |
| |
| .Lrestricted_syscall: |
| mov %rsp, %rdi |
| call syscall_from_restricted |
| // There is no path that returns from this call, but if it did, trap. |
| ud2 |
| |
| END_FUNCTION(x86_syscall) |
| |
| .balign 16 |
| // All the syscall wrapper routines return to here. |
| LOCAL_FUNCTION_LABEL(x86_syscall_cleanup_and_return) |
| .cfi_startproc simple |
| // At this point: |
| // rax = syscall result |
| // rdx = non-zero if thread was signaled |
| // rsp = address of syscall_regs_t |
| |
| // Save syscall result to the syscall_regs_t on the stack to ensure it's not trashed |
| // by upcoming function calls and to ensure debuggers can see and modify it if the thread was |
| // suspened. |
| movq %rax, (%rsp) |
| |
| // Move the thread-signaled indicator to a callee-saved register to ensure it's not trashed by |
| // upcoming function calls. |
| movq %rdx, %r12 |
| |
| // Spectre V1: If the syscall is going to return certain errors, flush the L1D$ |
| // TODO(https://fxbug.dev/42108888): Can this be folded together w/ MD_CLEAR below? |
| test %rax, %rax |
| jz 1f |
| movq %rax, %rdi |
| call x86_cpu_maybe_l1d_flush |
| 1: |
| |
| // Was the thread signaled? |
| test %r12, %r12 |
| jnz .Lthread_signaled |
| |
| .Lreturn_from_syscall: |
| #if LK_DEBUGLEVEL > 2 |
| // Ensure that interrupts are disabled on all paths to here. |
| // If they are not, enter a spinloop. |
| pushf |
| popq %rax |
| bt $9, %rax // RFLAGS.IF |
| 0: |
| jc 0b // Loop if we found RFLAGS.IF set (interrupts enabled) |
| #endif |
| |
| // If we are affected by the MDS speculative execution vulnerability, flush microarchitectural |
| // buffers via mds_buff_overwrite(). Patching will NOP out the flush where it is not required. |
| .global syscall_maybe_mds_buff_overwrite |
| syscall_maybe_mds_buff_overwrite: |
| // Mitigates MDS/TAA bugs. See <arch/code-patches/case-id.h> |
| .code_patching.start CASE_ID_MDS_TAA_MITIGATION |
| call mds_buff_overwrite |
| .code_patching.end |
| |
| // Restore general purpose registers just before returning. |
| // |
| // It is critical that all registers are reset. The callee-saved registers must be restored per |
| // the ABI. The other registers might contain private kernel data that must not be leaked to |
| // user mode. To ensure data is not leaked in call-clobbered registers, we restored them to |
| // their previous values. Alternatively, we could simply zero them out to ensure data is not |
| // leaked. However, this code path is shared with the path taken by a thread returning to user |
| // mode after its registers have been modified by a debugger so we restore them all to keep it |
| // simple (except for RCX and R11 which are clobbered by the SYSRET instruction). |
| // |
| // TODO(https://fxbug.dev/42141222): Make the restored register state completely capture the thread's state |
| // and make syscalls act more like atomic instructions. |
| pop_value %rax |
| pop_value %rbx |
| pop_value %rcx // Will be overwritten with RIP later on. |
| pop_value %rdx |
| pop_value %rsi |
| pop_value %rdi |
| pop_value %rbp |
| pop_value %r8 |
| pop_value %r9 |
| pop_value %r10 |
| pop_value %r11 // Will be overwritten with RFLAGS later on. |
| pop_value %r12 |
| pop_value %r13 |
| pop_value %r14 |
| pop_value %r15 |
| pop_value %rcx // RIP |
| pop_value %r11 // RFLAGS |
| pop_value %rsp // User stack |
| |
| // put the user gs back |
| swapgs |
| |
| // This will fault if the return address is non-canonical. See |
| // docs/sysret_problem.md for how we avoid that. |
| sysretq |
| |
| .Lthread_signaled: |
| // Pass a pointer to the syscall_regs_t struct as first arg. |
| movq %rsp, %rdi |
| call x86_syscall_process_pending_signals |
| JMP_AND_SPECULATION_POSTFENCE(.Lreturn_from_syscall) |
| END_FUNCTION(x86_syscall_cleanup_and_return) |
| |
| // One of these macros is invoked by kernel.inc for each syscall. |
| |
| // These don't have kernel entry points. |
| #define VDSO_SYSCALL(...) |
| |
| // These are the direct kernel entry points. |
| #define KERNEL_SYSCALL(name, type, attrs, nargs, arglist, prototype) \ |
| syscall_dispatch nargs, name |
| #define INTERNAL_SYSCALL(...) KERNEL_SYSCALL(__VA_ARGS__) |
| #define BLOCKING_SYSCALL(...) KERNEL_SYSCALL(__VA_ARGS__) |
| |
| start_syscall_dispatch |
| |
| #include <lib/syscalls/kernel.inc> |
| |
| #undef VDSO_SYSCALL |
| #undef KERNEL_SYSCALL |
| #undef INTERNAL_SYSCALL |
| #undef BLOCKING_SYSCALL |
