blob: d7bb09a24fe02dc0fbd3634f97c6702b56e39c13 [file] [log] [blame]
// Copyright 2017 The Fuchsia Authors
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
#include <arch/arm64/cache_loop.h>
#include <arch/asm_macros.h>
#include <arch/defines.h>
#include <asm.h>
#include <mexec.h>
/* Arguments Passed via x0 through x8 inclusive */
bootarg0 .req x25
bootarg1 .req x26
bootarg2 .req x27
boot_el .req x28
// This is the address to branch to once the copy is completed.
new_kernel_addr .req x24
// This is a null terminated list of memory regions to copy.
copy_list .req x23
// This many more entires in the current page to copy before we should
// interpret the entry as a pointer to the next page in the linked list.
page_remaining_entries .req x22
tmp .req x9
.section .text
// Turn off the caches and MMU
mrs tmp, sctlr_el1 // Read the SCTLR into a temp
bic tmp, tmp, #(1<<12) // Disable icache
bic tmp, tmp, #(1<<2) // Disable dcache/ucache
bic tmp, tmp, #(1<<0) // Disable the MMU
msr sctlr_el1, tmp // Write the temp back to the control register
// Stash the boot arguments to pass to the next kernel since we expect to trash
// x0 - x5
mov bootarg0, x0
mov bootarg1, x1
mov bootarg2, x2
// Stash the boot el, as we will need it later
mov boot_el, x3
// This list contains the memmove operations that we need to perform.
mov copy_list, x4
// This is the address of the kernel that we eventueally want to jump to.
mov new_kernel_addr, x5
// If we were originally booted in EL2, transition back into EL2
cmp boot_el, #2 cplt_transition_to_boot_el // We booted in EL1, no need to transition up
adr x0, cplt_transition_to_boot_el
hvc #1 // Branch into an EL2 trampoline that bounces to cplt_transition_to_boot_el
// Clean/Invalidate the cache early on.
// We want to make sure that there are no dirty cache entries hanging around
// in the cache before we start the memcpy.
// If these cache entries were to get written back later, they would corrupt
// the state of the system so we clean/invalidate them up front.
bl mexec_arch_clean_invalidate_cache_all
// Outer loop: Execute all the memmove ops in a given page.
ldr page_remaining_entries, =(MAX_OPS_PER_PAGE)
/* Mempy the new kernel over the old kernel. Keep in mind that since the MMU
* is disabled, unaligned accesses are no longer legal. All accesses must be
* word aligned.
// Load a copy operation into memory
ldr x0, [copy_list, MEMMOV_OPS_DST_OFFSET]
ldr x1, [copy_list, MEMMOV_OPS_SRC_OFFSET]
ldr x2, [copy_list, MEMMOV_OPS_LEN_OFFSET]
// Determine if this is the end of the list by checking if all three elems
// in the copy list are null.
orr tmp, x0, x1
orr tmp, tmp, x2
cbz tmp, .Lfinish_copy
// A page may contain up to 169* memmove ops, the 170th op is actually a
// pointer to the next page full of memmove ops.
cbnz page_remaining_entries, .Lcontinue_page // Have we processed 169 entries yet, branch if we haven't.
mov copy_list, x0 // If we've followed all the entries on the current page, follow the next pointer of the linked list.
b .Lcopypage // And execute the list of copies on the page that we just advanced to.
// The copy operation is not null, go ahead and memmove
bl memmove_mexec
// Advance the pointer to the next copy operation.
add copy_list, copy_list, MEMMOV_OPS_STRUCT_LEN
// Decrement the number of entries remaining in this page, if this reaches 0
// we must follow the linked list to the next page.
sub page_remaining_entries, page_remaining_entries, 1
b .Lcopy
bl mexec_arch_clean_invalidate_cache_all
// Restore the bootarguments for the next kernel.
mov x0, bootarg0
mov x1, bootarg1
mov x2, bootarg2
// Get everything out of the pipeline before branching to the new kernel.
dsb sy
// Branch to the next kernel.
br new_kernel_addr
// x6 contains the stride (1 word if we're copying forward
// -1 word if we're copying backwards)
mov x6, 1
// x3 is the start index of the copy, this is the front of the array if
// we're copying forward or the back of the array if we're copying
// backwards.
mov x3, 0
// Convert the length of the array from bytes to machine words
lsr x2, x2, 3
// If the source address and the destination address are the same then
// we can return because there's nothing to be done.
cmp x0, x1
beq .done
// Decide if we need to copy backwards.
blt .no_alias
mov x6, -1 // Set the stride to backwards
mov x3, x2 // Move the copy index to the back of the array
sub x3, x3, 1 // i = (len_wrds - 1); to start at the last word
mov x4, 0 // Loop iteration index
// Copy one word of data
// dst[i << 3] = src[i << 3]
ldr tmp, [x1, x3, lsl 3]
str tmp, [x0, x3, lsl 3]
lsl x7, x3, 3
add x7, x7, x0
// Increment the array index by the stride (backwards or forwards).
// i += stride
add x3, x3, x6
// Increment the number of words copied (we use this to decide when to
// stop)
// words_copied += 1
add x4, x4, 1
// If we've copied the whole buffer, then finish.
// if (words_copied == words_to_copy) break;
cmp x2, x4
bne .copy_loop
// Perform a bulk clean/invalidate across the whole cache
// Normally on ARM we can use the CIVAC, CVAC, CVAU and IVAC instructions to
// manipulate the cache but these ops only work against virtual memory addresses
// and since we have disabled the MMU, these instructions are no longer
// meaningful.
// As a result, we have to use the Level/Set/Way cache ops. Since the definition
// of the cache set is left up to the implementation, the only portable (safe)
// way to perform these cache ops is to operate against the whole cache.
// The following op cleans and invalidates every entry in each level of the
// cache.
cache_way_set_op cisw, clean_invalidate
// Invalidate the instruction cache as well
ic iallu
dsb sy
/* This .ltorg emits any immediate constants here. We need to put this before
* the mexec_asm_end symbol because we intend to relocate the assembly contained
* within the mexec_asm[_end] block. Any constants needed by this block should
* also be relocated so we need to ensure that they occur before mexec_asm_end.