kernel/arch/arm64/include/arch/arm64/mmu.h - zircon - Git at Google

 // Copyright 2016 The Fuchsia Authors
 // Copyright (c) 2014 Google Inc. All rights reserved
 //
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file or at
 // https://opensource.org/licenses/MIT


 #pragma once

 #include <arch/defines.h>

 #define IFTE(c,t,e) (!!(c) * (t) | !(c) * (e))
 #define NBITS01(n)      IFTE(n, 1, 0)
 #define NBITS02(n)      IFTE((n) >>  1,  1 + NBITS01((n) >>  1), NBITS01(n))
 #define NBITS04(n)      IFTE((n) >>  2,  2 + NBITS02((n) >>  2), NBITS02(n))
 #define NBITS08(n)      IFTE((n) >>  4,  4 + NBITS04((n) >>  4), NBITS04(n))
 #define NBITS16(n)      IFTE((n) >>  8,  8 + NBITS08((n) >>  8), NBITS08(n))
 #define NBITS32(n)      IFTE((n) >> 16, 16 + NBITS16((n) >> 16), NBITS16(n))
 #define NBITS(n)        IFTE((n) >> 32, 32 + NBITS32((n) >> 32), NBITS32(n))

 #ifndef MMU_KERNEL_SIZE_SHIFT
 #define KERNEL_ASPACE_BITS (NBITS(0xffffffffffffffff-KERNEL_ASPACE_BASE))
 #define KERNEL_BASE_BITS (NBITS(0xffffffffffffffff-KERNEL_BASE))
 #if KERNEL_BASE_BITS > KERNEL_ASPACE_BITS
 #define KERNEL_ASPACE_BITS KERNEL_BASE_BITS /* KERNEL_BASE should not be below KERNEL_ASPACE_BASE */
 #endif

 #if KERNEL_ASPACE_BITS < 25
 #define MMU_KERNEL_SIZE_SHIFT (25)
 #else
 #define MMU_KERNEL_SIZE_SHIFT (KERNEL_ASPACE_BITS)
 #endif
 #endif

 #ifndef MMU_USER_SIZE_SHIFT
 #define MMU_USER_SIZE_SHIFT 48
 #endif

 #ifndef MMU_IDENT_SIZE_SHIFT
 #define MMU_IDENT_SIZE_SHIFT 42 /* Max size supported by block mappings */
 #endif

 #define MMU_MAX_PAGE_SIZE_SHIFT 48

 #define MMU_KERNEL_PAGE_SIZE_SHIFT      (PAGE_SIZE_SHIFT)
 #define MMU_USER_PAGE_SIZE_SHIFT        (USER_PAGE_SIZE_SHIFT)

 #if MMU_IDENT_SIZE_SHIFT < 25
 #error MMU_IDENT_SIZE_SHIFT too small
 #elif MMU_IDENT_SIZE_SHIFT <= 29 /* Use 2MB block mappings (4K page size) */
 #define MMU_IDENT_PAGE_SIZE_SHIFT       (SHIFT_4K)
 #elif MMU_IDENT_SIZE_SHIFT <= 30 /* Use 512MB block mappings (64K page size) */
 #define MMU_IDENT_PAGE_SIZE_SHIFT       (SHIFT_64K)
 #elif MMU_IDENT_SIZE_SHIFT <= 39 /* Use 1GB block mappings (4K page size) */
 #define MMU_IDENT_PAGE_SIZE_SHIFT       (SHIFT_4K)
 #elif MMU_IDENT_SIZE_SHIFT <= 42 /* Use 512MB block mappings (64K page size) */
 #define MMU_IDENT_PAGE_SIZE_SHIFT       (SHIFT_64K)
 #else
 #error MMU_IDENT_SIZE_SHIFT too large
 #endif

 /*
  * TCR TGx values
  *
  * Page size:   4K      16K     64K
  * TG0:         0       2       1
  * TG1:         2       1       3
  */

 #define MMU_TG0(page_size_shift) ((((page_size_shift == 14) & 1) << 1) | \
                                   ((page_size_shift == 16) & 1))

 #define MMU_TG1(page_size_shift) ((((page_size_shift == 12) & 1) << 1) | \
                                   ((page_size_shift == 14) & 1) | \
                                   ((page_size_shift == 16) & 1) | \
                                   (((page_size_shift == 16) & 1) << 1))

 #define MMU_LX_X(page_shift, level) ((4 - (level)) * ((page_shift) - 3) + 3)

 #if MMU_USER_SIZE_SHIFT > MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 0)
 #define MMU_USER_TOP_SHIFT MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 0)
 #elif MMU_USER_SIZE_SHIFT > MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 1)
 #define MMU_USER_TOP_SHIFT MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 1)
 #elif MMU_USER_SIZE_SHIFT > MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 2)
 #define MMU_USER_TOP_SHIFT MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 2)
 #elif MMU_USER_SIZE_SHIFT > MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 3)
 #define MMU_USER_TOP_SHIFT MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 3)
 #else
 #error User address space size must be larger than page size
 #endif
 #define MMU_USER_PAGE_TABLE_ENTRIES_TOP (0x1 << (MMU_USER_SIZE_SHIFT - MMU_USER_TOP_SHIFT))

 #if MMU_KERNEL_SIZE_SHIFT > MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 0)
 #define MMU_KERNEL_TOP_SHIFT MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 0)
 #elif MMU_KERNEL_SIZE_SHIFT > MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 1)
 #define MMU_KERNEL_TOP_SHIFT MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 1)
 #elif MMU_KERNEL_SIZE_SHIFT > MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 2)
 #define MMU_KERNEL_TOP_SHIFT MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 2)
 #elif MMU_KERNEL_SIZE_SHIFT > MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 3)
 #define MMU_KERNEL_TOP_SHIFT MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 3)
 #else
 #error Kernel address space size must be larger than page size
 #endif
 #define MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP (0x1 << (MMU_KERNEL_SIZE_SHIFT - MMU_KERNEL_TOP_SHIFT))

 #if MMU_IDENT_SIZE_SHIFT > MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 0)
 #define MMU_IDENT_TOP_SHIFT MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 0)
 #elif MMU_IDENT_SIZE_SHIFT > MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 1)
 #define MMU_IDENT_TOP_SHIFT MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 1)
 #elif MMU_IDENT_SIZE_SHIFT > MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 2)
 #define MMU_IDENT_TOP_SHIFT MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 2)
 #elif MMU_IDENT_SIZE_SHIFT > MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 3)
 #define MMU_IDENT_TOP_SHIFT MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 3)
 #else
 #error Ident address space size must be larger than page size
 #endif
 #define MMU_PAGE_TABLE_ENTRIES_IDENT_SHIFT (MMU_IDENT_SIZE_SHIFT - MMU_IDENT_TOP_SHIFT)
 #define MMU_PAGE_TABLE_ENTRIES_IDENT (0x1 << MMU_PAGE_TABLE_ENTRIES_IDENT_SHIFT)

 #define MMU_PTE_DESCRIPTOR_BLOCK_MAX_SHIFT      (30)

 #ifndef ASSEMBLY
 #define BM(base, count, val) (((val) & ((1UL << (count)) - 1)) << (base))
 #else
 #define BM(base, count, val) (((val) & ((0x1 << (count)) - 1)) << (base))
 #endif

 #define MMU_SH_NON_SHAREABLE                    (0)
 #define MMU_SH_OUTER_SHAREABLE                  (2)
 #define MMU_SH_INNER_SHAREABLE                  (3)

 #define MMU_RGN_NON_CACHEABLE                   (0)
 #define MMU_RGN_WRITE_BACK_ALLOCATE             (1)
 #define MMU_RGN_WRITE_THROUGH_NO_ALLOCATE       (2)
 #define MMU_RGN_WRITE_BACK_NO_ALLOCATE          (3)

 #define MMU_TCR_TBI1                            BM(38, 1, 1)
 #define MMU_TCR_TBI0                            BM(37, 1, 1)
 #define MMU_TCR_AS                              BM(36, 1, 1)
 #define MMU_TCR_IPS(size)                       BM(32, 3, (size))
 #define MMU_TCR_TG1(granule_size)               BM(30, 2, (granule_size))
 #define MMU_TCR_SH1(shareability_flags)         BM(28, 2, (shareability_flags))
 #define MMU_TCR_ORGN1(cache_flags)              BM(26, 2, (cache_flags))
 #define MMU_TCR_IRGN1(cache_flags)              BM(24, 2, (cache_flags))
 #define MMU_TCR_EPD1                            BM(23, 1, 1)
 #define MMU_TCR_A1                              BM(22, 1, 1)
 #define MMU_TCR_T1SZ(size)                      BM(16, 6, (size))
 #define MMU_TCR_TG0(granule_size)               BM(14, 2, (granule_size))
 #define MMU_TCR_SH0(shareability_flags)         BM(12, 2, (shareability_flags))
 #define MMU_TCR_ORGN0(cache_flags)              BM(10, 2, (cache_flags))
 #define MMU_TCR_IRGN0(cache_flags)              BM( 8, 2, (cache_flags))
 #define MMU_TCR_EPD0                            BM( 7, 1, 1)
 #define MMU_TCR_T0SZ(size)                      BM( 0, 6, (size))

 #define MMU_MAIR_ATTR(index, attr)              BM(index * 8, 8, (attr))


 /* L0/L1/L2/L3 descriptor types */
 #define MMU_PTE_DESCRIPTOR_INVALID              BM(0, 2, 0)
 #define MMU_PTE_DESCRIPTOR_MASK                 BM(0, 2, 3)

 /* L0/L1/L2 descriptor types */
 #define MMU_PTE_L012_DESCRIPTOR_BLOCK           BM(0, 2, 1)
 #define MMU_PTE_L012_DESCRIPTOR_TABLE           BM(0, 2, 3)

 /* L3 descriptor types */
 #define MMU_PTE_L3_DESCRIPTOR_PAGE              BM(0, 2, 3)

 /* Output address mask */
 #define MMU_PTE_OUTPUT_ADDR_MASK                BM(12, 36, 0xfffffffff)

 /* Table attrs */
 #define MMU_PTE_ATTR_NS_TABLE                   BM(63, 1, 1)
 #define MMU_PTE_ATTR_AP_TABLE_NO_WRITE          BM(62, 1, 1)
 #define MMU_PTE_ATTR_AP_TABLE_NO_EL0            BM(61, 1, 1)
 #define MMU_PTE_ATTR_UXN_TABLE                  BM(60, 1, 1)
 #define MMU_PTE_ATTR_PXN_TABLE                  BM(59, 1, 1)

 /* Block/Page attrs */
 #define MMU_PTE_ATTR_RES_SOFTWARE               BM(55, 4, 0xf)
 #define MMU_PTE_ATTR_UXN                        BM(54, 1, 1)
 #define MMU_PTE_ATTR_PXN                        BM(53, 1, 1)
 #define MMU_PTE_ATTR_CONTIGUOUS                 BM(52, 1, 1)

 #define MMU_PTE_ATTR_NON_GLOBAL                 BM(11, 1, 1)
 #define MMU_PTE_ATTR_AF                         BM(10, 1, 1)

 #define MMU_PTE_ATTR_SH_NON_SHAREABLE           BM(8, 2, 0)
 #define MMU_PTE_ATTR_SH_OUTER_SHAREABLE         BM(8, 2, 2)
 #define MMU_PTE_ATTR_SH_INNER_SHAREABLE         BM(8, 2, 3)

 #define MMU_PTE_ATTR_AP_P_RW_U_NA               BM(6, 2, 0)
 #define MMU_PTE_ATTR_AP_P_RW_U_RW               BM(6, 2, 1)
 #define MMU_PTE_ATTR_AP_P_RO_U_NA               BM(6, 2, 2)
 #define MMU_PTE_ATTR_AP_P_RO_U_RO               BM(6, 2, 3)
 #define MMU_PTE_ATTR_AP_MASK                    BM(6, 2, 3)

 #define MMU_PTE_ATTR_NON_SECURE                 BM(5, 1, 1)

 #define MMU_PTE_ATTR_ATTR_INDEX(attrindex)      BM(2, 3, attrindex)
 #define MMU_PTE_ATTR_ATTR_INDEX_MASK            MMU_PTE_ATTR_ATTR_INDEX(7)

 #define MMU_PTE_PERMISSION_MASK (MMU_PTE_ATTR_AP_MASK | \
                                  MMU_PTE_ATTR_UXN | \
                                  MMU_PTE_ATTR_PXN)

 /* Default configuration for main kernel page table:
  *    - do cached translation walks
  */

 /* Device-nGnRnE memory */
 #define MMU_MAIR_ATTR0                  MMU_MAIR_ATTR(0, 0x00)
 #define MMU_PTE_ATTR_STRONGLY_ORDERED   MMU_PTE_ATTR_ATTR_INDEX(0)

 /* Device-nGnRE memory */
 #define MMU_MAIR_ATTR1                  MMU_MAIR_ATTR(1, 0x04)
 #define MMU_PTE_ATTR_DEVICE             MMU_PTE_ATTR_ATTR_INDEX(1)

 /* Normal Memory, Outer Write-back non-transient Read/Write allocate,
  * Inner Write-back non-transient Read/Write allocate
  */
 #define MMU_MAIR_ATTR2                  MMU_MAIR_ATTR(2, 0xff)
 #define MMU_PTE_ATTR_NORMAL_MEMORY      MMU_PTE_ATTR_ATTR_INDEX(2)

 #define MMU_MAIR_ATTR3                  (0)
 #define MMU_MAIR_ATTR4                  (0)
 #define MMU_MAIR_ATTR5                  (0)
 #define MMU_MAIR_ATTR6                  (0)
 #define MMU_MAIR_ATTR7                  (0)

 #define MMU_MAIR_VAL                    (MMU_MAIR_ATTR0 | MMU_MAIR_ATTR1 | \
                                          MMU_MAIR_ATTR2 | MMU_MAIR_ATTR3 | \
                                          MMU_MAIR_ATTR4 | MMU_MAIR_ATTR5 | \
                                          MMU_MAIR_ATTR6 | MMU_MAIR_ATTR7 )

 #define MMU_TCR_IPS_DEFAULT MMU_TCR_IPS(2) /* TODO: read at runtime, or configure per platform */

 /* Enable cached page table walks:
  * inner/outer (IRGN/ORGN): write-back + write-allocate
  */
 #define MMU_TCR_FLAGS1 (MMU_TCR_TG1(MMU_TG1(MMU_KERNEL_PAGE_SIZE_SHIFT)) | \
                         MMU_TCR_SH1(MMU_SH_INNER_SHAREABLE) | \
                         MMU_TCR_ORGN1(MMU_RGN_WRITE_BACK_ALLOCATE) | \
                         MMU_TCR_IRGN1(MMU_RGN_WRITE_BACK_ALLOCATE) | \
                         MMU_TCR_T1SZ(64 - MMU_KERNEL_SIZE_SHIFT))
 #define MMU_TCR_FLAGS0 (MMU_TCR_TG0(MMU_TG0(MMU_USER_PAGE_SIZE_SHIFT)) | \
                         MMU_TCR_SH0(MMU_SH_INNER_SHAREABLE) | \
                         MMU_TCR_ORGN0(MMU_RGN_WRITE_BACK_ALLOCATE) | \
                         MMU_TCR_IRGN0(MMU_RGN_WRITE_BACK_ALLOCATE) | \
                         MMU_TCR_T0SZ(64 - MMU_USER_SIZE_SHIFT))
 #define MMU_TCR_FLAGS0_IDENT \
                        (MMU_TCR_TG0(MMU_TG0(MMU_IDENT_PAGE_SIZE_SHIFT)) | \
                         MMU_TCR_SH0(MMU_SH_INNER_SHAREABLE) | \
                         MMU_TCR_ORGN0(MMU_RGN_WRITE_BACK_ALLOCATE) | \
                         MMU_TCR_IRGN0(MMU_RGN_WRITE_BACK_ALLOCATE) | \
                         MMU_TCR_T0SZ(64 - MMU_IDENT_SIZE_SHIFT))
 #define MMU_TCR_FLAGS_IDENT (MMU_TCR_IPS_DEFAULT | MMU_TCR_FLAGS1 | MMU_TCR_FLAGS0_IDENT)

 #define MMU_TCR_FLAGS_KERNEL (MMU_TCR_IPS_DEFAULT | \
                               MMU_TCR_FLAGS1 | \
                               MMU_TCR_FLAGS0 | \
                               MMU_TCR_EPD0 | \
                               MMU_TCR_AS | \
                               MMU_TCR_A1)

 #define MMU_TCR_FLAGS_USER (MMU_TCR_IPS_DEFAULT | \
                             MMU_TCR_FLAGS1 | \
                             MMU_TCR_FLAGS0 | \
                             MMU_TCR_AS)


 #if MMU_IDENT_SIZE_SHIFT > MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 2)
 #define MMU_PTE_IDENT_DESCRIPTOR MMU_PTE_L012_DESCRIPTOR_BLOCK
 #else
 #define MMU_PTE_IDENT_DESCRIPTOR MMU_PTE_L3_DESCRIPTOR_PAGE
 #endif
 #define MMU_PTE_IDENT_FLAGS \
     (MMU_PTE_IDENT_DESCRIPTOR | \
      MMU_PTE_ATTR_AF | \
      MMU_PTE_ATTR_SH_INNER_SHAREABLE | \
      MMU_PTE_ATTR_NORMAL_MEMORY | \
      MMU_PTE_ATTR_AP_P_RW_U_NA)

 #define MMU_PTE_KERNEL_RO_FLAGS \
     (MMU_PTE_ATTR_UXN | \
      MMU_PTE_ATTR_AF | \
      MMU_PTE_ATTR_SH_INNER_SHAREABLE | \
      MMU_PTE_ATTR_NORMAL_MEMORY | \
      MMU_PTE_ATTR_AP_P_RO_U_NA)

 #define MMU_PTE_KERNEL_DATA_FLAGS \
     (MMU_PTE_ATTR_UXN | \
      MMU_PTE_ATTR_PXN | \
      MMU_PTE_ATTR_AF | \
      MMU_PTE_ATTR_SH_INNER_SHAREABLE | \
      MMU_PTE_ATTR_NORMAL_MEMORY | \
      MMU_PTE_ATTR_AP_P_RW_U_NA)

 #define MMU_INITIAL_MAP_STRONGLY_ORDERED \
     (MMU_PTE_ATTR_UXN | \
      MMU_PTE_ATTR_PXN | \
      MMU_PTE_ATTR_AF | \
      MMU_PTE_ATTR_STRONGLY_ORDERED | \
      MMU_PTE_ATTR_AP_P_RW_U_NA)

 #define MMU_INITIAL_MAP_DEVICE \
     (MMU_PTE_ATTR_UXN | \
      MMU_PTE_ATTR_PXN | \
      MMU_PTE_ATTR_AF | \
      MMU_PTE_ATTR_DEVICE | \
      MMU_PTE_ATTR_AP_P_RW_U_NA)

 #ifndef ASSEMBLY

 #include <sys/types.h>
 #include <assert.h>
 #include <zircon/compiler.h>
 #include <arch/arm64.h>

 typedef uint64_t pte_t;

 __BEGIN_CDECLS

 #define ARM64_TLBI_NOADDR(op) \
 ({ \
     __asm__ volatile("tlbi " #op::); \
     ISB; \
 })

 #define ARM64_TLBI(op, val) \
 ({ \
     __asm__ volatile("tlbi " #op ", %0" :: "r" ((uint64_t)(val))); \
     ISB; \
 })

 #define MMU_ARM64_GLOBAL_ASID (~0U)
 #define MMU_ARM64_USER_ASID (0U)

 #define MMU_ARM64_ASID_BITS     (16U)


 __END_CDECLS
 #endif /* ASSEMBLY */
	// Copyright 2016 The Fuchsia Authors
	// Copyright (c) 2014 Google Inc. All rights reserved
	//
	// Use of this source code is governed by a MIT-style
	// license that can be found in the LICENSE file or at
	// https://opensource.org/licenses/MIT


	#pragma once

	#include <arch/defines.h>

	#define IFTE(c,t,e) (!!(c) * (t) \| !(c) * (e))
	#define NBITS01(n) IFTE(n, 1, 0)
	#define NBITS02(n) IFTE((n) >> 1, 1 + NBITS01((n) >> 1), NBITS01(n))
	#define NBITS04(n) IFTE((n) >> 2, 2 + NBITS02((n) >> 2), NBITS02(n))
	#define NBITS08(n) IFTE((n) >> 4, 4 + NBITS04((n) >> 4), NBITS04(n))
	#define NBITS16(n) IFTE((n) >> 8, 8 + NBITS08((n) >> 8), NBITS08(n))
	#define NBITS32(n) IFTE((n) >> 16, 16 + NBITS16((n) >> 16), NBITS16(n))
	#define NBITS(n) IFTE((n) >> 32, 32 + NBITS32((n) >> 32), NBITS32(n))

	#ifndef MMU_KERNEL_SIZE_SHIFT
	#define KERNEL_ASPACE_BITS (NBITS(0xffffffffffffffff-KERNEL_ASPACE_BASE))
	#define KERNEL_BASE_BITS (NBITS(0xffffffffffffffff-KERNEL_BASE))
	#if KERNEL_BASE_BITS > KERNEL_ASPACE_BITS
	#define KERNEL_ASPACE_BITS KERNEL_BASE_BITS /* KERNEL_BASE should not be below KERNEL_ASPACE_BASE */
	#endif

	#if KERNEL_ASPACE_BITS < 25
	#define MMU_KERNEL_SIZE_SHIFT (25)
	#else
	#define MMU_KERNEL_SIZE_SHIFT (KERNEL_ASPACE_BITS)
	#endif
	#endif

	#ifndef MMU_USER_SIZE_SHIFT
	#define MMU_USER_SIZE_SHIFT 48
	#endif

	#ifndef MMU_IDENT_SIZE_SHIFT
	#define MMU_IDENT_SIZE_SHIFT 42 /* Max size supported by block mappings */
	#endif

	#define MMU_MAX_PAGE_SIZE_SHIFT 48

	#define MMU_KERNEL_PAGE_SIZE_SHIFT (PAGE_SIZE_SHIFT)
	#define MMU_USER_PAGE_SIZE_SHIFT (USER_PAGE_SIZE_SHIFT)

	#if MMU_IDENT_SIZE_SHIFT < 25
	#error MMU_IDENT_SIZE_SHIFT too small
	#elif MMU_IDENT_SIZE_SHIFT <= 29 /* Use 2MB block mappings (4K page size) */
	#define MMU_IDENT_PAGE_SIZE_SHIFT (SHIFT_4K)
	#elif MMU_IDENT_SIZE_SHIFT <= 30 /* Use 512MB block mappings (64K page size) */
	#define MMU_IDENT_PAGE_SIZE_SHIFT (SHIFT_64K)
	#elif MMU_IDENT_SIZE_SHIFT <= 39 /* Use 1GB block mappings (4K page size) */
	#define MMU_IDENT_PAGE_SIZE_SHIFT (SHIFT_4K)
	#elif MMU_IDENT_SIZE_SHIFT <= 42 /* Use 512MB block mappings (64K page size) */
	#define MMU_IDENT_PAGE_SIZE_SHIFT (SHIFT_64K)
	#else
	#error MMU_IDENT_SIZE_SHIFT too large
	#endif

	/*
	* TCR TGx values
	*
	* Page size: 4K 16K 64K
	* TG0: 0 2 1
	* TG1: 2 1 3
	*/

	#define MMU_TG0(page_size_shift) ((((page_size_shift == 14) & 1) << 1) \| \
	((page_size_shift == 16) & 1))

	#define MMU_TG1(page_size_shift) ((((page_size_shift == 12) & 1) << 1) \| \
	((page_size_shift == 14) & 1) \| \
	((page_size_shift == 16) & 1) \| \
	(((page_size_shift == 16) & 1) << 1))

	#define MMU_LX_X(page_shift, level) ((4 - (level)) * ((page_shift) - 3) + 3)

	#if MMU_USER_SIZE_SHIFT > MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 0)
	#define MMU_USER_TOP_SHIFT MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 0)
	#elif MMU_USER_SIZE_SHIFT > MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 1)
	#define MMU_USER_TOP_SHIFT MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 1)
	#elif MMU_USER_SIZE_SHIFT > MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 2)
	#define MMU_USER_TOP_SHIFT MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 2)
	#elif MMU_USER_SIZE_SHIFT > MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 3)
	#define MMU_USER_TOP_SHIFT MMU_LX_X(MMU_USER_PAGE_SIZE_SHIFT, 3)
	#else
	#error User address space size must be larger than page size
	#endif
	#define MMU_USER_PAGE_TABLE_ENTRIES_TOP (0x1 << (MMU_USER_SIZE_SHIFT - MMU_USER_TOP_SHIFT))

	#if MMU_KERNEL_SIZE_SHIFT > MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 0)
	#define MMU_KERNEL_TOP_SHIFT MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 0)
	#elif MMU_KERNEL_SIZE_SHIFT > MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 1)
	#define MMU_KERNEL_TOP_SHIFT MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 1)
	#elif MMU_KERNEL_SIZE_SHIFT > MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 2)
	#define MMU_KERNEL_TOP_SHIFT MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 2)
	#elif MMU_KERNEL_SIZE_SHIFT > MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 3)
	#define MMU_KERNEL_TOP_SHIFT MMU_LX_X(MMU_KERNEL_PAGE_SIZE_SHIFT, 3)
	#else
	#error Kernel address space size must be larger than page size
	#endif
	#define MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP (0x1 << (MMU_KERNEL_SIZE_SHIFT - MMU_KERNEL_TOP_SHIFT))

	#if MMU_IDENT_SIZE_SHIFT > MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 0)
	#define MMU_IDENT_TOP_SHIFT MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 0)
	#elif MMU_IDENT_SIZE_SHIFT > MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 1)
	#define MMU_IDENT_TOP_SHIFT MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 1)
	#elif MMU_IDENT_SIZE_SHIFT > MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 2)
	#define MMU_IDENT_TOP_SHIFT MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 2)
	#elif MMU_IDENT_SIZE_SHIFT > MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 3)
	#define MMU_IDENT_TOP_SHIFT MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 3)
	#else
	#error Ident address space size must be larger than page size
	#endif
	#define MMU_PAGE_TABLE_ENTRIES_IDENT_SHIFT (MMU_IDENT_SIZE_SHIFT - MMU_IDENT_TOP_SHIFT)
	#define MMU_PAGE_TABLE_ENTRIES_IDENT (0x1 << MMU_PAGE_TABLE_ENTRIES_IDENT_SHIFT)

	#define MMU_PTE_DESCRIPTOR_BLOCK_MAX_SHIFT (30)

	#ifndef ASSEMBLY
	#define BM(base, count, val) (((val) & ((1UL << (count)) - 1)) << (base))
	#else
	#define BM(base, count, val) (((val) & ((0x1 << (count)) - 1)) << (base))
	#endif

	#define MMU_SH_NON_SHAREABLE (0)
	#define MMU_SH_OUTER_SHAREABLE (2)
	#define MMU_SH_INNER_SHAREABLE (3)

	#define MMU_RGN_NON_CACHEABLE (0)
	#define MMU_RGN_WRITE_BACK_ALLOCATE (1)
	#define MMU_RGN_WRITE_THROUGH_NO_ALLOCATE (2)
	#define MMU_RGN_WRITE_BACK_NO_ALLOCATE (3)

	#define MMU_TCR_TBI1 BM(38, 1, 1)
	#define MMU_TCR_TBI0 BM(37, 1, 1)
	#define MMU_TCR_AS BM(36, 1, 1)
	#define MMU_TCR_IPS(size) BM(32, 3, (size))
	#define MMU_TCR_TG1(granule_size) BM(30, 2, (granule_size))
	#define MMU_TCR_SH1(shareability_flags) BM(28, 2, (shareability_flags))
	#define MMU_TCR_ORGN1(cache_flags) BM(26, 2, (cache_flags))
	#define MMU_TCR_IRGN1(cache_flags) BM(24, 2, (cache_flags))
	#define MMU_TCR_EPD1 BM(23, 1, 1)
	#define MMU_TCR_A1 BM(22, 1, 1)
	#define MMU_TCR_T1SZ(size) BM(16, 6, (size))
	#define MMU_TCR_TG0(granule_size) BM(14, 2, (granule_size))
	#define MMU_TCR_SH0(shareability_flags) BM(12, 2, (shareability_flags))
	#define MMU_TCR_ORGN0(cache_flags) BM(10, 2, (cache_flags))
	#define MMU_TCR_IRGN0(cache_flags) BM( 8, 2, (cache_flags))
	#define MMU_TCR_EPD0 BM( 7, 1, 1)
	#define MMU_TCR_T0SZ(size) BM( 0, 6, (size))

	#define MMU_MAIR_ATTR(index, attr) BM(index * 8, 8, (attr))


	/* L0/L1/L2/L3 descriptor types */
	#define MMU_PTE_DESCRIPTOR_INVALID BM(0, 2, 0)
	#define MMU_PTE_DESCRIPTOR_MASK BM(0, 2, 3)

	/* L0/L1/L2 descriptor types */
	#define MMU_PTE_L012_DESCRIPTOR_BLOCK BM(0, 2, 1)
	#define MMU_PTE_L012_DESCRIPTOR_TABLE BM(0, 2, 3)

	/* L3 descriptor types */
	#define MMU_PTE_L3_DESCRIPTOR_PAGE BM(0, 2, 3)

	/* Output address mask */
	#define MMU_PTE_OUTPUT_ADDR_MASK BM(12, 36, 0xfffffffff)

	/* Table attrs */
	#define MMU_PTE_ATTR_NS_TABLE BM(63, 1, 1)
	#define MMU_PTE_ATTR_AP_TABLE_NO_WRITE BM(62, 1, 1)
	#define MMU_PTE_ATTR_AP_TABLE_NO_EL0 BM(61, 1, 1)
	#define MMU_PTE_ATTR_UXN_TABLE BM(60, 1, 1)
	#define MMU_PTE_ATTR_PXN_TABLE BM(59, 1, 1)

	/* Block/Page attrs */
	#define MMU_PTE_ATTR_RES_SOFTWARE BM(55, 4, 0xf)
	#define MMU_PTE_ATTR_UXN BM(54, 1, 1)
	#define MMU_PTE_ATTR_PXN BM(53, 1, 1)
	#define MMU_PTE_ATTR_CONTIGUOUS BM(52, 1, 1)

	#define MMU_PTE_ATTR_NON_GLOBAL BM(11, 1, 1)
	#define MMU_PTE_ATTR_AF BM(10, 1, 1)

	#define MMU_PTE_ATTR_SH_NON_SHAREABLE BM(8, 2, 0)
	#define MMU_PTE_ATTR_SH_OUTER_SHAREABLE BM(8, 2, 2)
	#define MMU_PTE_ATTR_SH_INNER_SHAREABLE BM(8, 2, 3)

	#define MMU_PTE_ATTR_AP_P_RW_U_NA BM(6, 2, 0)
	#define MMU_PTE_ATTR_AP_P_RW_U_RW BM(6, 2, 1)
	#define MMU_PTE_ATTR_AP_P_RO_U_NA BM(6, 2, 2)
	#define MMU_PTE_ATTR_AP_P_RO_U_RO BM(6, 2, 3)
	#define MMU_PTE_ATTR_AP_MASK BM(6, 2, 3)

	#define MMU_PTE_ATTR_NON_SECURE BM(5, 1, 1)

	#define MMU_PTE_ATTR_ATTR_INDEX(attrindex) BM(2, 3, attrindex)
	#define MMU_PTE_ATTR_ATTR_INDEX_MASK MMU_PTE_ATTR_ATTR_INDEX(7)

	#define MMU_PTE_PERMISSION_MASK (MMU_PTE_ATTR_AP_MASK \| \
	MMU_PTE_ATTR_UXN \| \
	MMU_PTE_ATTR_PXN)

	/* Default configuration for main kernel page table:
	* - do cached translation walks
	*/

	/* Device-nGnRnE memory */
	#define MMU_MAIR_ATTR0 MMU_MAIR_ATTR(0, 0x00)
	#define MMU_PTE_ATTR_STRONGLY_ORDERED MMU_PTE_ATTR_ATTR_INDEX(0)

	/* Device-nGnRE memory */
	#define MMU_MAIR_ATTR1 MMU_MAIR_ATTR(1, 0x04)
	#define MMU_PTE_ATTR_DEVICE MMU_PTE_ATTR_ATTR_INDEX(1)

	/* Normal Memory, Outer Write-back non-transient Read/Write allocate,
	* Inner Write-back non-transient Read/Write allocate
	*/
	#define MMU_MAIR_ATTR2 MMU_MAIR_ATTR(2, 0xff)
	#define MMU_PTE_ATTR_NORMAL_MEMORY MMU_PTE_ATTR_ATTR_INDEX(2)

	#define MMU_MAIR_ATTR3 (0)
	#define MMU_MAIR_ATTR4 (0)
	#define MMU_MAIR_ATTR5 (0)
	#define MMU_MAIR_ATTR6 (0)
	#define MMU_MAIR_ATTR7 (0)

	#define MMU_MAIR_VAL (MMU_MAIR_ATTR0 \| MMU_MAIR_ATTR1 \| \
	MMU_MAIR_ATTR2 \| MMU_MAIR_ATTR3 \| \
	MMU_MAIR_ATTR4 \| MMU_MAIR_ATTR5 \| \
	MMU_MAIR_ATTR6 \| MMU_MAIR_ATTR7 )

	#define MMU_TCR_IPS_DEFAULT MMU_TCR_IPS(2) /* TODO: read at runtime, or configure per platform */

	/* Enable cached page table walks:
	* inner/outer (IRGN/ORGN): write-back + write-allocate
	*/
	#define MMU_TCR_FLAGS1 (MMU_TCR_TG1(MMU_TG1(MMU_KERNEL_PAGE_SIZE_SHIFT)) \| \
	MMU_TCR_SH1(MMU_SH_INNER_SHAREABLE) \| \
	MMU_TCR_ORGN1(MMU_RGN_WRITE_BACK_ALLOCATE) \| \
	MMU_TCR_IRGN1(MMU_RGN_WRITE_BACK_ALLOCATE) \| \
	MMU_TCR_T1SZ(64 - MMU_KERNEL_SIZE_SHIFT))
	#define MMU_TCR_FLAGS0 (MMU_TCR_TG0(MMU_TG0(MMU_USER_PAGE_SIZE_SHIFT)) \| \
	MMU_TCR_SH0(MMU_SH_INNER_SHAREABLE) \| \
	MMU_TCR_ORGN0(MMU_RGN_WRITE_BACK_ALLOCATE) \| \
	MMU_TCR_IRGN0(MMU_RGN_WRITE_BACK_ALLOCATE) \| \
	MMU_TCR_T0SZ(64 - MMU_USER_SIZE_SHIFT))
	#define MMU_TCR_FLAGS0_IDENT \
	(MMU_TCR_TG0(MMU_TG0(MMU_IDENT_PAGE_SIZE_SHIFT)) \| \
	MMU_TCR_SH0(MMU_SH_INNER_SHAREABLE) \| \
	MMU_TCR_ORGN0(MMU_RGN_WRITE_BACK_ALLOCATE) \| \
	MMU_TCR_IRGN0(MMU_RGN_WRITE_BACK_ALLOCATE) \| \
	MMU_TCR_T0SZ(64 - MMU_IDENT_SIZE_SHIFT))
	#define MMU_TCR_FLAGS_IDENT (MMU_TCR_IPS_DEFAULT \| MMU_TCR_FLAGS1 \| MMU_TCR_FLAGS0_IDENT)

	#define MMU_TCR_FLAGS_KERNEL (MMU_TCR_IPS_DEFAULT \| \
	MMU_TCR_FLAGS1 \| \
	MMU_TCR_FLAGS0 \| \
	MMU_TCR_EPD0 \| \
	MMU_TCR_AS \| \
	MMU_TCR_A1)

	#define MMU_TCR_FLAGS_USER (MMU_TCR_IPS_DEFAULT \| \
	MMU_TCR_FLAGS1 \| \
	MMU_TCR_FLAGS0 \| \
	MMU_TCR_AS)


	#if MMU_IDENT_SIZE_SHIFT > MMU_LX_X(MMU_IDENT_PAGE_SIZE_SHIFT, 2)
	#define MMU_PTE_IDENT_DESCRIPTOR MMU_PTE_L012_DESCRIPTOR_BLOCK
	#else
	#define MMU_PTE_IDENT_DESCRIPTOR MMU_PTE_L3_DESCRIPTOR_PAGE
	#endif
	#define MMU_PTE_IDENT_FLAGS \
	(MMU_PTE_IDENT_DESCRIPTOR \| \
	MMU_PTE_ATTR_AF \| \
	MMU_PTE_ATTR_SH_INNER_SHAREABLE \| \
	MMU_PTE_ATTR_NORMAL_MEMORY \| \
	MMU_PTE_ATTR_AP_P_RW_U_NA)

	#define MMU_PTE_KERNEL_RO_FLAGS \
	(MMU_PTE_ATTR_UXN \| \
	MMU_PTE_ATTR_AF \| \
	MMU_PTE_ATTR_SH_INNER_SHAREABLE \| \
	MMU_PTE_ATTR_NORMAL_MEMORY \| \
	MMU_PTE_ATTR_AP_P_RO_U_NA)

	#define MMU_PTE_KERNEL_DATA_FLAGS \
	(MMU_PTE_ATTR_UXN \| \
	MMU_PTE_ATTR_PXN \| \
	MMU_PTE_ATTR_AF \| \
	MMU_PTE_ATTR_SH_INNER_SHAREABLE \| \
	MMU_PTE_ATTR_NORMAL_MEMORY \| \
	MMU_PTE_ATTR_AP_P_RW_U_NA)

	#define MMU_INITIAL_MAP_STRONGLY_ORDERED \
	(MMU_PTE_ATTR_UXN \| \
	MMU_PTE_ATTR_PXN \| \
	MMU_PTE_ATTR_AF \| \
	MMU_PTE_ATTR_STRONGLY_ORDERED \| \
	MMU_PTE_ATTR_AP_P_RW_U_NA)

	#define MMU_INITIAL_MAP_DEVICE \
	(MMU_PTE_ATTR_UXN \| \
	MMU_PTE_ATTR_PXN \| \
	MMU_PTE_ATTR_AF \| \
	MMU_PTE_ATTR_DEVICE \| \
	MMU_PTE_ATTR_AP_P_RW_U_NA)

	#ifndef ASSEMBLY

	#include <sys/types.h>
	#include <assert.h>
	#include <zircon/compiler.h>
	#include <arch/arm64.h>

	typedef uint64_t pte_t;

	__BEGIN_CDECLS

	#define ARM64_TLBI_NOADDR(op) \
	({ \
	__asm__ volatile("tlbi " #op::); \
	ISB; \
	})

	#define ARM64_TLBI(op, val) \
	({ \
	__asm__ volatile("tlbi " #op ", %0" :: "r" ((uint64_t)(val))); \
	ISB; \
	})

	#define MMU_ARM64_GLOBAL_ASID (~0U)
	#define MMU_ARM64_USER_ASID (0U)

	#define MMU_ARM64_ASID_BITS (16U)


	__END_CDECLS
	#endif /* ASSEMBLY */