use a 32-bit xfreed field
diff --git a/include/mimalloc/alloc/page.h b/include/mimalloc/alloc/page.h index 21c966c..2010585 100644 --- a/include/mimalloc/alloc/page.h +++ b/include/mimalloc/alloc/page.h
@@ -55,7 +55,7 @@ // Page start static inline uint8_t* mi_page_start(const mi_page_t* page) { // multiplication must be done in `size_t` - return (uint8_t*)page + page->page_offset; + return (uint8_t*)page + ((size_t)page->page_woffset * MI_SIZE_SIZE); } static inline size_t mi_page_size(const mi_page_t* page) {
diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index efd2b6f..260bb5b 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h
@@ -448,9 +448,9 @@ } le; // little-endian layout } mi_used_t; -static inline size_t mi_xused_used_count(mi_used_t xused) { return (xused.used_alloc & 0xFFFF); } -static inline size_t mi_xused_alloc_count(mi_used_t xused) { return ((xused.used_alloc>>16) & 0xFFFF); } -static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc = xused.used_alloc & ~0xFFFF; return xused; } +static inline uint16_t mi_xused_used_count(mi_used_t xused) { return (uint16_t)(xused.used_alloc); } +static inline uint16_t mi_xused_alloc_count(mi_used_t xused) { return (uint16_t)(xused.used_alloc>>16); } +static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc = xused.used_alloc & ~0xFFFF; return xused; } // A page contains blocks of one specific size (`block_size`). // Each page has three list of free blocks: @@ -493,8 +493,8 @@ uint16_t xlast_alloc; // for sampling; on 64-bit platforms it is in bits 48..63 of xused. #endif - size_t page_offset; // const: relative offset from the page to the start of the blocks - size_t xfreed; // local frees not yet subtracted from `xused` (so `mi_free` does not update `xused` written by `mi_malloc`) + uint32_t xfreed; // local frees not yet subtracted from `xused` (so `mi_free` does not update `xused` written by `mi_malloc`) (32-bit to avoid store-forwarding stalls from wider loads) + uint32_t page_woffset; // const: offset from the page to the start of the blocks (in `size_t` units, so up to 32 GiB) uint16_t capacity; // number of blocks committed uint16_t reserved; // number of blocks reserved in memory uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already)
diff --git a/src/alloc/free.c b/src/alloc/free.c index ac5050c..b9be2bd 100644 --- a/src/alloc/free.c +++ b/src/alloc/free.c
@@ -59,12 +59,13 @@ // actual free: push on the local free list fast-path // (we count frees in `xfreed` instead of decrementing `xused` to avoid a store-to-load dependency between malloc and free) - const size_t freed = page->xfreed + 1; mi_block_t* lfree = page->local_free; + const uint16_t used = mi_xused_used_count(page->xused); + const size_t freed = (size_t)page->xfreed + 1; + page->xfreed = (uint32_t)freed; mi_block_set_next(page, block, lfree); - page->xfreed = freed; page->local_free = block; - const bool is_empty = (mi_xused_used_count(page->xused) == freed); + const bool is_empty = (used == freed); mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); if mi_unlikely(is_empty) { // is used count zero ? _mi_page_qsbr_clear(page); // a new empty interval must not reuse an earlier grace period
diff --git a/src/arena/arena-page.c b/src/arena/arena-page.c index 8db2624..b90acaf 100644 --- a/src/arena/arena-page.c +++ b/src/arena/arena-page.c
@@ -587,7 +587,9 @@ // initialize the page start uint8_t* const start = slice_start + block_start; mi_assert_internal(start > (uint8_t*)page); - page->page_offset = start - (uint8_t*)page; + const size_t page_offset = (size_t)(start - (uint8_t*)page); + mi_assert_internal((page_offset % MI_SIZE_SIZE) == 0 && (page_offset / MI_SIZE_SIZE) <= UINT32_MAX); + page->page_woffset = (uint32_t)(page_offset / MI_SIZE_SIZE); // initialize page meta-data page->reserved = (uint16_t)reserved;
diff --git a/src/heap/init.c b/src/heap/init.c index 8313481..45f8760 100644 --- a/src/heap/init.c +++ b/src/heap/init.c
@@ -32,8 +32,8 @@ 0, // xlast_used 0, // xlast_alloc #endif - 0, // page_offset 0, // xfreed + 0, // page_woffset 0, // capacity 0, // reserved capacity 0, // slice_pcommitted