free batching never creates a theap for, or reads, a non-main heap on a cache missĀ
diff --git a/src/alloc/free.c b/src/alloc/free.c index 57ac420..f790eb5 100644 --- a/src/alloc/free.c +++ b/src/alloc/free.c
@@ -899,27 +899,22 @@ } } +// Called if the cached theap does not belong to the heap of the page. +// A free never creates a theap for (nor dereferences) a non-main heap: a thread that did not +// allocate in it may free concurrently with a `mi_heap_delete` (see `mi_thread_heap_release`). static mi_decl_noinline void mi_free_block_mt_batch_init(mi_page_t* page, mi_block_t* block, mi_heap_t* heap, size_t block_size) { // check first as meta-data pages are also freed during thread teardown - if mi_unlikely(mi_page_thread_id(page) == MI_THREAD_ID_DETACHED || mi_heap_qsbr_is_enabled(heap)) { + if mi_unlikely(mi_page_thread_id(page) == MI_THREAD_ID_DETACHED) { mi_free_block_mt_publish(page, block, block, true); return; } - mi_theap_t* main = _mi_theap_main(); - if mi_unlikely(!mi_theap_is_initialized(main) && !_mi_thread_is_done()) { + mi_theap_t* theap = _mi_theap_main(); + if mi_unlikely(!mi_theap_is_initialized(theap) && !_mi_thread_is_done() && mi_is_process_heap_main(heap)) { mi_thread_init(); // initialize free-only threads so they can batch as well (but don't re-initialize after `mi_thread_done`) - main = _mi_theap_main(); + theap = _mi_theap_main(); } - mi_tld_t* const tld = (mi_theap_is_initialized(main) ? main->tld : NULL); - if mi_unlikely(tld == NULL || tld->free_batch_disabled || tld->subproc != heap->subproc) { - mi_free_block_mt_publish(page, block, block, true); - return; - } - - tld->free_batch_disabled = true; - mi_theap_t* const theap = _mi_heap_theap_get_or_init(heap); - tld->free_batch_disabled = false; - if mi_unlikely(!mi_theap_is_initialized(theap) || theap->tld != tld || mi_theap_heap_peek(theap) != heap) { + if mi_unlikely(!mi_theap_is_initialized(theap) || mi_theap_heap_peek(theap) != heap || + theap->tld == NULL || theap->tld->free_batch_disabled) { mi_free_block_mt_publish(page, block, block, true); return; } @@ -933,7 +928,7 @@ return; } mi_heap_t* const heap = mi_page_heap(page); - mi_theap_t* const theap = _mi_heap_theap_cached(heap); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); // only compares the heap pointer (as the heap may be deleted concurrently) if mi_unlikely(theap == NULL) { mi_free_block_mt_batch_init(page, block, heap, block_size); return;
diff --git a/test/test-free-batch.c b/test/test-free-batch.c index d100a1f..f2973a4 100644 --- a/test/test-free-batch.c +++ b/test/test-free-batch.c
@@ -150,7 +150,10 @@ trace_phase("worker heap lookup"); mi_heap_t* const heap = (disposable == NULL ? mi_heap_main() : disposable); mi_theap_t* theap = _mi_heap_theap_peek(heap); - if (disposable != NULL && theap != NULL) return false; + if (disposable != NULL) { + // frees only batch into a non-main heap if this thread uses it (and thus has a theap for it) + if (theap != NULL || mi_heap_theap(disposable) == NULL) return false; + } bool pending = false; trace_phase("worker free loop"); for (size_t i = 0; i < block_count; i++) { @@ -260,13 +263,21 @@ static void* separate_blocks[2][33]; static mi_theap_t* consumer_theap; +// use the heap (so this thread has a theap for it) and free a block of it +static void free_separate_block(size_t h, size_t i) { + mi_heap_theap(separate_heaps[h]); + mi_free(separate_blocks[h][i]); +} + static bool free_separate_heaps(void) { mi_thread_init(); + // a thread without a theap for a (non-main) heap does not create one on free for (size_t h = 0; h < 2; h++) { + mi_free(separate_blocks[h][0]); if (_mi_heap_theap_peek(separate_heaps[h]) != NULL) return false; } - for (size_t i = 0; i < 32; i++) { - for (size_t h = 0; h < 2; h++) mi_free(separate_blocks[h][i]); + for (size_t i = 1; i < 32; i++) { + for (size_t h = 0; h < 2; h++) free_separate_block(h, i); } mi_theap_t* first = _mi_heap_theap_peek(separate_heaps[0]); mi_theap_t* second = _mi_heap_theap_peek(separate_heaps[1]); @@ -277,7 +288,7 @@ if (first->free_batch.batched_total != 0 || second->free_batch.batched_total == 0) return false; mi_theap_delete(second); if (_mi_heap_theap_peek(separate_heaps[1]) != NULL) return false; - for (size_t h = 0; h < 2; h++) mi_free(separate_blocks[h][32]); + for (size_t h = 0; h < 2; h++) free_separate_block(h, 32); second = _mi_heap_theap_peek(separate_heaps[1]); if (second == NULL || first->free_batch.batched_total == 0 || second->free_batch.batched_total == 0) return false; @@ -306,6 +317,7 @@ static bool pending_heap_consumer(void) { mi_thread_init(); + if (mi_heap_theap(disposable) == NULL) return false; // use the heap so frees are batched for (size_t i = 0; i + 1 < block_count; i++) mi_free(blocks[i]); mi_theap_t* const theap = _mi_heap_theap_peek(disposable); consumer_theap = theap;