test/unit/batch_alloc.c

#include "test/jemalloc_test.h"

#define BATCH_MAX ((1U << 16) + 1024)
static void *ptrs[BATCH_MAX];

#define PAGE_ALIGNED(ptr) (((uintptr_t)ptr & PAGE_MASK) == 0)

static void
verify_batch_basic(tsd_t *tsd, void **ptrs, size_t batch, size_t usize,
    bool zero) {
	for (size_t i = 0; i < batch; ++i) {
		void *p = ptrs[i];
		expect_zu_eq(isalloc(tsd_tsdn(tsd), p), usize, "");
		if (zero) {
			for (size_t k = 0; k < usize; ++k) {
				expect_true(*((unsigned char *)p + k) == 0, "");
			}
		}
	}
}

static void
verify_batch_locality(tsd_t *tsd, void **ptrs, size_t batch, size_t usize,
    arena_t *arena, unsigned nregs) {
	if (config_prof && opt_prof) {
		/*
		 * Checking batch locality when prof is on is feasible but
		 * complicated, while checking the non-prof case suffices for
		 * unit-test purpose.
		 */
		return;
	}
	for (size_t i = 0, j = 0; i < batch; ++i, ++j) {
		if (j == nregs) {
			j = 0;
		}
		if (j == 0 && batch - i < nregs) {
			break;
		}
		void *p = ptrs[i];
		expect_ptr_eq(iaalloc(tsd_tsdn(tsd), p), arena, "");
		if (j == 0) {
			expect_true(PAGE_ALIGNED(p), "");
			continue;
		}
		assert(i > 0);
		void *q = ptrs[i - 1];
		expect_true((uintptr_t)p > (uintptr_t)q
		    && (size_t)((uintptr_t)p - (uintptr_t)q) == usize, "");
	}
}

static void
release_batch(void **ptrs, size_t batch, size_t size) {
	for (size_t i = 0; i < batch; ++i) {
		sdallocx(ptrs[i], size, 0);
	}
}

typedef struct batch_alloc_packet_s batch_alloc_packet_t;
struct batch_alloc_packet_s {
	void **ptrs;
	size_t num;
	size_t size;
	int flags;
};

static size_t
batch_alloc_wrapper(void **ptrs, size_t num, size_t size, int flags) {
	batch_alloc_packet_t batch_alloc_packet = {ptrs, num, size, flags};
	size_t filled;
	size_t len = sizeof(size_t);
	assert_d_eq(mallctl("experimental.batch_alloc", &filled, &len,
	    &batch_alloc_packet, sizeof(batch_alloc_packet)), 0, "");
	return filled;
}

static void
test_wrapper(size_t size, size_t alignment, bool zero, unsigned arena_flag) {
	tsd_t *tsd = tsd_fetch();
	assert(tsd != NULL);
	const size_t usize =
	    (alignment != 0 ? sz_sa2u(size, alignment) : sz_s2u(size));
	const szind_t ind = sz_size2index(usize);
	const bin_info_t *bin_info = &bin_infos[ind];
	const unsigned nregs = bin_info->nregs;
	assert(nregs > 0);
	arena_t *arena;
	if (arena_flag != 0) {
		arena = arena_get(tsd_tsdn(tsd), MALLOCX_ARENA_GET(arena_flag),
		    false);
	} else {
		arena = arena_choose(tsd, NULL);
	}
	assert(arena != NULL);
	int flags = arena_flag;
	if (alignment != 0) {
		flags |= MALLOCX_ALIGN(alignment);
	}
	if (zero) {
		flags |= MALLOCX_ZERO;
	}

	/*
	 * Allocate for the purpose of bootstrapping arena_tdata, so that the
	 * change in bin stats won't contaminate the stats to be verified below.
	 */
	void *p = mallocx(size, flags | MALLOCX_TCACHE_NONE);

	for (size_t i = 0; i < 4; ++i) {
		size_t base = 0;
		if (i == 1) {
			base = nregs;
		} else if (i == 2) {
			base = nregs * 2;
		} else if (i == 3) {
			base = (1 << 16);
		}
		for (int j = -1; j <= 1; ++j) {
			if (base == 0 && j == -1) {
				continue;
			}
			size_t batch = base + (size_t)j;
			assert(batch < BATCH_MAX);
			size_t filled = batch_alloc_wrapper(ptrs, batch, size,
			    flags);
			assert_zu_eq(filled, batch, "");
			verify_batch_basic(tsd, ptrs, batch, usize, zero);
			verify_batch_locality(tsd, ptrs, batch, usize, arena,
			    nregs);
			release_batch(ptrs, batch, usize);
		}
	}

	free(p);
}

TEST_BEGIN(test_batch_alloc) {
	test_wrapper(11, 0, false, 0);
}
TEST_END

TEST_BEGIN(test_batch_alloc_zero) {
	test_wrapper(11, 0, true, 0);
}
TEST_END

TEST_BEGIN(test_batch_alloc_aligned) {
	test_wrapper(7, 16, false, 0);
}
TEST_END

TEST_BEGIN(test_batch_alloc_manual_arena) {
	unsigned arena_ind;
	size_t len_unsigned = sizeof(unsigned);
	assert_d_eq(mallctl("arenas.create", &arena_ind, &len_unsigned, NULL,
	    0), 0, "");
	test_wrapper(11, 0, false, MALLOCX_ARENA(arena_ind));
}
TEST_END

TEST_BEGIN(test_batch_alloc_large) {
	size_t size = SC_LARGE_MINCLASS;
	for (size_t batch = 0; batch < 4; ++batch) {
		assert(batch < BATCH_MAX);
		size_t filled = batch_alloc(ptrs, batch, size, 0);
		assert_zu_eq(filled, batch, "");
		release_batch(ptrs, batch, size);
	}
	size = tcache_maxclass + 1;
	for (size_t batch = 0; batch < 4; ++batch) {
		assert(batch < BATCH_MAX);
		size_t filled = batch_alloc(ptrs, batch, size, 0);
		assert_zu_eq(filled, batch, "");
		release_batch(ptrs, batch, size);
	}
}
TEST_END

int
main(void) {
	return test(
	    test_batch_alloc,
	    test_batch_alloc_zero,
	    test_batch_alloc_aligned,
	    test_batch_alloc_manual_arena,
	    test_batch_alloc_large);
}