src/pa_extra.c

#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"

/*
 * This file is logically part of the PA module.  While pa.c contains the core
 * allocator functionality, this file contains boring integration functionality;
 * things like the pre- and post- fork handlers, and stats merging for CTL
 * refreshes.
 */

void
pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
	malloc_mutex_prefork(tsdn, &shard->pac.decay_dirty.mtx);
	malloc_mutex_prefork(tsdn, &shard->pac.decay_muzzy.mtx);
}

void
pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
	if (shard->ever_used_hpa) {
		sec_prefork2(tsdn, &shard->hpa_sec);
	}
}

void
pa_shard_prefork3(tsdn_t *tsdn, pa_shard_t *shard) {
	malloc_mutex_prefork(tsdn, &shard->pac.grow_mtx);
	if (shard->ever_used_hpa) {
		hpa_shard_prefork3(tsdn, &shard->hpa_shard);
	}
}

void
pa_shard_prefork4(tsdn_t *tsdn, pa_shard_t *shard) {
	ecache_prefork(tsdn, &shard->pac.ecache_dirty);
	ecache_prefork(tsdn, &shard->pac.ecache_muzzy);
	ecache_prefork(tsdn, &shard->pac.ecache_retained);
	if (shard->ever_used_hpa) {
		hpa_shard_prefork4(tsdn, &shard->hpa_shard);
	}
}

void
pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard) {
	edata_cache_prefork(tsdn, &shard->edata_cache);
}

void
pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
	edata_cache_postfork_parent(tsdn, &shard->edata_cache);
	ecache_postfork_parent(tsdn, &shard->pac.ecache_dirty);
	ecache_postfork_parent(tsdn, &shard->pac.ecache_muzzy);
	ecache_postfork_parent(tsdn, &shard->pac.ecache_retained);
	malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx);
	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
	if (shard->ever_used_hpa) {
		sec_postfork_parent(tsdn, &shard->hpa_sec);
		hpa_shard_postfork_parent(tsdn, &shard->hpa_shard);
	}
}

void
pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
	edata_cache_postfork_child(tsdn, &shard->edata_cache);
	ecache_postfork_child(tsdn, &shard->pac.ecache_dirty);
	ecache_postfork_child(tsdn, &shard->pac.ecache_muzzy);
	ecache_postfork_child(tsdn, &shard->pac.ecache_retained);
	malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx);
	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
	if (shard->ever_used_hpa) {
		sec_postfork_child(tsdn, &shard->hpa_sec);
		hpa_shard_postfork_child(tsdn, &shard->hpa_shard);
	}
}

void
pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, size_t *ndirty,
    size_t *nmuzzy) {
	*nactive += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
	*ndirty += ecache_npages_get(&shard->pac.ecache_dirty);
	*nmuzzy += ecache_npages_get(&shard->pac.ecache_muzzy);
}

void
pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
    pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
    hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
    size_t *resident) {
	cassert(config_stats);

	pa_shard_stats_out->pac_stats.retained +=
	    ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE;
	pa_shard_stats_out->edata_avail += atomic_load_zu(
	    &shard->edata_cache.count, ATOMIC_RELAXED);

	size_t resident_pgs = 0;
	resident_pgs += atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
	resident_pgs += ecache_npages_get(&shard->pac.ecache_dirty);
	*resident += (resident_pgs << LG_PAGE);

	/* Dirty decay stats */
	locked_inc_u64_unsynchronized(
	    &pa_shard_stats_out->pac_stats.decay_dirty.npurge,
	    locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
	    &shard->pac.stats->decay_dirty.npurge));
	locked_inc_u64_unsynchronized(
	    &pa_shard_stats_out->pac_stats.decay_dirty.nmadvise,
	    locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
	    &shard->pac.stats->decay_dirty.nmadvise));
	locked_inc_u64_unsynchronized(
	    &pa_shard_stats_out->pac_stats.decay_dirty.purged,
	    locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
	    &shard->pac.stats->decay_dirty.purged));

	/* Muzzy decay stats */
	locked_inc_u64_unsynchronized(
	    &pa_shard_stats_out->pac_stats.decay_muzzy.npurge,
	    locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
	    &shard->pac.stats->decay_muzzy.npurge));
	locked_inc_u64_unsynchronized(
	    &pa_shard_stats_out->pac_stats.decay_muzzy.nmadvise,
	    locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
	    &shard->pac.stats->decay_muzzy.nmadvise));
	locked_inc_u64_unsynchronized(
	    &pa_shard_stats_out->pac_stats.decay_muzzy.purged,
	    locked_read_u64(tsdn, LOCKEDINT_MTX(*shard->stats_mtx),
	    &shard->pac.stats->decay_muzzy.purged));

	atomic_load_add_store_zu(&pa_shard_stats_out->pac_stats.abandoned_vm,
	    atomic_load_zu(&shard->pac.stats->abandoned_vm, ATOMIC_RELAXED));

	for (pszind_t i = 0; i < SC_NPSIZES; i++) {
		size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes,
		    retained_bytes;
		dirty = ecache_nextents_get(&shard->pac.ecache_dirty, i);
		muzzy = ecache_nextents_get(&shard->pac.ecache_muzzy, i);
		retained = ecache_nextents_get(&shard->pac.ecache_retained, i);
		dirty_bytes = ecache_nbytes_get(&shard->pac.ecache_dirty, i);
		muzzy_bytes = ecache_nbytes_get(&shard->pac.ecache_muzzy, i);
		retained_bytes = ecache_nbytes_get(&shard->pac.ecache_retained,
		    i);

		estats_out[i].ndirty = dirty;
		estats_out[i].nmuzzy = muzzy;
		estats_out[i].nretained = retained;
		estats_out[i].dirty_bytes = dirty_bytes;
		estats_out[i].muzzy_bytes = muzzy_bytes;
		estats_out[i].retained_bytes = retained_bytes;
	}

	if (shard->ever_used_hpa) {
		hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out);
		sec_stats_merge(tsdn, &shard->hpa_sec, sec_stats_out);
	}
}

static void
pa_shard_mtx_stats_read_single(tsdn_t *tsdn, mutex_prof_data_t *mutex_prof_data,
    malloc_mutex_t *mtx, int ind) {
	malloc_mutex_lock(tsdn, mtx);
	malloc_mutex_prof_read(tsdn, &mutex_prof_data[ind], mtx);
	malloc_mutex_unlock(tsdn, mtx);
}

void
pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
    mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]) {
	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
	    &shard->edata_cache.mtx, arena_prof_mutex_extent_avail);
	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
	    &shard->pac.ecache_dirty.mtx, arena_prof_mutex_extents_dirty);
	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
	    &shard->pac.ecache_muzzy.mtx, arena_prof_mutex_extents_muzzy);
	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
	    &shard->pac.ecache_retained.mtx, arena_prof_mutex_extents_retained);
	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
	    &shard->pac.decay_dirty.mtx, arena_prof_mutex_decay_dirty);
	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
	    &shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy);

	if (shard->ever_used_hpa) {
		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
		    &shard->hpa_shard.mtx, arena_prof_mutex_hpa_shard);
		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
		    &shard->hpa_shard.grow_mtx,
		    arena_prof_mutex_hpa_shard_grow);
		sec_mutex_stats_read(tsdn, &shard->hpa_sec,
		    &mutex_prof_data[arena_prof_mutex_hpa_sec]);
	}
}