blob: eb4eebefdfb5d18b0c8a19f575b292e8e5e03260 [file] [log] [blame]
/*
* QEMU coroutines
*
* Copyright IBM, Corp. 2011
*
* Authors:
* Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
* Kevin Wolf <kwolf@redhat.com>
*
* This work is licensed under the terms of the GNU LGPL, version 2 or later.
* See the COPYING.LIB file in the top-level directory.
*
*/
#include "qemu/osdep.h"
#include "trace.h"
#include "qemu/thread.h"
#include "qemu/atomic.h"
#include "qemu/coroutine_int.h"
#include "qemu/coroutine-tls.h"
#include "qemu/cutils.h"
#include "block/aio.h"
enum {
COROUTINE_POOL_BATCH_MAX_SIZE = 128,
};
/*
* Coroutine creation and deletion is expensive so a pool of unused coroutines
* is kept as a cache. When the pool has coroutines available, they are
* recycled instead of creating new ones from scratch. Coroutines are added to
* the pool upon termination.
*
* The pool is global but each thread maintains a small local pool to avoid
* global pool contention. Threads fetch and return batches of coroutines from
* the global pool to maintain their local pool. The local pool holds up to two
* batches whereas the maximum size of the global pool is controlled by the
* qemu_coroutine_inc_pool_size() API.
*
* .-----------------------------------.
* | Batch 1 | Batch 2 | Batch 3 | ... | global_pool
* `-----------------------------------'
*
* .-------------------.
* | Batch 1 | Batch 2 | per-thread local_pool (maximum 2 batches)
* `-------------------'
*/
typedef struct CoroutinePoolBatch {
/* Batches are kept in a list */
QSLIST_ENTRY(CoroutinePoolBatch) next;
/* This batch holds up to @COROUTINE_POOL_BATCH_MAX_SIZE coroutines */
QSLIST_HEAD(, Coroutine) list;
unsigned int size;
} CoroutinePoolBatch;
typedef QSLIST_HEAD(, CoroutinePoolBatch) CoroutinePool;
/* Host operating system limit on number of pooled coroutines */
static unsigned int global_pool_hard_max_size;
static QemuMutex global_pool_lock; /* protects the following variables */
static CoroutinePool global_pool = QSLIST_HEAD_INITIALIZER(global_pool);
static unsigned int global_pool_size;
static unsigned int global_pool_max_size = COROUTINE_POOL_BATCH_MAX_SIZE;
QEMU_DEFINE_STATIC_CO_TLS(CoroutinePool, local_pool);
QEMU_DEFINE_STATIC_CO_TLS(Notifier, local_pool_cleanup_notifier);
static CoroutinePoolBatch *coroutine_pool_batch_new(void)
{
CoroutinePoolBatch *batch = g_new(CoroutinePoolBatch, 1);
QSLIST_INIT(&batch->list);
batch->size = 0;
return batch;
}
static void coroutine_pool_batch_delete(CoroutinePoolBatch *batch)
{
Coroutine *co;
Coroutine *tmp;
QSLIST_FOREACH_SAFE(co, &batch->list, pool_next, tmp) {
QSLIST_REMOVE_HEAD(&batch->list, pool_next);
qemu_coroutine_delete(co);
}
g_free(batch);
}
static void local_pool_cleanup(Notifier *n, void *value)
{
CoroutinePool *local_pool = get_ptr_local_pool();
CoroutinePoolBatch *batch;
CoroutinePoolBatch *tmp;
QSLIST_FOREACH_SAFE(batch, local_pool, next, tmp) {
QSLIST_REMOVE_HEAD(local_pool, next);
coroutine_pool_batch_delete(batch);
}
}
/* Ensure the atexit notifier is registered */
static void local_pool_cleanup_init_once(void)
{
Notifier *notifier = get_ptr_local_pool_cleanup_notifier();
if (!notifier->notify) {
notifier->notify = local_pool_cleanup;
qemu_thread_atexit_add(notifier);
}
}
/* Helper to get the next unused coroutine from the local pool */
static Coroutine *coroutine_pool_get_local(void)
{
CoroutinePool *local_pool = get_ptr_local_pool();
CoroutinePoolBatch *batch = QSLIST_FIRST(local_pool);
Coroutine *co;
if (unlikely(!batch)) {
return NULL;
}
co = QSLIST_FIRST(&batch->list);
QSLIST_REMOVE_HEAD(&batch->list, pool_next);
batch->size--;
if (batch->size == 0) {
QSLIST_REMOVE_HEAD(local_pool, next);
coroutine_pool_batch_delete(batch);
}
return co;
}
/* Get the next batch from the global pool */
static void coroutine_pool_refill_local(void)
{
CoroutinePool *local_pool = get_ptr_local_pool();
CoroutinePoolBatch *batch;
WITH_QEMU_LOCK_GUARD(&global_pool_lock) {
batch = QSLIST_FIRST(&global_pool);
if (batch) {
QSLIST_REMOVE_HEAD(&global_pool, next);
global_pool_size -= batch->size;
}
}
if (batch) {
QSLIST_INSERT_HEAD(local_pool, batch, next);
local_pool_cleanup_init_once();
}
}
/* Add a batch of coroutines to the global pool */
static void coroutine_pool_put_global(CoroutinePoolBatch *batch)
{
WITH_QEMU_LOCK_GUARD(&global_pool_lock) {
unsigned int max = MIN(global_pool_max_size,
global_pool_hard_max_size);
if (global_pool_size < max) {
QSLIST_INSERT_HEAD(&global_pool, batch, next);
/* Overshooting the max pool size is allowed */
global_pool_size += batch->size;
return;
}
}
/* The global pool was full, so throw away this batch */
coroutine_pool_batch_delete(batch);
}
/* Get the next unused coroutine from the pool or return NULL */
static Coroutine *coroutine_pool_get(void)
{
Coroutine *co;
co = coroutine_pool_get_local();
if (!co) {
coroutine_pool_refill_local();
co = coroutine_pool_get_local();
}
return co;
}
static void coroutine_pool_put(Coroutine *co)
{
CoroutinePool *local_pool = get_ptr_local_pool();
CoroutinePoolBatch *batch = QSLIST_FIRST(local_pool);
if (unlikely(!batch)) {
batch = coroutine_pool_batch_new();
QSLIST_INSERT_HEAD(local_pool, batch, next);
local_pool_cleanup_init_once();
}
if (unlikely(batch->size >= COROUTINE_POOL_BATCH_MAX_SIZE)) {
CoroutinePoolBatch *next = QSLIST_NEXT(batch, next);
/* Is the local pool full? */
if (next) {
QSLIST_REMOVE_HEAD(local_pool, next);
coroutine_pool_put_global(batch);
}
batch = coroutine_pool_batch_new();
QSLIST_INSERT_HEAD(local_pool, batch, next);
}
QSLIST_INSERT_HEAD(&batch->list, co, pool_next);
batch->size++;
}
Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque)
{
Coroutine *co = NULL;
if (IS_ENABLED(CONFIG_COROUTINE_POOL)) {
co = coroutine_pool_get();
}
if (!co) {
co = qemu_coroutine_new();
}
co->entry = entry;
co->entry_arg = opaque;
QSIMPLEQ_INIT(&co->co_queue_wakeup);
return co;
}
static void coroutine_delete(Coroutine *co)
{
co->caller = NULL;
if (IS_ENABLED(CONFIG_COROUTINE_POOL)) {
coroutine_pool_put(co);
} else {
qemu_coroutine_delete(co);
}
}
void qemu_aio_coroutine_enter(AioContext *ctx, Coroutine *co)
{
QSIMPLEQ_HEAD(, Coroutine) pending = QSIMPLEQ_HEAD_INITIALIZER(pending);
Coroutine *from = qemu_coroutine_self();
QSIMPLEQ_INSERT_TAIL(&pending, co, co_queue_next);
/* Run co and any queued coroutines */
while (!QSIMPLEQ_EMPTY(&pending)) {
Coroutine *to = QSIMPLEQ_FIRST(&pending);
CoroutineAction ret;
/*
* Read to before to->scheduled; pairs with qatomic_cmpxchg in
* qemu_co_sleep(), aio_co_schedule() etc.
*/
smp_read_barrier_depends();
const char *scheduled = qatomic_read(&to->scheduled);
QSIMPLEQ_REMOVE_HEAD(&pending, co_queue_next);
trace_qemu_aio_coroutine_enter(ctx, from, to, to->entry_arg);
/* if the Coroutine has already been scheduled, entering it again will
* cause us to enter it twice, potentially even after the coroutine has
* been deleted */
if (scheduled) {
fprintf(stderr,
"%s: Co-routine was already scheduled in '%s'\n",
__func__, scheduled);
abort();
}
if (to->caller) {
fprintf(stderr, "Co-routine re-entered recursively\n");
abort();
}
to->caller = from;
to->ctx = ctx;
/* Store to->ctx before anything that stores to. Matches
* barrier in aio_co_wake and qemu_co_mutex_wake.
*/
smp_wmb();
ret = qemu_coroutine_switch(from, to, COROUTINE_ENTER);
/* Queued coroutines are run depth-first; previously pending coroutines
* run after those queued more recently.
*/
QSIMPLEQ_PREPEND(&pending, &to->co_queue_wakeup);
switch (ret) {
case COROUTINE_YIELD:
break;
case COROUTINE_TERMINATE:
assert(!to->locks_held);
trace_qemu_coroutine_terminate(to);
coroutine_delete(to);
break;
default:
abort();
}
}
}
void qemu_coroutine_enter(Coroutine *co)
{
qemu_aio_coroutine_enter(qemu_get_current_aio_context(), co);
}
void qemu_coroutine_enter_if_inactive(Coroutine *co)
{
if (!qemu_coroutine_entered(co)) {
qemu_coroutine_enter(co);
}
}
void coroutine_fn qemu_coroutine_yield(void)
{
Coroutine *self = qemu_coroutine_self();
Coroutine *to = self->caller;
trace_qemu_coroutine_yield(self, to);
if (!to) {
fprintf(stderr, "Co-routine is yielding to no one\n");
abort();
}
self->caller = NULL;
qemu_coroutine_switch(self, to, COROUTINE_YIELD);
}
bool qemu_coroutine_entered(Coroutine *co)
{
return co->caller;
}
AioContext *qemu_coroutine_get_aio_context(Coroutine *co)
{
return co->ctx;
}
void qemu_coroutine_inc_pool_size(unsigned int additional_pool_size)
{
QEMU_LOCK_GUARD(&global_pool_lock);
global_pool_max_size += additional_pool_size;
}
void qemu_coroutine_dec_pool_size(unsigned int removing_pool_size)
{
QEMU_LOCK_GUARD(&global_pool_lock);
global_pool_max_size -= removing_pool_size;
}
static unsigned int get_global_pool_hard_max_size(void)
{
#ifdef __linux__
g_autofree char *contents = NULL;
int max_map_count;
/*
* Linux processes can have up to max_map_count virtual memory areas
* (VMAs). mmap(2), mprotect(2), etc fail with ENOMEM beyond this limit. We
* must limit the coroutine pool to a safe size to avoid running out of
* VMAs.
*/
if (g_file_get_contents("/proc/sys/vm/max_map_count", &contents, NULL,
NULL) &&
qemu_strtoi(contents, NULL, 10, &max_map_count) == 0) {
/*
* This is an upper bound that avoids exceeding max_map_count. Leave a
* fixed amount for non-coroutine users like library dependencies,
* vhost-user, etc. Each coroutine takes up 2 VMAs so halve the
* remaining amount.
*/
if (max_map_count > 5000) {
return (max_map_count - 5000) / 2;
} else {
/* Disable the global pool but threads still have local pools */
return 0;
}
}
#endif
return UINT_MAX;
}
static void __attribute__((constructor)) qemu_coroutine_init(void)
{
qemu_mutex_init(&global_pool_lock);
global_pool_hard_max_size = get_global_pool_hard_max_size();
}