src/inline_internal.h

/*
 * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
 *
 * @APPLE_APACHE_LICENSE_HEADER_START@
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @APPLE_APACHE_LICENSE_HEADER_END@
 */

/*
 * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
 * which are subject to change in future releases of Mac OS X. Any applications
 * relying on these interfaces WILL break.
 */

#ifndef __DISPATCH_INLINE_INTERNAL__
#define __DISPATCH_INLINE_INTERNAL__

#ifndef __DISPATCH_INDIRECT__
#error "Please #include <dispatch/dispatch.h> instead of this file directly."
#include <dispatch/base.h> // for HeaderDoc
#endif

#if DISPATCH_USE_CLIENT_CALLOUT

DISPATCH_NOTHROW void
_dispatch_client_callout(void *ctxt, dispatch_function_t f);
DISPATCH_NOTHROW void
_dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t));
#if HAVE_MACH
DISPATCH_NOTHROW void
_dispatch_client_callout3(void *ctxt, dispatch_mach_reason_t reason,
		dispatch_mach_msg_t dmsg, dispatch_mach_async_reply_callback_t f);
DISPATCH_NOTHROW void
_dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason,
		dispatch_mach_msg_t dmsg, mach_error_t error,
		dispatch_mach_handler_function_t f);
#endif // HAVE_MACH

#else // !DISPATCH_USE_CLIENT_CALLOUT

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_client_callout(void *ctxt, dispatch_function_t f)
{
	return f(ctxt);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t))
{
	return f(ctxt, i);
}

#if HAVE_MACH
DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_client_callout3(void *ctxt, dispatch_mach_reason_t reason,
		dispatch_mach_msg_t dmsg, dispatch_mach_async_reply_callback_t f)
{
	return f(ctxt, reason, dmsg);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason,
		dispatch_mach_msg_t dmsg, mach_error_t error,
		dispatch_mach_handler_function_t f)
{
	return f(ctxt, reason, dmsg, error);
}
#endif // HAVE_MACH

#endif // !DISPATCH_USE_CLIENT_CALLOUT

#pragma mark -
#pragma mark _os_object_t & dispatch_object_t
#if DISPATCH_PURE_C

DISPATCH_ALWAYS_INLINE
static inline const char *
_dispatch_object_class_name(dispatch_object_t dou)
{
#if USE_OBJC
	return object_getClassName((id)dou._do) + strlen("OS_dispatch_");
#else
	return dx_vtable(dou._do)->do_kind;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_is_global(dispatch_object_t dou)
{
	return dou._do->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_is_root_or_base_queue(dispatch_object_t dou)
{
	return dx_hastypeflag(dou._do, QUEUE_ROOT) ||
			dx_hastypeflag(dou._do, QUEUE_BASE);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_has_vtable(dispatch_object_t dou)
{
	// vtables are pointers far away from the low page in memory
	return dou._dc->dc_flags > 0xffful;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_is_queue(dispatch_object_t dou)
{
	return _dispatch_object_has_vtable(dou) && dx_vtable(dou._dq)->dq_push;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_is_continuation(dispatch_object_t dou)
{
	if (_dispatch_object_has_vtable(dou)) {
		return dx_metatype(dou._do) == _DISPATCH_CONTINUATION_TYPE;
	}
	return true;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_has_type(dispatch_object_t dou, unsigned long type)
{
	return _dispatch_object_has_vtable(dou) && dx_type(dou._do) == type;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_is_redirection(dispatch_object_t dou)
{
	return _dispatch_object_has_type(dou,
			DISPATCH_CONTINUATION_TYPE(ASYNC_REDIRECT));
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_is_barrier(dispatch_object_t dou)
{
	dispatch_queue_flags_t dq_flags;

	if (!_dispatch_object_has_vtable(dou)) {
		return (dou._dc->dc_flags & DC_FLAG_BARRIER);
	}
	if (dx_cluster(dou._do) != _DISPATCH_QUEUE_CLUSTER) {
		return false;
	}
	dq_flags = os_atomic_load2o(dou._dq, dq_atomic_flags, relaxed);
	return dq_flags & DQF_BARRIER_BIT;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_is_waiter(dispatch_object_t dou)
{
	if (_dispatch_object_has_vtable(dou)) {
		return false;
	}
	return (dou._dc->dc_flags & (DC_FLAG_SYNC_WAITER | DC_FLAG_ASYNC_AND_WAIT));
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_is_sync_waiter(dispatch_object_t dou)
{
	if (_dispatch_object_has_vtable(dou)) {
		return false;
	}
	return (dou._dc->dc_flags & DC_FLAG_SYNC_WAITER);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_object_is_sync_waiter_non_barrier(dispatch_object_t dou)
{
	if (_dispatch_object_has_vtable(dou)) {
		return false;
	}
	return ((dou._dc->dc_flags & (DC_FLAG_BARRIER | DC_FLAG_SYNC_WAITER)) ==
				(DC_FLAG_SYNC_WAITER));
}

DISPATCH_ALWAYS_INLINE
static inline _os_object_t
_os_object_retain_internal_n_inline(_os_object_t obj, int n)
{
	int ref_cnt = _os_object_refcnt_add_orig(obj, n);
	if (unlikely(ref_cnt < 0)) {
		_OS_OBJECT_CLIENT_CRASH("Resurrection of an object");
	}
	return obj;
}

DISPATCH_ALWAYS_INLINE
static inline void
_os_object_release_internal_n_no_dispose_inline(_os_object_t obj, int n)
{
	int ref_cnt = _os_object_refcnt_sub(obj, n);
	if (likely(ref_cnt >= 0)) {
		return;
	}
	_OS_OBJECT_CLIENT_CRASH("Over-release of an object");
}

DISPATCH_ALWAYS_INLINE
static inline void
_os_object_release_internal_n_inline(_os_object_t obj, int n)
{
	int ref_cnt = _os_object_refcnt_sub(obj, n);
	if (likely(ref_cnt >= 0)) {
		return;
	}
	if (unlikely(ref_cnt < -1)) {
		_OS_OBJECT_CLIENT_CRASH("Over-release of an object");
	}
#if DISPATCH_DEBUG
	int xref_cnt = obj->os_obj_xref_cnt;
	if (unlikely(xref_cnt >= 0)) {
		DISPATCH_INTERNAL_CRASH(xref_cnt,
				"Release while external references exist");
	}
#endif
	// _os_object_refcnt_dispose_barrier() is in _os_object_dispose()
	return _os_object_dispose(obj);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_retain(dispatch_object_t dou)
{
	(void)_os_object_retain_internal_n_inline(dou._os_obj, 1);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_retain_2(dispatch_object_t dou)
{
	(void)_os_object_retain_internal_n_inline(dou._os_obj, 2);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_retain_n(dispatch_object_t dou, int n)
{
	(void)_os_object_retain_internal_n_inline(dou._os_obj, n);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_retain_n_unsafe(dispatch_object_t dou, int n)
{
	// _dispatch_retain_*_unsafe assumes:
	// - the object is not global
	// - there's no refcount management bug
	//
	// This is meant to be used only when called between the update_tail and
	// update_prev os_mpsc methods, so that the assembly of that critical window
	// is as terse as possible (this window is a possible dequeuer starvation).
	//
	// Other code should use the safe variants at all times.
	os_atomic_add2o(dou._os_obj, os_obj_ref_cnt, n, relaxed);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_retain_2_unsafe(dispatch_object_t dou)
{
	_dispatch_retain_n_unsafe(dou, 2);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_release(dispatch_object_t dou)
{
	_os_object_release_internal_n_inline(dou._os_obj, 1);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_release_2(dispatch_object_t dou)
{
	_os_object_release_internal_n_inline(dou._os_obj, 2);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_release_n(dispatch_object_t dou, int n)
{
	_os_object_release_internal_n_inline(dou._os_obj, n);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_release_no_dispose(dispatch_object_t dou)
{
	_os_object_release_internal_n_no_dispose_inline(dou._os_obj, 1);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_release_2_no_dispose(dispatch_object_t dou)
{
	_os_object_release_internal_n_no_dispose_inline(dou._os_obj, 2);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_release_tailcall(dispatch_object_t dou)
{
	_os_object_release_internal(dou._os_obj);
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_release_2_tailcall(dispatch_object_t dou)
{
	_os_object_release_internal_n(dou._os_obj, 2);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_retain_unote_owner(dispatch_unote_t du)
{
	_dispatch_retain_2(_dispatch_wref2ptr(du._du->du_owner_wref));
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_release_unote_owner_tailcall(dispatch_unote_t du)
{
	_dispatch_release_2_tailcall(_dispatch_wref2ptr(du._du->du_owner_wref));
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_retain_storage(dispatch_queue_class_t dqu)
{
	int ref_cnt = os_atomic_inc2o(dqu._dq, dq_sref_cnt, relaxed);
	if (unlikely(ref_cnt <= 0)) {
		_OS_OBJECT_CLIENT_CRASH("Resurrection of an object");
	}
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_release_storage(dispatch_queue_class_t dqu)
{
	// this refcount only delays the _dispatch_object_dealloc() and there's no
	// need for visibility wrt to the allocation, the internal refcount already
	// gives us that, and the object becomes immutable after the last internal
	// refcount release.
	int ref_cnt = os_atomic_dec2o(dqu._dq, dq_sref_cnt, relaxed);
	if (unlikely(ref_cnt >= 0)) {
		return;
	}
	if (unlikely(ref_cnt < -1)) {
		_OS_OBJECT_CLIENT_CRASH("Over-release of an object");
	}
	dqu._dq->dq_state = 0xdead000000000000;
	_dispatch_object_dealloc(dqu._dq);
}

DISPATCH_ALWAYS_INLINE DISPATCH_NONNULL_ALL
static inline void
_dispatch_object_set_target_queue_inline(dispatch_object_t dou,
		dispatch_queue_t tq)
{
	_dispatch_retain(tq);
	tq = os_atomic_xchg2o(dou._do, do_targetq, tq, release);
	if (tq) _dispatch_release(tq);
}

#endif // DISPATCH_PURE_C
#pragma mark -
#pragma mark dispatch_thread
#if DISPATCH_PURE_C

DISPATCH_ALWAYS_INLINE
static inline dispatch_thread_context_t
_dispatch_thread_context_find(const void *key)
{
	dispatch_thread_context_t dtc =
			_dispatch_thread_getspecific(dispatch_context_key);
	while (dtc) {
		if (dtc->dtc_key == key) {
			return dtc;
		}
		dtc = dtc->dtc_prev;
	}
	return NULL;
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_context_push(dispatch_thread_context_t ctxt)
{
	ctxt->dtc_prev = _dispatch_thread_getspecific(dispatch_context_key);
	_dispatch_thread_setspecific(dispatch_context_key, ctxt);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_context_pop(dispatch_thread_context_t ctxt)
{
	dispatch_assert(_dispatch_thread_getspecific(dispatch_context_key) == ctxt);
	_dispatch_thread_setspecific(dispatch_context_key, ctxt->dtc_prev);
}

typedef struct dispatch_thread_frame_iterator_s {
	dispatch_queue_t dtfi_queue;
	dispatch_thread_frame_t dtfi_frame;
} *dispatch_thread_frame_iterator_t;

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_frame_iterate_start(dispatch_thread_frame_iterator_t it)
{
	_dispatch_thread_getspecific_pair(
			dispatch_queue_key, (void **)&it->dtfi_queue,
			dispatch_frame_key, (void **)&it->dtfi_frame);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_frame_iterate_next(dispatch_thread_frame_iterator_t it)
{
	dispatch_thread_frame_t dtf = it->dtfi_frame;
	dispatch_queue_t dq = it->dtfi_queue;

	if (dtf) {
		dispatch_queue_t tq = dq->do_targetq;
		if (tq) {
			// redirections or dispatch_sync may skip frames,
			// so we need to simulate seeing the missing links
			it->dtfi_queue = tq;
			if (dq == dtf->dtf_queue) {
				it->dtfi_frame = dtf->dtf_prev;
			}
		} else {
			it->dtfi_queue = dtf->dtf_queue;
			it->dtfi_frame = dtf->dtf_prev;
		}
	} else if (dq) {
		it->dtfi_queue = dq->do_targetq;
	}
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_thread_frame_find_queue(dispatch_queue_t dq)
{
	struct dispatch_thread_frame_iterator_s it;

	_dispatch_thread_frame_iterate_start(&it);
	while (it.dtfi_queue) {
		if (it.dtfi_queue == dq) {
			return true;
		}
		_dispatch_thread_frame_iterate_next(&it);
	}
	return false;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_thread_frame_t
_dispatch_thread_frame_get_current(void)
{
	return _dispatch_thread_getspecific(dispatch_frame_key);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_frame_save_state(dispatch_thread_frame_t dtf)
{
	_dispatch_thread_getspecific_packed_pair(
			dispatch_queue_key, dispatch_frame_key, dtf->dtf_pair);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_frame_push(dispatch_thread_frame_t dtf,
		dispatch_queue_class_t dqu)
{
	_dispatch_thread_frame_save_state(dtf);
	_dispatch_thread_setspecific_pair(dispatch_queue_key, dqu._dq,
			dispatch_frame_key, dtf);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_frame_push_and_rebase(dispatch_thread_frame_t dtf,
		dispatch_queue_class_t dqu, dispatch_thread_frame_t new_base)
{
	_dispatch_thread_frame_save_state(dtf);
	_dispatch_thread_setspecific_pair(dispatch_queue_key, dqu._dq,
			dispatch_frame_key, new_base);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_frame_pop(dispatch_thread_frame_t dtf)
{
	_dispatch_thread_setspecific_packed_pair(
			dispatch_queue_key, dispatch_frame_key, dtf->dtf_pair);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_queue_t
_dispatch_thread_frame_stash(dispatch_thread_frame_t dtf)
{
	_dispatch_thread_getspecific_pair(
			dispatch_queue_key, &dtf->dtf_pair[0],
			dispatch_frame_key, &dtf->dtf_pair[1]);
	_dispatch_thread_frame_pop(dtf->dtf_prev);
	return dtf->dtf_queue;
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_frame_unstash(dispatch_thread_frame_t dtf)
{
	_dispatch_thread_frame_pop(dtf);
}

DISPATCH_ALWAYS_INLINE
static inline int
_dispatch_wqthread_override_start_check_owner(mach_port_t thread,
		dispatch_qos_t qos, mach_port_t *ulock_addr)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	if (!_dispatch_set_qos_class_enabled) return 0;
	return _pthread_workqueue_override_start_direct_check_owner(thread,
			_dispatch_qos_to_pp(qos), ulock_addr);
#else
	(void)thread; (void)qos; (void)ulock_addr;
	return 0;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_wqthread_override_start(mach_port_t thread, dispatch_qos_t qos)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	if (!_dispatch_set_qos_class_enabled) return;
	(void)_pthread_workqueue_override_start_direct(thread,
			_dispatch_qos_to_pp(qos));
#else
	(void)thread; (void)qos;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_wqthread_override_reset(void)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	if (!_dispatch_set_qos_class_enabled) return;
	(void)_pthread_workqueue_override_reset();
#endif
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_override_start(mach_port_t thread, pthread_priority_t pp,
		void *resource)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	if (!_dispatch_set_qos_class_enabled) return;
	(void)_pthread_qos_override_start_direct(thread, pp, resource);
#else
	(void)thread; (void)pp; (void)resource;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_override_end(mach_port_t thread, void *resource)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	if (!_dispatch_set_qos_class_enabled) return;
	(void)_pthread_qos_override_end_direct(thread, resource);
#else
	(void)thread; (void)resource;
#endif
}

#endif // DISPATCH_PURE_C
#pragma mark -
#pragma mark dispatch_queue_t state accessors
#if DISPATCH_PURE_C

DISPATCH_ALWAYS_INLINE
static inline dispatch_queue_flags_t
_dispatch_queue_atomic_flags(dispatch_queue_class_t dqu)
{
	return os_atomic_load2o(dqu._dq, dq_atomic_flags, relaxed);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_queue_flags_t
_dispatch_queue_atomic_flags_set(dispatch_queue_class_t dqu,
		dispatch_queue_flags_t bits)
{
	return os_atomic_or2o(dqu._dq, dq_atomic_flags, bits, relaxed);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_queue_flags_t
_dispatch_queue_atomic_flags_set_and_clear_orig(dispatch_queue_class_t dqu,
		dispatch_queue_flags_t add_bits, dispatch_queue_flags_t clr_bits)
{
	dispatch_queue_flags_t oflags, nflags;
	os_atomic_rmw_loop2o(dqu._dq, dq_atomic_flags, oflags, nflags, relaxed, {
		nflags = (oflags | add_bits) & ~clr_bits;
		if (nflags == oflags) os_atomic_rmw_loop_give_up(return oflags);
	});
	return oflags;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_queue_flags_t
_dispatch_queue_atomic_flags_set_and_clear(dispatch_queue_class_t dqu,
		dispatch_queue_flags_t add_bits, dispatch_queue_flags_t clr_bits)
{
	dispatch_queue_flags_t oflags, nflags;
	os_atomic_rmw_loop2o(dqu._dq, dq_atomic_flags, oflags, nflags, relaxed, {
		nflags = (oflags | add_bits) & ~clr_bits;
		if (nflags == oflags) os_atomic_rmw_loop_give_up(return oflags);
	});
	return nflags;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_queue_flags_t
_dispatch_queue_atomic_flags_set_orig(dispatch_queue_class_t dqu,
		dispatch_queue_flags_t bits)
{
	return os_atomic_or_orig2o(dqu._dq, dq_atomic_flags, bits, relaxed);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_queue_flags_t
_dispatch_queue_atomic_flags_clear(dispatch_queue_class_t dqu,
		dispatch_queue_flags_t bits)
{
	return os_atomic_and2o(dqu._dq, dq_atomic_flags, ~bits, relaxed);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_queue_is_thread_bound(dispatch_queue_class_t dqu)
{
	return _dispatch_queue_atomic_flags(dqu) & DQF_THREAD_BOUND;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_queue_label_needs_free(dispatch_queue_class_t dqu)
{
	return _dispatch_queue_atomic_flags(dqu) & DQF_LABEL_NEEDS_FREE;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_invoke_flags_t
_dispatch_queue_autorelease_frequency(dispatch_queue_class_t dqu)
{
	const unsigned long factor =
			DISPATCH_INVOKE_AUTORELEASE_ALWAYS / DQF_AUTORELEASE_ALWAYS;
	dispatch_assert(factor > 0);

	dispatch_queue_flags_t qaf = _dispatch_queue_atomic_flags(dqu);

	qaf &= (dispatch_queue_flags_t)_DQF_AUTORELEASE_MASK;
	return (dispatch_invoke_flags_t)qaf * factor;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_invoke_flags_t
_dispatch_queue_merge_autorelease_frequency(dispatch_queue_class_t dqu,
		dispatch_invoke_flags_t flags)
{
	dispatch_invoke_flags_t qaf = _dispatch_queue_autorelease_frequency(dqu);

	if (qaf) {
		flags &= ~_DISPATCH_INVOKE_AUTORELEASE_MASK;
		flags |= qaf;
	}
	return flags;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_queue_is_mutable(dispatch_queue_class_t dqu)
{
	return _dispatch_queue_atomic_flags(dqu) & DQF_MUTABLE;
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_wlh_retain(dispatch_wlh_t wlh)
{
	if (wlh && wlh != DISPATCH_WLH_ANON) {
		_dispatch_queue_retain_storage((dispatch_queue_t)wlh);
	}
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_wlh_release(dispatch_wlh_t wlh)
{
	if (wlh && wlh != DISPATCH_WLH_ANON) {
		_dispatch_queue_release_storage((dispatch_queue_t)wlh);
	}
}

#define DISPATCH_WLH_STORAGE_REF 1ul

DISPATCH_ALWAYS_INLINE DISPATCH_PURE
static inline dispatch_wlh_t
_dispatch_get_wlh(void)
{
	return _dispatch_thread_getspecific(dispatch_wlh_key);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_workloop_t
_dispatch_wlh_to_workloop(dispatch_wlh_t wlh)
{
	if (wlh == DISPATCH_WLH_ANON) {
		return NULL;
	}
	if (dx_metatype((dispatch_workloop_t)wlh) == _DISPATCH_WORKLOOP_TYPE) {
		return (dispatch_workloop_t)wlh;
	}
	return NULL;
}

DISPATCH_ALWAYS_INLINE DISPATCH_PURE
static inline dispatch_wlh_t
_dispatch_get_event_wlh(void)
{
	dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
	if (ddi) {
		DISPATCH_COMPILER_CAN_ASSUME(ddi->ddi_wlh != DISPATCH_WLH_ANON);
		return ddi->ddi_wlh;
	}
	return DISPATCH_WLH_ANON;
}

DISPATCH_ALWAYS_INLINE DISPATCH_PURE
static inline dispatch_wlh_t
_dispatch_get_wlh_reference(void)
{
	dispatch_wlh_t wlh = _dispatch_get_wlh();
	return (dispatch_wlh_t)((uintptr_t)wlh & ~DISPATCH_WLH_STORAGE_REF);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_adopt_wlh_anon_recurse(void)
{
	dispatch_wlh_t cur_wlh = _dispatch_get_wlh_reference();
	if (cur_wlh == DISPATCH_WLH_ANON) return false;
	_dispatch_debug("wlh[anon]: set current (releasing %p)", cur_wlh);
	_dispatch_wlh_release(cur_wlh);
	_dispatch_thread_setspecific(dispatch_wlh_key, (void *)DISPATCH_WLH_ANON);
	return true;
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_adopt_wlh_anon(void)
{
	if (unlikely(!_dispatch_adopt_wlh_anon_recurse())) {
		DISPATCH_INTERNAL_CRASH(0, "Lingering DISPATCH_WLH_ANON");
	}
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_adopt_wlh(dispatch_wlh_t wlh)
{
	dispatch_wlh_t cur_wlh = _dispatch_get_wlh_reference();
	_dispatch_debug("wlh[%p]: adopt current (releasing %p)", wlh, cur_wlh);
	if (cur_wlh == DISPATCH_WLH_ANON) {
		DISPATCH_INTERNAL_CRASH(0, "Lingering DISPATCH_WLH_ANON");
	}
	if (cur_wlh != wlh) {
		dispatch_assert(wlh);
		_dispatch_wlh_release(cur_wlh);
		_dispatch_wlh_retain(wlh);
	}
	_dispatch_thread_setspecific(dispatch_wlh_key, (void *)wlh);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_preserve_wlh_storage_reference(dispatch_wlh_t wlh)
{
	dispatch_assert(wlh != DISPATCH_WLH_ANON);
	dispatch_assert(wlh == _dispatch_get_wlh());
	_dispatch_thread_setspecific(dispatch_wlh_key,
			(void *)((uintptr_t)wlh | DISPATCH_WLH_STORAGE_REF));
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_reset_wlh(void)
{
	dispatch_assert(_dispatch_get_wlh() == DISPATCH_WLH_ANON);
	_dispatch_debug("wlh[anon]: clear current");
	_dispatch_thread_setspecific(dispatch_wlh_key, NULL);
	_dispatch_clear_return_to_kernel();
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_wlh_should_poll_unote(dispatch_unote_t du)
{
	dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
	return _dispatch_needs_to_return_to_kernel() && ddi &&
			ddi->ddi_wlh != DISPATCH_WLH_ANON &&
			_dispatch_unote_wlh(du) == ddi->ddi_wlh;
}

#endif // DISPATCH_PURE_C
#ifndef __cplusplus

DISPATCH_ALWAYS_INLINE
static inline uint32_t
_dq_state_suspend_cnt(uint64_t dq_state)
{
	return (uint32_t)(dq_state / DISPATCH_QUEUE_SUSPEND_INTERVAL);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_has_side_suspend_cnt(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_HAS_SIDE_SUSPEND_CNT;
}

DISPATCH_ALWAYS_INLINE
static inline int32_t
_dq_state_extract_width_bits(uint64_t dq_state)
{
	dq_state &= DISPATCH_QUEUE_WIDTH_MASK;
	return (int32_t)(dq_state >> DISPATCH_QUEUE_WIDTH_SHIFT);
}

DISPATCH_ALWAYS_INLINE
static inline int32_t
_dq_state_available_width(uint64_t dq_state)
{
	int32_t full = DISPATCH_QUEUE_WIDTH_FULL;
	if (likely(!(dq_state & DISPATCH_QUEUE_WIDTH_FULL_BIT))) {
		return full - _dq_state_extract_width_bits(dq_state);
	}
	return 0;
}

DISPATCH_ALWAYS_INLINE
static inline int32_t
_dq_state_used_width(uint64_t dq_state, uint16_t dq_width)
{
	int32_t full = DISPATCH_QUEUE_WIDTH_FULL;
	int32_t width = _dq_state_extract_width_bits(dq_state);

	if (dq_state & DISPATCH_QUEUE_PENDING_BARRIER) {
		// DISPATCH_QUEUE_PENDING_BARRIER means (dq_width - 1) of the used width
		// is pre-reservation that we want to ignore
		return width - (full - dq_width) - (dq_width - 1);
	}
	return width - (full - dq_width);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_suspended(uint64_t dq_state)
{
	return dq_state >= DISPATCH_QUEUE_NEEDS_ACTIVATION;
}
#define DISPATCH_QUEUE_IS_SUSPENDED(x) \
		_dq_state_is_suspended(os_atomic_load2o(x, dq_state, relaxed))

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_inactive(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_INACTIVE;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_needs_activation(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_NEEDS_ACTIVATION;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_in_barrier(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_IN_BARRIER;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_has_available_width(uint64_t dq_state)
{
	return !(dq_state & DISPATCH_QUEUE_WIDTH_FULL_BIT);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_has_pending_barrier(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_PENDING_BARRIER;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_dirty(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_DIRTY;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_base_wlh(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_ROLE_BASE_WLH;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_base_anon(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_ROLE_BASE_ANON;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_inner_queue(uint64_t dq_state)
{
	return (dq_state & DISPATCH_QUEUE_ROLE_MASK) == DISPATCH_QUEUE_ROLE_INNER;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_enqueued(uint64_t dq_state)
{
	return dq_state & (DISPATCH_QUEUE_ENQUEUED|DISPATCH_QUEUE_ENQUEUED_ON_MGR);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_enqueued_on_target(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_ENQUEUED;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_enqueued_on_manager(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_ENQUEUED_ON_MGR;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_in_sync_transfer(uint64_t dq_state)
{
	return dq_state & DISPATCH_QUEUE_SYNC_TRANSFER;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_received_override(uint64_t dq_state)
{
	return _dq_state_is_base_anon(dq_state) &&
			(dq_state & DISPATCH_QUEUE_RECEIVED_OVERRIDE);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_received_sync_wait(uint64_t dq_state)
{
	return _dq_state_is_base_wlh(dq_state) &&
			(dq_state & DISPATCH_QUEUE_RECEIVED_SYNC_WAIT);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dq_state_max_qos(uint64_t dq_state)
{
	dq_state &= DISPATCH_QUEUE_MAX_QOS_MASK;
	return (dispatch_qos_t)(dq_state >> DISPATCH_QUEUE_MAX_QOS_SHIFT);
}

DISPATCH_ALWAYS_INLINE
static inline uint64_t
_dq_state_from_qos(dispatch_qos_t qos)
{
	return (uint64_t)(qos) << DISPATCH_QUEUE_MAX_QOS_SHIFT;
}

DISPATCH_ALWAYS_INLINE
static inline uint64_t
_dq_state_merge_qos(uint64_t dq_state, dispatch_qos_t qos)
{
	uint64_t qos_bits = _dq_state_from_qos(qos);
	if ((dq_state & DISPATCH_QUEUE_MAX_QOS_MASK) < qos_bits) {
		dq_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
		dq_state |= qos_bits;
		if (unlikely(_dq_state_is_base_anon(dq_state))) {
			dq_state |= DISPATCH_QUEUE_RECEIVED_OVERRIDE;
		}
	}
	return dq_state;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_tid
_dq_state_drain_owner(uint64_t dq_state)
{
	return _dispatch_lock_owner((dispatch_lock)dq_state);
}
#define DISPATCH_QUEUE_DRAIN_OWNER(dq) \
	_dq_state_drain_owner(os_atomic_load2o(dq, dq_state, relaxed))

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_drain_locked_by(uint64_t dq_state, dispatch_tid tid)
{
	return _dispatch_lock_is_locked_by((dispatch_lock)dq_state, tid);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_drain_locked_by_self(uint64_t dq_state)
{
	return _dispatch_lock_is_locked_by_self((dispatch_lock)dq_state);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_drain_locked(uint64_t dq_state)
{
	return _dispatch_lock_is_locked((dispatch_lock)dq_state);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_sync_runnable(uint64_t dq_state)
{
	return dq_state < DISPATCH_QUEUE_IN_BARRIER;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_is_runnable(uint64_t dq_state)
{
	return dq_state < DISPATCH_QUEUE_WIDTH_FULL_BIT;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_should_override(uint64_t dq_state)
{
	if (_dq_state_is_suspended(dq_state) ||
			_dq_state_is_enqueued_on_manager(dq_state)) {
		return false;
	}
	if (_dq_state_is_enqueued_on_target(dq_state)) {
		return true;
	}
	if (_dq_state_is_base_wlh(dq_state)) {
		return false;
	}
	return _dq_state_drain_locked(dq_state);
}


#endif // __cplusplus
#pragma mark -
#pragma mark dispatch_queue_t state machine

static inline pthread_priority_t _dispatch_get_priority(void);
static inline dispatch_priority_t _dispatch_get_basepri(void);
static inline dispatch_qos_t _dispatch_get_basepri_override_qos_floor(void);
static inline void _dispatch_set_basepri_override_qos(dispatch_qos_t qos);
static inline void _dispatch_reset_basepri(dispatch_priority_t dbp);
static inline dispatch_priority_t _dispatch_set_basepri(dispatch_priority_t dbp);

#if DISPATCH_PURE_C

// Note to later developers: ensure that any initialization changes are
// made for statically allocated queues (i.e. _dispatch_main_q).
static inline dispatch_queue_class_t
_dispatch_queue_init(dispatch_queue_class_t dqu, dispatch_queue_flags_t dqf,
		uint16_t width, uint64_t initial_state_bits)
{
	uint64_t dq_state = DISPATCH_QUEUE_STATE_INIT_VALUE(width);
	dispatch_queue_t dq = dqu._dq;

	dispatch_assert((initial_state_bits & ~(DISPATCH_QUEUE_ROLE_MASK |
			DISPATCH_QUEUE_INACTIVE)) == 0);

	if (initial_state_bits & DISPATCH_QUEUE_INACTIVE) {
		dq_state |= DISPATCH_QUEUE_INACTIVE + DISPATCH_QUEUE_NEEDS_ACTIVATION;
		dq->do_ref_cnt += 2; // rdar://8181908 see _dispatch_lane_resume
		if (dx_metatype(dq) == _DISPATCH_SOURCE_TYPE) {
			dq->do_ref_cnt++; // released when DSF_DELETED is set
		}
	}

	dq_state |= (initial_state_bits & DISPATCH_QUEUE_ROLE_MASK);
	dq->do_next = DISPATCH_OBJECT_LISTLESS;
	dqf |= DQF_WIDTH(width);
	os_atomic_store2o(dq, dq_atomic_flags, dqf, relaxed);
	dq->dq_state = dq_state;
	dq->dq_serialnum =
			os_atomic_inc_orig(&_dispatch_queue_serial_numbers, relaxed);
	return dqu;
}
#define _dispatch_queue_alloc(name, dqf, w, initial_state_bits) \
		_dispatch_queue_init(_dispatch_object_alloc(DISPATCH_VTABLE(name),\
				sizeof(struct dispatch_##name##_s)), dqf, w, initial_state_bits)

/* Used by:
 * - _dispatch_lane_set_target_queue
 * - changing dispatch source handlers
 *
 * Tries to prevent concurrent wakeup of an inactive queue by suspending it.
 */
DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline bool
_dispatch_lane_try_inactive_suspend(dispatch_lane_class_t dqu)
{
	uint64_t old_state, new_state;

	(void)os_atomic_rmw_loop2o(dqu._dl, dq_state, old_state, new_state, relaxed, {
		if (unlikely(!_dq_state_is_inactive(old_state))) {
			os_atomic_rmw_loop_give_up(return false);
		}
		new_state = old_state + DISPATCH_QUEUE_SUSPEND_INTERVAL;
	});
	if (unlikely(!_dq_state_is_suspended(old_state) ||
			_dq_state_has_side_suspend_cnt(old_state))) {
		// Crashing here means that 128+ dispatch_suspend() calls have been
		// made on an inactive object and then dispatch_set_target_queue() or
		// dispatch_set_*_handler() has been called.
		//
		// We don't want to handle the side suspend count in a codepath that
		// needs to be fast.
		DISPATCH_CLIENT_CRASH(0, "Too many calls to dispatch_suspend() "
				"prior to calling dispatch_set_target_queue() "
				"or dispatch_set_*_handler()");
	}
	return true;
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_needs_lock_override(uint64_t dq_state, dispatch_qos_t qos)
{
	return _dq_state_is_base_anon(dq_state) &&
			qos < _dq_state_max_qos(dq_state);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dispatch_queue_override_self(uint64_t dq_state)
{
	dispatch_qos_t qos = _dq_state_max_qos(dq_state);
	_dispatch_wqthread_override_start(_dispatch_tid_self(), qos);
	// ensure that the root queue sees
	// that this thread was overridden.
	_dispatch_set_basepri_override_qos(qos);
	return qos;
}

DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline uint64_t
_dispatch_queue_drain_try_lock(dispatch_queue_t dq,
		dispatch_invoke_flags_t flags)
{
	uint64_t pending_barrier_width =
			(dq->dq_width - 1) * DISPATCH_QUEUE_WIDTH_INTERVAL;
	uint64_t set_owner_and_set_full_width =
			_dispatch_lock_value_for_self() | DISPATCH_QUEUE_WIDTH_FULL_BIT;
	uint64_t lock_fail_mask, old_state, new_state, dequeue_mask;

	// same as !_dq_state_is_runnable()
	lock_fail_mask  = ~(DISPATCH_QUEUE_WIDTH_FULL_BIT - 1);
	// same as _dq_state_drain_locked()
	lock_fail_mask |= DISPATCH_QUEUE_DRAIN_OWNER_MASK;

	if (flags & DISPATCH_INVOKE_STEALING) {
		lock_fail_mask |= DISPATCH_QUEUE_ENQUEUED_ON_MGR;
		dequeue_mask = 0;
	} else if (flags & DISPATCH_INVOKE_MANAGER_DRAIN) {
		dequeue_mask = DISPATCH_QUEUE_ENQUEUED_ON_MGR;
	} else {
		lock_fail_mask |= DISPATCH_QUEUE_ENQUEUED_ON_MGR;
		dequeue_mask = DISPATCH_QUEUE_ENQUEUED;
	}
	dispatch_assert(!(flags & DISPATCH_INVOKE_WLH));

	dispatch_qos_t oq_floor = _dispatch_get_basepri_override_qos_floor();
retry:
	os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, acquire, {
		new_state = old_state;
		if (likely(!(old_state & lock_fail_mask))) {
			if (unlikely(_dq_state_needs_lock_override(old_state, oq_floor))) {
				os_atomic_rmw_loop_give_up({
					oq_floor = _dispatch_queue_override_self(old_state);
					goto retry;
				});
			}
			//
			// Only keep the HAS_WAITER, MAX_QOS and ENQUEUED bits
			// In particular acquiring the drain lock clears the DIRTY and
			// RECEIVED_OVERRIDE bits.
			//
			new_state &= DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK;
			new_state |= set_owner_and_set_full_width;
			if (_dq_state_has_pending_barrier(old_state) ||
					old_state + pending_barrier_width <
					DISPATCH_QUEUE_WIDTH_FULL_BIT) {
				new_state |= DISPATCH_QUEUE_IN_BARRIER;
			}
		} else if (dequeue_mask) {
			// dequeue_mask is in a register, xor yields better assembly
			new_state ^= dequeue_mask;
		} else {
			os_atomic_rmw_loop_give_up(break);
		}
	});

	dispatch_assert((old_state & dequeue_mask) == dequeue_mask);
	if (likely(!(old_state & lock_fail_mask))) {
		new_state &= DISPATCH_QUEUE_IN_BARRIER | DISPATCH_QUEUE_WIDTH_FULL_BIT |
				dequeue_mask;
		old_state &= DISPATCH_QUEUE_WIDTH_MASK;
		return new_state - old_state;
	}
	return 0;
}

DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline bool
_dispatch_queue_drain_try_lock_wlh(dispatch_queue_t dq, uint64_t *dq_state)
{
	uint64_t old_state, new_state;
	uint64_t lock_bits = _dispatch_lock_value_for_self() |
			DISPATCH_QUEUE_WIDTH_FULL_BIT | DISPATCH_QUEUE_IN_BARRIER;

	os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, acquire, {
		new_state = old_state;
		if (unlikely(_dq_state_is_suspended(old_state))) {
			new_state &= ~DISPATCH_QUEUE_ENQUEUED;
		} else if (unlikely(_dq_state_drain_locked(old_state))) {
			os_atomic_rmw_loop_give_up(break);
		} else {
			new_state &= DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK;
			new_state |= lock_bits;
		}
	});
	if (unlikely(!_dq_state_is_base_wlh(old_state) ||
			!_dq_state_is_enqueued_on_target(old_state) ||
			_dq_state_is_enqueued_on_manager(old_state))) {
#if DISPATCH_SIZEOF_PTR == 4
		old_state >>= 32;
#endif
		DISPATCH_INTERNAL_CRASH(old_state, "Invalid wlh state");
	}

	if (dq_state) *dq_state = new_state;
	return !_dq_state_is_suspended(old_state) &&
			!_dq_state_drain_locked(old_state);
}

/* Used by _dispatch_barrier_{try,}sync
 *
 * Note, this fails if any of e:1 or dl!=0, but that allows this code to be a
 * simple cmpxchg which is significantly faster on Intel, and makes a
 * significant difference on the uncontended codepath.
 *
 * See discussion for DISPATCH_QUEUE_DIRTY in queue_internal.h
 *
 * Initial state must be `completely idle`
 * Final state forces { ib:1, qf:1, w:0 }
 */
DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline bool
_dispatch_queue_try_acquire_barrier_sync_and_suspend(dispatch_lane_t dq,
		uint32_t tid, uint64_t suspend_count)
{
	uint64_t init  = DISPATCH_QUEUE_STATE_INIT_VALUE(dq->dq_width);
	uint64_t value = DISPATCH_QUEUE_WIDTH_FULL_BIT | DISPATCH_QUEUE_IN_BARRIER |
			_dispatch_lock_value_from_tid(tid) |
			(suspend_count * DISPATCH_QUEUE_SUSPEND_INTERVAL);
	uint64_t old_state, new_state;

	return os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, acquire, {
		uint64_t role = old_state & DISPATCH_QUEUE_ROLE_MASK;
		if (old_state != (init | role)) {
			os_atomic_rmw_loop_give_up(break);
		}
		new_state = value | role;
	});
}

DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline bool
_dispatch_queue_try_acquire_barrier_sync(dispatch_queue_class_t dq, uint32_t tid)
{
	return _dispatch_queue_try_acquire_barrier_sync_and_suspend(dq._dl, tid, 0);
}

/* Used by _dispatch_sync for root queues and some drain codepaths
 *
 * Root queues have no strict orderning and dispatch_sync() always goes through.
 * Drain is the sole setter of `dl` hence can use this non failing version of
 * _dispatch_queue_try_acquire_sync().
 *
 * Final state: { w += 1 }
 */
DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_reserve_sync_width(dispatch_lane_t dq)
{
	os_atomic_add2o(dq, dq_state, DISPATCH_QUEUE_WIDTH_INTERVAL, relaxed);
}

/* Used by _dispatch_sync on non-serial queues
 *
 * Initial state must be { sc:0, ib:0, pb:0, d:0 }
 * Final state: { w += 1 }
 */
DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline bool
_dispatch_queue_try_reserve_sync_width(dispatch_lane_t dq)
{
	uint64_t old_state, new_state;

	// <rdar://problem/24738102&24743140> reserving non barrier width
	// doesn't fail if only the ENQUEUED bit is set (unlike its barrier width
	// equivalent), so we have to check that this thread hasn't enqueued
	// anything ahead of this call or we can break ordering
	if (unlikely(dq->dq_items_tail)) {
		return false;
	}

	return os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, relaxed, {
		if (unlikely(!_dq_state_is_sync_runnable(old_state)) ||
				_dq_state_is_dirty(old_state) ||
				_dq_state_has_pending_barrier(old_state)) {
			os_atomic_rmw_loop_give_up(return false);
		}
		new_state = old_state + DISPATCH_QUEUE_WIDTH_INTERVAL;
	});
}

/* Used by target-queue recursing code
 *
 * Initial state must be { sc:0, ib:0, qf:0, pb:0, d:0 }
 * Final state: { w += 1 }
 */
DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline bool
_dispatch_queue_try_acquire_async(dispatch_lane_t dq)
{
	uint64_t old_state, new_state;

	return os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, acquire, {
		if (unlikely(!_dq_state_is_runnable(old_state) ||
				_dq_state_is_dirty(old_state) ||
				_dq_state_has_pending_barrier(old_state))) {
			os_atomic_rmw_loop_give_up(return false);
		}
		new_state = old_state + DISPATCH_QUEUE_WIDTH_INTERVAL;
	});
}

/* Used by concurrent drain
 *
 * Either acquires the full barrier width, in which case the Final state is:
 *   { ib:1 qf:1 pb:0 d:0 }
 * Or if there isn't enough width prepare the queue with the PENDING_BARRIER bit
 *   { ib:0 pb:1 d:0}
 *
 * This always clears the dirty bit as we know for sure we shouldn't reevaluate
 * the state machine here
 */
DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline bool
_dispatch_queue_try_upgrade_full_width(dispatch_lane_t dq, uint64_t owned)
{
	uint64_t old_state, new_state;
	uint64_t pending_barrier_width = DISPATCH_QUEUE_PENDING_BARRIER +
			(dq->dq_width - 1) * DISPATCH_QUEUE_WIDTH_INTERVAL;

	os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, acquire, {
		new_state = old_state - owned;
		if (likely(!_dq_state_has_pending_barrier(old_state))) {
			new_state += pending_barrier_width;
		}
		if (likely(_dq_state_is_runnable(new_state))) {
			new_state += DISPATCH_QUEUE_WIDTH_INTERVAL;
			new_state += DISPATCH_QUEUE_IN_BARRIER;
			new_state -= DISPATCH_QUEUE_PENDING_BARRIER;
		}
		new_state &= ~DISPATCH_QUEUE_DIRTY;
	});
	return new_state & DISPATCH_QUEUE_IN_BARRIER;
}

/* Used at the end of Drainers
 *
 * This adjusts the `owned` width when the next continuation is already known
 * to account for its barrierness.
 */
DISPATCH_ALWAYS_INLINE
static inline uint64_t
_dispatch_queue_adjust_owned(dispatch_queue_class_t dq, uint64_t owned,
		struct dispatch_object_s *next_dc)
{
	uint16_t dq_width = dq._dq->dq_width;
	uint64_t reservation;

	if (unlikely(dq_width > 1)) {
		if (next_dc && _dispatch_object_is_barrier(next_dc)) {
			reservation  = DISPATCH_QUEUE_PENDING_BARRIER;
			reservation += (dq_width - 1) * DISPATCH_QUEUE_WIDTH_INTERVAL;
			owned -= reservation;
		}
	}
	return owned;
}

/* Used at the end of Drainers
 *
 * Unlocking fails if the DIRTY bit is seen (and the queue is not suspended).
 * In that case, only the DIRTY bit is cleared. The DIRTY bit is therefore used
 * as a signal to renew the drain lock instead of releasing it.
 *
 * Successful unlock forces { dl:0, d:!done, qo:0 } and gives back `owned`
 */
DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline bool
_dispatch_queue_drain_try_unlock(dispatch_queue_t dq, uint64_t owned, bool done)
{
	uint64_t old_state, new_state;

	os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
		new_state  = old_state - owned;
		new_state &= ~DISPATCH_QUEUE_DRAIN_UNLOCK_MASK;
		if (unlikely(_dq_state_is_suspended(old_state))) {
			// nothing to do
		} else if (unlikely(_dq_state_is_dirty(old_state))) {
			os_atomic_rmw_loop_give_up({
				// just renew the drain lock with an acquire barrier, to see
				// what the enqueuer that set DIRTY has done.
				// the xor generates better assembly as DISPATCH_QUEUE_DIRTY
				// is already in a register
				os_atomic_xor2o(dq, dq_state, DISPATCH_QUEUE_DIRTY, acquire);
				return false;
			});
		} else if (likely(done)) {
			new_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
		} else {
			new_state |= DISPATCH_QUEUE_DIRTY;
		}
	});

	if (_dq_state_received_override(old_state)) {
		// Ensure that the root queue sees that this thread was overridden.
		_dispatch_set_basepri_override_qos(_dq_state_max_qos(old_state));
	}
	return true;
}

#pragma mark -
#pragma mark os_mpsc_queue

#define _os_mpsc_head(q, _ns, ...)   &(q)->_ns##_head ##__VA_ARGS__
#define _os_mpsc_tail(q, _ns, ...)   &(q)->_ns##_tail ##__VA_ARGS__

#define os_mpsc(q, _ns, ...)   (q, _ns, __VA_ARGS__)

#define os_mpsc_node_type(Q) _os_atomic_basetypeof(_os_mpsc_head Q)

//
// Multi Producer calls, can be used safely concurrently
//

// Returns true when the queue was empty and the head must be set
#define os_mpsc_push_update_tail(Q, tail, _o_next)  ({ \
		os_mpsc_node_type(Q) _tl = (tail); \
		os_atomic_store2o(_tl, _o_next, NULL, relaxed); \
		os_atomic_xchg(_os_mpsc_tail Q, _tl, release); \
	})

#define os_mpsc_push_was_empty(prev) ((prev) == NULL)

#define os_mpsc_push_update_prev(Q, prev, head, _o_next)  ({ \
		os_mpsc_node_type(Q) _prev = (prev); \
		if (likely(_prev)) { \
			(void)os_atomic_store2o(_prev, _o_next, (head), relaxed); \
		} else { \
			(void)os_atomic_store(_os_mpsc_head Q, (head), relaxed); \
		} \
	})

#define os_mpsc_push_list(Q, head, tail, _o_next)  ({ \
		os_mpsc_node_type(Q) _token; \
		_token = os_mpsc_push_update_tail(Q, tail, _o_next); \
		os_mpsc_push_update_prev(Q, _token, head, _o_next); \
		os_mpsc_push_was_empty(_token); \
	})

// Returns true when the queue was empty and the head must be set
#define os_mpsc_push_item(Q, tail, _o_next)  ({ \
		os_mpsc_node_type(Q) _tail = (tail); \
		os_mpsc_push_list(Q, _tail, _tail, _o_next); \
	})

//
// Single Consumer calls, can NOT be used safely concurrently
//

#define os_mpsc_looks_empty(Q) \
		(os_atomic_load(_os_mpsc_tail Q, relaxed) == NULL)

#define os_mpsc_get_head(Q)  ({ \
		__typeof__(_os_mpsc_head Q) __n = _os_mpsc_head Q; \
		os_mpsc_node_type(Q) _node; \
		_node = os_atomic_load(__n, dependency); \
		if (unlikely(_node == NULL)) { \
			_node = _dispatch_wait_for_enqueuer((void **)__n); \
		} \
		_node; \
	})

#define os_mpsc_get_next(_n, _o_next)  ({ \
		__typeof__(_n) __n = (_n); \
		_os_atomic_basetypeof(&__n->_o_next) _node; \
		_node = os_atomic_load(&__n->_o_next, dependency); \
		if (unlikely(_node == NULL)) { \
			_node = _dispatch_wait_for_enqueuer((void **)&__n->_o_next); \
		} \
		_node; \
	})

#define os_mpsc_pop_head(Q, head, _o_next)  ({ \
		os_mpsc_node_type(Q) _head = (head), _n; \
		_n = os_atomic_load2o(_head, _o_next, dependency); \
		os_atomic_store(_os_mpsc_head Q, _n, relaxed); \
		/* 22708742: set tail to NULL with release, so that NULL write */ \
		/* to head above doesn't clobber head from concurrent enqueuer */ \
		if (unlikely(!_n && \
				!os_atomic_cmpxchg(_os_mpsc_tail Q, _head, NULL, release))) { \
			_n = os_mpsc_get_next(_head, _o_next); \
			os_atomic_store(_os_mpsc_head Q, _n, relaxed); \
		} \
		_n; \
	})

#define os_mpsc_undo_pop_list(Q, head, tail, next, _o_next)  ({ \
		os_mpsc_node_type(Q) _hd = (head), _tl = (tail), _n = (next); \
		os_atomic_store2o(_tl, _o_next, _n, relaxed); \
		if (unlikely(!_n && \
				!os_atomic_cmpxchg(_os_mpsc_tail Q, NULL, _tl, release))) { \
			_n = os_mpsc_get_head(Q); \
			os_atomic_store2o(_tl, _o_next, _n, relaxed); \
		} \
		os_atomic_store(_os_mpsc_head Q, _hd, relaxed); \
	})

#define os_mpsc_undo_pop_head(Q, head, next, _o_next) ({ \
		os_mpsc_node_type(Q) _head = (head); \
		os_mpsc_undo_pop_list(Q, _head, _head, next, _o_next); \
	})

#define os_mpsc_capture_snapshot(Q, tail)  ({ \
		os_mpsc_node_type(Q) _head = os_mpsc_get_head(Q); \
		os_atomic_store(_os_mpsc_head Q, NULL, relaxed); \
		/* 22708742: set tail to NULL with release, so that NULL write */ \
		/* to head above doesn't clobber head from concurrent enqueuer */ \
		*(tail) = os_atomic_xchg(_os_mpsc_tail Q, NULL, release); \
		_head; \
	})

#define os_mpsc_pop_snapshot_head(head, tail, _o_next) ({ \
		__typeof__(head) _head = (head), _tail = (tail), _n = NULL; \
		if (_head != _tail) _n = os_mpsc_get_next(_head, _o_next); \
		_n; \
	})

#define os_mpsc_prepend(Q, head, tail, _o_next)  ({ \
		os_mpsc_node_type(Q) _n = os_atomic_load(_os_mpsc_head Q, relaxed); \
		os_mpsc_undo_pop_list(Q, head, tail, _n, _o_next); \
	})

#pragma mark -
#pragma mark dispatch_queue_t tq lock

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_queue_sidelock_trylock(dispatch_lane_t dq, dispatch_qos_t qos)
{
	dispatch_tid owner;
	if (_dispatch_unfair_lock_trylock(&dq->dq_sidelock, &owner)) {
		return true;
	}
	_dispatch_wqthread_override_start_check_owner(owner, qos,
			&dq->dq_sidelock.dul_lock);
	return false;
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_sidelock_lock(dispatch_lane_t dq)
{
	return _dispatch_unfair_lock_lock(&dq->dq_sidelock);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_queue_sidelock_tryunlock(dispatch_lane_t dq)
{
	if (_dispatch_unfair_lock_tryunlock(&dq->dq_sidelock)) {
		return true;
	}
	// Ensure that the root queue sees that this thread was overridden.
	// Since we don't know which override QoS was used, use MAINTENANCE
	// as a marker for _dispatch_reset_basepri_override()
	_dispatch_set_basepri_override_qos(DISPATCH_QOS_MAINTENANCE);
	return false;
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_sidelock_unlock(dispatch_lane_t dq)
{
	if (_dispatch_unfair_lock_unlock_had_failed_trylock(&dq->dq_sidelock)) {
		// Ensure that the root queue sees that this thread was overridden.
		// Since we don't know which override QoS was used, use MAINTENANCE
		// as a marker for _dispatch_reset_basepri_override()
		_dispatch_set_basepri_override_qos(DISPATCH_QOS_MAINTENANCE);
	}
}

#pragma mark -
#pragma mark dispatch_queue_t misc

DISPATCH_ALWAYS_INLINE
static inline dispatch_queue_t
_dispatch_queue_get_current(void)
{
	return (dispatch_queue_t)_dispatch_thread_getspecific(dispatch_queue_key);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_queue_t
_dispatch_queue_get_current_or_default(void)
{
	int idx = DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT;
	return _dispatch_queue_get_current() ?: _dispatch_root_queues[idx]._as_dq;
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_set_current(dispatch_queue_class_t dqu)
{
	_dispatch_thread_setspecific(dispatch_queue_key, dqu._dq);
}

DISPATCH_ALWAYS_INLINE
static inline struct dispatch_object_s*
_dispatch_queue_get_head(dispatch_lane_class_t dq)
{
	return os_mpsc_get_head(os_mpsc(dq._dl, dq_items));
}

DISPATCH_ALWAYS_INLINE
static inline struct dispatch_object_s*
_dispatch_queue_pop_head(dispatch_lane_class_t dq, struct dispatch_object_s *dc)
{
	return os_mpsc_pop_head(os_mpsc(dq._dl, dq_items), dc, do_next);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_queue_push_item(dispatch_lane_class_t dqu, dispatch_object_t dou)
{
	return os_mpsc_push_item(os_mpsc(dqu._dl, dq_items), dou._do, do_next);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_root_queue_push_inline(dispatch_queue_global_t dq,
		dispatch_object_t _head, dispatch_object_t _tail, int n)
{
	struct dispatch_object_s *hd = _head._do, *tl = _tail._do;
	if (unlikely(os_mpsc_push_list(os_mpsc(dq, dq_items), hd, tl, do_next))) {
		return _dispatch_root_queue_poke(dq, n, 0);
	}
}

#include "trace.h"

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_push_queue(dispatch_queue_t tq, dispatch_queue_class_t dq,
		uint64_t dq_state)
{
#if DISPATCH_USE_KEVENT_WORKLOOP
	if (likely(_dq_state_is_base_wlh(dq_state))) {
		_dispatch_trace_runtime_event(worker_request, dq._dq, 1);
		return _dispatch_event_loop_poke((dispatch_wlh_t)dq._dq, dq_state,
				DISPATCH_EVENT_LOOP_CONSUME_2);
	}
#endif // DISPATCH_USE_KEVENT_WORKLOOP
	_dispatch_trace_item_push(tq, dq);
	return dx_push(tq, dq, _dq_state_max_qos(dq_state));
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_priority_t
_dispatch_root_queue_identity_assume(dispatch_queue_global_t assumed_rq)
{
	dispatch_priority_t old_dbp = _dispatch_get_basepri();
	dispatch_assert(dx_hastypeflag(assumed_rq, QUEUE_ROOT));
	_dispatch_reset_basepri(assumed_rq->dq_priority);
	_dispatch_queue_set_current(assumed_rq);
	return old_dbp;
}

typedef dispatch_queue_wakeup_target_t
_dispatch_queue_class_invoke_handler_t(dispatch_queue_class_t,
		dispatch_invoke_context_t dic, dispatch_invoke_flags_t,
		uint64_t *owned);

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_class_invoke(dispatch_queue_class_t dqu,
		dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags,
		dispatch_invoke_flags_t const_restrict_flags,
		_dispatch_queue_class_invoke_handler_t invoke)
{
	dispatch_queue_t dq = dqu._dq;
	dispatch_queue_wakeup_target_t tq = DISPATCH_QUEUE_WAKEUP_NONE;
	bool owning = !(flags & DISPATCH_INVOKE_STEALING);
	uint64_t owned = 0;

	// When called from a plain _dispatch_queue_drain:
	//   overriding = false
	//   owning = true
	//
	// When called from an override continuation:
	//   overriding = true
	//   owning depends on whether the override embedded the queue or steals

	if (!(flags & (DISPATCH_INVOKE_STEALING | DISPATCH_INVOKE_WLH))) {
		dq->do_next = DISPATCH_OBJECT_LISTLESS;
		_dispatch_trace_item_pop(_dispatch_queue_get_current(), dq);
	}
	flags |= const_restrict_flags;
	if (likely(flags & DISPATCH_INVOKE_WLH)) {
		owned = DISPATCH_QUEUE_SERIAL_DRAIN_OWNED | DISPATCH_QUEUE_ENQUEUED;
	} else {
		owned = _dispatch_queue_drain_try_lock(dq, flags);
	}
	if (likely(owned)) {
		dispatch_priority_t old_dbp;
		if (!(flags & DISPATCH_INVOKE_MANAGER_DRAIN)) {
			old_dbp = _dispatch_set_basepri(dq->dq_priority);
		} else {
			old_dbp = 0;
		}
		if (flags & DISPATCH_INVOKE_WORKLOOP_DRAIN) {
			if (unlikely(_dispatch_queue_atomic_flags(dqu) & DQF_MUTABLE)) {
				_dispatch_queue_atomic_flags_clear(dqu, DQF_MUTABLE);
			}
		}

		flags = _dispatch_queue_merge_autorelease_frequency(dq, flags);
attempt_running_slow_head:
#if DISPATCH_COCOA_COMPAT
		if ((flags & DISPATCH_INVOKE_WLH) &&
				!(flags & DISPATCH_INVOKE_AUTORELEASE_ALWAYS)) {
			_dispatch_last_resort_autorelease_pool_push(dic);
		}
#endif // DISPATCH_COCOA_COMPAT
		tq = invoke(dq, dic, flags, &owned);
#if DISPATCH_COCOA_COMPAT
		if ((flags & DISPATCH_INVOKE_WLH) &&
				!(flags & DISPATCH_INVOKE_AUTORELEASE_ALWAYS)) {
			dispatch_thread_frame_s dtf;
			_dispatch_thread_frame_push(&dtf, dq);
			_dispatch_last_resort_autorelease_pool_pop(dic);
			_dispatch_thread_frame_pop(&dtf);
		}
#endif // DISPATCH_COCOA_COMPAT
		dispatch_assert(tq != DISPATCH_QUEUE_WAKEUP_TARGET);
		if (unlikely(tq != DISPATCH_QUEUE_WAKEUP_NONE &&
				tq != DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT)) {
			// Either dc is set, which is a deferred invoke case
			//
			// or only tq is and it means a reenqueue is required, because of:
			// a retarget, a suspension, or a width change.
			//
			// In both cases, we want to bypass the check for DIRTY.
			// That may cause us to leave DIRTY in place but all drain lock
			// acquirers clear it
		} else if (!_dispatch_queue_drain_try_unlock(dq, owned,
				tq == DISPATCH_QUEUE_WAKEUP_NONE)) {
			tq = _dispatch_queue_get_current();
			if (dx_hastypeflag(tq, QUEUE_ROOT) || !owning) {
				goto attempt_running_slow_head;
			}
			DISPATCH_COMPILER_CAN_ASSUME(tq != DISPATCH_QUEUE_WAKEUP_NONE);
		} else {
			owned = 0;
			tq = NULL;
		}
		if (!(flags & DISPATCH_INVOKE_MANAGER_DRAIN)) {
			_dispatch_reset_basepri(old_dbp);
		}
	}
	if (likely(owning)) {
		_dispatch_trace_item_complete(dq);
	}

	if (tq) {
		return _dispatch_queue_invoke_finish(dq, dic, tq, owned);
	}

	return _dispatch_release_2_tailcall(dq);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_queue_class_probe(dispatch_lane_class_t dqu)
{
	struct dispatch_object_s *tail;
	// seq_cst wrt atomic store to dq_state <rdar://problem/14637483>
	// seq_cst wrt atomic store to dq_flags <rdar://problem/22623242>
	tail = os_atomic_load2o(dqu._dl, dq_items_tail, ordered);
	return unlikely(tail != NULL);
}

DISPATCH_ALWAYS_INLINE DISPATCH_CONST
static inline bool
_dispatch_is_in_root_queues_array(dispatch_queue_class_t dqu)
{
	return (dqu._dgq >= _dispatch_root_queues) &&
			(dqu._dgq < _dispatch_root_queues + _DISPATCH_ROOT_QUEUE_IDX_COUNT);
}

DISPATCH_ALWAYS_INLINE DISPATCH_CONST
static inline dispatch_queue_global_t
_dispatch_get_root_queue(dispatch_qos_t qos, bool overcommit)
{
	if (unlikely(qos < DISPATCH_QOS_MIN || qos > DISPATCH_QOS_MAX)) {
		DISPATCH_CLIENT_CRASH(qos, "Corrupted priority");
	}
	return &_dispatch_root_queues[2 * (qos - 1) + overcommit];
}

#define _dispatch_get_default_queue(overcommit) \
		_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS + \
				!!(overcommit)]._as_dq

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_set_bound_thread(dispatch_queue_class_t dqu)
{
	// Tag thread-bound queues with the owning thread
	dispatch_assert(_dispatch_queue_is_thread_bound(dqu));
	uint64_t old_state, new_state;
	os_atomic_rmw_loop2o(dqu._dq, dq_state, old_state, new_state, relaxed, {
		new_state = old_state;
		new_state &= ~DISPATCH_QUEUE_DRAIN_OWNER_MASK;
		new_state |= _dispatch_lock_value_for_self();
	});
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_clear_bound_thread(dispatch_queue_class_t dqu)
{
	dispatch_assert(_dispatch_queue_is_thread_bound(dqu));
	os_atomic_and2o(dqu._dq, dq_state,
			~DISPATCH_QUEUE_DRAIN_OWNER_MASK, relaxed);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_pthread_root_queue_observer_hooks_t
_dispatch_get_pthread_root_queue_observer_hooks(void)
{
	return _dispatch_thread_getspecific(
			dispatch_pthread_root_queue_observer_hooks_key);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_set_pthread_root_queue_observer_hooks(
		dispatch_pthread_root_queue_observer_hooks_t observer_hooks)
{
	_dispatch_thread_setspecific(dispatch_pthread_root_queue_observer_hooks_key,
			observer_hooks);
}

#pragma mark -
#pragma mark dispatch_priority

DISPATCH_ALWAYS_INLINE
static inline dispatch_priority_t
_dispatch_get_basepri(void)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	return (dispatch_priority_t)(uintptr_t)_dispatch_thread_getspecific(
			dispatch_basepri_key);
#else
	return 0;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_reset_basepri(dispatch_priority_t dbp)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	dispatch_priority_t old_dbp = _dispatch_get_basepri();
	// If an inner-loop or'd in the override flag to the per-thread priority,
	// it needs to be propagated up the chain.
	dbp &= ~DISPATCH_PRIORITY_OVERRIDE_MASK;
	dbp |= (old_dbp & DISPATCH_PRIORITY_OVERRIDE_MASK);
	_dispatch_thread_setspecific(dispatch_basepri_key, (void*)(uintptr_t)dbp);
#else
	(void)dbp;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dispatch_get_basepri_override_qos_floor(void)
{
	dispatch_priority_t dbp = _dispatch_get_basepri();
	dispatch_qos_t qos = _dispatch_priority_qos(dbp);
	dispatch_qos_t oqos = _dispatch_priority_override_qos(dbp);
	return MAX(qos, oqos);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_set_basepri_override_qos(dispatch_qos_t qos)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	dispatch_priority_t dbp = _dispatch_get_basepri();
	if (_dispatch_priority_override_qos(dbp) >= qos) return;
	dbp &= ~DISPATCH_PRIORITY_OVERRIDE_MASK;
	dbp |= qos << DISPATCH_PRIORITY_OVERRIDE_SHIFT;
	_dispatch_thread_setspecific(dispatch_basepri_key, (void*)(uintptr_t)dbp);
#else
	(void)qos;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_reset_basepri_override(void)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	dispatch_priority_t dbp = _dispatch_get_basepri();
	dispatch_qos_t oqos = _dispatch_priority_override_qos(dbp);
	if (oqos) {
		dbp &= ~DISPATCH_PRIORITY_OVERRIDE_MASK;
		_dispatch_thread_setspecific(dispatch_basepri_key, (void*)(uintptr_t)dbp);
		return oqos != DISPATCH_QOS_SATURATED;
	}
#endif
	return false;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_priority_t
_dispatch_set_basepri(dispatch_priority_t dq_dbp)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	dispatch_priority_t old_dbp = _dispatch_get_basepri();
	dispatch_priority_t dbp = old_dbp;

	if (unlikely(!old_dbp)) {
		dbp = dq_dbp & ~DISPATCH_PRIORITY_OVERRIDE_MASK;
	} else if (dq_dbp & DISPATCH_PRIORITY_REQUESTED_MASK) {
		dbp &= (DISPATCH_PRIORITY_OVERRIDE_MASK |
				DISPATCH_PRIORITY_FLAG_OVERCOMMIT);
		dbp |= MAX(old_dbp & DISPATCH_PRIORITY_REQUESTED_MASK,
				dq_dbp & DISPATCH_PRIORITY_REQUESTED_MASK);
		if (_dispatch_priority_fallback_qos(dq_dbp) >
				_dispatch_priority_qos(dbp)) {
			dq_dbp &= (DISPATCH_PRIORITY_FALLBACK_QOS_MASK |
					DISPATCH_PRIORITY_FLAG_FALLBACK |
					DISPATCH_PRIORITY_FLAG_FLOOR);
		} else {
			dq_dbp &= DISPATCH_PRIORITY_FLAG_FLOOR;
		}
		dbp |= dq_dbp;
	} else {
		if (dbp & DISPATCH_PRIORITY_REQUESTED_MASK) {
			dbp |= DISPATCH_PRIORITY_FLAG_FLOOR;
		}
		if (_dispatch_priority_fallback_qos(dq_dbp) >
				_dispatch_priority_qos(dbp)) {
			dbp &= ~DISPATCH_PRIORITY_FALLBACK_QOS_MASK;
			dbp |= (dq_dbp & (DISPATCH_PRIORITY_FALLBACK_QOS_MASK |
					DISPATCH_PRIORITY_FLAG_FALLBACK));
		}
	}
	_dispatch_thread_setspecific(dispatch_basepri_key, (void*)(uintptr_t)dbp);
	return old_dbp;
#else
	(void)dq_dbp;
	return 0;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_init_basepri(dispatch_priority_t dbp)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	dispatch_assert(!_dispatch_get_basepri());
	_dispatch_thread_setspecific(dispatch_basepri_key, (void*)(uintptr_t)dbp);
#else
	(void)dbp;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_init_basepri_wlh(dispatch_priority_t dbp)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	dispatch_assert(!_dispatch_get_basepri());
	dbp |= _dispatch_priority_make_override(DISPATCH_QOS_SATURATED);
	_dispatch_thread_setspecific(dispatch_basepri_key, (void*)(uintptr_t)dbp);
#else
	(void)dbp;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_clear_basepri(void)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	_dispatch_thread_setspecific(dispatch_basepri_key, (void*)(uintptr_t)0);
#endif
}

DISPATCH_ALWAYS_INLINE
static inline pthread_priority_t
_dispatch_priority_adopt(pthread_priority_t pp, unsigned long flags)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	dispatch_priority_t dbp = _dispatch_get_basepri();
	pthread_priority_t basepp = _dispatch_priority_to_pp_strip_flags(dbp);
	pthread_priority_t minbasepp = basepp &
			~(pthread_priority_t)_PTHREAD_PRIORITY_PRIORITY_MASK;
	bool enforce = (flags & DISPATCH_PRIORITY_ENFORCE) ||
			(pp & _PTHREAD_PRIORITY_ENFORCE_FLAG);
	pp &= ~_PTHREAD_PRIORITY_FLAGS_MASK;

	if (unlikely(!pp)) {
		dispatch_qos_t fallback = _dispatch_priority_fallback_qos(dbp);
		return fallback ? _dispatch_qos_to_pp(fallback) : basepp;
	} else if (pp < minbasepp) {
		return basepp;
	} else if (enforce || (dbp & (DISPATCH_PRIORITY_FLAG_FLOOR |
			DISPATCH_PRIORITY_FLAG_FALLBACK))) {
		return pp;
	} else {
		return basepp;
	}
#else
	(void)pp; (void)flags;
	return 0;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline pthread_priority_t
_dispatch_get_priority(void)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	pthread_priority_t pp = (uintptr_t)
			_dispatch_thread_getspecific(dispatch_priority_key);
	return pp;
#else
	return 0;
#endif
}

#if HAVE_PTHREAD_WORKQUEUE_QOS
DISPATCH_ALWAYS_INLINE
static inline pthread_priority_t
_dispatch_priority_compute_update(pthread_priority_t pp)
{
	dispatch_assert(pp != DISPATCH_NO_PRIORITY);
	if (!_dispatch_set_qos_class_enabled) return 0;
	// the priority in _dispatch_get_priority() only tracks manager-ness
	// and overcommit, which is inherited from the current value for each update
	// however if the priority had the NEEDS_UNBIND flag set we need to clear it
	// the first chance we get
	//
	// the manager bit is invalid input, but we keep it to get meaningful
	// assertions in _dispatch_set_priority_and_voucher_slow()
	pp &= _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG | ~_PTHREAD_PRIORITY_FLAGS_MASK;
	pthread_priority_t cur_priority = _dispatch_get_priority();
	pthread_priority_t unbind = _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG;
	pthread_priority_t overcommit = _PTHREAD_PRIORITY_OVERCOMMIT_FLAG;
	if (unlikely(cur_priority & unbind)) {
		// else we always need an update if the NEEDS_UNBIND flag is set
		// the slow path in _dispatch_set_priority_and_voucher_slow() will
		// adjust the priority further with the proper overcommitness
		return pp ? pp : (cur_priority & ~unbind);
	} else {
		cur_priority &= ~overcommit;
	}
	if (unlikely(pp != cur_priority)) return pp;
	return 0;
}
#endif

DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline voucher_t
_dispatch_set_priority_and_voucher(pthread_priority_t pp,
		voucher_t v, dispatch_thread_set_self_t flags)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	pp = _dispatch_priority_compute_update(pp);
	if (likely(!pp)) {
		if (v == DISPATCH_NO_VOUCHER) {
			return DISPATCH_NO_VOUCHER;
		}
		if (likely(v == _voucher_get())) {
			bool retained = flags & DISPATCH_VOUCHER_CONSUME;
			if (flags & DISPATCH_VOUCHER_REPLACE) {
				if (retained && v) _voucher_release_no_dispose(v);
				v = DISPATCH_NO_VOUCHER;
			} else {
				if (!retained && v) _voucher_retain(v);
			}
			return v;
		}
	}
	return _dispatch_set_priority_and_voucher_slow(pp, v, flags);
#else
	(void)pp; (void)v; (void)flags;
	return DISPATCH_NO_VOUCHER;
#endif
}

DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline voucher_t
_dispatch_adopt_priority_and_set_voucher(pthread_priority_t pp,
		voucher_t v, dispatch_thread_set_self_t flags)
{
	pthread_priority_t p = 0;
	if (pp != DISPATCH_NO_PRIORITY) {
		p = _dispatch_priority_adopt(pp, flags);
	}
	return _dispatch_set_priority_and_voucher(p, v, flags);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_reset_priority_and_voucher(pthread_priority_t pp, voucher_t v)
{
	if (pp == DISPATCH_NO_PRIORITY) pp = 0;
	(void)_dispatch_set_priority_and_voucher(pp, v,
			DISPATCH_VOUCHER_CONSUME | DISPATCH_VOUCHER_REPLACE);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_reset_voucher(voucher_t v, dispatch_thread_set_self_t flags)
{
	flags |= DISPATCH_VOUCHER_CONSUME | DISPATCH_VOUCHER_REPLACE;
	(void)_dispatch_set_priority_and_voucher(0, v, flags);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dispatch_queue_push_qos(dispatch_queue_class_t dq, dispatch_qos_t qos)
{
	if (qos > _dispatch_priority_qos(dq._dl->dq_priority)) {
		return qos;
	}
	return DISPATCH_QOS_UNSPECIFIED;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dispatch_queue_wakeup_qos(dispatch_queue_class_t dq, dispatch_qos_t qos)
{
	if (!qos) qos = _dispatch_priority_fallback_qos(dq._dl->dq_priority);
	// for asynchronous workitems, queue priority is the floor for overrides
	return MAX(qos, _dispatch_priority_qos(dq._dl->dq_priority));
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dispatch_queue_max_qos(dispatch_queue_class_t dq)
{
	// Note: the non atomic load allows to avoid CAS on 32bit architectures
	//       which doesn't give us much as the bits we want are in a single byte
	//       and can't quite be read non atomically. Given that this function is
	//       called in various critical codepaths (such as _dispatch_lane_push()
	//       between the tail exchange and updating the `prev` pointer), we care
	//       deeply about avoiding this.
	return _dq_state_max_qos((uint64_t)dq._dl->dq_state_bits << 32);
}

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_queue_need_override(dispatch_queue_class_t dq, dispatch_qos_t qos)
{
	dispatch_qos_t max_qos = _dispatch_queue_max_qos(dq);
	return max_qos == DISPATCH_QOS_UNSPECIFIED || max_qos < qos;
}

#define DISPATCH_PRIORITY_PROPAGATE_CURRENT 0x1
#define DISPATCH_PRIORITY_PROPAGATE_FOR_SYNC_IPC 0x2

DISPATCH_ALWAYS_INLINE
static inline pthread_priority_t
_dispatch_priority_compute_propagated(pthread_priority_t pp,
		unsigned int flags)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	if (flags & DISPATCH_PRIORITY_PROPAGATE_CURRENT) {
		pp = _dispatch_get_priority();
	}
	pp &= ~_PTHREAD_PRIORITY_FLAGS_MASK;
	if (!(flags & DISPATCH_PRIORITY_PROPAGATE_FOR_SYNC_IPC) &&
			pp > _dispatch_qos_to_pp(DISPATCH_QOS_USER_INITIATED)) {
		// Cap QOS for propagation at user-initiated <rdar://16681262&16998036>
		return _dispatch_qos_to_pp(DISPATCH_QOS_USER_INITIATED);
	}
	return pp;
#else
	(void)pp; (void)flags;
	return 0;
#endif
}

DISPATCH_ALWAYS_INLINE
static inline pthread_priority_t
_dispatch_priority_propagate(void)
{
	return _dispatch_priority_compute_propagated(0,
			DISPATCH_PRIORITY_PROPAGATE_CURRENT);
}

// including maintenance
DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_is_background_thread(void)
{
#if HAVE_PTHREAD_WORKQUEUE_QOS
	pthread_priority_t pp = _dispatch_get_priority();
	return _dispatch_qos_is_background(_dispatch_qos_from_pp(pp));
#else
	return false;
#endif
}

#pragma mark -
#pragma mark dispatch_block_t

#ifdef __BLOCKS__

DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_block_has_private_data(const dispatch_block_t block)
{
	return (_dispatch_Block_invoke(block) == _dispatch_block_special_invoke);
}

DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline pthread_priority_t
_dispatch_block_invoke_should_set_priority(dispatch_block_flags_t flags,
		pthread_priority_t new_pri)
{
	pthread_priority_t old_pri, p = 0;  // 0 means do not change priority.
	if ((flags & DISPATCH_BLOCK_HAS_PRIORITY)
			&& ((flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS) ||
			!(flags & DISPATCH_BLOCK_INHERIT_QOS_CLASS))) {
		new_pri &= ~_PTHREAD_PRIORITY_FLAGS_MASK;
		old_pri = _dispatch_get_priority() & ~_PTHREAD_PRIORITY_FLAGS_MASK;
		if (old_pri && old_pri < new_pri) p = old_pri;
	}
	return p;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_block_private_data_t
_dispatch_block_get_data(const dispatch_block_t db)
{
	if (!_dispatch_block_has_private_data(db)) {
		return NULL;
	}
	// Keep in sync with _dispatch_block_create implementation
	uint8_t *x = (uint8_t *)db;
	// x points to base of struct Block_layout
	x += sizeof(struct Block_layout);
	// x points to base of captured dispatch_block_private_data_s object
	dispatch_block_private_data_t dbpd = (dispatch_block_private_data_t)x;
	if (dbpd->dbpd_magic != DISPATCH_BLOCK_PRIVATE_DATA_MAGIC) {
		DISPATCH_CLIENT_CRASH(dbpd->dbpd_magic,
				"Corruption of dispatch block object");
	}
	return dbpd;
}

DISPATCH_ALWAYS_INLINE
static inline pthread_priority_t
_dispatch_block_get_priority(const dispatch_block_t db)
{
	dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db);
	return dbpd ? dbpd->dbpd_priority : 0;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_block_flags_t
_dispatch_block_get_flags(const dispatch_block_t db)
{
	dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db);
	return dbpd ? dbpd->dbpd_flags : 0;
}

#endif

#pragma mark -
#pragma mark dispatch_continuation_t

DISPATCH_ALWAYS_INLINE
static inline dispatch_continuation_t
_dispatch_continuation_alloc_cacheonly(void)
{
	dispatch_continuation_t dc = (dispatch_continuation_t)
			_dispatch_thread_getspecific(dispatch_cache_key);
	if (likely(dc)) {
		_dispatch_thread_setspecific(dispatch_cache_key, dc->do_next);
	}
	return dc;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_continuation_t
_dispatch_continuation_alloc(void)
{
	dispatch_continuation_t dc =
			_dispatch_continuation_alloc_cacheonly();
	if (unlikely(!dc)) {
		return _dispatch_continuation_alloc_from_heap();
	}
	return dc;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_continuation_t
_dispatch_continuation_free_cacheonly(dispatch_continuation_t dc)
{
	dispatch_continuation_t prev_dc = (dispatch_continuation_t)
			_dispatch_thread_getspecific(dispatch_cache_key);
	int cnt = prev_dc ? prev_dc->dc_cache_cnt + 1 : 1;
	// Cap continuation cache
	if (unlikely(cnt > _dispatch_continuation_cache_limit)) {
		return dc;
	}
	dc->do_next = prev_dc;
	dc->dc_cache_cnt = cnt;
#if DISPATCH_ALLOCATOR
	// This magical value helps memory tools to recognize continuations on
	// the various free lists that are really free.
	dc->dc_flags = (uintptr_t)(void *)&_dispatch_main_heap;
#endif
	_dispatch_thread_setspecific(dispatch_cache_key, dc);
	return NULL;
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_continuation_free(dispatch_continuation_t dc)
{
	dc = _dispatch_continuation_free_cacheonly(dc);
	if (unlikely(dc)) {
		_dispatch_continuation_free_to_cache_limit(dc);
	}
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_continuation_with_group_invoke(dispatch_continuation_t dc)
{
	struct dispatch_object_s *dou = dc->dc_data;
	unsigned long type = dx_type(dou);
	if (type == DISPATCH_GROUP_TYPE) {
		_dispatch_client_callout(dc->dc_ctxt, dc->dc_func);
		_dispatch_trace_item_complete(dc);
		dispatch_group_leave((dispatch_group_t)dou);
	} else {
		DISPATCH_INTERNAL_CRASH(dx_type(dou), "Unexpected object type");
	}
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_continuation_invoke_inline(dispatch_object_t dou,
		dispatch_invoke_flags_t flags, dispatch_queue_class_t dqu)
{
	dispatch_continuation_t dc = dou._dc, dc1;
	dispatch_invoke_with_autoreleasepool(flags, {
		uintptr_t dc_flags = dc->dc_flags;
		// Add the item back to the cache before calling the function. This
		// allows the 'hot' continuation to be used for a quick callback.
		//
		// The ccache version is per-thread.
		// Therefore, the object has not been reused yet.
		// This generates better assembly.
		_dispatch_continuation_voucher_adopt(dc, dc_flags);
		if (!(dc_flags & DC_FLAG_NO_INTROSPECTION)) {
			_dispatch_trace_item_pop(dqu, dou);
		}
		if (dc_flags & DC_FLAG_CONSUME) {
			dc1 = _dispatch_continuation_free_cacheonly(dc);
		} else {
			dc1 = NULL;
		}
		if (unlikely(dc_flags & DC_FLAG_GROUP_ASYNC)) {
			_dispatch_continuation_with_group_invoke(dc);
		} else {
			_dispatch_client_callout(dc->dc_ctxt, dc->dc_func);
			_dispatch_trace_item_complete(dc);
		}
		if (unlikely(dc1)) {
			_dispatch_continuation_free_to_cache_limit(dc1);
		}
	});
	_dispatch_perfmon_workitem_inc();
}

DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_continuation_pop_inline(dispatch_object_t dou,
		dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags,
		dispatch_queue_class_t dqu)
{
	dispatch_pthread_root_queue_observer_hooks_t observer_hooks =
			_dispatch_get_pthread_root_queue_observer_hooks();
	if (observer_hooks) observer_hooks->queue_will_execute(dqu._dq);
	flags &= _DISPATCH_INVOKE_PROPAGATE_MASK;
	if (_dispatch_object_has_vtable(dou)) {
		dx_invoke(dou._dq, dic, flags);
	} else {
		_dispatch_continuation_invoke_inline(dou, flags, dqu);
	}
	if (observer_hooks) observer_hooks->queue_did_execute(dqu._dq);
}

// used to forward the do_invoke of a continuation with a vtable to its real
// implementation.
#define _dispatch_continuation_pop_forwarded(dc, dc_flags, dq, ...) \
	({ \
		dispatch_continuation_t _dc = (dc), _dc1; \
		uintptr_t _dc_flags = (dc_flags); \
		_dispatch_continuation_voucher_adopt(_dc, _dc_flags); \
		if (!(_dc_flags & DC_FLAG_NO_INTROSPECTION)) { \
			_dispatch_trace_item_pop(dq, dc); \
		} \
		if (_dc_flags & DC_FLAG_CONSUME) { \
			_dc1 = _dispatch_continuation_free_cacheonly(_dc); \
		} else { \
			_dc1 = NULL; \
		} \
		__VA_ARGS__; \
		if (!(_dc_flags & DC_FLAG_NO_INTROSPECTION)) { \
			_dispatch_trace_item_complete(_dc); \
		} \
		if (unlikely(_dc1)) { \
			_dispatch_continuation_free_to_cache_limit(_dc1); \
		} \
	})

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dispatch_continuation_priority_set(dispatch_continuation_t dc,
		dispatch_queue_class_t dqu,
		pthread_priority_t pp, dispatch_block_flags_t flags)
{
	dispatch_qos_t qos = DISPATCH_QOS_UNSPECIFIED;
#if HAVE_PTHREAD_WORKQUEUE_QOS
	dispatch_queue_t dq = dqu._dq;

	if (likely(pp)) {
		bool enforce = (flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS);
		bool is_floor = (dq->dq_priority & DISPATCH_PRIORITY_FLAG_FLOOR);
		bool dq_has_qos = (dq->dq_priority & DISPATCH_PRIORITY_REQUESTED_MASK);
		if (enforce) {
			pp |= _PTHREAD_PRIORITY_ENFORCE_FLAG;
			qos = _dispatch_qos_from_pp_unsafe(pp);
		} else if (!is_floor && dq_has_qos) {
			pp = 0;
		} else {
			qos = _dispatch_qos_from_pp_unsafe(pp);
		}
	}
	dc->dc_priority = pp;
#else
	(void)dc; (void)dqu; (void)pp; (void)flags;
#endif
	return qos;
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dispatch_continuation_init_f(dispatch_continuation_t dc,
		dispatch_queue_class_t dqu, void *ctxt, dispatch_function_t f,
		dispatch_block_flags_t flags, uintptr_t dc_flags)
{
	pthread_priority_t pp = 0;
	dc->dc_flags = dc_flags | DC_FLAG_ALLOCATED;
	dc->dc_func = f;
	dc->dc_ctxt = ctxt;
	// in this context DISPATCH_BLOCK_HAS_PRIORITY means that the priority
	// should not be propagated, only taken from the handler if it has one
	if (!(flags & DISPATCH_BLOCK_HAS_PRIORITY)) {
		pp = _dispatch_priority_propagate();
	}
	_dispatch_continuation_voucher_set(dc, flags);
	return _dispatch_continuation_priority_set(dc, dqu, pp, flags);
}

DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dispatch_continuation_init(dispatch_continuation_t dc,
		dispatch_queue_class_t dqu, dispatch_block_t work,
		dispatch_block_flags_t flags, uintptr_t dc_flags)
{
	void *ctxt = _dispatch_Block_copy(work);

	dc_flags |= DC_FLAG_BLOCK | DC_FLAG_ALLOCATED;
	if (unlikely(_dispatch_block_has_private_data(work))) {
		dc->dc_flags = dc_flags;
		dc->dc_ctxt = ctxt;
		// will initialize all fields but requires dc_flags & dc_ctxt to be set
		return _dispatch_continuation_init_slow(dc, dqu, flags);
	}

	dispatch_function_t func = _dispatch_Block_invoke(work);
	if (dc_flags & DC_FLAG_CONSUME) {
		func = _dispatch_call_block_and_release;
	}
	return _dispatch_continuation_init_f(dc, dqu, ctxt, func, flags, dc_flags);
}

DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_continuation_async(dispatch_queue_class_t dqu,
		dispatch_continuation_t dc, dispatch_qos_t qos, uintptr_t dc_flags)
{
#if DISPATCH_INTROSPECTION
	if (!(dc_flags & DC_FLAG_NO_INTROSPECTION)) {
		_dispatch_trace_item_push(dqu, dc);
	}
#else
	(void)dc_flags;
#endif
	return dx_push(dqu._dq, dc, qos);
}

#endif // DISPATCH_PURE_C

#endif /* __DISPATCH_INLINE_INTERNAL__ */