| /* |
| * Copyright (c) 2008-2013 Apple Inc. All rights reserved. |
| * |
| * @APPLE_APACHE_LICENSE_HEADER_START@ |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| * |
| * @APPLE_APACHE_LICENSE_HEADER_END@ |
| */ |
| |
| /* |
| * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch |
| * which are subject to change in future releases of Mac OS X. Any applications |
| * relying on these interfaces WILL break. |
| */ |
| |
| #ifndef __DISPATCH_INLINE_INTERNAL__ |
| #define __DISPATCH_INLINE_INTERNAL__ |
| |
| #ifndef __DISPATCH_INDIRECT__ |
| #error "Please #include <dispatch/dispatch.h> instead of this file directly." |
| #include <dispatch/base.h> // for HeaderDoc |
| #endif |
| |
| #if DISPATCH_USE_CLIENT_CALLOUT |
| |
| DISPATCH_NOTHROW void |
| _dispatch_client_callout(void *ctxt, dispatch_function_t f); |
| DISPATCH_NOTHROW void |
| _dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)); |
| #if HAVE_MACH |
| DISPATCH_NOTHROW void |
| _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, |
| dispatch_mach_msg_t dmsg, mach_error_t error, |
| dispatch_mach_handler_function_t f); |
| #endif // HAVE_MACH |
| |
| #else // !DISPATCH_USE_CLIENT_CALLOUT |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_client_callout(void *ctxt, dispatch_function_t f) |
| { |
| return f(ctxt); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)) |
| { |
| return f(ctxt, i); |
| } |
| |
| #if HAVE_MACH |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_client_callout4(void *ctxt, dispatch_mach_reason_t reason, |
| dispatch_mach_msg_t dmsg, mach_error_t error, |
| dispatch_mach_handler_function_t f) |
| { |
| return f(ctxt, reason, dmsg, error); |
| } |
| #endif // HAVE_MACH |
| |
| #endif // !DISPATCH_USE_CLIENT_CALLOUT |
| |
| #pragma mark - |
| #pragma mark _os_object_t & dispatch_object_t |
| #if DISPATCH_PURE_C |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_object_has_vtable(dispatch_object_t dou) |
| { |
| uintptr_t dc_flags = dou._dc->dc_flags; |
| |
| // vtables are pointers far away from the low page in memory |
| return dc_flags > 0xffful; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_object_is_continuation(dispatch_object_t dou) |
| { |
| if (_dispatch_object_has_vtable(dou)) { |
| return dx_metatype(dou._do) == _DISPATCH_CONTINUATION_TYPE; |
| } |
| return true; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_object_has_type(dispatch_object_t dou, unsigned long type) |
| { |
| return _dispatch_object_has_vtable(dou) && dx_type(dou._do) == type; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_object_is_redirection(dispatch_object_t dou) |
| { |
| return _dispatch_object_has_type(dou, |
| DISPATCH_CONTINUATION_TYPE(ASYNC_REDIRECT)); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_object_is_barrier(dispatch_object_t dou) |
| { |
| dispatch_queue_flags_t dq_flags; |
| |
| if (!_dispatch_object_has_vtable(dou)) { |
| return (dou._dc->dc_flags & DISPATCH_OBJ_BARRIER_BIT); |
| } |
| switch (dx_metatype(dou._do)) { |
| case _DISPATCH_QUEUE_TYPE: |
| case _DISPATCH_SOURCE_TYPE: |
| dq_flags = os_atomic_load2o(dou._dq, dq_atomic_flags, relaxed); |
| return dq_flags & DQF_BARRIER_BIT; |
| default: |
| return false; |
| } |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_object_is_slow_item(dispatch_object_t dou) |
| { |
| if (_dispatch_object_has_vtable(dou)) { |
| return false; |
| } |
| return (dou._dc->dc_flags & DISPATCH_OBJ_SYNC_SLOW_BIT); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_object_is_slow_non_barrier(dispatch_object_t dou) |
| { |
| if (_dispatch_object_has_vtable(dou)) { |
| return false; |
| } |
| return ((dou._dc->dc_flags & |
| (DISPATCH_OBJ_BARRIER_BIT | DISPATCH_OBJ_SYNC_SLOW_BIT)) == |
| (DISPATCH_OBJ_SYNC_SLOW_BIT)); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_object_is_slow_barrier(dispatch_object_t dou) |
| { |
| if (_dispatch_object_has_vtable(dou)) { |
| return false; |
| } |
| return ((dou._dc->dc_flags & |
| (DISPATCH_OBJ_BARRIER_BIT | DISPATCH_OBJ_SYNC_SLOW_BIT)) == |
| (DISPATCH_OBJ_BARRIER_BIT | DISPATCH_OBJ_SYNC_SLOW_BIT)); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline _os_object_t |
| _os_object_retain_internal_inline(_os_object_t obj) |
| { |
| int ref_cnt = _os_object_refcnt_inc(obj); |
| if (unlikely(ref_cnt <= 0)) { |
| _OS_OBJECT_CLIENT_CRASH("Resurrection of an object"); |
| } |
| return obj; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _os_object_release_internal_inline_no_dispose(_os_object_t obj) |
| { |
| int ref_cnt = _os_object_refcnt_dec(obj); |
| if (likely(ref_cnt >= 0)) { |
| return; |
| } |
| if (ref_cnt == 0) { |
| _OS_OBJECT_CLIENT_CRASH("Unexpected release of an object"); |
| } |
| _OS_OBJECT_CLIENT_CRASH("Over-release of an object"); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _os_object_release_internal_inline(_os_object_t obj) |
| { |
| int ref_cnt = _os_object_refcnt_dec(obj); |
| if (likely(ref_cnt >= 0)) { |
| return; |
| } |
| if (unlikely(ref_cnt < -1)) { |
| _OS_OBJECT_CLIENT_CRASH("Over-release of an object"); |
| } |
| #if DISPATCH_DEBUG |
| int xref_cnt = obj->os_obj_xref_cnt; |
| if (unlikely(xref_cnt >= 0)) { |
| DISPATCH_INTERNAL_CRASH(xref_cnt, |
| "Release while external references exist"); |
| } |
| #endif |
| // _os_object_refcnt_dispose_barrier() is in _os_object_dispose() |
| return _os_object_dispose(obj); |
| } |
| |
| DISPATCH_ALWAYS_INLINE_NDEBUG |
| static inline void |
| _dispatch_retain(dispatch_object_t dou) |
| { |
| (void)_os_object_retain_internal_inline(dou._os_obj); |
| } |
| |
| DISPATCH_ALWAYS_INLINE_NDEBUG |
| static inline void |
| _dispatch_release(dispatch_object_t dou) |
| { |
| _os_object_release_internal_inline(dou._os_obj); |
| } |
| |
| DISPATCH_ALWAYS_INLINE_NDEBUG |
| static inline void |
| _dispatch_release_tailcall(dispatch_object_t dou) |
| { |
| _os_object_release_internal(dou._os_obj); |
| } |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_NONNULL_ALL |
| static inline void |
| _dispatch_object_set_target_queue_inline(dispatch_object_t dou, |
| dispatch_queue_t tq) |
| { |
| _dispatch_retain(tq); |
| tq = os_atomic_xchg2o(dou._do, do_targetq, tq, release); |
| if (tq) _dispatch_release(tq); |
| _dispatch_object_debug(dou._do, "%s", __func__); |
| } |
| |
| #endif // DISPATCH_PURE_C |
| #pragma mark - |
| #pragma mark dispatch_thread |
| #if DISPATCH_PURE_C |
| |
| #define DISPATCH_DEFERRED_ITEMS_MAGIC 0xdefe55edul /* deferred */ |
| #define DISPATCH_DEFERRED_ITEMS_EVENT_COUNT 8 |
| #ifdef WORKQ_KEVENT_EVENT_BUFFER_LEN |
| _Static_assert(WORKQ_KEVENT_EVENT_BUFFER_LEN >= |
| DISPATCH_DEFERRED_ITEMS_EVENT_COUNT, |
| "our list should not be longer than the kernel's"); |
| #endif |
| |
| typedef struct dispatch_deferred_items_s { |
| uint32_t ddi_magic; |
| dispatch_queue_t ddi_stashed_dq; |
| struct dispatch_object_s *ddi_stashed_dou; |
| dispatch_priority_t ddi_stashed_pp; |
| int ddi_nevents; |
| int ddi_maxevents; |
| _dispatch_kevent_qos_s ddi_eventlist[DISPATCH_DEFERRED_ITEMS_EVENT_COUNT]; |
| } dispatch_deferred_items_s, *dispatch_deferred_items_t; |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_deferred_items_set(dispatch_deferred_items_t ddi) |
| { |
| _dispatch_thread_setspecific(dispatch_deferred_items_key, (void *)ddi); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_deferred_items_t |
| _dispatch_deferred_items_get(void) |
| { |
| dispatch_deferred_items_t ddi = (dispatch_deferred_items_t) |
| _dispatch_thread_getspecific(dispatch_deferred_items_key); |
| if (ddi && ddi->ddi_magic == DISPATCH_DEFERRED_ITEMS_MAGIC) { |
| return ddi; |
| } |
| return NULL; |
| } |
| |
| #endif // DISPATCH_PURE_C |
| #pragma mark - |
| #pragma mark dispatch_thread |
| #if DISPATCH_PURE_C |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_thread_context_t |
| _dispatch_thread_context_find(const void *key) |
| { |
| dispatch_thread_context_t dtc = |
| _dispatch_thread_getspecific(dispatch_context_key); |
| while (dtc) { |
| if (dtc->dtc_key == key) { |
| return dtc; |
| } |
| dtc = dtc->dtc_prev; |
| } |
| return NULL; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_context_push(dispatch_thread_context_t ctxt) |
| { |
| ctxt->dtc_prev = _dispatch_thread_getspecific(dispatch_context_key); |
| _dispatch_thread_setspecific(dispatch_context_key, ctxt); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_context_pop(dispatch_thread_context_t ctxt) |
| { |
| dispatch_assert(_dispatch_thread_getspecific(dispatch_context_key) == ctxt); |
| _dispatch_thread_setspecific(dispatch_context_key, ctxt->dtc_prev); |
| } |
| |
| typedef struct dispatch_thread_frame_iterator_s { |
| dispatch_queue_t dtfi_queue; |
| dispatch_thread_frame_t dtfi_frame; |
| } *dispatch_thread_frame_iterator_t; |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_frame_iterate_start(dispatch_thread_frame_iterator_t it) |
| { |
| _dispatch_thread_getspecific_pair( |
| dispatch_queue_key, (void **)&it->dtfi_queue, |
| dispatch_frame_key, (void **)&it->dtfi_frame); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_frame_iterate_next(dispatch_thread_frame_iterator_t it) |
| { |
| dispatch_thread_frame_t dtf = it->dtfi_frame; |
| dispatch_queue_t dq = it->dtfi_queue; |
| |
| if (dtf) { |
| if (dq->do_targetq) { |
| // redirections and trysync_f may skip some frames, |
| // so we need to simulate seeing the missing links |
| // however the bottom root queue is always present |
| it->dtfi_queue = dq->do_targetq; |
| if (it->dtfi_queue == dtf->dtf_queue) { |
| it->dtfi_frame = dtf->dtf_prev; |
| } |
| } else { |
| it->dtfi_queue = dtf->dtf_queue; |
| it->dtfi_frame = dtf->dtf_prev; |
| } |
| } else if (dq) { |
| it->dtfi_queue = dq->do_targetq; |
| } |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_thread_frame_find_queue(dispatch_queue_t dq) |
| { |
| struct dispatch_thread_frame_iterator_s it; |
| |
| _dispatch_thread_frame_iterate_start(&it); |
| while (it.dtfi_queue) { |
| if (it.dtfi_queue == dq) { |
| return true; |
| } |
| _dispatch_thread_frame_iterate_next(&it); |
| } |
| return false; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_thread_frame_t |
| _dispatch_thread_frame_get_current(void) |
| { |
| return _dispatch_thread_getspecific(dispatch_frame_key); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_frame_set_current(dispatch_thread_frame_t dtf) |
| { |
| _dispatch_thread_setspecific(dispatch_frame_key, dtf); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_frame_save_state(dispatch_thread_frame_t dtf) |
| { |
| _dispatch_thread_getspecific_packed_pair( |
| dispatch_queue_key, dispatch_frame_key, (void **)&dtf->dtf_queue); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_frame_push(dispatch_thread_frame_t dtf, dispatch_queue_t dq) |
| { |
| _dispatch_thread_frame_save_state(dtf); |
| _dispatch_thread_setspecific_pair(dispatch_queue_key, dq, |
| dispatch_frame_key, dtf); |
| dtf->dtf_deferred = NULL; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_frame_push_and_rebase(dispatch_thread_frame_t dtf, |
| dispatch_queue_t dq, dispatch_thread_frame_t new_base) |
| { |
| _dispatch_thread_frame_save_state(dtf); |
| _dispatch_thread_setspecific_pair(dispatch_queue_key, dq, |
| dispatch_frame_key, new_base); |
| dtf->dtf_deferred = NULL; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_frame_pop(dispatch_thread_frame_t dtf) |
| { |
| _dispatch_thread_setspecific_packed_pair( |
| dispatch_queue_key, dispatch_frame_key, (void **)&dtf->dtf_queue); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_queue_t |
| _dispatch_thread_frame_stash(dispatch_thread_frame_t dtf) |
| { |
| _dispatch_thread_getspecific_pair( |
| dispatch_queue_key, (void **)&dtf->dtf_queue, |
| dispatch_frame_key, (void **)&dtf->dtf_prev); |
| _dispatch_thread_frame_pop(dtf->dtf_prev); |
| return dtf->dtf_queue; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_frame_unstash(dispatch_thread_frame_t dtf) |
| { |
| _dispatch_thread_frame_pop(dtf); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline int |
| _dispatch_wqthread_override_start_check_owner(mach_port_t thread, |
| pthread_priority_t pp, mach_port_t *ulock_addr) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| if (!_dispatch_set_qos_class_enabled) return 0; |
| return _pthread_workqueue_override_start_direct_check_owner(thread, |
| pp, ulock_addr); |
| #else |
| (void)thread; (void)pp; (void)ulock_addr; |
| return 0; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_wqthread_override_start(mach_port_t thread, |
| pthread_priority_t pp) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| if (!_dispatch_set_qos_class_enabled) return; |
| (void)_pthread_workqueue_override_start_direct(thread, pp); |
| #else |
| (void)thread; (void)pp; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_wqthread_override_reset(void) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| if (!_dispatch_set_qos_class_enabled) return; |
| (void)_pthread_workqueue_override_reset(); |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_override_start(mach_port_t thread, pthread_priority_t pp, |
| void *resource) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| if (!_dispatch_set_qos_class_enabled) return; |
| (void)_pthread_qos_override_start_direct(thread, pp, resource); |
| #else |
| (void)thread; (void)pp; (void)resource; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_thread_override_end(mach_port_t thread, void *resource) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| if (!_dispatch_set_qos_class_enabled) return; |
| (void)_pthread_qos_override_end_direct(thread, resource); |
| #else |
| (void)thread; (void)resource; |
| #endif |
| } |
| |
| #if DISPATCH_DEBUG_QOS && HAVE_PTHREAD_WORKQUEUE_QOS |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_qos_class_is_valid(pthread_priority_t pp) |
| { |
| pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK; |
| if (pp > (1UL << (DISPATCH_QUEUE_QOS_COUNT + |
| _PTHREAD_PRIORITY_QOS_CLASS_SHIFT))) { |
| return false; |
| } |
| return true; |
| } |
| #define _dispatch_assert_is_valid_qos_class(pp) ({ typeof(pp) _pp = (pp); \ |
| if (unlikely(!_dispatch_qos_class_is_valid(_pp))) { \ |
| DISPATCH_INTERNAL_CRASH(_pp, "Invalid qos class"); \ |
| } \ |
| }) |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_qos_override_is_valid(pthread_priority_t pp) |
| { |
| if (pp & (pthread_priority_t)_PTHREAD_PRIORITY_FLAGS_MASK) { |
| return false; |
| } |
| return _dispatch_qos_class_is_valid(pp); |
| } |
| #define _dispatch_assert_is_valid_qos_override(pp) ({ typeof(pp) _pp = (pp); \ |
| if (unlikely(!_dispatch_qos_override_is_valid(_pp))) { \ |
| DISPATCH_INTERNAL_CRASH(_pp, "Invalid override"); \ |
| } \ |
| }) |
| #else |
| #define _dispatch_assert_is_valid_qos_override(pp) (void)(pp) |
| #define _dispatch_assert_is_valid_qos_class(pp) (void)(pp) |
| #endif |
| |
| #endif // DISPATCH_PURE_C |
| #pragma mark - |
| #pragma mark dispatch_queue_t state accessors |
| #if DISPATCH_PURE_C |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_queue_flags_t |
| _dispatch_queue_atomic_flags(dispatch_queue_t dq) |
| { |
| return os_atomic_load2o(dq, dq_atomic_flags, relaxed); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_queue_flags_t |
| _dispatch_queue_atomic_flags_set(dispatch_queue_t dq, |
| dispatch_queue_flags_t bits) |
| { |
| return os_atomic_or2o(dq, dq_atomic_flags, bits, relaxed); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_queue_flags_t |
| _dispatch_queue_atomic_flags_set_and_clear_orig(dispatch_queue_t dq, |
| dispatch_queue_flags_t add_bits, dispatch_queue_flags_t clr_bits) |
| { |
| dispatch_queue_flags_t oflags, nflags; |
| os_atomic_rmw_loop2o(dq, dq_atomic_flags, oflags, nflags, relaxed, { |
| nflags = (oflags | add_bits) & ~clr_bits; |
| }); |
| return oflags; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_queue_flags_t |
| _dispatch_queue_atomic_flags_set_and_clear(dispatch_queue_t dq, |
| dispatch_queue_flags_t add_bits, dispatch_queue_flags_t clr_bits) |
| { |
| dispatch_queue_flags_t oflags, nflags; |
| os_atomic_rmw_loop2o(dq, dq_atomic_flags, oflags, nflags, relaxed, { |
| nflags = (oflags | add_bits) & ~clr_bits; |
| }); |
| return nflags; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_queue_flags_t |
| _dispatch_queue_atomic_flags_set_orig(dispatch_queue_t dq, |
| dispatch_queue_flags_t bits) |
| { |
| return os_atomic_or_orig2o(dq, dq_atomic_flags, bits, relaxed); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_queue_flags_t |
| _dispatch_queue_atomic_flags_clear(dispatch_queue_t dq, |
| dispatch_queue_flags_t bits) |
| { |
| return os_atomic_and2o(dq, dq_atomic_flags, ~bits, relaxed); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_is_thread_bound(dispatch_queue_t dq) |
| { |
| return _dispatch_queue_atomic_flags(dq) & DQF_THREAD_BOUND; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_cannot_trysync(dispatch_queue_t dq) |
| { |
| return _dispatch_queue_atomic_flags(dq) & DQF_CANNOT_TRYSYNC; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_label_needs_free(dispatch_queue_t dq) |
| { |
| return _dispatch_queue_atomic_flags(dq) & DQF_LABEL_NEEDS_FREE; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_invoke_flags_t |
| _dispatch_queue_autorelease_frequency(dispatch_queue_t dq) |
| { |
| const unsigned long factor = |
| DISPATCH_INVOKE_AUTORELEASE_ALWAYS / DQF_AUTORELEASE_ALWAYS; |
| dispatch_static_assert(factor > 0); |
| |
| dispatch_queue_flags_t qaf = _dispatch_queue_atomic_flags(dq); |
| |
| qaf &= _DQF_AUTORELEASE_MASK; |
| return (dispatch_invoke_flags_t)qaf * factor; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_invoke_flags_t |
| _dispatch_queue_merge_autorelease_frequency(dispatch_queue_t dq, |
| dispatch_invoke_flags_t flags) |
| { |
| dispatch_invoke_flags_t qaf = _dispatch_queue_autorelease_frequency(dq); |
| |
| if (qaf) { |
| flags &= ~_DISPATCH_INVOKE_AUTORELEASE_MASK; |
| flags |= qaf; |
| } |
| return flags; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_has_immutable_target(dispatch_queue_t dq) |
| { |
| if (dx_metatype(dq) != _DISPATCH_QUEUE_TYPE) { |
| return false; |
| } |
| return dx_type(dq) != DISPATCH_QUEUE_LEGACY_TYPE; |
| } |
| |
| #endif // DISPATCH_PURE_C |
| #ifndef __cplusplus |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline uint32_t |
| _dq_state_suspend_cnt(uint64_t dq_state) |
| { |
| return (uint32_t)(dq_state / DISPATCH_QUEUE_SUSPEND_INTERVAL); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_has_side_suspend_cnt(uint64_t dq_state) |
| { |
| return dq_state & DISPATCH_QUEUE_HAS_SIDE_SUSPEND_CNT; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline uint32_t |
| _dq_state_extract_width_bits(uint64_t dq_state) |
| { |
| dq_state &= DISPATCH_QUEUE_WIDTH_MASK; |
| return (uint32_t)(dq_state >> DISPATCH_QUEUE_WIDTH_SHIFT); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline uint32_t |
| _dq_state_available_width(uint64_t dq_state) |
| { |
| uint32_t full = DISPATCH_QUEUE_WIDTH_FULL; |
| if (fastpath(!(dq_state & DISPATCH_QUEUE_WIDTH_FULL_BIT))) { |
| return full - _dq_state_extract_width_bits(dq_state); |
| } |
| return 0; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline uint32_t |
| _dq_state_used_width(uint64_t dq_state, uint16_t dq_width) |
| { |
| uint32_t full = DISPATCH_QUEUE_WIDTH_FULL; |
| uint32_t width = _dq_state_extract_width_bits(dq_state); |
| |
| if (dq_state & DISPATCH_QUEUE_PENDING_BARRIER) { |
| // DISPATCH_QUEUE_PENDING_BARRIER means (dq_width - 1) of the used width |
| // is pre-reservation that we want to ignore |
| return width - (full - dq_width) - (dq_width - 1); |
| } |
| return width - (full - dq_width); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_is_suspended(uint64_t dq_state) |
| { |
| return dq_state >= DISPATCH_QUEUE_NEEDS_ACTIVATION; |
| } |
| #define DISPATCH_QUEUE_IS_SUSPENDED(x) _dq_state_is_suspended((x)->dq_state) |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_is_inactive(uint64_t dq_state) |
| { |
| return dq_state & DISPATCH_QUEUE_INACTIVE; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_needs_activation(uint64_t dq_state) |
| { |
| return dq_state & DISPATCH_QUEUE_NEEDS_ACTIVATION; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_is_in_barrier(uint64_t dq_state) |
| { |
| return dq_state & DISPATCH_QUEUE_IN_BARRIER; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_has_available_width(uint64_t dq_state) |
| { |
| return !(dq_state & DISPATCH_QUEUE_WIDTH_FULL_BIT); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_has_pending_barrier(uint64_t dq_state) |
| { |
| return dq_state & DISPATCH_QUEUE_PENDING_BARRIER; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_is_dirty(uint64_t dq_state) |
| { |
| return dq_state & DISPATCH_QUEUE_DIRTY; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_is_enqueued(uint64_t dq_state) |
| { |
| return dq_state & DISPATCH_QUEUE_ENQUEUED; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_has_override(uint64_t dq_state) |
| { |
| return dq_state & DISPATCH_QUEUE_HAS_OVERRIDE; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_lock_owner |
| _dq_state_drain_owner(uint64_t dq_state) |
| { |
| return _dispatch_lock_owner((dispatch_lock)dq_state); |
| } |
| #define DISPATCH_QUEUE_DRAIN_OWNER(dq) \ |
| _dq_state_drain_owner(os_atomic_load2o(dq, dq_state, relaxed)) |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_drain_pended(uint64_t dq_state) |
| { |
| return (dq_state & DISPATCH_QUEUE_DRAIN_PENDED); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_drain_locked_by(uint64_t dq_state, uint32_t owner) |
| { |
| if (_dq_state_drain_pended(dq_state)) { |
| return false; |
| } |
| return _dq_state_drain_owner(dq_state) == owner; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_drain_locked(uint64_t dq_state) |
| { |
| return (dq_state & DISPATCH_QUEUE_DRAIN_OWNER_MASK) != 0; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_has_waiters(uint64_t dq_state) |
| { |
| return _dispatch_lock_has_waiters((dispatch_lock)dq_state); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_is_sync_runnable(uint64_t dq_state) |
| { |
| return dq_state < DISPATCH_QUEUE_IN_BARRIER; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_is_runnable(uint64_t dq_state) |
| { |
| return dq_state < DISPATCH_QUEUE_WIDTH_FULL_BIT; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dq_state_should_wakeup(uint64_t dq_state) |
| { |
| return _dq_state_is_runnable(dq_state) && |
| !_dq_state_is_enqueued(dq_state) && |
| !_dq_state_drain_locked(dq_state); |
| } |
| |
| #endif // __cplusplus |
| #pragma mark - |
| #pragma mark dispatch_queue_t state machine |
| #ifndef __cplusplus |
| |
| static inline bool _dispatch_queue_need_override(dispatch_queue_class_t dqu, |
| pthread_priority_t pp); |
| static inline bool _dispatch_queue_need_override_retain( |
| dispatch_queue_class_t dqu, pthread_priority_t pp); |
| static inline dispatch_priority_t _dispatch_queue_reset_override_priority( |
| dispatch_queue_class_t dqu, bool qp_is_floor); |
| static inline bool _dispatch_queue_reinstate_override_priority(dispatch_queue_class_t dqu, |
| dispatch_priority_t new_op); |
| static inline pthread_priority_t _dispatch_get_defaultpriority(void); |
| static inline void _dispatch_set_defaultpriority_override(void); |
| static inline void _dispatch_reset_defaultpriority(pthread_priority_t pp); |
| static inline pthread_priority_t _dispatch_get_priority(void); |
| static inline pthread_priority_t _dispatch_set_defaultpriority( |
| pthread_priority_t pp, pthread_priority_t *new_pp); |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_xref_dispose(struct dispatch_queue_s *dq) |
| { |
| if (slowpath(DISPATCH_QUEUE_IS_SUSPENDED(dq))) { |
| // Arguments for and against this assert are within 6705399 |
| DISPATCH_CLIENT_CRASH(dq, "Release of a suspended object"); |
| } |
| os_atomic_or2o(dq, dq_atomic_flags, DQF_RELEASED, relaxed); |
| } |
| |
| #endif |
| #if DISPATCH_PURE_C |
| |
| // Note to later developers: ensure that any initialization changes are |
| // made for statically allocated queues (i.e. _dispatch_main_q). |
| static inline void |
| _dispatch_queue_init(dispatch_queue_t dq, dispatch_queue_flags_t dqf, |
| uint16_t width, bool inactive) |
| { |
| uint64_t dq_state = DISPATCH_QUEUE_STATE_INIT_VALUE(width); |
| |
| if (inactive) { |
| dq_state += DISPATCH_QUEUE_INACTIVE + DISPATCH_QUEUE_NEEDS_ACTIVATION; |
| dq->do_ref_cnt++; // rdar://8181908 see _dispatch_queue_resume |
| } |
| dq->do_next = (struct dispatch_queue_s *)DISPATCH_OBJECT_LISTLESS; |
| dqf |= (dispatch_queue_flags_t)width << DQF_WIDTH_SHIFT; |
| os_atomic_store2o(dq, dq_atomic_flags, dqf, relaxed); |
| dq->dq_state = dq_state; |
| dq->dq_override_voucher = DISPATCH_NO_VOUCHER; |
| dq->dq_serialnum = |
| os_atomic_inc_orig(&_dispatch_queue_serial_numbers, relaxed); |
| } |
| |
| /* Used by: |
| * - _dispatch_queue_set_target_queue |
| * - changing dispatch source handlers |
| * |
| * Tries to prevent concurrent wakeup of an inactive queue by suspending it. |
| */ |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline bool |
| _dispatch_queue_try_inactive_suspend(dispatch_queue_t dq) |
| { |
| uint64_t dq_state, value; |
| |
| (void)os_atomic_rmw_loop2o(dq, dq_state, dq_state, value, relaxed, { |
| if (!fastpath(_dq_state_is_inactive(dq_state))) { |
| os_atomic_rmw_loop_give_up(return false); |
| } |
| value = dq_state + DISPATCH_QUEUE_SUSPEND_INTERVAL; |
| }); |
| if (slowpath(!_dq_state_is_suspended(dq_state)) || |
| slowpath(_dq_state_has_side_suspend_cnt(dq_state))) { |
| // Crashing here means that 128+ dispatch_suspend() calls have been |
| // made on an inactive object and then dispatch_set_target_queue() or |
| // dispatch_set_*_handler() has been called. |
| // |
| // We don't want to handle the side suspend count in a codepath that |
| // needs to be fast. |
| DISPATCH_CLIENT_CRASH(dq, "Too many calls to dispatch_suspend() " |
| "prior to calling dispatch_set_target_queue() " |
| "or dispatch_set_*_handler()"); |
| } |
| return true; |
| } |
| |
| /* Must be used by any caller meaning to do a speculative wakeup when the caller |
| * was preventing other wakeups (for example dispatch_resume() or a drainer not |
| * doing a drain_try_unlock() and not observing DIRTY) |
| * |
| * In that case this call loads DIRTY with an acquire barrier so that when |
| * other threads have made changes (such as dispatch_source_cancel()) the |
| * caller can take these state machine changes into account in its decision to |
| * wake up the object. |
| */ |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_try_wakeup(dispatch_queue_t dq, uint64_t dq_state, |
| dispatch_wakeup_flags_t flags) |
| { |
| if (_dq_state_should_wakeup(dq_state)) { |
| if (slowpath(_dq_state_is_dirty(dq_state))) { |
| // <rdar://problem/14637483> |
| // seq_cst wrt state changes that were flushed and not acted upon |
| os_atomic_thread_fence(acquire); |
| } |
| return dx_wakeup(dq, 0, flags); |
| } |
| if (flags & DISPATCH_WAKEUP_CONSUME) { |
| return _dispatch_release_tailcall(dq); |
| } |
| } |
| |
| /* Used by: |
| * - _dispatch_queue_class_invoke (normal path) |
| * - _dispatch_queue_override_invoke (stealer) |
| * |
| * Initial state must be { sc:0, ib:0, qf:0, dl:0 } |
| * Final state forces { dl:self, qf:1, d: 0 } |
| * ib:1 is forced when the width acquired is equivalent to the barrier width |
| */ |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline uint64_t |
| _dispatch_queue_drain_try_lock(dispatch_queue_t dq, |
| dispatch_invoke_flags_t flags, uint64_t *dq_state) |
| { |
| uint64_t pending_barrier_width = |
| (dq->dq_width - 1) * DISPATCH_QUEUE_WIDTH_INTERVAL; |
| uint64_t xor_owner_and_set_full_width = |
| _dispatch_tid_self() | DISPATCH_QUEUE_WIDTH_FULL_BIT; |
| uint64_t clear_enqueued_bit, old_state, new_state; |
| |
| if (flags & DISPATCH_INVOKE_STEALING) { |
| clear_enqueued_bit = 0; |
| } else { |
| clear_enqueued_bit = DISPATCH_QUEUE_ENQUEUED; |
| } |
| |
| os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, acquire, { |
| new_state = old_state; |
| new_state ^= clear_enqueued_bit; |
| if (likely(_dq_state_is_runnable(old_state) && |
| !_dq_state_drain_locked(old_state))) { |
| // |
| // Only keep the HAS_WAITER bit (and ENQUEUED if stealing). |
| // In particular acquiring the drain lock clears the DIRTY bit |
| // |
| new_state &= DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK; |
| // |
| // For the NOWAITERS_BIT case, the thread identity |
| // has NOWAITERS_BIT set, and NOWAITERS_BIT was kept above, |
| // so the xor below flips the NOWAITERS_BIT to 0 as expected. |
| // |
| // For the non inverted WAITERS_BIT case, WAITERS_BIT is not set in |
| // the thread identity, and the xor leaves the bit alone. |
| // |
| new_state ^= xor_owner_and_set_full_width; |
| if (_dq_state_has_pending_barrier(old_state) || |
| old_state + pending_barrier_width < |
| DISPATCH_QUEUE_WIDTH_FULL_BIT) { |
| new_state |= DISPATCH_QUEUE_IN_BARRIER; |
| } |
| } else if (!clear_enqueued_bit) { |
| os_atomic_rmw_loop_give_up(break); |
| } |
| }); |
| |
| if (dq_state) *dq_state = new_state; |
| if (likely(_dq_state_is_runnable(old_state) && |
| !_dq_state_drain_locked(old_state))) { |
| new_state &= DISPATCH_QUEUE_IN_BARRIER | DISPATCH_QUEUE_WIDTH_FULL_BIT; |
| old_state &= DISPATCH_QUEUE_WIDTH_MASK; |
| return new_state - old_state; |
| } |
| return 0; |
| } |
| |
| /* Used by _dispatch_barrier_{try,}sync |
| * |
| * Note, this fails if any of e:1 or dl!=0, but that allows this code to be a |
| * simple cmpxchg which is significantly faster on Intel, and makes a |
| * significant difference on the uncontended codepath. |
| * |
| * See discussion for DISPATCH_QUEUE_DIRTY in queue_internal.h |
| * |
| * Initial state must be `completely idle` |
| * Final state forces { ib:1, qf:1, w:0 } |
| */ |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline bool |
| _dispatch_queue_try_acquire_barrier_sync(dispatch_queue_t dq) |
| { |
| uint64_t value = DISPATCH_QUEUE_WIDTH_FULL_BIT | DISPATCH_QUEUE_IN_BARRIER; |
| value |= _dispatch_tid_self(); |
| |
| return os_atomic_cmpxchg2o(dq, dq_state, |
| DISPATCH_QUEUE_STATE_INIT_VALUE(dq->dq_width), value, acquire); |
| } |
| |
| /* Used by _dispatch_sync for root queues and some drain codepaths |
| * |
| * Root queues have no strict orderning and dispatch_sync() always goes through. |
| * Drain is the sole setter of `dl` hence can use this non failing version of |
| * _dispatch_queue_try_acquire_sync(). |
| * |
| * Final state: { w += 1 } |
| */ |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_reserve_sync_width(dispatch_queue_t dq) |
| { |
| (void)os_atomic_add2o(dq, dq_state, |
| DISPATCH_QUEUE_WIDTH_INTERVAL, relaxed); |
| } |
| |
| /* Used by _dispatch_sync on non-serial queues |
| * |
| * Initial state must be { sc:0, ib:0, pb:0, d:0 } |
| * Final state: { w += 1 } |
| */ |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline bool |
| _dispatch_queue_try_reserve_sync_width(dispatch_queue_t dq) |
| { |
| uint64_t dq_state, value; |
| |
| return os_atomic_rmw_loop2o(dq, dq_state, dq_state, value, relaxed, { |
| if (!fastpath(_dq_state_is_sync_runnable(dq_state)) || |
| slowpath(_dq_state_is_dirty(dq_state)) || |
| slowpath(_dq_state_has_pending_barrier(dq_state))) { |
| os_atomic_rmw_loop_give_up(return false); |
| } |
| value = dq_state + DISPATCH_QUEUE_WIDTH_INTERVAL; |
| }); |
| } |
| |
| /* Used by _dispatch_apply_redirect |
| * |
| * Try to acquire at most da_width and returns what could be acquired, |
| * possibly 0 |
| */ |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline uint32_t |
| _dispatch_queue_try_reserve_apply_width(dispatch_queue_t dq, uint32_t da_width) |
| { |
| uint64_t dq_state, value; |
| uint32_t width; |
| |
| (void)os_atomic_rmw_loop2o(dq, dq_state, dq_state, value, relaxed, { |
| width = _dq_state_available_width(dq_state); |
| if (!fastpath(width)) { |
| os_atomic_rmw_loop_give_up(return 0); |
| } |
| if (width > da_width) { |
| width = da_width; |
| } |
| value = dq_state + width * DISPATCH_QUEUE_WIDTH_INTERVAL; |
| }); |
| return width; |
| } |
| |
| /* Used by _dispatch_apply_redirect |
| * |
| * Release width acquired by _dispatch_queue_try_acquire_width |
| */ |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_relinquish_width(dispatch_queue_t dq, uint32_t da_width) |
| { |
| (void)os_atomic_sub2o(dq, dq_state, |
| da_width * DISPATCH_QUEUE_WIDTH_INTERVAL, relaxed); |
| } |
| |
| /* Used by target-queue recursing code |
| * |
| * Initial state must be { sc:0, ib:0, qf:0, pb:0, d:0 } |
| * Final state: { w += 1 } |
| */ |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline bool |
| _dispatch_queue_try_acquire_async(dispatch_queue_t dq) |
| { |
| uint64_t dq_state, value; |
| |
| return os_atomic_rmw_loop2o(dq, dq_state, dq_state, value, acquire, { |
| if (!fastpath(_dq_state_is_runnable(dq_state)) || |
| slowpath(_dq_state_is_dirty(dq_state)) || |
| slowpath(_dq_state_has_pending_barrier(dq_state))) { |
| os_atomic_rmw_loop_give_up(return false); |
| } |
| value = dq_state + DISPATCH_QUEUE_WIDTH_INTERVAL; |
| }); |
| } |
| |
| /* Used at the end of Drainers |
| * |
| * This adjusts the `owned` width when the next continuation is already known |
| * to account for its barrierness. |
| */ |
| DISPATCH_ALWAYS_INLINE |
| static inline uint64_t |
| _dispatch_queue_adjust_owned(dispatch_queue_t dq, uint64_t owned, |
| struct dispatch_object_s *next_dc) |
| { |
| uint64_t reservation; |
| |
| if (slowpath(dq->dq_width > 1)) { |
| if (next_dc && _dispatch_object_is_barrier(next_dc)) { |
| reservation = DISPATCH_QUEUE_PENDING_BARRIER; |
| reservation += (dq->dq_width - 1) * DISPATCH_QUEUE_WIDTH_INTERVAL; |
| owned -= reservation; |
| } |
| } |
| return owned; |
| } |
| |
| /* Used at the end of Drainers |
| * |
| * Unlocking fails if the DIRTY bit is seen (and the queue is not suspended). |
| * In that case, only the DIRTY bit is cleared. The DIRTY bit is therefore used |
| * as a signal to renew the drain lock instead of releasing it. |
| * |
| * Successful unlock forces { dl:0, d:0, qo:0 } and gives back `owned` |
| */ |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline bool |
| _dispatch_queue_drain_try_unlock(dispatch_queue_t dq, uint64_t owned) |
| { |
| uint64_t old_state = os_atomic_load2o(dq, dq_state, relaxed); |
| uint64_t new_state; |
| dispatch_priority_t pp = 0, op; |
| |
| do { |
| if (unlikely(_dq_state_is_dirty(old_state) && |
| !_dq_state_is_suspended(old_state))) { |
| // just renew the drain lock with an acquire barrier, to see |
| // what the enqueuer that set DIRTY has done. |
| os_atomic_and2o(dq, dq_state, ~DISPATCH_QUEUE_DIRTY, acquire); |
| _dispatch_queue_reinstate_override_priority(dq, pp); |
| return false; |
| } |
| new_state = old_state - owned; |
| if ((new_state & DISPATCH_QUEUE_WIDTH_FULL_BIT) || |
| _dq_state_is_suspended(old_state)) { |
| // the test for the WIDTH_FULL_BIT is about narrow concurrent queues |
| // releasing the drain lock while being at the width limit |
| // |
| // _non_barrier_complete() will set the DIRTY bit when going back |
| // under the limit which will cause the try_unlock to fail |
| new_state = DISPATCH_QUEUE_DRAIN_UNLOCK_PRESERVE_WAITERS_BIT(new_state); |
| } else { |
| new_state &= ~DISPATCH_QUEUE_DIRTY; |
| new_state &= ~DISPATCH_QUEUE_DRAIN_UNLOCK_MASK; |
| // This current owner is the only one that can clear HAS_OVERRIDE, |
| // so accumulating reset overrides here is valid. |
| if (unlikely(_dq_state_has_override(new_state))) { |
| new_state &= ~DISPATCH_QUEUE_HAS_OVERRIDE; |
| dispatch_assert(!_dispatch_queue_is_thread_bound(dq)); |
| op = _dispatch_queue_reset_override_priority(dq, false); |
| if (op > pp) pp = op; |
| } |
| } |
| } while (!fastpath(os_atomic_cmpxchgvw2o(dq, dq_state, |
| old_state, new_state, &old_state, release))); |
| |
| if (_dq_state_has_override(old_state)) { |
| // Ensure that the root queue sees that this thread was overridden. |
| _dispatch_set_defaultpriority_override(); |
| } |
| return true; |
| } |
| |
| /* Used at the end of Drainers when the next work item is known |
| * and that the dirty-head check isn't needed. |
| * |
| * This releases `owned`, clears DIRTY, and handles HAS_OVERRIDE when seen. |
| */ |
| DISPATCH_ALWAYS_INLINE |
| static inline uint64_t |
| _dispatch_queue_drain_lock_transfer_or_unlock(dispatch_queue_t dq, |
| uint64_t owned, mach_port_t next_owner, uint64_t *orig_state) |
| { |
| uint64_t dq_state, value; |
| |
| #ifdef DLOCK_NOWAITERS_BIT |
| // The NOWAITERS_BIT state must not change through the transfer. It means |
| // that if next_owner is 0 the bit must be flipped in the rmw_loop below, |
| // and if next_owner is set, then the bit must be left unchanged. |
| // |
| // - when next_owner is 0, the xor below sets NOWAITERS_BIT in next_owner, |
| // which causes the second xor to flip the bit as expected. |
| // - if next_owner is not 0, it has the NOWAITERS_BIT set, so we have to |
| // clear it so that the second xor leaves the NOWAITERS_BIT alone. |
| next_owner ^= DLOCK_NOWAITERS_BIT; |
| #endif |
| os_atomic_rmw_loop2o(dq, dq_state, dq_state, value, release, { |
| value = dq_state - owned; |
| // same as DISPATCH_QUEUE_DRAIN_UNLOCK_PRESERVE_WAITERS_BIT |
| // but we want to be more efficient wrt the WAITERS_BIT |
| value &= ~DISPATCH_QUEUE_DRAIN_OWNER_MASK; |
| value &= ~DISPATCH_QUEUE_DRAIN_PENDED; |
| value &= ~DISPATCH_QUEUE_DIRTY; |
| value ^= next_owner; |
| }); |
| |
| if (_dq_state_has_override(dq_state)) { |
| // Ensure that the root queue sees that this thread was overridden. |
| _dispatch_set_defaultpriority_override(); |
| } |
| if (orig_state) *orig_state = dq_state; |
| return value; |
| } |
| #define _dispatch_queue_drain_unlock(dq, owned, orig) \ |
| _dispatch_queue_drain_lock_transfer_or_unlock(dq, owned, 0, orig) |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_drain_transfer_lock(dispatch_queue_t dq, |
| uint64_t to_unlock, dispatch_object_t dou) |
| { |
| mach_port_t th_next = 0; |
| if (dou._dc->dc_flags & DISPATCH_OBJ_BARRIER_BIT) { |
| th_next = (mach_port_t)dou._dc->dc_data; |
| } |
| _dispatch_queue_drain_lock_transfer_or_unlock(dq, to_unlock, th_next, NULL); |
| } |
| |
| |
| #pragma mark - |
| #pragma mark os_mpsc_queue |
| |
| // type_t * {volatile,const,_Atomic,...} -> type_t * |
| // type_t[] -> type_t * |
| #define os_unqualified_pointer_type(expr) \ |
| typeof(typeof(*(expr)) *) |
| |
| #define os_mpsc_node_type(q, _ns) \ |
| os_unqualified_pointer_type((q)->_ns##_head) |
| |
| // |
| // Multi Producer calls, can be used safely concurrently |
| // |
| |
| // Returns true when the queue was empty and the head must be set |
| #define os_mpsc_push_update_tail_list(q, _ns, head, tail, _o_next) ({ \ |
| os_mpsc_node_type(q, _ns) _head = (head), _tail = (tail), _prev; \ |
| _tail->_o_next = NULL; \ |
| _prev = os_atomic_xchg2o((q), _ns##_tail, _tail, release); \ |
| if (fastpath(_prev)) { \ |
| os_atomic_store2o(_prev, _o_next, _head, relaxed); \ |
| } \ |
| (_prev == NULL); \ |
| }) |
| |
| // Returns true when the queue was empty and the head must be set |
| #define os_mpsc_push_update_tail(q, _ns, o, _o_next) ({ \ |
| os_mpsc_node_type(q, _ns) _o = (o); \ |
| os_mpsc_push_update_tail_list(q, _ns, _o, _o, _o_next); \ |
| }) |
| |
| #define os_mpsc_push_update_head(q, _ns, o) ({ \ |
| os_atomic_store2o((q), _ns##_head, o, relaxed); \ |
| }) |
| |
| // |
| // Single Consumer calls, can NOT be used safely concurrently |
| // |
| |
| #define os_mpsc_get_head(q, _ns) ({ \ |
| os_mpsc_node_type(q, _ns) _head; \ |
| _dispatch_wait_until(_head = (q)->_ns##_head); \ |
| _head; \ |
| }) |
| |
| #define os_mpsc_pop_head(q, _ns, head, _o_next) ({ \ |
| typeof(q) _q = (q); \ |
| os_mpsc_node_type(_q, _ns) _head = (head), _n = fastpath(_head->_o_next); \ |
| os_atomic_store2o(_q, _ns##_head, _n, relaxed); \ |
| /* 22708742: set tail to NULL with release, so that NULL write */ \ |
| /* to head above doesn't clobber head from concurrent enqueuer */ \ |
| if (!_n && !os_atomic_cmpxchg2o(_q, _ns##_tail, _head, NULL, release)) { \ |
| _dispatch_wait_until(_n = fastpath(_head->_o_next)); \ |
| os_atomic_store2o(_q, _ns##_head, _n, relaxed); \ |
| } \ |
| _n; \ |
| }) |
| |
| #define os_mpsc_undo_pop_head(q, _ns, head, next, _o_next) ({ \ |
| typeof(q) _q = (q); \ |
| os_mpsc_node_type(_q, _ns) _head = (head), _n = (next); \ |
| if (!_n && !os_atomic_cmpxchg2o(_q, _ns##_tail, NULL, _head, relaxed)) { \ |
| _dispatch_wait_until(_n = _q->_ns##_head); \ |
| _head->_o_next = _n; \ |
| } \ |
| os_atomic_store2o(_q, _ns##_head, _head, relaxed); \ |
| }) |
| |
| #define os_mpsc_capture_snapshot(q, _ns, tail) ({ \ |
| typeof(q) _q = (q); \ |
| os_mpsc_node_type(_q, _ns) _head; \ |
| _dispatch_wait_until(_head = _q->_ns##_head); \ |
| os_atomic_store2o(_q, _ns##_head, NULL, relaxed); \ |
| /* 22708742: set tail to NULL with release, so that NULL write */ \ |
| /* to head above doesn't clobber head from concurrent enqueuer */ \ |
| *(tail) = os_atomic_xchg2o(_q, _ns##_tail, NULL, release); \ |
| _head; \ |
| }) |
| |
| #define os_mpsc_pop_snapshot_head(head, tail, _o_next) ({ \ |
| os_unqualified_pointer_type(head) _head = (head), _n = NULL; \ |
| if (_head != (tail)) { \ |
| _dispatch_wait_until(_n = _head->_o_next); \ |
| }; \ |
| _n; }) |
| |
| #define os_mpsc_prepend(q, _ns, head, tail, _o_next) ({ \ |
| typeof(q) _q = (q); \ |
| os_mpsc_node_type(_q, _ns) _head = (head), _tail = (tail), _n; \ |
| _tail->_o_next = NULL; \ |
| if (!os_atomic_cmpxchg2o(_q, _ns##_tail, NULL, _tail, release)) { \ |
| _dispatch_wait_until(_n = _q->_ns##_head); \ |
| _tail->_o_next = _n; \ |
| } \ |
| os_atomic_store2o(_q, _ns##_head, _head, relaxed); \ |
| }) |
| |
| #pragma mark - |
| #pragma mark dispatch_queue_t tq lock |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_sidelock_trylock(dispatch_queue_t dq, pthread_priority_t pp) |
| { |
| dispatch_lock_owner owner; |
| if (_dispatch_unfair_lock_trylock(&dq->dq_sidelock, &owner)) { |
| return true; |
| } |
| _dispatch_wqthread_override_start_check_owner(owner, pp, |
| &dq->dq_sidelock.dul_lock); |
| return false; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_sidelock_lock(dispatch_queue_t dq) |
| { |
| return _dispatch_unfair_lock_lock(&dq->dq_sidelock); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_sidelock_tryunlock(dispatch_queue_t dq) |
| { |
| if (_dispatch_unfair_lock_tryunlock(&dq->dq_sidelock)) { |
| return true; |
| } |
| // Ensure that the root queue sees that this thread was overridden. |
| _dispatch_set_defaultpriority_override(); |
| return false; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_sidelock_unlock(dispatch_queue_t dq) |
| { |
| if (_dispatch_unfair_lock_unlock_had_failed_trylock(&dq->dq_sidelock)) { |
| // Ensure that the root queue sees that this thread was overridden. |
| _dispatch_set_defaultpriority_override(); |
| } |
| } |
| |
| #pragma mark - |
| #pragma mark dispatch_queue_t misc |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_queue_t |
| _dispatch_queue_get_current(void) |
| { |
| return (dispatch_queue_t)_dispatch_thread_getspecific(dispatch_queue_key); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_set_current(dispatch_queue_t dq) |
| { |
| _dispatch_thread_setspecific(dispatch_queue_key, dq); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline struct dispatch_object_s* |
| _dispatch_queue_head(dispatch_queue_t dq) |
| { |
| return os_mpsc_get_head(dq, dq_items); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline struct dispatch_object_s* |
| _dispatch_queue_next(dispatch_queue_t dq, struct dispatch_object_s *dc) |
| { |
| return os_mpsc_pop_head(dq, dq_items, dc, do_next); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_push_update_tail(dispatch_queue_t dq, |
| struct dispatch_object_s *tail) |
| { |
| // if we crash here with a value less than 0x1000, then we are |
| // at a known bug in client code. for example, see |
| // _dispatch_queue_dispose or _dispatch_atfork_child |
| return os_mpsc_push_update_tail(dq, dq_items, tail, do_next); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_push_update_tail_list(dispatch_queue_t dq, |
| struct dispatch_object_s *head, struct dispatch_object_s *tail) |
| { |
| // if we crash here with a value less than 0x1000, then we are |
| // at a known bug in client code. for example, see |
| // _dispatch_queue_dispose or _dispatch_atfork_child |
| return os_mpsc_push_update_tail_list(dq, dq_items, head, tail, do_next); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_push_update_head(dispatch_queue_t dq, |
| struct dispatch_object_s *head, bool retained) |
| { |
| if (dx_type(dq) == DISPATCH_QUEUE_GLOBAL_ROOT_TYPE) { |
| dispatch_assert(!retained); |
| // Lie about "retained" here, it generates better assembly in this |
| // hotpath, and _dispatch_root_queue_wakeup knows to ignore this |
| // fake "WAKEUP_CONSUME" bit when it also sees WAKEUP_FLUSH. |
| // |
| // We need to bypass the retain below because pthread root queues |
| // are not global and retaining them would be wrong. |
| // |
| // We should eventually have a typeflag for "POOL" kind of root queues. |
| retained = true; |
| } |
| // The queue must be retained before dq_items_head is written in order |
| // to ensure that the reference is still valid when _dispatch_queue_wakeup |
| // is called. Otherwise, if preempted between the assignment to |
| // dq_items_head and _dispatch_queue_wakeup, the blocks submitted to the |
| // queue may release the last reference to the queue when invoked by |
| // _dispatch_queue_drain. <rdar://problem/6932776> |
| if (!retained) _dispatch_retain(dq); |
| os_mpsc_push_update_head(dq, dq_items, head); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_push_list(dispatch_queue_t dq, dispatch_object_t _head, |
| dispatch_object_t _tail, pthread_priority_t pp, unsigned int n) |
| { |
| struct dispatch_object_s *head = _head._do, *tail = _tail._do; |
| bool override = _dispatch_queue_need_override_retain(dq, pp); |
| dispatch_queue_flags_t flags; |
| if (slowpath(_dispatch_queue_push_update_tail_list(dq, head, tail))) { |
| _dispatch_queue_push_update_head(dq, head, override); |
| if (fastpath(dx_type(dq) == DISPATCH_QUEUE_GLOBAL_ROOT_TYPE)) { |
| return _dispatch_queue_push_list_slow(dq, n); |
| } |
| flags = DISPATCH_WAKEUP_CONSUME | DISPATCH_WAKEUP_FLUSH; |
| } else if (override) { |
| flags = DISPATCH_WAKEUP_CONSUME | DISPATCH_WAKEUP_OVERRIDING; |
| } else { |
| return; |
| } |
| dx_wakeup(dq, pp, flags); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_push_inline(dispatch_queue_t dq, dispatch_object_t _tail, |
| pthread_priority_t pp, dispatch_wakeup_flags_t flags) |
| { |
| struct dispatch_object_s *tail = _tail._do; |
| bool override = _dispatch_queue_need_override(dq, pp); |
| if (flags & DISPATCH_WAKEUP_SLOW_WAITER) { |
| // when SLOW_WAITER is set, we borrow the reference of the caller |
| if (unlikely(_dispatch_queue_push_update_tail(dq, tail))) { |
| _dispatch_queue_push_update_head(dq, tail, true); |
| flags = DISPATCH_WAKEUP_SLOW_WAITER | DISPATCH_WAKEUP_FLUSH; |
| } else if (override) { |
| flags = DISPATCH_WAKEUP_SLOW_WAITER | DISPATCH_WAKEUP_OVERRIDING; |
| } else { |
| flags = DISPATCH_WAKEUP_SLOW_WAITER; |
| } |
| } else { |
| if (override) _dispatch_retain(dq); |
| if (unlikely(_dispatch_queue_push_update_tail(dq, tail))) { |
| _dispatch_queue_push_update_head(dq, tail, override); |
| flags = DISPATCH_WAKEUP_CONSUME | DISPATCH_WAKEUP_FLUSH; |
| } else if (override) { |
| flags = DISPATCH_WAKEUP_CONSUME | DISPATCH_WAKEUP_OVERRIDING; |
| } else { |
| return; |
| } |
| } |
| return dx_wakeup(dq, pp, flags); |
| } |
| |
| struct _dispatch_identity_s { |
| pthread_priority_t old_pp; |
| }; |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_root_queue_identity_assume(struct _dispatch_identity_s *di, |
| pthread_priority_t pp) |
| { |
| // assumed_rq was set by the caller, we need to fake the priorities |
| dispatch_queue_t assumed_rq = _dispatch_queue_get_current(); |
| |
| dispatch_assert(dx_type(assumed_rq) == DISPATCH_QUEUE_GLOBAL_ROOT_TYPE); |
| |
| di->old_pp = _dispatch_get_defaultpriority(); |
| |
| if (!(assumed_rq->dq_priority & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG)) { |
| if (!pp) { |
| pp = _dispatch_get_priority(); |
| // _dispatch_root_queue_drain_deferred_item() may turn a manager |
| // thread into a regular root queue, and we must never try to |
| // restore the manager flag once we became a regular work queue |
| // thread. |
| pp &= ~(pthread_priority_t)_PTHREAD_PRIORITY_EVENT_MANAGER_FLAG; |
| } |
| if ((pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK) > |
| (assumed_rq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { |
| _dispatch_wqthread_override_start(_dispatch_tid_self(), pp); |
| // Ensure that the root queue sees that this thread was overridden. |
| _dispatch_set_defaultpriority_override(); |
| } |
| } |
| _dispatch_reset_defaultpriority(assumed_rq->dq_priority); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_root_queue_identity_restore(struct _dispatch_identity_s *di) |
| { |
| _dispatch_reset_defaultpriority(di->old_pp); |
| } |
| |
| typedef dispatch_queue_t |
| _dispatch_queue_class_invoke_handler_t(dispatch_object_t, |
| dispatch_invoke_flags_t, uint64_t *owned, struct dispatch_object_s **); |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_class_invoke(dispatch_object_t dou, |
| dispatch_invoke_flags_t flags, |
| _dispatch_queue_class_invoke_handler_t invoke) |
| { |
| dispatch_queue_t dq = dou._dq; |
| struct dispatch_object_s *dc = NULL; |
| dispatch_queue_t tq = NULL; |
| uint64_t dq_state, to_unlock = 0; |
| bool owning = !slowpath(flags & DISPATCH_INVOKE_STEALING); |
| bool overriding = slowpath(flags & DISPATCH_INVOKE_OVERRIDING); |
| |
| // When called from a plain _dispatch_queue_drain: |
| // overriding = false |
| // owning = true |
| // |
| // When called from an override continuation: |
| // overriding = true |
| // owning depends on whether the override embedded the queue or steals |
| DISPATCH_COMPILER_CAN_ASSUME(owning || overriding); |
| |
| if (owning) { |
| dq->do_next = DISPATCH_OBJECT_LISTLESS; |
| } |
| to_unlock = _dispatch_queue_drain_try_lock(dq, flags, &dq_state); |
| if (likely(to_unlock)) { |
| struct _dispatch_identity_s di; |
| pthread_priority_t old_dp; |
| |
| drain_pending_barrier: |
| if (overriding) { |
| _dispatch_object_debug(dq, "stolen onto thread 0x%x, 0x%lx", |
| _dispatch_tid_self(), _dispatch_get_defaultpriority()); |
| _dispatch_root_queue_identity_assume(&di, 0); |
| } |
| |
| if (!(flags & DISPATCH_INVOKE_MANAGER_DRAIN)) { |
| pthread_priority_t op, dp; |
| |
| old_dp = _dispatch_set_defaultpriority(dq->dq_priority, &dp); |
| op = dq->dq_override; |
| if (op > (dp & _PTHREAD_PRIORITY_QOS_CLASS_MASK)) { |
| _dispatch_wqthread_override_start(_dispatch_tid_self(), op); |
| // Ensure that the root queue sees that this thread was overridden. |
| _dispatch_set_defaultpriority_override(); |
| } |
| } |
| |
| flags = _dispatch_queue_merge_autorelease_frequency(dq, flags); |
| attempt_running_slow_head: |
| tq = invoke(dq, flags, &to_unlock, &dc); |
| if (slowpath(tq)) { |
| // Either dc is set, which is a deferred invoke case |
| // |
| // or only tq is and it means a reenqueue is required, because of: |
| // a retarget, a suspension, or a width change. |
| // |
| // In both cases, we want to bypass the check for DIRTY. |
| // That may cause us to leave DIRTY in place but all drain lock |
| // acquirers clear it |
| } else { |
| if (!_dispatch_queue_drain_try_unlock(dq, to_unlock)) { |
| goto attempt_running_slow_head; |
| } |
| to_unlock = 0; |
| } |
| if (overriding) { |
| _dispatch_root_queue_identity_restore(&di); |
| } |
| if (!(flags & DISPATCH_INVOKE_MANAGER_DRAIN)) { |
| _dispatch_reset_defaultpriority(old_dp); |
| } |
| } else if (overriding) { |
| uint32_t owner = _dq_state_drain_owner(dq_state); |
| pthread_priority_t p = dq->dq_override; |
| if (owner && p) { |
| _dispatch_object_debug(dq, "overriding thr 0x%x to priority 0x%lx", |
| owner, p); |
| _dispatch_wqthread_override_start_check_owner(owner, p, |
| &dq->dq_state_lock); |
| } |
| } |
| |
| if (owning) { |
| _dispatch_introspection_queue_item_complete(dq); |
| } |
| |
| if (tq && dc) { |
| return _dispatch_queue_drain_deferred_invoke(dq, flags, to_unlock, dc); |
| } |
| |
| if (tq) { |
| bool full_width_upgrade_allowed = (tq == _dispatch_queue_get_current()); |
| uint64_t old_state, new_state; |
| |
| os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, { |
| new_state = old_state - to_unlock; |
| if (full_width_upgrade_allowed && _dq_state_is_runnable(new_state) && |
| _dq_state_has_pending_barrier(new_state)) { |
| new_state += DISPATCH_QUEUE_IN_BARRIER; |
| new_state += DISPATCH_QUEUE_WIDTH_INTERVAL; |
| new_state -= DISPATCH_QUEUE_PENDING_BARRIER; |
| new_state += to_unlock & DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK; |
| } else { |
| new_state = DISPATCH_QUEUE_DRAIN_UNLOCK_PRESERVE_WAITERS_BIT(new_state); |
| if (_dq_state_should_wakeup(new_state)) { |
| // drain was not interupted for suspension |
| // we will reenqueue right away, just put ENQUEUED back |
| new_state |= DISPATCH_QUEUE_ENQUEUED; |
| new_state |= DISPATCH_QUEUE_DIRTY; |
| } |
| } |
| }); |
| if (_dq_state_is_in_barrier(new_state)) { |
| // we did a "full width upgrade" and just added IN_BARRIER |
| // so adjust what we own and drain again |
| to_unlock &= DISPATCH_QUEUE_ENQUEUED; |
| to_unlock += DISPATCH_QUEUE_IN_BARRIER; |
| to_unlock += dq->dq_width * DISPATCH_QUEUE_WIDTH_INTERVAL; |
| goto drain_pending_barrier; |
| } |
| if (_dq_state_has_override(old_state)) { |
| // Ensure that the root queue sees that this thread was overridden. |
| _dispatch_set_defaultpriority_override(); |
| } |
| |
| if ((old_state ^ new_state) & DISPATCH_QUEUE_ENQUEUED) { |
| return _dispatch_queue_push(tq, dq, 0); |
| } |
| } |
| |
| return _dispatch_release_tailcall(dq); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_class_probe(dispatch_queue_class_t dqu) |
| { |
| struct dispatch_object_s *tail; |
| // seq_cst wrt atomic store to dq_state <rdar://problem/14637483> |
| // seq_cst wrt atomic store to dq_flags <rdar://problem/22623242> |
| tail = os_atomic_load2o(dqu._oq, oq_items_tail, ordered); |
| return slowpath(tail != NULL); |
| } |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_CONST |
| static inline bool |
| _dispatch_is_in_root_queues_array(dispatch_queue_t dq) |
| { |
| return (dq >= _dispatch_root_queues) && |
| (dq < _dispatch_root_queues + _DISPATCH_ROOT_QUEUE_IDX_COUNT); |
| } |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_CONST |
| static inline dispatch_queue_t |
| _dispatch_get_root_queue(qos_class_t priority, bool overcommit) |
| { |
| if (overcommit) switch (priority) { |
| case _DISPATCH_QOS_CLASS_MAINTENANCE: |
| return &_dispatch_root_queues[ |
| DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT]; |
| case _DISPATCH_QOS_CLASS_BACKGROUND: |
| return &_dispatch_root_queues[ |
| DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT]; |
| case _DISPATCH_QOS_CLASS_UTILITY: |
| return &_dispatch_root_queues[ |
| DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT]; |
| case _DISPATCH_QOS_CLASS_DEFAULT: |
| return &_dispatch_root_queues[ |
| DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT]; |
| case _DISPATCH_QOS_CLASS_USER_INITIATED: |
| return &_dispatch_root_queues[ |
| DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT]; |
| case _DISPATCH_QOS_CLASS_USER_INTERACTIVE: |
| return &_dispatch_root_queues[ |
| DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT]; |
| } else switch (priority) { |
| case _DISPATCH_QOS_CLASS_MAINTENANCE: |
| return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS]; |
| case _DISPATCH_QOS_CLASS_BACKGROUND: |
| return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS]; |
| case _DISPATCH_QOS_CLASS_UTILITY: |
| return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS]; |
| case _DISPATCH_QOS_CLASS_DEFAULT: |
| return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS]; |
| case _DISPATCH_QOS_CLASS_USER_INITIATED: |
| return &_dispatch_root_queues[ |
| DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS]; |
| case _DISPATCH_QOS_CLASS_USER_INTERACTIVE: |
| return &_dispatch_root_queues[ |
| DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS]; |
| } |
| return NULL; |
| } |
| |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| DISPATCH_ALWAYS_INLINE DISPATCH_CONST |
| static inline dispatch_queue_t |
| _dispatch_get_root_queue_for_priority(pthread_priority_t pp, bool overcommit) |
| { |
| uint32_t idx; |
| |
| pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK; |
| idx = (uint32_t)__builtin_ffs((int)pp); |
| if (unlikely(!_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS] |
| .dq_priority)) { |
| // If kernel doesn't support maintenance, bottom bit is background. |
| // Shift to our idea of where background bit is. |
| idx++; |
| } |
| // ffs starts at 1, and account for the QOS_CLASS_SHIFT |
| // if pp is 0, idx is 0 or 1 and this will wrap to a value larger than |
| // DISPATCH_QOS_COUNT |
| idx -= (_PTHREAD_PRIORITY_QOS_CLASS_SHIFT + 1); |
| if (unlikely(idx >= DISPATCH_QUEUE_QOS_COUNT)) { |
| DISPATCH_CLIENT_CRASH(pp, "Corrupted priority"); |
| } |
| return &_dispatch_root_queues[2 * idx + overcommit]; |
| } |
| #endif |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_CONST |
| static inline dispatch_queue_t |
| _dispatch_get_root_queue_with_overcommit(dispatch_queue_t rq, bool overcommit) |
| { |
| bool rq_overcommit = (rq->dq_priority & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG); |
| // root queues in _dispatch_root_queues are not overcommit for even indices |
| // and overcommit for odd ones, so fixing overcommit is either returning |
| // the same queue, or picking its neighbour in _dispatch_root_queues |
| if (overcommit && !rq_overcommit) { |
| return rq + 1; |
| } |
| if (!overcommit && rq_overcommit) { |
| return rq - 1; |
| } |
| return rq; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_set_bound_thread(dispatch_queue_t dq) |
| { |
| // Tag thread-bound queues with the owning thread |
| dispatch_assert(_dispatch_queue_is_thread_bound(dq)); |
| mach_port_t old_owner, self = _dispatch_tid_self(); |
| uint64_t dq_state = os_atomic_or_orig2o(dq, dq_state, self, relaxed); |
| if (unlikely(old_owner = _dq_state_drain_owner(dq_state))) { |
| DISPATCH_INTERNAL_CRASH(old_owner, "Queue bound twice"); |
| } |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_clear_bound_thread(dispatch_queue_t dq) |
| { |
| uint64_t dq_state, value; |
| |
| dispatch_assert(_dispatch_queue_is_thread_bound(dq)); |
| os_atomic_rmw_loop2o(dq, dq_state, dq_state, value, relaxed, { |
| value = DISPATCH_QUEUE_DRAIN_UNLOCK_PRESERVE_WAITERS_BIT(dq_state); |
| }); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_pthread_root_queue_observer_hooks_t |
| _dispatch_get_pthread_root_queue_observer_hooks(void) |
| { |
| return _dispatch_thread_getspecific( |
| dispatch_pthread_root_queue_observer_hooks_key); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_set_pthread_root_queue_observer_hooks( |
| dispatch_pthread_root_queue_observer_hooks_t observer_hooks) |
| { |
| _dispatch_thread_setspecific(dispatch_pthread_root_queue_observer_hooks_key, |
| observer_hooks); |
| } |
| |
| #pragma mark - |
| #pragma mark dispatch_priority |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _dispatch_get_defaultpriority(void) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pthread_priority_t pp = (uintptr_t)_dispatch_thread_getspecific( |
| dispatch_defaultpriority_key); |
| return pp; |
| #else |
| return 0; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_reset_defaultpriority(pthread_priority_t pp) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pthread_priority_t old_pp = _dispatch_get_defaultpriority(); |
| // If an inner-loop or'd in the override flag to the per-thread priority, |
| // it needs to be propagated up the chain. |
| pp |= old_pp & _PTHREAD_PRIORITY_OVERRIDE_FLAG; |
| _dispatch_thread_setspecific(dispatch_defaultpriority_key, (void*)pp); |
| #else |
| (void)pp; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_set_defaultpriority_override(void) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pthread_priority_t old_pp = _dispatch_get_defaultpriority(); |
| pthread_priority_t pp = old_pp | _PTHREAD_PRIORITY_OVERRIDE_FLAG; |
| |
| _dispatch_thread_setspecific(dispatch_defaultpriority_key, (void*)pp); |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_reset_defaultpriority_override(void) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pthread_priority_t old_pp = _dispatch_get_defaultpriority(); |
| pthread_priority_t pp = old_pp & |
| ~((pthread_priority_t)_PTHREAD_PRIORITY_OVERRIDE_FLAG); |
| |
| _dispatch_thread_setspecific(dispatch_defaultpriority_key, (void*)pp); |
| return unlikely(pp != old_pp); |
| #endif |
| return false; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_priority_inherit_from_target(dispatch_queue_t dq, |
| dispatch_queue_t tq) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| const dispatch_priority_t rootqueue_flag = _PTHREAD_PRIORITY_ROOTQUEUE_FLAG; |
| const dispatch_priority_t inherited_flag = _PTHREAD_PRIORITY_INHERIT_FLAG; |
| const dispatch_priority_t defaultqueue_flag = |
| _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG; |
| dispatch_priority_t dqp = dq->dq_priority, tqp = tq->dq_priority; |
| if ((!(dqp & ~_PTHREAD_PRIORITY_FLAGS_MASK) || (dqp & inherited_flag)) && |
| (tqp & rootqueue_flag)) { |
| if (tqp & defaultqueue_flag) { |
| dq->dq_priority = 0; |
| } else { |
| dq->dq_priority = (tqp & ~rootqueue_flag) | inherited_flag; |
| } |
| } |
| #else |
| (void)dq; (void)tq; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _dispatch_set_defaultpriority(pthread_priority_t pp, pthread_priority_t *new_pp) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| const pthread_priority_t default_priority_preserved_flags = |
| _PTHREAD_PRIORITY_OVERRIDE_FLAG|_PTHREAD_PRIORITY_OVERCOMMIT_FLAG; |
| pthread_priority_t old_pp = _dispatch_get_defaultpriority(); |
| if (old_pp) { |
| pthread_priority_t flags, defaultqueue, basepri; |
| flags = (pp & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG); |
| defaultqueue = (old_pp & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG); |
| basepri = (old_pp & ~_PTHREAD_PRIORITY_FLAGS_MASK); |
| pp &= ~_PTHREAD_PRIORITY_FLAGS_MASK; |
| if (!pp) { |
| flags = _PTHREAD_PRIORITY_INHERIT_FLAG | defaultqueue; |
| pp = basepri; |
| } else if (pp < basepri && !defaultqueue) { // rdar://16349734 |
| pp = basepri; |
| } |
| pp |= flags | (old_pp & default_priority_preserved_flags); |
| } |
| _dispatch_thread_setspecific(dispatch_defaultpriority_key, (void*)pp); |
| if (new_pp) *new_pp = pp; |
| return old_pp; |
| #else |
| (void)pp; (void)new_pp; |
| return 0; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _dispatch_priority_adopt(pthread_priority_t pp, unsigned long flags) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pthread_priority_t defaultpri = _dispatch_get_defaultpriority(); |
| bool enforce, inherited, defaultqueue; |
| enforce = (flags & DISPATCH_PRIORITY_ENFORCE) || |
| (pp & _PTHREAD_PRIORITY_ENFORCE_FLAG); |
| inherited = (defaultpri & _PTHREAD_PRIORITY_INHERIT_FLAG); |
| defaultqueue = (defaultpri & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG); |
| defaultpri &= ~_PTHREAD_PRIORITY_FLAGS_MASK; |
| pp &= ~_PTHREAD_PRIORITY_FLAGS_MASK; |
| |
| if (!pp) { |
| return defaultpri; |
| } else if (defaultqueue) { // rdar://16349734 |
| return pp; |
| } else if (pp < defaultpri) { |
| return defaultpri; |
| } else if (enforce || inherited) { |
| return pp; |
| } else { |
| return defaultpri; |
| } |
| #else |
| (void)pp; (void)flags; |
| return 0; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _dispatch_priority_inherit_from_root_queue(pthread_priority_t pp, |
| dispatch_queue_t rq) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pthread_priority_t p = pp & ~_PTHREAD_PRIORITY_FLAGS_MASK; |
| pthread_priority_t rqp = rq->dq_priority & ~_PTHREAD_PRIORITY_FLAGS_MASK; |
| pthread_priority_t defaultqueue = |
| rq->dq_priority & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG; |
| |
| if (!p || (!defaultqueue && p < rqp)) { |
| p = rqp | defaultqueue; |
| } |
| return p | (rq->dq_priority & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG); |
| #else |
| (void)rq; (void)pp; |
| return 0; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _dispatch_get_priority(void) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pthread_priority_t pp = (uintptr_t) |
| _dispatch_thread_getspecific(dispatch_priority_key); |
| return pp; |
| #else |
| return 0; |
| #endif |
| } |
| |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _dispatch_priority_compute_update(pthread_priority_t pp) |
| { |
| dispatch_assert(pp != DISPATCH_NO_PRIORITY); |
| if (!_dispatch_set_qos_class_enabled) return 0; |
| // the priority in _dispatch_get_priority() only tracks manager-ness |
| // and overcommit, which is inherited from the current value for each update |
| // however if the priority had the NEEDS_UNBIND flag set we need to clear it |
| // the first chance we get |
| // |
| // the manager bit is invalid input, but we keep it to get meaningful |
| // assertions in _dispatch_set_priority_and_voucher_slow() |
| pp &= _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG | ~_PTHREAD_PRIORITY_FLAGS_MASK; |
| pthread_priority_t cur_priority = _dispatch_get_priority(); |
| pthread_priority_t unbind = _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG; |
| pthread_priority_t overcommit = _PTHREAD_PRIORITY_OVERCOMMIT_FLAG; |
| if (unlikely(cur_priority & unbind)) { |
| // else we always need an update if the NEEDS_UNBIND flag is set |
| // the slowpath in _dispatch_set_priority_and_voucher_slow() will |
| // adjust the priority further with the proper overcommitness |
| return pp ? pp : (cur_priority & ~unbind); |
| } else { |
| cur_priority &= ~overcommit; |
| } |
| if (unlikely(pp != cur_priority)) return pp; |
| return 0; |
| } |
| #endif |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline voucher_t |
| _dispatch_set_priority_and_voucher(pthread_priority_t pp, |
| voucher_t v, _dispatch_thread_set_self_t flags) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pp = _dispatch_priority_compute_update(pp); |
| if (likely(!pp)) { |
| if (v == DISPATCH_NO_VOUCHER) { |
| return DISPATCH_NO_VOUCHER; |
| } |
| if (likely(v == _voucher_get())) { |
| bool retained = flags & DISPATCH_VOUCHER_CONSUME; |
| if (flags & DISPATCH_VOUCHER_REPLACE) { |
| if (retained && v) _voucher_release_no_dispose(v); |
| v = DISPATCH_NO_VOUCHER; |
| } else { |
| if (!retained && v) _voucher_retain(v); |
| } |
| return v; |
| } |
| } |
| return _dispatch_set_priority_and_voucher_slow(pp, v, flags); |
| #else |
| (void)pp; (void)v; (void)flags; |
| return DISPATCH_NO_VOUCHER; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline voucher_t |
| _dispatch_adopt_priority_and_set_voucher(pthread_priority_t pp, |
| voucher_t v, _dispatch_thread_set_self_t flags) |
| { |
| pthread_priority_t p = 0; |
| if (pp != DISPATCH_NO_PRIORITY) { |
| p = _dispatch_priority_adopt(pp, flags); |
| } |
| return _dispatch_set_priority_and_voucher(p, v, flags); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_reset_priority_and_voucher(pthread_priority_t pp, voucher_t v) |
| { |
| if (pp == DISPATCH_NO_PRIORITY) pp = 0; |
| (void)_dispatch_set_priority_and_voucher(pp, v, |
| DISPATCH_VOUCHER_CONSUME | DISPATCH_VOUCHER_REPLACE); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_reset_voucher(voucher_t v, _dispatch_thread_set_self_t flags) |
| { |
| flags |= DISPATCH_VOUCHER_CONSUME | DISPATCH_VOUCHER_REPLACE; |
| (void)_dispatch_set_priority_and_voucher(0, v, flags); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_need_override(dispatch_queue_class_t dqu, pthread_priority_t pp) |
| { |
| // global queues have their override set to DISPATCH_SATURATED_OVERRIDE |
| // which makes this test always return false for them. |
| return dqu._oq->oq_override < (pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_received_override(dispatch_queue_class_t dqu, |
| pthread_priority_t pp) |
| { |
| dispatch_assert(dqu._oq->oq_override != DISPATCH_SATURATED_OVERRIDE); |
| return dqu._oq->oq_override > (pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_need_override_retain(dispatch_queue_class_t dqu, |
| pthread_priority_t pp) |
| { |
| if (_dispatch_queue_need_override(dqu, pp)) { |
| _os_object_retain_internal_inline(dqu._oq->_as_os_obj); |
| return true; |
| } |
| return false; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_queue_reinstate_override_priority(dispatch_queue_class_t dqu, |
| dispatch_priority_t new_op) |
| { |
| dispatch_priority_t old_op; |
| new_op &= _PTHREAD_PRIORITY_QOS_CLASS_MASK; |
| if (!new_op) return false; |
| os_atomic_rmw_loop2o(dqu._oq, oq_override, old_op, new_op, relaxed, { |
| if (new_op <= old_op) { |
| os_atomic_rmw_loop_give_up(return false); |
| } |
| }); |
| return true; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_queue_override_priority(dispatch_queue_class_t dqu, |
| pthread_priority_t *pp, dispatch_wakeup_flags_t *flags) |
| { |
| os_mpsc_queue_t oq = dqu._oq; |
| dispatch_priority_t qp = oq->oq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK; |
| dispatch_priority_t np = (*pp & _PTHREAD_PRIORITY_QOS_CLASS_MASK); |
| dispatch_priority_t o; |
| |
| _dispatch_assert_is_valid_qos_override(np); |
| if (oq->oq_priority & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG) { |
| qp = 0; |
| } else if (*flags & DISPATCH_WAKEUP_SLOW_WAITER) { |
| // when a queue is used as a lock its priority doesn't count |
| } else if (np < qp) { |
| // for asynchronous workitems, queue priority is the floor for overrides |
| np = qp; |
| } |
| *flags &= ~_DISPATCH_WAKEUP_OVERRIDE_BITS; |
| |
| // this optimizes for the case when no update of the override is required |
| // os_atomic_rmw_loop2o optimizes for the case when the update happens, |
| // and can't be used. |
| o = os_atomic_load2o(oq, oq_override, relaxed); |
| do { |
| if (likely(np <= o)) break; |
| } while (unlikely(!os_atomic_cmpxchgvw2o(oq, oq_override, o, np, &o, relaxed))); |
| |
| if (np <= o) { |
| *pp = o; |
| } else { |
| *flags |= DISPATCH_WAKEUP_OVERRIDING; |
| *pp = np; |
| } |
| if (o > qp) { |
| *flags |= DISPATCH_WAKEUP_WAS_OVERRIDDEN; |
| } |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_priority_t |
| _dispatch_queue_reset_override_priority(dispatch_queue_class_t dqu, |
| bool qp_is_floor) |
| { |
| os_mpsc_queue_t oq = dqu._oq; |
| dispatch_priority_t p = 0; |
| if (qp_is_floor) { |
| // thread bound queues floor their dq_override to their |
| // priority to avoid receiving useless overrides |
| p = oq->oq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK; |
| } |
| dispatch_priority_t o = os_atomic_xchg2o(oq, oq_override, p, relaxed); |
| dispatch_assert(o != DISPATCH_SATURATED_OVERRIDE); |
| return (o > p) ? o : 0; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _dispatch_priority_propagate(void) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pthread_priority_t pp = _dispatch_get_priority(); |
| pp &= ~_PTHREAD_PRIORITY_FLAGS_MASK; |
| if (pp > _dispatch_user_initiated_priority) { |
| // Cap QOS for propagation at user-initiated <rdar://16681262&16998036> |
| pp = _dispatch_user_initiated_priority; |
| } |
| return pp; |
| #else |
| return 0; |
| #endif |
| } |
| |
| // including maintenance |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_is_background_thread(void) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pthread_priority_t pp = _dispatch_get_priority(); |
| pp &= ~_PTHREAD_PRIORITY_FLAGS_MASK; |
| return pp && (pp <= _dispatch_background_priority); |
| #else |
| return false; |
| #endif |
| } |
| |
| #pragma mark - |
| #pragma mark dispatch_block_t |
| |
| #ifdef __BLOCKS__ |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_block_has_private_data(const dispatch_block_t block) |
| { |
| extern void (*_dispatch_block_special_invoke)(void*); |
| return (_dispatch_Block_invoke(block) == _dispatch_block_special_invoke); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_block_sync_should_enforce_qos_class(dispatch_block_flags_t flags) |
| { |
| /* |
| * Generates better assembly than the actual readable test: |
| * (flags & ENFORCE_QOS_CLASS) || !(flags & INHERIT_QOS_FLAGS) |
| */ |
| flags &= DISPATCH_BLOCK_ENFORCE_QOS_CLASS | DISPATCH_BLOCK_INHERIT_QOS_CLASS; |
| return flags != DISPATCH_BLOCK_INHERIT_QOS_CLASS; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_block_private_data_t |
| _dispatch_block_get_data(const dispatch_block_t db) |
| { |
| if (!_dispatch_block_has_private_data(db)) { |
| return NULL; |
| } |
| // Keep in sync with _dispatch_block_create implementation |
| uint8_t *x = (uint8_t *)db; |
| // x points to base of struct Block_layout |
| x += sizeof(struct Block_layout); |
| // x points to base of captured dispatch_block_private_data_s object |
| dispatch_block_private_data_t dbpd = (dispatch_block_private_data_t)x; |
| if (dbpd->dbpd_magic != DISPATCH_BLOCK_PRIVATE_DATA_MAGIC) { |
| DISPATCH_CLIENT_CRASH(dbpd->dbpd_magic, |
| "Corruption of dispatch block object"); |
| } |
| return dbpd; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _dispatch_block_get_priority(const dispatch_block_t db) |
| { |
| dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); |
| return dbpd ? dbpd->dbpd_priority : 0; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_block_flags_t |
| _dispatch_block_get_flags(const dispatch_block_t db) |
| { |
| dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); |
| return dbpd ? dbpd->dbpd_flags : 0; |
| } |
| |
| #endif |
| |
| #pragma mark - |
| #pragma mark dispatch_continuation_t |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_continuation_t |
| _dispatch_continuation_alloc_cacheonly(void) |
| { |
| dispatch_continuation_t dc = (dispatch_continuation_t) |
| _dispatch_thread_getspecific(dispatch_cache_key); |
| if (likely(dc)) { |
| _dispatch_thread_setspecific(dispatch_cache_key, dc->do_next); |
| } |
| return dc; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_continuation_t |
| _dispatch_continuation_alloc(void) |
| { |
| dispatch_continuation_t dc = |
| _dispatch_continuation_alloc_cacheonly(); |
| if (unlikely(!dc)) { |
| return _dispatch_continuation_alloc_from_heap(); |
| } |
| return dc; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_continuation_t |
| _dispatch_continuation_free_cacheonly(dispatch_continuation_t dc) |
| { |
| dispatch_continuation_t prev_dc = (dispatch_continuation_t) |
| _dispatch_thread_getspecific(dispatch_cache_key); |
| int cnt = prev_dc ? prev_dc->dc_cache_cnt + 1 : 1; |
| // Cap continuation cache |
| if (unlikely(cnt > _dispatch_continuation_cache_limit)) { |
| return dc; |
| } |
| dc->do_next = prev_dc; |
| dc->dc_cache_cnt = cnt; |
| _dispatch_thread_setspecific(dispatch_cache_key, dc); |
| return NULL; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_continuation_free(dispatch_continuation_t dc) |
| { |
| dc = _dispatch_continuation_free_cacheonly(dc); |
| if (unlikely(dc)) { |
| _dispatch_continuation_free_to_cache_limit(dc); |
| } |
| } |
| |
| #include "trace.h" |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_continuation_with_group_invoke(dispatch_continuation_t dc) |
| { |
| struct dispatch_object_s *dou = dc->dc_data; |
| unsigned long type = dx_type(dou); |
| if (type == DISPATCH_GROUP_TYPE) { |
| _dispatch_client_callout(dc->dc_ctxt, dc->dc_func); |
| _dispatch_introspection_queue_item_complete(dou); |
| dispatch_group_leave((dispatch_group_t)dou); |
| } else { |
| DISPATCH_INTERNAL_CRASH(dx_type(dou), "Unexpected object type"); |
| } |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_continuation_invoke_inline(dispatch_object_t dou, voucher_t ov, |
| dispatch_invoke_flags_t flags) |
| { |
| dispatch_continuation_t dc = dou._dc, dc1; |
| dispatch_invoke_with_autoreleasepool(flags, { |
| uintptr_t dc_flags = dc->dc_flags; |
| // Add the item back to the cache before calling the function. This |
| // allows the 'hot' continuation to be used for a quick callback. |
| // |
| // The ccache version is per-thread. |
| // Therefore, the object has not been reused yet. |
| // This generates better assembly. |
| _dispatch_continuation_voucher_adopt(dc, ov, dc_flags); |
| if (dc_flags & DISPATCH_OBJ_CONSUME_BIT) { |
| dc1 = _dispatch_continuation_free_cacheonly(dc); |
| } else { |
| dc1 = NULL; |
| } |
| if (unlikely(dc_flags & DISPATCH_OBJ_GROUP_BIT)) { |
| _dispatch_continuation_with_group_invoke(dc); |
| } else { |
| _dispatch_client_callout(dc->dc_ctxt, dc->dc_func); |
| _dispatch_introspection_queue_item_complete(dou); |
| } |
| if (unlikely(dc1)) { |
| _dispatch_continuation_free_to_cache_limit(dc1); |
| } |
| }); |
| } |
| |
| DISPATCH_ALWAYS_INLINE_NDEBUG |
| static inline void |
| _dispatch_continuation_pop_inline(dispatch_object_t dou, dispatch_queue_t dq, |
| dispatch_invoke_flags_t flags) |
| { |
| dispatch_pthread_root_queue_observer_hooks_t observer_hooks = |
| _dispatch_get_pthread_root_queue_observer_hooks(); |
| if (observer_hooks) observer_hooks->queue_will_execute(dq); |
| _dispatch_trace_continuation_pop(dq, dou); |
| flags &= _DISPATCH_INVOKE_PROPAGATE_MASK; |
| if (_dispatch_object_has_vtable(dou)) { |
| dx_invoke(dou._do, flags); |
| } else { |
| voucher_t ov = dq->dq_override_voucher; |
| _dispatch_continuation_invoke_inline(dou, ov, flags); |
| } |
| if (observer_hooks) observer_hooks->queue_did_execute(dq); |
| } |
| |
| // used to forward the do_invoke of a continuation with a vtable to its real |
| // implementation. |
| #define _dispatch_continuation_pop_forwarded(dc, ov, dc_flags, ...) \ |
| ({ \ |
| dispatch_continuation_t _dc = (dc), _dc1; \ |
| uintptr_t _dc_flags = (dc_flags); \ |
| _dispatch_continuation_voucher_adopt(_dc, ov, _dc_flags); \ |
| if (_dc_flags & DISPATCH_OBJ_CONSUME_BIT) { \ |
| _dc1 = _dispatch_continuation_free_cacheonly(_dc); \ |
| } else { \ |
| _dc1 = NULL; \ |
| } \ |
| __VA_ARGS__; \ |
| _dispatch_introspection_queue_item_complete(_dc); \ |
| if (unlikely(_dc1)) { \ |
| _dispatch_continuation_free_to_cache_limit(_dc1); \ |
| } \ |
| }) |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_continuation_priority_set(dispatch_continuation_t dc, |
| pthread_priority_t pp, dispatch_block_flags_t flags) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| if (likely(!(flags & DISPATCH_BLOCK_HAS_PRIORITY))) { |
| pp = _dispatch_priority_propagate(); |
| } |
| if (flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS) { |
| pp |= _PTHREAD_PRIORITY_ENFORCE_FLAG; |
| } |
| dc->dc_priority = pp; |
| #else |
| (void)dc; (void)pp; (void)flags; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _dispatch_continuation_get_override_priority(dispatch_queue_t dq, |
| dispatch_continuation_t dc) |
| { |
| #if HAVE_PTHREAD_WORKQUEUE_QOS |
| pthread_priority_t p = dc->dc_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK; |
| bool enforce = dc->dc_priority & _PTHREAD_PRIORITY_ENFORCE_FLAG; |
| pthread_priority_t dqp = dq->dq_priority & _PTHREAD_PRIORITY_QOS_CLASS_MASK; |
| bool defaultqueue = dq->dq_priority & _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG; |
| |
| dispatch_assert(dc->dc_priority != DISPATCH_NO_PRIORITY); |
| if (p && (enforce || !dqp || defaultqueue)) { |
| return p; |
| } |
| return dqp; |
| #else |
| (void)dq; (void)dc; |
| return 0; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_continuation_init_f(dispatch_continuation_t dc, |
| dispatch_queue_class_t dqu, void *ctxt, dispatch_function_t func, |
| pthread_priority_t pp, dispatch_block_flags_t flags, uintptr_t dc_flags) |
| { |
| dc->dc_flags = dc_flags; |
| dc->dc_func = func; |
| dc->dc_ctxt = ctxt; |
| _dispatch_continuation_voucher_set(dc, dqu, flags); |
| _dispatch_continuation_priority_set(dc, pp, flags); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_continuation_init(dispatch_continuation_t dc, |
| dispatch_queue_class_t dqu, dispatch_block_t work, |
| pthread_priority_t pp, dispatch_block_flags_t flags, uintptr_t dc_flags) |
| { |
| dc->dc_flags = dc_flags | DISPATCH_OBJ_BLOCK_BIT; |
| dc->dc_ctxt = _dispatch_Block_copy(work); |
| _dispatch_continuation_priority_set(dc, pp, flags); |
| |
| if (unlikely(_dispatch_block_has_private_data(work))) { |
| // always sets dc_func & dc_voucher |
| // may update dc_priority & do_vtable |
| return _dispatch_continuation_init_slow(dc, dqu, flags); |
| } |
| |
| if (dc_flags & DISPATCH_OBJ_CONSUME_BIT) { |
| dc->dc_func = _dispatch_call_block_and_release; |
| } else { |
| dc->dc_func = _dispatch_Block_invoke(work); |
| } |
| _dispatch_continuation_voucher_set(dc, dqu, flags); |
| } |
| |
| #endif // DISPATCH_PURE_C |
| |
| #endif /* __DISPATCH_INLINE_INTERNAL__ */ |