| /* |
| * Copyright (c) 2013 Apple Inc. All rights reserved. |
| * |
| * @APPLE_APACHE_LICENSE_HEADER_START@ |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| * |
| * @APPLE_APACHE_LICENSE_HEADER_END@ |
| */ |
| |
| /* |
| * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch |
| * which are subject to change in future releases of Mac OS X. Any applications |
| * relying on these interfaces WILL break. |
| */ |
| |
| #ifndef __DISPATCH_VOUCHER_INTERNAL__ |
| #define __DISPATCH_VOUCHER_INTERNAL__ |
| |
| #ifndef __DISPATCH_INDIRECT__ |
| #error "Please #include <dispatch/dispatch.h> instead of this file directly." |
| #include <dispatch/base.h> // for HeaderDoc |
| #endif |
| |
| #pragma mark - |
| #pragma mark voucher_recipe_t (disabled) |
| |
| #if VOUCHER_ENABLE_RECIPE_OBJECTS |
| /*! |
| * @group Voucher Creation SPI |
| * SPI intended for clients that need to create vouchers. |
| */ |
| OS_OBJECT_DECL_CLASS(voucher_recipe); |
| |
| /*! |
| * @function voucher_create |
| * |
| * @abstract |
| * Creates a new voucher object from a recipe. |
| * |
| * @discussion |
| * Error handling TBD |
| * |
| * @result |
| * The newly created voucher object. |
| */ |
| API_AVAILABLE(macos(10.10), ios(8.0)) |
| OS_EXPORT OS_OBJECT_RETURNS_RETAINED OS_WARN_RESULT OS_NOTHROW |
| voucher_t |
| voucher_create(voucher_recipe_t recipe); |
| #endif // VOUCHER_ENABLE_RECIPE_OBJECTS |
| |
| #if VOUCHER_ENABLE_GET_MACH_VOUCHER |
| /*! |
| * @function voucher_get_mach_voucher |
| * |
| * @abstract |
| * Returns the mach voucher port underlying the specified voucher object. |
| * |
| * @discussion |
| * The caller must either maintain a reference on the voucher object while the |
| * returned mach voucher port is in use to ensure it stays valid for the |
| * duration, or it must retain the mach voucher port with mach_port_mod_refs(). |
| * |
| * @param voucher |
| * The voucher object to query. |
| * |
| * @result |
| * A mach voucher port. |
| */ |
| API_AVAILABLE(macos(10.10), ios(8.0)) |
| OS_VOUCHER_EXPORT OS_WARN_RESULT OS_NOTHROW |
| mach_voucher_t |
| voucher_get_mach_voucher(voucher_t voucher); |
| #endif // VOUCHER_ENABLE_GET_MACH_VOUCHER |
| |
| #pragma mark - |
| #pragma mark voucher_t |
| |
| void _voucher_init(void); |
| void _voucher_atfork_child(void); |
| void _voucher_activity_debug_channel_init(void); |
| #if OS_VOUCHER_ACTIVITY_SPI && OS_VOUCHER_ACTIVITY_GENERATE_SWAPS |
| void _voucher_activity_swap(firehose_activity_id_t old_id, |
| firehose_activity_id_t new_id); |
| #endif |
| void _voucher_xref_dispose(voucher_t voucher); |
| void _voucher_dispose(voucher_t voucher); |
| size_t _voucher_debug(voucher_t v, char* buf, size_t bufsiz); |
| void _voucher_thread_cleanup(void *voucher); |
| mach_voucher_t _voucher_get_mach_voucher(voucher_t voucher); |
| voucher_t _voucher_create_without_importance(voucher_t voucher); |
| voucher_t _voucher_create_accounting_voucher(voucher_t voucher); |
| mach_voucher_t _voucher_create_mach_voucher_with_priority(voucher_t voucher, |
| pthread_priority_t priority); |
| voucher_t _voucher_create_with_priority_and_mach_voucher(voucher_t voucher, |
| pthread_priority_t priority, mach_voucher_t kv); |
| void _voucher_dealloc_mach_voucher(mach_voucher_t kv); |
| |
| #if VOUCHER_ENABLE_RECIPE_OBJECTS |
| _OS_OBJECT_DECL_SUBCLASS_INTERFACE(voucher_recipe, object) |
| #endif |
| |
| voucher_t voucher_retain(voucher_t voucher); |
| void voucher_release(voucher_t voucher); |
| |
| #define VOUCHER_NO_MACH_VOUCHER MACH_PORT_DEAD |
| |
| #if VOUCHER_USE_MACH_VOUCHER |
| |
| #if DISPATCH_DEBUG |
| #define DISPATCH_VOUCHER_DEBUG 1 |
| #define DISPATCH_VOUCHER_ACTIVITY_DEBUG 1 |
| #endif |
| |
| #include <voucher/ipc_pthread_priority_types.h> |
| |
| typedef uint32_t _voucher_magic_t; |
| typedef uint32_t _voucher_priority_t; |
| |
| #define VOUCHER_MAGIC_V3 ((_voucher_magic_t)0x0390cefa) // FACE9003 |
| |
| typedef struct _voucher_mach_udata_s { |
| _voucher_magic_t vmu_magic; |
| _voucher_priority_t vmu_priority; |
| uint8_t _vmu_after_priority[0]; |
| firehose_activity_id_t vmu_activity; |
| uint64_t vmu_activity_pid; |
| firehose_activity_id_t vmu_parent_activity; |
| uint8_t _vmu_after_activity[0]; |
| } _voucher_mach_udata_s; |
| |
| OS_ENUM(voucher_fields, uint16_t, |
| VOUCHER_FIELD_NONE = 0, |
| VOUCHER_FIELD_KVOUCHER = 1u << 0, |
| VOUCHER_FIELD_PRIORITY = 1u << 1, |
| VOUCHER_FIELD_ACTIVITY = 1u << 2, |
| |
| #if VOUCHER_ENABLE_RECIPE_OBJECTS |
| VOUCHER_FIELD_EXTRA = 1u << 15, |
| #else |
| VOUCHER_FIELD_EXTRA = 0, |
| #endif |
| ); |
| |
| typedef struct voucher_s { |
| _OS_OBJECT_HEADER( |
| struct voucher_vtable_s *os_obj_isa, |
| os_obj_ref_cnt, |
| os_obj_xref_cnt); |
| struct voucher_hash_entry_s { |
| uintptr_t vhe_next; |
| uintptr_t vhe_prev_ptr; |
| } v_list; |
| mach_voucher_t v_kvoucher, v_ipc_kvoucher; // if equal, only one reference |
| voucher_t v_kvbase; // if non-NULL, v_kvoucher is a borrowed reference |
| firehose_activity_id_t v_activity; |
| uint64_t v_activity_creator; |
| firehose_activity_id_t v_parent_activity; |
| _voucher_priority_t v_priority; |
| unsigned int v_kv_has_importance:1; |
| #if VOUCHER_ENABLE_RECIPE_OBJECTS |
| size_t v_recipe_extra_offset; |
| mach_voucher_attr_recipe_size_t v_recipe_extra_size; |
| #endif |
| } voucher_s; |
| |
| typedef struct voucher_hash_head_s { |
| uintptr_t vhh_first; |
| uintptr_t vhh_last_ptr; |
| } voucher_hash_head_s; |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _voucher_hash_is_enqueued(const struct voucher_s *v) |
| { |
| return v->v_list.vhe_prev_ptr != 0; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_hash_mark_not_enqueued(struct voucher_s *v) |
| { |
| v->v_list.vhe_prev_ptr = 0; |
| v->v_list.vhe_next = (uintptr_t)DISPATCH_OBJECT_LISTLESS; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_hash_set_next(uintptr_t *next, struct voucher_s *v) |
| { |
| *next = ~(uintptr_t)v; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline voucher_t |
| _voucher_hash_get_next(uintptr_t next) |
| { |
| return (voucher_t)~next; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_hash_set_prev_ptr(uintptr_t *prev_ptr, uintptr_t *addr) |
| { |
| *prev_ptr = ~(uintptr_t)addr; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_hash_store_to_prev_ptr(uintptr_t prev_ptr, struct voucher_s *v) |
| { |
| *(uintptr_t *)~prev_ptr = ~(uintptr_t)v; |
| } |
| |
| #if VOUCHER_ENABLE_RECIPE_OBJECTS |
| #define _voucher_extra_size(v) ((v)->v_recipe_extra_size) |
| #define _voucher_extra_recipes(v) ((char*)(v) + (v)->v_recipe_extra_offset) |
| #else |
| #define _voucher_extra_size(v) 0 |
| #define _voucher_extra_recipes(v) NULL |
| #endif |
| |
| #if VOUCHER_ENABLE_RECIPE_OBJECTS |
| typedef struct voucher_recipe_s { |
| _OS_OBJECT_HEADER( |
| const _os_object_vtable_s *os_obj_isa, |
| os_obj_ref_cnt, |
| os_obj_xref_cnt); |
| size_t vr_allocation_size; |
| mach_voucher_attr_recipe_size_t volatile vr_size; |
| mach_voucher_attr_recipe_t vr_data; |
| } voucher_recipe_s; |
| #endif |
| |
| #if TARGET_OS_EMBEDDED |
| #define VL_HASH_SIZE 64u // must be a power of two |
| #else |
| #define VL_HASH_SIZE 256u // must be a power of two |
| #endif |
| #define VL_HASH(kv) (MACH_PORT_INDEX(kv) & (VL_HASH_SIZE - 1)) |
| |
| #if DISPATCH_DEBUG && DISPATCH_VOUCHER_DEBUG |
| #define _dispatch_voucher_debug(msg, v, ...) \ |
| _dispatch_debug("voucher[%p]: " msg, v, ##__VA_ARGS__) |
| #define _dispatch_kvoucher_debug(msg, kv, ...) \ |
| _dispatch_debug("kvoucher[0x%08x]: " msg, kv, ##__VA_ARGS__) |
| #define _dispatch_voucher_debug_machport(name) _dispatch_debug_machport(name) |
| #else |
| #define _dispatch_voucher_debug(msg, v, ...) |
| #define _dispatch_kvoucher_debug(msg, kv, ...) |
| #define _dispatch_voucher_debug_machport(name) ((void)(name)) |
| #endif |
| |
| #ifndef DISPATCH_VOUCHER_OBJC_DEBUG |
| #if DISPATCH_INTROSPECTION || DISPATCH_DEBUG |
| #define DISPATCH_VOUCHER_OBJC_DEBUG 1 |
| #else |
| #define DISPATCH_VOUCHER_OBJC_DEBUG 0 |
| #endif |
| #endif // DISPATCH_VOUCHER_OBJC_DEBUG |
| |
| #if DISPATCH_PURE_C |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline voucher_t |
| _voucher_retain(voucher_t voucher) |
| { |
| #if !DISPATCH_VOUCHER_OBJC_DEBUG |
| // not using _os_object_refcnt* because we don't need barriers: |
| // vouchers are immutable and are in a hash table with a lock |
| int xref_cnt = os_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed); |
| _dispatch_voucher_debug("retain -> %d", voucher, xref_cnt + 1); |
| if (unlikely(xref_cnt <= 0)) { |
| _OS_OBJECT_CLIENT_CRASH("Voucher resurrection"); |
| } |
| #else |
| os_retain(voucher); |
| _dispatch_voucher_debug("retain -> %d", voucher, |
| voucher->os_obj_xref_cnt + 1); |
| #endif // DISPATCH_DEBUG |
| return voucher; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_release(voucher_t voucher) |
| { |
| #if !DISPATCH_VOUCHER_OBJC_DEBUG |
| // not using _os_object_refcnt* because we don't need barriers: |
| // vouchers are immutable and are in a hash table with a lock |
| int xref_cnt = os_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed); |
| _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1); |
| if (likely(xref_cnt >= 0)) { |
| return; |
| } |
| if (unlikely(xref_cnt < -1)) { |
| _OS_OBJECT_CLIENT_CRASH("Voucher over-release"); |
| } |
| return _os_object_xref_dispose((_os_object_t)voucher); |
| #else |
| _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt); |
| return os_release(voucher); |
| #endif // DISPATCH_DEBUG |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_release_no_dispose(voucher_t voucher) |
| { |
| #if !DISPATCH_VOUCHER_OBJC_DEBUG |
| // not using _os_object_refcnt* because we don't need barriers: |
| // vouchers are immutable and are in a hash table with a lock |
| int xref_cnt = os_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed); |
| _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1); |
| if (likely(xref_cnt >= 0)) { |
| return; |
| } |
| _OS_OBJECT_CLIENT_CRASH("Voucher over-release"); |
| #else |
| _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt); |
| return os_release(voucher); |
| #endif // DISPATCH_DEBUG |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline voucher_t |
| _voucher_get(void) |
| { |
| return _dispatch_thread_getspecific(dispatch_voucher_key); |
| } |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline voucher_t |
| _voucher_copy(void) |
| { |
| voucher_t voucher = _voucher_get(); |
| if (voucher) _voucher_retain(voucher); |
| return voucher; |
| } |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline voucher_t |
| _voucher_copy_without_importance(void) |
| { |
| voucher_t voucher = _voucher_get(); |
| if (voucher) voucher = _voucher_create_without_importance(voucher); |
| return voucher; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_mach_voucher_set(mach_voucher_t kv) |
| { |
| if (kv == VOUCHER_NO_MACH_VOUCHER) return; |
| _dispatch_set_priority_and_mach_voucher_slow(0, kv); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline mach_voucher_t |
| _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher) |
| { |
| if (ov == voucher) return VOUCHER_NO_MACH_VOUCHER; |
| _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov); |
| _dispatch_thread_setspecific(dispatch_voucher_key, voucher); |
| mach_voucher_t kv = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL; |
| mach_voucher_t okv = ov ? ov->v_kvoucher : MACH_VOUCHER_NULL; |
| #if OS_VOUCHER_ACTIVITY_GENERATE_SWAPS |
| firehose_activity_id_t aid = voucher ? voucher->v_activity : 0; |
| firehose_activity_id_t oaid = ov ? ov->v_activity : 0; |
| if (aid != oaid) _voucher_activity_swap(aid, oaid); |
| #endif |
| return (kv != okv) ? kv : VOUCHER_NO_MACH_VOUCHER; |
| } |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline voucher_t |
| _voucher_adopt(voucher_t voucher) |
| { |
| voucher_t ov = _voucher_get(); |
| _voucher_mach_voucher_set(_voucher_swap_and_get_mach_voucher(ov, voucher)); |
| return ov; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_replace(voucher_t voucher) |
| { |
| voucher_t ov = _voucher_adopt(voucher); |
| if (ov) _voucher_release(ov); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_clear(void) |
| { |
| _voucher_replace(NULL); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _voucher_get_priority(voucher_t v) |
| { |
| return v ? (pthread_priority_t)v->v_priority : 0; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline firehose_activity_id_t |
| _voucher_get_activity_id(voucher_t v, uint64_t *creator_pid) |
| { |
| if (creator_pid) *creator_pid = v ? v->v_activity_creator : 0; |
| return v ? v->v_activity : 0; |
| } |
| |
| void _voucher_task_mach_voucher_init(void* ctxt); |
| extern dispatch_once_t _voucher_task_mach_voucher_pred; |
| extern mach_voucher_t _voucher_task_mach_voucher; |
| #if VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER |
| #define _voucher_default_task_mach_voucher MACH_VOUCHER_NULL |
| #else |
| extern mach_voucher_t _voucher_default_task_mach_voucher; |
| #endif |
| DISPATCH_ALWAYS_INLINE |
| static inline mach_voucher_t |
| _voucher_get_task_mach_voucher(void) |
| { |
| dispatch_once_f(&_voucher_task_mach_voucher_pred, NULL, |
| _voucher_task_mach_voucher_init); |
| return _voucher_task_mach_voucher; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv, |
| bool move_send) |
| { |
| if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false; |
| if (!kv) return false; |
| msg->msgh_voucher_port = kv; |
| msg->msgh_bits |= MACH_MSGH_BITS_SET_PORTS(0, 0, move_send ? |
| MACH_MSG_TYPE_MOVE_SEND : MACH_MSG_TYPE_COPY_SEND); |
| _dispatch_kvoucher_debug("msg[%p] set %s", kv, msg, move_send ? |
| "move-send" : "copy-send"); |
| _dispatch_voucher_debug_machport(kv); |
| return true; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher) |
| { |
| if (MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) return false; |
| mach_voucher_t kv; |
| if (voucher) { |
| kv = _voucher_get_mach_voucher(voucher); |
| } else { |
| kv = _voucher_get_task_mach_voucher(); |
| } |
| return _voucher_mach_msg_set_mach_voucher(msg, kv, false); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline mach_voucher_t |
| _voucher_mach_msg_get(mach_msg_header_t *msg, mach_msg_bits_t *msgh_bits) |
| { |
| if (!MACH_MSGH_BITS_HAS_VOUCHER(msg->msgh_bits)) { |
| *msgh_bits = 0; |
| return MACH_VOUCHER_NULL; |
| } |
| mach_voucher_t kv = msg->msgh_voucher_port; |
| msg->msgh_voucher_port = MACH_VOUCHER_NULL; |
| mach_msg_bits_t mask = MACH_MSGH_BITS_VOUCHER_MASK|MACH_MSGH_BITS_RAISEIMP; |
| *msgh_bits = msg->msgh_bits & mask; |
| msg->msgh_bits &= ~mask; |
| return kv; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline mach_voucher_t |
| _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send) |
| { |
| mach_msg_bits_t kvbits = MACH_MSGH_BITS_VOUCHER(msg->msgh_bits); |
| mach_voucher_t kv = msg->msgh_voucher_port, kvm = MACH_VOUCHER_NULL; |
| if ((kvbits == MACH_MSG_TYPE_COPY_SEND || |
| kvbits == MACH_MSG_TYPE_MOVE_SEND) && kv) { |
| _dispatch_kvoucher_debug("msg[%p] clear %s", kv, msg, move_send ? |
| "move-send" : "copy-send"); |
| _dispatch_voucher_debug_machport(kv); |
| if (kvbits == MACH_MSG_TYPE_MOVE_SEND) { |
| // <rdar://problem/15694142> return/drop received or pseudo-received |
| // voucher reference (e.g. due to send failure). |
| if (move_send) { |
| kvm = kv; |
| } else { |
| _voucher_dealloc_mach_voucher(kv); |
| } |
| } |
| msg->msgh_voucher_port = MACH_VOUCHER_NULL; |
| msg->msgh_bits &= (mach_msg_bits_t)~MACH_MSGH_BITS_VOUCHER_MASK; |
| } |
| return kvm; |
| } |
| |
| #pragma mark - |
| #pragma mark dispatch_continuation_t + voucher_t |
| |
| #if DISPATCH_USE_VOUCHER_KDEBUG_TRACE |
| #define DISPATCH_VOUCHER_CODE(code) DISPATCH_CODE(VOUCHER, code) |
| #else |
| #define DISPATCH_VOUCHER_CODE(code) 0 |
| #endif // DISPATCH_USE_VOUCHER_KDEBUG_TRACE |
| |
| #define DISPATCH_TRACE_VOUCHER_DC_PUSH DISPATCH_VOUCHER_CODE(0x1) |
| #define DISPATCH_TRACE_VOUCHER_DC_POP DISPATCH_VOUCHER_CODE(0x2) |
| #define DISPATCH_TRACE_VOUCHER_DMSG_PUSH DISPATCH_VOUCHER_CODE(0x3) |
| #define DISPATCH_TRACE_VOUCHER_DMSG_POP DISPATCH_VOUCHER_CODE(0x4) |
| #define DISPATCH_TRACE_VOUCHER_ACTIVITY_ADOPT DISPATCH_VOUCHER_CODE(0x5) |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_voucher_ktrace(uint32_t code, voucher_t v, const void *container) |
| { |
| if (v == DISPATCH_NO_VOUCHER) return; |
| natural_t voucher = v ? v->v_kvoucher : MACH_VOUCHER_NULL; |
| _dispatch_ktrace2(code, voucher, (uintptr_t)container); |
| } |
| #define _dispatch_voucher_ktrace(code, v, container) \ |
| _dispatch_voucher_ktrace(DISPATCH_TRACE_VOUCHER_##code, v, container) |
| #define _dispatch_voucher_ktrace_dc_push(dc) \ |
| _dispatch_voucher_ktrace(DC_PUSH, (dc)->dc_voucher, (dc)) |
| #define _dispatch_voucher_ktrace_dc_pop(dc, v) \ |
| _dispatch_voucher_ktrace(DC_POP, v, (dc)) |
| #define _dispatch_voucher_ktrace_dmsg_push(dmsg) \ |
| _dispatch_voucher_ktrace(DMSG_PUSH, (dmsg)->dmsg_voucher, (dmsg)) |
| #define _dispatch_voucher_ktrace_dmsg_pop(dmsg) \ |
| _dispatch_voucher_ktrace(DMSG_POP, (dmsg)->dmsg_voucher, (dmsg)) |
| #define _dispatch_voucher_ktrace_activity_adopt(aid) \ |
| _dispatch_ktrace1(DISPATCH_TRACE_VOUCHER_ACTIVITY_ADOPT, aid); |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_continuation_voucher_set(dispatch_continuation_t dc, |
| dispatch_queue_class_t dqu, dispatch_block_flags_t flags) |
| { |
| voucher_t v = NULL; |
| |
| (void)dqu; |
| // _dispatch_continuation_voucher_set is never called for blocks with |
| // private data or with the DISPATCH_BLOCK_HAS_VOUCHER flag set. |
| // only _dispatch_continuation_init_slow handles this bit. |
| dispatch_assert(!(flags & DISPATCH_BLOCK_HAS_VOUCHER)); |
| |
| if (!(flags & DISPATCH_BLOCK_NO_VOUCHER)) { |
| v = _voucher_copy(); |
| } |
| dc->dc_voucher = v; |
| _dispatch_voucher_debug("continuation[%p] set", dc->dc_voucher, dc); |
| _dispatch_voucher_ktrace_dc_push(dc); |
| } |
| |
| static inline dispatch_queue_t _dispatch_queue_get_current(void); |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc, |
| voucher_t ov, uintptr_t dc_flags) |
| { |
| voucher_t v = dc->dc_voucher; |
| dispatch_thread_set_self_t consume = (dc_flags & DISPATCH_OBJ_CONSUME_BIT); |
| dispatch_assert(DISPATCH_OBJ_CONSUME_BIT == DISPATCH_VOUCHER_CONSUME); |
| |
| if (consume) { |
| dc->dc_voucher = VOUCHER_INVALID; |
| } |
| if (likely(v != DISPATCH_NO_VOUCHER)) { |
| _dispatch_voucher_ktrace_dc_pop(dc, v); |
| _dispatch_voucher_debug("continuation[%p] adopt", v, dc); |
| |
| if (likely(!(dc_flags & DISPATCH_OBJ_ENFORCE_VOUCHER))) { |
| if (unlikely(ov != DISPATCH_NO_VOUCHER && v != ov)) { |
| if (consume && v) _voucher_release(v); |
| consume = 0; |
| v = ov; |
| } |
| } |
| } else { |
| consume = 0; |
| v = ov; |
| } |
| (void)_dispatch_adopt_priority_and_set_voucher(dc->dc_priority, v, |
| consume | DISPATCH_VOUCHER_REPLACE); |
| } |
| |
| #pragma mark - |
| #pragma mark _voucher activity subsystem |
| |
| extern dispatch_once_t _firehose_task_buffer_pred; |
| extern union firehose_buffer_u *_firehose_task_buffer; |
| extern uint64_t _voucher_unique_pid; |
| extern dispatch_mach_t _voucher_activity_debug_channel; |
| extern voucher_activity_hooks_t _voucher_libtrace_hooks; |
| |
| #endif // DISPATCH_PURE_C |
| |
| #else // VOUCHER_USE_MACH_VOUCHER |
| |
| #pragma mark - |
| #pragma mark Simulator / vouchers disabled |
| |
| #define _dispatch_voucher_debug(msg, v, ...) |
| #define _dispatch_kvoucher_debug(msg, kv, ...) |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline voucher_t |
| _voucher_retain(voucher_t voucher) |
| { |
| return voucher; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_release(voucher_t voucher) |
| { |
| (void)voucher; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline voucher_t |
| _voucher_get(void) |
| { |
| return NULL; |
| } |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline voucher_t |
| _voucher_copy(void) |
| { |
| return NULL; |
| } |
| |
| DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT |
| static inline voucher_t |
| _voucher_copy_without_importance(void) |
| { |
| return NULL; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline mach_voucher_t |
| _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher) |
| { |
| (void)ov; (void)voucher; |
| return MACH_VOUCHER_NULL; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline voucher_t |
| _voucher_adopt(voucher_t voucher) |
| { |
| return voucher; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_replace(voucher_t voucher) |
| { |
| (void)voucher; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _voucher_clear(void) |
| { |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline pthread_priority_t |
| _voucher_get_priority(voucher_t voucher) |
| { |
| (void)voucher; |
| return 0; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _voucher_mach_msg_set_mach_voucher(mach_msg_header_t *msg, mach_voucher_t kv, |
| bool move_send) |
| { |
| (void)msg; (void)kv; (void)move_send; |
| return false; |
| |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _voucher_mach_msg_set(mach_msg_header_t *msg, voucher_t voucher) |
| { |
| (void)msg; (void)voucher; |
| return false; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline mach_voucher_t |
| _voucher_mach_msg_get(mach_msg_header_t *msg, mach_msg_bits_t *msgh_bits) |
| { |
| (void)msg;(void)msgh_bits; |
| return 0; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline mach_voucher_t |
| _voucher_mach_msg_clear(mach_msg_header_t *msg, bool move_send) |
| { |
| (void)msg; (void)move_send; |
| return MACH_VOUCHER_NULL; |
| } |
| |
| #define _dispatch_voucher_ktrace_dc_push(dc) |
| #define _dispatch_voucher_ktrace_dc_pop(dc, v) |
| #define _dispatch_voucher_ktrace_dmsg_push(dmsg) |
| #define _dispatch_voucher_ktrace_dmsg_pop(dmsg) |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_continuation_voucher_set(dispatch_continuation_t dc, |
| dispatch_queue_class_t dqu, dispatch_block_flags_t flags) |
| { |
| (void)dc; (void)dqu; (void)flags; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_continuation_voucher_adopt(dispatch_continuation_t dc, voucher_t ov, |
| uintptr_t dc_flags) |
| { |
| (void)dc; (void)ov; (void)dc_flags; |
| } |
| |
| #endif // VOUCHER_USE_MACH_VOUCHER |
| |
| #endif /* __DISPATCH_VOUCHER_INTERNAL__ */ |