| /* |
| * Copyright (c) 2010-2013 Apple Inc. All rights reserved. |
| * |
| * @APPLE_APACHE_LICENSE_HEADER_START@ |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| * |
| * @APPLE_APACHE_LICENSE_HEADER_END@ |
| */ |
| |
| /* |
| * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch |
| * which are subject to change in future releases of Mac OS X. Any applications |
| * relying on these interfaces WILL break. |
| */ |
| |
| #ifndef __DISPATCH_TRACE__ |
| #define __DISPATCH_TRACE__ |
| |
| #if DISPATCH_PURE_C |
| |
| #if DISPATCH_USE_DTRACE_INTROSPECTION |
| #define _dispatch_trace_callout(_c, _f, _dcc) do { \ |
| if (unlikely(DISPATCH_CALLOUT_ENTRY_ENABLED() || \ |
| DISPATCH_CALLOUT_RETURN_ENABLED())) { \ |
| dispatch_queue_t _dq = _dispatch_queue_get_current(); \ |
| const char *_label = _dq && _dq->dq_label ? _dq->dq_label : ""; \ |
| dispatch_function_t _func = (dispatch_function_t)(_f); \ |
| void *_ctxt = (_c); \ |
| DISPATCH_CALLOUT_ENTRY(_dq, _label, _func, _ctxt); \ |
| _dcc; \ |
| DISPATCH_CALLOUT_RETURN(_dq, _label, _func, _ctxt); \ |
| } else { \ |
| _dcc; \ |
| } \ |
| } while (0) |
| #elif DISPATCH_INTROSPECTION |
| #define _dispatch_trace_callout(_c, _f, _dcc) \ |
| do { (void)(_c); (void)(_f); _dcc; } while (0) |
| #endif // DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION |
| |
| #if DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_client_callout(void *ctxt, dispatch_function_t f) |
| { |
| dispatch_function_t func = (f == _dispatch_call_block_and_release && |
| ctxt ? _dispatch_Block_invoke(ctxt) : f); |
| _dispatch_introspection_callout_entry(ctxt, func); |
| _dispatch_trace_callout(ctxt, func, _dispatch_client_callout(ctxt, f)); |
| _dispatch_introspection_callout_return(ctxt, func); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_client_callout2(void *ctxt, size_t i, void (*f)(void *, size_t)) |
| { |
| dispatch_function_t func = (dispatch_function_t)f; |
| _dispatch_introspection_callout_entry(ctxt, func); |
| _dispatch_trace_callout(ctxt, func, _dispatch_client_callout2(ctxt, i, f)); |
| _dispatch_introspection_callout_return(ctxt, func); |
| } |
| |
| #define _dispatch_client_callout _dispatch_trace_client_callout |
| #define _dispatch_client_callout2 _dispatch_trace_client_callout2 |
| #endif // DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION |
| |
| #ifdef _COMM_PAGE_KDEBUG_ENABLE |
| #define DISPATCH_KTRACE_ENABLED \ |
| (*(volatile uint32_t *)_COMM_PAGE_KDEBUG_ENABLE != 0) |
| |
| #if DISPATCH_INTROSPECTION |
| #define _dispatch_only_if_ktrace_enabled(...) \ |
| if (unlikely(DISPATCH_KTRACE_ENABLED)) ({ __VA_ARGS__; }) |
| #else |
| #define _dispatch_only_if_ktrace_enabled(...) (void)0 |
| #endif /* DISPATCH_INTROSPECTION */ |
| |
| #else /* _COMM_PAGE_KDEBUG_ENABLE */ |
| |
| #define DISPATCH_KTRACE_ENABLED 0 |
| #define _dispatch_only_if_ktrace_enabled(...) (void)0 |
| #endif /* _COMM_PAGE_KDEBUG_ENABLE */ |
| |
| |
| #if DISPATCH_USE_DTRACE_INTROSPECTION |
| #define _dispatch_trace_continuation(_q, _o, _t) do { \ |
| dispatch_queue_t _dq = (_q); \ |
| const char *_label = _dq && _dq->dq_label ? _dq->dq_label : ""; \ |
| struct dispatch_object_s *_do = (_o); \ |
| dispatch_continuation_t _dc; \ |
| char *_kind; \ |
| dispatch_function_t _func; \ |
| void *_ctxt; \ |
| if (_dispatch_object_has_vtable(_do)) { \ |
| _kind = (char*)_dispatch_object_class_name(_do); \ |
| if ((dx_metatype(_do) == _DISPATCH_SOURCE_TYPE) && \ |
| _dq != _dispatch_mgr_q._as_dq) { \ |
| dispatch_source_t _ds = (dispatch_source_t)_do; \ |
| _dc = os_atomic_load(&_ds->ds_refs->ds_handler[ \ |
| DS_EVENT_HANDLER], relaxed); \ |
| _func = _dc ? _dc->dc_func : NULL; \ |
| _ctxt = _dc ? _dc->dc_ctxt : NULL; \ |
| } else { \ |
| _func = (dispatch_function_t)_dispatch_lane_invoke; \ |
| _ctxt = _do->do_ctxt; \ |
| } \ |
| } else { \ |
| _dc = (void*)_do; \ |
| _ctxt = _dc->dc_ctxt; \ |
| if (_dc->dc_flags & DC_FLAG_SYNC_WAITER) { \ |
| _kind = "semaphore"; \ |
| _func = (dispatch_function_t)dispatch_semaphore_signal; \ |
| } else if (_dc->dc_flags & DC_FLAG_BLOCK) { \ |
| _kind = "block"; \ |
| _func = _dispatch_Block_invoke(_dc->dc_ctxt); \ |
| } else { \ |
| _kind = "function"; \ |
| _func = _dc->dc_func; \ |
| } \ |
| } \ |
| _t(_dq, _label, _do, _kind, _func, _ctxt); \ |
| } while (0) |
| #elif DISPATCH_INTROSPECTION |
| #define _dispatch_trace_continuation(_q, _o, _t) \ |
| do { (void)(_q); (void)(_o); } while(0) |
| #define DISPATCH_QUEUE_PUSH_ENABLED() 0 |
| #define DISPATCH_QUEUE_POP_ENABLED() 0 |
| #endif // DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION |
| |
| #if DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_queue_class_t |
| _dispatch_trace_queue_create(dispatch_queue_class_t dqu) |
| { |
| _dispatch_only_if_ktrace_enabled({ |
| uint64_t dq_label[4] = {0}; // So that we get the right null termination |
| dispatch_queue_t dq = dqu._dq; |
| strncpy((char *)dq_label, (char *)dq->dq_label ?: "", sizeof(dq_label)); |
| |
| _dispatch_ktrace2(DISPATCH_QOS_TRACE_queue_creation_start, |
| dq->dq_serialnum, |
| _dispatch_priority_to_pp_prefer_fallback(dq->dq_priority)); |
| |
| _dispatch_ktrace4(DISPATCH_QOS_TRACE_queue_creation_end, |
| dq_label[0], dq_label[1], dq_label[2], dq_label[3]); |
| }); |
| |
| return _dispatch_introspection_queue_create(dqu); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_queue_dispose(dispatch_queue_class_t dqu) |
| { |
| _dispatch_ktrace1(DISPATCH_QOS_TRACE_queue_dispose, (dqu._dq)->dq_serialnum); |
| _dispatch_introspection_queue_dispose(dqu); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_source_dispose(dispatch_source_t ds) |
| { |
| _dispatch_ktrace1(DISPATCH_QOS_TRACE_src_dispose, (uintptr_t)ds); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_block_create_with_voucher_and_priority(dispatch_block_t db, |
| void *func, dispatch_block_flags_t original_flags, |
| pthread_priority_t original_priority, |
| pthread_priority_t thread_prio, pthread_priority_t final_block_prio) |
| { |
| _dispatch_ktrace4(DISPATCH_QOS_TRACE_private_block_creation, |
| (uintptr_t)db, |
| (uintptr_t)func, |
| BITPACK_UINT32_PAIR(original_flags, original_priority), |
| BITPACK_UINT32_PAIR(thread_prio, final_block_prio)); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_firehose_reserver_gave_up(uint8_t stream, uint8_t ref, |
| bool waited, uint64_t old_state, uint64_t new_state) |
| { |
| uint64_t first = ((uint64_t)ref << 8) | (uint64_t)stream; |
| uint64_t second = waited; |
| _dispatch_ktrace4(DISPATCH_FIREHOSE_TRACE_reserver_gave_up, first, second, |
| old_state, new_state); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_firehose_reserver_wait(uint8_t stream, uint8_t ref, |
| bool waited, uint64_t old_state, uint64_t new_state, bool reliable) |
| { |
| uint64_t first = ((uint64_t)ref << 8) | (uint64_t)stream; |
| uint64_t second = ((uint64_t)reliable << 1) | waited; |
| _dispatch_ktrace4(DISPATCH_FIREHOSE_TRACE_reserver_wait, first, second, |
| old_state, new_state); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_firehose_allocator(uint64_t ask0, uint64_t ask1, |
| uint64_t old_state, uint64_t new_state) |
| { |
| _dispatch_ktrace4(DISPATCH_FIREHOSE_TRACE_allocator, ask0, ask1, old_state, |
| new_state); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_firehose_wait_for_logd(uint8_t stream, uint64_t timestamp, |
| uint64_t old_state, uint64_t new_state) |
| { |
| _dispatch_ktrace4(DISPATCH_FIREHOSE_TRACE_wait_for_logd, stream, timestamp, |
| old_state, new_state); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_firehose_chunk_install(uint64_t ask0, uint64_t ask1, |
| uint64_t old_state, uint64_t new_state) |
| { |
| _dispatch_ktrace4(DISPATCH_FIREHOSE_TRACE_chunk_install, ask0, ask1, |
| old_state, new_state); |
| } |
| |
| /* Implemented in introspection.c */ |
| void |
| _dispatch_trace_item_push_internal(dispatch_queue_t dq, dispatch_object_t dou); |
| |
| #define _dispatch_trace_item_push_inline(...) \ |
| _dispatch_only_if_ktrace_enabled({ \ |
| _dispatch_trace_item_push_internal(__VA_ARGS__); \ |
| }) |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_item_push_list(dispatch_queue_global_t dq, |
| dispatch_object_t _head, dispatch_object_t _tail) |
| { |
| if (unlikely(DISPATCH_QUEUE_PUSH_ENABLED() || DISPATCH_KTRACE_ENABLED)) { |
| struct dispatch_object_s *dou = _head._do; |
| do { |
| if (unlikely(DISPATCH_QUEUE_PUSH_ENABLED())) { |
| _dispatch_trace_continuation(dq->_as_dq, dou, DISPATCH_QUEUE_PUSH); |
| } |
| |
| _dispatch_trace_item_push_inline(dq->_as_dq, dou); |
| } while (dou != _tail._do && (dou = dou->do_next)); |
| } |
| _dispatch_introspection_queue_push_list(dq, _head, _tail); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_item_push(dispatch_queue_class_t dqu, dispatch_object_t _tail) |
| { |
| if (unlikely(DISPATCH_QUEUE_PUSH_ENABLED())) { |
| _dispatch_trace_continuation(dqu._dq, _tail._do, DISPATCH_QUEUE_PUSH); |
| } |
| |
| _dispatch_trace_item_push_inline(dqu._dq, _tail._do); |
| _dispatch_introspection_queue_push(dqu, _tail); |
| } |
| |
| /* Implemented in introspection.c */ |
| void |
| _dispatch_trace_item_pop_internal(dispatch_queue_t dq, dispatch_object_t dou); |
| |
| #define _dispatch_trace_item_pop_inline(...) \ |
| _dispatch_only_if_ktrace_enabled({ \ |
| _dispatch_trace_item_pop_internal(__VA_ARGS__); \ |
| }) |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_item_pop(dispatch_queue_class_t dqu, dispatch_object_t dou) |
| { |
| if (unlikely(DISPATCH_QUEUE_POP_ENABLED())) { |
| _dispatch_trace_continuation(dqu._dq, dou._do, DISPATCH_QUEUE_POP); |
| } |
| |
| _dispatch_trace_item_pop_inline(dqu._dq, dou); |
| _dispatch_introspection_queue_pop(dqu, dou); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_item_complete_inline(dispatch_object_t dou) |
| { |
| _dispatch_ktrace1(DISPATCH_QOS_TRACE_queue_item_complete, dou._do_value); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_item_complete(dispatch_object_t dou) |
| { |
| _dispatch_trace_item_complete_inline(dou); |
| _dispatch_introspection_queue_item_complete(dou); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline struct dispatch_object_s * |
| _dispatch_trace_item_sync_push_pop(dispatch_queue_class_t dqu, |
| void *ctx, dispatch_function_t f, uintptr_t dc_flags) |
| { |
| // No need to add tracing here since the introspection calls out to |
| // _trace_item_push and _trace_item_pop |
| return _dispatch_introspection_queue_fake_sync_push_pop(dqu._dq, ctx, |
| f, dc_flags); |
| } |
| |
| /* Implemented in introspection.c */ |
| void |
| _dispatch_trace_source_callout_entry_internal(dispatch_source_t ds, long kind, |
| dispatch_queue_t dq, dispatch_continuation_t dc); |
| |
| #define _dispatch_trace_source_callout_entry(...) \ |
| _dispatch_only_if_ktrace_enabled({ \ |
| _dispatch_trace_source_callout_entry_internal(__VA_ARGS__); \ |
| }) |
| |
| #define _dispatch_trace_runtime_event(evt, ptr, value) \ |
| _dispatch_introspection_runtime_event(\ |
| dispatch_introspection_runtime_event_##evt, ptr, value) |
| |
| #define DISPATCH_TRACE_ARG(arg) , arg |
| #else |
| #define _dispatch_trace_queue_create _dispatch_introspection_queue_create |
| #define _dispatch_trace_queue_dispose _dispatch_introspection_queue_dispose |
| #define _dispatch_trace_source_dispose(ds) ((void)0) |
| #define _dispatch_trace_block_create_with_voucher_and_priority(_db, _func, \ |
| _flags, _pri, _tpri, _bpri) \ |
| do { (void)_db; (void)_func; (void) _flags; (void) _pri; (void) _tpri; \ |
| (void) _bpri; } while (0) |
| #define _dispatch_trace_firehose_reserver_gave_up(stream, ref, waited, \ |
| old_state, new_state) \ |
| do { (void)(stream); (void)(ref); (void)(waited); (void)(old_state); \ |
| (void)(new_state); } while (0) |
| #define _dispatch_trace_firehose_reserver_wait(stream, ref, waited, \ |
| old_state, new_state, reliable) \ |
| do { (void)(stream); (void)(ref); (void)(waited); (void)(old_state); \ |
| (void)(new_state); (void)(reliable); } while (0) |
| #define _dispatch_trace_firehose_allocator(ask0, ask1, old_state, new_state) \ |
| do { (void)(ask0); (void)(ask1); (void)(old_state); \ |
| (void)(new_state); } while (0) |
| #define _dispatch_trace_firehose_wait_for_logd(stream, timestamp, old_state, \ |
| new_state) \ |
| do { (void)(stream); (void)(timestamp); (void)(old_state); \ |
| (void)(new_state); } while (0) |
| #define _dispatch_trace_firehose_chunk_install(ask0, ask1, old_state, \ |
| new_state) \ |
| do { (void)(ask0); (void)(ask1); (void)(old_state); \ |
| (void)(new_state); } while (0) |
| #define _dispatch_trace_item_push(dq, dou) \ |
| do { (void)(dq); (void)(dou); } while(0) |
| #define _dispatch_trace_item_push_list(dq, head, tail) \ |
| do { (void)(dq); (void)(head); (void)tail; } while(0) |
| #define _dispatch_trace_item_pop(dq, dou) \ |
| do { (void)(dq); (void)(dou); } while(0) |
| #define _dispatch_trace_item_complete(dou) ((void)0) |
| #define _dispatch_trace_item_sync_push_pop(dq, ctxt, func, flags) \ |
| do { (void)(dq); (void)(ctxt); (void)(func); (void)(flags); } while(0) |
| #define _dispatch_trace_source_callout_entry(ds, k, dq, dc) ((void)0) |
| #define _dispatch_trace_runtime_event(evt, ptr, value) \ |
| do { (void)(ptr); (void)(value); } while(0) |
| #define DISPATCH_TRACE_ARG(arg) |
| #endif // DISPATCH_USE_DTRACE_INTROSPECTION || DISPATCH_INTROSPECTION |
| |
| #if DISPATCH_USE_DTRACE |
| static inline dispatch_function_t |
| _dispatch_trace_timer_function(dispatch_timer_source_refs_t dr) |
| { |
| dispatch_continuation_t dc; |
| dc = os_atomic_load(&dr->ds_handler[DS_EVENT_HANDLER], relaxed); |
| return dc ? dc->dc_func : NULL; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline uint64_t |
| _dispatch_time_clock_to_nsecs(dispatch_clock_t clock, uint64_t t) |
| { |
| #if !DISPATCH_TIME_UNIT_USES_NANOSECONDS |
| switch (clock) { |
| case DISPATCH_CLOCK_MONOTONIC: |
| case DISPATCH_CLOCK_UPTIME: |
| return _dispatch_time_mach2nano(t); |
| case DISPATCH_CLOCK_WALL: |
| return t; |
| } |
| #else |
| (void)clock; |
| return t; |
| #endif |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline dispatch_trace_timer_params_t |
| _dispatch_trace_timer_params(dispatch_clock_t clock, |
| struct dispatch_timer_source_s *values, uint64_t deadline, |
| dispatch_trace_timer_params_t params) |
| { |
| #define _dispatch_trace_time2nano3(t) \ |
| (_dispatch_time_clock_to_nsecs(clock, t)) |
| #define _dispatch_trace_time2nano2(v, t) ({ uint64_t _t = (t); \ |
| (v) >= INT64_MAX ? -1ll : (int64_t)_dispatch_trace_time2nano3(_t);}) |
| #define _dispatch_trace_time2nano(v) ({ uint64_t _t; \ |
| _t = _dispatch_trace_time2nano3(v); _t >= INT64_MAX ? -1ll : \ |
| (int64_t)_t; }) |
| if (deadline) { |
| params->deadline = (int64_t)deadline; |
| } else { |
| uint64_t now = _dispatch_time_now(clock); |
| params->deadline = _dispatch_trace_time2nano2(values->target, |
| values->target < now ? 0 : values->target - now); |
| } |
| uint64_t leeway = values->deadline - values->target; |
| params->interval = _dispatch_trace_time2nano(values->interval); |
| params->leeway = _dispatch_trace_time2nano(leeway); |
| return params; |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline bool |
| _dispatch_trace_timer_configure_enabled(void) |
| { |
| return DISPATCH_TIMER_CONFIGURE_ENABLED(); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_timer_configure(dispatch_source_t ds, dispatch_clock_t clock, |
| struct dispatch_timer_source_s *values) |
| { |
| dispatch_timer_source_refs_t dr = ds->ds_timer_refs; |
| struct dispatch_trace_timer_params_s params; |
| DISPATCH_TIMER_CONFIGURE(ds, _dispatch_trace_timer_function(dr), |
| _dispatch_trace_timer_params(clock, values, 0, ¶ms)); |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_timer_program(dispatch_timer_source_refs_t dr, uint64_t deadline) |
| { |
| if (unlikely(DISPATCH_TIMER_PROGRAM_ENABLED())) { |
| if (deadline && dr) { |
| dispatch_source_t ds = _dispatch_source_from_refs(dr); |
| dispatch_clock_t clock = DISPATCH_TIMER_CLOCK(dr->du_ident); |
| struct dispatch_trace_timer_params_s params; |
| DISPATCH_TIMER_PROGRAM(ds, _dispatch_trace_timer_function(dr), |
| _dispatch_trace_timer_params(clock, &dr->dt_timer, |
| deadline, ¶ms)); |
| } |
| } |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_timer_wake(dispatch_timer_source_refs_t dr) |
| { |
| if (unlikely(DISPATCH_TIMER_WAKE_ENABLED())) { |
| if (dr) { |
| dispatch_source_t ds = _dispatch_source_from_refs(dr); |
| DISPATCH_TIMER_WAKE(ds, _dispatch_trace_timer_function(dr)); |
| } |
| } |
| } |
| |
| DISPATCH_ALWAYS_INLINE |
| static inline void |
| _dispatch_trace_timer_fire(dispatch_timer_source_refs_t dr, uint64_t data, |
| uint64_t missed) |
| { |
| if (unlikely(DISPATCH_TIMER_FIRE_ENABLED())) { |
| if (!(data - missed) && dr) { |
| dispatch_source_t ds = _dispatch_source_from_refs(dr); |
| DISPATCH_TIMER_FIRE(ds, _dispatch_trace_timer_function(dr)); |
| } |
| } |
| } |
| |
| #else |
| |
| #define _dispatch_trace_timer_configure_enabled() false |
| #define _dispatch_trace_timer_configure(ds, clock, values) \ |
| do { (void)(ds); (void)(clock); (void)(values); } while(0) |
| #define _dispatch_trace_timer_program(dr, deadline) \ |
| do { (void)(dr); (void)(deadline); } while(0) |
| #define _dispatch_trace_timer_wake(dr) \ |
| do { (void)(dr); } while(0) |
| #define _dispatch_trace_timer_fire(dr, data, missed) \ |
| do { (void)(dr); (void)(data); (void)(missed); } while(0) |
| |
| #endif // DISPATCH_USE_DTRACE |
| |
| #endif // DISPATCH_PURE_C |
| |
| #endif // __DISPATCH_TRACE__ |