| /* |
| * Copyright (c) 2013 Apple Inc. All rights reserved. |
| * |
| * @APPLE_APACHE_LICENSE_HEADER_START@ |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| * |
| * @APPLE_APACHE_LICENSE_HEADER_END@ |
| */ |
| |
| /* |
| * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch |
| * which are subject to change in future releases of Mac OS X. Any applications |
| * relying on these interfaces WILL break. |
| */ |
| |
| #ifndef __DISPATCH_SHIMS_YIELD__ |
| #define __DISPATCH_SHIMS_YIELD__ |
| |
| #pragma mark - |
| #pragma mark _dispatch_wait_until |
| |
| // _dispatch_wait_until() is used for cases when we're waiting on a thread to |
| // finish a critical section that is a few instructions long and cannot fail |
| // (IOW has a guarantee of making forward progress). |
| // |
| // Using _dispatch_wait_until() has two implications: |
| // - there's a single waiter for the specified condition, |
| // - the thing it is waiting on has a strong guarantee of forward progress |
| // toward resolving the condition. |
| // |
| // For these reasons, we spin shortly for the likely case when the other thread |
| // is on core and we just caught it in the inconsistency window. If the |
| // condition we're waiting for doesn't resolve quickly, then we yield because |
| // it's very likely the other thread that can unblock us is preempted, and we |
| // need to wait for it to be scheduled again. |
| // |
| // Its typical client is the enqueuer/dequeuer starvation issue for the dispatch |
| // enqueue algorithm where there is typically a 1-10 instruction gap between the |
| // exchange at the tail and setting the head/prev pointer. |
| #if DISPATCH_HW_CONFIG_UP |
| #define _dispatch_wait_until(c) ({ \ |
| __typeof__(c) _c; \ |
| int _spins = 0; \ |
| for (;;) { \ |
| if (likely(_c = (c))) break; \ |
| _spins++; \ |
| _dispatch_preemption_yield(_spins); \ |
| } \ |
| _c; }) |
| #else |
| #ifndef DISPATCH_WAIT_SPINS_WFE |
| #define DISPATCH_WAIT_SPINS_WFE 10 |
| #endif |
| #ifndef DISPATCH_WAIT_SPINS // <rdar://problem/15440575> |
| #define DISPATCH_WAIT_SPINS 1024 |
| #endif |
| #define _dispatch_wait_until(c) ({ \ |
| __typeof__(c) _c; \ |
| int _spins = -(DISPATCH_WAIT_SPINS); \ |
| for (;;) { \ |
| if (likely(_c = (c))) break; \ |
| if (unlikely(_spins++ >= 0)) { \ |
| _dispatch_preemption_yield(_spins); \ |
| } else { \ |
| dispatch_hardware_pause(); \ |
| } \ |
| } \ |
| _c; }) |
| #endif |
| |
| DISPATCH_NOT_TAIL_CALLED DISPATCH_EXPORT |
| void *_dispatch_wait_for_enqueuer(void **ptr); |
| |
| #pragma mark - |
| #pragma mark _dispatch_contention_wait_until |
| |
| #if DISPATCH_HW_CONFIG_UP |
| #define _dispatch_contention_wait_until(c) false |
| #else |
| #ifndef DISPATCH_CONTENTION_SPINS_MAX |
| #define DISPATCH_CONTENTION_SPINS_MAX (128 - 1) |
| #endif |
| #ifndef DISPATCH_CONTENTION_SPINS_MIN |
| #define DISPATCH_CONTENTION_SPINS_MIN (32 - 1) |
| #endif |
| #if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR |
| #define _dispatch_contention_spins() \ |
| ((DISPATCH_CONTENTION_SPINS_MIN) + ((DISPATCH_CONTENTION_SPINS_MAX) - \ |
| (DISPATCH_CONTENTION_SPINS_MIN)) / 2) |
| #elif defined(_WIN32) |
| // Use randomness to prevent threads from resonating at the same frequency and |
| // permanently contending. Windows doesn't provide rand_r(), so use a simple |
| // LCG. (msvcrt has rand_s(), but its security guarantees aren't optimal here.) |
| #define _dispatch_contention_spins() ({ \ |
| static os_atomic(unsigned int) _seed = 1; \ |
| unsigned int _next = os_atomic_load(&_seed, relaxed); \ |
| os_atomic_store(&_seed, _next * 1103515245 + 12345, relaxed); \ |
| ((_next >> 24) & (DISPATCH_CONTENTION_SPINS_MAX)) | \ |
| (DISPATCH_CONTENTION_SPINS_MIN); }) |
| #else |
| // Use randomness to prevent threads from resonating at the same |
| // frequency and permanently contending. |
| #define _dispatch_contention_spins() ({ \ |
| ((unsigned int)rand() & (DISPATCH_CONTENTION_SPINS_MAX)) | \ |
| (DISPATCH_CONTENTION_SPINS_MIN); }) |
| #endif |
| #define _dispatch_contention_wait_until(c) ({ \ |
| bool _out = false; \ |
| unsigned int _spins = _dispatch_contention_spins(); \ |
| while (_spins--) { \ |
| dispatch_hardware_pause(); \ |
| if (likely(_out = (c))) break; \ |
| }; _out; }) |
| #endif |
| |
| #pragma mark - |
| #pragma mark dispatch_hardware_pause |
| |
| #if defined(__x86_64__) || defined(__i386__) |
| #define dispatch_hardware_pause() __asm__("pause") |
| #elif (defined(__arm__) && defined(_ARM_ARCH_7) && defined(__thumb__)) || \ |
| defined(__arm64__) |
| #define dispatch_hardware_pause() __asm__("yield") |
| #define dispatch_hardware_wfe() __asm__("wfe") |
| #else |
| #define dispatch_hardware_pause() __asm__("") |
| #endif |
| |
| #pragma mark - |
| #pragma mark _dispatch_preemption_yield |
| |
| #if HAVE_MACH |
| #if defined(SWITCH_OPTION_OSLOCK_DEPRESS) |
| #define DISPATCH_YIELD_THREAD_SWITCH_OPTION SWITCH_OPTION_OSLOCK_DEPRESS |
| #else |
| #define DISPATCH_YIELD_THREAD_SWITCH_OPTION SWITCH_OPTION_DEPRESS |
| #endif |
| #define _dispatch_preemption_yield(n) thread_switch(MACH_PORT_NULL, \ |
| DISPATCH_YIELD_THREAD_SWITCH_OPTION, (mach_msg_timeout_t)(n)) |
| #define _dispatch_preemption_yield_to(th, n) thread_switch(th, \ |
| DISPATCH_YIELD_THREAD_SWITCH_OPTION, (mach_msg_timeout_t)(n)) |
| #elif HAVE_PTHREAD_YIELD_NP |
| #define _dispatch_preemption_yield(n) { (void)n; pthread_yield_np(); } |
| #define _dispatch_preemption_yield_to(th, n) { (void)n; pthread_yield_np(); } |
| #elif defined(_WIN32) |
| #define _dispatch_preemption_yield(n) { (void)n; Sleep(0); } |
| #define _dispatch_preemption_yield_to(th, n) { (void)n; Sleep(0); } |
| #else |
| #define _dispatch_preemption_yield(n) { (void)n; sched_yield(); } |
| #define _dispatch_preemption_yield_to(th, n) { (void)n; sched_yield(); } |
| #endif // HAVE_MACH |
| |
| #pragma mark - |
| #pragma mark _dispatch_contention_usleep |
| |
| #ifndef DISPATCH_CONTENTION_USLEEP_START |
| #if defined(_WIN32) |
| #define DISPATCH_CONTENTION_USLEEP_START 1000 // Must be >= 1ms for Sleep() |
| #else |
| #define DISPATCH_CONTENTION_USLEEP_START 500 |
| #endif |
| #endif |
| #ifndef DISPATCH_CONTENTION_USLEEP_MAX |
| #define DISPATCH_CONTENTION_USLEEP_MAX 100000 |
| #endif |
| |
| #if HAVE_MACH |
| #if defined(SWITCH_OPTION_DISPATCH_CONTENTION) |
| #define _dispatch_contention_usleep(u) thread_switch(MACH_PORT_NULL, \ |
| SWITCH_OPTION_DISPATCH_CONTENTION, (u)) |
| #else |
| #define _dispatch_contention_usleep(u) thread_switch(MACH_PORT_NULL, \ |
| SWITCH_OPTION_WAIT, (((u)-1)/1000)+1) |
| #endif |
| #else |
| #if defined(_WIN32) |
| #define _dispatch_contention_usleep(u) Sleep((u) / 1000) |
| #else |
| #define _dispatch_contention_usleep(u) usleep((u)) |
| #endif |
| #endif // HAVE_MACH |
| |
| #endif // __DISPATCH_SHIMS_YIELD__ |