Merge pull request #255 from apple/revert-254-das-darwin-libdispatch-890-merge-master
Revert "Merge darwin/libdispatch-890 to master"
diff --git a/INSTALL.md b/INSTALL.md
index 9940c2c..b758f2b 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -87,6 +87,11 @@
Specify the path to Apple's libplatform package, so that appropriate headers
can be found and used.
+`--with-apple-libclosure-source`
+
+Specify the path to Apple's Libclosure package, so that appropriate headers
+ can be found and used.
+
`--with-apple-xnu-source`
Specify the path to Apple's XNU package, so that appropriate headers can be
@@ -99,6 +104,11 @@
The following options are likely to only be useful when building libdispatch on
OS X as a replacement for /usr/lib/system/libdispatch.dylib:
+`--with-apple-objc4-source`
+
+Specify the path to Apple's objc4 package, so that appropriate headers can
+ be found and used.
+
`--disable-libdispatch-init-constructor`
Do not tag libdispatch's init routine as __constructor, in which case it must be run manually before libdispatch routines can be called. This is the default when building on OS X. For /usr/lib/system/libdispatch.dylib the init routine is called automatically during process start.
@@ -121,7 +131,9 @@
--enable-apple-tsd-optimizations \
--with-apple-libpthread-source=/path/to/10.11.0/libpthread-137.1.1 \
--with-apple-libplatform-source=/path/to/10.11.0/libplatform-73.1.1 \
+ --with-apple-libclosure-source=/path/to/10.11.0/libclosure-65 \
--with-apple-xnu-source=/path/to/10.11.0/xnu-3247.1.106 \
+ --with-apple-objc4-source=/path/to/10.11.0/objc4-680
make check
### Building and installing for FreeBSD
diff --git a/configure.ac b/configure.ac
index 5599cb0..3221112 100644
--- a/configure.ac
+++ b/configure.ac
@@ -125,6 +125,13 @@
CPPFLAGS="$CPPFLAGS -isystem $apple_libplatform_source_include_path"
])
+AC_ARG_WITH([apple-libclosure-source],
+ [AS_HELP_STRING([--with-apple-libclosure-source],
+ [Specify path to Apple libclosure source])], [
+ apple_libclosure_source_path=${withval}
+ CPPFLAGS="$CPPFLAGS -isystem $apple_libclosure_source_path"
+])
+
AC_ARG_WITH([apple-xnu-source],
[AS_HELP_STRING([--with-apple-xnu-source],
[Specify path to Apple XNU source])], [
@@ -136,6 +143,12 @@
CPPFLAGS="$CPPFLAGS -idirafter $apple_xnu_source_libkern_path -isystem $apple_xnu_source_bsd_path -isystem $apple_xnu_source_libsyscall_path -isystem $apple_xnu_source_libproc_path "
])
+AC_ARG_WITH([apple-objc4-source],
+ [AS_HELP_STRING([--with-apple-objc4-source],
+ [Specify path to Apple objc4 source])], [
+ apple_objc4_source_runtime_path=${withval}/runtime
+])
+
AC_CACHE_CHECK([for System.framework/PrivateHeaders], dispatch_cv_system_privateheaders,
[AS_IF([test -d /System/Library/Frameworks/System.framework/PrivateHeaders],
[dispatch_cv_system_privateheaders=yes], [dispatch_cv_system_privateheaders=no])]
@@ -374,11 +387,24 @@
[have_foundation=true], [have_foundation=false]
)
AM_CONDITIONAL(HAVE_FOUNDATION, $have_foundation)
-AC_CHECK_HEADER([objc/runtime.h], [
+# hack for objc4/runtime/objc-internal.h
+AS_IF([test -n "$apple_objc4_source_runtime_path"], [
+ saveCPPFLAGS="$CPPFLAGS"
+ CPPFLAGS="$CPPFLAGS -I."
+ ln -fsh "$apple_objc4_source_runtime_path" objc
+])
+AC_CHECK_HEADER([objc/objc-internal.h], [
AC_DEFINE(HAVE_OBJC, 1, [Define if you have the Objective-C runtime])
have_objc=true], [have_objc=false],
[#include <objc/runtime.h>]
)
+AS_IF([test -n "$apple_objc4_source_runtime_path"], [
+ rm -f objc
+ CPPFLAGS="$saveCPPFLAGS"
+ AC_CONFIG_COMMANDS([src/objc],
+ [ln -fsh "$apple_objc4_source_runtime_path" src/objc],
+ [apple_objc4_source_runtime_path="$apple_objc4_source_runtime_path"])
+])
AM_CONDITIONAL(USE_OBJC, $have_objc)
AC_LANG_POP([Objective C])
diff --git a/dispatch/dispatch.h b/dispatch/dispatch.h
index 2d45b83..6f8b31b 100644
--- a/dispatch/dispatch.h
+++ b/dispatch/dispatch.h
@@ -23,7 +23,6 @@
#ifdef __APPLE__
#include <Availability.h>
-#include <os/availability.h>
#include <TargetConditionals.h>
#include <os/base.h>
#elif defined(__linux__)
@@ -48,7 +47,7 @@
#endif
#endif
-#define DISPATCH_API_VERSION 20170124
+#define DISPATCH_API_VERSION 20160831
#ifndef __DISPATCH_BUILDING_DISPATCH__
diff --git a/dispatch/queue.h b/dispatch/queue.h
index 606bd30..b1dd8e5 100644
--- a/dispatch/queue.h
+++ b/dispatch/queue.h
@@ -206,49 +206,15 @@
void *_Nullable context,
dispatch_function_t work);
-
-#if !defined(__APPLE__) || TARGET_OS_WATCH || TARGET_OS_TV || \
- (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && \
- __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_7_0) || \
- (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \
- __MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_9)
-#define DISPATCH_APPLY_AUTO_AVAILABLE 1
-#else
-#define DISPATCH_APPLY_AUTO_AVAILABLE 0
-#endif
-
-/*!
- * @constant DISPATCH_APPLY_AUTO
- *
- * @abstract
- * Constant to pass to dispatch_apply() or dispatch_apply_f() to request that
- * the system automatically use worker threads that match the configuration of
- * the current thread most closely.
- *
- * @discussion
- * When submitting a block for parallel invocation, passing this constant as the
- * queue argument will automatically use the global concurrent queue that
- * matches the Quality of Service of the caller most closely.
- *
- * No assumptions should be made about which global concurrent queue will
- * actually be used.
- *
- * Using this constant deploys backward to macOS 10.9, iOS 7.0 and any tvOS or
- * watchOS version.
- */
-#if DISPATCH_APPLY_AUTO_AVAILABLE
-#define DISPATCH_APPLY_AUTO ((dispatch_queue_t _Nonnull)0)
-#endif
-
/*!
* @function dispatch_apply
*
* @abstract
- * Submits a block to a dispatch queue for parallel invocation.
+ * Submits a block to a dispatch queue for multiple invocations.
*
* @discussion
- * Submits a block to a dispatch queue for parallel invocation. This function
- * waits for the task block to complete before returning. If the specified queue
+ * Submits a block to a dispatch queue for multiple invocations. This function
+ * waits for the task block to complete before returning. If the target queue
* is concurrent, the block may be invoked concurrently, and it must therefore
* be reentrant safe.
*
@@ -258,9 +224,8 @@
* The number of iterations to perform.
*
* @param queue
- * The dispatch queue to which the block is submitted.
- * The preferred value to pass is DISPATCH_APPLY_AUTO to automatically use
- * a queue appropriate for the calling thread.
+ * The target dispatch queue to which the block is submitted.
+ * The result of passing NULL in this parameter is undefined.
*
* @param block
* The block to be invoked the specified number of iterations.
@@ -278,7 +243,7 @@
* @function dispatch_apply_f
*
* @abstract
- * Submits a function to a dispatch queue for parallel invocation.
+ * Submits a function to a dispatch queue for multiple invocations.
*
* @discussion
* See dispatch_apply() for details.
@@ -287,15 +252,14 @@
* The number of iterations to perform.
*
* @param queue
- * The dispatch queue to which the function is submitted.
- * The preferred value to pass is DISPATCH_APPLY_AUTO to automatically use
- * a queue appropriate for the calling thread.
+ * The target dispatch queue to which the function is submitted.
+ * The result of passing NULL in this parameter is undefined.
*
* @param context
* The application-defined context parameter to pass to the function.
*
* @param work
- * The application-defined function to invoke on the specified queue. The first
+ * The application-defined function to invoke on the target queue. The first
* parameter passed to this function is the context provided to
* dispatch_apply_f(). The second parameter passed to this function is the
* current index of iteration.
diff --git a/libdispatch.xcodeproj/project.pbxproj b/libdispatch.xcodeproj/project.pbxproj
index 361994f..ce73d95 100644
--- a/libdispatch.xcodeproj/project.pbxproj
+++ b/libdispatch.xcodeproj/project.pbxproj
@@ -331,37 +331,6 @@
E48EC97C1835BADD00EAC4F1 /* yield.h in Headers */ = {isa = PBXBuildFile; fileRef = E48EC97B1835BADD00EAC4F1 /* yield.h */; };
E48EC97D1835BADD00EAC4F1 /* yield.h in Headers */ = {isa = PBXBuildFile; fileRef = E48EC97B1835BADD00EAC4F1 /* yield.h */; };
E48EC97E1835BADD00EAC4F1 /* yield.h in Headers */ = {isa = PBXBuildFile; fileRef = E48EC97B1835BADD00EAC4F1 /* yield.h */; };
- E49BB6D11E70748100868613 /* provider.d in Sources */ = {isa = PBXBuildFile; fileRef = E43570B8126E93380097AB9F /* provider.d */; };
- E49BB6D21E70748100868613 /* protocol.defs in Sources */ = {isa = PBXBuildFile; fileRef = FC7BED950E8361E600161930 /* protocol.defs */; settings = {ATTRIBUTES = (Client, Server, ); }; };
- E49BB6D31E70748100868613 /* venture.c in Sources */ = {isa = PBXBuildFile; fileRef = 6E9955CE1C3B218E0071D40C /* venture.c */; };
- E49BB6D41E70748100868613 /* firehose.defs in Sources */ = {isa = PBXBuildFile; fileRef = 72DEAA9B1AE1B0BD00289540 /* firehose.defs */; };
- E49BB6D51E70748100868613 /* firehose_buffer.c in Sources */ = {isa = PBXBuildFile; fileRef = 72DEAA971AE181D300289540 /* firehose_buffer.c */; };
- E49BB6D61E70748100868613 /* event_kevent.c in Sources */ = {isa = PBXBuildFile; fileRef = 6E5ACCB01D3C4CFB007DA2B4 /* event_kevent.c */; };
- E49BB6D71E70748100868613 /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE371251656400645D88 /* resolver.c */; };
- E49BB6D81E70748100868613 /* mach.c in Sources */ = {isa = PBXBuildFile; fileRef = 6E4BACBC1D48A41500B562AE /* mach.c */; };
- E49BB6D91E70748100868613 /* init.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE3B1251659900645D88 /* init.c */; };
- E49BB6DA1E70748100868613 /* queue.c in Sources */ = {isa = PBXBuildFile; fileRef = FC7BED8A0E8361E600161930 /* queue.c */; };
- E49BB6DB1E70748100868613 /* semaphore.c in Sources */ = {isa = PBXBuildFile; fileRef = 721F5CCE0F15553500FF03A6 /* semaphore.c */; };
- E49BB6DC1E70748100868613 /* lock.c in Sources */ = {isa = PBXBuildFile; fileRef = 6EF2CAAB1C8899D5001ABE83 /* lock.c */; };
- E49BB6DD1E70748100868613 /* firehose_reply.defs in Sources */ = {isa = PBXBuildFile; fileRef = 72406A031AF95DF800DF4E2B /* firehose_reply.defs */; settings = {ATTRIBUTES = (Server, ); }; };
- E49BB6DE1E70748100868613 /* once.c in Sources */ = {isa = PBXBuildFile; fileRef = 96DF70BD0F38FE3C0074BD99 /* once.c */; };
- E49BB6DF1E70748100868613 /* apply.c in Sources */ = {isa = PBXBuildFile; fileRef = 9676A0E00F3E755D00713ADB /* apply.c */; };
- E49BB6E01E70748100868613 /* object.c in Sources */ = {isa = PBXBuildFile; fileRef = 9661E56A0F3E7DDF00749F3E /* object.c */; };
- E49BB6E11E70748100868613 /* benchmark.c in Sources */ = {isa = PBXBuildFile; fileRef = 965CD6340F3E806200D4E28D /* benchmark.c */; };
- E49BB6E21E70748100868613 /* event_epoll.c in Sources */ = {isa = PBXBuildFile; fileRef = 6EA7937D1D456D1300929B1B /* event_epoll.c */; };
- E49BB6E31E70748100868613 /* source.c in Sources */ = {isa = PBXBuildFile; fileRef = 96A8AA860F41E7A400CD570B /* source.c */; };
- E49BB6E41E70748100868613 /* time.c in Sources */ = {isa = PBXBuildFile; fileRef = 96032E4A0F5CC8C700241C5F /* time.c */; };
- E49BB6E51E70748100868613 /* data.c in Sources */ = {isa = PBXBuildFile; fileRef = 5AAB45BF10D30B79004407EA /* data.c */; };
- E49BB6E61E70748100868613 /* io.c in Sources */ = {isa = PBXBuildFile; fileRef = 5A27262510F26F1900751FBC /* io.c */; };
- E49BB6E71E70748100868613 /* block.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E43A724F1AF85BBC00BAA921 /* block.cpp */; };
- E49BB6E81E70748100868613 /* event.c in Sources */ = {isa = PBXBuildFile; fileRef = 6E5ACCBD1D3C6719007DA2B4 /* event.c */; };
- E49BB6E91E70748100868613 /* transform.c in Sources */ = {isa = PBXBuildFile; fileRef = C9C5F80D143C1771006DC718 /* transform.c */; };
- E49BB6EA1E70748100868613 /* object.m in Sources */ = {isa = PBXBuildFile; fileRef = E4FC3263145F46C9002FBDDB /* object.m */; };
- E49BB6EB1E70748100868613 /* allocator.c in Sources */ = {isa = PBXBuildFile; fileRef = 2BBF5A62154B64F5002B20F9 /* allocator.c */; };
- E49BB6EC1E70748100868613 /* data.m in Sources */ = {isa = PBXBuildFile; fileRef = E420866F16027AE500EEE210 /* data.m */; };
- E49BB6ED1E70748100868613 /* voucher.c in Sources */ = {isa = PBXBuildFile; fileRef = E44A8E6A1805C3E0009FFDB6 /* voucher.c */; };
- E49BB7091E70A39700868613 /* venture.c in Sources */ = {isa = PBXBuildFile; fileRef = 6E9955CE1C3B218E0071D40C /* venture.c */; };
- E49BB70A1E70A3B000868613 /* venture.c in Sources */ = {isa = PBXBuildFile; fileRef = 6E9955CE1C3B218E0071D40C /* venture.c */; };
E49F2423125D3C960057C971 /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE371251656400645D88 /* resolver.c */; };
E49F2424125D3C970057C971 /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE371251656400645D88 /* resolver.c */; };
E49F2499125D48D80057C971 /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = E44EBE371251656400645D88 /* resolver.c */; };
@@ -590,13 +559,6 @@
remoteGlobalIDString = E4EC121612514715000DDBD1;
remoteInfo = "libdispatch mp resolved";
};
- E49BB6F71E7074C100868613 /* PBXContainerItemProxy */ = {
- isa = PBXContainerItemProxy;
- containerPortal = 08FB7793FE84155DC02AAC07 /* Project object */;
- proxyType = 1;
- remoteGlobalIDString = E49BB6CE1E70748100868613;
- remoteInfo = "libdispatch alt resolved";
- };
E4B515DA164B317700E003AF /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = 08FB7793FE84155DC02AAC07 /* Project object */;
@@ -658,6 +620,7 @@
6E326B161C239431002A6505 /* dispatch_timer_short.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = dispatch_timer_short.c; sourceTree = "<group>"; };
6E326B171C239431002A6505 /* dispatch_timer_timeout.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = dispatch_timer_timeout.c; sourceTree = "<group>"; };
6E326B441C239B61002A6505 /* dispatch_priority.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = dispatch_priority.c; sourceTree = "<group>"; };
+ 6E4130C91B431697001A152D /* backward-compat.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = "backward-compat.xcconfig"; sourceTree = "<group>"; };
6E4BACBC1D48A41500B562AE /* mach.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = mach.c; sourceTree = "<group>"; };
6E4BACC91D48A89500B562AE /* mach_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = mach_internal.h; sourceTree = "<group>"; };
6E4FC9D11C84123600520351 /* os_venture_basic.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = os_venture_basic.c; sourceTree = "<group>"; };
@@ -695,7 +658,6 @@
6EB4E4421BA8BD7800D7B9D2 /* libfirehose.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = libfirehose.xcconfig; sourceTree = "<group>"; };
6EB60D291BBB19640092FA94 /* firehose_inline_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = firehose_inline_internal.h; sourceTree = "<group>"; };
6EC5ABE31D4436E4004F8674 /* dispatch_deadname.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = dispatch_deadname.c; sourceTree = "<group>"; };
- 6EC670C61E37E201004F10D6 /* dispatch_network_event_thread.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = dispatch_network_event_thread.c; sourceTree = "<group>"; };
6EC670C71E37E201004F10D6 /* perf_mach_async.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = perf_mach_async.c; sourceTree = "<group>"; };
6EC670C81E37E201004F10D6 /* perf_pipepingpong.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = perf_pipepingpong.c; sourceTree = "<group>"; };
6EDB888D1CB73BDC006776D6 /* dispatch_kevent_cancel_races.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = dispatch_kevent_cancel_races.c; sourceTree = "<group>"; };
@@ -739,9 +701,6 @@
96BC39BC0F3EBAB100C59689 /* queue_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = queue_private.h; sourceTree = "<group>"; };
96C9553A0F3EAEDD000D2CA4 /* once.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = once.h; sourceTree = "<group>"; };
96DF70BD0F38FE3C0074BD99 /* once.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; lineEnding = 0; path = once.c; sourceTree = "<group>"; xcLanguageSpecificationIdentifier = xcode.lang.c; };
- B63B793F1E8F004F0060C1E1 /* dispatch_no_blocks.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = dispatch_no_blocks.c; sourceTree = "<group>"; };
- B68330BC1EBCF6080003E71C /* dispatch_wl.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = dispatch_wl.c; sourceTree = "<group>"; };
- B6AC73FD1EB10973009FB2F2 /* perf_thread_request.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = perf_thread_request.c; sourceTree = "<group>"; };
B6AE9A4A1D7F53B300AC007F /* dispatch_queue_create.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = dispatch_queue_create.c; sourceTree = "<group>"; };
B6AE9A561D7F53C100AC007F /* perf_async_bench.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = perf_async_bench.m; sourceTree = "<group>"; };
B6AE9A581D7F53CB00AC007F /* perf_bench.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = perf_bench.m; sourceTree = "<group>"; };
@@ -780,13 +739,12 @@
E44F9DA816543F79001DCD38 /* introspection_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = introspection_internal.h; sourceTree = "<group>"; };
E454569214746F1B00106147 /* object_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = object_private.h; sourceTree = "<group>"; };
E463024F1761603C00E11F4C /* atomic_sfb.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = atomic_sfb.h; sourceTree = "<group>"; };
- E46DBC5714EE10C80001F9F6 /* libdispatch_up.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libdispatch_up.a; sourceTree = BUILT_PRODUCTS_DIR; };
+ E46DBC5714EE10C80001F9F6 /* libdispatch.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libdispatch.a; sourceTree = BUILT_PRODUCTS_DIR; };
E46DBC5814EE11BC0001F9F6 /* libdispatch-up-static.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = "libdispatch-up-static.xcconfig"; sourceTree = "<group>"; };
E47D6BB5125F0F800070D91C /* resolved.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = resolved.h; sourceTree = "<group>"; };
E482F1CD12DBAB590030614D /* postprocess-headers.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; path = "postprocess-headers.sh"; sourceTree = "<group>"; };
E48AF55916E70FD9004105FF /* io_private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = io_private.h; path = private/io_private.h; sourceTree = SOURCE_ROOT; tabWidth = 8; };
E48EC97B1835BADD00EAC4F1 /* yield.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = yield.h; sourceTree = "<group>"; };
- E49BB6F21E70748100868613 /* libdispatch_alt.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libdispatch_alt.a; sourceTree = BUILT_PRODUCTS_DIR; };
E49F24DF125D57FA0057C971 /* libdispatch.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = libdispatch.dylib; sourceTree = BUILT_PRODUCTS_DIR; };
E49F251D125D630A0057C971 /* install-manpages.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; path = "install-manpages.sh"; sourceTree = "<group>"; };
E49F251E125D631D0057C971 /* mig-headers.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; path = "mig-headers.sh"; sourceTree = "<group>"; };
@@ -955,16 +913,15 @@
isa = PBXGroup;
children = (
D2AAC046055464E500DB518D /* libdispatch.dylib */,
- E4B515D6164B2DA300E003AF /* libdispatch.dylib */,
- E49F24DF125D57FA0057C971 /* libdispatch.dylib */,
- E4EC122D12514715000DDBD1 /* libdispatch_mp.a */,
E4EC11C312514302000DDBD1 /* libdispatch_up.a */,
- E49BB6F21E70748100868613 /* libdispatch_alt.a */,
- E46DBC5714EE10C80001F9F6 /* libdispatch_up.a */,
+ E4EC122D12514715000DDBD1 /* libdispatch_mp.a */,
+ E49F24DF125D57FA0057C971 /* libdispatch.dylib */,
+ E46DBC5714EE10C80001F9F6 /* libdispatch.a */,
+ E4B515D6164B2DA300E003AF /* libdispatch.dylib */,
+ 6EB4E4091BA8BCAD00D7B9D2 /* libfirehose_server.a */,
+ 6E040C631C499B1B00411A2E /* libfirehose_kernel.a */,
C01866BD1C5973210040FC07 /* libdispatch.a */,
C00B0E0A1C5AEBBE000330B3 /* libdispatch_dyld_stub.a */,
- 6E040C631C499B1B00411A2E /* libfirehose_kernel.a */,
- 6EB4E4091BA8BCAD00D7B9D2 /* libfirehose_server.a */,
);
name = Products;
sourceTree = "<group>";
@@ -1053,8 +1010,6 @@
6E326ABD1C22A577002A6505 /* dispatch_io_net.c */,
6E326ABE1C22A577002A6505 /* dispatch_io.c */,
6EDB888D1CB73BDC006776D6 /* dispatch_kevent_cancel_races.c */,
- 6EC670C61E37E201004F10D6 /* dispatch_network_event_thread.c */,
- B63B793F1E8F004F0060C1E1 /* dispatch_no_blocks.c */,
C96CE17A1CEB851600F4B8E6 /* dispatch_objc.m */,
6E67D9131C17676D00FC98AC /* dispatch_overcommit.c */,
6E67D9151C1768B300FC98AC /* dispatch_pingpong.c */,
@@ -1084,14 +1039,12 @@
6E62B0531C55806200D2C7C0 /* dispatch_trysync.c */,
6E8E4EC91C1A670B0004F5CC /* dispatch_vm.c */,
6E326AB71C225FCA002A6505 /* dispatch_vnode.c */,
- B68330BC1EBCF6080003E71C /* dispatch_wl.c */,
6E67D9171C17BA7200FC98AC /* nsoperation.m */,
6E4FC9D11C84123600520351 /* os_venture_basic.c */,
B6AE9A561D7F53C100AC007F /* perf_async_bench.m */,
B6AE9A581D7F53CB00AC007F /* perf_bench.m */,
6EC670C71E37E201004F10D6 /* perf_mach_async.c */,
6EC670C81E37E201004F10D6 /* perf_pipepingpong.c */,
- B6AC73FD1EB10973009FB2F2 /* perf_thread_request.c */,
92F3FE921BEC686300025962 /* Makefile */,
6E8E4E6E1C1A35EE0004F5CC /* test_lib.c */,
6E8E4E6F1C1A35EE0004F5CC /* test_lib.h */,
@@ -1136,6 +1089,7 @@
E40041E4125E71150022B135 /* xcodeconfig */ = {
isa = PBXGroup;
children = (
+ 6E4130C91B431697001A152D /* backward-compat.xcconfig */,
E43D93F11097917E004F6A62 /* libdispatch.xcconfig */,
E40041AA125D705F0022B135 /* libdispatch-resolver.xcconfig */,
E40041A9125D70590022B135 /* libdispatch-resolved.xcconfig */,
@@ -1633,9 +1587,8 @@
);
dependencies = (
6EF0B27E1BA8C5BF007FA4F6 /* PBXTargetDependency */,
- E47D6ECD125FEBA10070D91C /* PBXTargetDependency */,
E47D6ECB125FEB9D0070D91C /* PBXTargetDependency */,
- E49BB6F81E7074C100868613 /* PBXTargetDependency */,
+ E47D6ECD125FEBA10070D91C /* PBXTargetDependency */,
E4B515DB164B317700E003AF /* PBXTargetDependency */,
C01866C21C597AEA0040FC07 /* PBXTargetDependency */,
E437F0D614F7441F00F0B997 /* PBXTargetDependency */,
@@ -1658,24 +1611,7 @@
);
name = "libdispatch up static";
productName = libdispatch;
- productReference = E46DBC5714EE10C80001F9F6 /* libdispatch_up.a */;
- productType = "com.apple.product-type.library.static";
- };
- E49BB6CE1E70748100868613 /* libdispatch alt resolved */ = {
- isa = PBXNativeTarget;
- buildConfigurationList = E49BB6EF1E70748100868613 /* Build configuration list for PBXNativeTarget "libdispatch alt resolved" */;
- buildPhases = (
- E49BB6CF1E70748100868613 /* Mig Headers */,
- E49BB6D01E70748100868613 /* Sources */,
- E49BB6EE1E70748100868613 /* Symlink normal variant */,
- );
- buildRules = (
- );
- dependencies = (
- );
- name = "libdispatch alt resolved";
- productName = libdispatch;
- productReference = E49BB6F21E70748100868613 /* libdispatch_alt.a */;
+ productReference = E46DBC5714EE10C80001F9F6 /* libdispatch.a */;
productType = "com.apple.product-type.library.static";
};
E49F24A9125D57FA0057C971 /* libdispatch no resolver */ = {
@@ -1756,7 +1692,7 @@
isa = PBXProject;
attributes = {
BuildIndependentTargetsInParallel = YES;
- LastUpgradeCheck = 0900;
+ LastUpgradeCheck = 0820;
TargetAttributes = {
3F3C9326128E637B0042B1F7 = {
ProvisioningStyle = Manual;
@@ -1840,7 +1776,6 @@
E49F24A9125D57FA0057C971 /* libdispatch no resolver */,
E4EC121612514715000DDBD1 /* libdispatch mp resolved */,
E4EC118F12514302000DDBD1 /* libdispatch up resolved */,
- E49BB6CE1E70748100868613 /* libdispatch alt resolved */,
E4B51595164B2DA300E003AF /* libdispatch introspection */,
E46DBC1A14EE10C80001F9F6 /* libdispatch up static */,
C01866A41C5973210040FC07 /* libdispatch mp static */,
@@ -1996,47 +1931,6 @@
shellScript = ". \"${SCRIPT_INPUT_FILE_0}\"";
showEnvVarsInLog = 0;
};
- E49BB6CF1E70748100868613 /* Mig Headers */ = {
- isa = PBXShellScriptBuildPhase;
- buildActionMask = 2147483647;
- files = (
- );
- inputPaths = (
- "$(SRCROOT)/src/protocol.defs",
- "$(SRCROOT)/src/firehose/firehose.defs",
- "$(SRCROOT)/src/firehose/firehose_reply.defs",
- "$(SRCROOT)/xcodescripts/mig-headers.sh",
- );
- name = "Mig Headers";
- outputPaths = (
- "$(DERIVED_FILE_DIR)/protocol.h",
- "$(DERIVED_FILE_DIR)/protocolServer.h",
- "$(DERIVED_FILE_DIR)/firehose.h",
- "$(DERIVED_FILE_DIR)/firehoseServer.h",
- "$(DERIVED_FILE_DIR)/firehose_reply.h",
- "$(DERIVED_FILE_DIR)/firehose_replyServer.h",
- );
- runOnlyForDeploymentPostprocessing = 0;
- shellPath = "/bin/bash -e";
- shellScript = ". \"${SCRIPT_INPUT_FILE_3}\"";
- showEnvVarsInLog = 0;
- };
- E49BB6EE1E70748100868613 /* Symlink normal variant */ = {
- isa = PBXShellScriptBuildPhase;
- buildActionMask = 2147483647;
- files = (
- );
- inputPaths = (
- );
- name = "Symlink normal variant";
- outputPaths = (
- "$(CONFIGURATION_BUILD_DIR)/$(PRODUCT_NAME)_normal.a",
- );
- runOnlyForDeploymentPostprocessing = 0;
- shellPath = "/bin/bash -e";
- shellScript = "ln -fs \"${PRODUCT_NAME}.a\" \"${SCRIPT_OUTPUT_FILE_0}\"";
- showEnvVarsInLog = 0;
- };
E49F24D7125D57FA0057C971 /* Install Manpages */ = {
isa = PBXShellScriptBuildPhase;
buildActionMask = 8;
@@ -2202,10 +2096,10 @@
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
- 6E90269C1BB9BD50004DC3AD /* firehose.defs in Sources */,
- 6EF0B2781BA8C56E007FA4F6 /* firehose_reply.defs in Sources */,
6EF0B27A1BA8C57D007FA4F6 /* firehose_server_object.m in Sources */,
+ 6E90269C1BB9BD50004DC3AD /* firehose.defs in Sources */,
6E21F2E91BBB240E0000C6A5 /* firehose_server.c in Sources */,
+ 6EF0B2781BA8C56E007FA4F6 /* firehose_reply.defs in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -2214,31 +2108,30 @@
buildActionMask = 2147483647;
files = (
C00B0DF21C5AEBBE000330B3 /* protocol.defs in Sources */,
- C00B0DF71C5AEBBE000330B3 /* firehose.defs in Sources */,
- C00B0DFA1C5AEBBE000330B3 /* firehose_reply.defs in Sources */,
C00B0DF31C5AEBBE000330B3 /* resolver.c in Sources */,
- C00B0DF41C5AEBBE000330B3 /* init.c in Sources */,
- C00B0DFE1C5AEBBE000330B3 /* object.c in Sources */,
- C00B0DF81C5AEBBE000330B3 /* block.cpp in Sources */,
- 6EF2CAB31C8899ED001ABE83 /* lock.c in Sources */,
- C00B0DF91C5AEBBE000330B3 /* semaphore.c in Sources */,
- C00B0DFB1C5AEBBE000330B3 /* once.c in Sources */,
- C00B0DF51C5AEBBE000330B3 /* queue.c in Sources */,
- C00B0DFD1C5AEBBE000330B3 /* apply.c in Sources */,
- C00B0E001C5AEBBE000330B3 /* source.c in Sources */,
- 6E4BACC81D48A42400B562AE /* mach.c in Sources */,
- 6EA9629E1D48622C00759D53 /* event.c in Sources */,
- 6EA962A61D48625500759D53 /* event_kevent.c in Sources */,
6E4BACFC1D49A04A00B562AE /* event_epoll.c in Sources */,
- C00B0DFC1C5AEBBE000330B3 /* voucher.c in Sources */,
+ 6EF2CAB31C8899ED001ABE83 /* lock.c in Sources */,
+ C00B0DF41C5AEBBE000330B3 /* init.c in Sources */,
+ C00B0DF51C5AEBBE000330B3 /* queue.c in Sources */,
C00B0DF61C5AEBBE000330B3 /* firehose_buffer.c in Sources */,
- C00B0E031C5AEBBE000330B3 /* io.c in Sources */,
- C00B0E021C5AEBBE000330B3 /* data.c in Sources */,
- C00B0E041C5AEBBE000330B3 /* transform.c in Sources */,
- C00B0E011C5AEBBE000330B3 /* time.c in Sources */,
- C00B0E051C5AEBBE000330B3 /* allocator.c in Sources */,
+ C00B0DF71C5AEBBE000330B3 /* firehose.defs in Sources */,
+ C00B0DF81C5AEBBE000330B3 /* block.cpp in Sources */,
+ C00B0DF91C5AEBBE000330B3 /* semaphore.c in Sources */,
+ 6E4BACC81D48A42400B562AE /* mach.c in Sources */,
+ C00B0DFA1C5AEBBE000330B3 /* firehose_reply.defs in Sources */,
+ C00B0DFB1C5AEBBE000330B3 /* once.c in Sources */,
+ C00B0DFC1C5AEBBE000330B3 /* voucher.c in Sources */,
+ C00B0DFD1C5AEBBE000330B3 /* apply.c in Sources */,
+ C00B0DFE1C5AEBBE000330B3 /* object.c in Sources */,
C00B0DFF1C5AEBBE000330B3 /* benchmark.c in Sources */,
- E49BB70A1E70A3B000868613 /* venture.c in Sources */,
+ C00B0E001C5AEBBE000330B3 /* source.c in Sources */,
+ C00B0E011C5AEBBE000330B3 /* time.c in Sources */,
+ C00B0E021C5AEBBE000330B3 /* data.c in Sources */,
+ 6EA962A61D48625500759D53 /* event_kevent.c in Sources */,
+ C00B0E031C5AEBBE000330B3 /* io.c in Sources */,
+ C00B0E041C5AEBBE000330B3 /* transform.c in Sources */,
+ 6EA9629E1D48622C00759D53 /* event.c in Sources */,
+ C00B0E051C5AEBBE000330B3 /* allocator.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -2247,31 +2140,30 @@
buildActionMask = 2147483647;
files = (
C01866A61C5973210040FC07 /* protocol.defs in Sources */,
- C01866AB1C5973210040FC07 /* firehose.defs in Sources */,
- C01866AE1C5973210040FC07 /* firehose_reply.defs in Sources */,
C01866A71C5973210040FC07 /* resolver.c in Sources */,
- C01866A81C5973210040FC07 /* init.c in Sources */,
- C01866B21C5973210040FC07 /* object.c in Sources */,
- C01866AC1C5973210040FC07 /* block.cpp in Sources */,
- 6EF2CAB21C8899EC001ABE83 /* lock.c in Sources */,
- C01866AD1C5973210040FC07 /* semaphore.c in Sources */,
- C01866AF1C5973210040FC07 /* once.c in Sources */,
- C01866A91C5973210040FC07 /* queue.c in Sources */,
- C01866B11C5973210040FC07 /* apply.c in Sources */,
- C01866B41C5973210040FC07 /* source.c in Sources */,
- 6E4BACC71D48A42300B562AE /* mach.c in Sources */,
- 6EA9629D1D48622B00759D53 /* event.c in Sources */,
- 6EA962A51D48625400759D53 /* event_kevent.c in Sources */,
6E4BACFB1D49A04A00B562AE /* event_epoll.c in Sources */,
- C01866B01C5973210040FC07 /* voucher.c in Sources */,
+ 6EF2CAB21C8899EC001ABE83 /* lock.c in Sources */,
+ C01866A81C5973210040FC07 /* init.c in Sources */,
+ C01866A91C5973210040FC07 /* queue.c in Sources */,
C01866AA1C5973210040FC07 /* firehose_buffer.c in Sources */,
- C01866B71C5973210040FC07 /* io.c in Sources */,
- C01866B61C5973210040FC07 /* data.c in Sources */,
- C01866B81C5973210040FC07 /* transform.c in Sources */,
- C01866B51C5973210040FC07 /* time.c in Sources */,
- C01866B91C5973210040FC07 /* allocator.c in Sources */,
+ C01866AB1C5973210040FC07 /* firehose.defs in Sources */,
+ C01866AC1C5973210040FC07 /* block.cpp in Sources */,
+ C01866AD1C5973210040FC07 /* semaphore.c in Sources */,
+ 6E4BACC71D48A42300B562AE /* mach.c in Sources */,
+ C01866AE1C5973210040FC07 /* firehose_reply.defs in Sources */,
+ C01866AF1C5973210040FC07 /* once.c in Sources */,
+ C01866B01C5973210040FC07 /* voucher.c in Sources */,
+ C01866B11C5973210040FC07 /* apply.c in Sources */,
+ C01866B21C5973210040FC07 /* object.c in Sources */,
C01866B31C5973210040FC07 /* benchmark.c in Sources */,
- E49BB7091E70A39700868613 /* venture.c in Sources */,
+ C01866B41C5973210040FC07 /* source.c in Sources */,
+ C01866B51C5973210040FC07 /* time.c in Sources */,
+ C01866B61C5973210040FC07 /* data.c in Sources */,
+ 6EA962A51D48625400759D53 /* event_kevent.c in Sources */,
+ C01866B71C5973210040FC07 /* io.c in Sources */,
+ C01866B81C5973210040FC07 /* transform.c in Sources */,
+ 6EA9629D1D48622B00759D53 /* event.c in Sources */,
+ C01866B91C5973210040FC07 /* allocator.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -2281,33 +2173,33 @@
files = (
E43570B9126E93380097AB9F /* provider.d in Sources */,
FC7BEDA40E8361E600161930 /* protocol.defs in Sources */,
- 6ED64B471BBD89AF00C35F4D /* firehose.defs in Sources */,
- 6ED64B491BBD89BC00C35F4D /* firehose_reply.defs in Sources */,
- E49F2499125D48D80057C971 /* resolver.c in Sources */,
- E44EBE3E1251659900645D88 /* init.c in Sources */,
- 9661E56B0F3E7DDF00749F3E /* object.c in Sources */,
- E4FC3264145F46C9002FBDDB /* object.m in Sources */,
- E43A72501AF85BBC00BAA921 /* block.cpp in Sources */,
- 6EF2CAAC1C8899D5001ABE83 /* lock.c in Sources */,
- 721F5CCF0F15553500FF03A6 /* semaphore.c in Sources */,
- 96DF70BE0F38FE3C0074BD99 /* once.c in Sources */,
- FC7BED990E8361E600161930 /* queue.c in Sources */,
- 9676A0E10F3E755D00713ADB /* apply.c in Sources */,
- 96A8AA870F41E7A400CD570B /* source.c in Sources */,
- 6E4BACBD1D48A41500B562AE /* mach.c in Sources */,
- 6EA962971D48622600759D53 /* event.c in Sources */,
- 6EA9629F1D48625000759D53 /* event_kevent.c in Sources */,
- 6E4BACF51D49A04600B562AE /* event_epoll.c in Sources */,
- E44A8E6B1805C3E0009FFDB6 /* voucher.c in Sources */,
- 6ED64B441BBD898700C35F4D /* firehose_buffer.c in Sources */,
- 5A27262610F26F1900751FBC /* io.c in Sources */,
- 5AAB45C010D30B79004407EA /* data.c in Sources */,
- E420867016027AE500EEE210 /* data.m in Sources */,
- C9C5F80E143C1771006DC718 /* transform.c in Sources */,
- 96032E4B0F5CC8C700241C5F /* time.c in Sources */,
- 2BBF5A63154B64F5002B20F9 /* allocator.c in Sources */,
- 965CD6350F3E806200D4E28D /* benchmark.c in Sources */,
6E9955CF1C3B218E0071D40C /* venture.c in Sources */,
+ 6ED64B471BBD89AF00C35F4D /* firehose.defs in Sources */,
+ 6ED64B441BBD898700C35F4D /* firehose_buffer.c in Sources */,
+ 6EA9629F1D48625000759D53 /* event_kevent.c in Sources */,
+ E49F2499125D48D80057C971 /* resolver.c in Sources */,
+ 6E4BACBD1D48A41500B562AE /* mach.c in Sources */,
+ E44EBE3E1251659900645D88 /* init.c in Sources */,
+ FC7BED990E8361E600161930 /* queue.c in Sources */,
+ 721F5CCF0F15553500FF03A6 /* semaphore.c in Sources */,
+ 6EF2CAAC1C8899D5001ABE83 /* lock.c in Sources */,
+ 6ED64B491BBD89BC00C35F4D /* firehose_reply.defs in Sources */,
+ 96DF70BE0F38FE3C0074BD99 /* once.c in Sources */,
+ 9676A0E10F3E755D00713ADB /* apply.c in Sources */,
+ 9661E56B0F3E7DDF00749F3E /* object.c in Sources */,
+ 965CD6350F3E806200D4E28D /* benchmark.c in Sources */,
+ 6E4BACF51D49A04600B562AE /* event_epoll.c in Sources */,
+ 96A8AA870F41E7A400CD570B /* source.c in Sources */,
+ 96032E4B0F5CC8C700241C5F /* time.c in Sources */,
+ 5AAB45C010D30B79004407EA /* data.c in Sources */,
+ 5A27262610F26F1900751FBC /* io.c in Sources */,
+ E43A72501AF85BBC00BAA921 /* block.cpp in Sources */,
+ 6EA962971D48622600759D53 /* event.c in Sources */,
+ C9C5F80E143C1771006DC718 /* transform.c in Sources */,
+ E4FC3264145F46C9002FBDDB /* object.m in Sources */,
+ 2BBF5A63154B64F5002B20F9 /* allocator.c in Sources */,
+ E420867016027AE500EEE210 /* data.m in Sources */,
+ E44A8E6B1805C3E0009FFDB6 /* voucher.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -2315,68 +2207,32 @@
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
- E46DBC4014EE10C80001F9F6 /* protocol.defs in Sources */,
- 6EBEC7E71BBDD30F009B1596 /* firehose.defs in Sources */,
- 6EBEC7EA1BBDD326009B1596 /* firehose_reply.defs in Sources */,
- E46DBC4114EE10C80001F9F6 /* resolver.c in Sources */,
- E46DBC4214EE10C80001F9F6 /* init.c in Sources */,
- E46DBC4714EE10C80001F9F6 /* object.c in Sources */,
- E43A72881AF85BE900BAA921 /* block.cpp in Sources */,
- 6EF2CAB11C8899EC001ABE83 /* lock.c in Sources */,
- E46DBC4414EE10C80001F9F6 /* semaphore.c in Sources */,
- E46DBC4514EE10C80001F9F6 /* once.c in Sources */,
- E46DBC4314EE10C80001F9F6 /* queue.c in Sources */,
- E46DBC4614EE10C80001F9F6 /* apply.c in Sources */,
- E46DBC4914EE10C80001F9F6 /* source.c in Sources */,
6E4BACC61D48A42300B562AE /* mach.c in Sources */,
- 6EA9629C1D48622A00759D53 /* event.c in Sources */,
+ E46DBC4014EE10C80001F9F6 /* protocol.defs in Sources */,
+ E46DBC4114EE10C80001F9F6 /* resolver.c in Sources */,
+ 6EF2CAB11C8899EC001ABE83 /* lock.c in Sources */,
+ E46DBC4214EE10C80001F9F6 /* init.c in Sources */,
+ E46DBC4314EE10C80001F9F6 /* queue.c in Sources */,
6EA962A41D48625300759D53 /* event_kevent.c in Sources */,
- 6E4BACFA1D49A04900B562AE /* event_epoll.c in Sources */,
- E44A8E701805C3E0009FFDB6 /* voucher.c in Sources */,
6EE664271BE2FD5C00ED7B1C /* firehose_buffer.c in Sources */,
- E46DBC4C14EE10C80001F9F6 /* io.c in Sources */,
- E46DBC4B14EE10C80001F9F6 /* data.c in Sources */,
- E46DBC4D14EE10C80001F9F6 /* transform.c in Sources */,
- E46DBC4A14EE10C80001F9F6 /* time.c in Sources */,
- 2BBF5A67154B64F5002B20F9 /* allocator.c in Sources */,
- E46DBC4814EE10C80001F9F6 /* benchmark.c in Sources */,
+ 6EA9629C1D48622A00759D53 /* event.c in Sources */,
+ 6EBEC7E71BBDD30F009B1596 /* firehose.defs in Sources */,
+ E43A72881AF85BE900BAA921 /* block.cpp in Sources */,
+ E46DBC4414EE10C80001F9F6 /* semaphore.c in Sources */,
6E9956011C3B21980071D40C /* venture.c in Sources */,
- );
- runOnlyForDeploymentPostprocessing = 0;
- };
- E49BB6D01E70748100868613 /* Sources */ = {
- isa = PBXSourcesBuildPhase;
- buildActionMask = 2147483647;
- files = (
- E49BB6D11E70748100868613 /* provider.d in Sources */,
- E49BB6D21E70748100868613 /* protocol.defs in Sources */,
- E49BB6D41E70748100868613 /* firehose.defs in Sources */,
- E49BB6DD1E70748100868613 /* firehose_reply.defs in Sources */,
- E49BB6D71E70748100868613 /* resolver.c in Sources */,
- E49BB6D91E70748100868613 /* init.c in Sources */,
- E49BB6E01E70748100868613 /* object.c in Sources */,
- E49BB6EA1E70748100868613 /* object.m in Sources */,
- E49BB6E71E70748100868613 /* block.cpp in Sources */,
- E49BB6DC1E70748100868613 /* lock.c in Sources */,
- E49BB6DB1E70748100868613 /* semaphore.c in Sources */,
- E49BB6DE1E70748100868613 /* once.c in Sources */,
- E49BB6D81E70748100868613 /* mach.c in Sources */,
- E49BB6DA1E70748100868613 /* queue.c in Sources */,
- E49BB6DF1E70748100868613 /* apply.c in Sources */,
- E49BB6E31E70748100868613 /* source.c in Sources */,
- E49BB6E81E70748100868613 /* event.c in Sources */,
- E49BB6D61E70748100868613 /* event_kevent.c in Sources */,
- E49BB6E21E70748100868613 /* event_epoll.c in Sources */,
- E49BB6ED1E70748100868613 /* voucher.c in Sources */,
- E49BB6D51E70748100868613 /* firehose_buffer.c in Sources */,
- E49BB6E61E70748100868613 /* io.c in Sources */,
- E49BB6E51E70748100868613 /* data.c in Sources */,
- E49BB6EC1E70748100868613 /* data.m in Sources */,
- E49BB6E91E70748100868613 /* transform.c in Sources */,
- E49BB6E41E70748100868613 /* time.c in Sources */,
- E49BB6EB1E70748100868613 /* allocator.c in Sources */,
- E49BB6E11E70748100868613 /* benchmark.c in Sources */,
- E49BB6D31E70748100868613 /* venture.c in Sources */,
+ 6EBEC7EA1BBDD326009B1596 /* firehose_reply.defs in Sources */,
+ E46DBC4514EE10C80001F9F6 /* once.c in Sources */,
+ E44A8E701805C3E0009FFDB6 /* voucher.c in Sources */,
+ E46DBC4614EE10C80001F9F6 /* apply.c in Sources */,
+ E46DBC4714EE10C80001F9F6 /* object.c in Sources */,
+ E46DBC4814EE10C80001F9F6 /* benchmark.c in Sources */,
+ E46DBC4914EE10C80001F9F6 /* source.c in Sources */,
+ E46DBC4A14EE10C80001F9F6 /* time.c in Sources */,
+ E46DBC4B14EE10C80001F9F6 /* data.c in Sources */,
+ E46DBC4C14EE10C80001F9F6 /* io.c in Sources */,
+ E46DBC4D14EE10C80001F9F6 /* transform.c in Sources */,
+ 2BBF5A67154B64F5002B20F9 /* allocator.c in Sources */,
+ 6E4BACFA1D49A04900B562AE /* event_epoll.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -2386,33 +2242,33 @@
files = (
E43570BA126E93380097AB9F /* provider.d in Sources */,
E49F24C8125D57FA0057C971 /* protocol.defs in Sources */,
- 6ED64B461BBD89AF00C35F4D /* firehose.defs in Sources */,
- 6ED64B4A1BBD89BD00C35F4D /* firehose_reply.defs in Sources */,
- E49F24C9125D57FA0057C971 /* resolver.c in Sources */,
- E49F24CA125D57FA0057C971 /* init.c in Sources */,
- E49F24CF125D57FA0057C971 /* object.c in Sources */,
- E4FC3265145F46C9002FBDDB /* object.m in Sources */,
- E43A72841AF85BCB00BAA921 /* block.cpp in Sources */,
- 6EF2CAAD1C8899E9001ABE83 /* lock.c in Sources */,
- E49F24CC125D57FA0057C971 /* semaphore.c in Sources */,
- E49F24CD125D57FA0057C971 /* once.c in Sources */,
- E49F24CB125D57FA0057C971 /* queue.c in Sources */,
- E49F24CE125D57FA0057C971 /* apply.c in Sources */,
- E49F24D1125D57FA0057C971 /* source.c in Sources */,
- 6E4BACC21D48A42000B562AE /* mach.c in Sources */,
- 6EA962981D48622700759D53 /* event.c in Sources */,
- 6EA962A01D48625100759D53 /* event_kevent.c in Sources */,
- 6E4BACF61D49A04700B562AE /* event_epoll.c in Sources */,
- E44A8E6C1805C3E0009FFDB6 /* voucher.c in Sources */,
- 6ED64B401BBD898300C35F4D /* firehose_buffer.c in Sources */,
- E49F24D4125D57FA0057C971 /* io.c in Sources */,
- E49F24D3125D57FA0057C971 /* data.c in Sources */,
- E420867116027AE500EEE210 /* data.m in Sources */,
- C93D6165143E190E00EB9023 /* transform.c in Sources */,
- E49F24D2125D57FA0057C971 /* time.c in Sources */,
- 2BBF5A64154B64F5002B20F9 /* allocator.c in Sources */,
- E49F24D0125D57FA0057C971 /* benchmark.c in Sources */,
6E9956051C3B219B0071D40C /* venture.c in Sources */,
+ 6ED64B461BBD89AF00C35F4D /* firehose.defs in Sources */,
+ 6ED64B401BBD898300C35F4D /* firehose_buffer.c in Sources */,
+ 6EA962A01D48625100759D53 /* event_kevent.c in Sources */,
+ E49F24C9125D57FA0057C971 /* resolver.c in Sources */,
+ 6E4BACC21D48A42000B562AE /* mach.c in Sources */,
+ E49F24CA125D57FA0057C971 /* init.c in Sources */,
+ E49F24CB125D57FA0057C971 /* queue.c in Sources */,
+ E49F24CC125D57FA0057C971 /* semaphore.c in Sources */,
+ 6EF2CAAD1C8899E9001ABE83 /* lock.c in Sources */,
+ 6ED64B4A1BBD89BD00C35F4D /* firehose_reply.defs in Sources */,
+ E49F24CD125D57FA0057C971 /* once.c in Sources */,
+ E49F24CE125D57FA0057C971 /* apply.c in Sources */,
+ E49F24CF125D57FA0057C971 /* object.c in Sources */,
+ E49F24D0125D57FA0057C971 /* benchmark.c in Sources */,
+ 6E4BACF61D49A04700B562AE /* event_epoll.c in Sources */,
+ E49F24D1125D57FA0057C971 /* source.c in Sources */,
+ E49F24D2125D57FA0057C971 /* time.c in Sources */,
+ E49F24D3125D57FA0057C971 /* data.c in Sources */,
+ E49F24D4125D57FA0057C971 /* io.c in Sources */,
+ E43A72841AF85BCB00BAA921 /* block.cpp in Sources */,
+ 6EA962981D48622700759D53 /* event.c in Sources */,
+ C93D6165143E190E00EB9023 /* transform.c in Sources */,
+ E4FC3265145F46C9002FBDDB /* object.m in Sources */,
+ 2BBF5A64154B64F5002B20F9 /* allocator.c in Sources */,
+ E420867116027AE500EEE210 /* data.m in Sources */,
+ E44A8E6C1805C3E0009FFDB6 /* voucher.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -2421,34 +2277,34 @@
buildActionMask = 2147483647;
files = (
E4B515BD164B2DA300E003AF /* provider.d in Sources */,
+ 6EA962A31D48625300759D53 /* event_kevent.c in Sources */,
E4B515BE164B2DA300E003AF /* protocol.defs in Sources */,
- 6ED64B481BBD89B100C35F4D /* firehose.defs in Sources */,
- 6ED64B4B1BBD89BE00C35F4D /* firehose_reply.defs in Sources */,
E4B515BF164B2DA300E003AF /* resolver.c in Sources */,
+ 6ED64B4B1BBD89BE00C35F4D /* firehose_reply.defs in Sources */,
+ 6ED64B481BBD89B100C35F4D /* firehose.defs in Sources */,
E4B515C0164B2DA300E003AF /* init.c in Sources */,
- E4B515C5164B2DA300E003AF /* object.c in Sources */,
- E4B515CC164B2DA300E003AF /* object.m in Sources */,
- E43A72871AF85BCD00BAA921 /* block.cpp in Sources */,
- 6EF2CAB01C8899EB001ABE83 /* lock.c in Sources */,
+ 6EA9629B1D48622900759D53 /* event.c in Sources */,
+ E4B515C1164B2DA300E003AF /* queue.c in Sources */,
+ 6E9956021C3B21990071D40C /* venture.c in Sources */,
E4B515C2164B2DA300E003AF /* semaphore.c in Sources */,
E4B515C3164B2DA300E003AF /* once.c in Sources */,
- E4B515C1164B2DA300E003AF /* queue.c in Sources */,
+ E43A72871AF85BCD00BAA921 /* block.cpp in Sources */,
E4B515C4164B2DA300E003AF /* apply.c in Sources */,
- E4B515C7164B2DA300E003AF /* source.c in Sources */,
- 6E4BACC51D48A42200B562AE /* mach.c in Sources */,
- 6EA9629B1D48622900759D53 /* event.c in Sources */,
- 6EA962A31D48625300759D53 /* event_kevent.c in Sources */,
- 6E4BACF91D49A04800B562AE /* event_epoll.c in Sources */,
- E44A8E6F1805C3E0009FFDB6 /* voucher.c in Sources */,
+ E4B515C5164B2DA300E003AF /* object.c in Sources */,
6ED64B431BBD898600C35F4D /* firehose_buffer.c in Sources */,
- E4B515CA164B2DA300E003AF /* io.c in Sources */,
- E4B515C9164B2DA300E003AF /* data.c in Sources */,
- E4B515CE164B2DA300E003AF /* data.m in Sources */,
- E4B515CB164B2DA300E003AF /* transform.c in Sources */,
- E4B515C8164B2DA300E003AF /* time.c in Sources */,
- E4B515CD164B2DA300E003AF /* allocator.c in Sources */,
E4B515C6164B2DA300E003AF /* benchmark.c in Sources */,
- 6E9956021C3B21990071D40C /* venture.c in Sources */,
+ E4B515C7164B2DA300E003AF /* source.c in Sources */,
+ E4B515C8164B2DA300E003AF /* time.c in Sources */,
+ 6E4BACC51D48A42200B562AE /* mach.c in Sources */,
+ E4B515C9164B2DA300E003AF /* data.c in Sources */,
+ E4B515CA164B2DA300E003AF /* io.c in Sources */,
+ E44A8E6F1805C3E0009FFDB6 /* voucher.c in Sources */,
+ E4B515CB164B2DA300E003AF /* transform.c in Sources */,
+ 6EF2CAB01C8899EB001ABE83 /* lock.c in Sources */,
+ E4B515CC164B2DA300E003AF /* object.m in Sources */,
+ E4B515CD164B2DA300E003AF /* allocator.c in Sources */,
+ 6E4BACF91D49A04800B562AE /* event_epoll.c in Sources */,
+ E4B515CE164B2DA300E003AF /* data.m in Sources */,
E4B515DD164B32E000E003AF /* introspection.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
@@ -2459,33 +2315,33 @@
files = (
E417A38412A472C4004D659D /* provider.d in Sources */,
E44EBE5412517EBE00645D88 /* protocol.defs in Sources */,
- 6EBEC7E61BBDD30D009B1596 /* firehose.defs in Sources */,
- 6EBEC7E91BBDD325009B1596 /* firehose_reply.defs in Sources */,
- E49F2424125D3C970057C971 /* resolver.c in Sources */,
- E44EBE5512517EBE00645D88 /* init.c in Sources */,
- E4EC11B212514302000DDBD1 /* object.c in Sources */,
- E4FC3266145F46C9002FBDDB /* object.m in Sources */,
- E43A72861AF85BCC00BAA921 /* block.cpp in Sources */,
- 6EF2CAAF1C8899EB001ABE83 /* lock.c in Sources */,
- E4EC11AF12514302000DDBD1 /* semaphore.c in Sources */,
- E4EC11B012514302000DDBD1 /* once.c in Sources */,
- E4EC11AE12514302000DDBD1 /* queue.c in Sources */,
- E4EC11B112514302000DDBD1 /* apply.c in Sources */,
- E4EC11B412514302000DDBD1 /* source.c in Sources */,
- 6E4BACC41D48A42200B562AE /* mach.c in Sources */,
- 6EA9629A1D48622900759D53 /* event.c in Sources */,
- 6EA962A21D48625200759D53 /* event_kevent.c in Sources */,
- 6E4BACF81D49A04800B562AE /* event_epoll.c in Sources */,
- E44A8E6E1805C3E0009FFDB6 /* voucher.c in Sources */,
- 6ED64B421BBD898500C35F4D /* firehose_buffer.c in Sources */,
- E4EC11B812514302000DDBD1 /* io.c in Sources */,
- E4EC11B712514302000DDBD1 /* data.c in Sources */,
- E420867316027AE500EEE210 /* data.m in Sources */,
- C93D6166143E190F00EB9023 /* transform.c in Sources */,
- E4EC11B512514302000DDBD1 /* time.c in Sources */,
- 2BBF5A65154B64F5002B20F9 /* allocator.c in Sources */,
- E4EC11B312514302000DDBD1 /* benchmark.c in Sources */,
6E9956031C3B219A0071D40C /* venture.c in Sources */,
+ 6EBEC7E61BBDD30D009B1596 /* firehose.defs in Sources */,
+ 6ED64B421BBD898500C35F4D /* firehose_buffer.c in Sources */,
+ 6EA962A21D48625200759D53 /* event_kevent.c in Sources */,
+ E49F2424125D3C970057C971 /* resolver.c in Sources */,
+ 6E4BACC41D48A42200B562AE /* mach.c in Sources */,
+ E44EBE5512517EBE00645D88 /* init.c in Sources */,
+ E4EC11AE12514302000DDBD1 /* queue.c in Sources */,
+ E4EC11AF12514302000DDBD1 /* semaphore.c in Sources */,
+ 6EF2CAAF1C8899EB001ABE83 /* lock.c in Sources */,
+ 6EBEC7E91BBDD325009B1596 /* firehose_reply.defs in Sources */,
+ E4EC11B012514302000DDBD1 /* once.c in Sources */,
+ E4EC11B112514302000DDBD1 /* apply.c in Sources */,
+ E4EC11B212514302000DDBD1 /* object.c in Sources */,
+ E4EC11B312514302000DDBD1 /* benchmark.c in Sources */,
+ 6E4BACF81D49A04800B562AE /* event_epoll.c in Sources */,
+ E4EC11B412514302000DDBD1 /* source.c in Sources */,
+ E4EC11B512514302000DDBD1 /* time.c in Sources */,
+ E4EC11B712514302000DDBD1 /* data.c in Sources */,
+ E4EC11B812514302000DDBD1 /* io.c in Sources */,
+ E43A72861AF85BCC00BAA921 /* block.cpp in Sources */,
+ 6EA9629A1D48622900759D53 /* event.c in Sources */,
+ C93D6166143E190F00EB9023 /* transform.c in Sources */,
+ E4FC3266145F46C9002FBDDB /* object.m in Sources */,
+ 2BBF5A65154B64F5002B20F9 /* allocator.c in Sources */,
+ E420867316027AE500EEE210 /* data.m in Sources */,
+ E44A8E6E1805C3E0009FFDB6 /* voucher.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -2495,33 +2351,33 @@
files = (
E417A38512A472C5004D659D /* provider.d in Sources */,
E44EBE5612517EBE00645D88 /* protocol.defs in Sources */,
- 6EBEC7E51BBDD30C009B1596 /* firehose.defs in Sources */,
- 6EBEC7E81BBDD324009B1596 /* firehose_reply.defs in Sources */,
- E49F2423125D3C960057C971 /* resolver.c in Sources */,
- E44EBE5712517EBE00645D88 /* init.c in Sources */,
- E4EC121E12514715000DDBD1 /* object.c in Sources */,
- E4FC3267145F46C9002FBDDB /* object.m in Sources */,
- E43A72851AF85BCC00BAA921 /* block.cpp in Sources */,
- 6EF2CAAE1C8899EA001ABE83 /* lock.c in Sources */,
- E4EC121B12514715000DDBD1 /* semaphore.c in Sources */,
- E4EC121C12514715000DDBD1 /* once.c in Sources */,
- E4EC121A12514715000DDBD1 /* queue.c in Sources */,
- E4EC121D12514715000DDBD1 /* apply.c in Sources */,
- E4EC122012514715000DDBD1 /* source.c in Sources */,
- 6E4BACC31D48A42100B562AE /* mach.c in Sources */,
- 6EA962991D48622800759D53 /* event.c in Sources */,
- 6EA962A11D48625100759D53 /* event_kevent.c in Sources */,
- 6E4BACF71D49A04700B562AE /* event_epoll.c in Sources */,
- E44A8E6D1805C3E0009FFDB6 /* voucher.c in Sources */,
- 6ED64B411BBD898400C35F4D /* firehose_buffer.c in Sources */,
- E4EC122412514715000DDBD1 /* io.c in Sources */,
- E4EC122312514715000DDBD1 /* data.c in Sources */,
- E420867216027AE500EEE210 /* data.m in Sources */,
- C93D6167143E190F00EB9023 /* transform.c in Sources */,
- E4EC122112514715000DDBD1 /* time.c in Sources */,
- 2BBF5A66154B64F5002B20F9 /* allocator.c in Sources */,
- E4EC121F12514715000DDBD1 /* benchmark.c in Sources */,
6E9956041C3B219B0071D40C /* venture.c in Sources */,
+ 6EBEC7E51BBDD30C009B1596 /* firehose.defs in Sources */,
+ 6ED64B411BBD898400C35F4D /* firehose_buffer.c in Sources */,
+ 6EA962A11D48625100759D53 /* event_kevent.c in Sources */,
+ E49F2423125D3C960057C971 /* resolver.c in Sources */,
+ 6E4BACC31D48A42100B562AE /* mach.c in Sources */,
+ E44EBE5712517EBE00645D88 /* init.c in Sources */,
+ E4EC121A12514715000DDBD1 /* queue.c in Sources */,
+ E4EC121B12514715000DDBD1 /* semaphore.c in Sources */,
+ 6EF2CAAE1C8899EA001ABE83 /* lock.c in Sources */,
+ 6EBEC7E81BBDD324009B1596 /* firehose_reply.defs in Sources */,
+ E4EC121C12514715000DDBD1 /* once.c in Sources */,
+ E4EC121D12514715000DDBD1 /* apply.c in Sources */,
+ E4EC121E12514715000DDBD1 /* object.c in Sources */,
+ E4EC121F12514715000DDBD1 /* benchmark.c in Sources */,
+ 6E4BACF71D49A04700B562AE /* event_epoll.c in Sources */,
+ E4EC122012514715000DDBD1 /* source.c in Sources */,
+ E4EC122112514715000DDBD1 /* time.c in Sources */,
+ E4EC122312514715000DDBD1 /* data.c in Sources */,
+ E4EC122412514715000DDBD1 /* io.c in Sources */,
+ E43A72851AF85BCC00BAA921 /* block.cpp in Sources */,
+ 6EA962991D48622800759D53 /* event.c in Sources */,
+ C93D6167143E190F00EB9023 /* transform.c in Sources */,
+ E4FC3267145F46C9002FBDDB /* object.m in Sources */,
+ 2BBF5A66154B64F5002B20F9 /* allocator.c in Sources */,
+ E420867216027AE500EEE210 /* data.m in Sources */,
+ E44A8E6D1805C3E0009FFDB6 /* voucher.c in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -2583,11 +2439,6 @@
target = E4EC121612514715000DDBD1 /* libdispatch mp resolved */;
targetProxy = E47D6ECC125FEBA10070D91C /* PBXContainerItemProxy */;
};
- E49BB6F81E7074C100868613 /* PBXTargetDependency */ = {
- isa = PBXTargetDependency;
- target = E49BB6CE1E70748100868613 /* libdispatch alt resolved */;
- targetProxy = E49BB6F71E7074C100868613 /* PBXContainerItemProxy */;
- };
E4B515DB164B317700E003AF /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = E4B51595164B2DA300E003AF /* libdispatch introspection */;
@@ -2708,6 +2559,7 @@
isa = XCBuildConfiguration;
baseConfigurationReference = C00B0E121C5AEBF7000330B3 /* libdispatch-dyld-stub.xcconfig */;
buildSettings = {
+ PRODUCT_NAME = "$(PRODUCT_NAME)";
};
name = Release;
};
@@ -2715,6 +2567,7 @@
isa = XCBuildConfiguration;
baseConfigurationReference = C00B0E121C5AEBF7000330B3 /* libdispatch-dyld-stub.xcconfig */;
buildSettings = {
+ PRODUCT_NAME = "$(PRODUCT_NAME)";
};
name = Debug;
};
@@ -2722,6 +2575,7 @@
isa = XCBuildConfiguration;
baseConfigurationReference = C01866BE1C59735B0040FC07 /* libdispatch-mp-static.xcconfig */;
buildSettings = {
+ PRODUCT_NAME = "$(PRODUCT_NAME)";
};
name = Release;
};
@@ -2729,6 +2583,7 @@
isa = XCBuildConfiguration;
baseConfigurationReference = C01866BE1C59735B0040FC07 /* libdispatch-mp-static.xcconfig */;
buildSettings = {
+ PRODUCT_NAME = "$(PRODUCT_NAME)";
};
name = Debug;
};
@@ -2758,22 +2613,6 @@
};
name = Debug;
};
- E49BB6F01E70748100868613 /* Release */ = {
- isa = XCBuildConfiguration;
- baseConfigurationReference = E40041A9125D70590022B135 /* libdispatch-resolved.xcconfig */;
- buildSettings = {
- DISPATCH_RESOLVED_VARIANT = alt;
- };
- name = Release;
- };
- E49BB6F11E70748100868613 /* Debug */ = {
- isa = XCBuildConfiguration;
- baseConfigurationReference = E40041A9125D70590022B135 /* libdispatch-resolved.xcconfig */;
- buildSettings = {
- DISPATCH_RESOLVED_VARIANT = alt;
- };
- name = Debug;
- };
E49F24D9125D57FA0057C971 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
@@ -2787,7 +2626,6 @@
E49F24DA125D57FA0057C971 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
- ONLY_ACTIVE_ARCH = YES;
WARNING_CFLAGS = (
"-Weverything",
"$(inherited)",
@@ -2975,15 +2813,6 @@
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
- E49BB6EF1E70748100868613 /* Build configuration list for PBXNativeTarget "libdispatch alt resolved" */ = {
- isa = XCConfigurationList;
- buildConfigurations = (
- E49BB6F01E70748100868613 /* Release */,
- E49BB6F11E70748100868613 /* Debug */,
- );
- defaultConfigurationIsVisible = 0;
- defaultConfigurationName = Release;
- };
E49F24D8125D57FA0057C971 /* Build configuration list for PBXNativeTarget "libdispatch no resolver" */ = {
isa = XCConfigurationList;
buildConfigurations = (
diff --git a/man/dispatch_queue_create.3 b/man/dispatch_queue_create.3
index 833e564..f3c3051 100644
--- a/man/dispatch_queue_create.3
+++ b/man/dispatch_queue_create.3
@@ -72,8 +72,7 @@
By convention, clients should pass a reverse DNS style label. For example:
.Pp
.Bd -literal -offset indent
-my_queue = dispatch_queue_create("com.example.subsystem.taskXYZ",
- DISPATCH_QUEUE_SERIAL);
+my_queue = dispatch_queue_create("com.example.subsystem.taskXYZ", NULL);
.Ed
.Pp
The
diff --git a/os/firehose_buffer_private.h b/os/firehose_buffer_private.h
index d131d6d..29b80c3 100644
--- a/os/firehose_buffer_private.h
+++ b/os/firehose_buffer_private.h
@@ -31,7 +31,7 @@
#include <dispatch/dispatch.h>
#endif
-#define OS_FIREHOSE_SPI_VERSION 20170222
+#define OS_FIREHOSE_SPI_VERSION 20160318
/*!
* @group Firehose SPI
diff --git a/os/firehose_server_private.h b/os/firehose_server_private.h
index fc352da..441bb52 100644
--- a/os/firehose_server_private.h
+++ b/os/firehose_server_private.h
@@ -228,23 +228,6 @@
firehose_client_set_context(firehose_client_t client, void *ctxt);
/*!
- * @function firehose_client_initiate_quarantine
- *
- * @abstract
- * Starts the procedure to move the given client to the high volume quarantine
- *
- * @discussion
- * When the client is in the high volume quarantine, their firehose chunks
- * have the fcp_quarantined bit set to 1.
- *
- * @param client
- * The specified client.
- */
-OS_NOTHROW OS_NONNULL1
-void
-firehose_client_initiate_quarantine(firehose_client_t client);
-
-/*!
* @function firehose_client_metadata_stream_peek
*
* @abstract
@@ -378,36 +361,6 @@
dispatch_queue_t
firehose_server_copy_queue(firehose_server_queue_t which);
-/*!
- * @function firehose_server_quarantined_suspend
- *
- * @abstract
- * Suspends processing of quarantined clients until
- * firehose_server_quarantined_resume() is called for the same queue.
- *
- * @discussion
- * Suspending processing of quarantined clients causes firehose_snapshot()
- * to block until the processing is enabled again.
- *
- * However if this is used to pace the processing, it is a good idea to disable
- * this pacing until the snapshot has completed.
- *
- * Similarly, quarantine suspension must be off during shutdown.
- */
-OS_NOTHROW
-void
-firehose_server_quarantined_suspend(firehose_server_queue_t q);
-
-/*!
- * @function firehose_server_quarantined_resume
- *
- * @abstract
- * Resumes processing of quarantined clients.
- */
-OS_NOTHROW
-void
-firehose_server_quarantined_resume(firehose_server_queue_t q);
-
#pragma mark - Firehose Snapshot
/*!
diff --git a/os/object_private.h b/os/object_private.h
index 215c3d1..2f8cdf4 100644
--- a/os/object_private.h
+++ b/os/object_private.h
@@ -36,9 +36,7 @@
#define OS_OBJECT_NONNULL __attribute__((__nonnull__))
#define OS_OBJECT_WARN_RESULT __attribute__((__warn_unused_result__))
#define OS_OBJECT_MALLOC __attribute__((__malloc__))
-#ifndef OS_OBJECT_EXPORT
#define OS_OBJECT_EXPORT extern __attribute__((visibility("default")))
-#endif
#else
/*! @parseOnly */
#define OS_OBJECT_NOTHROW
@@ -48,11 +46,8 @@
#define OS_OBJECT_WARN_RESULT
/*! @parseOnly */
#define OS_OBJECT_MALLOC
-#ifndef OS_OBJECT_EXPORT
-/*! @parseOnly */
#define OS_OBJECT_EXPORT extern
#endif
-#endif
#if OS_OBJECT_USE_OBJC && __has_feature(objc_arc)
#define _OS_OBJECT_OBJC_ARC 1
@@ -184,18 +179,6 @@
void
_os_object_release_internal(_os_object_t object);
-API_AVAILABLE(macos(10.12), ios(10.0), tvos(10.0), watchos(3.0))
-OS_OBJECT_EXPORT OS_OBJECT_NONNULL OS_OBJECT_NOTHROW
-OS_SWIFT_UNAVAILABLE("Unavailable in Swift")
-_os_object_t
-_os_object_retain_internal_n(_os_object_t object, uint16_t n);
-
-API_AVAILABLE(macos(10.12), ios(10.0), tvos(10.0), watchos(3.0))
-OS_OBJECT_EXPORT OS_OBJECT_NONNULL OS_OBJECT_NOTHROW
-OS_SWIFT_UNAVAILABLE("Unavailable in Swift")
-void
-_os_object_release_internal_n(_os_object_t object, uint16_t n);
-
#endif // !_OS_OBJECT_OBJC_ARC
__END_DECLS
diff --git a/os/voucher_activity_private.h b/os/voucher_activity_private.h
index 8ce0ef5..28effc9 100644
--- a/os/voucher_activity_private.h
+++ b/os/voucher_activity_private.h
@@ -282,13 +282,12 @@
const void *privdata, size_t privlen);
typedef const struct voucher_activity_hooks_s {
-#define VOUCHER_ACTIVITY_HOOKS_VERSION 5
+#define VOUCHER_ACTIVITY_HOOKS_VERSION 4
long vah_version;
mach_port_t (*vah_get_logd_port)(void);
dispatch_mach_handler_function_t vah_debug_channel_handler;
kern_return_t (*vah_get_reconnect_info)(mach_vm_address_t *, mach_vm_size_t *);
void (*vah_metadata_init)(void *metadata_buffer, size_t size);
- void (*vah_quarantine_starts)(void);
} *voucher_activity_hooks_t;
/*!
diff --git a/private/mach_private.h b/private/mach_private.h
index bc53223..6ca891d 100644
--- a/private/mach_private.h
+++ b/private/mach_private.h
@@ -114,9 +114,7 @@
* A SIGTERM signal has been received. This notification is delivered at most
* once during the lifetime of the channel. This event is sent only for XPC
* channels (i.e. channels that were created by calling
- * dispatch_mach_create_4libxpc()) and only if the
- * dmxh_enable_sigterm_notification function in the XPC hooks structure is not
- * set or it returned true when it was called at channel activation time.
+ * dispatch_mach_create_4libxpc()).
*
* @const DISPATCH_MACH_ASYNC_WAITER_DISCONNECTED
* The channel has been disconnected by a call to dispatch_mach_reconnect() or
@@ -813,7 +811,7 @@
API_AVAILABLE(macos(10.12), ios(10.0), tvos(10.0), watchos(3.0))
typedef const struct dispatch_mach_xpc_hooks_s {
-#define DISPATCH_MACH_XPC_HOOKS_VERSION 3
+#define DISPATCH_MACH_XPC_HOOKS_VERSION 2
unsigned long version;
/* Fields available in version 1. */
@@ -829,8 +827,8 @@
* throw an exception.
*/
bool (* _Nonnull dmxh_direct_message_handler)(void *_Nullable context,
- dispatch_mach_reason_t reason, dispatch_mach_msg_t message,
- mach_error_t error);
+ dispatch_mach_reason_t reason, dispatch_mach_msg_t message,
+ mach_error_t error);
/* Fields available in version 2. */
@@ -846,7 +844,7 @@
* other code.
*/
dispatch_queue_t _Nullable (*_Nonnull dmxh_msg_context_reply_queue)(
- void *_Nonnull msg_context);
+ void *_Nonnull msg_context);
/*
* Called when a reply to a message sent by
@@ -863,15 +861,6 @@
* details.
*/
dispatch_mach_async_reply_callback_t dmxh_async_reply_handler;
-
- /* Fields available in version 3. */
- /**
- * Called once when the Mach channel has been activated. If this function
- * returns true, a DISPATCH_MACH_SIGTERM_RECEIVED notification will be
- * delivered to the channel's event handler when a SIGTERM is received.
- */
- bool (* _Nullable dmxh_enable_sigterm_notification)(
- void *_Nullable context);
} *dispatch_mach_xpc_hooks_t;
#define DISPATCH_MACH_XPC_SUPPORTS_ASYNC_REPLIES(hooks) ((hooks)->version >= 2)
diff --git a/private/private.h b/private/private.h
index cc9d578..82da15e 100644
--- a/private/private.h
+++ b/private/private.h
@@ -66,7 +66,7 @@
#endif /* !__DISPATCH_BUILDING_DISPATCH__ */
// <rdar://problem/9627726> Check that public and private dispatch headers match
-#if DISPATCH_API_VERSION != 20170124 // Keep in sync with <dispatch/dispatch.h>
+#if DISPATCH_API_VERSION != 20160831 // Keep in sync with <dispatch/dispatch.h>
#error "Dispatch header mismatch between /usr/include and /usr/local/include"
#endif
@@ -214,16 +214,6 @@
DISPATCH_EXPORT DISPATCH_WARN_RESULT DISPATCH_NOTHROW
mach_port_t
_dispatch_runloop_root_queue_get_port_4CF(dispatch_queue_t queue);
-
-#ifdef __BLOCKS__
-API_AVAILABLE(macos(10.12), ios(10.0), tvos(10.0), watchos(3.0))
-DISPATCH_EXPORT DISPATCH_MALLOC DISPATCH_RETURNS_RETAINED DISPATCH_WARN_RESULT
-DISPATCH_NOTHROW
-dispatch_queue_t
-_dispatch_network_root_queue_create_4NW(const char *_Nullable label,
- const pthread_attr_t *_Nullable attrs,
- dispatch_block_t _Nullable configure);
-#endif
#endif
API_AVAILABLE(macos(10.9), ios(7.0))
@@ -252,11 +242,6 @@
#endif /* DISPATCH_COCOA_COMPAT */
-API_AVAILABLE(macos(10.12), ios(10.0), tvos(10.0), watchos(3.0))
-DISPATCH_EXPORT DISPATCH_NOTHROW
-void
-_dispatch_poll_for_events_4launchd(void);
-
__END_DECLS
DISPATCH_ASSUME_NONNULL_END
diff --git a/private/queue_private.h b/private/queue_private.h
index 98c7f5e..14d6477 100644
--- a/private/queue_private.h
+++ b/private/queue_private.h
@@ -278,12 +278,11 @@
/*!
* @constant DISPATCH_APPLY_CURRENT_ROOT_QUEUE
- *
- * @discussion
- * This constant is deprecated, please use DISPATCH_APPLY_AUTO.
- *
- * DISPATCH_APPLY_AUTO also selects the current pthread root queue if
- * applicable.
+ * @discussion Constant to pass to the dispatch_apply() and dispatch_apply_f()
+ * functions to indicate that the root queue for the current thread should be
+ * used (i.e. one of the global concurrent queues or a queue created with
+ * dispatch_pthread_root_queue_create()). If there is no such queue, the
+ * default priority global concurrent queue will be used.
*/
#define DISPATCH_APPLY_CURRENT_ROOT_QUEUE ((dispatch_queue_t _Nonnull)0)
diff --git a/private/source_private.h b/private/source_private.h
index 019f648..f01287b 100644
--- a/private/source_private.h
+++ b/private/source_private.h
@@ -165,6 +165,15 @@
DISPATCH_SOCK_NOTIFY_ACK = 0x00004000,
};
+/*!
+ * @enum dispatch_source_nw_channel_flags_t
+ *
+ * @constant DISPATCH_NW_CHANNEL_FLOW_ADV_UPDATE
+ * Received network channel flow advisory.
+ */
+enum {
+ DISPATCH_NW_CHANNEL_FLOW_ADV_UPDATE = 0x00000001,
+};
/*!
* @enum dispatch_source_vfs_flags_t
@@ -332,16 +341,11 @@
*
* @constant DISPATCH_MEMORYPRESSURE_PROC_LIMIT_CRITICAL
* The memory of the process has reached 100% of its high watermark limit.
- *
- * @constant DISPATCH_MEMORYPRESSURE_MSL_STATUS
- * Mask for enabling/disabling malloc stack logging.
*/
enum {
- DISPATCH_MEMORYPRESSURE_PROC_LIMIT_WARN DISPATCH_ENUM_API_AVAILABLE(macos(10.12), ios(10.0), tvos(10.0), watchos(3.0)) = 0x10,
+ DISPATCH_MEMORYPRESSURE_PROC_LIMIT_WARN DISPATCH_ENUM_API_AVAILABLE(macos(10.12), ios(10.10), tvos(10.10), watchos(3.0)) = 0x10,
- DISPATCH_MEMORYPRESSURE_PROC_LIMIT_CRITICAL DISPATCH_ENUM_API_AVAILABLE(macos(10.12), ios(10.0), tvos(10.0), watchos(3.0)) = 0x20,
-
- DISPATCH_MEMORYPRESSURE_MSL_STATUS DISPATCH_ENUM_API_AVAILABLE(macos(10.12), ios(10.0), tvos(10.0), watchos(3.0)) = 0xf0000000,
+ DISPATCH_MEMORYPRESSURE_PROC_LIMIT_CRITICAL DISPATCH_ENUM_API_AVAILABLE(macos(10.12), ios(10.10), tvos(10.10), watchos(3.0)) = 0x20,
};
/*!
diff --git a/src/apply.c b/src/apply.c
index 9d64522..40e6f32 100644
--- a/src/apply.c
+++ b/src/apply.c
@@ -35,7 +35,7 @@
size_t idx, done = 0;
idx = os_atomic_inc_orig2o(da, da_index, acquire);
- if (unlikely(idx >= iter)) goto out;
+ if (!fastpath(idx < iter)) goto out;
// da_dc is only safe to access once the 'index lock' has been acquired
dispatch_apply_function_t const func = (void *)da->da_dc->dc_func;
@@ -67,7 +67,7 @@
done++;
idx = os_atomic_inc_orig2o(da, da_index, relaxed);
});
- } while (likely(idx < iter));
+ } while (fastpath(idx < iter));
if (invoke_flags & DISPATCH_APPLY_INVOKE_REDIRECT) {
_dispatch_reset_basepri(old_dbp);
@@ -124,7 +124,7 @@
while (dq && !qaf) {
qaf = _dispatch_queue_autorelease_frequency(dq);
- dq = dq->do_targetq;
+ dq = slowpath(dq->do_targetq);
}
return qaf;
}
@@ -198,13 +198,13 @@
do {
int32_t width = _dispatch_queue_try_reserve_apply_width(rq, da_width);
- if (unlikely(da_width > width)) {
+ if (slowpath(da_width > width)) {
int32_t excess = da_width - width;
for (tq = dq; tq != rq; tq = tq->do_targetq) {
_dispatch_queue_relinquish_width(tq, excess);
}
da_width -= excess;
- if (unlikely(!da_width)) {
+ if (slowpath(!da_width)) {
return _dispatch_apply_serial(da);
}
da->da_thr_cnt -= excess;
@@ -216,41 +216,22 @@
da->da_flags = _dispatch_queue_autorelease_frequency(dq);
}
rq = rq->do_targetq;
- } while (unlikely(rq->do_targetq));
+ } while (slowpath(rq->do_targetq));
_dispatch_apply_f2(rq, da, _dispatch_apply_redirect_invoke);
do {
_dispatch_queue_relinquish_width(dq, da_width);
dq = dq->do_targetq;
- } while (unlikely(dq->do_targetq));
+ } while (slowpath(dq->do_targetq));
}
#define DISPATCH_APPLY_MAX UINT16_MAX // must be < sqrt(SIZE_MAX)
-DISPATCH_ALWAYS_INLINE
-static inline dispatch_queue_t
-_dispatch_apply_root_queue(dispatch_queue_t dq)
-{
- if (dq) {
- while (unlikely(dq->do_targetq)) {
- dq = dq->do_targetq;
- }
- // if the current root queue is a pthread root queue, select it
- if (!_dispatch_priority_qos(dq->dq_priority)) {
- return dq;
- }
- }
-
- pthread_priority_t pp = _dispatch_get_priority();
- dispatch_qos_t qos = _dispatch_qos_from_pp(pp);
- return _dispatch_get_root_queue(qos ? qos : DISPATCH_QOS_DEFAULT, false);
-}
-
DISPATCH_NOINLINE
void
dispatch_apply_f(size_t iterations, dispatch_queue_t dq, void *ctxt,
void (*func)(void *, size_t))
{
- if (unlikely(iterations == 0)) {
+ if (slowpath(iterations == 0)) {
return;
}
int32_t thr_cnt = (int32_t)dispatch_hw_config(active_cpus);
@@ -259,7 +240,7 @@
size_t nested = dtctxt ? dtctxt->dtc_apply_nesting : 0;
dispatch_queue_t old_dq = _dispatch_queue_get_current();
- if (likely(!nested)) {
+ if (!slowpath(nested)) {
nested = iterations;
} else {
thr_cnt = nested < (size_t)thr_cnt ? thr_cnt / (int32_t)nested : 1;
@@ -269,8 +250,12 @@
if (iterations < (size_t)thr_cnt) {
thr_cnt = (int32_t)iterations;
}
- if (likely(dq == DISPATCH_APPLY_AUTO)) {
- dq = _dispatch_apply_root_queue(old_dq);
+ if (slowpath(dq == DISPATCH_APPLY_CURRENT_ROOT_QUEUE)) {
+ dq = old_dq ? old_dq : _dispatch_get_root_queue(
+ DISPATCH_QOS_DEFAULT, false);
+ while (slowpath(dq->do_targetq)) {
+ dq = dq->do_targetq;
+ }
}
struct dispatch_continuation_s dc = {
.dc_func = (void*)func,
@@ -291,11 +276,11 @@
#endif
da->da_flags = 0;
- if (unlikely(dq->dq_width == 1 || thr_cnt <= 1)) {
+ if (slowpath(dq->dq_width == 1) || slowpath(thr_cnt <= 1)) {
return dispatch_sync_f(dq, da, _dispatch_apply_serial);
}
- if (unlikely(dq->do_targetq)) {
- if (unlikely(dq == old_dq)) {
+ if (slowpath(dq->do_targetq)) {
+ if (slowpath(dq == old_dq)) {
return dispatch_sync_f(dq, da, _dispatch_apply_serial);
} else {
return dispatch_sync_f(dq, da, _dispatch_apply_redirect);
diff --git a/src/block.cpp b/src/block.cpp
index 2a6f007..3060a2a 100644
--- a/src/block.cpp
+++ b/src/block.cpp
@@ -32,8 +32,6 @@
#include "internal.h"
}
-// NOTE: this file must not contain any atomic operations
-
#if DISPATCH_DEBUG && DISPATCH_BLOCK_PRIVATE_DATA_DEBUG
#define _dispatch_block_private_data_debug(msg, ...) \
_dispatch_debug("block_private[%p]: " msg, (this), ##__VA_ARGS__)
@@ -85,8 +83,7 @@
((void (*)(dispatch_group_t))dispatch_release)(dbpd_group);
}
if (dbpd_queue) {
- ((void (*)(os_mpsc_queue_t, uint16_t))
- _os_object_release_internal_n)(dbpd_queue, 2);
+ ((void (*)(os_mpsc_queue_t))_os_object_release_internal)(dbpd_queue);
}
if (dbpd_block) Block_release(dbpd_block);
if (dbpd_voucher) voucher_release(dbpd_voucher);
diff --git a/src/data.c b/src/data.c
index 240309f..adcfbb2 100644
--- a/src/data.c
+++ b/src/data.c
@@ -100,22 +100,51 @@
#define _dispatch_data_release(x) dispatch_release(x)
#endif
+const dispatch_block_t _dispatch_data_destructor_free = ^{
+ DISPATCH_INTERNAL_CRASH(0, "free destructor called");
+};
+
+const dispatch_block_t _dispatch_data_destructor_none = ^{
+ DISPATCH_INTERNAL_CRASH(0, "none destructor called");
+};
+
+#if !HAVE_MACH
+const dispatch_block_t _dispatch_data_destructor_munmap = ^{
+ DISPATCH_INTERNAL_CRASH(0, "munmap destructor called");
+};
+#else
+// _dispatch_data_destructor_munmap is a linker alias to the following
+const dispatch_block_t _dispatch_data_destructor_vm_deallocate = ^{
+ DISPATCH_INTERNAL_CRASH(0, "vmdeallocate destructor called");
+};
+#endif
+
+const dispatch_block_t _dispatch_data_destructor_inline = ^{
+ DISPATCH_INTERNAL_CRASH(0, "inline destructor called");
+};
+
+struct dispatch_data_s _dispatch_data_empty = {
+#if DISPATCH_DATA_IS_BRIDGED_TO_NSDATA
+ .do_vtable = DISPATCH_DATA_EMPTY_CLASS,
+#else
+ DISPATCH_GLOBAL_OBJECT_HEADER(data),
+ .do_next = DISPATCH_OBJECT_LISTLESS,
+#endif
+};
+
DISPATCH_ALWAYS_INLINE
static inline dispatch_data_t
_dispatch_data_alloc(size_t n, size_t extra)
{
dispatch_data_t data;
size_t size;
- size_t base_size;
- if (os_add_overflow(sizeof(struct dispatch_data_s), extra, &base_size)) {
- return DISPATCH_OUT_OF_MEMORY;
- }
- if (os_mul_and_add_overflow(n, sizeof(range_record), base_size, &size)) {
+ if (os_mul_and_add_overflow(n, sizeof(range_record),
+ sizeof(struct dispatch_data_s) + extra, &size)) {
return DISPATCH_OUT_OF_MEMORY;
}
- data = _dispatch_object_alloc(DISPATCH_DATA_CLASS, size);
+ data = _dispatch_alloc(DISPATCH_DATA_CLASS, size);
data->num_records = n;
#if !DISPATCH_DATA_IS_BRIDGED_TO_NSDATA
data->do_targetq = dispatch_get_global_queue(
@@ -163,8 +192,8 @@
}
void
-_dispatch_data_init_with_bytes(dispatch_data_t data, const void *buffer,
- size_t size, dispatch_block_t destructor)
+dispatch_data_init(dispatch_data_t data, const void *buffer, size_t size,
+ dispatch_block_t destructor)
{
if (!buffer || !size) {
if (destructor) {
@@ -255,7 +284,7 @@
}
void
-_dispatch_data_dispose(dispatch_data_t dd, DISPATCH_UNUSED bool *allow_free)
+_dispatch_data_dispose(dispatch_data_t dd)
{
if (_dispatch_data_leaf(dd)) {
_dispatch_data_destroy_buffer(dd->buf, dd->size, dd->do_targetq,
@@ -269,18 +298,6 @@
}
}
-void
-_dispatch_data_set_target_queue(dispatch_data_t dd, dispatch_queue_t tq)
-{
-#if DISPATCH_DATA_IS_BRIDGED_TO_NSDATA
- _dispatch_retain(tq);
- tq = os_atomic_xchg2o(dd, do_targetq, tq, release);
- if (tq) _dispatch_release(tq);
-#else
- _dispatch_object_set_target_queue_inline(dd, tq);
-#endif
-}
-
size_t
_dispatch_data_debug(dispatch_data_t dd, char* buf, size_t bufsiz)
{
diff --git a/src/data.m b/src/data.m
index 1d024ff..9971f18 100644
--- a/src/data.m
+++ b/src/data.m
@@ -28,8 +28,6 @@
#include <Foundation/NSString.h>
-// NOTE: this file must not contain any atomic operations
-
@interface DISPATCH_CLASS(data) () <DISPATCH_CLASS(data)>
@property (readonly,nonatomic) NSUInteger length;
@property (readonly,nonatomic) const void *bytes NS_RETURNS_INNER_POINTER;
@@ -68,26 +66,29 @@
} else {
destructor = DISPATCH_DATA_DESTRUCTOR_NONE;
}
- _dispatch_data_init_with_bytes(self, bytes, length, destructor);
+ dispatch_data_init(self, bytes, length, destructor);
return self;
}
+#define _dispatch_data_objc_dispose(selector) \
+ struct dispatch_data_s *dd = (void*)self; \
+ _dispatch_data_dispose(self); \
+ dispatch_queue_t tq = dd->do_targetq; \
+ dispatch_function_t func = dd->finalizer; \
+ void *ctxt = dd->ctxt; \
+ [super selector]; \
+ if (func && ctxt) { \
+ if (!tq) { \
+ tq = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT,0);\
+ } \
+ dispatch_async_f(tq, ctxt, func); \
+ } \
+ if (tq) { \
+ _os_object_release_internal((_os_object_t)tq); \
+ }
+
- (void)dealloc {
- struct dispatch_data_s *dd = (void*)self;
- _dispatch_data_dispose(self, NULL);
- dispatch_queue_t tq = dd->do_targetq;
- dispatch_function_t func = dd->finalizer;
- void *ctxt = dd->ctxt;
- [super dealloc];
- if (func && ctxt) {
- if (!tq) {
- tq = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT,0);
- }
- dispatch_async_f(tq, ctxt, func);
- }
- if (tq) {
- _os_object_release_internal((_os_object_t)tq);
- }
+ _dispatch_data_objc_dispose(dealloc);
}
- (BOOL)_bytesAreVM {
@@ -112,7 +113,10 @@
- (void)_setTargetQueue:(dispatch_queue_t)queue {
struct dispatch_data_s *dd = (void*)self;
- return _dispatch_data_set_target_queue(dd, queue);
+ _os_object_retain_internal((_os_object_t)queue);
+ dispatch_queue_t prev;
+ prev = os_atomic_xchg2o(dd, do_targetq, queue, release);
+ if (prev) _os_object_release_internal((_os_object_t)prev);
}
- (NSString *)debugDescription {
diff --git a/src/data_internal.h b/src/data_internal.h
index 19fc3d9..bbef21e 100644
--- a/src/data_internal.h
+++ b/src/data_internal.h
@@ -100,13 +100,12 @@
dispatch_transform_t encode;
};
-void _dispatch_data_init_with_bytes(dispatch_data_t data, const void *buffer,
- size_t size, dispatch_block_t destructor);
-void _dispatch_data_dispose(dispatch_data_t data, bool *allow_free);
-void _dispatch_data_set_target_queue(struct dispatch_data_s *dd,
- dispatch_queue_t tq);
+void dispatch_data_init(dispatch_data_t data, const void *buffer, size_t size,
+ dispatch_block_t destructor);
+void _dispatch_data_dispose(dispatch_data_t data);
size_t _dispatch_data_debug(dispatch_data_t data, char* buf, size_t bufsiz);
-const void* _dispatch_data_get_flattened_bytes(struct dispatch_data_s *dd);
+const void*
+_dispatch_data_get_flattened_bytes(struct dispatch_data_s *dd);
#if !defined(__cplusplus)
extern const dispatch_block_t _dispatch_data_destructor_inline;
diff --git a/src/event/event.c b/src/event/event.c
index 34abbf0..2a8a8c3 100644
--- a/src/event/event.c
+++ b/src/event/event.c
@@ -46,7 +46,6 @@
du = _dispatch_unote_linkage_get_unote(dul)._du;
}
du->du_type = dst;
- du->du_can_be_wlh = dst->dst_per_trigger_qos;
du->du_ident = (uint32_t)handle;
du->du_filter = dst->dst_filter;
du->du_fflags = (typeof(du->du_fflags))mask;
@@ -109,13 +108,8 @@
}
#endif
if (du._du->du_is_timer) {
- if (unlikely(du._dt->dt_heap_entry[DTH_TARGET_ID] != DTH_INVALID_ID ||
- du._dt->dt_heap_entry[DTH_DEADLINE_ID] != DTH_INVALID_ID)) {
- DISPATCH_INTERNAL_CRASH(0, "Disposing of timer still in its heap");
- }
- if (unlikely(du._dt->dt_pending_config)) {
+ if (du._dt->dt_pending_config) {
free(du._dt->dt_pending_config);
- du._dt->dt_pending_config = NULL;
}
} else if (!du._du->du_is_direct) {
ptr = _dispatch_unote_get_linkage(du);
@@ -286,8 +280,6 @@
du._dt->dt_timer.target = UINT64_MAX;
du._dt->dt_timer.deadline = UINT64_MAX;
du._dt->dt_timer.interval = UINT64_MAX;
- du._dt->dt_heap_entry[DTH_TARGET_ID] = DTH_INVALID_ID;
- du._dt->dt_heap_entry[DTH_DEADLINE_ID] = DTH_INVALID_ID;
}
return du;
}
diff --git a/src/event/event_config.h b/src/event/event_config.h
index 2ac3c42..7f7761c 100644
--- a/src/event/event_config.h
+++ b/src/event/event_config.h
@@ -35,17 +35,12 @@
#if DISPATCH_DEBUG
#define DISPATCH_MGR_QUEUE_DEBUG 1
-#define DISPATCH_WLH_DEBUG 1
#endif
#ifndef DISPATCH_MGR_QUEUE_DEBUG
#define DISPATCH_MGR_QUEUE_DEBUG 0
#endif
-#ifndef DISPATCH_WLH_DEBUG
-#define DISPATCH_WLH_DEBUG 0
-#endif
-
#ifndef DISPATCH_MACHPORT_DEBUG
#define DISPATCH_MACHPORT_DEBUG 0
#endif
@@ -105,6 +100,31 @@
# ifndef VQ_DESIRED_DISK
# undef HAVE_DECL_VQ_DESIRED_DISK
# endif // VQ_DESIRED_DISK
+
+# ifndef NOTE_MEMORYSTATUS_LOW_SWAP
+# define NOTE_MEMORYSTATUS_LOW_SWAP 0x8
+# endif
+
+# if !defined(NOTE_MEMORYSTATUS_PROC_LIMIT_WARN) || \
+ !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+# undef NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
+# define NOTE_MEMORYSTATUS_PROC_LIMIT_WARN 0
+# endif // NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
+
+# if !defined(NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL) || \
+ !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+# undef NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
+# define NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL 0
+# endif // NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
+
+# ifndef DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS
+# if TARGET_OS_MAC && !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+ // deferred delete can return bogus ENOENTs on older kernels
+# define DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS 1
+# else
+# define DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS 0
+# endif
+# endif
#else // DISPATCH_EVENT_BACKEND_KEVENT
# define EV_ADD 0x0001
# define EV_DELETE 0x0002
@@ -175,14 +195,6 @@
# define MACH_MSG_PRIORITY_UNSPECIFIED ((mach_msg_priority_t)0)
# endif // MACH_SEND_OVERRIDE
-# ifndef MACH_SEND_SYNC_OVERRIDE
-# define MACH_SEND_SYNC_OVERRIDE 0x00100000
-# endif // MACH_SEND_SYNC_OVERRIDE
-
-# ifndef MACH_RCV_SYNC_WAIT
-# define MACH_RCV_SYNC_WAIT 0x00004000
-# endif // MACH_RCV_SYNC_WAIT
-
# define DISPATCH_MACH_TRAILER_SIZE sizeof(dispatch_mach_trailer_t)
# define DISPATCH_MACH_RCV_TRAILER MACH_RCV_TRAILER_CTX
# define DISPATCH_MACH_RCV_OPTIONS ( \
diff --git a/src/event/event_epoll.c b/src/event/event_epoll.c
index 68140d5..647552f 100644
--- a/src/event/event_epoll.c
+++ b/src/event/event_epoll.c
@@ -211,8 +211,8 @@
}
bool
-_dispatch_unote_register(dispatch_unote_t du,
- DISPATCH_UNUSED dispatch_wlh_t wlh, dispatch_priority_t pri)
+_dispatch_unote_register(dispatch_unote_t du, dispatch_wlh_t wlh,
+ dispatch_priority_t pri)
{
struct dispatch_muxnote_bucket_s *dmb;
dispatch_muxnote_t dmn;
@@ -225,7 +225,7 @@
case DISPATCH_EVFILT_CUSTOM_ADD:
case DISPATCH_EVFILT_CUSTOM_OR:
case DISPATCH_EVFILT_CUSTOM_REPLACE:
- du._du->du_wlh = DISPATCH_WLH_ANON;
+ du._du->du_wlh = wlh;
return true;
case EVFILT_WRITE:
events |= EPOLLOUT;
@@ -268,8 +268,7 @@
TAILQ_INSERT_TAIL(&dmn->dmn_readers_head, dul, du_link);
}
dul->du_muxnote = dmn;
- dispatch_assert(du._du->du_wlh == NULL);
- du._du->du_wlh = DISPATCH_WLH_ANON;
+ du._du->du_wlh = DISPATCH_WLH_GLOBAL;
}
return dmn != NULL;
}
@@ -322,7 +321,6 @@
TAILQ_REMOVE(_dispatch_unote_muxnote_bucket(du), dmn, dmn_list);
_dispatch_muxnote_dispose(dmn);
}
- dispatch_assert(du._du->du_wlh == DISPATCH_WLH_ANON);
du._du->du_wlh = NULL;
}
return true;
@@ -420,6 +418,11 @@
{
}
+void
+_dispatch_event_loop_init(void)
+{
+}
+
static void
_dispatch_epoll_init(void *context DISPATCH_UNUSED)
{
@@ -456,7 +459,7 @@
void
_dispatch_event_loop_poke(dispatch_wlh_t wlh DISPATCH_UNUSED,
- uint64_t dq_state DISPATCH_UNUSED, uint32_t flags DISPATCH_UNUSED)
+ dispatch_priority_t pri DISPATCH_UNUSED, uint32_t flags DISPATCH_UNUSED)
{
dispatch_once_f(&epoll_init_pred, NULL, _dispatch_epoll_init);
dispatch_assume_zero(eventfd_write(_dispatch_eventfd, 1));
@@ -578,40 +581,4 @@
}
}
-void
-_dispatch_event_loop_wake_owner(dispatch_sync_context_t dsc,
- dispatch_wlh_t wlh, uint64_t old_state, uint64_t new_state)
-{
- (void)dsc; (void)wlh; (void)old_state; (void)new_state;
-}
-
-void
-_dispatch_event_loop_wait_for_ownership(dispatch_sync_context_t dsc)
-{
- if (dsc->dsc_release_storage) {
- _dispatch_queue_release_storage(dsc->dc_data);
- }
-}
-
-void
-_dispatch_event_loop_end_ownership(dispatch_wlh_t wlh, uint64_t old_state,
- uint64_t new_state, uint32_t flags)
-{
- (void)wlh; (void)old_state; (void)new_state; (void)flags;
-}
-
-#if DISPATCH_WLH_DEBUG
-void
-_dispatch_event_loop_assert_not_owned(dispatch_wlh_t wlh)
-{
- (void)wlh;
-}
-#endif
-
-void
-_dispatch_event_loop_leave_immediate(dispatch_wlh_t wlh, uint64_t dq_state)
-{
- (void)wlh; (void)dq_state;
-}
-
#endif // DISPATCH_EVENT_BACKEND_EPOLL
diff --git a/src/event/event_internal.h b/src/event/event_internal.h
index 842c4ee..c84b353 100644
--- a/src/event/event_internal.h
+++ b/src/event/event_internal.h
@@ -29,10 +29,9 @@
#include "event_config.h"
-struct dispatch_sync_context_s;
typedef struct dispatch_wlh_s *dispatch_wlh_t; // opaque handle
-#define DISPATCH_WLH_ANON ((dispatch_wlh_t)(void*)(~0ul))
-#define DISPATCH_WLH_MANAGER ((dispatch_wlh_t)(void*)(~2ul))
+#define DISPATCH_WLH_GLOBAL ((dispatch_wlh_t)(void*)(~0ul))
+#define DISPATCH_WLH_MANAGER ((dispatch_wlh_t)(void*)(~2ul))
#define DISPATCH_UNOTE_DATA_ACTION_SIZE 2
@@ -41,17 +40,15 @@
uintptr_t du_owner_wref; /* "weak" back reference to the owner object */ \
dispatch_wlh_t du_wlh; \
uint32_t du_ident; \
- int8_t du_filter; \
+ int16_t du_filter; \
+ uint8_t du_data_action : DISPATCH_UNOTE_DATA_ACTION_SIZE; \
+ uint8_t du_is_direct : 1; \
+ uint8_t du_is_timer : 1; \
+ uint8_t du_memorypressure_override : 1; \
+ uint8_t du_vmpressure_override : 1; \
+ uint8_t dmr_async_reply : 1; \
+ uint8_t dmrr_handler_is_block : 1; \
os_atomic(bool) dmsr_notification_armed; \
- uint16_t du_data_action : DISPATCH_UNOTE_DATA_ACTION_SIZE; \
- uint16_t du_is_direct : 1; \
- uint16_t du_is_timer : 1; \
- uint16_t du_memorypressure_override : 1; \
- uint16_t du_vmpressure_override : 1; \
- uint16_t du_can_be_wlh : 1; \
- uint16_t dmr_async_reply : 1; \
- uint16_t dmrr_handler_is_block : 1; \
- uint16_t du_unused : 7; \
uint32_t du_fflags; \
dispatch_priority_t du_priority
@@ -96,7 +93,6 @@
uint64_t delay, leeway;
} dispatch_timer_delay_s;
-#define DTH_INVALID_ID (~0u)
#define DTH_TARGET_ID 0u
#define DTH_DEADLINE_ID 1u
#define DTH_ID_COUNT 2u
@@ -227,11 +223,11 @@
#define DU_UNREGISTER_ALREADY_DELETED 0x02
#define DU_UNREGISTER_DISCONNECTED 0x04
#define DU_UNREGISTER_REPLY_REMOVE 0x08
+#define DU_UNREGISTER_WAKEUP 0x10
typedef struct dispatch_source_type_s {
const char *dst_kind;
- int8_t dst_filter;
- uint8_t dst_per_trigger_qos : 1;
+ int16_t dst_filter;
uint16_t dst_flags;
uint32_t dst_fflags;
uint32_t dst_mask;
@@ -260,10 +256,14 @@
extern const dispatch_source_type_s _dispatch_source_type_after;
#if HAVE_MACH
+extern const dispatch_source_type_s _dispatch_source_type_mach_recv_pset;
extern const dispatch_source_type_s _dispatch_source_type_mach_recv_direct;
+extern const dispatch_source_type_s _dispatch_source_type_mach_recv_direct_pset;
extern const dispatch_source_type_s _dispatch_mach_type_send;
extern const dispatch_source_type_s _dispatch_mach_type_recv;
+extern const dispatch_source_type_s _dispatch_mach_type_recv_pset;
extern const dispatch_source_type_s _dispatch_mach_type_reply;
+extern const dispatch_source_type_s _dispatch_mach_type_reply_pset;
extern const dispatch_source_type_s _dispatch_xpc_type_sigterm;
#endif
@@ -282,17 +282,13 @@
#define DISPATCH_DEFERRED_ITEMS_EVENT_COUNT 16
typedef struct dispatch_deferred_items_s {
+#define DISPATCH_PRIORITY_NOSTASH ((dispatch_priority_t)~0u)
+ dispatch_priority_t ddi_stashed_pri;
dispatch_queue_t ddi_stashed_rq;
- dispatch_object_t ddi_stashed_dou;
- dispatch_qos_t ddi_stashed_qos;
+ dispatch_queue_t ddi_stashed_dq;
#if DISPATCH_EVENT_BACKEND_KEVENT
- dispatch_kevent_t ddi_eventlist;
- uint16_t ddi_nevents;
- uint16_t ddi_maxevents;
- bool ddi_can_stash;
- uint16_t ddi_wlh_needs_delete : 1;
- uint16_t ddi_wlh_needs_update : 1;
- uint16_t ddi_wlh_servicing : 1;
+ int ddi_nevents;
+ dispatch_kevent_s ddi_eventlist[DISPATCH_DEFERRED_ITEMS_EVENT_COUNT];
#endif
} dispatch_deferred_items_s, *dispatch_deferred_items_t;
@@ -337,6 +333,32 @@
_dispatch_thread_setspecific(dispatch_r2k_key, (void *)0);
}
+DISPATCH_ALWAYS_INLINE DISPATCH_PURE
+static inline dispatch_wlh_t
+_dispatch_get_wlh(void)
+{
+ return _dispatch_thread_getspecific(dispatch_wlh_key);
+}
+
+DISPATCH_ALWAYS_INLINE
+static inline void
+_dispatch_set_wlh(dispatch_wlh_t wlh)
+{
+ dispatch_assert(_dispatch_get_wlh() == NULL);
+ dispatch_assert(wlh);
+ _dispatch_debug("wlh[%p]: set current ", wlh);
+ _dispatch_thread_setspecific(dispatch_wlh_key, (void *)wlh);
+}
+
+DISPATCH_ALWAYS_INLINE
+static inline void
+_dispatch_reset_wlh(void)
+{
+ _dispatch_debug("wlh[%p]: clear current ", _dispatch_get_wlh());
+ _dispatch_thread_setspecific(dispatch_wlh_key, NULL);
+ _dispatch_clear_return_to_kernel();
+}
+
DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_unote_registered(dispatch_unote_t du)
@@ -345,14 +367,6 @@
}
DISPATCH_ALWAYS_INLINE
-static inline bool
-_dispatch_unote_wlh_changed(dispatch_unote_t du, dispatch_wlh_t expected_wlh)
-{
- dispatch_wlh_t wlh = du._du->du_wlh;
- return wlh && wlh != DISPATCH_WLH_ANON && wlh != expected_wlh;
-}
-
-DISPATCH_ALWAYS_INLINE
static inline dispatch_unote_linkage_t
_dispatch_unote_get_linkage(dispatch_unote_t du)
{
@@ -419,29 +433,14 @@
void _dispatch_unote_dispose(dispatch_unote_t du);
void _dispatch_event_loop_atfork_child(void);
-#define DISPATCH_EVENT_LOOP_CONSUME_2 DISPATCH_WAKEUP_CONSUME_2
-#define DISPATCH_EVENT_LOOP_OVERRIDE 0x80000000
-void _dispatch_event_loop_poke(dispatch_wlh_t wlh, uint64_t dq_state,
+void _dispatch_event_loop_init(void);
+void _dispatch_event_loop_poke(dispatch_wlh_t wlh, dispatch_priority_t pri,
uint32_t flags);
-void _dispatch_event_loop_wake_owner(struct dispatch_sync_context_s *dsc,
- dispatch_wlh_t wlh, uint64_t old_state, uint64_t new_state);
-void _dispatch_event_loop_wait_for_ownership(
- struct dispatch_sync_context_s *dsc);
-void _dispatch_event_loop_end_ownership(dispatch_wlh_t wlh,
- uint64_t old_state, uint64_t new_state, uint32_t flags);
-#if DISPATCH_WLH_DEBUG
-void _dispatch_event_loop_assert_not_owned(dispatch_wlh_t wlh);
-#else
-#undef _dispatch_event_loop_assert_not_owned
-#define _dispatch_event_loop_assert_not_owned(wlh) ((void)wlh)
-#endif
-void _dispatch_event_loop_leave_immediate(dispatch_wlh_t wlh, uint64_t dq_state);
+void _dispatch_event_loop_drain(uint32_t flags);
#if DISPATCH_EVENT_BACKEND_KEVENT
-void _dispatch_event_loop_leave_deferred(dispatch_wlh_t wlh,
- uint64_t dq_state);
+void _dispatch_event_loop_update(void);
void _dispatch_event_loop_merge(dispatch_kevent_t events, int nevents);
#endif
-void _dispatch_event_loop_drain(uint32_t flags);
void _dispatch_event_loop_timer_arm(unsigned int tidx,
dispatch_timer_delay_s range, dispatch_clock_now_cache_t nows);
void _dispatch_event_loop_timer_delete(unsigned int tidx);
diff --git a/src/event/event_kevent.c b/src/event/event_kevent.c
index c15a397..3275888 100644
--- a/src/event/event_kevent.c
+++ b/src/event/event_kevent.c
@@ -30,7 +30,6 @@
#endif
#define DISPATCH_KEVENT_MUXED_MARKER 1ul
-#define DISPATCH_MACH_AUDIT_TOKEN_PID (5)
typedef struct dispatch_muxnote_s {
TAILQ_ENTRY(dispatch_muxnote_s) dmn_list;
@@ -39,7 +38,6 @@
dispatch_kevent_s dmn_kev;
} *dispatch_muxnote_t;
-static bool _dispatch_timers_force_max_leeway;
static int _dispatch_kq = -1;
static struct {
dispatch_once_t pred;
@@ -79,6 +77,7 @@
};
static void _dispatch_kevent_timer_drain(dispatch_kevent_t ke);
+static void _dispatch_kevent_poke_drain(dispatch_kevent_t ke);
#pragma mark -
#pragma mark kevent debug
@@ -221,12 +220,7 @@
#define _dispatch_kevent_mgr_debug(verb, kev) _dispatch_kevent_debug(verb, kev)
#else
#define _dispatch_kevent_mgr_debug(verb, kev) ((void)verb, (void)kev)
-#endif // DISPATCH_MGR_QUEUE_DEBUG
-#if DISPATCH_WLH_DEBUG
-#define _dispatch_kevent_wlh_debug(verb, kev) _dispatch_kevent_debug(verb, kev)
-#else
-#define _dispatch_kevent_wlh_debug(verb, kev) ((void)verb, (void)kev)
-#endif // DISPATCH_WLH_DEBUG
+#endif
#if DISPATCH_MACHPORT_DEBUG
#ifndef MACH_PORT_TYPE_SPREQUEST
@@ -311,6 +305,9 @@
return (mach_msg_size_t)ke->ext[1];
}
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+static void _dispatch_mach_kevent_portset_drain(dispatch_kevent_t ke);
+#endif
static void _dispatch_kevent_mach_msg_drain(dispatch_kevent_t ke);
static inline void _dispatch_mach_host_calendar_change_register(void);
@@ -345,14 +342,20 @@
static void
_dispatch_kevent_print_error(dispatch_kevent_t ke)
{
- _dispatch_debug("kevent[0x%llx]: handling error",
- (unsigned long long)ke->udata);
+ dispatch_kevent_t kev = NULL;
+
if (ke->flags & EV_DELETE) {
if (ke->flags & EV_UDATA_SPECIFIC) {
if (ke->data == EINPROGRESS) {
// deferred EV_DELETE
return;
}
+#if DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS
+ if (ke->data == ENOENT) {
+ // deferred EV_DELETE
+ return;
+ }
+#endif
}
// for EV_DELETE if the update was deferred we may have reclaimed
// the udata already, and it is unsafe to dereference it now.
@@ -366,7 +369,8 @@
#if HAVE_MACH
if (ke->filter == EVFILT_MACHPORT && ke->data == ENOTSUP &&
- (ke->flags & EV_ADD) && (ke->fflags & MACH_RCV_MSG)) {
+ (ke->flags & EV_ADD) && _dispatch_evfilt_machport_direct_enabled &&
+ kev && (kev->fflags & MACH_RCV_MSG)) {
DISPATCH_INTERNAL_CRASH(ke->ident,
"Missing EVFILT_MACHPORT support for ports");
}
@@ -434,7 +438,7 @@
{
if (ke->filter == EVFILT_USER) {
_dispatch_kevent_mgr_debug("received", ke);
- return;
+ return _dispatch_kevent_poke_drain(ke);
}
_dispatch_kevent_debug("received", ke);
if (unlikely(ke->flags & EV_ERROR)) {
@@ -448,6 +452,8 @@
ke->data = 0;
_dispatch_kevent_debug("synthetic NOTE_EXIT", ke);
} else {
+ _dispatch_debug("kevent[0x%llx]: handling error",
+ (unsigned long long)ke->udata);
return _dispatch_kevent_print_error(ke);
}
}
@@ -457,6 +463,11 @@
#if HAVE_MACH
if (ke->filter == EVFILT_MACHPORT) {
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+ if (ke->udata == 0) {
+ return _dispatch_mach_kevent_portset_drain(ke);
+ }
+#endif
if (_dispatch_kevent_mach_msg_size(ke)) {
return _dispatch_kevent_mach_msg_drain(ke);
}
@@ -522,30 +533,31 @@
#endif
static void
-_dispatch_kq_init(void *context)
+_dispatch_kq_init(void *context DISPATCH_UNUSED)
{
- bool *kq_initialized = context;
-
_dispatch_fork_becomes_unsafe();
- if (unlikely(getenv("LIBDISPATCH_TIMERS_FORCE_MAX_LEEWAY"))) {
- _dispatch_timers_force_max_leeway = true;
- }
- *kq_initialized = true;
-
#if DISPATCH_USE_KEVENT_WORKQUEUE
_dispatch_kevent_workqueue_init();
if (_dispatch_kevent_workqueue_enabled) {
int r;
int kqfd = _dispatch_kq;
- const dispatch_kevent_s ke = {
- .ident = 1,
- .filter = EVFILT_USER,
- .flags = EV_ADD|EV_CLEAR,
- .qos = _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG,
- .udata = (uintptr_t)DISPATCH_WLH_MANAGER,
+ const dispatch_kevent_s kev[] = {
+ [0] = {
+ .ident = 1,
+ .filter = EVFILT_USER,
+ .flags = EV_ADD|EV_CLEAR,
+ .qos = _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG,
+ .udata = (uintptr_t)DISPATCH_WLH_MANAGER,
+ },
+ [1] = {
+ .ident = 1,
+ .filter = EVFILT_USER,
+ .fflags = NOTE_TRIGGER,
+ .udata = (uintptr_t)DISPATCH_WLH_MANAGER,
+ },
};
retry:
- r = kevent_qos(kqfd, &ke, 1, NULL, 0, NULL, NULL,
+ r = kevent_qos(kqfd, kev, 2, NULL, 0, NULL, NULL,
KEVENT_FLAG_WORKQ|KEVENT_FLAG_IMMEDIATE);
if (unlikely(r == -1)) {
int err = errno;
@@ -567,116 +579,88 @@
#endif // DISPATCH_USE_MGR_THREAD
}
-#if DISPATCH_USE_MEMORYPRESSURE_SOURCE
-static void _dispatch_memorypressure_init(void);
-#else
-#define _dispatch_memorypressure_init() ((void)0)
-#endif
-
DISPATCH_NOINLINE
static int
-_dispatch_kq_poll(dispatch_wlh_t wlh, dispatch_kevent_t ke, int n,
- dispatch_kevent_t ke_out, int n_out, void *buf, size_t *avail,
+_dispatch_kq_update(dispatch_wlh_t wlh, dispatch_kevent_t ke, int n,
uint32_t flags)
{
static dispatch_once_t pred;
- bool kq_initialized = false;
- int r = 0;
+ dispatch_once_f(&pred, NULL, _dispatch_kq_init);
- dispatch_once_f(&pred, &kq_initialized, _dispatch_kq_init);
- if (unlikely(kq_initialized)) {
- // The calling thread was the one doing the initialization
- //
- // The event loop needs the memory pressure source and debug channel,
- // however creating these will recursively call _dispatch_kq_poll(),
- // so we can't quite initialize them under the dispatch once.
- _dispatch_memorypressure_init();
- _voucher_activity_debug_channel_init();
- }
+ dispatch_kevent_s ke_out[DISPATCH_DEFERRED_ITEMS_EVENT_COUNT];
+ int i, out_n = countof(ke_out), r = 0;
+#if DISPATCH_USE_KEVENT_QOS
+ size_t size, *avail = NULL;
+ void *buf = NULL;
+#endif
-
-#if !DISPATCH_USE_KEVENT_QOS
- if (flags & KEVENT_FLAG_ERROR_EVENTS) {
- // emulate KEVENT_FLAG_ERROR_EVENTS
- for (r = 0; r < n; r++) {
- ke[r].flags |= EV_RECEIPT;
+#if DISPATCH_DEBUG
+ dispatch_assert(wlh);
+ dispatch_assert((size_t)n <= countof(ke_out));
+ for (i = 0; i < n; i++) {
+ if (ke[i].filter != EVFILT_USER || DISPATCH_MGR_QUEUE_DEBUG) {
+ _dispatch_kevent_debug_n(NULL, ke + i, i, n);
}
- out_n = n;
}
#endif
+ wlh = DISPATCH_WLH_GLOBAL;
+
+ if (flags & KEVENT_FLAG_ERROR_EVENTS) {
+#if !DISPATCH_USE_KEVENT_QOS
+ // emulate KEVENT_FLAG_ERROR_EVENTS
+ for (i = 0; i < n; i++) {
+ ke[i].flags |= EV_RECEIPT;
+ }
+ out_n = n;
+#endif
+ } else {
+#if DISPATCH_USE_KEVENT_QOS
+ size = DISPATCH_MACH_RECEIVE_MAX_INLINE_MESSAGE_SIZE +
+ DISPATCH_MACH_TRAILER_SIZE;
+ buf = alloca(size);
+ avail = &size;
+#endif
+ }
+
retry:
- if (wlh == DISPATCH_WLH_ANON) {
+ _dispatch_clear_return_to_kernel();
+ if (wlh == DISPATCH_WLH_GLOBAL) {
int kqfd = _dispatch_kq;
#if DISPATCH_USE_KEVENT_QOS
if (_dispatch_kevent_workqueue_enabled) {
flags |= KEVENT_FLAG_WORKQ;
}
- r = kevent_qos(kqfd, ke, n, ke_out, n_out, buf, avail, flags);
+ r = kevent_qos(kqfd, ke, n, ke_out, out_n, buf, avail, flags);
#else
const struct timespec timeout_immediately = {}, *timeout = NULL;
if (flags & KEVENT_FLAG_IMMEDIATE) timeout = &timeout_immediately;
- r = kevent(kqfd, ke, n, ke_out, n_out, timeout);
+ r = kevent(kqfd, ke, n, ke_out, out_n, timeout);
#endif
}
if (unlikely(r == -1)) {
int err = errno;
switch (err) {
- case ENOMEM:
- _dispatch_temporary_resource_shortage();
- /* FALLTHROUGH */
case EINTR:
goto retry;
case EBADF:
DISPATCH_CLIENT_CRASH(err, "Do not close random Unix descriptors");
+ break;
default:
- DISPATCH_CLIENT_CRASH(err, "Unexpected error from kevent");
+ (void)dispatch_assume_zero(err);
+ break;
}
+ return err;
}
- return r;
-}
-DISPATCH_NOINLINE
-static int
-_dispatch_kq_drain(dispatch_wlh_t wlh, dispatch_kevent_t ke, int n,
- uint32_t flags)
-{
- dispatch_kevent_s ke_out[DISPATCH_DEFERRED_ITEMS_EVENT_COUNT];
- bool poll_for_events = !(flags & KEVENT_FLAG_ERROR_EVENTS);
- int i, n_out = countof(ke_out), r = 0;
- size_t *avail = NULL;
- void *buf = NULL;
-
-#if DISPATCH_USE_KEVENT_QOS
- size_t size;
- if (poll_for_events) {
- size = DISPATCH_MACH_RECEIVE_MAX_INLINE_MESSAGE_SIZE +
- DISPATCH_MACH_TRAILER_SIZE;
- buf = alloca(size);
- avail = &size;
- }
-#endif
-
-#if DISPATCH_DEBUG
- for (r = 0; r < n; r++) {
- if (ke[r].filter != EVFILT_USER || DISPATCH_MGR_QUEUE_DEBUG) {
- _dispatch_kevent_debug_n(NULL, ke + r, r, n);
- }
- }
-#endif
-
- if (poll_for_events) _dispatch_clear_return_to_kernel();
- n = _dispatch_kq_poll(wlh, ke, n, ke_out, n_out, buf, avail, flags);
- if (n == 0) {
- r = 0;
- } else if (flags & KEVENT_FLAG_ERROR_EVENTS) {
- for (i = 0, r = 0; i < n; i++) {
+ if (flags & KEVENT_FLAG_ERROR_EVENTS) {
+ for (i = 0, n = r, r = 0; i < n; i++) {
if ((ke_out[i].flags & EV_ERROR) && (r = (int)ke_out[i].data)) {
_dispatch_kevent_drain(&ke_out[i]);
}
}
} else {
- for (i = 0, r = 0; i < n; i++) {
+ for (i = 0, n = r, r = 0; i < n; i++) {
_dispatch_kevent_drain(&ke_out[i]);
}
}
@@ -687,7 +671,7 @@
static inline int
_dispatch_kq_update_one(dispatch_wlh_t wlh, dispatch_kevent_t ke)
{
- return _dispatch_kq_drain(wlh, ke, 1,
+ return _dispatch_kq_update(wlh, ke, 1,
KEVENT_FLAG_IMMEDIATE | KEVENT_FLAG_ERROR_EVENTS);
}
@@ -695,7 +679,7 @@
static inline void
_dispatch_kq_update_all(dispatch_wlh_t wlh, dispatch_kevent_t ke, int n)
{
- (void)_dispatch_kq_drain(wlh, ke, n,
+ (void)_dispatch_kq_update(wlh, ke, n,
KEVENT_FLAG_IMMEDIATE | KEVENT_FLAG_ERROR_EVENTS);
}
@@ -747,8 +731,8 @@
_dispatch_kq_deferred_reuse_slot(dispatch_wlh_t wlh,
dispatch_deferred_items_t ddi, int slot)
{
- if (wlh != DISPATCH_WLH_ANON) _dispatch_set_return_to_kernel();
- if (unlikely(slot == ddi->ddi_maxevents)) {
+ if (wlh != DISPATCH_WLH_GLOBAL) _dispatch_set_return_to_kernel();
+ if (unlikely(slot == countof(ddi->ddi_eventlist))) {
int nevents = ddi->ddi_nevents;
ddi->ddi_nevents = 1;
_dispatch_kq_update_all(wlh, ddi->ddi_eventlist, nevents);
@@ -778,13 +762,13 @@
{
dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
- if (ddi && ddi->ddi_maxevents && wlh == _dispatch_get_wlh()) {
+ if (ddi && wlh == _dispatch_get_wlh()) {
int slot = _dispatch_kq_deferred_find_slot(ddi, ke->filter, ke->ident,
ke->udata);
dispatch_kevent_t dk = _dispatch_kq_deferred_reuse_slot(wlh, ddi, slot);
*dk = *ke;
- if (ke->filter != EVFILT_USER) {
- _dispatch_kevent_mgr_debug("deferred", ke);
+ if (ke->filter != EVFILT_USER || DISPATCH_MGR_QUEUE_DEBUG) {
+ _dispatch_kevent_debug("deferred", ke);
}
} else {
_dispatch_kq_update_one(wlh, ke);
@@ -816,9 +800,7 @@
if (action_flags & EV_ADD) {
// as soon as we register we may get an event delivery and it has to
- // see du_wlh already set, else it will not unregister the kevent
- dispatch_assert(du->du_wlh == NULL);
- _dispatch_wlh_retain(wlh);
+ // see this bit already set, else it will not unregister the kevent
du->du_wlh = wlh;
}
@@ -852,7 +834,6 @@
done:
if (action_flags & EV_ADD) {
if (unlikely(r)) {
- _dispatch_wlh_release(du->du_wlh);
du->du_wlh = NULL;
}
return r == 0;
@@ -861,8 +842,11 @@
if (action_flags & EV_DELETE) {
if (r == EINPROGRESS) {
return false;
+#if DISPATCH_KEVENT_TREAT_ENOENT_AS_EINPROGRESS
+ } else if (r == ENOENT) {
+ return false;
+#endif
}
- _dispatch_wlh_release(du->du_wlh);
du->du_wlh = NULL;
}
@@ -929,7 +913,7 @@
{
struct dispatch_muxnote_bucket_s *dmb;
dmb = _dispatch_muxnote_bucket(name, filter);
- return _dispatch_muxnote_find(dmb, DISPATCH_WLH_ANON, name, filter);
+ return _dispatch_muxnote_find(dmb, DISPATCH_WLH_GLOBAL, name, filter);
}
DISPATCH_NOINLINE
@@ -987,7 +971,7 @@
bool armed = DISPATCH_MACH_NOTIFICATION_ARMED(&dmn->dmn_kev);
os_atomic_store2o(du._dmsr, dmsr_notification_armed, armed,relaxed);
}
- du._du->du_wlh = DISPATCH_WLH_ANON;
+ du._du->du_wlh = DISPATCH_WLH_GLOBAL;
}
return installed;
}
@@ -1002,11 +986,11 @@
case DISPATCH_EVFILT_CUSTOM_ADD:
case DISPATCH_EVFILT_CUSTOM_OR:
case DISPATCH_EVFILT_CUSTOM_REPLACE:
- du._du->du_wlh = DISPATCH_WLH_ANON;
+ du._du->du_wlh = wlh;
return true;
}
if (!du._du->du_is_direct) {
- return _dispatch_unote_register_muxed(du, DISPATCH_WLH_ANON);
+ return _dispatch_unote_register_muxed(du, DISPATCH_WLH_GLOBAL);
}
return _dispatch_kq_unote_update(wlh, du, EV_ADD | EV_ENABLE);
}
@@ -1040,7 +1024,6 @@
if (dmn->dmn_kev.filter == DISPATCH_EVFILT_MACH_NOTIFICATION) {
os_atomic_store2o(du._dmsr, dmsr_notification_armed, false, relaxed);
}
- dispatch_assert(du._du->du_wlh == DISPATCH_WLH_ANON);
du._du->du_wlh = NULL;
TAILQ_REMOVE(&dmn->dmn_unotes_head, dul, du_link);
_TAILQ_TRASH_ENTRY(dul, du_link);
@@ -1107,7 +1090,14 @@
}
#pragma mark -
-#pragma mark dispatch_event_loop
+#pragma mark dispatch_loop
+
+#if DISPATCH_USE_MEMORYPRESSURE_SOURCE
+static void _dispatch_memorypressure_init(void);
+#else
+#define _dispatch_memorypressure_init()
+#endif
+static bool _dispatch_timers_force_max_leeway;
void
_dispatch_event_loop_atfork_child(void)
@@ -1118,118 +1108,77 @@
#endif
}
+DISPATCH_NOINLINE
+void
+_dispatch_event_loop_init(void)
+{
+ if (unlikely(getenv("LIBDISPATCH_TIMERS_FORCE_MAX_LEEWAY"))) {
+ _dispatch_timers_force_max_leeway = true;
+ }
+ _dispatch_memorypressure_init();
+ _voucher_activity_debug_channel_init();
+}
DISPATCH_NOINLINE
void
-_dispatch_event_loop_poke(dispatch_wlh_t wlh, uint64_t dq_state, uint32_t flags)
+_dispatch_event_loop_poke(dispatch_wlh_t wlh, dispatch_priority_t pri,
+ uint32_t flags)
{
if (wlh == DISPATCH_WLH_MANAGER) {
- dispatch_kevent_s ke = (dispatch_kevent_s){
+ dispatch_assert(!flags);
+ dispatch_kevent_s ke = {
.ident = 1,
.filter = EVFILT_USER,
.fflags = NOTE_TRIGGER,
.udata = (uintptr_t)DISPATCH_WLH_MANAGER,
};
- return _dispatch_kq_deferred_update(DISPATCH_WLH_ANON, &ke);
- } else if (wlh && wlh != DISPATCH_WLH_ANON) {
- (void)dq_state; (void)flags;
+ return _dispatch_kq_deferred_update(DISPATCH_WLH_GLOBAL, &ke);
+ } else if (wlh && wlh != DISPATCH_WLH_GLOBAL) {
+ dispatch_assert(flags);
+ dispatch_assert(pri);
}
DISPATCH_INTERNAL_CRASH(wlh, "Unsupported wlh configuration");
}
DISPATCH_NOINLINE
+static void
+_dispatch_kevent_poke_drain(dispatch_kevent_t ke)
+{
+ dispatch_assert(ke->filter == EVFILT_USER);
+ dispatch_wlh_t wlh = (dispatch_wlh_t)ke->udata;
+ dispatch_assert(wlh);
+}
+
+DISPATCH_NOINLINE
void
_dispatch_event_loop_drain(uint32_t flags)
{
dispatch_wlh_t wlh = _dispatch_get_wlh();
dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
- int n;
-
-again:
- n = ddi->ddi_nevents;
+ int n = ddi->ddi_nevents;
ddi->ddi_nevents = 0;
- _dispatch_kq_drain(wlh, ddi->ddi_eventlist, n, flags);
-
- if ((flags & KEVENT_FLAG_IMMEDIATE) &&
- !(flags & KEVENT_FLAG_ERROR_EVENTS) &&
- _dispatch_needs_to_return_to_kernel()) {
- goto again;
- }
+ _dispatch_kq_update(wlh, ddi->ddi_eventlist, n, flags);
}
void
-_dispatch_event_loop_merge(dispatch_kevent_t events, int nevents)
+_dispatch_event_loop_update(void)
{
- dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
- dispatch_kevent_s kev[nevents];
-
- // now we can re-use the whole event list, but we need to save one slot
- // for the event loop poke
- memcpy(kev, events, sizeof(kev));
- ddi->ddi_maxevents = DISPATCH_DEFERRED_ITEMS_EVENT_COUNT - 1;
-
- for (int i = 0; i < nevents; i++) {
- _dispatch_kevent_drain(&kev[i]);
- }
-
dispatch_wlh_t wlh = _dispatch_get_wlh();
- if (wlh == DISPATCH_WLH_ANON && ddi->ddi_stashed_dou._do) {
- if (ddi->ddi_nevents) {
- // We will drain the stashed item and not return to the kernel
- // right away. As a consequence, do not delay these updates.
- _dispatch_event_loop_drain(KEVENT_FLAG_IMMEDIATE |
- KEVENT_FLAG_ERROR_EVENTS);
- }
- _dispatch_trace_continuation_push(ddi->ddi_stashed_rq,
- ddi->ddi_stashed_dou);
+ dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
+ int n = ddi->ddi_nevents;
+ ddi->ddi_nevents = 0;
+ _dispatch_kq_update_all(wlh, ddi->ddi_eventlist, n);
+ dispatch_assert(ddi->ddi_nevents == 0);
+}
+
+void
+_dispatch_event_loop_merge(dispatch_kevent_t ke, int n)
+{
+ while (n-- > 0) {
+ _dispatch_kevent_drain(ke++);
}
}
-void
-_dispatch_event_loop_leave_immediate(dispatch_wlh_t wlh, uint64_t dq_state)
-{
- (void)wlh; (void)dq_state;
-}
-
-void
-_dispatch_event_loop_leave_deferred(dispatch_wlh_t wlh, uint64_t dq_state)
-{
- (void)wlh; (void)dq_state;
-}
-
-void
-_dispatch_event_loop_wake_owner(dispatch_sync_context_t dsc,
- dispatch_wlh_t wlh, uint64_t old_state, uint64_t new_state)
-{
- (void)dsc; (void)wlh; (void)old_state; (void)new_state;
-}
-
-void
-_dispatch_event_loop_wait_for_ownership(dispatch_sync_context_t dsc)
-{
- if (dsc->dsc_release_storage) {
- _dispatch_queue_release_storage(dsc->dc_data);
- }
-}
-
-void
-_dispatch_event_loop_end_ownership(dispatch_wlh_t wlh, uint64_t old_state,
- uint64_t new_state, uint32_t flags)
-{
- (void)wlh; (void)old_state; (void)new_state; (void)flags;
-}
-
-#if DISPATCH_WLH_DEBUG
-void
-_dispatch_event_loop_assert_not_owned(dispatch_wlh_t wlh)
-{
- (void)wlh;
-}
-#endif // DISPATCH_WLH_DEBUG
-
-#pragma mark -
-#pragma mark dispatch_event_loop timers
-
#define DISPATCH_KEVENT_TIMEOUT_IDENT_MASK (~0ull << 8)
DISPATCH_NOINLINE
@@ -1270,7 +1219,7 @@
#endif
};
- _dispatch_kq_deferred_update(DISPATCH_WLH_ANON, &ke);
+ _dispatch_kq_deferred_update(DISPATCH_WLH_GLOBAL, &ke);
}
void
@@ -1302,7 +1251,6 @@
_dispatch_event_loop_timer_program(tidx, 0, 0, EV_DELETE);
}
-#pragma mark -
#pragma mark kevent specific sources
static dispatch_unote_t
@@ -1416,16 +1364,12 @@
DISPATCH_MEMORYPRESSURE_WARN | \
DISPATCH_MEMORYPRESSURE_CRITICAL | \
DISPATCH_MEMORYPRESSURE_PROC_LIMIT_WARN | \
- DISPATCH_MEMORYPRESSURE_PROC_LIMIT_CRITICAL | \
- DISPATCH_MEMORYPRESSURE_MSL_STATUS)
-
+ DISPATCH_MEMORYPRESSURE_PROC_LIMIT_CRITICAL)
#define DISPATCH_MEMORYPRESSURE_MALLOC_MASK ( \
DISPATCH_MEMORYPRESSURE_WARN | \
DISPATCH_MEMORYPRESSURE_CRITICAL | \
DISPATCH_MEMORYPRESSURE_PROC_LIMIT_WARN | \
- DISPATCH_MEMORYPRESSURE_PROC_LIMIT_CRITICAL | \
- DISPATCH_MEMORYPRESSURE_MSL_STATUS)
-
+ DISPATCH_MEMORYPRESSURE_PROC_LIMIT_CRITICAL)
static void
_dispatch_memorypressure_handler(void *context)
@@ -1465,7 +1409,8 @@
{
dispatch_source_t ds = dispatch_source_create(
DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, 0,
- DISPATCH_MEMORYPRESSURE_SOURCE_MASK, &_dispatch_mgr_q);
+ DISPATCH_MEMORYPRESSURE_SOURCE_MASK,
+ _dispatch_get_root_queue(DISPATCH_QOS_DEFAULT, true));
dispatch_set_context(ds, ds);
dispatch_source_set_event_handler_f(ds, _dispatch_memorypressure_handler);
dispatch_activate(ds);
@@ -1515,8 +1460,7 @@
.dst_mask = NOTE_MEMORYSTATUS_PRESSURE_NORMAL
|NOTE_MEMORYSTATUS_PRESSURE_WARN|NOTE_MEMORYSTATUS_PRESSURE_CRITICAL
|NOTE_MEMORYSTATUS_LOW_SWAP|NOTE_MEMORYSTATUS_PROC_LIMIT_WARN
- |NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL
- |NOTE_MEMORYSTATUS_MSL_STATUS,
+ |NOTE_MEMORYSTATUS_PROC_LIMIT_CRITICAL,
.dst_size = sizeof(struct dispatch_source_refs_s),
#if TARGET_OS_SIMULATOR
@@ -1579,42 +1523,15 @@
}
}
-static mach_msg_audit_trailer_t *
-_dispatch_mach_msg_get_audit_trailer(mach_msg_header_t *hdr)
-{
- mach_msg_trailer_t *tlr = NULL;
- mach_msg_audit_trailer_t *audit_tlr = NULL;
- tlr = (mach_msg_trailer_t *)((unsigned char *)hdr +
- round_msg(hdr->msgh_size));
- // The trailer should always be of format zero.
- if (tlr->msgh_trailer_type == MACH_MSG_TRAILER_FORMAT_0) {
- if (tlr->msgh_trailer_size >= sizeof(mach_msg_audit_trailer_t)) {
- audit_tlr = (mach_msg_audit_trailer_t *)tlr;
- }
- }
- return audit_tlr;
-}
-
DISPATCH_NOINLINE
static void
_dispatch_mach_notify_source_invoke(mach_msg_header_t *hdr)
{
mig_reply_error_t reply;
- mach_msg_audit_trailer_t *tlr = NULL;
dispatch_assert(sizeof(mig_reply_error_t) == sizeof(union
__ReplyUnion___dispatch_libdispatch_internal_protocol_subsystem));
dispatch_assert(sizeof(mig_reply_error_t) <
DISPATCH_MACH_RECEIVE_MAX_INLINE_MESSAGE_SIZE);
- tlr = _dispatch_mach_msg_get_audit_trailer(hdr);
- if (!tlr) {
- DISPATCH_INTERNAL_CRASH(0, "message received without expected trailer");
- }
- if (tlr->msgh_audit.val[DISPATCH_MACH_AUDIT_TOKEN_PID] != 0) {
- (void)dispatch_assume_zero(
- tlr->msgh_audit.val[DISPATCH_MACH_AUDIT_TOKEN_PID]);
- mach_msg_destroy(hdr);
- return;
- }
boolean_t success = libdispatch_internal_protocol_server(hdr, &reply.Head);
if (!success && reply.RetCode == MIG_BAD_ID &&
(hdr->msgh_id == HOST_CALENDAR_SET_REPLYID ||
@@ -1839,7 +1756,7 @@
_dispatch_debug_machport(name);
dmn = _dispatch_mach_muxnote_find(name, DISPATCH_EVFILT_MACH_NOTIFICATION);
- if (!dmn) {
+ if (!dispatch_assume(dmn)) {
return;
}
@@ -1988,6 +1905,204 @@
#pragma mark mach recv / reply
#if HAVE_MACH
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+static mach_port_t _dispatch_mach_portset, _dispatch_mach_recv_portset;
+static dispatch_kevent_s _dispatch_mach_recv_kevent;
+
+static void
+_dispatch_mach_portset_init(void *context DISPATCH_UNUSED)
+{
+ kern_return_t kr = mach_port_allocate(mach_task_self(),
+ MACH_PORT_RIGHT_PORT_SET, &_dispatch_mach_portset);
+ DISPATCH_VERIFY_MIG(kr);
+ if (unlikely(kr)) {
+ DISPATCH_CLIENT_CRASH(kr,
+ "mach_port_allocate() failed: cannot create port set");
+ }
+
+ dispatch_kevent_s kev = {
+ .filter = EVFILT_MACHPORT,
+ .flags = EV_ADD|EV_ENABLE,
+ .ident = _dispatch_mach_portset,
+ .qos = _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG,
+ };
+ _dispatch_kq_deferred_update(DISPATCH_WLH_GLOBAL, &kev);
+}
+
+static bool
+_dispatch_mach_portset_update(mach_port_t mp, mach_port_t mps)
+{
+ kern_return_t kr;
+
+ _dispatch_debug_machport(mp);
+ kr = mach_port_move_member(mach_task_self(), mp, mps);
+ if (unlikely(kr)) {
+ DISPATCH_VERIFY_MIG(kr);
+ switch (kr) {
+ case KERN_INVALID_RIGHT:
+ if (mps) {
+ _dispatch_bug_mach_client("_dispatch_kevent_machport_enable: "
+ "mach_port_move_member() failed ", kr);
+ break;
+ }
+ //fall through
+ case KERN_INVALID_NAME:
+#if DISPATCH_DEBUG
+ _dispatch_log("Corruption: Mach receive right 0x%x destroyed "
+ "prematurely", mp);
+#endif
+ break;
+ default:
+ (void)dispatch_assume_zero(kr);
+ break;
+ }
+ }
+ if (mps) {
+ return kr == KERN_SUCCESS;
+ }
+ return true;
+}
+
+static mach_port_t
+_dispatch_mach_get_portset(void)
+{
+ static dispatch_once_t pred;
+ dispatch_once_f(&pred, NULL, _dispatch_mach_portset_init);
+ return _dispatch_mach_portset;
+}
+
+static bool
+_dispatch_mach_recv_update_portset_mux(dispatch_muxnote_t dmn)
+{
+ mach_port_t mp = (mach_port_t)dmn->dmn_kev.ident;
+ mach_port_t mps = MACH_PORT_NULL;
+ if (!(dmn->dmn_kev.flags & EV_DELETE)) {
+ mps = _dispatch_mach_get_portset();
+ }
+ return _dispatch_mach_portset_update(mp, mps);
+}
+
+static void
+_dispatch_mach_recv_msg_buf_init(dispatch_kevent_t ke)
+{
+ mach_vm_size_t vm_size = mach_vm_round_page(
+ DISPATCH_MACH_RECEIVE_MAX_INLINE_MESSAGE_SIZE +
+ DISPATCH_MACH_TRAILER_SIZE);
+ mach_vm_address_t vm_addr = vm_page_size;
+ kern_return_t kr;
+
+ while (unlikely(kr = mach_vm_allocate(mach_task_self(), &vm_addr, vm_size,
+ VM_FLAGS_ANYWHERE))) {
+ if (kr != KERN_NO_SPACE) {
+ DISPATCH_CLIENT_CRASH(kr,
+ "Could not allocate mach msg receive buffer");
+ }
+ _dispatch_temporary_resource_shortage();
+ vm_addr = vm_page_size;
+ }
+ ke->ext[0] = (uintptr_t)vm_addr;
+ ke->ext[1] = vm_size;
+}
+
+static void
+_dispatch_mach_recv_portset_init(void *context DISPATCH_UNUSED)
+{
+ kern_return_t kr = mach_port_allocate(mach_task_self(),
+ MACH_PORT_RIGHT_PORT_SET, &_dispatch_mach_recv_portset);
+ DISPATCH_VERIFY_MIG(kr);
+ if (unlikely(kr)) {
+ DISPATCH_CLIENT_CRASH(kr,
+ "mach_port_allocate() failed: cannot create port set");
+ }
+
+ dispatch_assert(DISPATCH_MACH_TRAILER_SIZE ==
+ REQUESTED_TRAILER_SIZE_NATIVE(MACH_RCV_TRAILER_ELEMENTS(
+ DISPATCH_MACH_RCV_TRAILER)));
+
+ _dispatch_mach_recv_kevent = (dispatch_kevent_s){
+ .filter = EVFILT_MACHPORT,
+ .ident = _dispatch_mach_recv_portset,
+ .flags = EV_ADD|EV_ENABLE|EV_DISPATCH,
+ .fflags = DISPATCH_MACH_RCV_OPTIONS,
+ .qos = _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG,
+ };
+ if (!_dispatch_kevent_workqueue_enabled) {
+ _dispatch_mach_recv_msg_buf_init(&_dispatch_mach_recv_kevent);
+ }
+ _dispatch_kq_deferred_update(DISPATCH_WLH_GLOBAL,
+ &_dispatch_mach_recv_kevent);
+}
+
+static mach_port_t
+_dispatch_mach_get_recv_portset(void)
+{
+ static dispatch_once_t pred;
+ dispatch_once_f(&pred, NULL, _dispatch_mach_recv_portset_init);
+ return _dispatch_mach_recv_portset;
+}
+
+static bool
+_dispatch_mach_recv_direct_update_portset_mux(dispatch_muxnote_t dmn)
+{
+ mach_port_t mp = (mach_port_t)dmn->dmn_kev.ident;
+ mach_port_t mps = MACH_PORT_NULL;
+ if (!(dmn->dmn_kev.flags & EV_DELETE)) {
+ mps = _dispatch_mach_get_recv_portset();
+ }
+ return _dispatch_mach_portset_update(mp, mps);
+}
+
+static dispatch_unote_t
+_dispatch_mach_kevent_mach_recv_direct_find(mach_port_t name)
+{
+ dispatch_muxnote_t dmn;
+ dispatch_unote_linkage_t dul;
+
+ dmn = _dispatch_mach_muxnote_find(name, EVFILT_MACHPORT);
+ TAILQ_FOREACH(dul, &dmn->dmn_unotes_head, du_link) {
+ dispatch_unote_t du = _dispatch_unote_linkage_get_unote(dul);
+ if (du._du->du_type->dst_fflags & MACH_RCV_MSG) {
+ return du;
+ }
+ }
+ return DISPATCH_UNOTE_NULL;
+}
+
+DISPATCH_NOINLINE
+static void
+_dispatch_mach_kevent_portset_merge(dispatch_kevent_t ke)
+{
+ mach_port_t name = (mach_port_name_t)ke->data;
+ dispatch_unote_linkage_t dul, dul_next;
+ dispatch_muxnote_t dmn;
+
+ _dispatch_debug_machport(name);
+ dmn = _dispatch_mach_muxnote_find(name, EVFILT_MACHPORT);
+ if (!dispatch_assume(dmn)) {
+ return;
+ }
+ _dispatch_mach_portset_update(name, MACH_PORT_NULL); // emulate EV_DISPATCH
+
+ TAILQ_FOREACH_SAFE(dul, &dmn->dmn_unotes_head, du_link, dul_next) {
+ dispatch_unote_t du = _dispatch_unote_linkage_get_unote(dul);
+ dux_merge_evt(du._du, EV_ENABLE | EV_DISPATCH,
+ DISPATCH_MACH_RECV_MESSAGE, 0, 0);
+ }
+}
+
+DISPATCH_NOINLINE
+static void
+_dispatch_mach_kevent_portset_drain(dispatch_kevent_t ke)
+{
+ if (ke->ident == _dispatch_mach_recv_portset) {
+ return _dispatch_kevent_mach_msg_drain(ke);
+ } else {
+ dispatch_assert(ke->ident == _dispatch_mach_portset);
+ return _dispatch_mach_kevent_portset_merge(ke);
+ }
+}
+#endif // DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+
static void
_dispatch_kevent_mach_msg_recv(dispatch_unote_t du, uint32_t flags,
mach_msg_header_t *hdr)
@@ -2004,6 +2119,11 @@
"received message with MACH_PORT_NULL port");
} else {
_dispatch_debug_machport(name);
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+ if (du._du == NULL) {
+ du = _dispatch_mach_kevent_mach_recv_direct_find(name);
+ }
+#endif
if (likely(du._du)) {
return dux_merge_msg(du._du, flags, hdr, siz);
}
@@ -2074,6 +2194,25 @@
_dispatch_bug_mach_client("_dispatch_kevent_mach_msg_drain: "
"message reception failed", kr);
}
+
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+ if (!(flags & EV_UDATA_SPECIFIC)) {
+ _dispatch_kq_deferred_update(DISPATCH_WLH_GLOBAL,
+ &_dispatch_mach_recv_kevent);
+ }
+#endif
+}
+
+static dispatch_unote_t
+_dispatch_source_mach_recv_create(dispatch_source_type_t dst,
+ uintptr_t handle, unsigned long mask)
+{
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+ if (!_dispatch_evfilt_machport_direct_enabled) {
+ dst = &_dispatch_source_type_mach_recv_pset;
+ }
+#endif
+ return _dispatch_unote_create_with_handle(dst, handle, mask);
}
const dispatch_source_type_s _dispatch_source_type_mach_recv = {
@@ -2083,13 +2222,26 @@
.dst_fflags = 0,
.dst_size = sizeof(struct dispatch_source_refs_s),
- .dst_create = _dispatch_unote_create_with_handle,
+ .dst_create = _dispatch_source_mach_recv_create,
.dst_merge_evt = _dispatch_source_merge_evt,
.dst_merge_msg = NULL, // never receives messages directly
-
- .dst_per_trigger_qos = true,
};
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+const dispatch_source_type_s _dispatch_source_type_mach_recv_pset = {
+ .dst_kind = "mach_recv (portset)",
+ .dst_filter = EVFILT_MACHPORT,
+ .dst_flags = EV_DISPATCH,
+ .dst_fflags = 0,
+ .dst_size = sizeof(struct dispatch_source_refs_s),
+
+ .dst_create = NULL, // never created directly
+ .dst_update_mux = _dispatch_mach_recv_update_portset_mux,
+ .dst_merge_evt = _dispatch_source_merge_evt,
+ .dst_merge_msg = NULL, // never receives messages directly
+};
+#endif
+
static void
_dispatch_source_mach_recv_direct_merge_msg(dispatch_unote_t du, uint32_t flags,
mach_msg_header_t *msg, mach_msg_size_t msgsz DISPATCH_UNUSED)
@@ -2114,6 +2266,18 @@
}
}
+static dispatch_unote_t
+_dispatch_source_mach_recv_direct_create(dispatch_source_type_t dst,
+ uintptr_t handle, unsigned long mask)
+{
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+ if (!_dispatch_evfilt_machport_direct_enabled) {
+ dst = &_dispatch_source_type_mach_recv_direct_pset;
+ }
+#endif
+ return _dispatch_unote_create_with_handle(dst, handle, mask);
+}
+
static void
_dispatch_mach_recv_direct_merge(dispatch_unote_t du,
uint32_t flags, uintptr_t data,
@@ -2134,13 +2298,40 @@
.dst_fflags = DISPATCH_MACH_RCV_OPTIONS,
.dst_size = sizeof(struct dispatch_source_refs_s),
- .dst_create = _dispatch_unote_create_with_handle,
+ .dst_create = _dispatch_source_mach_recv_direct_create,
.dst_merge_evt = _dispatch_mach_recv_direct_merge,
.dst_merge_msg = _dispatch_source_mach_recv_direct_merge_msg,
-
- .dst_per_trigger_qos = true,
};
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+const dispatch_source_type_s _dispatch_source_type_mach_recv_direct_pset = {
+ .dst_kind = "direct mach_recv (portset)",
+ .dst_filter = EVFILT_MACHPORT,
+ .dst_flags = 0,
+ .dst_fflags = DISPATCH_MACH_RCV_OPTIONS,
+ .dst_size = sizeof(struct dispatch_source_refs_s),
+
+ .dst_create = NULL, // never created directly
+ .dst_update_mux = _dispatch_mach_recv_direct_update_portset_mux,
+ .dst_merge_evt = _dispatch_mach_recv_direct_merge,
+ .dst_merge_msg = _dispatch_source_mach_recv_direct_merge_msg,
+};
+#endif
+
+static dispatch_unote_t
+_dispatch_mach_recv_create(dispatch_source_type_t dst,
+ uintptr_t handle, unsigned long mask)
+{
+ // mach channels pass MACH_PORT_NULL until connect
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+ if (!_dispatch_evfilt_machport_direct_enabled) {
+ dst = &_dispatch_mach_type_recv_pset;
+ }
+#endif
+ // without handle because the mach code will set the ident later
+ return _dispatch_unote_create_without_handle(dst, handle, mask);
+}
+
const dispatch_source_type_s _dispatch_mach_type_recv = {
.dst_kind = "mach_recv (channel)",
.dst_filter = EVFILT_MACHPORT,
@@ -2148,14 +2339,38 @@
.dst_fflags = DISPATCH_MACH_RCV_OPTIONS,
.dst_size = sizeof(struct dispatch_mach_recv_refs_s),
- // without handle because the mach code will set the ident after connect
- .dst_create = _dispatch_unote_create_without_handle,
+ .dst_create = _dispatch_mach_recv_create,
.dst_merge_evt = _dispatch_mach_recv_direct_merge,
.dst_merge_msg = _dispatch_mach_merge_msg,
-
- .dst_per_trigger_qos = true,
};
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+const dispatch_source_type_s _dispatch_mach_type_recv_pset = {
+ .dst_kind = "mach_recv (channel, portset)",
+ .dst_filter = EVFILT_MACHPORT,
+ .dst_flags = 0,
+ .dst_fflags = DISPATCH_MACH_RCV_OPTIONS,
+ .dst_size = sizeof(struct dispatch_mach_recv_refs_s),
+
+ .dst_create = NULL, // never created directly
+ .dst_update_mux = _dispatch_mach_recv_direct_update_portset_mux,
+ .dst_merge_evt = _dispatch_mach_recv_direct_merge,
+ .dst_merge_msg = _dispatch_mach_merge_msg,
+};
+#endif
+
+static dispatch_unote_t
+_dispatch_mach_reply_create(dispatch_source_type_t dst,
+ uintptr_t handle, unsigned long mask)
+{
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+ if (!_dispatch_evfilt_machport_direct_enabled) {
+ dst = &_dispatch_mach_type_reply_pset;
+ }
+#endif
+ return _dispatch_unote_create_with_handle(dst, handle, mask);
+}
+
DISPATCH_NORETURN
static void
_dispatch_mach_reply_merge_evt(dispatch_unote_t du,
@@ -2173,11 +2388,26 @@
.dst_fflags = DISPATCH_MACH_RCV_OPTIONS,
.dst_size = sizeof(struct dispatch_mach_reply_refs_s),
- .dst_create = _dispatch_unote_create_with_handle,
+ .dst_create = _dispatch_mach_reply_create,
.dst_merge_evt = _dispatch_mach_reply_merge_evt,
.dst_merge_msg = _dispatch_mach_reply_merge_msg,
};
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+const dispatch_source_type_s _dispatch_mach_type_reply_pset = {
+ .dst_kind = "mach reply (portset)",
+ .dst_filter = EVFILT_MACHPORT,
+ .dst_flags = EV_ONESHOT,
+ .dst_fflags = DISPATCH_MACH_RCV_OPTIONS,
+ .dst_size = sizeof(struct dispatch_mach_reply_refs_s),
+
+ .dst_create = NULL, // never created directly
+ .dst_update_mux = _dispatch_mach_recv_direct_update_portset_mux,
+ .dst_merge_evt = _dispatch_mach_reply_merge_evt,
+ .dst_merge_msg = _dispatch_mach_reply_merge_msg,
+};
+#endif
+
#pragma mark Mach channel SIGTERM notification (for XPC channels only)
const dispatch_source_type_s _dispatch_xpc_type_sigterm = {
diff --git a/src/event/workqueue_internal.h b/src/event/workqueue_internal.h
index 9f8fc3a..012e554 100644
--- a/src/event/workqueue_internal.h
+++ b/src/event/workqueue_internal.h
@@ -37,8 +37,6 @@
#define WORKQ_NUM_PRIORITIES 6
-#define WORKQ_ADDTHREADS_OPTION_OVERCOMMIT 0x1
-
#define DISPATCH_WORKQ_MAX_PTHREAD_COUNT 255
void _dispatch_workq_worker_register(dispatch_queue_t root_q, int priority);
diff --git a/src/firehose/firehose.defs b/src/firehose/firehose.defs
index e4fdf33..7ed7958 100644
--- a/src/firehose/firehose.defs
+++ b/src/firehose/firehose.defs
@@ -40,13 +40,12 @@
);
routine
-push_and_wait(
+push(
RequestPort comm_port : mach_port_t;
SReplyPort reply_port : mach_port_make_send_once_t;
qos_class : qos_class_t;
for_io : boolean_t;
-out push_reply : firehose_push_reply_t;
-out quarantinedOut : boolean_t
+out push_reply : firehose_push_reply_t
);
simpleroutine
diff --git a/src/firehose/firehose_buffer.c b/src/firehose/firehose_buffer.c
index 3bb790c..21692b9 100644
--- a/src/firehose/firehose_buffer.c
+++ b/src/firehose/firehose_buffer.c
@@ -69,8 +69,6 @@
#define DLOCK_LOCK_DATA_CONTENTION 0
static void _dispatch_gate_wait(dispatch_gate_t l, uint32_t flags);
-#define fcp_quarntined fcp_quarantined
-
#include <kern/debug.h>
#include <machine/cpu_number.h>
#include <kern/thread.h>
@@ -452,58 +450,23 @@
}
}
}
-
-OS_NOINLINE
-static void
-firehose_client_start_quarantine(firehose_buffer_t fb)
-{
- if (_voucher_libtrace_hooks->vah_version < 5) return;
- if (!_voucher_libtrace_hooks->vah_quarantine_starts) return;
-
- _voucher_libtrace_hooks->vah_quarantine_starts();
-
- fb->fb_header.fbh_quarantined = true;
- firehose_buffer_stream_flush(fb, firehose_stream_special);
- firehose_buffer_stream_flush(fb, firehose_stream_persist);
- firehose_buffer_stream_flush(fb, firehose_stream_memory);
-}
#endif // !KERNEL
static void
firehose_client_merge_updates(firehose_buffer_t fb, bool async_notif,
- firehose_push_reply_t reply, bool quarantined,
- firehose_bank_state_u *state_out)
+ firehose_push_reply_t reply, firehose_bank_state_u *state_out)
{
- firehose_buffer_header_t fbh = &fb->fb_header;
firehose_bank_state_u state;
firehose_ring_tail_u otail, ntail;
uint64_t old_flushed_pos, bank_updates;
uint16_t io_delta = 0;
uint16_t mem_delta = 0;
- if (quarantined) {
-#ifndef KERNEL
- // this isn't a dispatch_once so that the upcall to libtrace
- // can actually log itself without blocking on the gate.
- if (async_notif) {
- if (os_atomic_xchg(&fbh->fbh_quarantined_state,
- FBH_QUARANTINE_STARTED, relaxed) !=
- FBH_QUARANTINE_STARTED) {
- firehose_client_start_quarantine(fb);
- }
- } else if (os_atomic_load(&fbh->fbh_quarantined_state, relaxed) ==
- FBH_QUARANTINE_NONE) {
- os_atomic_cmpxchg(&fbh->fbh_quarantined_state, FBH_QUARANTINE_NONE,
- FBH_QUARANTINE_PENDING, relaxed);
- }
-#endif
- }
-
- if (firehose_atomic_maxv2o(fbh, fbh_bank.fbb_mem_flushed,
+ if (firehose_atomic_maxv2o(&fb->fb_header, fbh_bank.fbb_mem_flushed,
reply.fpr_mem_flushed_pos, &old_flushed_pos, relaxed)) {
mem_delta = (uint16_t)(reply.fpr_mem_flushed_pos - old_flushed_pos);
}
- if (firehose_atomic_maxv2o(fbh, fbh_bank.fbb_io_flushed,
+ if (firehose_atomic_maxv2o(&fb->fb_header, fbh_bank.fbb_io_flushed,
reply.fpr_io_flushed_pos, &old_flushed_pos, relaxed)) {
io_delta = (uint16_t)(reply.fpr_io_flushed_pos - old_flushed_pos);
}
@@ -515,14 +478,14 @@
if (!mem_delta && !io_delta) {
if (state_out) {
- state_out->fbs_atomic_state = os_atomic_load2o(fbh,
+ state_out->fbs_atomic_state = os_atomic_load2o(&fb->fb_header,
fbh_bank.fbb_state.fbs_atomic_state, relaxed);
}
return;
}
__firehose_critical_region_enter();
- os_atomic_rmw_loop2o(fbh, fbh_ring_tail.frp_atomic_tail,
+ os_atomic_rmw_loop2o(&fb->fb_header, fbh_ring_tail.frp_atomic_tail,
otail.frp_atomic_tail, ntail.frp_atomic_tail, relaxed, {
ntail = otail;
// overflow handles the generation wraps
@@ -532,7 +495,7 @@
bank_updates = ((uint64_t)mem_delta << FIREHOSE_BANK_SHIFT(0)) |
((uint64_t)io_delta << FIREHOSE_BANK_SHIFT(1));
- state.fbs_atomic_state = os_atomic_sub2o(fbh,
+ state.fbs_atomic_state = os_atomic_sub2o(&fb->fb_header,
fbh_bank.fbb_state.fbs_atomic_state, bank_updates, release);
__firehose_critical_region_leave();
@@ -540,32 +503,29 @@
if (async_notif) {
if (io_delta) {
- os_atomic_inc2o(fbh, fbh_bank.fbb_io_notifs, relaxed);
+ os_atomic_inc2o(&fb->fb_header, fbh_bank.fbb_io_notifs, relaxed);
}
if (mem_delta) {
- os_atomic_inc2o(fbh, fbh_bank.fbb_mem_notifs, relaxed);
+ os_atomic_inc2o(&fb->fb_header, fbh_bank.fbb_mem_notifs, relaxed);
}
}
}
#ifndef KERNEL
-OS_NOT_TAIL_CALLED OS_NOINLINE
static void
-firehose_client_send_push_and_wait(firehose_buffer_t fb, bool for_io,
+firehose_client_send_push(firehose_buffer_t fb, bool for_io,
firehose_bank_state_u *state_out)
{
mach_port_t sendp = fb->fb_header.fbh_sendp;
firehose_push_reply_t push_reply = { };
qos_class_t qos = qos_class_self();
- boolean_t quarantined = false;
kern_return_t kr;
if (slowpath(sendp == MACH_PORT_DEAD)) {
return;
}
if (fastpath(sendp)) {
- kr = firehose_send_push_and_wait(sendp, qos, for_io,
- &push_reply, &quarantined);
+ kr = firehose_send_push(sendp, qos, for_io, &push_reply);
if (likely(kr == KERN_SUCCESS)) {
goto success;
}
@@ -577,8 +537,7 @@
sendp = firehose_client_reconnect(fb, sendp);
if (fastpath(MACH_PORT_VALID(sendp))) {
- kr = firehose_send_push_and_wait(sendp, qos, for_io,
- &push_reply, &quarantined);
+ kr = firehose_send_push(sendp, qos, for_io, &push_reply);
if (likely(kr == KERN_SUCCESS)) {
goto success;
}
@@ -614,22 +573,12 @@
// There only is a point for multithreaded clients if:
// - enough samples (total_flushes above some limits)
// - the ratio is really bad (a push per cycle is definitely a problem)
- return firehose_client_merge_updates(fb, false, push_reply, quarantined,
- state_out);
-}
-
-OS_NOT_TAIL_CALLED OS_NOINLINE
-static void
-__FIREHOSE_CLIENT_THROTTLED_DUE_TO_HEAVY_LOGGING__(firehose_buffer_t fb,
- bool for_io, firehose_bank_state_u *state_out)
-{
- firehose_client_send_push_and_wait(fb, for_io, state_out);
+ return firehose_client_merge_updates(fb, false, push_reply, state_out);
}
kern_return_t
firehose_client_push_reply(mach_port_t req_port OS_UNUSED,
- kern_return_t rtc, firehose_push_reply_t push_reply OS_UNUSED,
- boolean_t quarantined OS_UNUSED)
+ kern_return_t rtc, firehose_push_reply_t push_reply OS_UNUSED)
{
DISPATCH_INTERNAL_CRASH(rtc, "firehose_push_reply should never be sent "
"to the buffer receive port");
@@ -637,12 +586,12 @@
kern_return_t
firehose_client_push_notify_async(mach_port_t server_port OS_UNUSED,
- firehose_push_reply_t push_reply, boolean_t quarantined)
+ firehose_push_reply_t push_reply)
{
// see _dispatch_source_merge_mach_msg_direct
dispatch_queue_t dq = _dispatch_queue_get_current();
firehose_buffer_t fb = dispatch_get_context(dq);
- firehose_client_merge_updates(fb, true, push_reply, quarantined, NULL);
+ firehose_client_merge_updates(fb, true, push_reply, NULL);
return KERN_SUCCESS;
}
@@ -704,7 +653,6 @@
.fcp_qos = firehose_buffer_qos_bits_propagate(),
.fcp_stream = ask->stream,
.fcp_flag_io = ask->for_io,
- .fcp_quarantined = ask->quarantined,
};
if (privptr) {
@@ -720,8 +668,7 @@
{
firehose_stream_state_u state, new_state;
firehose_tracepoint_t ft;
- firehose_buffer_header_t fbh = &fb->fb_header;
- firehose_buffer_stream_t fbs = &fbh->fbh_stream[ask->stream];
+ firehose_buffer_stream_t fbs = &fb->fb_header.fbh_stream[ask->stream];
uint64_t stamp_and_len;
if (fastpath(ref)) {
@@ -738,7 +685,7 @@
ft->ft_thread = _pthread_threadid_self_np_direct();
#endif
if (ask->stream == firehose_stream_metadata) {
- os_atomic_or2o(fbh, fbh_bank.fbb_metadata_bitmap,
+ os_atomic_or2o(fb, fb_header.fbh_bank.fbb_metadata_bitmap,
1ULL << ref, relaxed);
}
// release barrier to make the chunk init visible
@@ -769,11 +716,8 @@
ft = NULL;
}
- // pairs with the one in firehose_buffer_tracepoint_reserve()
- __firehose_critical_region_leave();
-
#ifndef KERNEL
- if (unlikely(_dispatch_lock_is_locked_by_self(state.fss_gate.dgl_lock))) {
+ if (unlikely(state.fss_gate.dgl_lock != _dispatch_tid_self())) {
_dispatch_gate_broadcast_slow(&fbs->fbs_state.fss_gate,
state.fss_gate.dgl_lock);
}
@@ -781,16 +725,10 @@
if (unlikely(state.fss_current == FIREHOSE_STREAM_STATE_PRISTINE)) {
firehose_buffer_update_limits(fb);
}
-
- if (unlikely(os_atomic_load2o(fbh, fbh_quarantined_state, relaxed) ==
- FBH_QUARANTINE_PENDING)) {
- if (os_atomic_cmpxchg2o(fbh, fbh_quarantined_state,
- FBH_QUARANTINE_PENDING, FBH_QUARANTINE_STARTED, relaxed)) {
- firehose_client_start_quarantine(fb);
- }
- }
#endif // KERNEL
+ // pairs with the one in firehose_buffer_tracepoint_reserve()
+ __firehose_critical_region_leave();
return ft;
}
@@ -1029,12 +967,7 @@
state.fbs_atomic_state =
os_atomic_load2o(fbb, fbb_state.fbs_atomic_state, relaxed);
while ((state.fbs_atomic_state - bank_inc) & bank_unavail_mask) {
- if (ask->quarantined) {
- __FIREHOSE_CLIENT_THROTTLED_DUE_TO_HEAVY_LOGGING__(fb,
- ask->for_io, &state);
- } else {
- firehose_client_send_push_and_wait(fb, ask->for_io, &state);
- }
+ firehose_client_send_push(fb, ask->for_io, &state);
if (slowpath(fb->fb_header.fbh_sendp == MACH_PORT_DEAD)) {
// logd was unloaded, give up
return NULL;
@@ -1066,12 +999,7 @@
if (fastpath(ref = firehose_buffer_ring_try_grow(fbb, fbs_max_ref))) {
break;
}
- if (ask->quarantined) {
- __FIREHOSE_CLIENT_THROTTLED_DUE_TO_HEAVY_LOGGING__(fb,
- ask->for_io, &state);
- } else {
- firehose_client_send_push_and_wait(fb, ask->for_io, NULL);
- }
+ firehose_client_send_push(fb, ask->for_io, NULL);
if (slowpath(fb->fb_header.fbh_sendp == MACH_PORT_DEAD)) {
// logd was unloaded, give up
break;
@@ -1180,7 +1108,7 @@
{
firehose_buffer_t fb = kernel_firehose_buffer;
if (fastpath(fb)) {
- firehose_client_merge_updates(fb, true, update, false, NULL);
+ firehose_client_merge_updates(fb, true, update, NULL);
}
}
#endif // KERNEL
diff --git a/src/firehose/firehose_buffer_internal.h b/src/firehose/firehose_buffer_internal.h
index e41d9cb..7679c8c 100644
--- a/src/firehose/firehose_buffer_internal.h
+++ b/src/firehose/firehose_buffer_internal.h
@@ -171,11 +171,6 @@
dispatch_once_t fbh_notifs_pred OS_ALIGNED(64);
dispatch_source_t fbh_notifs_source;
dispatch_unfair_lock_s fbh_logd_lock;
-#define FBH_QUARANTINE_NONE 0
-#define FBH_QUARANTINE_PENDING 1
-#define FBH_QUARANTINE_STARTED 2
- uint8_t volatile fbh_quarantined_state;
- bool fbh_quarantined;
#endif
uint64_t fbh_unused[0];
} OS_ALIGNED(FIREHOSE_CHUNK_SIZE) *firehose_buffer_header_t;
@@ -192,7 +187,6 @@
firehose_stream_t stream;
bool is_bank_ok;
bool for_io;
- bool quarantined;
uint64_t stamp;
} *firehose_tracepoint_query_t;
diff --git a/src/firehose/firehose_inline_internal.h b/src/firehose/firehose_inline_internal.h
index 3939ee2..abc5f9e 100644
--- a/src/firehose/firehose_inline_internal.h
+++ b/src/firehose/firehose_inline_internal.h
@@ -319,7 +319,7 @@
#if KERNEL
new_state.fss_allocator = (uint32_t)cpu_number();
#else
- new_state.fss_allocator = _dispatch_lock_value_for_self();
+ new_state.fss_allocator = _dispatch_tid_self();
#endif
success = os_atomic_cmpxchgv2o(fbs, fbs_state.fss_atomic_state,
old_state.fss_atomic_state, new_state.fss_atomic_state,
@@ -335,9 +335,6 @@
.privsize = privsize,
.stream = stream,
.for_io = (firehose_stream_uses_io_bank & (1UL << stream)) != 0,
-#ifndef KERNEL
- .quarantined = fb->fb_header.fbh_quarantined,
-#endif
.stamp = stamp,
};
return firehose_buffer_tracepoint_reserve_slow(fb, &ask, privptr);
diff --git a/src/firehose/firehose_internal.h b/src/firehose/firehose_internal.h
index 7040995..29d1ad2 100644
--- a/src/firehose/firehose_internal.h
+++ b/src/firehose/firehose_internal.h
@@ -29,8 +29,6 @@
#define __MigTypeCheck 1
#endif
-#define fcp_quarntined fcp_quarantined
-
#include <limits.h>
#include <machine/endian.h>
#include <mach/mach_types.h>
diff --git a/src/firehose/firehose_reply.defs b/src/firehose/firehose_reply.defs
index c080545..124defa 100644
--- a/src/firehose/firehose_reply.defs
+++ b/src/firehose/firehose_reply.defs
@@ -33,13 +33,11 @@
simpleroutine push_reply(
RequestPort req_port : mach_port_move_send_once_t;
in rtc : kern_return_t;
-in push_reply : firehose_push_reply_t;
-in quarantined : boolean_t
+in push_reply : firehose_push_reply_t
);
simpleroutine push_notify_async(
RequestPort comm_port : mach_port_t;
in push_reply : firehose_push_reply_t;
-in quarantined : boolean_t;
WaitTime timeout : natural_t
);
diff --git a/src/firehose/firehose_server.c b/src/firehose/firehose_server.c
index ba335db..52397d6 100644
--- a/src/firehose/firehose_server.c
+++ b/src/firehose/firehose_server.c
@@ -31,11 +31,6 @@
% 8 == 0, "Make sure atomic fields are properly aligned");
#endif
-typedef struct fs_client_queue_s {
- struct firehose_client_s *volatile fs_client_head;
- struct firehose_client_s *volatile fs_client_tail;
-} fs_client_queue_s, *fs_client_queue_t;
-
static struct firehose_server_s {
mach_port_t fs_bootstrap_port;
dispatch_mach_t fs_mach_channel;
@@ -46,161 +41,26 @@
firehose_handler_t fs_handler;
firehose_snapshot_t fs_snapshot;
+ bool fs_io_snapshot_started;
+ bool fs_mem_snapshot_started;
+
int fs_kernel_fd;
firehose_client_t fs_kernel_client;
TAILQ_HEAD(, firehose_client_s) fs_clients;
- os_unfair_lock fs_clients_lock;
- fs_client_queue_s fs_queues[4];
- dispatch_source_t fs_sources[4];
} server_config = {
.fs_clients = TAILQ_HEAD_INITIALIZER(server_config.fs_clients),
- .fs_clients_lock = OS_UNFAIR_LOCK_INIT,
.fs_kernel_fd = -1,
};
-OS_ALWAYS_INLINE
-static inline void
-fs_clients_lock(void)
-{
- os_unfair_lock_lock_with_options(&server_config.fs_clients_lock,
- OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION);
-}
-
-OS_ALWAYS_INLINE
-static inline void
-fs_clients_unlock(void)
-{
- os_unfair_lock_unlock(&server_config.fs_clients_lock);
-}
+#pragma mark -
+#pragma mark firehose client state machine
static void firehose_server_demux(firehose_client_t fc,
mach_msg_header_t *msg_hdr);
static void firehose_client_cancel(firehose_client_t fc);
static void firehose_client_snapshot_finish(firehose_client_t fc,
firehose_snapshot_t snapshot, bool for_io);
-static void firehose_client_handle_death(void *ctxt);
-
-#pragma mark -
-#pragma mark firehose client enqueueing
-
-OS_ALWAYS_INLINE
-static inline bool
-fs_idx_is_for_io(size_t idx)
-{
- return idx & 1;
-}
-
-OS_ALWAYS_INLINE
-static inline bool
-fs_queue_is_for_io(fs_client_queue_t q)
-{
- return (q - server_config.fs_queues) & 1;
-}
-
-OS_ALWAYS_INLINE
-static inline bool
-fs_queue_is_for_quarantined(fs_client_queue_t q)
-{
- return (q - server_config.fs_queues) & 2;
-}
-
-OS_ALWAYS_INLINE
-static inline fs_client_queue_t
-fs_queue(bool quarantined, bool for_io)
-{
- return &server_config.fs_queues[quarantined * 2 + for_io];
-}
-
-OS_ALWAYS_INLINE
-static inline dispatch_source_t
-fs_source(bool quarantined, bool for_io)
-{
- return server_config.fs_sources[quarantined * 2 + for_io];
-}
-
-OS_ALWAYS_INLINE
-static inline void
-firehose_client_push(firehose_client_t fc, pthread_priority_t pp,
- bool quarantined, bool for_io)
-{
- fs_client_queue_t queue = fs_queue(quarantined, for_io);
- if (fc && os_mpsc_push_update_tail(queue, fs_client, fc, fc_next[for_io])) {
- os_mpsc_push_update_head(queue, fs_client, fc);
- _dispatch_source_merge_data(fs_source(quarantined, for_io), pp, 1);
- } else if (pp) {
- _dispatch_source_merge_data(fs_source(quarantined, for_io), pp, 1);
- }
-}
-
-OS_ALWAYS_INLINE
-static inline bool
-firehose_client_wakeup(firehose_client_t fc, pthread_priority_t pp,
- bool for_io)
-{
- uintptr_t canceled_bit = FC_STATE_CANCELED(for_io);
- uintptr_t enqueued_bit = FC_STATE_ENQUEUED(for_io);
- uintptr_t old_state, new_state;
-
- os_atomic_rmw_loop(&fc->fc_state, old_state, new_state, relaxed, {
- if (old_state & canceled_bit) {
- os_atomic_rmw_loop_give_up(return false);
- }
- if (old_state & enqueued_bit) {
- os_atomic_rmw_loop_give_up(break);
- }
- new_state = old_state | enqueued_bit;
- });
- firehose_client_push(old_state & enqueued_bit ? NULL : fc, pp,
- fc->fc_quarantined, for_io);
- return true;
-}
-
-OS_ALWAYS_INLINE
-static inline void
-firehose_client_start_cancel(firehose_client_t fc, bool for_io)
-{
- uintptr_t canceling_bit = FC_STATE_CANCELING(for_io);
- uintptr_t canceled_bit = FC_STATE_CANCELED(for_io);
- uintptr_t enqueued_bit = FC_STATE_ENQUEUED(for_io);
- uintptr_t old_state, new_state;
-
- os_atomic_rmw_loop(&fc->fc_state, old_state, new_state, relaxed, {
- if (old_state & (canceled_bit | canceling_bit)) {
- os_atomic_rmw_loop_give_up(return);
- }
- new_state = old_state | enqueued_bit | canceling_bit;
- });
- firehose_client_push(old_state & enqueued_bit ? NULL : fc, 0,
- fc->fc_quarantined, for_io);
-}
-
-OS_ALWAYS_INLINE
-static inline bool
-firehose_client_dequeue(firehose_client_t fc, bool for_io)
-{
- uintptr_t canceling_bit = FC_STATE_CANCELING(for_io);
- uintptr_t canceled_bit = FC_STATE_CANCELED(for_io);
- uintptr_t enqueued_bit = FC_STATE_ENQUEUED(for_io);
- uintptr_t old_state, new_state;
-
- os_atomic_rmw_loop(&fc->fc_state, old_state, new_state, relaxed, {
- new_state = old_state & ~(canceling_bit | enqueued_bit);
- if (old_state & canceling_bit) {
- new_state |= canceled_bit;
- }
- });
-
- if (((old_state ^ new_state) & FC_STATE_CANCELED_MASK) &&
- (new_state & FC_STATE_CANCELED_MASK) == FC_STATE_CANCELED_MASK) {
- dispatch_async_f(server_config.fs_io_drain_queue, fc,
- firehose_client_handle_death);
- }
- return !(new_state & canceled_bit);
-}
-
-#pragma mark -
-#pragma mark firehose client state machine
static void
firehose_client_notify(firehose_client_t fc, mach_port_t reply_port)
@@ -222,11 +82,9 @@
}
} else {
if (reply_port == fc->fc_sendp) {
- kr = firehose_send_push_notify_async(reply_port, push_reply,
- fc->fc_quarantined, 0);
+ kr = firehose_send_push_notify_async(reply_port, push_reply, 0);
} else {
- kr = firehose_send_push_reply(reply_port, KERN_SUCCESS, push_reply,
- fc->fc_quarantined);
+ kr = firehose_send_push_reply(reply_port, KERN_SUCCESS, push_reply);
}
if (kr != MACH_SEND_INVALID_DEST) {
DISPATCH_VERIFY_MIG(kr);
@@ -248,6 +106,18 @@
return head;
}
+OS_ALWAYS_INLINE
+static inline void
+firehose_client_push_async_merge(firehose_client_t fc, pthread_priority_t pp,
+ bool for_io)
+{
+ if (for_io) {
+ _dispatch_source_merge_data(fc->fc_io_source, pp, 1);
+ } else {
+ _dispatch_source_merge_data(fc->fc_mem_source, pp, 1);
+ }
+}
+
OS_NOINLINE OS_COLD
static void
firehose_client_mark_corrupted(firehose_client_t fc, mach_port_t reply_port)
@@ -261,7 +131,7 @@
if (reply_port) {
kern_return_t kr = firehose_send_push_reply(reply_port, 0,
- FIREHOSE_PUSH_REPLY_CORRUPTED, false);
+ FIREHOSE_PUSH_REPLY_CORRUPTED);
DISPATCH_VERIFY_MIG(kr);
dispatch_assume_zero(kr);
}
@@ -286,7 +156,7 @@
OS_NOINLINE
static void
-firehose_client_drain_one(firehose_client_t fc, mach_port_t port, uint32_t flags)
+firehose_client_drain(firehose_client_t fc, mach_port_t port, uint32_t flags)
{
firehose_buffer_t fb = fc->fc_buffer;
firehose_chunk_t fbc;
@@ -304,7 +174,9 @@
fbh_ring = fb->fb_header.fbh_io_ring;
sent_flushed = (uint16_t)fc->fc_io_sent_flushed_pos;
flushed = (uint16_t)fc->fc_io_flushed_pos;
- if (fc->fc_needs_io_snapshot) snapshot = server_config.fs_snapshot;
+ if (fc->fc_needs_io_snapshot && server_config.fs_io_snapshot_started) {
+ snapshot = server_config.fs_snapshot;
+ }
} else {
evt = FIREHOSE_EVENT_MEM_BUFFER_RECEIVED;
_Static_assert(FIREHOSE_EVENT_MEM_BUFFER_RECEIVED ==
@@ -312,7 +184,9 @@
fbh_ring = fb->fb_header.fbh_mem_ring;
sent_flushed = (uint16_t)fc->fc_mem_sent_flushed_pos;
flushed = (uint16_t)fc->fc_mem_flushed_pos;
- if (fc->fc_needs_mem_snapshot) snapshot = server_config.fs_snapshot;
+ if (fc->fc_needs_mem_snapshot && server_config.fs_mem_snapshot_started) {
+ snapshot = server_config.fs_snapshot;
+ }
}
if (slowpath(fc->fc_memory_corrupted)) {
@@ -399,12 +273,12 @@
// and there's more to drain, so optimistically schedule draining
// again this is cheap since the queue is hot, and is fair for other
// clients
- firehose_client_wakeup(fc, 0, for_io);
+ firehose_client_push_async_merge(fc, 0, for_io);
}
if (count && server_config.fs_kernel_client) {
// the kernel is special because it can drop messages, so if we're
// draining, poll the kernel each time while we're bound to a thread
- firehose_client_drain_one(server_config.fs_kernel_client,
+ firehose_client_drain(server_config.fs_kernel_client,
MACH_PORT_NULL, flags | FIREHOSE_DRAIN_POLL);
}
}
@@ -419,36 +293,20 @@
// (needs_<for_io>_snapshot: false, memory_corrupted: true). we can safely
// silence the corresponding source of drain wake-ups.
if (fc->fc_pid) {
- firehose_client_start_cancel(fc, for_io);
+ dispatch_source_cancel(for_io ? fc->fc_io_source : fc->fc_mem_source);
}
}
static void
-firehose_client_drain(void *ctxt)
+firehose_client_drain_io_async(void *ctx)
{
- fs_client_queue_t queue = ctxt;
- bool for_io = fs_queue_is_for_io(queue);
- bool quarantined = fs_queue_is_for_quarantined(queue);
- firehose_client_t fc, fc_next;
- size_t clients = 0;
+ firehose_client_drain(ctx, MACH_PORT_NULL, FIREHOSE_DRAIN_FOR_IO);
+}
- while (queue->fs_client_tail) {
- fc = os_mpsc_get_head(queue, fs_client);
- do {
- fc_next = os_mpsc_pop_head(queue, fs_client, fc, fc_next[for_io]);
- if (firehose_client_dequeue(fc, for_io)) {
- firehose_client_drain_one(fc, MACH_PORT_NULL,
- for_io ? FIREHOSE_DRAIN_FOR_IO : 0);
- }
- // process quarantined clients 4 times as slow as the other ones
- // also reasyncing every 4 clients allows for discovering
- // quarantined suspension faster
- if (++clients == (quarantined ? 1 : 4)) {
- dispatch_source_merge_data(fs_source(quarantined, for_io), 1);
- return;
- }
- } while ((fc = fc_next));
- }
+static void
+firehose_client_drain_mem_async(void *ctx)
+{
+ firehose_client_drain(ctx, MACH_PORT_NULL, 0);
}
OS_NOINLINE
@@ -477,10 +335,7 @@
}
server_config.fs_handler(fc, FIREHOSE_EVENT_CLIENT_DIED, NULL);
- fs_clients_lock();
TAILQ_REMOVE(&server_config.fs_clients, fc, fc_entry);
- fs_clients_unlock();
-
dispatch_release(fc->fc_mach_channel);
fc->fc_mach_channel = NULL;
fc->fc_entry.tqe_next = DISPATCH_OBJECT_LISTLESS;
@@ -558,7 +413,7 @@
continue;
}
server_config.fs_handler(fc, FIREHOSE_EVENT_IO_BUFFER_RECEIVED, fbc);
- if (fc->fc_needs_io_snapshot) {
+ if (fc->fc_needs_io_snapshot && server_config.fs_io_snapshot_started) {
snapshot->handler(fc, FIREHOSE_SNAPSHOT_EVENT_IO_BUFFER, fbc);
}
}
@@ -576,7 +431,7 @@
mem_bitmap_copy &= ~(1ULL << ref);
server_config.fs_handler(fc, FIREHOSE_EVENT_MEM_BUFFER_RECEIVED, fbc);
- if (fc->fc_needs_mem_snapshot) {
+ if (fc->fc_needs_mem_snapshot && server_config.fs_mem_snapshot_started) {
snapshot->handler(fc, FIREHOSE_SNAPSHOT_EVENT_MEM_BUFFER, fbc);
}
}
@@ -592,11 +447,16 @@
{
mach_msg_header_t *msg_hdr = NULL;
firehose_client_t fc = ctx;
- mach_port_t port;
+ mach_port_t oldsendp = 0, oldrecvp = 0;
+
+ if (dmsg) {
+ msg_hdr = dispatch_mach_msg_get_msg(dmsg, NULL);
+ oldsendp = msg_hdr->msgh_remote_port;
+ oldrecvp = msg_hdr->msgh_local_port;
+ }
switch (reason) {
case DISPATCH_MACH_MESSAGE_RECEIVED:
- msg_hdr = dispatch_mach_msg_get_msg(dmsg, NULL);
if (msg_hdr->msgh_id == MACH_NOTIFY_NO_SENDERS) {
_dispatch_debug("FIREHOSE NO_SENDERS (unique_pid: 0x%llx)",
firehose_client_get_unique_pid(fc, NULL));
@@ -607,33 +467,25 @@
break;
case DISPATCH_MACH_DISCONNECTED:
- msg_hdr = dispatch_mach_msg_get_msg(dmsg, NULL);
- port = msg_hdr->msgh_remote_port;
- if (MACH_PORT_VALID(port)) {
- if (port != fc->fc_sendp) {
- DISPATCH_INTERNAL_CRASH(port, "Unknown send-right");
+ if (oldsendp) {
+ if (slowpath(oldsendp != fc->fc_sendp)) {
+ DISPATCH_INTERNAL_CRASH(oldsendp,
+ "disconnect event about unknown send-right");
}
firehose_mach_port_send_release(fc->fc_sendp);
fc->fc_sendp = MACH_PORT_NULL;
}
- port = msg_hdr->msgh_local_port;
- if (MACH_PORT_VALID(port)) {
- if (port != fc->fc_recvp) {
- DISPATCH_INTERNAL_CRASH(port, "Unknown recv-right");
+ if (oldrecvp) {
+ if (slowpath(oldrecvp != fc->fc_recvp)) {
+ DISPATCH_INTERNAL_CRASH(oldrecvp,
+ "disconnect event about unknown receive-right");
}
firehose_mach_port_recv_dispose(fc->fc_recvp, fc);
fc->fc_recvp = MACH_PORT_NULL;
}
- break;
-
- case DISPATCH_MACH_CANCELED:
- if (MACH_PORT_VALID(fc->fc_sendp)) {
- DISPATCH_INTERNAL_CRASH(fc->fc_sendp, "send-right leak");
+ if (fc->fc_recvp == MACH_PORT_NULL && fc->fc_sendp == MACH_PORT_NULL) {
+ firehose_client_cancel(fc);
}
- if (MACH_PORT_VALID(fc->fc_recvp)) {
- DISPATCH_INTERNAL_CRASH(fc->fc_recvp, "recv-right leak");
- }
- firehose_client_cancel(fc);
break;
}
}
@@ -647,8 +499,10 @@
// resumed in firehose_client_drain for both memory and I/O
dispatch_suspend(fc->fc_kernel_source);
dispatch_suspend(fc->fc_kernel_source);
- firehose_client_wakeup(fc, 0, false);
- firehose_client_wakeup(fc, 0, true);
+ dispatch_async_f(server_config.fs_mem_drain_queue,
+ fc, firehose_client_drain_mem_async);
+ dispatch_async_f(server_config.fs_io_drain_queue,
+ fc, firehose_client_drain_io_async);
}
#endif
@@ -657,37 +511,36 @@
const struct firehose_client_connected_info_s *fcci)
{
dispatch_assert_queue(server_config.fs_io_drain_queue);
-
- fs_clients_lock();
TAILQ_INSERT_TAIL(&server_config.fs_clients, fc, fc_entry);
- fs_clients_unlock();
-
server_config.fs_handler(fc, FIREHOSE_EVENT_CLIENT_CONNECTED, (void *)fcci);
if (!fc->fc_pid) {
dispatch_activate(fc->fc_kernel_source);
} else {
dispatch_mach_connect(fc->fc_mach_channel,
fc->fc_recvp, fc->fc_sendp, NULL);
+ dispatch_activate(fc->fc_io_source);
+ dispatch_activate(fc->fc_mem_source);
}
}
static void
firehose_client_cancel(firehose_client_t fc)
{
+ dispatch_block_t block;
+
_dispatch_debug("client died (unique_pid: 0x%llx",
firehose_client_get_unique_pid(fc, NULL));
- if (MACH_PORT_VALID(fc->fc_sendp)) {
- firehose_mach_port_send_release(fc->fc_sendp);
- fc->fc_sendp = MACH_PORT_NULL;
- }
- if (MACH_PORT_VALID(fc->fc_recvp)) {
- firehose_mach_port_recv_dispose(fc->fc_recvp, fc);
- fc->fc_recvp = MACH_PORT_NULL;
- }
fc->fc_use_notifs = false;
- firehose_client_start_cancel(fc, false);
- firehose_client_start_cancel(fc, true);
+ dispatch_source_cancel(fc->fc_io_source);
+ dispatch_source_cancel(fc->fc_mem_source);
+
+ block = dispatch_block_create(DISPATCH_BLOCK_DETACHED, ^{
+ dispatch_async_f(server_config.fs_io_drain_queue, fc,
+ firehose_client_handle_death);
+ });
+ dispatch_async(server_config.fs_mem_drain_queue, block);
+ _Block_release(block);
}
static firehose_client_t
@@ -725,10 +578,28 @@
uint64_t unique_pid = fb->fb_header.fbh_uniquepid;
firehose_client_t fc = _firehose_client_create(fb);
dispatch_mach_t dm;
+ dispatch_source_t ds;
fc->fc_pid = token->pid ? token->pid : ~0;
fc->fc_euid = token->euid;
fc->fc_pidversion = token->execcnt;
+ ds = dispatch_source_create(DISPATCH_SOURCE_TYPE_DATA_OR, 0, 0,
+ server_config.fs_mem_drain_queue);
+ _os_object_retain_internal_inline(&fc->fc_as_os_object);
+ dispatch_set_context(ds, fc);
+ dispatch_set_finalizer_f(ds,
+ (dispatch_function_t)_os_object_release_internal);
+ dispatch_source_set_event_handler_f(ds, firehose_client_drain_mem_async);
+ fc->fc_mem_source = ds;
+
+ ds = dispatch_source_create(DISPATCH_SOURCE_TYPE_DATA_OR, 0, 0,
+ server_config.fs_io_drain_queue);
+ _os_object_retain_internal_inline(&fc->fc_as_os_object);
+ dispatch_set_context(ds, fc);
+ dispatch_set_finalizer_f(ds,
+ (dispatch_function_t)_os_object_release_internal);
+ dispatch_source_set_event_handler_f(ds, firehose_client_drain_io_async);
+ fc->fc_io_source = ds;
_dispatch_debug("FIREHOSE_REGISTER (unique_pid: 0x%llx)", unique_pid);
fc->fc_recvp = comm_recvp;
@@ -801,6 +672,12 @@
{
_dispatch_debug("Cleaning up client info for unique_pid 0x%llx",
firehose_client_get_unique_pid(fc, NULL));
+
+ dispatch_release(fc->fc_io_source);
+ fc->fc_io_source = NULL;
+
+ dispatch_release(fc->fc_mem_source);
+ fc->fc_mem_source = NULL;
}
uint64_t
@@ -845,12 +722,6 @@
return os_atomic_xchg2o(fc, fc_ctxt, ctxt, relaxed);
}
-void
-firehose_client_initiate_quarantine(firehose_client_t fc)
-{
- fc->fc_quarantined = true;
-}
-
#pragma mark -
#pragma mark firehose server
@@ -879,24 +750,22 @@
firehose_server_init(mach_port_t comm_port, firehose_handler_t handler)
{
struct firehose_server_s *fs = &server_config;
- dispatch_queue_attr_t attr = DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL;
- dispatch_queue_attr_t attr_ui;
+ dispatch_queue_attr_t attr;
dispatch_mach_t dm;
- dispatch_source_t ds;
// just reference the string so that it's captured
(void)os_atomic_load(&__libfirehose_serverVersionString[0], relaxed);
- attr_ui = dispatch_queue_attr_make_with_qos_class(attr,
+ attr = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL,
QOS_CLASS_USER_INITIATED, 0);
fs->fs_ipc_queue = dispatch_queue_create_with_target(
- "com.apple.firehose.ipc", attr_ui, NULL);
+ "com.apple.firehose.ipc", attr, NULL);
fs->fs_snapshot_gate_queue = dispatch_queue_create_with_target(
- "com.apple.firehose.snapshot-gate", attr, NULL);
+ "com.apple.firehose.snapshot-gate", DISPATCH_QUEUE_SERIAL, NULL);
fs->fs_io_drain_queue = dispatch_queue_create_with_target(
- "com.apple.firehose.drain-io", attr, NULL);
+ "com.apple.firehose.drain-io", DISPATCH_QUEUE_SERIAL, NULL);
fs->fs_mem_drain_queue = dispatch_queue_create_with_target(
- "com.apple.firehose.drain-mem", attr, NULL);
+ "com.apple.firehose.drain-mem", DISPATCH_QUEUE_SERIAL, NULL);
dm = dispatch_mach_create_f("com.apple.firehose.listener",
fs->fs_ipc_queue, NULL, firehose_server_handle_mach_event);
@@ -904,15 +773,6 @@
fs->fs_mach_channel = dm;
fs->fs_handler = _Block_copy(handler);
firehose_kernel_client_create();
-
- for (size_t i = 0; i < countof(fs->fs_sources); i++) {
- ds = dispatch_source_create(DISPATCH_SOURCE_TYPE_DATA_OR, 0, 0,
- fs_idx_is_for_io(i) ? server_config.fs_io_drain_queue :
- server_config.fs_mem_drain_queue);
- dispatch_set_context(ds, &fs->fs_queues[i]);
- dispatch_source_set_event_handler_f(ds, firehose_client_drain);
- fs->fs_sources[i] = ds;
- }
}
void
@@ -956,23 +816,24 @@
}
dispatch_mach_connect(fs->fs_mach_channel, fs->fs_bootstrap_port,
MACH_PORT_NULL, NULL);
- for (size_t i = 0; i < countof(fs->fs_sources); i++) {
- dispatch_activate(fs->fs_sources[i]);
+}
+
+OS_NOINLINE
+static void
+_firehose_server_cancel(void *ctxt OS_UNUSED)
+{
+ firehose_client_t fc;
+ TAILQ_FOREACH(fc, &server_config.fs_clients, fc_entry) {
+ dispatch_mach_cancel(fc->fc_mach_channel);
}
}
void
firehose_server_cancel(void)
{
- firehose_client_t fc;
-
dispatch_mach_cancel(server_config.fs_mach_channel);
-
- fs_clients_lock();
- TAILQ_FOREACH(fc, &server_config.fs_clients, fc_entry) {
- dispatch_mach_cancel(fc->fc_mach_channel);
- }
- fs_clients_unlock();
+ dispatch_async_f(server_config.fs_io_drain_queue, NULL,
+ _firehose_server_cancel);
}
dispatch_queue_t
@@ -993,37 +854,6 @@
return dq;
}
-void
-firehose_server_quarantined_suspend(firehose_server_queue_t which)
-{
- switch (which) {
- case FIREHOSE_SERVER_QUEUE_IO:
- dispatch_suspend(fs_source(true, true));
- break;
- case FIREHOSE_SERVER_QUEUE_MEMORY:
- dispatch_suspend(fs_source(true, false));
- break;
- default:
- DISPATCH_INTERNAL_CRASH(which, "Invalid firehose server queue type");
- }
-}
-
-void
-firehose_server_quarantined_resume(firehose_server_queue_t which)
-{
- switch (which) {
- case FIREHOSE_SERVER_QUEUE_IO:
- dispatch_resume(fs_source(true, true));
- break;
- case FIREHOSE_SERVER_QUEUE_MEMORY:
- dispatch_resume(fs_source(true, false));
- break;
- default:
- DISPATCH_INTERNAL_CRASH(which, "Invalid firehose server queue type");
- }
-}
-
-
#pragma mark -
#pragma mark firehose snapshot and peeking
@@ -1136,35 +966,73 @@
}
static void
-firehose_snapshot_tickle_clients(firehose_snapshot_t fs, bool for_io)
+firehose_snapshot_start(void *ctxt)
{
- firehose_client_t fc;
+ firehose_snapshot_t snapshot = ctxt;
+ firehose_client_t fci;
long n = 0;
- fs_clients_lock();
- TAILQ_FOREACH(fc, &server_config.fs_clients, fc_entry) {
- if (slowpath(fc->fc_memory_corrupted)) {
- continue;
- }
- if (!fc->fc_pid) {
+ // 0. we need to be on the IO queue so that client connection and/or death
+ // cannot happen concurrently
+ dispatch_assert_queue(server_config.fs_io_drain_queue);
+ server_config.fs_snapshot = snapshot;
+
+ // 1. mark all the clients participating in the current snapshot
+ // and enter the group for each bit set
+ TAILQ_FOREACH(fci, &server_config.fs_clients, fc_entry) {
+ if (!fci->fc_pid) {
#if TARGET_OS_SIMULATOR
continue;
#endif
- } else if (!firehose_client_wakeup(fc, 0, for_io)) {
+ }
+ if (slowpath(fci->fc_memory_corrupted)) {
continue;
}
- n++;
- if (for_io) {
- fc->fc_needs_io_snapshot = true;
- } else {
- fc->fc_needs_mem_snapshot = true;
- }
+ fci->fc_needs_io_snapshot = true;
+ fci->fc_needs_mem_snapshot = true;
+ n += 2;
}
- fs_clients_unlock();
+ if (n) {
+ // cheating: equivalent to dispatch_group_enter() n times
+ // without the acquire barriers that we don't need
+ os_atomic_add2o(snapshot->fs_group, dg_value, n, relaxed);
+ }
- // cheating: equivalent to dispatch_group_enter() n times
- // without the acquire barriers that we don't need
- if (n) os_atomic_add2o(fs->fs_group, dg_value, n, relaxed);
+ dispatch_async(server_config.fs_mem_drain_queue, ^{
+ // 2. start the fs_mem_snapshot, this is what triggers the snapshot
+ // logic from _drain() or handle_death()
+ server_config.fs_mem_snapshot_started = true;
+ snapshot->handler(NULL, FIREHOSE_SNAPSHOT_EVENT_MEM_START, NULL);
+
+ dispatch_async(server_config.fs_io_drain_queue, ^{
+ firehose_client_t fcj;
+
+ // 3. start the fs_io_snapshot, this is what triggers the snapshot
+ // logic from _drain() or handle_death()
+ // 29868879: must always happen after the memory snapshot started
+ server_config.fs_io_snapshot_started = true;
+ snapshot->handler(NULL, FIREHOSE_SNAPSHOT_EVENT_IO_START, NULL);
+
+ // match group_enter from firehose_snapshot() after MEM+IO_START
+ dispatch_group_leave(snapshot->fs_group);
+
+ // 3. tickle all the clients. the list of clients may have changed
+ // since step 1, but worry not - new clients don't have
+ // fc_needs_*_snapshot set so drain is harmless; clients that
+ // were removed from the list have already left the group
+ // (see firehose_client_finalize())
+ TAILQ_FOREACH(fcj, &server_config.fs_clients, fc_entry) {
+ if (!fcj->fc_pid) {
+#if !TARGET_OS_SIMULATOR
+ firehose_client_kernel_source_handle_event(fcj);
+#endif
+ } else {
+ dispatch_source_merge_data(fcj->fc_io_source, 1);
+ dispatch_source_merge_data(fcj->fc_mem_source, 1);
+ }
+ }
+ });
+ });
}
static void
@@ -1174,6 +1042,8 @@
fs->handler(NULL, FIREHOSE_SNAPSHOT_EVENT_COMPLETE, NULL);
server_config.fs_snapshot = NULL;
+ server_config.fs_mem_snapshot_started = false;
+ server_config.fs_io_snapshot_started = false;
dispatch_release(fs->fs_group);
Block_release(fs->handler);
@@ -1186,37 +1056,10 @@
static void
firehose_snapshot_gate(void *ctxt)
{
- firehose_snapshot_t fs = ctxt;
-
// prevent other snapshots from running until done
-
dispatch_suspend(server_config.fs_snapshot_gate_queue);
-
- server_config.fs_snapshot = fs;
- dispatch_group_async(fs->fs_group, server_config.fs_mem_drain_queue, ^{
- // start the fs_mem_snapshot, this is what triggers the snapshot
- // logic from _drain() or handle_death()
- fs->handler(NULL, FIREHOSE_SNAPSHOT_EVENT_MEM_START, NULL);
- firehose_snapshot_tickle_clients(fs, false);
-
- dispatch_group_async(fs->fs_group, server_config.fs_io_drain_queue, ^{
- // start the fs_io_snapshot, this is what triggers the snapshot
- // logic from _drain() or handle_death()
- // 29868879: must always happen after the memory snapshot started
- fs->handler(NULL, FIREHOSE_SNAPSHOT_EVENT_IO_START, NULL);
- firehose_snapshot_tickle_clients(fs, true);
-
-#if !TARGET_OS_SIMULATOR
- if (server_config.fs_kernel_client) {
- firehose_client_kernel_source_handle_event(
- server_config.fs_kernel_client);
- }
-#endif
- });
- });
-
- dispatch_group_notify_f(fs->fs_group, server_config.fs_io_drain_queue,
- fs, firehose_snapshot_finish);
+ dispatch_async_f(server_config.fs_io_drain_queue, ctxt,
+ firehose_snapshot_start);
}
void
@@ -1227,6 +1070,12 @@
snapshot->handler = Block_copy(handler);
snapshot->fs_group = dispatch_group_create();
+ // keep the group entered until IO_START and MEM_START have been sent
+ // See firehose_snapshot_start()
+ dispatch_group_enter(snapshot->fs_group);
+ dispatch_group_notify_f(snapshot->fs_group, server_config.fs_io_drain_queue,
+ snapshot, firehose_snapshot_finish);
+
dispatch_async_f(server_config.fs_snapshot_gate_queue, snapshot,
firehose_snapshot_gate);
}
@@ -1317,16 +1166,15 @@
if (expects_notifs && !fc->fc_use_notifs) {
fc->fc_use_notifs = true;
}
- firehose_client_wakeup(fc, pp, for_io);
+ firehose_client_push_async_merge(fc, pp, for_io);
}
return KERN_SUCCESS;
}
kern_return_t
-firehose_server_push_and_wait(mach_port_t server_port OS_UNUSED,
+firehose_server_push(mach_port_t server_port OS_UNUSED,
mach_port_t reply_port, qos_class_t qos, boolean_t for_io,
- firehose_push_reply_t *push_reply OS_UNUSED,
- boolean_t *quarantinedOut OS_UNUSED)
+ firehose_push_reply_t *push_reply OS_UNUSED)
{
firehose_client_t fc = cur_client_info;
dispatch_block_flags_t flags = DISPATCH_BLOCK_ENFORCE_QOS_CLASS;
@@ -1348,7 +1196,7 @@
}
block = dispatch_block_create_with_qos_class(flags, qos, 0, ^{
- firehose_client_drain_one(fc, reply_port,
+ firehose_client_drain(fc, reply_port,
for_io ? FIREHOSE_DRAIN_FOR_IO : 0);
});
dispatch_async(q, block);
diff --git a/src/firehose/firehose_server_internal.h b/src/firehose/firehose_server_internal.h
index 13f52b8..d805167 100644
--- a/src/firehose/firehose_server_internal.h
+++ b/src/firehose/firehose_server_internal.h
@@ -36,7 +36,6 @@
struct _os_object_s fc_as_os_object;
};
TAILQ_ENTRY(firehose_client_s) fc_entry;
- struct firehose_client_s *volatile fc_next[2];
firehose_buffer_t fc_buffer;
uint64_t volatile fc_mem_sent_flushed_pos;
@@ -44,27 +43,14 @@
uint64_t volatile fc_io_sent_flushed_pos;
uint64_t volatile fc_io_flushed_pos;
-#define FC_STATE_ENQUEUED(for_io) (0x0001u << (for_io))
-#define FC_STATE_MEM_ENQUEUED 0x0001
-#define FC_STATE_IO_ENQUEUED 0x0002
-
-#define FC_STATE_CANCELING(for_io) (0x0010u << (for_io))
-#define FC_STATE_MEM_CANCELING 0x0010
-#define FC_STATE_IO_CANCELING 0x0020
-
-#define FC_STATE_CANCELED(for_io) (0x0100u << (for_io))
-#define FC_STATE_MEM_CANCELED 0x0100
-#define FC_STATE_IO_CANCELED 0x0200
-#define FC_STATE_CANCELED_MASK 0x0300
-
- uintptr_t volatile fc_state;
-
void *volatile fc_ctxt;
union {
dispatch_mach_t fc_mach_channel;
dispatch_source_t fc_kernel_source;
};
+ dispatch_source_t fc_io_source;
+ dispatch_source_t fc_mem_source;
mach_port_t fc_recvp;
mach_port_t fc_sendp;
os_unfair_lock fc_lock;
@@ -75,7 +61,6 @@
bool fc_memory_corrupted;
bool fc_needs_io_snapshot;
bool fc_needs_mem_snapshot;
- bool fc_quarantined;
};
void
diff --git a/src/init.c b/src/init.c
index dea5e87..22a61e3 100644
--- a/src/init.c
+++ b/src/init.c
@@ -21,8 +21,6 @@
// Contains exported global data and initialization & other routines that must
// only exist once in the shared library even when resolvers are used.
-// NOTE: this file must not contain any atomic operations
-
#include "internal.h"
#if HAVE_MACH
@@ -148,6 +146,10 @@
#if DISPATCH_USE_KEVENT_WORKQUEUE && DISPATCH_USE_MGR_THREAD
int _dispatch_kevent_workqueue_enabled;
#endif
+#if DISPATCH_USE_EVFILT_MACHPORT_DIRECT && \
+ DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+int _dispatch_evfilt_machport_direct_enabled;
+#endif
DISPATCH_HW_CONFIG();
uint8_t _dispatch_unsafe_fork;
@@ -171,6 +173,33 @@
return _dispatch_child_of_unsafe_fork;
}
+DISPATCH_NOINLINE
+void
+_dispatch_fork_becomes_unsafe_slow(void)
+{
+ uint8_t value = os_atomic_or(&_dispatch_unsafe_fork,
+ _DISPATCH_UNSAFE_FORK_MULTITHREADED, relaxed);
+ if (value & _DISPATCH_UNSAFE_FORK_PROHIBIT) {
+ DISPATCH_CLIENT_CRASH(0, "Transition to multithreaded is prohibited");
+ }
+}
+
+DISPATCH_NOINLINE
+void
+_dispatch_prohibit_transition_to_multithreaded(bool prohibit)
+{
+ if (prohibit) {
+ uint8_t value = os_atomic_or(&_dispatch_unsafe_fork,
+ _DISPATCH_UNSAFE_FORK_PROHIBIT, relaxed);
+ if (value & _DISPATCH_UNSAFE_FORK_MULTITHREADED) {
+ DISPATCH_CLIENT_CRASH(0, "The executable is already multithreaded");
+ }
+ } else {
+ os_atomic_and(&_dispatch_unsafe_fork,
+ (uint8_t)~_DISPATCH_UNSAFE_FORK_PROHIBIT, relaxed);
+ }
+}
+
const struct dispatch_queue_offsets_s dispatch_queue_offsets = {
.dqo_version = 6,
.dqo_label = offsetof(struct dispatch_queue_s, dq_label),
@@ -209,10 +238,10 @@
.do_targetq = &_dispatch_root_queues[
DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT],
#endif
- .dq_state = DISPATCH_QUEUE_STATE_INIT_VALUE(1) |
- DISPATCH_QUEUE_ROLE_BASE_ANON,
+ .dq_state = DISPATCH_QUEUE_STATE_INIT_VALUE(1),
.dq_label = "com.apple.main-thread",
.dq_atomic_flags = DQF_THREAD_BOUND | DQF_CANNOT_TRYSYNC | DQF_WIDTH(1),
+ .dq_wlh = DISPATCH_WLH_GLOBAL, // TODO: main thread wlh
.dq_serialnum = 1,
};
@@ -397,7 +426,6 @@
.do_debug = dispatch_queue_debug,
);
-
DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_main, queue,
.do_type = DISPATCH_QUEUE_SERIAL_TYPE,
.do_kind = "main-queue",
@@ -421,7 +449,7 @@
DISPATCH_VTABLE_SUBCLASS_INSTANCE(queue_mgr, queue,
.do_type = DISPATCH_QUEUE_MGR_TYPE,
.do_kind = "mgr-queue",
- .do_push = _dispatch_mgr_queue_push,
+ .do_push = _dispatch_queue_push,
.do_invoke = _dispatch_mgr_thread,
.do_wakeup = _dispatch_mgr_queue_wakeup,
.do_debug = dispatch_queue_debug,
@@ -486,7 +514,6 @@
.do_kind = "data",
.do_dispose = _dispatch_data_dispose,
.do_debug = _dispatch_data_debug,
- .do_set_targetq = (void*)_dispatch_data_set_target_queue,
);
#endif
@@ -525,41 +552,6 @@
}
#pragma mark -
-#pragma mark dispatch_data globals
-
-const dispatch_block_t _dispatch_data_destructor_free = ^{
- DISPATCH_INTERNAL_CRASH(0, "free destructor called");
-};
-
-const dispatch_block_t _dispatch_data_destructor_none = ^{
- DISPATCH_INTERNAL_CRASH(0, "none destructor called");
-};
-
-#if !HAVE_MACH
-const dispatch_block_t _dispatch_data_destructor_munmap = ^{
- DISPATCH_INTERNAL_CRASH(0, "munmap destructor called");
-};
-#else
-// _dispatch_data_destructor_munmap is a linker alias to the following
-const dispatch_block_t _dispatch_data_destructor_vm_deallocate = ^{
- DISPATCH_INTERNAL_CRASH(0, "vmdeallocate destructor called");
-};
-#endif
-
-const dispatch_block_t _dispatch_data_destructor_inline = ^{
- DISPATCH_INTERNAL_CRASH(0, "inline destructor called");
-};
-
-struct dispatch_data_s _dispatch_data_empty = {
-#if DISPATCH_DATA_IS_BRIDGED_TO_NSDATA
- .do_vtable = DISPATCH_DATA_EMPTY_CLASS,
-#else
- DISPATCH_GLOBAL_OBJECT_HEADER(data),
- .do_next = DISPATCH_OBJECT_LISTLESS,
-#endif
-};
-
-#pragma mark -
#pragma mark dispatch_bug
static char _dispatch_build[16];
@@ -1155,17 +1147,16 @@
}
}
-void
-_dispatch_last_resort_autorelease_pool_push(dispatch_invoke_context_t dic)
+void*
+_dispatch_last_resort_autorelease_pool_push(void)
{
- dic->dic_autorelease_pool = _dispatch_autorelease_pool_push();
+ return _dispatch_autorelease_pool_push();
}
void
-_dispatch_last_resort_autorelease_pool_pop(dispatch_invoke_context_t dic)
+_dispatch_last_resort_autorelease_pool_pop(void *pool)
{
- _dispatch_autorelease_pool_pop(dic->dic_autorelease_pool);
- dic->dic_autorelease_pool = NULL;
+ _dispatch_autorelease_pool_pop(pool);
}
#endif // DISPATCH_COCOA_COMPAT
@@ -1208,16 +1199,22 @@
_dispatch_mach_notify_port_destroyed(mach_port_t notify DISPATCH_UNUSED,
mach_port_t name)
{
- DISPATCH_INTERNAL_CRASH(name, "unexpected receipt of port-destroyed");
- return KERN_FAILURE;
+ kern_return_t kr;
+ // this function should never be called
+ (void)dispatch_assume_zero(name);
+ kr = mach_port_mod_refs(mach_task_self(), name, MACH_PORT_RIGHT_RECEIVE,-1);
+ DISPATCH_VERIFY_MIG(kr);
+ (void)dispatch_assume_zero(kr);
+ return KERN_SUCCESS;
}
kern_return_t
-_dispatch_mach_notify_no_senders(mach_port_t notify DISPATCH_UNUSED,
- mach_port_mscount_t mscnt)
+_dispatch_mach_notify_no_senders(mach_port_t notify,
+ mach_port_mscount_t mscnt DISPATCH_UNUSED)
{
- DISPATCH_INTERNAL_CRASH(mscnt, "unexpected receipt of no-more-senders");
- return KERN_FAILURE;
+ // this function should never be called
+ (void)dispatch_assume_zero(notify);
+ return KERN_SUCCESS;
}
kern_return_t
diff --git a/src/inline_internal.h b/src/inline_internal.h
index 0ed9e51..53548ed 100644
--- a/src/inline_internal.h
+++ b/src/inline_internal.h
@@ -101,13 +101,6 @@
DISPATCH_ALWAYS_INLINE
static inline bool
-_dispatch_object_is_queue(dispatch_object_t dou)
-{
- return _dispatch_object_has_vtable(dou) && dx_vtable(dou._do)->do_push;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
_dispatch_object_is_continuation(dispatch_object_t dou)
{
if (_dispatch_object_has_vtable(dou)) {
@@ -174,9 +167,9 @@
DISPATCH_ALWAYS_INLINE
static inline _os_object_t
-_os_object_retain_internal_n_inline(_os_object_t obj, int n)
+_os_object_retain_internal_inline(_os_object_t obj)
{
- int ref_cnt = _os_object_refcnt_add(obj, n);
+ int ref_cnt = _os_object_refcnt_inc(obj);
if (unlikely(ref_cnt <= 0)) {
_OS_OBJECT_CLIENT_CRASH("Resurrection of an object");
}
@@ -185,20 +178,23 @@
DISPATCH_ALWAYS_INLINE
static inline void
-_os_object_release_internal_n_no_dispose_inline(_os_object_t obj, int n)
+_os_object_release_internal_inline_no_dispose(_os_object_t obj)
{
- int ref_cnt = _os_object_refcnt_sub(obj, n);
+ int ref_cnt = _os_object_refcnt_dec(obj);
if (likely(ref_cnt >= 0)) {
return;
}
+ if (ref_cnt == 0) {
+ _OS_OBJECT_CLIENT_CRASH("Unexpected release of an object");
+ }
_OS_OBJECT_CLIENT_CRASH("Over-release of an object");
}
DISPATCH_ALWAYS_INLINE
static inline void
-_os_object_release_internal_n_inline(_os_object_t obj, int n)
+_os_object_release_internal_inline(_os_object_t obj)
{
- int ref_cnt = _os_object_refcnt_sub(obj, n);
+ int ref_cnt = _os_object_refcnt_dec(obj);
if (likely(ref_cnt >= 0)) {
return;
}
@@ -220,56 +216,14 @@
static inline void
_dispatch_retain(dispatch_object_t dou)
{
- (void)_os_object_retain_internal_n_inline(dou._os_obj, 1);
-}
-
-DISPATCH_ALWAYS_INLINE_NDEBUG
-static inline void
-_dispatch_retain_2(dispatch_object_t dou)
-{
- (void)_os_object_retain_internal_n_inline(dou._os_obj, 2);
-}
-
-DISPATCH_ALWAYS_INLINE_NDEBUG
-static inline void
-_dispatch_retain_n(dispatch_object_t dou, int n)
-{
- (void)_os_object_retain_internal_n_inline(dou._os_obj, n);
+ (void)_os_object_retain_internal_inline(dou._os_obj);
}
DISPATCH_ALWAYS_INLINE_NDEBUG
static inline void
_dispatch_release(dispatch_object_t dou)
{
- _os_object_release_internal_n_inline(dou._os_obj, 1);
-}
-
-DISPATCH_ALWAYS_INLINE_NDEBUG
-static inline void
-_dispatch_release_2(dispatch_object_t dou)
-{
- _os_object_release_internal_n_inline(dou._os_obj, 2);
-}
-
-DISPATCH_ALWAYS_INLINE_NDEBUG
-static inline void
-_dispatch_release_n(dispatch_object_t dou, int n)
-{
- _os_object_release_internal_n_inline(dou._os_obj, n);
-}
-
-DISPATCH_ALWAYS_INLINE_NDEBUG
-static inline void
-_dispatch_release_no_dispose(dispatch_object_t dou)
-{
- _os_object_release_internal_n_no_dispose_inline(dou._os_obj, 1);
-}
-
-DISPATCH_ALWAYS_INLINE_NDEBUG
-static inline void
-_dispatch_release_2_no_dispose(dispatch_object_t dou)
-{
- _os_object_release_internal_n_no_dispose_inline(dou._os_obj, 2);
+ _os_object_release_internal_inline(dou._os_obj);
}
DISPATCH_ALWAYS_INLINE_NDEBUG
@@ -279,42 +233,6 @@
_os_object_release_internal(dou._os_obj);
}
-DISPATCH_ALWAYS_INLINE_NDEBUG
-static inline void
-_dispatch_release_2_tailcall(dispatch_object_t dou)
-{
- _os_object_release_internal_n(dou._os_obj, 2);
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_queue_retain_storage(dispatch_queue_t dq)
-{
- int ref_cnt = os_atomic_inc2o(dq, dq_sref_cnt, relaxed);
- if (unlikely(ref_cnt <= 0)) {
- _OS_OBJECT_CLIENT_CRASH("Resurrection of an object");
- }
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_queue_release_storage(dispatch_queue_t dq)
-{
- // this refcount only delays the _dispatch_object_dealloc() and there's no
- // need for visibility wrt to the allocation, the internal refcount already
- // gives us that, and the object becomes immutable after the last internal
- // refcount release.
- int ref_cnt = os_atomic_dec2o(dq, dq_sref_cnt, relaxed);
- if (unlikely(ref_cnt >= 0)) {
- return;
- }
- if (unlikely(ref_cnt < -1)) {
- _OS_OBJECT_CLIENT_CRASH("Over-release of an object");
- }
- dq->dq_state = 0xdead000000000000;
- _dispatch_object_dealloc(dq);
-}
-
DISPATCH_ALWAYS_INLINE DISPATCH_NONNULL_ALL
static inline void
_dispatch_object_set_target_queue_inline(dispatch_object_t dou,
@@ -656,113 +574,6 @@
return _dispatch_queue_atomic_flags(dq) & DQF_LEGACY;
}
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_wlh_retain(dispatch_wlh_t wlh)
-{
- if (wlh && wlh != DISPATCH_WLH_ANON) {
- _dispatch_queue_retain_storage((dispatch_queue_t)wlh);
- }
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_wlh_release(dispatch_wlh_t wlh)
-{
- if (wlh && wlh != DISPATCH_WLH_ANON) {
- _dispatch_queue_release_storage((dispatch_queue_t)wlh);
- }
-}
-
-#define DISPATCH_WLH_STORAGE_REF 1ul
-
-DISPATCH_ALWAYS_INLINE DISPATCH_PURE
-static inline dispatch_wlh_t
-_dispatch_get_wlh(void)
-{
- return _dispatch_thread_getspecific(dispatch_wlh_key);
-}
-
-DISPATCH_ALWAYS_INLINE DISPATCH_PURE
-static inline dispatch_wlh_t
-_dispatch_get_wlh_reference(void)
-{
- dispatch_wlh_t wlh = _dispatch_thread_getspecific(dispatch_wlh_key);
- if (wlh != DISPATCH_WLH_ANON) {
- wlh = (dispatch_wlh_t)((uintptr_t)wlh & ~DISPATCH_WLH_STORAGE_REF);
- }
- return wlh;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dispatch_adopt_wlh_anon_recurse(void)
-{
- dispatch_wlh_t cur_wlh = _dispatch_get_wlh_reference();
- if (cur_wlh == DISPATCH_WLH_ANON) return false;
- _dispatch_debug("wlh[anon]: set current (releasing %p)", cur_wlh);
- _dispatch_wlh_release(cur_wlh);
- _dispatch_thread_setspecific(dispatch_wlh_key, (void *)DISPATCH_WLH_ANON);
- return true;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_adopt_wlh_anon(void)
-{
- if (unlikely(!_dispatch_adopt_wlh_anon_recurse())) {
- DISPATCH_INTERNAL_CRASH(0, "Lingering DISPATCH_WLH_ANON");
- }
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_adopt_wlh(dispatch_wlh_t wlh)
-{
- dispatch_wlh_t cur_wlh = _dispatch_get_wlh_reference();
- _dispatch_debug("wlh[%p]: adopt current (releasing %p)", wlh, cur_wlh);
- if (cur_wlh == DISPATCH_WLH_ANON) {
- DISPATCH_INTERNAL_CRASH(0, "Lingering DISPATCH_WLH_ANON");
- }
- if (cur_wlh != wlh) {
- dispatch_assert(wlh);
- _dispatch_wlh_release(cur_wlh);
- _dispatch_wlh_retain(wlh);
- }
- _dispatch_thread_setspecific(dispatch_wlh_key, (void *)wlh);
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_preserve_wlh_storage_reference(dispatch_wlh_t wlh)
-{
- dispatch_assert(wlh != DISPATCH_WLH_ANON);
- dispatch_assert(wlh == _dispatch_get_wlh());
- _dispatch_thread_setspecific(dispatch_wlh_key,
- (void *)((uintptr_t)wlh | DISPATCH_WLH_STORAGE_REF));
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_reset_wlh(void)
-{
- dispatch_assert(_dispatch_get_wlh() == DISPATCH_WLH_ANON);
- _dispatch_debug("wlh[anon]: clear current");
- _dispatch_thread_setspecific(dispatch_wlh_key, NULL);
- _dispatch_clear_return_to_kernel();
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dispatch_wlh_should_poll_unote(dispatch_unote_t du)
-{
- if (likely(_dispatch_needs_to_return_to_kernel())) {
- dispatch_wlh_t wlh = _dispatch_get_wlh();
- return wlh != DISPATCH_WLH_ANON && du._du->du_wlh == wlh;
- }
- return false;
-}
-
#endif // DISPATCH_PURE_C
#ifndef __cplusplus
@@ -867,67 +678,16 @@
DISPATCH_ALWAYS_INLINE
static inline bool
-_dq_state_is_base_wlh(uint64_t dq_state)
-{
- return dq_state & DISPATCH_QUEUE_ROLE_BASE_WLH;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dq_state_is_base_anon(uint64_t dq_state)
-{
- return dq_state & DISPATCH_QUEUE_ROLE_BASE_ANON;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dq_state_is_inner_queue(uint64_t dq_state)
-{
- return (dq_state & DISPATCH_QUEUE_ROLE_MASK) == DISPATCH_QUEUE_ROLE_INNER;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
_dq_state_is_enqueued(uint64_t dq_state)
{
- return dq_state & (DISPATCH_QUEUE_ENQUEUED|DISPATCH_QUEUE_ENQUEUED_ON_MGR);
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dq_state_is_enqueued_on_target(uint64_t dq_state)
-{
return dq_state & DISPATCH_QUEUE_ENQUEUED;
}
DISPATCH_ALWAYS_INLINE
static inline bool
-_dq_state_is_enqueued_on_manager(uint64_t dq_state)
-{
- return dq_state & DISPATCH_QUEUE_ENQUEUED_ON_MGR;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dq_state_in_sync_transfer(uint64_t dq_state)
-{
- return dq_state & DISPATCH_QUEUE_SYNC_TRANSFER;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
_dq_state_received_override(uint64_t dq_state)
{
- return _dq_state_is_base_anon(dq_state) &&
- (dq_state & DISPATCH_QUEUE_RECEIVED_OVERRIDE);
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dq_state_received_sync_wait(uint64_t dq_state)
-{
- return _dq_state_is_base_wlh(dq_state) &&
- (dq_state & DISPATCH_QUEUE_RECEIVED_SYNC_WAIT);
+ return dq_state & DISPATCH_QUEUE_RECEIVED_OVERRIDE;
}
DISPATCH_ALWAYS_INLINE
@@ -952,16 +712,13 @@
uint64_t qos_bits = _dq_state_from_qos(qos);
if ((dq_state & DISPATCH_QUEUE_MAX_QOS_MASK) < qos_bits) {
dq_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
- dq_state |= qos_bits;
- if (unlikely(_dq_state_is_base_anon(dq_state))) {
- dq_state |= DISPATCH_QUEUE_RECEIVED_OVERRIDE;
- }
+ dq_state |= qos_bits | DISPATCH_QUEUE_RECEIVED_OVERRIDE;
}
return dq_state;
}
DISPATCH_ALWAYS_INLINE
-static inline dispatch_tid
+static inline dispatch_lock_owner
_dq_state_drain_owner(uint64_t dq_state)
{
return _dispatch_lock_owner((dispatch_lock)dq_state);
@@ -971,23 +728,33 @@
DISPATCH_ALWAYS_INLINE
static inline bool
-_dq_state_drain_locked_by(uint64_t dq_state, dispatch_tid tid)
+_dq_state_drain_pended(uint64_t dq_state)
{
- return _dispatch_lock_is_locked_by((dispatch_lock)dq_state, tid);
+ return (dq_state & DISPATCH_QUEUE_DRAIN_PENDED);
}
DISPATCH_ALWAYS_INLINE
static inline bool
-_dq_state_drain_locked_by_self(uint64_t dq_state)
+_dq_state_drain_locked_by(uint64_t dq_state, uint32_t owner)
{
- return _dispatch_lock_is_locked_by_self((dispatch_lock)dq_state);
+ if (_dq_state_drain_pended(dq_state)) {
+ return false;
+ }
+ return _dq_state_drain_owner(dq_state) == owner;
}
DISPATCH_ALWAYS_INLINE
static inline bool
_dq_state_drain_locked(uint64_t dq_state)
{
- return _dispatch_lock_is_locked((dispatch_lock)dq_state);
+ return (dq_state & DISPATCH_QUEUE_DRAIN_OWNER_MASK) != 0;
+}
+
+DISPATCH_ALWAYS_INLINE
+static inline bool
+_dq_state_has_waiters(uint64_t dq_state)
+{
+ return _dispatch_lock_has_waiters((dispatch_lock)dq_state);
}
DISPATCH_ALWAYS_INLINE
@@ -1006,25 +773,17 @@
DISPATCH_ALWAYS_INLINE
static inline bool
-_dq_state_should_override(uint64_t dq_state)
+_dq_state_should_wakeup(uint64_t dq_state)
{
- if (_dq_state_is_suspended(dq_state) ||
- _dq_state_is_enqueued_on_manager(dq_state)) {
- return false;
- }
- if (_dq_state_is_enqueued_on_target(dq_state)) {
- return true;
- }
- if (_dq_state_is_base_wlh(dq_state)) {
- return false;
- }
- return _dq_state_drain_locked(dq_state);
+ return _dq_state_is_runnable(dq_state) &&
+ !_dq_state_is_enqueued(dq_state) &&
+ !_dq_state_drain_locked(dq_state);
}
-
#endif // __cplusplus
#pragma mark -
#pragma mark dispatch_queue_t state machine
+#ifndef __cplusplus
static inline pthread_priority_t _dispatch_get_priority(void);
static inline dispatch_priority_t _dispatch_get_basepri(void);
@@ -1032,29 +791,43 @@
static inline void _dispatch_set_basepri_override_qos(dispatch_qos_t qos);
static inline void _dispatch_reset_basepri(dispatch_priority_t dbp);
static inline dispatch_priority_t _dispatch_set_basepri(dispatch_priority_t dbp);
+
static inline bool _dispatch_queue_need_override_retain(
dispatch_queue_class_t dqu, dispatch_qos_t qos);
+DISPATCH_ALWAYS_INLINE
+static inline void
+_dispatch_queue_xref_dispose(struct dispatch_queue_s *dq)
+{
+ uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed);
+ if (unlikely(_dq_state_is_suspended(dq_state))) {
+ long state = (long)dq_state;
+ if (sizeof(long) < sizeof(uint64_t)) state = (long)(dq_state >> 32);
+ if (unlikely(_dq_state_is_inactive(dq_state))) {
+ // Arguments for and against this assert are within 6705399
+ DISPATCH_CLIENT_CRASH(state, "Release of an inactive object");
+ }
+ DISPATCH_CLIENT_CRASH(dq_state, "Release of a suspended object");
+ }
+ os_atomic_or2o(dq, dq_atomic_flags, DQF_RELEASED, relaxed);
+}
+
+#endif
#if DISPATCH_PURE_C
// Note to later developers: ensure that any initialization changes are
// made for statically allocated queues (i.e. _dispatch_main_q).
static inline void
_dispatch_queue_init(dispatch_queue_t dq, dispatch_queue_flags_t dqf,
- uint16_t width, uint64_t initial_state_bits)
+ uint16_t width, bool inactive)
{
uint64_t dq_state = DISPATCH_QUEUE_STATE_INIT_VALUE(width);
- dispatch_assert((initial_state_bits & ~(DISPATCH_QUEUE_ROLE_MASK |
- DISPATCH_QUEUE_INACTIVE)) == 0);
-
- if (initial_state_bits & DISPATCH_QUEUE_INACTIVE) {
- dq_state |= DISPATCH_QUEUE_INACTIVE + DISPATCH_QUEUE_NEEDS_ACTIVATION;
- dq_state |= DLOCK_OWNER_MASK;
- dq->do_ref_cnt += 2; // rdar://8181908 see _dispatch_queue_resume
+ if (inactive) {
+ dq_state += DISPATCH_QUEUE_INACTIVE + DISPATCH_QUEUE_NEEDS_ACTIVATION;
+ dq_state += DLOCK_OWNER_INVALID;
+ dq->do_ref_cnt++; // rdar://8181908 see _dispatch_queue_resume
}
-
- dq_state |= (initial_state_bits & DISPATCH_QUEUE_ROLE_MASK);
dq->do_next = (struct dispatch_queue_s *)DISPATCH_OBJECT_LISTLESS;
dqf |= DQF_WIDTH(width);
os_atomic_store2o(dq, dq_atomic_flags, dqf, relaxed);
@@ -1096,13 +869,8 @@
return true;
}
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dq_state_needs_lock_override(uint64_t dq_state, dispatch_qos_t qos)
-{
- return _dq_state_is_base_anon(dq_state) &&
- qos < _dq_state_max_qos(dq_state);
-}
+#define _dispatch_queue_should_override_self(dq_state, qos) \
+ unlikely(qos < _dq_state_max_qos(dq_state))
DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
@@ -1116,139 +884,79 @@
return qos;
}
+/* Used by:
+ * - _dispatch_queue_class_invoke (normal path)
+ * - _dispatch_queue_override_invoke (stealer)
+ *
+ * Initial state must be { sc:0, ib:0, qf:0, dl:0 }
+ * Final state forces { dl:self, qf:1, d: 0 }
+ * ib:1 is forced when the width acquired is equivalent to the barrier width
+ */
DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
static inline uint64_t
_dispatch_queue_drain_try_lock(dispatch_queue_t dq,
- dispatch_invoke_flags_t flags)
+ dispatch_invoke_flags_t flags, uint64_t *dq_state)
{
uint64_t pending_barrier_width =
(dq->dq_width - 1) * DISPATCH_QUEUE_WIDTH_INTERVAL;
- uint64_t set_owner_and_set_full_width =
- _dispatch_lock_value_for_self() | DISPATCH_QUEUE_WIDTH_FULL_BIT;
- uint64_t lock_fail_mask, old_state, new_state, dequeue_mask;
-
- // same as !_dq_state_is_runnable()
- lock_fail_mask = ~(DISPATCH_QUEUE_WIDTH_FULL_BIT - 1);
- // same as _dq_state_drain_locked()
- lock_fail_mask |= DISPATCH_QUEUE_DRAIN_OWNER_MASK;
+ uint64_t xor_owner_and_set_full_width =
+ _dispatch_tid_self() | DISPATCH_QUEUE_WIDTH_FULL_BIT;
+ uint64_t clear_enqueued_bit, old_state, new_state;
if (flags & DISPATCH_INVOKE_STEALING) {
- lock_fail_mask |= DISPATCH_QUEUE_ENQUEUED_ON_MGR;
- dequeue_mask = 0;
- } else if (flags & DISPATCH_INVOKE_MANAGER_DRAIN) {
- dequeue_mask = DISPATCH_QUEUE_ENQUEUED_ON_MGR;
+ clear_enqueued_bit = 0;
} else {
- lock_fail_mask |= DISPATCH_QUEUE_ENQUEUED_ON_MGR;
- dequeue_mask = DISPATCH_QUEUE_ENQUEUED;
+ clear_enqueued_bit = DISPATCH_QUEUE_ENQUEUED;
}
- dispatch_assert(!(flags & DISPATCH_INVOKE_WLH));
dispatch_qos_t oq_floor = _dispatch_get_basepri_override_qos_floor();
retry:
os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, acquire, {
new_state = old_state;
- if (likely(!(old_state & lock_fail_mask))) {
- if (unlikely(_dq_state_needs_lock_override(old_state, oq_floor))) {
+ new_state ^= clear_enqueued_bit;
+ if (likely(_dq_state_is_runnable(old_state) &&
+ !_dq_state_drain_locked(old_state))) {
+ if (_dispatch_queue_should_override_self(old_state, oq_floor)) {
os_atomic_rmw_loop_give_up({
oq_floor = _dispatch_queue_override_self(old_state);
goto retry;
});
}
//
- // Only keep the HAS_WAITER, MAX_QOS and ENQUEUED bits
+ // Only keep the HAS_WAITER, MAX_QOS and ENQUEUED (if stealing) bits
// In particular acquiring the drain lock clears the DIRTY and
- // RECEIVED_OVERRIDE bits.
+ // RECEIVED_OVERRIDE
//
new_state &= DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK;
- new_state |= set_owner_and_set_full_width;
+ //
+ // For the NOWAITERS_BIT case, the thread identity
+ // has NOWAITERS_BIT set, and NOWAITERS_BIT was kept above,
+ // so the xor below flips the NOWAITERS_BIT to 0 as expected.
+ //
+ // For the non inverted WAITERS_BIT case, WAITERS_BIT is not set in
+ // the thread identity, and the xor leaves the bit alone.
+ //
+ new_state ^= xor_owner_and_set_full_width;
if (_dq_state_has_pending_barrier(old_state) ||
old_state + pending_barrier_width <
DISPATCH_QUEUE_WIDTH_FULL_BIT) {
new_state |= DISPATCH_QUEUE_IN_BARRIER;
}
- } else if (dequeue_mask) {
- // dequeue_mask is in a register, xor yields better assembly
- new_state ^= dequeue_mask;
- } else {
+ } else if (!clear_enqueued_bit) {
os_atomic_rmw_loop_give_up(break);
}
});
- dispatch_assert((old_state & dequeue_mask) == dequeue_mask);
- if (likely(!(old_state & lock_fail_mask))) {
- new_state &= DISPATCH_QUEUE_IN_BARRIER | DISPATCH_QUEUE_WIDTH_FULL_BIT |
- dequeue_mask;
+ if (dq_state) *dq_state = new_state;
+ if (likely(_dq_state_is_runnable(old_state) &&
+ !_dq_state_drain_locked(old_state))) {
+ new_state &= DISPATCH_QUEUE_IN_BARRIER | DISPATCH_QUEUE_WIDTH_FULL_BIT;
old_state &= DISPATCH_QUEUE_WIDTH_MASK;
return new_state - old_state;
}
return 0;
}
-DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT
-static inline bool
-_dispatch_queue_drain_try_lock_wlh(dispatch_queue_t dq, uint64_t *dq_state)
-{
- uint64_t old_state, new_state;
- uint64_t lock_bits = _dispatch_lock_value_for_self() |
- DISPATCH_QUEUE_WIDTH_FULL_BIT | DISPATCH_QUEUE_IN_BARRIER;
-
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, acquire, {
- new_state = old_state;
- if (unlikely(_dq_state_is_suspended(old_state))) {
- os_atomic_rmw_loop_give_up(break);
- } else if (unlikely(_dq_state_drain_locked(old_state))) {
- os_atomic_rmw_loop_give_up(break);
- } else {
- new_state &= DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK;
- new_state |= lock_bits;
- }
- });
- if (unlikely(!_dq_state_is_base_wlh(old_state) ||
- !_dq_state_is_enqueued_on_target(old_state) ||
- _dq_state_is_enqueued_on_manager(old_state))) {
-#if !__LP64__
- old_state >>= 32;
-#endif
- DISPATCH_INTERNAL_CRASH(old_state, "Invalid wlh state");
- }
-
- if (dq_state) *dq_state = new_state;
- return !_dq_state_is_suspended(old_state) &&
- !_dq_state_drain_locked(old_state);
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_queue_mgr_lock(dispatch_queue_t dq)
-{
- uint64_t old_state, new_state, set_owner_and_set_full_width =
- _dispatch_lock_value_for_self() | DISPATCH_QUEUE_SERIAL_DRAIN_OWNED;
-
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, acquire, {
- new_state = old_state;
- if (unlikely(!_dq_state_is_runnable(old_state) ||
- _dq_state_drain_locked(old_state))) {
- DISPATCH_INTERNAL_CRASH((uintptr_t)old_state,
- "Locking the manager should not fail");
- }
- new_state &= DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK;
- new_state |= set_owner_and_set_full_width;
- });
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline bool
-_dispatch_queue_mgr_unlock(dispatch_queue_t dq)
-{
- uint64_t old_state, new_state;
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
- new_state = old_state - DISPATCH_QUEUE_SERIAL_DRAIN_OWNED;
- new_state &= ~DISPATCH_QUEUE_DRAIN_UNLOCK_MASK;
- new_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
- });
- return _dq_state_is_dirty(old_state);
-}
-
/* Used by _dispatch_barrier_{try,}sync
*
* Note, this fails if any of e:1 or dl!=0, but that allows this code to be a
@@ -1264,18 +972,11 @@
static inline bool
_dispatch_queue_try_acquire_barrier_sync(dispatch_queue_t dq, uint32_t tid)
{
- uint64_t init = DISPATCH_QUEUE_STATE_INIT_VALUE(dq->dq_width);
- uint64_t value = DISPATCH_QUEUE_WIDTH_FULL_BIT | DISPATCH_QUEUE_IN_BARRIER |
- _dispatch_lock_value_from_tid(tid);
- uint64_t old_state, new_state;
+ uint64_t value = DISPATCH_QUEUE_WIDTH_FULL_BIT | DISPATCH_QUEUE_IN_BARRIER;
+ value |= tid;
- return os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, acquire, {
- uint64_t role = old_state & DISPATCH_QUEUE_ROLE_MASK;
- if (old_state != (init | role)) {
- os_atomic_rmw_loop_give_up(break);
- }
- new_state = value | role;
- });
+ return os_atomic_cmpxchg2o(dq, dq_state,
+ DISPATCH_QUEUE_STATE_INIT_VALUE(dq->dq_width), value, acquire);
}
/* Used by _dispatch_sync for root queues and some drain codepaths
@@ -1448,13 +1149,18 @@
static inline bool
_dispatch_queue_drain_try_unlock(dispatch_queue_t dq, uint64_t owned, bool done)
{
- uint64_t old_state, new_state;
+ uint64_t old_state = os_atomic_load2o(dq, dq_state, relaxed);
+ uint64_t new_state;
os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
- new_state = old_state - owned;
- new_state &= ~DISPATCH_QUEUE_DRAIN_UNLOCK_MASK;
- if (unlikely(_dq_state_is_suspended(old_state))) {
- new_state |= DLOCK_OWNER_MASK;
+ new_state = old_state - owned;
+ if (unlikely(_dq_state_is_suspended(new_state))) {
+#ifdef DLOCK_NOWAITERS_BIT
+ new_state = new_state | DISPATCH_QUEUE_DRAIN_OWNER_MASK;
+#else
+ new_state = new_state | DLOCK_OWNER_INVALID;
+#endif
+ new_state |= DISPATCH_QUEUE_DIRTY;
} else if (unlikely(_dq_state_is_dirty(old_state))) {
os_atomic_rmw_loop_give_up({
// just renew the drain lock with an acquire barrier, to see
@@ -1465,8 +1171,11 @@
return false;
});
} else if (likely(done)) {
+ new_state &= ~DISPATCH_QUEUE_DRAIN_UNLOCK_MASK;
+ new_state &= ~DISPATCH_QUEUE_RECEIVED_OVERRIDE;
new_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
} else {
+ new_state = DISPATCH_QUEUE_DRAIN_UNLOCK(new_state);
new_state |= DISPATCH_QUEUE_DIRTY;
}
});
@@ -1478,6 +1187,80 @@
return true;
}
+/* Used to transfer the drain lock to a next thread, because it is known
+ * and that the dirty-head check isn't needed.
+ *
+ * This releases `owned`, clears DIRTY, and handles overrides when seen.
+ */
+DISPATCH_ALWAYS_INLINE
+static inline void
+_dispatch_queue_drain_transfer_lock(dispatch_queue_t dq,
+ uint64_t owned, dispatch_object_t dou)
+{
+ uint64_t old_state, new_state;
+ mach_port_t next_owner = 0;
+ if (dou._dc->dc_flags & DISPATCH_OBJ_BARRIER_BIT) {
+ next_owner = (mach_port_t)dou._dc->dc_data;
+ }
+
+#ifdef DLOCK_NOWAITERS_BIT
+ // The NOWAITERS_BIT state must not change through the transfer. It means
+ // that if next_owner is 0 the bit must be flipped in the rmw_loop below,
+ // and if next_owner is set, then the bit must be left unchanged.
+ //
+ // - when next_owner is 0, the xor below sets NOWAITERS_BIT in next_owner,
+ // which causes the second xor to flip the bit as expected.
+ // - if next_owner is not 0, it has the NOWAITERS_BIT set, so we have to
+ // clear it so that the second xor leaves the NOWAITERS_BIT alone.
+ next_owner ^= DLOCK_NOWAITERS_BIT;
+#endif
+ os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
+ new_state = old_state - owned;
+ // same as DISPATCH_QUEUE_DRAIN_UNLOCK
+ // but we want to be more efficient wrt the WAITERS_BIT
+ new_state &= ~DISPATCH_QUEUE_DRAIN_OWNER_MASK;
+ new_state &= ~DISPATCH_QUEUE_DRAIN_PENDED;
+ new_state &= ~DISPATCH_QUEUE_RECEIVED_OVERRIDE;
+ new_state &= ~DISPATCH_QUEUE_DIRTY;
+ new_state ^= next_owner;
+ });
+ if (_dq_state_received_override(old_state)) {
+ // Ensure that the root queue sees that this thread was overridden.
+ _dispatch_set_basepri_override_qos(_dq_state_max_qos(old_state));
+ }
+}
+
+/* Used to forcefully unlock the drain lock, bypassing the dirty bit check.
+ * This usually is followed by a wakeup to re-evaluate the state machine
+ * of the queue/source.
+ *
+ * This releases `owned`, clears DIRTY, and handles overrides when seen.
+ */
+DISPATCH_ALWAYS_INLINE
+static inline uint64_t
+_dispatch_queue_drain_unlock(dispatch_queue_t dq, uint64_t owned)
+{
+ uint64_t old_state, new_state;
+
+ os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
+ new_state = old_state - owned;
+ // same as DISPATCH_QUEUE_DRAIN_UNLOCK
+ // but we want to be more efficient wrt the WAITERS_BIT
+#ifdef DLOCK_NOWAITERS_BIT
+ new_state ^= DLOCK_NOWAITERS_BIT;
+#endif
+ new_state &= ~DISPATCH_QUEUE_DRAIN_OWNER_MASK;
+ new_state &= ~DISPATCH_QUEUE_DRAIN_PENDED;
+ new_state &= ~DISPATCH_QUEUE_RECEIVED_OVERRIDE;
+ });
+
+ if (_dq_state_received_override(old_state)) {
+ // Ensure that the root queue sees that this thread was overridden.
+ _dispatch_set_basepri_override_qos(_dq_state_max_qos(old_state));
+ }
+ return old_state;
+}
+
#pragma mark -
#pragma mark os_mpsc_queue
@@ -1585,7 +1368,7 @@
static inline bool
_dispatch_queue_sidelock_trylock(dispatch_queue_t dq, dispatch_qos_t qos)
{
- dispatch_tid owner;
+ dispatch_lock_owner owner;
if (_dispatch_unfair_lock_trylock(&dq->dq_sidelock, &owner)) {
return true;
}
@@ -1716,11 +1499,11 @@
// queue when invoked by _dispatch_queue_drain. <rdar://problem/6932776>
bool overriding = _dispatch_queue_need_override_retain(dq, qos);
if (unlikely(_dispatch_queue_push_update_tail(dq, tail))) {
- if (!overriding) _dispatch_retain_2(dq->_as_os_obj);
+ if (!overriding) _dispatch_retain(dq);
_dispatch_queue_push_update_head(dq, tail);
- flags = DISPATCH_WAKEUP_CONSUME_2 | DISPATCH_WAKEUP_MAKE_DIRTY;
+ flags = DISPATCH_WAKEUP_CONSUME | DISPATCH_WAKEUP_FLUSH;
} else if (overriding) {
- flags = DISPATCH_WAKEUP_CONSUME_2;
+ flags = DISPATCH_WAKEUP_CONSUME | DISPATCH_WAKEUP_OVERRIDING;
} else {
return;
}
@@ -1728,14 +1511,6 @@
}
DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_queue_push_queue(dispatch_queue_t tq, dispatch_queue_t dq,
- uint64_t dq_state)
-{
- return dx_push(tq, dq, _dq_state_max_qos(dq_state));
-}
-
-DISPATCH_ALWAYS_INLINE
static inline dispatch_priority_t
_dispatch_root_queue_identity_assume(dispatch_queue_t assumed_rq)
{
@@ -1746,6 +1521,30 @@
return old_dbp;
}
+DISPATCH_ALWAYS_INLINE
+static inline bool
+_dispatch_root_queue_allows_wlh_for_queue(dispatch_queue_t rq,
+ dispatch_queue_class_t dqu)
+{
+ // This will discard:
+ // - queues already tagged with the global wlh
+ // - concurrent queues (width != 1)
+ // - non overcommit queues, which includes pthread root queues.
+ return dqu._dq->dq_wlh != DISPATCH_WLH_GLOBAL && dqu._dq->dq_width == 1 &&
+ (rq->dq_priority & DISPATCH_PRIORITY_FLAG_OVERCOMMIT);
+}
+
+DISPATCH_ALWAYS_INLINE
+static inline dispatch_wlh_t
+_dispatch_root_queue_wlh_for_queue(dispatch_queue_t rq,
+ dispatch_queue_class_t dqu)
+{
+ if (likely(_dispatch_root_queue_allows_wlh_for_queue(rq, dqu))) {
+ return (dispatch_wlh_t)dqu._dq;
+ }
+ return DISPATCH_WLH_GLOBAL;
+}
+
typedef dispatch_queue_wakeup_target_t
_dispatch_queue_class_invoke_handler_t(dispatch_object_t,
dispatch_invoke_context_t dic, dispatch_invoke_flags_t,
@@ -1755,13 +1554,13 @@
static inline void
_dispatch_queue_class_invoke(dispatch_object_t dou,
dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags,
- dispatch_invoke_flags_t const_restrict_flags,
_dispatch_queue_class_invoke_handler_t invoke)
{
dispatch_queue_t dq = dou._dq;
dispatch_queue_wakeup_target_t tq = DISPATCH_QUEUE_WAKEUP_NONE;
+ uint64_t dq_state, to_unlock = 0;
bool owning = !(flags & DISPATCH_INVOKE_STEALING);
- uint64_t owned = 0;
+ bool overriding = (flags & DISPATCH_INVOKE_OVERRIDING);
// When called from a plain _dispatch_queue_drain:
// overriding = false
@@ -1770,42 +1569,43 @@
// When called from an override continuation:
// overriding = true
// owning depends on whether the override embedded the queue or steals
+ DISPATCH_COMPILER_CAN_ASSUME(owning || overriding);
- if (!(flags & (DISPATCH_INVOKE_STEALING | DISPATCH_INVOKE_WLH))) {
+ if (likely(owning)) {
dq->do_next = DISPATCH_OBJECT_LISTLESS;
}
- flags |= const_restrict_flags;
- if (likely(flags & DISPATCH_INVOKE_WLH)) {
- owned = DISPATCH_QUEUE_SERIAL_DRAIN_OWNED | DISPATCH_QUEUE_ENQUEUED;
- } else {
- owned = _dispatch_queue_drain_try_lock(dq, flags);
- }
- if (likely(owned)) {
+ to_unlock = _dispatch_queue_drain_try_lock(dq, flags, &dq_state);
+ if (likely(to_unlock)) {
dispatch_priority_t old_dbp;
if (!(flags & DISPATCH_INVOKE_MANAGER_DRAIN)) {
+ if (unlikely(overriding)) {
+ _dispatch_object_debug(dq, "stolen onto thread 0x%x, 0x%x",
+ _dispatch_tid_self(), _dispatch_get_basepri());
+ }
old_dbp = _dispatch_set_basepri(dq->dq_priority);
+ dispatch_wlh_t wlh = _dispatch_get_wlh();
+ if (unlikely(dq->dq_wlh != wlh)) {
+ if (unlikely(dq->dq_wlh)) {
+ _dispatch_ktrace3(DISPATCH_PERF_wlh_change, dq,
+ dq->dq_wlh, wlh);
+ if (!(_dispatch_queue_atomic_flags_set_orig(dq,
+ DQF_WLH_CHANGED) & DQF_WLH_CHANGED)) {
+ _dispatch_bug_deprecated("Changing target queue "
+ "hierarchy after object has started executing");
+ }
+ }
+ dq->dq_wlh = wlh;
+#if DISPATCH_ENFORCE_STATIC_WLH_HIERARCHY
+ _dispatch_queue_atomic_flags_clear(dq, DQF_LEGACY);
+#endif
+ }
} else {
old_dbp = 0;
}
flags = _dispatch_queue_merge_autorelease_frequency(dq, flags);
attempt_running_slow_head:
-#if DISPATCH_COCOA_COMPAT
- if ((flags & DISPATCH_INVOKE_WLH) &&
- !(flags & DISPATCH_INVOKE_AUTORELEASE_ALWAYS)) {
- _dispatch_last_resort_autorelease_pool_push(dic);
- }
-#endif // DISPATCH_COCOA_COMPAT
- tq = invoke(dq, dic, flags, &owned);
-#if DISPATCH_COCOA_COMPAT
- if ((flags & DISPATCH_INVOKE_WLH) &&
- !(flags & DISPATCH_INVOKE_AUTORELEASE_ALWAYS)) {
- dispatch_thread_frame_s dtf;
- _dispatch_thread_frame_push(&dtf, dq);
- _dispatch_last_resort_autorelease_pool_pop(dic);
- _dispatch_thread_frame_pop(&dtf);
- }
-#endif // DISPATCH_COCOA_COMPAT
+ tq = invoke(dq, dic, flags, &to_unlock);
dispatch_assert(tq != DISPATCH_QUEUE_WAKEUP_TARGET);
if (unlikely(tq != DISPATCH_QUEUE_WAKEUP_NONE &&
tq != DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT)) {
@@ -1817,15 +1617,14 @@
// In both cases, we want to bypass the check for DIRTY.
// That may cause us to leave DIRTY in place but all drain lock
// acquirers clear it
- } else if (!_dispatch_queue_drain_try_unlock(dq, owned,
+ } else if (!_dispatch_queue_drain_try_unlock(dq, to_unlock,
tq == DISPATCH_QUEUE_WAKEUP_NONE)) {
tq = _dispatch_queue_get_current();
if (dx_hastypeflag(tq, QUEUE_ROOT) || !owning) {
goto attempt_running_slow_head;
}
- DISPATCH_COMPILER_CAN_ASSUME(tq != DISPATCH_QUEUE_WAKEUP_NONE);
} else {
- owned = 0;
+ to_unlock = 0;
tq = NULL;
}
if (!(flags & DISPATCH_INVOKE_MANAGER_DRAIN)) {
@@ -1836,43 +1635,32 @@
_dispatch_introspection_queue_item_complete(dq);
}
- if (tq) {
- if (const_restrict_flags & DISPATCH_INVOKE_DISALLOW_SYNC_WAITERS) {
- dispatch_assert(dic->dic_deferred == NULL);
- } else if (dic->dic_deferred) {
- return _dispatch_queue_drain_sync_waiter(dq, dic,
- flags, owned);
- }
+ if (tq && dic->dic_deferred) {
+ return _dispatch_queue_drain_deferred_invoke(dq, dic, flags, to_unlock);
+ }
- uint64_t old_state, new_state, enqueued = DISPATCH_QUEUE_ENQUEUED;
- if (tq == DISPATCH_QUEUE_WAKEUP_MGR) {
- enqueued = DISPATCH_QUEUE_ENQUEUED_ON_MGR;
- }
+ if (tq) {
+ uint64_t old_state, new_state;
+
os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
- new_state = old_state - owned;
- new_state &= ~DISPATCH_QUEUE_DRAIN_UNLOCK_MASK;
+ new_state = DISPATCH_QUEUE_DRAIN_UNLOCK(old_state - to_unlock);
new_state |= DISPATCH_QUEUE_DIRTY;
- if (_dq_state_is_suspended(new_state)) {
- new_state |= DLOCK_OWNER_MASK;
- } else if (_dq_state_is_runnable(new_state) &&
- !_dq_state_is_enqueued(new_state)) {
+ if (_dq_state_should_wakeup(new_state)) {
// drain was not interupted for suspension
// we will reenqueue right away, just put ENQUEUED back
- new_state |= enqueued;
+ new_state |= DISPATCH_QUEUE_ENQUEUED;
}
});
- old_state -= owned;
if (_dq_state_received_override(old_state)) {
// Ensure that the root queue sees that this thread was overridden.
- _dispatch_set_basepri_override_qos(_dq_state_max_qos(new_state));
+ _dispatch_set_basepri_override_qos(_dq_state_max_qos(old_state));
}
- if ((old_state ^ new_state) & enqueued) {
- dispatch_assert(_dq_state_is_enqueued(new_state));
- return _dispatch_queue_push_queue(tq, dq, new_state);
+ if ((old_state ^ new_state) & DISPATCH_QUEUE_ENQUEUED) {
+ return dx_push(tq, dq, _dq_state_max_qos(old_state));
}
}
- _dispatch_release_2_tailcall(dq);
+ return _dispatch_release_tailcall(dq);
}
DISPATCH_ALWAYS_INLINE
@@ -1910,21 +1698,23 @@
{
// Tag thread-bound queues with the owning thread
dispatch_assert(_dispatch_queue_is_thread_bound(dq));
- uint64_t old_state, new_state;
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, relaxed, {
- new_state = old_state;
- new_state &= ~DISPATCH_QUEUE_DRAIN_OWNER_MASK;
- new_state |= _dispatch_lock_value_for_self();
- });
+ mach_port_t old_owner, self = _dispatch_tid_self();
+ uint64_t dq_state = os_atomic_or_orig2o(dq, dq_state, self, relaxed);
+ if (unlikely(old_owner = _dq_state_drain_owner(dq_state))) {
+ DISPATCH_INTERNAL_CRASH(old_owner, "Queue bound twice");
+ }
}
DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_queue_clear_bound_thread(dispatch_queue_t dq)
{
+ uint64_t old_state, new_state;
+
dispatch_assert(_dispatch_queue_is_thread_bound(dq));
- _dispatch_queue_atomic_flags_clear(dq, DQF_THREAD_BOUND|DQF_CANNOT_TRYSYNC);
- os_atomic_and2o(dq, dq_state, ~DISPATCH_QUEUE_DRAIN_OWNER_MASK, relaxed);
+ os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, relaxed, {
+ new_state = DISPATCH_QUEUE_DRAIN_UNLOCK(old_state);
+ });
}
DISPATCH_ALWAYS_INLINE
@@ -2050,21 +1840,6 @@
}
DISPATCH_ALWAYS_INLINE
-static inline dispatch_priority_t
-_dispatch_set_basepri_wlh(dispatch_priority_t dbp)
-{
-#if HAVE_PTHREAD_WORKQUEUE_QOS
- dispatch_assert(!_dispatch_get_basepri());
- // _dispatch_set_basepri_override_qos(DISPATCH_QOS_SATURATED)
- dbp |= DISPATCH_QOS_SATURATED << DISPATCH_PRIORITY_OVERRIDE_SHIFT;
- _dispatch_thread_setspecific(dispatch_basepri_key, (void*)(uintptr_t)dbp);
-#else
- (void)dbp;
-#endif
- return 0;
-}
-
-DISPATCH_ALWAYS_INLINE
static inline pthread_priority_t
_dispatch_priority_adopt(pthread_priority_t pp, unsigned long flags)
{
@@ -2261,7 +2036,7 @@
dispatch_qos_t qos)
{
if (_dispatch_queue_need_override(dqu, qos)) {
- _os_object_retain_internal_n_inline(dqu._oq->_as_os_obj, 2);
+ _os_object_retain_internal_inline(dqu._oq->_as_os_obj);
return true;
}
return false;
@@ -2278,37 +2053,37 @@
return MAX(qos, _dispatch_priority_qos(dqu._oq->oq_priority));
}
-#define DISPATCH_PRIORITY_PROPAGATE_CURRENT 0x1
-#define DISPATCH_PRIORITY_PROPAGATE_FOR_SYNC_IPC 0x2
-
DISPATCH_ALWAYS_INLINE
-static inline pthread_priority_t
-_dispatch_priority_compute_propagated(pthread_priority_t pp,
- unsigned int flags)
+static inline dispatch_qos_t
+_dispatch_queue_reset_max_qos(dispatch_queue_class_t dqu)
{
-#if HAVE_PTHREAD_WORKQUEUE_QOS
- if (flags & DISPATCH_PRIORITY_PROPAGATE_CURRENT) {
- pp = _dispatch_get_priority();
- }
- pp &= ~_PTHREAD_PRIORITY_FLAGS_MASK;
- if (!(flags & DISPATCH_PRIORITY_PROPAGATE_FOR_SYNC_IPC) &&
- pp > _dispatch_qos_to_pp(DISPATCH_QOS_USER_INITIATED)) {
- // Cap QOS for propagation at user-initiated <rdar://16681262&16998036>
- return _dispatch_qos_to_pp(DISPATCH_QOS_USER_INITIATED);
- }
- return pp;
-#else
- (void)pp; (void)flags;
- return 0;
-#endif
+ uint64_t old_state, new_state;
+ os_atomic_rmw_loop2o(dqu._dq, dq_state, old_state, new_state, relaxed, {
+ new_state = old_state;
+ new_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
+ new_state &= ~DISPATCH_QUEUE_RECEIVED_OVERRIDE;
+ if (old_state == new_state) {
+ os_atomic_rmw_loop_give_up(return DISPATCH_QOS_UNSPECIFIED);
+ }
+ });
+ return _dq_state_max_qos(old_state);
}
DISPATCH_ALWAYS_INLINE
static inline pthread_priority_t
_dispatch_priority_propagate(void)
{
- return _dispatch_priority_compute_propagated(0,
- DISPATCH_PRIORITY_PROPAGATE_CURRENT);
+#if HAVE_PTHREAD_WORKQUEUE_QOS
+ pthread_priority_t pp = _dispatch_get_priority();
+ pp &= ~_PTHREAD_PRIORITY_FLAGS_MASK;
+ if (pp > _dispatch_qos_to_pp(DISPATCH_QOS_USER_INITIATED)) {
+ // Cap QOS for propagation at user-initiated <rdar://16681262&16998036>
+ return _dispatch_qos_to_pp(DISPATCH_QOS_USER_INITIATED);
+ }
+ return pp;
+#else
+ return 0;
+#endif
}
// including maintenance
@@ -2325,6 +2100,66 @@
}
#pragma mark -
+#pragma mark dispatch_wlh_t
+
+static inline dispatch_wlh_t
+_dispatch_queue_class_compute_wlh(dispatch_queue_class_t dqu)
+{
+ // TODO: combine with _dispatch_source_compute_kevent_priority
+ dispatch_queue_t dq = dqu._dq;
+ dispatch_queue_t tq = dq->do_targetq;
+
+ while (unlikely(!dx_hastypeflag(tq, QUEUE_ROOT))) {
+ if (tq->dq_wlh) {
+ return tq->dq_wlh;
+ }
+ dispatch_assert(!_dispatch_queue_is_thread_bound(tq));
+ if (unlikely(DISPATCH_QUEUE_IS_SUSPENDED(tq))) {
+ // this queue may not be activated yet, so the queue graph may not
+ // have stabilized yet
+ return NULL;
+ }
+ if (unlikely(_dispatch_queue_is_legacy(tq))) {
+ if (!_dispatch_is_in_root_queues_array(tq->do_targetq)) {
+ // we're not allowed to dereference tq->do_targetq
+ return NULL;
+ }
+ }
+ dq = tq;
+ tq = dq->do_targetq;
+ }
+ dispatch_assert(tq->dq_wlh);
+ return _dispatch_root_queue_wlh_for_queue(tq, dq);
+}
+
+static inline void
+_dispatch_queue_class_record_wlh_hierarchy(dispatch_queue_class_t dqu,
+ dispatch_wlh_t wlh)
+{
+ dispatch_queue_t dq = dqu._dq;
+ dispatch_queue_t tq = dq->do_targetq;
+
+ dispatch_assert(wlh);
+ dispatch_assert(!dq->dq_wlh);
+ dq->dq_wlh = wlh;
+#if DISPATCH_ENFORCE_STATIC_WLH_HIERARCHY
+ _dispatch_queue_atomic_flags_clear(dq, DQF_LEGACY);
+#endif
+ while (unlikely(!dx_hastypeflag(tq, QUEUE_ROOT))) {
+ if (tq->dq_wlh) {
+ return;
+ }
+ tq->dq_wlh = wlh;
+#if DISPATCH_ENFORCE_STATIC_WLH_HIERARCHY
+ _dispatch_queue_atomic_flags_set_and_clear(tq, DQF_TARGETED,DQF_LEGACY);
+#else
+ _dispatch_queue_atomic_flags_set(tq, DQF_TARGETED);
+#endif
+ tq = tq->do_targetq;
+ }
+}
+
+#pragma mark -
#pragma mark dispatch_block_t
#ifdef __BLOCKS__
diff --git a/src/internal.h b/src/internal.h
index 0536db1..688d5dd 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -38,7 +38,6 @@
#ifdef __APPLE__
#include <Availability.h>
-#include <os/availability.h>
#include <TargetConditionals.h>
#ifndef TARGET_OS_MAC_DESKTOP
@@ -49,15 +48,15 @@
#if TARGET_OS_MAC_DESKTOP
# define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) \
(__MAC_OS_X_VERSION_MIN_REQUIRED >= (x))
-# if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
-# error "OS X hosts older than OS X 10.12 aren't supported anymore"
-# endif // !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+# if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101100)
+# error "OS X hosts older than OS X 10.11 aren't supported anymore"
+# endif // !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101000)
#elif TARGET_OS_SIMULATOR
# define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) \
(IPHONE_SIMULATOR_HOST_MIN_VERSION_REQUIRED >= (x))
-# if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
-# error "Simulator hosts older than OS X 10.12 aren't supported anymore"
-# endif // !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+# if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101100)
+# error "Simulator hosts older than OS X 10.11 aren't supported anymore"
+# endif // !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101000)
#else
# define DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(x) 1
# if __IPHONE_OS_VERSION_MIN_REQUIRED < 90000
@@ -189,8 +188,6 @@
#define DISPATCH_USE_CLIENT_CALLOUT 1
#endif
-#define DISPATCH_ALLOW_NON_LEAF_RETARGET 1
-
/* The "_debug" library build */
#ifndef DISPATCH_DEBUG
#define DISPATCH_DEBUG 0
@@ -242,6 +239,9 @@
#if HAVE_MALLOC_MALLOC_H
#include <malloc/malloc.h>
#endif
+#if __has_include(<malloc_private.h>)
+#include <malloc_private.h>
+#endif // __has_include(<malloc_private.h)
#include <sys/stat.h>
@@ -260,11 +260,7 @@
#endif
#ifdef __BLOCKS__
-#if __has_include(<Block_private.h>)
#include <Block_private.h>
-#else
-#include "BlocksRuntime/Block_private.h"
-#endif // __has_include(<Block_private.h>)
#include <Block.h>
#endif /* __BLOCKS__ */
@@ -454,14 +450,14 @@
* For reporting bugs within libdispatch when using the "_debug" version of the
* library.
*/
-#if __APPLE__
+#if __GNUC__
#define dispatch_assert(e) do { \
if (__builtin_constant_p(e)) { \
dispatch_static_assert(e); \
} else { \
typeof(e) _e = fastpath(e); /* always eval 'e' */ \
- if (!_e) { \
- __assert_rtn(__func__, __FILE__, __LINE__, #e); \
+ if (DISPATCH_DEBUG && !_e) { \
+ _dispatch_abort(__LINE__, (long)_e); \
} \
} \
} while (0)
@@ -472,7 +468,7 @@
#define dispatch_assert(e) _dispatch_assert((long)(e), __LINE__)
#endif /* __GNUC__ */
-#if __APPLE__
+#if __GNUC__
/*
* A lot of API return zero upon success and not-zero on fail. Let's capture
* and log the non-zero value
@@ -482,8 +478,8 @@
dispatch_static_assert(e); \
} else { \
typeof(e) _e = slowpath(e); /* always eval 'e' */ \
- if (_e) { \
- __assert_rtn(__func__, __FILE__, __LINE__, #e); \
+ if (DISPATCH_DEBUG && _e) { \
+ _dispatch_abort(__LINE__, (long)_e); \
} \
} \
} while (0)
@@ -491,7 +487,7 @@
static inline void _dispatch_assert_zero(long e, long line) {
if (DISPATCH_DEBUG && e) _dispatch_abort(line, e);
}
-#define dispatch_assert_zero(e) _dispatch_assert_zero((long)(e), __LINE__)
+#define dispatch_assert_zero(e) _dispatch_assert((long)(e), __LINE__)
#endif /* __GNUC__ */
/*
@@ -600,7 +596,6 @@
const char *_dispatch_strdup_if_mutable(const char *str);
void _dispatch_vtable_init(void);
char *_dispatch_get_build(void);
-int _dispatch_sigmask(void);
uint64_t _dispatch_timeout(dispatch_time_t when);
uint64_t _dispatch_time_nanoseconds_since_epoch(dispatch_time_t when);
@@ -635,16 +630,35 @@
// Older Mac OS X and iOS Simulator fallbacks
+#if HAVE_PTHREAD_WORKQUEUES || DISPATCH_USE_INTERNAL_WORKQUEUE
+#ifndef WORKQ_ADDTHREADS_OPTION_OVERCOMMIT
+#define WORKQ_ADDTHREADS_OPTION_OVERCOMMIT 0x00000001
+#endif
+#endif // HAVE_PTHREAD_WORKQUEUES || DISPATCH_USE_INTERNAL_WORKQUEUE
#if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20140213 \
&& !defined(HAVE_PTHREAD_WORKQUEUE_QOS)
#define HAVE_PTHREAD_WORKQUEUE_QOS 1
#endif
-#if HAVE__PTHREAD_WORKQUEUE_INIT && PTHREAD_WORKQUEUE_SPI_VERSION >= 20150304 \
+#if HAVE__PTHREAD_WORKQUEUE_INIT && (PTHREAD_WORKQUEUE_SPI_VERSION >= 20150304 \
+ || (PTHREAD_WORKQUEUE_SPI_VERSION == 20140730 && \
+ defined(WORKQ_FEATURE_KEVENT))) \
&& !defined(HAVE_PTHREAD_WORKQUEUE_KEVENT)
+#if PTHREAD_WORKQUEUE_SPI_VERSION == 20140730
+// rdar://problem/20609877
+typedef pthread_worqueue_function_kevent_t pthread_workqueue_function_kevent_t;
+#endif
#define HAVE_PTHREAD_WORKQUEUE_KEVENT 1
#endif
+#ifndef PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK
+#if HAVE_PTHREAD_WORKQUEUE_QOS && DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+#define PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK 1
+#else
+#define PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK 0
+#endif
+#endif // PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK
+
#ifndef HAVE_PTHREAD_WORKQUEUE_NARROWING
#if !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(109900)
#define HAVE_PTHREAD_WORKQUEUE_NARROWING 0
@@ -667,29 +681,31 @@
#define DISPATCH_USE_MEMORYPRESSURE_SOURCE 1
#endif
#if DISPATCH_USE_MEMORYPRESSURE_SOURCE
-#if __has_include(<malloc_private.h>)
-#include <malloc_private.h>
-#else
-extern void malloc_memory_event_handler(unsigned long);
-#endif // __has_include(<malloc_private.h)
extern bool _dispatch_memory_warn;
#endif
#if HAVE_PTHREAD_WORKQUEUE_KEVENT && defined(KEVENT_FLAG_WORKQ) && \
+ DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200) && \
!defined(DISPATCH_USE_KEVENT_WORKQUEUE)
#define DISPATCH_USE_KEVENT_WORKQUEUE 1
#endif
-#if (!DISPATCH_USE_KEVENT_WORKQUEUE || DISPATCH_DEBUG || DISPATCH_PROFILE) && \
+
+#if (!DISPATCH_USE_KEVENT_WORKQUEUE || DISPATCH_DEBUG) && \
!defined(DISPATCH_USE_MGR_THREAD)
#define DISPATCH_USE_MGR_THREAD 1
#endif
+#if DISPATCH_USE_KEVENT_WORKQUEUE && \
+ DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200) && \
+ !defined(DISPATCH_USE_EVFILT_MACHPORT_DIRECT)
+#define DISPATCH_USE_EVFILT_MACHPORT_DIRECT 1
+#endif
-#if defined(MACH_SEND_SYNC_OVERRIDE) && defined(MACH_RCV_SYNC_WAIT) && \
- DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(109900) && \
- !defined(DISPATCH_USE_MACH_SEND_SYNC_OVERRIDE)
-#define DISPATCH_USE_MACH_SEND_SYNC_OVERRIDE 1
+
+#if (!DISPATCH_USE_EVFILT_MACHPORT_DIRECT || DISPATCH_DEBUG) && \
+ !defined(DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK)
+#define DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK 1
#endif
#if defined(F_SETNOSIGPIPE) && defined(F_GETNOSIGPIPE)
@@ -745,8 +761,7 @@
#else
#define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0
#endif
-#if !defined(DISPATCH_USE_VOUCHER_KDEBUG_TRACE) && \
- (DISPATCH_INTROSPECTION || DISPATCH_PROFILE)
+#if !defined(DISPATCH_USE_VOUCHER_KDEBUG_TRACE) && DISPATCH_INTROSPECTION
#define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 1
#endif
@@ -762,6 +777,7 @@
#define DISPATCH_PERF_delayed_registration DISPATCH_CODE(PERF, 4)
#define DISPATCH_PERF_mutable_target DISPATCH_CODE(PERF, 5)
#define DISPATCH_PERF_strict_bg_timer DISPATCH_CODE(PERF, 6)
+#define DISPATCH_PERF_wlh_change DISPATCH_CODE(PERF, 7)
#define DISPATCH_MACH_MSG_hdr_move DISPATCH_CODE(MACH_MSG, 1)
@@ -821,12 +837,32 @@
#endif
#endif // VOUCHER_USE_MACH_VOUCHER
-#ifndef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
#if RDAR_24272659 // FIXME: <rdar://problem/24272659>
+#if !VOUCHER_USE_MACH_VOUCHER || !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+#undef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
+#define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0
+#elif !defined(VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER)
#define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 1
+#endif
#else // RDAR_24272659
+#undef VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER
#define VOUCHER_USE_EMPTY_MACH_BASE_VOUCHER 0
#endif // RDAR_24272659
+
+#if !VOUCHER_USE_MACH_VOUCHER || !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+#undef VOUCHER_USE_BANK_AUTOREDEEM
+#define VOUCHER_USE_BANK_AUTOREDEEM 0
+#elif !defined(VOUCHER_USE_BANK_AUTOREDEEM)
+#define VOUCHER_USE_BANK_AUTOREDEEM 1
+#endif
+
+#if !VOUCHER_USE_MACH_VOUCHER || \
+ !__has_include(<voucher/ipc_pthread_priority_types.h>) || \
+ !DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+#undef VOUCHER_USE_MACH_VOUCHER_PRIORITY
+#define VOUCHER_USE_MACH_VOUCHER_PRIORITY 0
+#elif !defined(VOUCHER_USE_MACH_VOUCHER_PRIORITY)
+#define VOUCHER_USE_MACH_VOUCHER_PRIORITY 1
#endif
#ifndef VOUCHER_USE_PERSONA
@@ -940,6 +976,22 @@
#endif // DISPATCH_USE_KEVENT_WORKQUEUE
+#if DISPATCH_USE_EVFILT_MACHPORT_DIRECT
+#if !DISPATCH_USE_KEVENT_WORKQUEUE || !EV_UDATA_SPECIFIC
+#error Invalid build configuration
+#endif
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+extern int _dispatch_evfilt_machport_direct_enabled;
+#else
+#define _dispatch_evfilt_machport_direct_enabled (1)
+#endif
+#else
+#define _dispatch_evfilt_machport_direct_enabled (0)
+#endif // DISPATCH_USE_EVFILT_MACHPORT_DIRECT
+
+
+int _dispatch_sigmask(void);
+
/* #includes dependent on internal.h */
#include "object_internal.h"
#include "semaphore_internal.h"
diff --git a/src/introspection.c b/src/introspection.c
index 8692a8b..cd6bcff 100644
--- a/src/introspection.c
+++ b/src/introspection.c
@@ -219,7 +219,7 @@
} else {
if (flags & DISPATCH_OBJ_SYNC_WAITER_BIT) {
dispatch_sync_context_t dsc = (dispatch_sync_context_t)dc;
- waiter = pthread_from_mach_thread_np(dsc->dsc_waiter);
+ waiter = pthread_from_mach_thread_np((mach_port_t)dc->dc_data);
ctxt = dsc->dsc_ctxt;
func = dsc->dsc_func;
}
diff --git a/src/io.c b/src/io.c
index 2904373..f538862 100644
--- a/src/io.c
+++ b/src/io.c
@@ -233,7 +233,7 @@
static dispatch_io_t
_dispatch_io_create(dispatch_io_type_t type)
{
- dispatch_io_t channel = _dispatch_object_alloc(DISPATCH_VTABLE(io),
+ dispatch_io_t channel = _dispatch_alloc(DISPATCH_VTABLE(io),
sizeof(struct dispatch_io_s));
channel->do_next = DISPATCH_OBJECT_LISTLESS;
channel->do_targetq = _dispatch_get_root_queue(DISPATCH_QOS_DEFAULT, true);
@@ -278,7 +278,7 @@
}
void
-_dispatch_io_dispose(dispatch_io_t channel, DISPATCH_UNUSED bool *allow_free)
+_dispatch_io_dispose(dispatch_io_t channel)
{
_dispatch_object_debug(channel, "%s", __func__);
if (channel->fd_entry &&
@@ -682,9 +682,6 @@
_dispatch_channel_debug("stop cleanup", channel);
_dispatch_fd_entry_cleanup_operations(fd_entry, channel);
if (!(channel->atomic_flags & DIO_CLOSED)) {
- if (fd_entry->path_data) {
- fd_entry->path_data->channel = NULL;
- }
channel->fd_entry = NULL;
_dispatch_fd_entry_release(fd_entry);
}
@@ -735,10 +732,9 @@
relaxed);
dispatch_fd_entry_t fd_entry = channel->fd_entry;
if (fd_entry) {
- if (fd_entry->path_data) {
- fd_entry->path_data->channel = NULL;
+ if (!fd_entry->path_data) {
+ channel->fd_entry = NULL;
}
- channel->fd_entry = NULL;
_dispatch_fd_entry_release(fd_entry);
}
}
@@ -1023,13 +1019,14 @@
});
return NULL;
}
- dispatch_operation_t op = _dispatch_object_alloc(DISPATCH_VTABLE(operation),
+ dispatch_operation_t op = _dispatch_alloc(DISPATCH_VTABLE(operation),
sizeof(struct dispatch_operation_s));
_dispatch_channel_debug("operation create: %p", channel, op);
op->do_next = DISPATCH_OBJECT_LISTLESS;
op->do_xref_cnt = -1; // operation object is not exposed externally
- op->op_q = dispatch_queue_create_with_target("com.apple.libdispatch-io.opq",
- NULL, queue);
+ op->op_q = dispatch_queue_create("com.apple.libdispatch-io.opq", NULL);
+ op->op_q->do_targetq = queue;
+ _dispatch_retain(queue);
op->active = false;
op->direction = direction;
op->offset = offset + channel->f_ptr;
@@ -1050,8 +1047,7 @@
}
void
-_dispatch_operation_dispose(dispatch_operation_t op,
- DISPATCH_UNUSED bool *allow_free)
+_dispatch_operation_dispose(dispatch_operation_t op)
{
_dispatch_object_debug(op, "%s", __func__);
_dispatch_op_debug("dispose", op);
@@ -1309,10 +1305,12 @@
{
dispatch_fd_entry_t fd_entry;
fd_entry = _dispatch_calloc(1ul, sizeof(struct dispatch_fd_entry_s));
+ fd_entry->close_queue = dispatch_queue_create(
+ "com.apple.libdispatch-io.closeq", NULL);
// Use target queue to ensure that no concurrent lookups are going on when
// the close queue is running
- fd_entry->close_queue = dispatch_queue_create_with_target(
- "com.apple.libdispatch-io.closeq", NULL, q);
+ fd_entry->close_queue->do_targetq = q;
+ _dispatch_retain(q);
// Suspend the cleanup queue until closing
_dispatch_fd_entry_retain(fd_entry);
return fd_entry;
@@ -1586,9 +1584,11 @@
for (direction = 0; direction < DOP_DIR_MAX; direction++) {
dispatch_stream_t stream;
stream = _dispatch_calloc(1ul, sizeof(struct dispatch_stream_s));
- stream->dq = dispatch_queue_create_with_target(
- "com.apple.libdispatch-io.streamq", NULL, tq);
+ stream->dq = dispatch_queue_create("com.apple.libdispatch-io.streamq",
+ NULL);
dispatch_set_context(stream->dq, stream);
+ _dispatch_retain(tq);
+ stream->dq->do_targetq = tq;
TAILQ_INIT(&stream->operations[DISPATCH_IO_RANDOM]);
TAILQ_INIT(&stream->operations[DISPATCH_IO_STREAM]);
fd_entry->streams[direction] = stream;
@@ -1633,7 +1633,7 @@
}
// Otherwise create a new entry
size_t pending_reqs_depth = dispatch_io_defaults.max_pending_io_reqs;
- disk = _dispatch_object_alloc(DISPATCH_VTABLE(disk),
+ disk = _dispatch_alloc(DISPATCH_VTABLE(disk),
sizeof(struct dispatch_disk_s) +
(pending_reqs_depth * sizeof(dispatch_operation_t)));
disk->do_next = DISPATCH_OBJECT_LISTLESS;
@@ -1654,7 +1654,7 @@
}
void
-_dispatch_disk_dispose(dispatch_disk_t disk, DISPATCH_UNUSED bool *allow_free)
+_dispatch_disk_dispose(dispatch_disk_t disk)
{
uintptr_t hash = DIO_HASH(disk->dev);
TAILQ_REMOVE(&_dispatch_io_devs[hash], disk, disk_list);
diff --git a/src/io_internal.h b/src/io_internal.h
index 672727f..ad8259a 100644
--- a/src/io_internal.h
+++ b/src/io_internal.h
@@ -178,11 +178,10 @@
void _dispatch_io_set_target_queue(dispatch_io_t channel, dispatch_queue_t dq);
size_t _dispatch_io_debug(dispatch_io_t channel, char* buf, size_t bufsiz);
-void _dispatch_io_dispose(dispatch_io_t channel, bool *allow_free);
+void _dispatch_io_dispose(dispatch_io_t channel);
size_t _dispatch_operation_debug(dispatch_operation_t op, char* buf,
size_t bufsiz);
-void _dispatch_operation_dispose(dispatch_operation_t operation,
- bool *allow_free);
-void _dispatch_disk_dispose(dispatch_disk_t disk, bool *allow_free);
+void _dispatch_operation_dispose(dispatch_operation_t operation);
+void _dispatch_disk_dispose(dispatch_disk_t disk);
#endif // __DISPATCH_IO_INTERNAL__
diff --git a/src/libdispatch.codes b/src/libdispatch.codes
index 0ecc333..64f82b5 100644
--- a/src/libdispatch.codes
+++ b/src/libdispatch.codes
@@ -12,6 +12,7 @@
0x2e020010 DISPATCH_PERF_delayed_registration
0x2e020014 DISPATCH_PERF_mutable_target
0x2e020018 DISPATCH_PERF_strict_bg_timer
+0x2e02001c DISPATCH_PERF_wlh_change
0x2e030004 DISPATCH_MACH_MSG_hdr_move
diff --git a/src/mach.c b/src/mach.c
index 0f9e9a8..cc20645 100644
--- a/src/mach.c
+++ b/src/mach.c
@@ -33,7 +33,7 @@
DISPATCH_ENUM(dispatch_mach_send_invoke_flags, uint32_t,
DM_SEND_INVOKE_NONE = 0x0,
- DM_SEND_INVOKE_MAKE_DIRTY = 0x1,
+ DM_SEND_INVOKE_FLUSH = 0x1,
DM_SEND_INVOKE_NEEDS_BARRIER = 0x2,
DM_SEND_INVOKE_CANCEL = 0x4,
DM_SEND_INVOKE_CAN_RUN_BARRIER = 0x8,
@@ -43,6 +43,8 @@
((dispatch_mach_send_invoke_flags_t)DM_SEND_INVOKE_IMMEDIATE_SEND)
static inline mach_msg_option_t _dispatch_mach_checkin_options(void);
+static inline pthread_priority_t _dispatch_mach_priority_propagate(
+ mach_msg_option_t options);
static mach_port_t _dispatch_mach_msg_get_remote_port(dispatch_object_t dou);
static mach_port_t _dispatch_mach_msg_get_reply_port(dispatch_object_t dou);
static void _dispatch_mach_msg_disconnected(dispatch_mach_t dm,
@@ -121,14 +123,6 @@
"_dispatch_mach_default_async_reply_handler called");
}
-// Default dmxh_enable_sigterm_notification callback that enables delivery of
-// SIGTERM notifications (for backwards compatibility).
-static bool
-_dispatch_mach_enable_sigterm(void *_Nullable context DISPATCH_UNUSED)
-{
- return true;
-}
-
// Callbacks from dispatch to XPC. The default is to not support any callbacks.
static const struct dispatch_mach_xpc_hooks_s _dispatch_mach_xpc_hooks_default
= {
@@ -137,7 +131,6 @@
.dmxh_msg_context_reply_queue =
&_dispatch_mach_msg_context_no_async_reply_queue,
.dmxh_async_reply_handler = &_dispatch_mach_default_async_reply_handler,
- .dmxh_enable_sigterm_notification = &_dispatch_mach_enable_sigterm,
};
static dispatch_mach_xpc_hooks_t _dispatch_mach_xpc_hooks
@@ -164,17 +157,17 @@
dispatch_mach_recv_refs_t dmrr;
dispatch_mach_send_refs_t dmsr;
dispatch_mach_t dm;
- dm = _dispatch_object_alloc(DISPATCH_VTABLE(mach),
+ // ensure _dispatch_evfilt_machport_direct_enabled is initialized
+ _dispatch_root_queues_init();
+ dm = _dispatch_alloc(DISPATCH_VTABLE(mach),
sizeof(struct dispatch_mach_s));
- _dispatch_queue_init(dm->_as_dq, DQF_LEGACY, 1,
- DISPATCH_QUEUE_INACTIVE | DISPATCH_QUEUE_ROLE_INNER);
+ _dispatch_queue_init(dm->_as_dq, DQF_LEGACY, 1, true);
dm->dq_label = label;
dm->do_ref_cnt++; // the reference _dispatch_mach_cancel_invoke holds
dm->dm_is_xpc = is_xpc;
dmrr = dux_create(&_dispatch_mach_type_recv, 0, 0)._dmrr;
- dispatch_assert(dmrr->du_is_direct);
dmrr->du_owner_wref = _dispatch_ptr2wref(dm);
dmrr->dmrr_handler_func = handler;
dmrr->dmrr_handler_ctxt = context;
@@ -186,6 +179,13 @@
dmsr->du_owner_wref = _dispatch_ptr2wref(dm);
dm->dm_send_refs = dmsr;
+ if (is_xpc) {
+ dispatch_xpc_term_refs_t _dxtr =
+ dux_create(&_dispatch_xpc_type_sigterm, SIGTERM, 0)._dxtr;
+ _dxtr->du_owner_wref = _dispatch_ptr2wref(dm);
+ dm->dm_xpc_term_refs = _dxtr;
+ }
+
if (slowpath(!q)) {
q = _dispatch_get_root_queue(DISPATCH_QOS_DEFAULT, true);
} else {
@@ -221,7 +221,7 @@
}
void
-_dispatch_mach_dispose(dispatch_mach_t dm, bool *allow_free)
+_dispatch_mach_dispose(dispatch_mach_t dm)
{
_dispatch_object_debug(dm, "%s", __func__);
_dispatch_unote_dispose(dm->dm_recv_refs);
@@ -232,7 +232,7 @@
_dispatch_unote_dispose(dm->dm_xpc_term_refs);
dm->dm_xpc_term_refs = NULL;
}
- _dispatch_queue_destroy(dm->_as_dq, allow_free);
+ _dispatch_queue_destroy(dm->_as_dq);
}
void
@@ -309,66 +309,69 @@
if (dmsgr) {
return _dispatch_mach_handle_or_push_received_msg(dm, dmsgr);
}
+ dispatch_assert(!(options & DU_UNREGISTER_WAKEUP));
}
DISPATCH_NOINLINE
-static bool
-_dispatch_mach_reply_list_remove(dispatch_mach_t dm,
- dispatch_mach_reply_refs_t dmr) {
- // dmsr_replies_lock must be held by the caller.
- bool removed = false;
- if (likely(_TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
- TAILQ_REMOVE(&dm->dm_send_refs->dmsr_replies, dmr, dmr_list);
- _TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
- removed = true;
- }
- return removed;
-}
-
-DISPATCH_NOINLINE
-static bool
+static void
_dispatch_mach_reply_kevent_unregister(dispatch_mach_t dm,
dispatch_mach_reply_refs_t dmr, uint32_t options)
{
- dispatch_assert(!_TAILQ_IS_ENQUEUED(dmr, dmr_list));
-
- bool disconnected = (options & DU_UNREGISTER_DISCONNECTED);
- _dispatch_debug("machport[0x%08x]: unregistering for reply%s, ctxt %p",
- (mach_port_t)dmr->du_ident, disconnected ? " (disconnected)" : "",
- dmr->dmr_ctxt);
- if (!_dispatch_unote_unregister(dmr, options)) {
- _dispatch_debug("machport[0x%08x]: deferred delete kevent[%p]",
- (mach_port_t)dmr->du_ident, dmr);
- dispatch_assert(options == DU_UNREGISTER_DISCONNECTED);
- return false;
- }
-
dispatch_mach_msg_t dmsgr = NULL;
dispatch_queue_t drq = NULL;
+ bool replies_empty = false;
+ bool disconnected = (options & DU_UNREGISTER_DISCONNECTED);
+ if (options & DU_UNREGISTER_REPLY_REMOVE) {
+ _dispatch_unfair_lock_lock(&dm->dm_send_refs->dmsr_replies_lock);
+ if (unlikely(!_TAILQ_IS_ENQUEUED(dmr, dmr_list))) {
+ DISPATCH_INTERNAL_CRASH(0, "Could not find reply registration");
+ }
+ TAILQ_REMOVE(&dm->dm_send_refs->dmsr_replies, dmr, dmr_list);
+ _TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
+ replies_empty = TAILQ_EMPTY(&dm->dm_send_refs->dmsr_replies);
+ _dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);
+ }
if (disconnected) {
- // The next call is guaranteed to always transfer or consume the voucher
- // in the dmr, if there is one.
dmsgr = _dispatch_mach_msg_create_reply_disconnected(NULL, dmr,
dmr->dmr_async_reply ? DISPATCH_MACH_ASYNC_WAITER_DISCONNECTED
: DISPATCH_MACH_DISCONNECTED);
if (dmr->dmr_ctxt) {
drq = _dispatch_mach_msg_context_async_reply_queue(dmr->dmr_ctxt);
}
- dispatch_assert(dmr->dmr_voucher == NULL);
} else if (dmr->dmr_voucher) {
_voucher_release(dmr->dmr_voucher);
dmr->dmr_voucher = NULL;
}
+ _dispatch_debug("machport[0x%08x]: unregistering for reply%s, ctxt %p",
+ (mach_port_t)dmr->du_ident, disconnected ? " (disconnected)" : "",
+ dmr->dmr_ctxt);
+ if (!_dispatch_unote_unregister(dmr, options)) {
+ _dispatch_debug("machport[0x%08x]: deferred delete kevent[%p]",
+ (mach_port_t)dmr->du_ident, dmr);
+ dispatch_assert(options == DU_UNREGISTER_DISCONNECTED);
+ // dmr must be put back so that the event delivery finds it, the
+ // replies lock is held by the caller.
+ TAILQ_INSERT_HEAD(&dm->dm_send_refs->dmsr_replies, dmr, dmr_list);
+ if (dmsgr) {
+ dmr->dmr_voucher = dmsgr->dmsg_voucher;
+ dmsgr->dmsg_voucher = NULL;
+ _dispatch_release(dmsgr);
+ }
+ return; // deferred unregistration
+ }
_dispatch_unote_dispose(dmr);
-
if (dmsgr) {
if (drq) {
- _dispatch_mach_push_async_reply_msg(dm, dmsgr, drq);
+ return _dispatch_mach_push_async_reply_msg(dm, dmsgr, drq);
} else {
- _dispatch_mach_handle_or_push_received_msg(dm, dmsgr);
+ return _dispatch_mach_handle_or_push_received_msg(dm, dmsgr);
}
}
- return true;
+ if ((options & DU_UNREGISTER_WAKEUP) && replies_empty &&
+ (dm->dm_send_refs->dmsr_disconnect_cnt ||
+ (dm->dq_atomic_flags & DSF_CANCELED))) {
+ dx_wakeup(dm, 0, DISPATCH_WAKEUP_FLUSH);
+ }
}
DISPATCH_NOINLINE
@@ -409,11 +412,10 @@
dispatch_mach_msg_t dmsg)
{
dispatch_mach_reply_refs_t dmr;
- dispatch_priority_t mpri, pri, overcommit;
- dispatch_wlh_t wlh;
+ dispatch_priority_t mpri, pri, rpri;
+ dispatch_priority_t overcommit;
dmr = dux_create(&_dispatch_mach_type_reply, reply_port, 0)._dmr;
- dispatch_assert(dmr->du_is_direct);
dmr->du_owner_wref = _dispatch_ptr2wref(dm);
if (dmsg->dmsg_voucher) {
dmr->dmr_voucher = _voucher_retain(dmsg->dmsg_voucher);
@@ -428,22 +430,18 @@
drq = _dispatch_mach_msg_context_async_reply_queue(dmsg->do_ctxt);
}
- if (!drq) {
- pri = dm->dq_priority;
- wlh = dm->dm_recv_refs->du_wlh;
- } else if (dx_hastypeflag(drq, QUEUE_ROOT)) {
- pri = drq->dq_priority;
- wlh = DISPATCH_WLH_ANON;
- } else if (drq == dm->do_targetq) {
- pri = dm->dq_priority;
- wlh = dm->dm_recv_refs->du_wlh;
- } else if (!(pri = _dispatch_queue_compute_priority_and_wlh(drq, &wlh))) {
- pri = drq->dq_priority;
- wlh = DISPATCH_WLH_ANON;
+ dispatch_wlh_t wlh = dm->dq_wlh;
+ pri = (dm->dq_priority & DISPATCH_PRIORITY_REQUESTED_MASK);
+ overcommit = dm->dq_priority & DISPATCH_PRIORITY_FLAG_OVERCOMMIT;
+ if (drq) {
+ rpri = drq->dq_priority & DISPATCH_PRIORITY_REQUESTED_MASK;
+ if (rpri > pri) {
+ pri = rpri;
+ overcommit = drq->dq_priority & DISPATCH_PRIORITY_FLAG_OVERCOMMIT;
+ }
+ if (drq->dq_wlh) wlh = drq->dq_wlh;
}
- if (pri & DISPATCH_PRIORITY_REQUESTED_MASK) {
- overcommit = pri & DISPATCH_PRIORITY_FLAG_OVERCOMMIT;
- pri &= DISPATCH_PRIORITY_REQUESTED_MASK;
+ if (pri && dmr->du_is_direct) {
mpri = _dispatch_priority_from_pp_strip_flags(dmsg->dmsg_priority);
if (pri < mpri) pri = mpri;
pri |= overcommit;
@@ -462,54 +460,25 @@
_dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);
if (!_dispatch_unote_register(dmr, wlh, pri)) {
- _dispatch_unfair_lock_lock(&dm->dm_send_refs->dmsr_replies_lock);
- _dispatch_mach_reply_list_remove(dm, dmr);
- _dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);
_dispatch_mach_reply_kevent_unregister(dm, dmr,
- DU_UNREGISTER_DISCONNECTED);
+ DU_UNREGISTER_DISCONNECTED|DU_UNREGISTER_REPLY_REMOVE);
}
}
#pragma mark -
#pragma mark dispatch_mach_msg
-DISPATCH_ALWAYS_INLINE DISPATCH_CONST
-static inline bool
-_dispatch_use_mach_special_reply_port(void)
-{
-#if DISPATCH_USE_MACH_SEND_SYNC_OVERRIDE
- return true;
-#else
-#define thread_get_special_reply_port() ({__builtin_trap(); MACH_PORT_NULL;})
- return false;
-#endif
-}
-
static mach_port_t
_dispatch_get_thread_reply_port(void)
{
- mach_port_t reply_port, mrp;
- if (_dispatch_use_mach_special_reply_port()) {
- mrp = _dispatch_get_thread_special_reply_port();
- } else {
- mrp = _dispatch_get_thread_mig_reply_port();
- }
+ mach_port_t reply_port, mrp = _dispatch_get_thread_mig_reply_port();
if (mrp) {
reply_port = mrp;
_dispatch_debug("machport[0x%08x]: borrowed thread sync reply port",
reply_port);
} else {
- if (_dispatch_use_mach_special_reply_port()) {
- reply_port = thread_get_special_reply_port();
- _dispatch_set_thread_special_reply_port(reply_port);
- } else {
- reply_port = mach_reply_port();
- _dispatch_set_thread_mig_reply_port(reply_port);
- }
- if (unlikely(!MACH_PORT_VALID(reply_port))) {
- DISPATCH_CLIENT_CRASH(_dispatch_use_mach_special_reply_port(),
- "Unable to allocate reply port, possible port leak");
- }
+ reply_port = mach_reply_port();
+ _dispatch_set_thread_mig_reply_port(reply_port);
_dispatch_debug("machport[0x%08x]: allocated thread sync reply port",
reply_port);
}
@@ -520,12 +489,7 @@
static void
_dispatch_clear_thread_reply_port(mach_port_t reply_port)
{
- mach_port_t mrp;
- if (_dispatch_use_mach_special_reply_port()) {
- mrp = _dispatch_get_thread_special_reply_port();
- } else {
- mrp = _dispatch_get_thread_mig_reply_port();
- }
+ mach_port_t mrp = _dispatch_get_thread_mig_reply_port();
if (reply_port != mrp) {
if (mrp) {
_dispatch_debug("machport[0x%08x]: did not clear thread sync reply "
@@ -533,11 +497,7 @@
}
return;
}
- if (_dispatch_use_mach_special_reply_port()) {
- _dispatch_set_thread_special_reply_port(MACH_PORT_NULL);
- } else {
- _dispatch_set_thread_mig_reply_port(MACH_PORT_NULL);
- }
+ _dispatch_set_thread_mig_reply_port(MACH_PORT_NULL);
_dispatch_debug_machport(reply_port);
_dispatch_debug("machport[0x%08x]: cleared thread sync reply port",
reply_port);
@@ -547,12 +507,7 @@
_dispatch_set_thread_reply_port(mach_port_t reply_port)
{
_dispatch_debug_machport(reply_port);
- mach_port_t mrp;
- if (_dispatch_use_mach_special_reply_port()) {
- mrp = _dispatch_get_thread_special_reply_port();
- } else {
- mrp = _dispatch_get_thread_mig_reply_port();
- }
+ mach_port_t mrp = _dispatch_get_thread_mig_reply_port();
if (mrp) {
kern_return_t kr = mach_port_mod_refs(mach_task_self(), reply_port,
MACH_PORT_RIGHT_RECEIVE, -1);
@@ -561,11 +516,7 @@
_dispatch_debug("machport[0x%08x]: deallocated sync reply port "
"(found 0x%08x)", reply_port, mrp);
} else {
- if (_dispatch_use_mach_special_reply_port()) {
- _dispatch_set_thread_special_reply_port(reply_port);
- } else {
- _dispatch_set_thread_mig_reply_port(reply_port);
- }
+ _dispatch_set_thread_mig_reply_port(reply_port);
_dispatch_debug("machport[0x%08x]: restored thread sync reply port",
reply_port);
}
@@ -627,8 +578,7 @@
dmr->dmr_voucher = NULL; // transfer reference
} else {
voucher = voucher_create_with_mach_msg(hdr);
- pp = _dispatch_priority_compute_propagated(
- _voucher_get_priority(voucher), 0);
+ pp = _voucher_get_priority(voucher);
}
destructor = (flags & DISPATCH_EV_MSG_NEEDS_FREE) ?
@@ -659,6 +609,7 @@
dispatch_mach_recv_refs_t dmrr = du._dmrr;
dispatch_mach_t dm = _dispatch_wref2ptr(dmrr->du_owner_wref);
+ dispatch_wakeup_flags_t wflags = 0;
dispatch_queue_flags_t dqf;
dispatch_mach_msg_t dmsg;
@@ -669,24 +620,31 @@
"Unexpected EV_VANISHED (do not destroy random mach ports)");
}
- // once we modify the queue atomic flags below, it will allow concurrent
- // threads running _dispatch_mach_invoke2 to dispose of the source,
- // so we can't safely borrow the reference we get from the muxnote udata
- // anymore, and need our own
- dispatch_wakeup_flags_t wflags = DISPATCH_WAKEUP_CONSUME_2;
- _dispatch_retain_2(dm); // rdar://20382435
+ if (dmrr->du_is_direct || (flags & (EV_DELETE | EV_ONESHOT))) {
+ // once we modify the queue atomic flags below, it will allow concurrent
+ // threads running _dispatch_mach_invoke2 to dispose of the source,
+ // so we can't safely borrow the reference we get from the muxnote udata
+ // anymore, and need our own
+ wflags = DISPATCH_WAKEUP_CONSUME;
+ _dispatch_retain(dm); // rdar://20382435
+ }
if (unlikely((flags & EV_ONESHOT) && !(flags & EV_DELETE))) {
- dqf = _dispatch_queue_atomic_flags_set_and_clear(dm->_as_dq,
- DSF_DEFERRED_DELETE, DSF_ARMED);
+ dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
_dispatch_debug("kevent-source[%p]: deferred delete oneshot kevent[%p]",
dm, dmrr);
- } else if (unlikely(flags & (EV_ONESHOT | EV_DELETE))) {
+ } else if (unlikely(flags & EV_DELETE)) {
_dispatch_source_refs_unregister(dm->_as_ds,
DU_UNREGISTER_ALREADY_DELETED);
dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
_dispatch_debug("kevent-source[%p]: deleted kevent[%p]", dm, dmrr);
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+ } else if (unlikely(!dmrr->du_is_direct)) {
+ dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
+ _dispatch_unote_resume(du);
+#endif
} else {
+ dispatch_assert(dmrr->du_is_direct);
dqf = _dispatch_queue_atomic_flags_clear(dm->_as_dq, DSF_ARMED);
_dispatch_debug("kevent-source[%p]: disarmed kevent[%p]", dm, dmrr);
}
@@ -702,20 +660,14 @@
if (flags & DISPATCH_EV_MSG_NEEDS_FREE) {
free(hdr);
}
- return dx_wakeup(dm, 0, wflags | DISPATCH_WAKEUP_MAKE_DIRTY);
+ return dx_wakeup(dm, 0, wflags | DISPATCH_WAKEUP_FLUSH);
}
- // Once the mach channel disarming is visible, cancellation will switch to
- // immediate deletion. If we're preempted here, then the whole cancellation
- // sequence may be complete by the time we really enqueue the message.
- //
- // _dispatch_mach_msg_invoke_with_mach() is responsible for filtering it out
- // to keep the promise that DISPATCH_MACH_DISCONNECTED is the last
- // event sent.
-
dmsg = _dispatch_mach_msg_create_recv(hdr, siz, NULL, flags);
_dispatch_mach_handle_or_push_received_msg(dm, dmsg);
- return _dispatch_release_2_tailcall(dm);
+ if (wflags & DISPATCH_WAKEUP_CONSUME) {
+ return _dispatch_release_tailcall(dm);
+ }
}
void
@@ -731,11 +683,23 @@
_dispatch_debug("machport[0x%08x]: received msg id 0x%x, reply on 0x%08x",
hdr->msgh_local_port, hdr->msgh_id, hdr->msgh_remote_port);
- if (!canceled) {
+ uint32_t options = DU_UNREGISTER_IMMEDIATE_DELETE;
+ options |= DU_UNREGISTER_REPLY_REMOVE;
+ options |= DU_UNREGISTER_WAKEUP;
+ if (canceled) {
+ _dispatch_debug("machport[0x%08x]: drop msg id 0x%x, reply on 0x%08x",
+ hdr->msgh_local_port, hdr->msgh_id, hdr->msgh_remote_port);
+ options |= DU_UNREGISTER_DISCONNECTED;
+ mach_msg_destroy(hdr);
+ if (flags & DISPATCH_EV_MSG_NEEDS_FREE) {
+ free(hdr);
+ }
+ } else {
dmsg = _dispatch_mach_msg_create_recv(hdr, siz, dmr, flags);
}
+ _dispatch_mach_reply_kevent_unregister(dm, dmr, options);
- if (dmsg) {
+ if (!canceled) {
dispatch_queue_t drq = NULL;
if (dmsg->do_ctxt) {
drq = _dispatch_mach_msg_context_async_reply_queue(dmsg->do_ctxt);
@@ -745,45 +709,13 @@
} else {
_dispatch_mach_handle_or_push_received_msg(dm, dmsg);
}
- } else {
- _dispatch_debug("machport[0x%08x]: drop msg id 0x%x, reply on 0x%08x",
- hdr->msgh_local_port, hdr->msgh_id, hdr->msgh_remote_port);
- mach_msg_destroy(hdr);
- if (flags & DISPATCH_EV_MSG_NEEDS_FREE) {
- free(hdr);
- }
}
-
- dispatch_wakeup_flags_t wflags = 0;
- uint32_t options = DU_UNREGISTER_IMMEDIATE_DELETE;
- if (canceled) {
- options |= DU_UNREGISTER_DISCONNECTED;
- }
-
- _dispatch_unfair_lock_lock(&dm->dm_send_refs->dmsr_replies_lock);
- bool removed = _dispatch_mach_reply_list_remove(dm, dmr);
- dispatch_assert(removed);
- if (TAILQ_EMPTY(&dm->dm_send_refs->dmsr_replies) &&
- (dm->dm_send_refs->dmsr_disconnect_cnt ||
- (dm->dq_atomic_flags & DSF_CANCELED))) {
- // When the list is empty, _dispatch_mach_disconnect() may release the
- // last reference count on the Mach channel. To avoid this, take our
- // own reference before releasing the lock.
- wflags = DISPATCH_WAKEUP_MAKE_DIRTY | DISPATCH_WAKEUP_CONSUME_2;
- _dispatch_retain_2(dm);
- }
- _dispatch_unfair_lock_unlock(&dm->dm_send_refs->dmsr_replies_lock);
-
- bool result = _dispatch_mach_reply_kevent_unregister(dm, dmr, options);
- dispatch_assert(result);
- if (wflags) dx_wakeup(dm, 0, wflags);
}
DISPATCH_ALWAYS_INLINE
static inline dispatch_mach_msg_t
_dispatch_mach_msg_reply_recv(dispatch_mach_t dm,
- dispatch_mach_reply_refs_t dmr, mach_port_t reply_port,
- mach_port_t send)
+ dispatch_mach_reply_refs_t dmr, mach_port_t reply_port)
{
if (slowpath(!MACH_PORT_VALID(reply_port))) {
DISPATCH_CLIENT_CRASH(reply_port, "Invalid reply port");
@@ -794,7 +726,6 @@
mach_msg_size_t siz, msgsiz = 0;
mach_msg_return_t kr;
mach_msg_option_t options;
- mach_port_t notify = MACH_PORT_NULL;
siz = mach_vm_round_page(DISPATCH_MACH_RECEIVE_MAX_INLINE_MESSAGE_SIZE +
DISPATCH_MACH_TRAILER_SIZE);
hdr = alloca(siz);
@@ -803,17 +734,12 @@
*(char*)p = 0; // ensure alloca buffer doesn't overlap with stack guard
}
options = DISPATCH_MACH_RCV_OPTIONS & (~MACH_RCV_VOUCHER);
- if (MACH_PORT_VALID(send)) {
- notify = send;
- options |= MACH_RCV_SYNC_WAIT;
- }
-
retry:
_dispatch_debug_machport(reply_port);
_dispatch_debug("machport[0x%08x]: MACH_RCV_MSG %s", reply_port,
(options & MACH_RCV_TIMEOUT) ? "poll" : "wait");
kr = mach_msg(hdr, options, 0, siz, reply_port, MACH_MSG_TIMEOUT_NONE,
- notify);
+ MACH_PORT_NULL);
hdr_copyout_addr = hdr;
_dispatch_debug_machport(reply_port);
_dispatch_debug("machport[0x%08x]: MACH_RCV_MSG (size %u, opts 0x%x) "
@@ -861,9 +787,8 @@
if (shrink) hdr = hdr2 = shrink;
}
break;
- case MACH_RCV_INVALID_NOTIFY:
default:
- DISPATCH_INTERNAL_CRASH(kr, "Unexpected error from mach_msg_receive");
+ dispatch_assume_zero(kr);
break;
}
_dispatch_mach_msg_reply_received(dm, dmr, hdr->msgh_local_port);
@@ -1099,20 +1024,15 @@
} else {
clear_voucher = _voucher_mach_msg_set(msg, voucher);
}
- if (qos) {
+ if (qos && _dispatch_evfilt_machport_direct_enabled) {
opts |= MACH_SEND_OVERRIDE;
- msg_priority = (mach_msg_priority_t)
- _dispatch_priority_compute_propagated(
- _dispatch_qos_to_pp(qos), 0);
+ msg_priority = (mach_msg_priority_t)_dispatch_qos_to_pp(qos);
}
}
_dispatch_debug_machport(msg->msgh_remote_port);
if (reply_port) _dispatch_debug_machport(reply_port);
if (msg_opts & DISPATCH_MACH_WAIT_FOR_REPLY) {
if (msg_opts & DISPATCH_MACH_OWNED_REPLY_PORT) {
- if (_dispatch_use_mach_special_reply_port()) {
- opts |= MACH_SEND_SYNC_OVERRIDE;
- }
_dispatch_clear_thread_reply_port(reply_port);
}
_dispatch_mach_reply_waiter_register(dm, dmr, reply_port, dmsg,
@@ -1165,6 +1085,13 @@
if (!(msg_opts & DISPATCH_MACH_WAIT_FOR_REPLY) && !kr && reply_port &&
!(_dispatch_unote_registered(dmrr) &&
dmrr->du_ident == reply_port)) {
+ if (!dmrr->du_is_direct &&
+ _dispatch_queue_get_current() != &_dispatch_mgr_q) {
+ // reply receive kevent must be installed on the manager queue
+ dm->dm_needs_mgr = 1;
+ dmsg->dmsg_options = msg_opts | DISPATCH_MACH_REGISTER_FOR_REPLY;
+ goto out;
+ }
_dispatch_mach_reply_kevent_register(dm, reply_port, dmsg);
}
if (unlikely(!is_reply && dmsg == dsrr->dmsr_checkin &&
@@ -1204,9 +1131,6 @@
#pragma mark -
#pragma mark dispatch_mach_send_refs_t
-#define _dmsr_state_needs_lock_override(dq_state, qos) \
- unlikely(qos < _dq_state_max_qos(dq_state))
-
DISPATCH_ALWAYS_INLINE
static inline dispatch_qos_t
_dmsr_state_max_qos(uint64_t dmsr_state)
@@ -1246,8 +1170,11 @@
#define _dispatch_mach_send_pop_head(dmsr, head) \
os_mpsc_pop_head(dmsr, dmsr, head, do_next)
-#define dm_push(dm, dc, qos) \
- _dispatch_queue_push((dm)->_as_dq, dc, qos)
+#define dm_push(dm, dc, qos) ({ \
+ dispatch_queue_t _dq = (dm)->_as_dq; \
+ dispatch_assert(dx_vtable(_dq)->do_push == _dispatch_queue_push); \
+ _dispatch_queue_push(_dq, dc, qos); \
+ })
DISPATCH_ALWAYS_INLINE
static inline bool
@@ -1299,7 +1226,8 @@
dmsg = (dispatch_mach_msg_t)dc;
dmr = NULL;
} else {
- if (_dispatch_unote_registered(dmsr) &&
+ if ((_dispatch_unote_registered(dmsr) ||
+ !dm->dm_recv_refs->du_is_direct) &&
(_dispatch_queue_get_current() != &_dispatch_mgr_q)) {
// send kevent must be uninstalled on the manager queue
needs_mgr = true;
@@ -1393,7 +1321,7 @@
} else {
qos = 0;
}
- if (!disconnecting) dx_wakeup(dm, qos, DISPATCH_WAKEUP_MAKE_DIRTY);
+ if (!disconnecting) dx_wakeup(dm, qos, DISPATCH_WAKEUP_FLUSH);
}
return returning_send_result;
}
@@ -1404,7 +1332,7 @@
dispatch_mach_send_invoke_flags_t send_flags)
{
dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
- dispatch_lock owner_self = _dispatch_lock_value_for_self();
+ dispatch_lock_owner tid_self = _dispatch_tid_self();
uint64_t old_state, new_state;
uint64_t canlock_mask = DISPATCH_MACH_STATE_UNLOCK_MASK;
@@ -1422,18 +1350,18 @@
os_atomic_rmw_loop2o(dmsr, dmsr_state, old_state, new_state, acquire, {
new_state = old_state;
if (unlikely((old_state & canlock_mask) != canlock_state)) {
- if (!(send_flags & DM_SEND_INVOKE_MAKE_DIRTY)) {
+ if (!(send_flags & DM_SEND_INVOKE_FLUSH)) {
os_atomic_rmw_loop_give_up(break);
}
new_state |= DISPATCH_MACH_STATE_DIRTY;
} else {
- if (_dmsr_state_needs_lock_override(old_state, oq_floor)) {
+ if (_dispatch_queue_should_override_self(old_state, oq_floor)) {
os_atomic_rmw_loop_give_up({
oq_floor = _dispatch_queue_override_self(old_state);
goto retry;
});
}
- new_state |= owner_self;
+ new_state |= tid_self;
new_state &= ~DISPATCH_MACH_STATE_DIRTY;
new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
@@ -1491,14 +1419,14 @@
{
dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
uint64_t old_state, new_state, state_flags = 0;
- dispatch_tid owner;
+ dispatch_lock_owner owner;
bool wakeup;
// <rdar://problem/25896179> when pushing a send barrier that destroys
// the last reference to this channel, and the send queue is already
// draining on another thread, the send barrier may run as soon as
// _dispatch_mach_send_push_inline() returns.
- _dispatch_retain_2(dm);
+ _dispatch_retain(dm);
wakeup = _dispatch_mach_send_push_inline(dmsr, dc);
if (wakeup) {
@@ -1529,7 +1457,7 @@
_dispatch_wqthread_override_start_check_owner(owner, qos,
&dmsr->dmsr_state_lock.dul_lock);
}
- return _dispatch_release_2_tailcall(dm);
+ return _dispatch_release_tailcall(dm);
}
dispatch_wakeup_flags_t wflags = 0;
@@ -1537,14 +1465,14 @@
_dispatch_mach_send_barrier_drain_push(dm, qos);
} else if (wakeup || dmsr->dmsr_disconnect_cnt ||
(dm->dq_atomic_flags & DSF_CANCELED)) {
- wflags = DISPATCH_WAKEUP_MAKE_DIRTY | DISPATCH_WAKEUP_CONSUME_2;
+ wflags = DISPATCH_WAKEUP_FLUSH | DISPATCH_WAKEUP_CONSUME;
} else if (old_state & DISPATCH_MACH_STATE_PENDING_BARRIER) {
- wflags = DISPATCH_WAKEUP_CONSUME_2;
+ wflags = DISPATCH_WAKEUP_OVERRIDING | DISPATCH_WAKEUP_CONSUME;
}
if (wflags) {
return dx_wakeup(dm, qos, wflags);
}
- return _dispatch_release_2_tailcall(dm);
+ return _dispatch_release_tailcall(dm);
}
DISPATCH_NOINLINE
@@ -1554,9 +1482,9 @@
dispatch_mach_send_invoke_flags_t send_flags)
{
dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
- dispatch_lock owner_self = _dispatch_lock_value_for_self();
+ dispatch_lock_owner tid_self = _dispatch_tid_self();
uint64_t old_state, new_state, canlock_mask, state_flags = 0;
- dispatch_tid owner;
+ dispatch_lock_owner owner;
bool wakeup = _dispatch_mach_send_push_inline(dmsr, dou);
if (wakeup) {
@@ -1569,7 +1497,7 @@
new_state = _dmsr_state_merge_override(old_state, qos);
new_state |= state_flags;
});
- dx_wakeup(dm, qos, DISPATCH_WAKEUP_MAKE_DIRTY);
+ dx_wakeup(dm, qos, DISPATCH_WAKEUP_FLUSH);
return false;
}
@@ -1580,7 +1508,7 @@
new_state = _dmsr_state_merge_override(old_state, qos);
new_state |= state_flags;
if (likely((old_state & canlock_mask) == 0)) {
- new_state |= owner_self;
+ new_state |= tid_self;
new_state &= ~DISPATCH_MACH_STATE_DIRTY;
new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
@@ -1593,7 +1521,7 @@
os_atomic_rmw_loop_give_up(return false);
}
if (likely((old_state & canlock_mask) == 0)) {
- new_state |= owner_self;
+ new_state |= tid_self;
new_state &= ~DISPATCH_MACH_STATE_DIRTY;
new_state &= ~DISPATCH_MACH_STATE_RECEIVED_OVERRIDE;
new_state &= ~DISPATCH_MACH_STATE_PENDING_BARRIER;
@@ -1611,7 +1539,7 @@
}
if (old_state & DISPATCH_MACH_STATE_PENDING_BARRIER) {
- dx_wakeup(dm, qos, 0);
+ dx_wakeup(dm, qos, DISPATCH_WAKEUP_OVERRIDING);
return false;
}
@@ -1647,7 +1575,7 @@
DISPATCH_ASSERT_ON_MANAGER_QUEUE();
dm->dm_send_refs->du_ident = send;
dispatch_assume(_dispatch_unote_register(dm->dm_send_refs,
- DISPATCH_WLH_ANON, 0));
+ DISPATCH_WLH_MANAGER, 0));
}
void
@@ -1661,7 +1589,7 @@
if (data & dmsr->du_fflags) {
_dispatch_mach_send_invoke(dm, DISPATCH_INVOKE_MANAGER_DRAIN,
- DM_SEND_INVOKE_MAKE_DIRTY);
+ DM_SEND_INVOKE_FLUSH);
}
}
@@ -1672,7 +1600,7 @@
{
mach_error_t error;
dispatch_mach_reason_t reason = _dispatch_mach_msg_get_reason(dmsg, &error);
- if (reason == DISPATCH_MACH_MESSAGE_RECEIVED || !dm->dm_is_xpc ||
+ if (!dm->dm_is_xpc ||
!_dispatch_mach_xpc_hooks->dmxh_direct_message_handler(
dm->dm_recv_refs->dmrr_handler_ctxt, reason, dmsg, error)) {
// Not XPC client or not a message that XPC can handle inline - push
@@ -1721,25 +1649,15 @@
}
DISPATCH_ALWAYS_INLINE
-static inline dispatch_qos_t
-_dispatch_mach_priority_propagate(mach_msg_option_t options,
- pthread_priority_t *msg_pp)
+static inline pthread_priority_t
+_dispatch_mach_priority_propagate(mach_msg_option_t options)
{
#if DISPATCH_USE_NOIMPORTANCE_QOS
- if (options & MACH_SEND_NOIMPORTANCE) {
- *msg_pp = 0;
- return 0;
- }
+ if (options & MACH_SEND_NOIMPORTANCE) return 0;
+#else
+ (void)options;
#endif
- unsigned int flags = DISPATCH_PRIORITY_PROPAGATE_CURRENT;
- if ((options & DISPATCH_MACH_WAIT_FOR_REPLY) &&
- (options & DISPATCH_MACH_OWNED_REPLY_PORT) &&
- _dispatch_use_mach_special_reply_port()) {
- flags |= DISPATCH_PRIORITY_PROPAGATE_FOR_SYNC_IPC;
- }
- *msg_pp = _dispatch_priority_compute_propagated(0, flags);
- // TODO: remove QoS contribution of sync IPC messages to send queue
- return _dispatch_qos_from_pp(*msg_pp);
+ return _dispatch_priority_propagate();
}
DISPATCH_NOINLINE
@@ -1752,15 +1670,14 @@
DISPATCH_CLIENT_CRASH(dmsg->do_next, "Message already enqueued");
}
dispatch_retain(dmsg);
- pthread_priority_t msg_pp;
- dispatch_qos_t qos = _dispatch_mach_priority_propagate(options, &msg_pp);
+ pthread_priority_t priority = _dispatch_mach_priority_propagate(options);
options |= _dispatch_mach_send_options();
dmsg->dmsg_options = options;
mach_msg_header_t *msg = _dispatch_mach_msg_get_msg(dmsg);
dmsg->dmsg_reply = _dispatch_mach_msg_get_reply_port(dmsg);
bool is_reply = (MACH_MSGH_BITS_REMOTE(msg->msgh_bits) ==
MACH_MSG_TYPE_MOVE_SEND_ONCE);
- dmsg->dmsg_priority = msg_pp;
+ dmsg->dmsg_priority = priority;
dmsg->dmsg_voucher = _voucher_copy();
_dispatch_voucher_debug("mach-msg[%p] set", dmsg->dmsg_voucher, dmsg);
@@ -1783,7 +1700,7 @@
dispatch_object_t dou = { ._dmsg = dmsg };
if (dc_wait) dou._dc = dc_wait;
returning_send_result = _dispatch_mach_send_push_and_trydrain(dm, dou,
- qos, send_flags);
+ _dispatch_qos_from_pp(priority), send_flags);
}
if (returning_send_result) {
_dispatch_voucher_debug("mach-msg[%p] clear", dmsg->dmsg_voucher, dmsg);
@@ -1834,7 +1751,6 @@
dispatch_mach_msg_t dmsg, mach_msg_option_t options,
bool *returned_send_result)
{
- mach_port_t send = MACH_PORT_NULL;
mach_port_t reply_port = _dispatch_mach_msg_get_reply_port(dmsg);
if (!reply_port) {
// use per-thread mach reply port <rdar://24597802>
@@ -1845,7 +1761,6 @@
hdr->msgh_local_port = reply_port;
options |= DISPATCH_MACH_OWNED_REPLY_PORT;
}
- options |= DISPATCH_MACH_WAIT_FOR_REPLY;
dispatch_mach_reply_refs_t dmr;
#if DISPATCH_DEBUG
@@ -1866,13 +1781,8 @@
*returned_send_result = _dispatch_mach_send_msg(dm, dmsg, &dc_wait,options);
if (options & DISPATCH_MACH_OWNED_REPLY_PORT) {
_dispatch_clear_thread_reply_port(reply_port);
- if (_dispatch_use_mach_special_reply_port()) {
- // link special reply port to send right for remote receive right
- // TODO: extend to pre-connect phase <rdar://problem/31823384>
- send = dm->dm_send_refs->dmsr_send;
- }
}
- dmsg = _dispatch_mach_msg_reply_recv(dm, dmr, reply_port, send);
+ dmsg = _dispatch_mach_msg_reply_recv(dm, dmr, reply_port);
#if DISPATCH_DEBUG
free(dmr);
#endif
@@ -1888,6 +1798,7 @@
dispatch_mach_msg_t reply;
dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
options &= ~DISPATCH_MACH_OPTIONS_MASK;
+ options |= DISPATCH_MACH_WAIT_FOR_REPLY;
reply = _dispatch_mach_send_and_wait_for_reply(dm, dmsg, options,
&returned_send_result);
dispatch_assert(!returned_send_result);
@@ -1908,6 +1819,7 @@
dispatch_mach_msg_t reply;
dispatch_assert_zero(options & DISPATCH_MACH_OPTIONS_MASK);
options &= ~DISPATCH_MACH_OPTIONS_MASK;
+ options |= DISPATCH_MACH_WAIT_FOR_REPLY;
options |= DISPATCH_MACH_RETURN_IMMEDIATE_SEND_RESULT;
reply = _dispatch_mach_send_and_wait_for_reply(dm, dmsg, options,
&returned_send_result);
@@ -1965,8 +1877,8 @@
}
if (MACH_PORT_VALID(dmsr->dmsr_send)) {
_dispatch_mach_msg_disconnected(dm, MACH_PORT_NULL, dmsr->dmsr_send);
- dmsr->dmsr_send = MACH_PORT_NULL;
}
+ dmsr->dmsr_send = MACH_PORT_NULL;
if (dmsr->dmsr_checkin) {
_dispatch_mach_msg_not_sent(dm, dmsr->dmsr_checkin);
dmsr->dmsr_checkin = NULL;
@@ -1977,14 +1889,11 @@
TAILQ_REMOVE(&dm->dm_send_refs->dmsr_replies, dmr, dmr_list);
_TAILQ_MARK_NOT_ENQUEUED(dmr, dmr_list);
if (_dispatch_unote_registered(dmr)) {
- if (!_dispatch_mach_reply_kevent_unregister(dm, dmr,
- DU_UNREGISTER_DISCONNECTED)) {
- TAILQ_INSERT_HEAD(&dm->dm_send_refs->dmsr_replies, dmr,
- dmr_list);
- }
+ _dispatch_mach_reply_kevent_unregister(dm, dmr,
+ DU_UNREGISTER_DISCONNECTED);
} else {
_dispatch_mach_reply_waiter_unregister(dm, dmr,
- DU_UNREGISTER_DISCONNECTED);
+ DU_UNREGISTER_DISCONNECTED);
}
}
disconnected = TAILQ_EMPTY(&dm->dm_send_refs->dmsr_replies);
@@ -2008,18 +1917,8 @@
dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
mach_port_t local_port = (mach_port_t)dmrr->du_ident;
if (local_port) {
- // handle the deferred delete case properly, similar to what
- // _dispatch_source_invoke2() does
- dispatch_queue_flags_t dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
- if ((dqf & DSF_DEFERRED_DELETE) && !(dqf & DSF_ARMED)) {
- _dispatch_source_refs_unregister(dm->_as_ds,
- DU_UNREGISTER_IMMEDIATE_DELETE);
- dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
- } else if (!(dqf & DSF_DEFERRED_DELETE) && !(dqf & DSF_DELETED)) {
- _dispatch_source_refs_unregister(dm->_as_ds, 0);
- dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
- }
- if ((dqf & DSF_STATE_MASK) == DSF_DELETED) {
+ _dispatch_source_refs_unregister(dm->_as_ds, 0);
+ if ((dm->dq_atomic_flags & DSF_STATE_MASK) == DSF_DELETED) {
_dispatch_mach_msg_disconnected(dm, local_port, MACH_PORT_NULL);
dmrr->du_ident = 0;
} else {
@@ -2029,10 +1928,6 @@
_dispatch_queue_atomic_flags_set_and_clear(dm->_as_dq, DSF_DELETED,
DSF_ARMED | DSF_DEFERRED_DELETE);
}
-
- if (dm->dm_send_refs->dmsr_disconnect_cnt) {
- uninstalled = false; // <rdar://problem/31233110>
- }
if (uninstalled) dm->dm_uninstalled = uninstalled;
}
@@ -2128,21 +2023,8 @@
if (slowpath(!dm->dm_connect_handler_called)) {
_dispatch_mach_connect_invoke(dm);
}
- if (reason == DISPATCH_MACH_MESSAGE_RECEIVED &&
- (_dispatch_queue_atomic_flags(dm->_as_dq) & DSF_CANCELED)) {
- // <rdar://problem/32184699> Do not deliver message received
- // after cancellation: _dispatch_mach_merge_msg can be preempted
- // for a long time between clearing DSF_ARMED but before
- // enqueuing the message, allowing for cancellation to complete,
- // and then the message event to be delivered.
- //
- // This makes XPC unhappy because some of these messages are
- // port-destroyed notifications that can cause it to try to
- // reconnect on a channel that is almost fully canceled
- } else {
- _dispatch_client_callout4(dmrr->dmrr_handler_ctxt, reason, dmsg,
- err, dmrr->dmrr_handler_func);
- }
+ _dispatch_client_callout4(dmrr->dmrr_handler_ctxt, reason, dmsg,
+ err, dmrr->dmrr_handler_func);
}
_dispatch_perfmon_workitem_inc();
});
@@ -2292,37 +2174,22 @@
}
static void
-_dispatch_mach_install(dispatch_mach_t dm, dispatch_wlh_t wlh,
- dispatch_priority_t pri)
+_dispatch_mach_install(dispatch_mach_t dm, dispatch_priority_t pri,
+ dispatch_wlh_t wlh)
{
dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
uint32_t disconnect_cnt;
+ if (!dm->dq_wlh && wlh) {
+ _dispatch_queue_class_record_wlh_hierarchy(dm, wlh);
+ }
if (dmrr->du_ident) {
- _dispatch_source_refs_register(dm->_as_ds, wlh, pri);
- dispatch_assert(dmrr->du_is_direct);
+ _dispatch_source_refs_register(dm->_as_ds, pri);
}
-
- if (dm->dm_is_xpc) {
- bool monitor_sigterm;
- if (_dispatch_mach_xpc_hooks->version < 3) {
- monitor_sigterm = true;
- } else if (!_dispatch_mach_xpc_hooks->dmxh_enable_sigterm_notification){
- monitor_sigterm = true;
- } else {
- monitor_sigterm =
- _dispatch_mach_xpc_hooks->dmxh_enable_sigterm_notification(
- dm->dm_recv_refs->dmrr_handler_ctxt);
- }
- if (monitor_sigterm) {
- dispatch_xpc_term_refs_t _dxtr =
- dux_create(&_dispatch_xpc_type_sigterm, SIGTERM, 0)._dxtr;
- _dxtr->du_owner_wref = _dispatch_ptr2wref(dm);
- dm->dm_xpc_term_refs = _dxtr;
- _dispatch_unote_register(dm->dm_xpc_term_refs, wlh, pri);
- }
+ if (dm->dm_xpc_term_refs) {
+ _dispatch_unote_register(dm->dm_xpc_term_refs, dm->dq_wlh, pri);
}
- if (!dm->dq_priority) {
+ if (dmrr->du_is_direct && !dm->dq_priority) {
// _dispatch_mach_reply_kevent_register assumes this has been done
// which is unlike regular sources or queues, the DEFAULTQUEUE flag
// is used so that the priority of the channel doesn't act as
@@ -2337,17 +2204,21 @@
}
void
-_dispatch_mach_finalize_activation(dispatch_mach_t dm, bool *allow_resume)
+_dispatch_mach_finalize_activation(dispatch_mach_t dm)
{
- dispatch_priority_t pri;
- dispatch_wlh_t wlh;
+ dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
// call "super"
- _dispatch_queue_finalize_activation(dm->_as_dq, allow_resume);
+ _dispatch_queue_finalize_activation(dm->_as_dq);
- if (!dm->ds_is_installed) {
- pri = _dispatch_queue_compute_priority_and_wlh(dm->_as_dq, &wlh);
- if (pri) _dispatch_mach_install(dm, wlh, pri);
+ if (dmrr->du_is_direct && !dm->ds_is_installed) {
+ dispatch_source_t ds = dm->_as_ds;
+ dispatch_priority_t pri = _dispatch_source_compute_kevent_priority(ds);
+ if (pri) {
+ dispatch_wlh_t wlh = dm->dq_wlh;
+ if (!wlh) wlh = _dispatch_queue_class_compute_wlh(dm);
+ _dispatch_mach_install(dm, pri, wlh);
+ }
}
}
@@ -2379,24 +2250,8 @@
dispatch_mach_t dm = dou._dm;
dispatch_queue_wakeup_target_t retq = NULL;
dispatch_queue_t dq = _dispatch_queue_get_current();
- dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
- dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
- dispatch_queue_flags_t dqf = 0;
- if (!(flags & DISPATCH_INVOKE_MANAGER_DRAIN) && dmrr &&
- _dispatch_unote_wlh_changed(dmrr, _dispatch_get_wlh())) {
- dqf = _dispatch_queue_atomic_flags_set_orig(dm->_as_dq,
- DSF_WLH_CHANGED);
- if (!(dqf & DSF_WLH_CHANGED)) {
- if (dm->dm_is_xpc) {
- _dispatch_bug_deprecated("Changing target queue "
- "hierarchy after xpc connection was activated");
- } else {
- _dispatch_bug_deprecated("Changing target queue "
- "hierarchy after mach channel was activated");
- }
- }
- }
+ flags |= DISPATCH_INVOKE_DISALLOW_SYNC_WAITERS;
// This function performs all mach channel actions. Each action is
// responsible for verifying that it takes place on the appropriate queue.
@@ -2406,12 +2261,20 @@
// The order of tests here in invoke and in wakeup should be consistent.
+ dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
+ dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
+ dispatch_queue_t dkq = &_dispatch_mgr_q;
+
+ if (dmrr->du_is_direct) {
+ dkq = dm->do_targetq;
+ }
+
if (unlikely(!dm->ds_is_installed)) {
// The channel needs to be installed on the kevent queue.
- if (unlikely(flags & DISPATCH_INVOKE_MANAGER_DRAIN)) {
- return dm->do_targetq;
+ if (dq != dkq) {
+ return dkq;
}
- _dispatch_mach_install(dm, _dispatch_get_wlh(),_dispatch_get_basepri());
+ _dispatch_mach_install(dm, _dispatch_get_basepri(),_dispatch_get_wlh());
_dispatch_perfmon_workitem_inc();
}
@@ -2424,12 +2287,13 @@
}
}
- if (!retq && _dispatch_unote_registered(dmrr)) {
+ dispatch_queue_flags_t dqf = 0;
+ if (!retq && dmrr->du_is_direct) {
if (_dispatch_mach_tryarm(dm, &dqf)) {
_dispatch_unote_resume(dmrr);
if (dq == dm->do_targetq && !dq->do_targetq && !dmsr->dmsr_tail &&
(dq->dq_priority & DISPATCH_PRIORITY_FLAG_OVERCOMMIT) &&
- _dispatch_wlh_should_poll_unote(dmrr)) {
+ dmrr->du_wlh != DISPATCH_WLH_GLOBAL) {
// try to redrive the drain from under the lock for channels
// targeting an overcommit root queue to avoid parking
// when the next message has already fired
@@ -2443,7 +2307,7 @@
if (dmsr->dmsr_tail) {
bool requires_mgr = dm->dm_needs_mgr || (dmsr->dmsr_disconnect_cnt &&
- _dispatch_unote_registered(dmsr));
+ (_dispatch_unote_registered(dmsr) || !dmrr->du_is_direct));
if (!os_atomic_load2o(dmsr, dmsr_notification_armed, relaxed) ||
(dqf & DSF_CANCELED) || dmsr->dmsr_disconnect_cnt) {
// The channel has pending messages to send.
@@ -2493,8 +2357,7 @@
_dispatch_mach_invoke(dispatch_mach_t dm,
dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags)
{
- _dispatch_queue_class_invoke(dm, dic, flags,
- DISPATCH_INVOKE_DISALLOW_SYNC_WAITERS, _dispatch_mach_invoke2);
+ _dispatch_queue_class_invoke(dm, dic, flags, _dispatch_mach_invoke2);
}
void
@@ -2505,12 +2368,18 @@
// The order of tests here in probe and in invoke should be consistent.
dispatch_mach_send_refs_t dmsr = dm->dm_send_refs;
+ dispatch_mach_recv_refs_t dmrr = dm->dm_recv_refs;
+ dispatch_queue_wakeup_target_t dkq = DISPATCH_QUEUE_WAKEUP_MGR;
dispatch_queue_wakeup_target_t tq = DISPATCH_QUEUE_WAKEUP_NONE;
dispatch_queue_flags_t dqf = _dispatch_queue_atomic_flags(dm->_as_dq);
+ if (dmrr->du_is_direct) {
+ dkq = DISPATCH_QUEUE_WAKEUP_TARGET;
+ }
+
if (!dm->ds_is_installed) {
// The channel needs to be installed on the kevent queue.
- tq = DISPATCH_QUEUE_WAKEUP_TARGET;
+ tq = dkq;
goto done;
}
@@ -2527,7 +2396,7 @@
if (dmsr->dmsr_tail) {
bool requires_mgr = dm->dm_needs_mgr || (dmsr->dmsr_disconnect_cnt &&
- _dispatch_unote_registered(dmsr));
+ (_dispatch_unote_registered(dmsr) || !dmrr->du_is_direct));
if (!os_atomic_load2o(dmsr, dmsr_notification_armed, relaxed) ||
(dqf & DSF_CANCELED) || dmsr->dmsr_disconnect_cnt) {
if (unlikely(requires_mgr)) {
@@ -2552,12 +2421,13 @@
}
done:
- if ((tq == DISPATCH_QUEUE_WAKEUP_TARGET) &&
- dm->do_targetq == &_dispatch_mgr_q) {
- tq = DISPATCH_QUEUE_WAKEUP_MGR;
+ if (tq) {
+ return _dispatch_queue_class_wakeup(dm->_as_dq, qos, flags, tq);
+ } else if (qos) {
+ return _dispatch_queue_class_override_drainer(dm->_as_dq, qos, flags);
+ } else if (flags & DISPATCH_WAKEUP_CONSUME) {
+ return _dispatch_release_tailcall(dm);
}
-
- return _dispatch_queue_class_wakeup(dm->_as_dq, qos, flags, tq);
}
static void
@@ -2592,7 +2462,7 @@
_dispatch_barrier_async_detached_f(dm->_as_dq, dm,
_dispatch_mach_sigterm_invoke);
} else {
- dx_wakeup(dm, _dispatch_qos_from_pp(pp), DISPATCH_WAKEUP_MAKE_DIRTY);
+ dx_wakeup(dm, _dispatch_qos_from_pp(pp), DISPATCH_WAKEUP_FLUSH);
}
}
@@ -2607,15 +2477,9 @@
slowpath(destructor && !msg)) {
DISPATCH_CLIENT_CRASH(size, "Empty message");
}
-
- dispatch_mach_msg_t dmsg;
- size_t msg_size = sizeof(struct dispatch_mach_msg_s);
- if (!destructor && os_add_overflow(msg_size,
- (size - sizeof(dmsg->dmsg_msg)), &msg_size)) {
- DISPATCH_CLIENT_CRASH(size, "Message size too large");
- }
-
- dmsg = _dispatch_object_alloc(DISPATCH_VTABLE(mach_msg), msg_size);
+ dispatch_mach_msg_t dmsg = _dispatch_alloc(DISPATCH_VTABLE(mach_msg),
+ sizeof(struct dispatch_mach_msg_s) +
+ (destructor ? 0 : size - sizeof(dmsg->dmsg_msg)));
if (destructor) {
dmsg->dmsg_msg = msg;
} else if (msg) {
@@ -2632,8 +2496,7 @@
}
void
-_dispatch_mach_msg_dispose(dispatch_mach_msg_t dmsg,
- DISPATCH_UNUSED bool *allow_free)
+_dispatch_mach_msg_dispose(dispatch_mach_msg_t dmsg)
{
if (dmsg->dmsg_voucher) {
_voucher_release(dmsg->dmsg_voucher);
@@ -2676,7 +2539,8 @@
size_t offset = 0;
offset += dsnprintf(&buf[offset], bufsiz - offset, "%s[%p] = { ",
dx_kind(dmsg), dmsg);
- offset += _dispatch_object_debug_attr(dmsg, buf + offset, bufsiz - offset);
+ offset += dsnprintf(&buf[offset], bufsiz - offset, "xrefcnt = 0x%x, "
+ "refcnt = 0x%x, ", dmsg->do_xref_cnt + 1, dmsg->do_ref_cnt + 1);
offset += dsnprintf(&buf[offset], bufsiz - offset, "opts/err = 0x%x, "
"msgh[%p] = { ", dmsg->dmsg_options, dmsg->dmsg_buf);
mach_msg_header_t *hdr = _dispatch_mach_msg_get_msg(dmsg);
diff --git a/src/mach_internal.h b/src/mach_internal.h
index 8c8edd8..8600a38 100644
--- a/src/mach_internal.h
+++ b/src/mach_internal.h
@@ -99,8 +99,8 @@
void _dispatch_mach_msg_async_reply_invoke(dispatch_continuation_t dc,
dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags);
-void _dispatch_mach_dispose(dispatch_mach_t dm, bool *allow_free);
-void _dispatch_mach_finalize_activation(dispatch_mach_t dm, bool *allow_resume);
+void _dispatch_mach_dispose(dispatch_mach_t dm);
+void _dispatch_mach_finalize_activation(dispatch_mach_t dm);
void _dispatch_mach_invoke(dispatch_mach_t dm, dispatch_invoke_context_t dic,
dispatch_invoke_flags_t flags);
void _dispatch_mach_wakeup(dispatch_mach_t dm, dispatch_qos_t qos,
@@ -116,7 +116,7 @@
void _dispatch_xpc_sigterm_merge(dispatch_unote_t du, uint32_t flags,
uintptr_t data, uintptr_t status, pthread_priority_t pp);
-void _dispatch_mach_msg_dispose(dispatch_mach_msg_t dmsg, bool *allow_free);
+void _dispatch_mach_msg_dispose(dispatch_mach_msg_t dmsg);
void _dispatch_mach_msg_invoke(dispatch_mach_msg_t dmsg,
dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags);
size_t _dispatch_mach_msg_debug(dispatch_mach_msg_t dmsg, char* buf,
diff --git a/src/object.c b/src/object.c
index 43f580b..1ca41bc 100644
--- a/src/object.c
+++ b/src/object.c
@@ -37,28 +37,14 @@
_os_object_t
_os_object_retain_internal(_os_object_t obj)
{
- return _os_object_retain_internal_n_inline(obj, 1);
-}
-
-DISPATCH_NOINLINE
-_os_object_t
-_os_object_retain_internal_n(_os_object_t obj, uint16_t n)
-{
- return _os_object_retain_internal_n_inline(obj, n);
+ return _os_object_retain_internal_inline(obj);
}
DISPATCH_NOINLINE
void
_os_object_release_internal(_os_object_t obj)
{
- return _os_object_release_internal_n_inline(obj, 1);
-}
-
-DISPATCH_NOINLINE
-void
-_os_object_release_internal_n(_os_object_t obj, uint16_t n)
-{
- return _os_object_release_internal_n_inline(obj, n);
+ return _os_object_release_internal_inline(obj);
}
DISPATCH_NOINLINE
@@ -138,7 +124,7 @@
#pragma mark dispatch_object_t
void *
-_dispatch_object_alloc(const void *vtable, size_t size)
+_dispatch_alloc(const void *vtable, size_t size)
{
#if OS_OBJECT_HAVE_OBJC1
const struct dispatch_object_vtable_s *_vtable = vtable;
@@ -152,27 +138,6 @@
}
void
-_dispatch_object_finalize(dispatch_object_t dou)
-{
-#if USE_OBJC
- objc_destructInstance((id)dou._do);
-#else
- (void)dou;
-#endif
-}
-
-void
-_dispatch_object_dealloc(dispatch_object_t dou)
-{
- // so that ddt doesn't pick up bad objects when malloc reuses this memory
- dou._os_obj->os_obj_isa = NULL;
-#if OS_OBJECT_HAVE_OBJC1
- dou._do->do_vtable = NULL;
-#endif
- free(dou._os_obj);
-}
-
-void
dispatch_retain(dispatch_object_t dou)
{
DISPATCH_OBJECT_TFB(_dispatch_objc_retain, dou);
@@ -186,6 +151,24 @@
_os_object_release(dou._os_obj);
}
+static void
+_dispatch_dealloc(dispatch_object_t dou)
+{
+ dispatch_queue_t tq = dou._do->do_targetq;
+ dispatch_function_t func = dou._do->do_finalizer;
+ void *ctxt = dou._do->do_ctxt;
+#if OS_OBJECT_HAVE_OBJC1
+ // so that ddt doesn't pick up bad objects when malloc reuses this memory
+ dou._do->do_vtable = NULL;
+#endif
+ _os_object_dealloc(dou._os_obj);
+
+ if (func && ctxt) {
+ dispatch_async_f(tq, ctxt, func);
+ }
+ _dispatch_release_tailcall(tq);
+}
+
#if !USE_OBJC
void
_dispatch_xref_dispose(dispatch_object_t dou)
@@ -210,26 +193,11 @@
void
_dispatch_dispose(dispatch_object_t dou)
{
- dispatch_queue_t tq = dou._do->do_targetq;
- dispatch_function_t func = dou._do->do_finalizer;
- void *ctxt = dou._do->do_ctxt;
- bool allow_free = true;
-
if (slowpath(dou._do->do_next != DISPATCH_OBJECT_LISTLESS)) {
DISPATCH_INTERNAL_CRASH(dou._do->do_next, "Release while enqueued");
}
-
- dx_dispose(dou._do, &allow_free);
-
- // Past this point, the only thing left of the object is its memory
- if (likely(allow_free)) {
- _dispatch_object_finalize(dou);
- _dispatch_object_dealloc(dou);
- }
- if (func && ctxt) {
- dispatch_async_f(tq, ctxt, func);
- }
- if (tq) _dispatch_release_tailcall(tq);
+ dx_dispose(dou._do);
+ return _dispatch_dealloc(dou);
}
void *
@@ -302,9 +270,7 @@
dispatch_resume(dispatch_object_t dou)
{
DISPATCH_OBJECT_TFB(_dispatch_objc_resume, dou);
- // the do_suspend below is not a typo. Having a do_resume but no do_suspend
- // allows for objects to support activate, but have no-ops suspend/resume
- if (dx_vtable(dou._do)->do_suspend) {
+ if (dx_vtable(dou._do)->do_resume) {
dx_vtable(dou._do)->do_resume(dou._do, false);
}
}
@@ -312,6 +278,6 @@
size_t
_dispatch_object_debug_attr(dispatch_object_t dou, char* buf, size_t bufsiz)
{
- return dsnprintf(buf, bufsiz, "xref = %d, ref = %d, ",
+ return dsnprintf(buf, bufsiz, "xrefcnt = 0x%x, refcnt = 0x%x, ",
dou._do->do_xref_cnt + 1, dou._do->do_ref_cnt + 1);
}
diff --git a/src/object.m b/src/object.m
index cc97cc3..59cbc9d 100644
--- a/src/object.m
+++ b/src/object.m
@@ -29,21 +29,10 @@
#error Objective C GC isn't supported anymore
#endif
-#if __has_include(<objc/objc-internal.h>)
#include <objc/objc-internal.h>
-#else
-extern id _Nullable objc_retain(id _Nullable obj) __asm__("_objc_retain");
-extern void objc_release(id _Nullable obj) __asm__("_objc_release");
-extern void _objc_init(void);
-extern void _objc_atfork_prepare(void);
-extern void _objc_atfork_parent(void);
-extern void _objc_atfork_child(void);
-#endif // __has_include(<objc/objc-internal.h>)
#include <objc/objc-exception.h>
#include <Foundation/NSString.h>
-// NOTE: this file must not contain any atomic operations
-
#pragma mark -
#pragma mark _os_object_t
@@ -297,11 +286,6 @@
return [nsstring stringWithFormat:format, class_getName([self class]), buf];
}
-- (void)dealloc DISPATCH_NORETURN {
- DISPATCH_INTERNAL_CRASH(0, "Calling dealloc on a dispatch object");
- [super dealloc]; // make clang happy
-}
-
@end
@implementation DISPATCH_CLASS(queue)
@@ -429,20 +413,20 @@
#if DISPATCH_COCOA_COMPAT
-void
-_dispatch_last_resort_autorelease_pool_push(dispatch_invoke_context_t dic)
+void *
+_dispatch_last_resort_autorelease_pool_push(void)
{
if (!slowpath(_os_object_debug_missing_pools)) {
- dic->dic_autorelease_pool = _dispatch_autorelease_pool_push();
+ return _dispatch_autorelease_pool_push();
}
+ return NULL;
}
void
-_dispatch_last_resort_autorelease_pool_pop(dispatch_invoke_context_t dic)
+_dispatch_last_resort_autorelease_pool_pop(void *context)
{
if (!slowpath(_os_object_debug_missing_pools)) {
- _dispatch_autorelease_pool_pop(dic->dic_autorelease_pool);
- dic->dic_autorelease_pool = NULL;
+ return _dispatch_autorelease_pool_pop(context);
}
}
diff --git a/src/object_internal.h b/src/object_internal.h
index 0060f27..61caebf 100644
--- a/src/object_internal.h
+++ b/src/object_internal.h
@@ -188,14 +188,14 @@
DISPATCH_INVOKABLE_VTABLE_HEADER(x); \
void (*const do_wakeup)(struct x##_s *, \
dispatch_qos_t, dispatch_wakeup_flags_t); \
- void (*const do_dispose)(struct x##_s *, bool *allow_free)
+ void (*const do_dispose)(struct x##_s *)
#define DISPATCH_OBJECT_VTABLE_HEADER(x) \
DISPATCH_QUEUEABLE_VTABLE_HEADER(x); \
void (*const do_set_targetq)(struct x##_s *, dispatch_queue_t); \
void (*const do_suspend)(struct x##_s *); \
void (*const do_resume)(struct x##_s *, bool activate); \
- void (*const do_finalize_activation)(struct x##_s *, bool *allow_resume); \
+ void (*const do_finalize_activation)(struct x##_s *); \
size_t (*const do_debug)(struct x##_s *, char *, size_t)
#define dx_vtable(x) (&(x)->do_vtable->_os_obj_vtable)
@@ -205,7 +205,7 @@
#define dx_hastypeflag(x, f) (dx_vtable(x)->do_type & _DISPATCH_##f##_TYPEFLAG)
#define dx_kind(x) dx_vtable(x)->do_kind
#define dx_debug(x, y, z) dx_vtable(x)->do_debug((x), (y), (z))
-#define dx_dispose(x, y) dx_vtable(x)->do_dispose(x, y)
+#define dx_dispose(x) dx_vtable(x)->do_dispose(x)
#define dx_invoke(x, y, z) dx_vtable(x)->do_invoke(x, y, z)
#define dx_push(x, y, z) dx_vtable(x)->do_push(x, y, z)
#define dx_wakeup(x, y, z) dx_vtable(x)->do_wakeup(x, y, z)
@@ -230,29 +230,32 @@
// we sign extend the 64-bit version so that a better instruction encoding is
// generated on Intel
#define DISPATCH_OBJECT_LISTLESS ((void *)0xffffffff89abcdef)
+#define DISPATCH_OBJECT_WLH_REQ ((void *)0xffffffff7009cdef)
#else
#define DISPATCH_OBJECT_LISTLESS ((void *)0x89abcdef)
+#define DISPATCH_OBJECT_WLH_REQ ((void *)0x7009cdef)
#endif
DISPATCH_ENUM(dispatch_wakeup_flags, uint32_t,
- // The caller of dx_wakeup owns two internal refcounts on the object being
- // woken up. Two are needed for WLH wakeups where two threads need
- // the object to remain valid in a non-coordinated way
- // - the thread doing the poke for the duration of the poke
- // - drainers for the duration of their drain
- DISPATCH_WAKEUP_CONSUME_2 = 0x00000001,
+ // The caller of dx_wakeup owns an internal refcount on the object being
+ // woken up
+ DISPATCH_WAKEUP_CONSUME = 0x00000001,
// Some change to the object needs to be published to drainers.
// If the drainer isn't the same thread, some scheme such as the dispatch
// queue DIRTY bit must be used and a release barrier likely has to be
// involved before dx_wakeup returns
- DISPATCH_WAKEUP_MAKE_DIRTY = 0x00000002,
+ DISPATCH_WAKEUP_FLUSH = 0x00000002,
- // This wakeup is made by a sync owner that still holds the drain lock
- DISPATCH_WAKEUP_BARRIER_COMPLETE = 0x00000004,
+ // The caller desires to apply an override on the object being woken up.
+ // When this flag is passed, the qos passed to dx_wakeup() should not be 0
+ DISPATCH_WAKEUP_OVERRIDING = 0x00000004,
+
+ // This wakeup is caused by a handoff from a slow waiter.
+ DISPATCH_WAKEUP_WAITER_HANDOFF = 0x00000008,
// This wakeup is caused by a dispatch_block_wait()
- DISPATCH_WAKEUP_BLOCK_WAIT = 0x00000008,
+ DISPATCH_WAKEUP_BLOCK_WAIT = 0x00000010,
);
typedef struct dispatch_invoke_context_s {
@@ -260,12 +263,11 @@
#if HAVE_PTHREAD_WORKQUEUE_NARROWING
uint64_t dic_next_narrow_check;
#endif
-#if DISPATCH_COCOA_COMPAT
- void *dic_autorelease_pool;
-#endif
} dispatch_invoke_context_s, *dispatch_invoke_context_t;
#if HAVE_PTHREAD_WORKQUEUE_NARROWING
+#define DISPATCH_NARROW_CHECK_INTERVAL \
+ _dispatch_time_nano2mach(50 * NSEC_PER_MSEC)
#define DISPATCH_THREAD_IS_NARROWING 1
#define dispatch_with_disabled_narrowing(dic, ...) ({ \
@@ -287,11 +289,12 @@
// This invoke is a stealer, meaning that it doesn't own the
// enqueue lock at drain lock time.
//
- // @const DISPATCH_INVOKE_WLH
- // This invoke is for a bottom WLH
+ // @const DISPATCH_INVOKE_OVERRIDING
+ // This invoke is draining the hierarchy on another root queue and needs
+ // to fake the identity of the original one.
//
DISPATCH_INVOKE_STEALING = 0x00000001,
- DISPATCH_INVOKE_WLH = 0x00000002,
+ DISPATCH_INVOKE_OVERRIDING = 0x00000002,
// Misc flags
//
@@ -358,31 +361,29 @@
#define DISPATCH_CONTINUATION_TYPE(name) \
(_DISPATCH_CONTINUATION_TYPE | DC_##name##_TYPE)
- DISPATCH_DATA_TYPE = 1 | _DISPATCH_NODE_TYPE,
- DISPATCH_MACH_MSG_TYPE = 2 | _DISPATCH_NODE_TYPE,
- DISPATCH_QUEUE_ATTR_TYPE = 3 | _DISPATCH_NODE_TYPE,
+ DISPATCH_DATA_TYPE = 1 | _DISPATCH_NODE_TYPE,
+ DISPATCH_MACH_MSG_TYPE = 2 | _DISPATCH_NODE_TYPE,
+ DISPATCH_QUEUE_ATTR_TYPE = 3 | _DISPATCH_NODE_TYPE,
- DISPATCH_IO_TYPE = 0 | _DISPATCH_IO_TYPE,
- DISPATCH_OPERATION_TYPE = 0 | _DISPATCH_OPERATION_TYPE,
- DISPATCH_DISK_TYPE = 0 | _DISPATCH_DISK_TYPE,
+ DISPATCH_IO_TYPE = 0 | _DISPATCH_IO_TYPE,
+ DISPATCH_OPERATION_TYPE = 0 | _DISPATCH_OPERATION_TYPE,
+ DISPATCH_DISK_TYPE = 0 | _DISPATCH_DISK_TYPE,
- DISPATCH_QUEUE_LEGACY_TYPE = 1 | _DISPATCH_QUEUE_TYPE,
- DISPATCH_QUEUE_SERIAL_TYPE = 2 | _DISPATCH_QUEUE_TYPE,
- DISPATCH_QUEUE_CONCURRENT_TYPE = 3 | _DISPATCH_QUEUE_TYPE,
- DISPATCH_QUEUE_GLOBAL_ROOT_TYPE = 4 | _DISPATCH_QUEUE_TYPE |
+ DISPATCH_QUEUE_LEGACY_TYPE = 1 | _DISPATCH_QUEUE_TYPE,
+ DISPATCH_QUEUE_SERIAL_TYPE = 2 | _DISPATCH_QUEUE_TYPE,
+ DISPATCH_QUEUE_CONCURRENT_TYPE = 3 | _DISPATCH_QUEUE_TYPE,
+ DISPATCH_QUEUE_GLOBAL_ROOT_TYPE = 4 | _DISPATCH_QUEUE_TYPE |
_DISPATCH_QUEUE_ROOT_TYPEFLAG,
- DISPATCH_QUEUE_NETWORK_EVENT_TYPE = 5 | _DISPATCH_QUEUE_TYPE |
+ DISPATCH_QUEUE_RUNLOOP_TYPE = 5 | _DISPATCH_QUEUE_TYPE |
_DISPATCH_QUEUE_ROOT_TYPEFLAG,
- DISPATCH_QUEUE_RUNLOOP_TYPE = 6 | _DISPATCH_QUEUE_TYPE |
- _DISPATCH_QUEUE_ROOT_TYPEFLAG,
- DISPATCH_QUEUE_MGR_TYPE = 7 | _DISPATCH_QUEUE_TYPE,
- DISPATCH_QUEUE_SPECIFIC_TYPE = 8 | _DISPATCH_QUEUE_TYPE,
+ DISPATCH_QUEUE_MGR_TYPE = 6 | _DISPATCH_QUEUE_TYPE,
+ DISPATCH_QUEUE_SPECIFIC_TYPE = 7 | _DISPATCH_QUEUE_TYPE,
- DISPATCH_SEMAPHORE_TYPE = 1 | _DISPATCH_SEMAPHORE_TYPE,
- DISPATCH_GROUP_TYPE = 2 | _DISPATCH_SEMAPHORE_TYPE,
+ DISPATCH_SEMAPHORE_TYPE = 1 | _DISPATCH_SEMAPHORE_TYPE,
+ DISPATCH_GROUP_TYPE = 2 | _DISPATCH_SEMAPHORE_TYPE,
- DISPATCH_SOURCE_KEVENT_TYPE = 1 | _DISPATCH_SOURCE_TYPE,
- DISPATCH_MACH_CHANNEL_TYPE = 2 | _DISPATCH_SOURCE_TYPE,
+ DISPATCH_SOURCE_KEVENT_TYPE = 1 | _DISPATCH_SOURCE_TYPE,
+ DISPATCH_MACH_CHANNEL_TYPE = 2 | _DISPATCH_SOURCE_TYPE,
};
@@ -449,9 +450,9 @@
struct dispatch_object_s *volatile ns##_items_head; \
unsigned long ns##_serialnum; \
const char *ns##_label; \
+ dispatch_wlh_t ns##_wlh; \
struct dispatch_object_s *volatile ns##_items_tail; \
- dispatch_priority_t ns##_priority; \
- int volatile ns##_sref_cnt
+ dispatch_priority_t ns##_priority
#else
#define _OS_MPSC_QUEUE_FIELDS(ns, __state_field__) \
struct dispatch_object_s *volatile ns##_items_head; \
@@ -462,9 +463,10 @@
/* LP64 global queue cacheline boundary */ \
unsigned long ns##_serialnum; \
const char *ns##_label; \
+ dispatch_wlh_t ns##_wlh; \
struct dispatch_object_s *volatile ns##_items_tail; \
- dispatch_priority_t ns##_priority; \
- int volatile ns##_sref_cnt
+ dispatch_priority_t ns##_priority
+ /* LP64: 32bit hole */
#endif
OS_OBJECT_INTERNAL_CLASS_DECL(os_mpsc_queue, object,
@@ -482,9 +484,7 @@
size_t _dispatch_object_debug_attr(dispatch_object_t dou, char* buf,
size_t bufsiz);
-void *_dispatch_object_alloc(const void *vtable, size_t size);
-void _dispatch_object_finalize(dispatch_object_t dou);
-void _dispatch_object_dealloc(dispatch_object_t dou);
+void *_dispatch_alloc(const void *vtable, size_t size);
#if !USE_OBJC
void _dispatch_xref_dispose(dispatch_object_t dou);
#endif
@@ -492,22 +492,17 @@
#if DISPATCH_COCOA_COMPAT
#if USE_OBJC
#include <objc/runtime.h>
-#if __has_include(<objc/objc-internal.h>)
#include <objc/objc-internal.h>
-#else
-extern void *objc_autoreleasePoolPush(void);
-extern void objc_autoreleasePoolPop(void *context);
-#endif // __has_include(<objc/objc-internal.h>)
#define _dispatch_autorelease_pool_push() \
- objc_autoreleasePoolPush()
+ objc_autoreleasePoolPush()
#define _dispatch_autorelease_pool_pop(context) \
- objc_autoreleasePoolPop(context)
+ objc_autoreleasePoolPop(context)
#else
void *_dispatch_autorelease_pool_push(void);
void _dispatch_autorelease_pool_pop(void *context);
#endif
-void _dispatch_last_resort_autorelease_pool_push(dispatch_invoke_context_t dic);
-void _dispatch_last_resort_autorelease_pool_pop(dispatch_invoke_context_t dic);
+void *_dispatch_last_resort_autorelease_pool_push(void);
+void _dispatch_last_resort_autorelease_pool_pop(void *context);
#define dispatch_invoke_with_autoreleasepool(flags, ...) ({ \
void *pool = NULL; \
@@ -523,6 +518,7 @@
do { (void)flags; __VA_ARGS__; } while (0)
#endif
+
#if USE_OBJC
OS_OBJECT_OBJC_CLASS_DECL(object);
#endif
@@ -586,20 +582,20 @@
* a barrier to perform prior to tearing down an object when the refcount
* reached -1.
*/
-#define _os_atomic_refcnt_perform2o(o, f, op, n, m) ({ \
+#define _os_atomic_refcnt_perform2o(o, f, op, m) ({ \
typeof(o) _o = (o); \
int _ref_cnt = _o->f; \
if (fastpath(_ref_cnt != _OS_OBJECT_GLOBAL_REFCNT)) { \
- _ref_cnt = os_atomic_##op##2o(_o, f, n, m); \
+ _ref_cnt = os_atomic_##op##2o(_o, f, m); \
} \
_ref_cnt; \
})
-#define _os_atomic_refcnt_add2o(o, m, n) \
- _os_atomic_refcnt_perform2o(o, m, add, n, relaxed)
+#define _os_atomic_refcnt_inc2o(o, m) \
+ _os_atomic_refcnt_perform2o(o, m, inc, relaxed)
-#define _os_atomic_refcnt_sub2o(o, m, n) \
- _os_atomic_refcnt_perform2o(o, m, sub, n, release)
+#define _os_atomic_refcnt_dec2o(o, m) \
+ _os_atomic_refcnt_perform2o(o, m, dec, release)
#define _os_atomic_refcnt_dispose_barrier2o(o, m) \
(void)os_atomic_load2o(o, m, acquire)
@@ -622,19 +618,19 @@
*
*/
#define _os_object_xrefcnt_inc(o) \
- _os_atomic_refcnt_add2o(o, os_obj_xref_cnt, 1)
+ _os_atomic_refcnt_inc2o(o, os_obj_xref_cnt)
#define _os_object_xrefcnt_dec(o) \
- _os_atomic_refcnt_sub2o(o, os_obj_xref_cnt, 1)
+ _os_atomic_refcnt_dec2o(o, os_obj_xref_cnt)
#define _os_object_xrefcnt_dispose_barrier(o) \
_os_atomic_refcnt_dispose_barrier2o(o, os_obj_xref_cnt)
-#define _os_object_refcnt_add(o, n) \
- _os_atomic_refcnt_add2o(o, os_obj_ref_cnt, n)
+#define _os_object_refcnt_inc(o) \
+ _os_atomic_refcnt_inc2o(o, os_obj_ref_cnt)
-#define _os_object_refcnt_sub(o, n) \
- _os_atomic_refcnt_sub2o(o, os_obj_ref_cnt, n)
+#define _os_object_refcnt_dec(o) \
+ _os_atomic_refcnt_dec2o(o, os_obj_ref_cnt)
#define _os_object_refcnt_dispose_barrier(o) \
_os_atomic_refcnt_dispose_barrier2o(o, os_obj_ref_cnt)
diff --git a/src/queue.c b/src/queue.c
index 435ac96..6d74b79 100644
--- a/src/queue.c
+++ b/src/queue.c
@@ -50,16 +50,12 @@
static void _dispatch_deferred_items_cleanup(void *ctxt);
static void _dispatch_frame_cleanup(void *ctxt);
static void _dispatch_context_cleanup(void *ctxt);
-static void _dispatch_queue_barrier_complete(dispatch_queue_t dq,
- dispatch_qos_t qos, dispatch_wakeup_flags_t flags);
-static void _dispatch_queue_non_barrier_complete(dispatch_queue_t dq);
+static void _dispatch_non_barrier_complete(dispatch_queue_t dq);
static void _dispatch_queue_push_sync_waiter(dispatch_queue_t dq,
- dispatch_sync_context_t dsc, dispatch_qos_t qos);
+ dispatch_sync_context_t dsc);
#if HAVE_PTHREAD_WORKQUEUE_QOS
-static void _dispatch_root_queue_push_override_stealer(dispatch_queue_t orig_rq,
- dispatch_queue_t dq, dispatch_qos_t qos);
-static inline void _dispatch_queue_class_wakeup_with_override(dispatch_queue_t,
- uint64_t dq_state, dispatch_wakeup_flags_t flags);
+static void _dispatch_root_queue_push_queue_override(dispatch_queue_t rq,
+ dispatch_queue_class_t dqu, dispatch_qos_t qos);
#endif
#if HAVE_PTHREAD_WORKQUEUES
static void _dispatch_worker_thread4(void *context);
@@ -332,6 +328,7 @@
.do_ctxt = &_dispatch_root_queue_contexts[ \
_DISPATCH_ROOT_QUEUE_IDX(n, flags)], \
.dq_atomic_flags = DQF_WIDTH(DISPATCH_QUEUE_WIDTH_POOL), \
+ .dq_wlh = DISPATCH_WLH_GLOBAL, \
.dq_priority = _dispatch_priority_make(DISPATCH_QOS_##n, 0) | flags | \
DISPATCH_PRIORITY_FLAG_ROOTQUEUE | \
((flags & DISPATCH_PRIORITY_FLAG_DEFAULTQUEUE) ? 0 : \
@@ -426,13 +423,13 @@
DISPATCH_CACHELINE_ALIGN
struct dispatch_queue_s _dispatch_mgr_q = {
DISPATCH_GLOBAL_OBJECT_HEADER(queue_mgr),
- .dq_state = DISPATCH_QUEUE_STATE_INIT_VALUE(1) |
- DISPATCH_QUEUE_ROLE_BASE_ANON,
+ .dq_state = DISPATCH_QUEUE_STATE_INIT_VALUE(1),
.do_targetq = &_dispatch_mgr_root_queue,
.dq_label = "com.apple.libdispatch-manager",
.dq_atomic_flags = DQF_WIDTH(1),
.dq_priority = DISPATCH_PRIORITY_FLAG_MANAGER |
DISPATCH_PRIORITY_SATURATED_OVERRIDE,
+ .dq_wlh = DISPATCH_WLH_GLOBAL,
.dq_serialnum = 2,
};
@@ -497,7 +494,10 @@
"dispatch_assert_queue()");
}
uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed);
- if (likely(_dq_state_drain_locked_by_self(dq_state))) {
+ if (unlikely(_dq_state_drain_pended(dq_state))) {
+ goto fail;
+ }
+ if (likely(_dq_state_drain_owner(dq_state) == _dispatch_tid_self())) {
return;
}
// we can look at the width: if it is changing while we read it,
@@ -511,6 +511,7 @@
return;
}
}
+fail:
_dispatch_assert_queue_fail(dq, true);
}
@@ -523,7 +524,10 @@
"dispatch_assert_queue_not()");
}
uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed);
- if (likely(!_dq_state_drain_locked_by_self(dq_state))) {
+ if (_dq_state_drain_pended(dq_state)) {
+ return;
+ }
+ if (likely(_dq_state_drain_owner(dq_state) != _dispatch_tid_self())) {
// we can look at the width: if it is changing while we read it,
// it means that a barrier is running on `dq` concurrently, which
// proves that we're not on `dq`. Hence reading a stale '1' is ok.
@@ -588,7 +592,7 @@
#endif
#if DISPATCH_USE_KEVENT_WORKQUEUE
bool disable_kevent_wq = false;
-#if DISPATCH_DEBUG || DISPATCH_PROFILE
+#if DISPATCH_DEBUG
disable_kevent_wq = slowpath(getenv("LIBDISPATCH_DISABLE_KEVENT_WQ"));
#endif
#endif
@@ -604,6 +608,9 @@
#if DISPATCH_USE_MGR_THREAD
_dispatch_kevent_workqueue_enabled = !r;
#endif
+#if DISPATCH_EVFILT_MACHPORT_PORTSET_FALLBACK
+ _dispatch_evfilt_machport_direct_enabled = !r;
+#endif
result = !r;
} else
#endif // DISPATCH_USE_KEVENT_WORKQUEUE
@@ -780,6 +787,7 @@
dispatch_assert(sizeof(struct dispatch_root_queue_context_s) %
DISPATCH_CACHELINE_SIZE == 0);
+
#if HAVE_PTHREAD_WORKQUEUE_QOS
dispatch_qos_t qos = _dispatch_qos_from_qos_class(qos_class_main());
dispatch_priority_t pri = _dispatch_priority_make(qos, 0);
@@ -827,7 +835,6 @@
dispatch_atfork_parent, dispatch_atfork_child));
#endif
_dispatch_hw_config_init();
- _dispatch_time_init();
_dispatch_vtable_init();
_os_object_init();
_voucher_init();
@@ -895,18 +902,13 @@
void
_dispatch_queue_atfork_child(void)
{
- dispatch_queue_t main_q = &_dispatch_main_q;
void *crash = (void *)0x100;
size_t i;
- if (_dispatch_queue_is_thread_bound(main_q)) {
- _dispatch_queue_set_bound_thread(main_q);
- }
-
if (!_dispatch_is_multithreaded_inline()) return;
- main_q->dq_items_head = crash;
- main_q->dq_items_tail = crash;
+ _dispatch_main_q.dq_items_head = crash;
+ _dispatch_main_q.dq_items_tail = crash;
_dispatch_mgr_q.dq_items_head = crash;
_dispatch_mgr_q.dq_items_tail = crash;
@@ -917,33 +919,6 @@
}
}
-DISPATCH_NOINLINE
-void
-_dispatch_fork_becomes_unsafe_slow(void)
-{
- uint8_t value = os_atomic_or(&_dispatch_unsafe_fork,
- _DISPATCH_UNSAFE_FORK_MULTITHREADED, relaxed);
- if (value & _DISPATCH_UNSAFE_FORK_PROHIBIT) {
- DISPATCH_CLIENT_CRASH(0, "Transition to multithreaded is prohibited");
- }
-}
-
-DISPATCH_NOINLINE
-void
-_dispatch_prohibit_transition_to_multithreaded(bool prohibit)
-{
- if (prohibit) {
- uint8_t value = os_atomic_or(&_dispatch_unsafe_fork,
- _DISPATCH_UNSAFE_FORK_PROHIBIT, relaxed);
- if (value & _DISPATCH_UNSAFE_FORK_MULTITHREADED) {
- DISPATCH_CLIENT_CRASH(0, "The executable is already multithreaded");
- }
- } else {
- os_atomic_and(&_dispatch_unsafe_fork,
- (uint8_t)~_DISPATCH_UNSAFE_FORK_PROHIBIT, relaxed);
- }
-}
-
#pragma mark -
#pragma mark dispatch_queue_attr_t
@@ -1100,107 +1075,13 @@
dq->dq_label = label;
}
-static inline bool
-_dispatch_base_queue_is_wlh(dispatch_queue_t dq, dispatch_queue_t tq)
-{
- (void)dq; (void)tq;
- return false;
-}
-
-static void
-_dispatch_queue_inherit_wlh_from_target(dispatch_queue_t dq,
- dispatch_queue_t tq)
-{
- uint64_t old_state, new_state, role;
-
- if (!dx_hastypeflag(tq, QUEUE_ROOT)) {
- role = DISPATCH_QUEUE_ROLE_INNER;
- } else if (_dispatch_base_queue_is_wlh(dq, tq)) {
- role = DISPATCH_QUEUE_ROLE_BASE_WLH;
- } else {
- role = DISPATCH_QUEUE_ROLE_BASE_ANON;
- }
-
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, relaxed, {
- new_state = old_state & ~DISPATCH_QUEUE_ROLE_MASK;
- new_state |= role;
- if (old_state == new_state) {
- os_atomic_rmw_loop_give_up(break);
- }
- });
-
- dispatch_wlh_t cur_wlh = _dispatch_get_wlh();
- if (cur_wlh == (dispatch_wlh_t)dq && !_dq_state_is_base_wlh(new_state)) {
- _dispatch_event_loop_leave_immediate(cur_wlh, new_state);
- }
- if (!dx_hastypeflag(tq, QUEUE_ROOT)) {
-#if DISPATCH_ALLOW_NON_LEAF_RETARGET
- _dispatch_queue_atomic_flags_set(tq, DQF_TARGETED);
-#else
- _dispatch_queue_atomic_flags_set_and_clear(tq, DQF_TARGETED, DQF_LEGACY);
-#endif
- }
-}
-
-unsigned long volatile _dispatch_queue_serial_numbers =
- DISPATCH_QUEUE_SERIAL_NUMBER_INIT;
-
-dispatch_priority_t
-_dispatch_queue_compute_priority_and_wlh(dispatch_queue_t dq,
- dispatch_wlh_t *wlh_out)
-{
- dispatch_priority_t p = dq->dq_priority & DISPATCH_PRIORITY_REQUESTED_MASK;
- dispatch_queue_t tq = dq->do_targetq;
- dispatch_priority_t tqp = tq->dq_priority &DISPATCH_PRIORITY_REQUESTED_MASK;
- dispatch_wlh_t wlh = DISPATCH_WLH_ANON;
-
- if (_dq_state_is_base_wlh(dq->dq_state)) {
- wlh = (dispatch_wlh_t)dq;
- }
-
- while (unlikely(!dx_hastypeflag(tq, QUEUE_ROOT))) {
- if (unlikely(tq == &_dispatch_mgr_q)) {
- if (wlh_out) *wlh_out = DISPATCH_WLH_ANON;
- return DISPATCH_PRIORITY_FLAG_MANAGER;
- }
- if (unlikely(_dispatch_queue_is_thread_bound(tq))) {
- // thread-bound hierarchies are weird, we need to install
- // from the context of the thread this hierarchy is bound to
- if (wlh_out) *wlh_out = NULL;
- return 0;
- }
- if (unlikely(DISPATCH_QUEUE_IS_SUSPENDED(tq))) {
- // this queue may not be activated yet, so the queue graph may not
- // have stabilized yet
- _dispatch_ktrace1(DISPATCH_PERF_delayed_registration, dq);
- if (wlh_out) *wlh_out = NULL;
- return 0;
- }
-
- if (_dq_state_is_base_wlh(tq->dq_state)) {
- wlh = (dispatch_wlh_t)tq;
- } else if (unlikely(_dispatch_queue_is_legacy(tq))) {
- // we're not allowed to dereference tq->do_targetq
- _dispatch_ktrace1(DISPATCH_PERF_delayed_registration, dq);
- if (wlh_out) *wlh_out = NULL;
- return 0;
- }
-
- if (!(tq->dq_priority & DISPATCH_PRIORITY_FLAG_INHERIT)) {
- if (p < tqp) p = tqp;
- }
- tq = tq->do_targetq;
- tqp = tq->dq_priority & DISPATCH_PRIORITY_REQUESTED_MASK;
- }
-
- if (unlikely(!tqp)) {
- // pthread root queues opt out of QoS
- if (wlh_out) *wlh_out = DISPATCH_WLH_ANON;
- return DISPATCH_PRIORITY_FLAG_MANAGER;
- }
- if (wlh_out) *wlh_out = wlh;
- return _dispatch_priority_inherit_from_root_queue(p, tq);
-}
+// skip zero
+// 1 - main_q
+// 2 - mgr_q
+// 3 - mgr_root_q
+// 4,5,6,7,8,9,10,11,12,13,14,15 - global queues
+// we use 'xadd' on Intel, so the initial value == next assigned
+unsigned long volatile _dispatch_queue_serial_numbers = 16;
DISPATCH_NOINLINE
static dispatch_queue_t
@@ -1319,13 +1200,13 @@
}
}
- dispatch_queue_t dq = _dispatch_object_alloc(vtable,
+ dispatch_queue_t dq = _dispatch_alloc(vtable,
sizeof(struct dispatch_queue_s) - DISPATCH_QUEUE_CACHELINE_PAD);
_dispatch_queue_init(dq, dqf, dqa->dqa_concurrent ?
- DISPATCH_QUEUE_WIDTH_MAX : 1, DISPATCH_QUEUE_ROLE_INNER |
- (dqa->dqa_inactive ? DISPATCH_QUEUE_INACTIVE : 0));
+ DISPATCH_QUEUE_WIDTH_MAX : 1, dqa->dqa_inactive);
dq->dq_label = label;
+
#if HAVE_PTHREAD_WORKQUEUE_QOS
dq->dq_priority = dqa->dqa_qos_and_relpri;
if (overcommit == _dispatch_queue_attr_overcommit_enabled) {
@@ -1337,10 +1218,17 @@
// legacy way of inherithing the QoS from the target
_dispatch_queue_priority_inherit_from_target(dq, tq);
}
- if (!dqa->dqa_inactive) {
- _dispatch_queue_inherit_wlh_from_target(dq, tq);
+ if (!dqa->dqa_inactive && !dx_hastypeflag(tq, QUEUE_ROOT)) {
+ _dispatch_queue_atomic_flags_set(tq, DQF_TARGETED);
}
dq->do_targetq = tq;
+ if (!_dispatch_queue_is_legacy(dq) && !dqa->dqa_inactive) {
+ if (dx_hastypeflag(tq, QUEUE_ROOT)) {
+ dq->dq_wlh = _dispatch_root_queue_wlh_for_queue(tq, dq);
+ } else {
+ dq->dq_wlh = tq->dq_wlh;
+ }
+ }
_dispatch_object_debug(dq, "%s", __func__);
return _dispatch_introspection_queue_create(dq);
}
@@ -1368,7 +1256,7 @@
}
void
-_dispatch_queue_destroy(dispatch_queue_t dq, bool *allow_free)
+_dispatch_queue_destroy(dispatch_queue_t dq)
{
uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed);
uint64_t initial_state = DISPATCH_QUEUE_STATE_INIT_VALUE(dq->dq_width);
@@ -1376,13 +1264,21 @@
if (dx_type(dq) == DISPATCH_QUEUE_GLOBAL_ROOT_TYPE) {
initial_state = DISPATCH_ROOT_QUEUE_STATE_INIT_VALUE;
}
- dq_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
- dq_state &= ~DISPATCH_QUEUE_DIRTY;
- dq_state &= ~DISPATCH_QUEUE_ROLE_MASK;
+ if (dx_metatype(dq) == _DISPATCH_SOURCE_TYPE) {
+ // dispatch_cancel_and_wait may apply overrides in a racy way with
+ // the source cancellation finishing. This race is expensive and not
+ // really worthwhile to resolve since the source becomes dead anyway.
+ //
+ // In a similar way using DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT causes
+ // DIRTY & MAX_QOS bits to stay with the channel or source sometimes
+ // never woken up before it dies, so we have to ignore them.
+ dq_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
+ dq_state &= ~DISPATCH_QUEUE_DIRTY;
+ dq_state &= ~DISPATCH_QUEUE_RECEIVED_OVERRIDE;
+ }
if (slowpath(dq_state != initial_state)) {
if (_dq_state_drain_locked(dq_state)) {
- DISPATCH_CLIENT_CRASH((uintptr_t)dq_state,
- "Release of a locked queue");
+ DISPATCH_CLIENT_CRASH(dq, "Release of a locked queue");
}
#ifndef __LP64__
dq_state >>= 32;
@@ -1390,6 +1286,9 @@
DISPATCH_CLIENT_CRASH((uintptr_t)dq_state,
"Release of a queue with corrupt state");
}
+ if (slowpath(dq == _dispatch_queue_get_current())) {
+ DISPATCH_CLIENT_CRASH(dq, "Release of a queue by itself");
+ }
if (slowpath(dq->dq_items_tail)) {
DISPATCH_CLIENT_CRASH(dq->dq_items_tail,
"Release of a queue while items are enqueued");
@@ -1398,61 +1297,30 @@
// trash the queue so that use after free will crash
dq->dq_items_head = (void *)0x200;
dq->dq_items_tail = (void *)0x200;
+ // poison the state with something that is suspended and is easy to spot
+ dq->dq_state = 0xdead000000000000;
dispatch_queue_t dqsq = os_atomic_xchg2o(dq, dq_specific_q,
(void *)0x200, relaxed);
if (dqsq) {
_dispatch_release(dqsq);
}
-
- // fastpath for queues that never got their storage retained
- if (likely(os_atomic_load2o(dq, dq_sref_cnt, relaxed) == 0)) {
- // poison the state with something that is suspended and is easy to spot
- dq->dq_state = 0xdead000000000000;
- return;
+ if (dq->dq_wlh) {
+ dq->dq_wlh = NULL;
}
-
- // Take over freeing the memory from _dispatch_object_dealloc()
- //
- // As soon as we call _dispatch_queue_release_storage(), we forfeit
- // the possibility for the caller of dx_dispose() to finalize the object
- // so that responsibility is ours.
- _dispatch_object_finalize(dq);
- *allow_free = false;
- dq->dq_label = "<released queue, pending free>";
- dq->do_targetq = NULL;
- dq->do_finalizer = NULL;
- dq->do_ctxt = NULL;
- return _dispatch_queue_release_storage(dq);
}
// 6618342 Contact the team that owns the Instrument DTrace probe before
// renaming this symbol
void
-_dispatch_queue_dispose(dispatch_queue_t dq, bool *allow_free)
+_dispatch_queue_dispose(dispatch_queue_t dq)
{
_dispatch_object_debug(dq, "%s", __func__);
_dispatch_introspection_queue_dispose(dq);
if (dq->dq_label && _dispatch_queue_label_needs_free(dq)) {
free((void*)dq->dq_label);
}
- _dispatch_queue_destroy(dq, allow_free);
-}
-
-void
-_dispatch_queue_xref_dispose(dispatch_queue_t dq)
-{
- uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed);
- if (unlikely(_dq_state_is_suspended(dq_state))) {
- long state = (long)dq_state;
- if (sizeof(long) < sizeof(uint64_t)) state = (long)(dq_state >> 32);
- if (unlikely(_dq_state_is_inactive(dq_state))) {
- // Arguments for and against this assert are within 6705399
- DISPATCH_CLIENT_CRASH(state, "Release of an inactive object");
- }
- DISPATCH_CLIENT_CRASH(dq_state, "Release of a suspended object");
- }
- os_atomic_or2o(dq, dq_atomic_flags, DQF_RELEASED, relaxed);
+ _dispatch_queue_destroy(dq);
}
DISPATCH_NOINLINE
@@ -1506,15 +1374,21 @@
return _dispatch_queue_suspend_slow(dq);
});
}
- if (!_dq_state_drain_locked(dq_state)) {
- value |= DLOCK_OWNER_MASK;
+#ifdef DLOCK_NOWAITERS_BIT
+ if (_dq_state_drain_locked(dq_state)) {
+ value |= DISPATCH_QUEUE_DRAIN_OWNER_MASK;
+ } else {
+ value ^= DLOCK_OWNER_INVALID;
}
+#else
+ value |= DLOCK_OWNER_INVALID;
+#endif
});
if (!_dq_state_is_suspended(dq_state)) {
// rdar://8181908 we need to extend the queue life for the duration
// of the call to wakeup at _dispatch_queue_resume() time.
- _dispatch_retain_2(dq);
+ _dispatch_retain(dq);
}
}
@@ -1559,15 +1433,12 @@
static void
_dispatch_queue_resume_finalize_activation(dispatch_queue_t dq)
{
- bool allow_resume = true;
// Step 2: run the activation finalizer
if (dx_vtable(dq)->do_finalize_activation) {
- dx_vtable(dq)->do_finalize_activation(dq, &allow_resume);
+ dx_vtable(dq)->do_finalize_activation(dq);
}
// Step 3: consume the suspend count
- if (allow_resume) {
- return dx_vtable(dq)->do_resume(dq, false);
- }
+ return dx_vtable(dq)->do_resume(dq, false);
}
void
@@ -1575,15 +1446,12 @@
{
// covers all suspend and inactive bits, including side suspend bit
const uint64_t suspend_bits = DISPATCH_QUEUE_SUSPEND_BITS_MASK;
- uint64_t pending_barrier_width =
- (dq->dq_width - 1) * DISPATCH_QUEUE_WIDTH_INTERVAL;
- uint64_t set_owner_and_set_full_width_and_in_barrier =
- _dispatch_lock_value_for_self() | DISPATCH_QUEUE_WIDTH_FULL_BIT |
- DISPATCH_QUEUE_IN_BARRIER;
-
+ // covers all suspend and inactive bits and owner mask
+ const uint64_t suspend_owner_bits = DISPATCH_QUEUE_SUSPEND_BITS_MASK |
+ DISPATCH_QUEUE_DRAIN_OWNER_MASK;
// backward compatibility: only dispatch sources can abuse
// dispatch_resume() to really mean dispatch_activate()
- bool is_source = (dx_metatype(dq) == _DISPATCH_SOURCE_TYPE);
+ bool resume_can_activate = (dx_type(dq) == DISPATCH_SOURCE_KEVENT_TYPE);
uint64_t dq_state, value;
dispatch_assert(dq->do_ref_cnt != DISPATCH_OBJECT_GLOBAL_REFCNT);
@@ -1633,49 +1501,51 @@
+ DISPATCH_QUEUE_NEEDS_ACTIVATION) {
// { sc:1 i:0 na:1 } -> { sc:1 i:0 na:0 }
value = dq_state - DISPATCH_QUEUE_NEEDS_ACTIVATION;
- } else if (is_source && (dq_state & suspend_bits) ==
+ } else if (resume_can_activate && (dq_state & suspend_bits) ==
DISPATCH_QUEUE_NEEDS_ACTIVATION + DISPATCH_QUEUE_INACTIVE) {
// { sc:0 i:1 na:1 } -> { sc:1 i:0 na:0 }
value = dq_state - DISPATCH_QUEUE_INACTIVE
- DISPATCH_QUEUE_NEEDS_ACTIVATION
+ DISPATCH_QUEUE_SUSPEND_INTERVAL;
- } else if (unlikely(os_sub_overflow(dq_state,
- DISPATCH_QUEUE_SUSPEND_INTERVAL, &value))) {
- // underflow means over-resume or a suspend count transfer
- // to the side count is needed
- os_atomic_rmw_loop_give_up({
- if (!(dq_state & DISPATCH_QUEUE_HAS_SIDE_SUSPEND_CNT)) {
- goto over_resume;
- }
- return _dispatch_queue_resume_slow(dq);
- });
- //
- // below this, value = dq_state - DISPATCH_QUEUE_SUSPEND_INTERVAL
- //
- } else if (!_dq_state_is_runnable(value)) {
- // Out of width or still suspended.
- // For the former, force _dispatch_queue_non_barrier_complete
- // to reconsider whether it has work to do
- value |= DISPATCH_QUEUE_DIRTY;
- } else if (!_dq_state_drain_locked_by(value, DLOCK_OWNER_MASK)) {
- dispatch_assert(_dq_state_drain_locked(value));
- // still locked by someone else, make drain_try_unlock() fail
- // and reconsider whether it has work to do
- value |= DISPATCH_QUEUE_DIRTY;
- } else if (!is_source && (_dq_state_has_pending_barrier(value) ||
- value + pending_barrier_width <
- DISPATCH_QUEUE_WIDTH_FULL_BIT)) {
- // if we can, acquire the full width drain lock
- // and then perform a lock transfer
- //
- // However this is never useful for a source where there are no
- // sync waiters, so never take the lock and do a plain wakeup
- value &= DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK;
- value |= set_owner_and_set_full_width_and_in_barrier;
+ } else if ((dq_state & suspend_owner_bits) == (suspend_owner_bits &
+ (DISPATCH_QUEUE_SUSPEND_INTERVAL + DLOCK_OWNER_INVALID))) {
+ value = dq_state;
+ value ^= DISPATCH_QUEUE_SUSPEND_INTERVAL + DLOCK_OWNER_INVALID;
+ uint64_t full_width = value;
+ if (_dq_state_has_pending_barrier(full_width)) {
+ full_width -= DISPATCH_QUEUE_PENDING_BARRIER;
+ full_width += DISPATCH_QUEUE_WIDTH_INTERVAL;
+ full_width += DISPATCH_QUEUE_IN_BARRIER;
+ } else {
+ full_width += dq->dq_width * DISPATCH_QUEUE_WIDTH_INTERVAL;
+ full_width += DISPATCH_QUEUE_IN_BARRIER;
+ }
+ if ((full_width & DISPATCH_QUEUE_WIDTH_MASK) ==
+ DISPATCH_QUEUE_WIDTH_FULL_BIT) {
+ value = full_width;
+ value &= ~DISPATCH_QUEUE_DIRTY;
+ value ^= _dispatch_tid_self();
+ } else {
+ value &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
+ value &= ~DISPATCH_QUEUE_RECEIVED_OVERRIDE;
+ }
} else {
- // clear overrides and force a wakeup
- value &= ~DISPATCH_QUEUE_DRAIN_UNLOCK_MASK;
- value &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
+ value = DISPATCH_QUEUE_SUSPEND_INTERVAL;
+ if (unlikely(os_sub_overflow(dq_state, value, &value))) {
+ // underflow means over-resume or a suspend count transfer
+ // to the side count is needed
+ os_atomic_rmw_loop_give_up({
+ if (!(dq_state & DISPATCH_QUEUE_HAS_SIDE_SUSPEND_CNT)) {
+ goto over_resume;
+ }
+ return _dispatch_queue_resume_slow(dq);
+ });
+ }
+ if (unlikely(_dq_state_is_runnable(value))) {
+ // make drain_try_unlock() fail and reconsider whether
+ // it has work to do
+ value |= DISPATCH_QUEUE_DIRTY;
+ }
}
});
}
@@ -1698,26 +1568,21 @@
return;
}
- if (_dq_state_is_dirty(dq_state)) {
+ if ((dq_state ^ value) & DISPATCH_QUEUE_IN_BARRIER) {
+ _dispatch_try_lock_transfer_or_wakeup(dq);
+ } else if (_dq_state_should_wakeup(value)) {
// <rdar://problem/14637483>
// dependency ordering for dq state changes that were flushed
// and not acted upon
os_atomic_thread_fence(dependency);
- dq = os_atomic_force_dependency_on(dq, dq_state);
+ dq = os_atomic_force_dependency_on(dq, value);
+ dispatch_qos_t qos = _dq_state_max_qos(dq_state);
+ // Balancing the retain() done in suspend() for rdar://8181908
+ return dx_wakeup(dq, qos, DISPATCH_WAKEUP_CONSUME);
}
- // Balancing the retain_2 done in suspend() for rdar://8181908
- dispatch_wakeup_flags_t flags = DISPATCH_WAKEUP_CONSUME_2;
- if ((dq_state ^ value) & DISPATCH_QUEUE_IN_BARRIER) {
- flags |= DISPATCH_WAKEUP_BARRIER_COMPLETE;
- } else if (!_dq_state_is_runnable(value)) {
- if (_dq_state_is_base_wlh(dq_state)) {
- _dispatch_event_loop_assert_not_owned((dispatch_wlh_t)dq);
- }
- return _dispatch_release_2(dq);
- }
- dispatch_assert(!_dq_state_received_sync_wait(dq_state));
- dispatch_assert(!_dq_state_in_sync_transfer(dq_state));
- return dx_wakeup(dq, _dq_state_max_qos(dq_state), flags);
+
+ // Balancing the retain() done in suspend() for rdar://8181908
+ return _dispatch_release_tailcall(dq);
over_resume:
if (unlikely(_dq_state_is_inactive(dq_state))) {
@@ -1778,7 +1643,6 @@
os_atomic_rmw_loop2o(dq, dq_atomic_flags, old_dqf, new_dqf, relaxed, {
new_dqf = (old_dqf & DQF_FLAGS_MASK) | DQF_WIDTH(tmp);
});
- _dispatch_queue_inherit_wlh_from_target(dq, dq->do_targetq);
_dispatch_object_debug(dq, "%s", __func__);
}
@@ -1813,18 +1677,16 @@
dispatch_queue_t otq = dq->do_targetq;
if (_dispatch_queue_atomic_flags(dq) & DQF_TARGETED) {
-#if DISPATCH_ALLOW_NON_LEAF_RETARGET
_dispatch_ktrace3(DISPATCH_PERF_non_leaf_retarget, dq, otq, tq);
_dispatch_bug_deprecated("Changing the target of a queue "
"already targeted by other dispatch objects");
-#else
- DISPATCH_CLIENT_CRASH(0, "Cannot change the target of a queue "
- "already targeted by other dispatch objects");
-#endif
}
_dispatch_queue_priority_inherit_from_target(dq, tq);
- _dispatch_queue_inherit_wlh_from_target(dq, tq);
+ if (!dx_hastypeflag(tq, QUEUE_ROOT)) {
+ _dispatch_queue_atomic_flags_set(tq, DQF_TARGETED);
+ }
+
#if HAVE_PTHREAD_WORKQUEUE_QOS
// see _dispatch_queue_class_wakeup()
_dispatch_queue_sidelock_lock(dq);
@@ -1856,33 +1718,22 @@
return dx_vtable(dq)->do_resume(dq, false);
}
-#if !DISPATCH_ALLOW_NON_LEAF_RETARGET
- if (_dispatch_queue_atomic_flags(dq) & DQF_TARGETED) {
- DISPATCH_CLIENT_CRASH(0, "Cannot change the target of a queue "
- "already targeted by other dispatch objects");
- }
-#endif
-
if (unlikely(!_dispatch_queue_is_legacy(dq))) {
-#if DISPATCH_ALLOW_NON_LEAF_RETARGET
if (_dispatch_queue_atomic_flags(dq) & DQF_TARGETED) {
- DISPATCH_CLIENT_CRASH(0, "Cannot change the target of a queue "
+ DISPATCH_CLIENT_CRASH(dq, "Cannot change the target of a queue "
"already targeted by other dispatch objects");
}
-#endif
- DISPATCH_CLIENT_CRASH(0, "Cannot change the target of this object "
+ DISPATCH_CLIENT_CRASH(dq, "Cannot change the target of this object "
"after it has been activated");
}
unsigned long type = dx_type(dq);
switch (type) {
case DISPATCH_QUEUE_LEGACY_TYPE:
-#if DISPATCH_ALLOW_NON_LEAF_RETARGET
if (_dispatch_queue_atomic_flags(dq) & DQF_TARGETED) {
_dispatch_bug_deprecated("Changing the target of a queue "
"already targeted by other dispatch objects");
}
-#endif
break;
case DISPATCH_SOURCE_KEVENT_TYPE:
case DISPATCH_MACH_CHANNEL_TYPE:
@@ -1920,6 +1771,7 @@
.do_ctxt = &_dispatch_mgr_root_queue_context,
.dq_label = "com.apple.root.libdispatch-manager",
.dq_atomic_flags = DQF_WIDTH(DISPATCH_QUEUE_WIDTH_POOL),
+ .dq_wlh = DISPATCH_WLH_GLOBAL,
.dq_priority = DISPATCH_PRIORITY_FLAG_MANAGER |
DISPATCH_PRIORITY_SATURATED_OVERRIDE,
.dq_serialnum = 3,
@@ -2143,7 +1995,7 @@
dqs = sizeof(struct dispatch_queue_s) - DISPATCH_QUEUE_CACHELINE_PAD;
dqs = roundup(dqs, _Alignof(struct dispatch_root_queue_context_s));
- dq = _dispatch_object_alloc(DISPATCH_VTABLE(queue_root), dqs +
+ dq = _dispatch_alloc(DISPATCH_VTABLE(queue_root), dqs +
sizeof(struct dispatch_root_queue_context_s) +
sizeof(struct dispatch_pthread_root_queue_context_s));
qc = (void*)dq + dqs;
@@ -2158,11 +2010,13 @@
}
}
- _dispatch_queue_init(dq, dqf, DISPATCH_QUEUE_WIDTH_POOL, 0);
+ _dispatch_queue_init(dq, dqf, DISPATCH_QUEUE_WIDTH_POOL, false);
dq->dq_label = label;
dq->dq_state = DISPATCH_ROOT_QUEUE_STATE_INIT_VALUE;
dq->do_ctxt = qc;
+ dq->do_targetq = NULL;
dq->dq_priority = DISPATCH_PRIORITY_SATURATED_OVERRIDE;
+ dq->dq_wlh = DISPATCH_WLH_GLOBAL;
pqc->dpq_thread_mediator.do_vtable = DISPATCH_VTABLE(semaphore);
qc->dgq_ctxt = pqc;
@@ -2231,7 +2085,7 @@
#endif // DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES
void
-_dispatch_pthread_root_queue_dispose(dispatch_queue_t dq, bool *allow_free)
+_dispatch_pthread_root_queue_dispose(dispatch_queue_t dq)
{
if (slowpath(dq->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT)) {
DISPATCH_INTERNAL_CRASH(dq, "Global root queue disposed");
@@ -2243,7 +2097,7 @@
dispatch_pthread_root_queue_context_t pqc = qc->dgq_ctxt;
pthread_attr_destroy(&pqc->dpq_thread_attr);
- _dispatch_semaphore_dispose(&pqc->dpq_thread_mediator, NULL);
+ _dispatch_semaphore_dispose(&pqc->dpq_thread_mediator);
if (pqc->dpq_thread_configure) {
Block_release(pqc->dpq_thread_configure);
}
@@ -2252,7 +2106,7 @@
if (dq->dq_label && _dispatch_queue_label_needs_free(dq)) {
free((void*)dq->dq_label);
}
- _dispatch_queue_destroy(dq, allow_free);
+ _dispatch_queue_destroy(dq);
}
#pragma mark -
@@ -2273,8 +2127,7 @@
DISPATCH_DECL(dispatch_queue_specific);
void
-_dispatch_queue_specific_queue_dispose(dispatch_queue_specific_queue_t dqsq,
- bool *allow_free)
+_dispatch_queue_specific_queue_dispose(dispatch_queue_specific_queue_t dqsq)
{
dispatch_queue_specific_t dqs, tmp;
dispatch_queue_t rq = _dispatch_get_root_queue(DISPATCH_QOS_DEFAULT, false);
@@ -2285,7 +2138,7 @@
}
free(dqs);
}
- _dispatch_queue_destroy(dqsq->_as_dq, allow_free);
+ _dispatch_queue_destroy(dqsq->_as_dq);
}
static void
@@ -2293,13 +2146,12 @@
{
dispatch_queue_specific_queue_t dqsq;
- dqsq = _dispatch_object_alloc(DISPATCH_VTABLE(queue_specific_queue),
+ dqsq = _dispatch_alloc(DISPATCH_VTABLE(queue_specific_queue),
sizeof(struct dispatch_queue_specific_queue_s));
- _dispatch_queue_init(dqsq->_as_dq, DQF_NONE, DISPATCH_QUEUE_WIDTH_MAX,
- DISPATCH_QUEUE_ROLE_BASE_ANON);
+ _dispatch_queue_init(dqsq->_as_dq, DQF_NONE,
+ DISPATCH_QUEUE_WIDTH_MAX, false);
dqsq->do_xref_cnt = -1;
- dqsq->do_targetq = _dispatch_get_root_queue(
- DISPATCH_QOS_USER_INITIATED, true);
+ dqsq->do_targetq = _dispatch_get_root_queue(DISPATCH_QOS_USER_INITIATED, true);
dqsq->dq_label = "queue-specific";
TAILQ_INIT(&dqsq->dqsq_contexts);
if (slowpath(!os_atomic_cmpxchg2o(dq, dq_specific_q, NULL,
@@ -2427,7 +2279,7 @@
DISPATCH_CLIENT_CRASH(dq->dq_width, "Invalid queue type");
}
uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed);
- return _dq_state_drain_locked_by_self(dq_state);
+ return _dq_state_drain_locked_by(dq_state, _dispatch_tid_self());
}
#endif
@@ -2439,13 +2291,12 @@
{
size_t offset = 0;
dispatch_queue_t target = dq->do_targetq;
- const char *tlabel = target && target->dq_label ? target->dq_label : "";
uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed);
- offset += dsnprintf(&buf[offset], bufsiz - offset, "sref = %d, "
+ offset += dsnprintf(&buf[offset], bufsiz - offset,
"target = %s[%p], width = 0x%x, state = 0x%016llx",
- dq->dq_sref_cnt + 1, tlabel, target, dq->dq_width,
- (unsigned long long)dq_state);
+ target && target->dq_label ? target->dq_label : "", target,
+ dq->dq_width, (unsigned long long)dq_state);
if (_dq_state_is_suspended(dq_state)) {
offset += dsnprintf(&buf[offset], bufsiz - offset, ", suspended = %d",
_dq_state_suspend_cnt(dq_state));
@@ -2510,15 +2361,11 @@
#endif
#if DISPATCH_PERF_MON
-
-#define DISPATCH_PERF_MON_BUCKETS 8
-
static struct {
uint64_t volatile time_total;
uint64_t volatile count_total;
uint64_t volatile thread_total;
-} _dispatch_stats[DISPATCH_PERF_MON_BUCKETS];
-DISPATCH_USED static size_t _dispatch_stat_buckets = DISPATCH_PERF_MON_BUCKETS;
+} _dispatch_stats[65];
void
_dispatch_queue_merge_stats(uint64_t start, bool trace, perfmon_thread_type type)
@@ -2526,14 +2373,15 @@
uint64_t delta = _dispatch_absolute_time() - start;
unsigned long count;
int bucket = 0;
+
count = (unsigned long)_dispatch_thread_getspecific(dispatch_bcounter_key);
_dispatch_thread_setspecific(dispatch_bcounter_key, NULL);
+
if (count == 0) {
bucket = 0;
if (trace) _dispatch_ktrace1(DISPATCH_PERF_MON_worker_useless, type);
} else {
- bucket = MIN(DISPATCH_PERF_MON_BUCKETS - 1,
- (int)sizeof(count) * CHAR_BIT - __builtin_clzl(count));
+ bucket = (int)sizeof(count) * CHAR_BIT - __builtin_clzl(count);
os_atomic_add(&_dispatch_stats[bucket].count_total, count, relaxed);
}
os_atomic_add(&_dispatch_stats[bucket].time_total, delta, relaxed);
@@ -2571,9 +2419,8 @@
if (likely(old_pri & ~_PTHREAD_PRIORITY_FLAGS_MASK)) {
pflags |= _PTHREAD_SET_SELF_QOS_FLAG;
}
- uint64_t mgr_dq_state =
- os_atomic_load2o(&_dispatch_mgr_q, dq_state, relaxed);
- if (unlikely(_dq_state_drain_locked_by_self(mgr_dq_state))) {
+ if (unlikely(DISPATCH_QUEUE_DRAIN_OWNER(&_dispatch_mgr_q) ==
+ _dispatch_tid_self())) {
DISPATCH_INTERNAL_CRASH(pp,
"Changing the QoS while on the manager queue");
}
@@ -2617,6 +2464,9 @@
kv = _voucher_swap_and_get_mach_voucher(ov, v);
}
}
+#if !PTHREAD_WORKQUEUE_RESETS_VOUCHER_AND_PRIORITY_ON_PARK
+ flags &= ~(dispatch_thread_set_self_t)DISPATCH_THREAD_PARK;
+#endif
if (!(flags & DISPATCH_THREAD_PARK)) {
_dispatch_set_priority_and_mach_voucher_slow(priority, kv);
}
@@ -2915,7 +2765,7 @@
oq = os_atomic_xchg2o(dbpd, dbpd_queue, NULL, relaxed);
if (oq) {
// balances dispatch_{,barrier_,}sync
- _os_object_release_internal_n(oq->_as_os_obj, 2);
+ _os_object_release_internal(oq->_as_os_obj);
}
}
@@ -2941,7 +2791,7 @@
oq = os_atomic_xchg2o(dbpd, dbpd_queue, NULL, relaxed);
if (oq) {
// balances dispatch_{,barrier_,group_}async
- _os_object_release_internal_n_inline(oq->_as_os_obj, 2);
+ _os_object_release_internal_inline(oq->_as_os_obj);
}
if (release) {
Block_release(b);
@@ -3014,7 +2864,8 @@
// that times out, subsequent waits will not boost the qos of the
// still-running block.
dx_wakeup(boost_oq, _dispatch_qos_from_pp(pp),
- DISPATCH_WAKEUP_BLOCK_WAIT | DISPATCH_WAKEUP_CONSUME_2);
+ DISPATCH_WAKEUP_BLOCK_WAIT | DISPATCH_WAKEUP_OVERRIDING |
+ DISPATCH_WAKEUP_CONSUME);
}
mach_port_t boost_th = dbpd->dbpd_thread;
@@ -3078,7 +2929,7 @@
// balanced in d_block_async_invoke_and_release or d_block_wait
if (os_atomic_cmpxchg2o(dbpd, dbpd_queue, NULL, oq, relaxed)) {
- _os_object_retain_internal_n_inline(oq->_as_os_obj, 2);
+ _os_object_retain_internal_inline(oq->_as_os_obj);
}
if (dc_flags & DISPATCH_OBJ_CONSUME_BIT) {
@@ -3235,12 +3086,17 @@
rq = dq->do_targetq;
while (slowpath(rq->do_targetq) && rq != old_dq) {
- _dispatch_queue_non_barrier_complete(rq);
+ _dispatch_non_barrier_complete(rq);
rq = rq->do_targetq;
}
- _dispatch_queue_non_barrier_complete(dq);
- _dispatch_release_tailcall(dq); // pairs with _dispatch_async_redirect_wrap
+ _dispatch_non_barrier_complete(dq);
+
+ if (dic->dic_deferred) {
+ return _dispatch_queue_drain_deferred_invoke(dq, dic, flags, 0);
+ }
+
+ _dispatch_release_tailcall(dq);
}
DISPATCH_ALWAYS_INLINE
@@ -3257,7 +3113,7 @@
dc->dc_other = dou._do;
dc->dc_voucher = DISPATCH_NO_VOUCHER;
dc->dc_priority = DISPATCH_NO_PRIORITY;
- _dispatch_retain(dq); // released in _dispatch_async_redirect_invoke
+ _dispatch_retain(dq);
return dc;
}
@@ -3408,11 +3264,36 @@
#pragma mark -
#pragma mark _dispatch_sync_invoke / _dispatch_sync_complete
+DISPATCH_ALWAYS_INLINE
+static inline void
+_dispatch_barrier_complete_inline(dispatch_queue_t dq)
+{
+ uint64_t owned = DISPATCH_QUEUE_IN_BARRIER +
+ dq->dq_width * DISPATCH_QUEUE_WIDTH_INTERVAL;
+
+ if (unlikely(dq->dq_items_tail)) {
+ return _dispatch_try_lock_transfer_or_wakeup(dq);
+ }
+
+ if (unlikely(!_dispatch_queue_drain_try_unlock(dq, owned, true))) {
+ // someone enqueued a slow item at the head
+ // looping may be its last chance
+ return _dispatch_try_lock_transfer_or_wakeup(dq);
+ }
+}
+
DISPATCH_NOINLINE
static void
-_dispatch_queue_non_barrier_complete(dispatch_queue_t dq)
+_dispatch_barrier_complete(dispatch_queue_t dq)
{
- uint64_t old_state, new_state, owner_self = _dispatch_lock_value_for_self();
+ _dispatch_barrier_complete_inline(dq);
+}
+
+DISPATCH_NOINLINE
+static void
+_dispatch_non_barrier_complete(dispatch_queue_t dq)
+{
+ uint64_t old_state, new_state;
// see _dispatch_queue_resume()
os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, relaxed, {
@@ -3435,7 +3316,7 @@
DISPATCH_QUEUE_WIDTH_FULL_BIT) {
new_state = full_width;
new_state &= ~DISPATCH_QUEUE_DIRTY;
- new_state |= owner_self;
+ new_state ^= _dispatch_tid_self();
} else if (_dq_state_is_dirty(old_state)) {
new_state |= DISPATCH_QUEUE_ENQUEUED;
}
@@ -3443,19 +3324,11 @@
});
if ((old_state ^ new_state) & DISPATCH_QUEUE_IN_BARRIER) {
- if (_dq_state_is_dirty(old_state)) {
- // <rdar://problem/14637483>
- // dependency ordering for dq state changes that were flushed
- // and not acted upon
- os_atomic_thread_fence(dependency);
- dq = os_atomic_force_dependency_on(dq, old_state);
- }
- return _dispatch_queue_barrier_complete(dq, 0, 0);
+ return _dispatch_try_lock_transfer_or_wakeup(dq);
}
if ((old_state ^ new_state) & DISPATCH_QUEUE_ENQUEUED) {
- _dispatch_retain_2(dq);
- dispatch_assert(!_dq_state_is_base_wlh(new_state));
+ _dispatch_retain(dq);
return dx_push(dq->do_targetq, dq, _dq_state_max_qos(new_state));
}
}
@@ -3490,9 +3363,9 @@
do {
if (dq == stop_dq) return;
if (barrier) {
- _dispatch_queue_barrier_complete(dq, 0, 0);
+ _dispatch_barrier_complete(dq);
} else {
- _dispatch_queue_non_barrier_complete(dq);
+ _dispatch_non_barrier_complete(dq);
}
dq = dq->do_targetq;
barrier = (dq->dq_width == 1);
@@ -3514,7 +3387,7 @@
dispatch_function_t func)
{
_dispatch_sync_function_invoke_inline(dq, ctxt, func);
- _dispatch_queue_non_barrier_complete(dq);
+ _dispatch_non_barrier_complete(dq);
}
DISPATCH_NOINLINE
@@ -3523,274 +3396,60 @@
dispatch_function_t func)
{
_dispatch_sync_function_invoke_inline(dq, ctxt, func);
- dx_wakeup(dq, 0, DISPATCH_WAKEUP_BARRIER_COMPLETE);
-}
-
-/*
- * This is an optimized version of _dispatch_barrier_sync_invoke_and_complete
- *
- * For queues we can cheat and inline the unlock code, which is invalid
- * for objects with a more complex state machine (sources or mach channels)
- */
-DISPATCH_NOINLINE
-static void
-_dispatch_queue_barrier_sync_invoke_and_complete(dispatch_queue_t dq,
- void *ctxt, dispatch_function_t func)
-{
- _dispatch_sync_function_invoke_inline(dq, ctxt, func);
- if (unlikely(dq->dq_items_tail || dq->dq_width > 1)) {
- return _dispatch_queue_barrier_complete(dq, 0, 0);
- }
-
- // Presence of any of these bits requires more work that only
- // _dispatch_queue_barrier_complete() handles properly
- //
- // Note: testing for RECEIVED_OVERRIDE or RECEIVED_SYNC_WAIT without
- // checking the role is sloppy, but is a super fast check, and neither of
- // these bits should be set if the lock was never contended/discovered.
- const uint64_t fail_unlock_mask = DISPATCH_QUEUE_SUSPEND_BITS_MASK |
- DISPATCH_QUEUE_ENQUEUED | DISPATCH_QUEUE_DIRTY |
- DISPATCH_QUEUE_RECEIVED_OVERRIDE | DISPATCH_QUEUE_SYNC_TRANSFER |
- DISPATCH_QUEUE_RECEIVED_SYNC_WAIT;
- uint64_t old_state, new_state;
-
- // similar to _dispatch_queue_drain_try_unlock
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
- new_state = old_state - DISPATCH_QUEUE_SERIAL_DRAIN_OWNED;
- new_state &= ~DISPATCH_QUEUE_DRAIN_UNLOCK_MASK;
- new_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
- if (unlikely(old_state & fail_unlock_mask)) {
- os_atomic_rmw_loop_give_up({
- return _dispatch_queue_barrier_complete(dq, 0, 0);
- });
- }
- });
- if (_dq_state_is_base_wlh(old_state)) {
- _dispatch_event_loop_assert_not_owned((dispatch_wlh_t)dq);
- }
+ _dispatch_barrier_complete_inline(dq);
}
#pragma mark -
#pragma mark _dispatch_sync_wait / _dispatch_sync_waiter_wake
-#define DISPATCH_SYNC_WAITER_NO_UNLOCK (~0ull)
-
DISPATCH_NOINLINE
static void
-_dispatch_sync_waiter_wake(dispatch_sync_context_t dsc,
- dispatch_wlh_t wlh, uint64_t old_state, uint64_t new_state)
+_dispatch_sync_waiter_wake(OS_UNUSED dispatch_queue_t dq,
+ dispatch_sync_context_t dsc)
{
- dispatch_wlh_t waiter_wlh = dsc->dc_data;
-
- if (_dq_state_in_sync_transfer(old_state) ||
- _dq_state_in_sync_transfer(new_state) ||
- (waiter_wlh != DISPATCH_WLH_ANON)) {
- _dispatch_event_loop_wake_owner(dsc, wlh, old_state, new_state);
+ if (dsc->dsc_override_qos > dsc->dsc_override_qos_floor) {
+ _dispatch_wqthread_override_start((mach_port_t)&dsc->dc_data,
+ dsc->dsc_override_qos);
}
- if (waiter_wlh == DISPATCH_WLH_ANON) {
- if (dsc->dsc_override_qos > dsc->dsc_override_qos_floor) {
- _dispatch_wqthread_override_start(dsc->dsc_waiter,
- dsc->dsc_override_qos);
- }
- _dispatch_thread_event_signal(&dsc->dsc_event);
- }
+ _dispatch_thread_event_signal(&dsc->dsc_event);
_dispatch_introspection_queue_item_complete(dsc->_as_dc);
}
DISPATCH_NOINLINE
static void
-_dispatch_sync_waiter_redirect_or_wake(dispatch_queue_t dq, uint64_t owned,
+_dispatch_sync_waiter_redirect_or_wake(dispatch_queue_t dq,
dispatch_object_t dou)
{
- dispatch_sync_context_t dsc = (dispatch_sync_context_t)dou._dc;
- uint64_t next_owner = 0, old_state, new_state;
- dispatch_wlh_t wlh = NULL;
+ dispatch_sync_context_t dsc = (dispatch_sync_context_t )dou._dc;
+ uint32_t tid = (uint32_t)(uintptr_t)dsc->dc_data;
+ if (likely(dsc->dsc_override_qos)) {
+ uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed);
+ if (dsc->dsc_override_qos < _dq_state_max_qos(dq_state)) {
+ dsc->dsc_override_qos = _dq_state_max_qos(dq_state);
+ }
+ }
_dispatch_trace_continuation_pop(dq, dsc->_as_dc);
- if (owned == DISPATCH_SYNC_WAITER_NO_UNLOCK) {
- dispatch_assert(!(dsc->dc_flags & DISPATCH_OBJ_BARRIER_BIT));
- new_state = old_state = os_atomic_load2o(dq, dq_state, relaxed);
- } else {
- if (dsc->dc_flags & DISPATCH_OBJ_BARRIER_BIT) {
- next_owner = _dispatch_lock_value_from_tid(dsc->dsc_waiter);
- }
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
- new_state = old_state - owned;
- new_state &= ~DISPATCH_QUEUE_DRAIN_UNLOCK_MASK;
- new_state &= ~DISPATCH_QUEUE_DIRTY;
- new_state |= next_owner;
- if (_dq_state_is_base_wlh(old_state)) {
- new_state |= DISPATCH_QUEUE_SYNC_TRANSFER;
- }
- });
- if (_dq_state_is_base_wlh(old_state)) {
- wlh = (dispatch_wlh_t)dq;
- } else if (_dq_state_received_override(old_state)) {
- // Ensure that the root queue sees that this thread was overridden.
- _dispatch_set_basepri_override_qos(_dq_state_max_qos(old_state));
- }
- }
-
- if (dsc->dc_data == DISPATCH_WLH_ANON) {
- if (dsc->dsc_override_qos < _dq_state_max_qos(old_state)) {
- dsc->dsc_override_qos = _dq_state_max_qos(old_state);
- }
- }
-
- if (unlikely(_dq_state_is_inner_queue(old_state))) {
- dispatch_queue_t tq = dq->do_targetq;
- if (likely(tq->dq_width == 1)) {
+ while (unlikely(dq->do_targetq->do_targetq)) {
+ dq = dq->do_targetq;
+ if (likely(dq->dq_width == 1)) {
dsc->dc_flags = DISPATCH_OBJ_BARRIER_BIT |
DISPATCH_OBJ_SYNC_WAITER_BIT;
+ if (unlikely(!_dispatch_queue_try_acquire_barrier_sync(dq, tid))) {
+ _dispatch_introspection_queue_item_complete(dsc->_as_dc);
+ return _dispatch_queue_push_sync_waiter(dq, dsc);
+ }
} else {
dsc->dc_flags = DISPATCH_OBJ_SYNC_WAITER_BIT;
- }
- _dispatch_introspection_queue_item_complete(dsc->_as_dc);
- return _dispatch_queue_push_sync_waiter(tq, dsc, 0);
- }
-
- return _dispatch_sync_waiter_wake(dsc, wlh, old_state, new_state);
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_queue_class_barrier_complete(dispatch_queue_t dq, dispatch_qos_t qos,
- dispatch_wakeup_flags_t flags, dispatch_queue_wakeup_target_t target,
- uint64_t owned)
-{
- uint64_t old_state, new_state, enqueue;
- dispatch_queue_t tq;
-
- if (target == DISPATCH_QUEUE_WAKEUP_MGR) {
- tq = &_dispatch_mgr_q;
- enqueue = DISPATCH_QUEUE_ENQUEUED_ON_MGR;
- } else if (target) {
- tq = (target == DISPATCH_QUEUE_WAKEUP_TARGET) ? dq->do_targetq : target;
- enqueue = DISPATCH_QUEUE_ENQUEUED;
- } else {
- tq = NULL;
- enqueue = 0;
- }
-
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
- new_state = _dq_state_merge_qos(old_state - owned, qos);
- new_state &= ~DISPATCH_QUEUE_DRAIN_UNLOCK_MASK;
- if (unlikely(_dq_state_is_suspended(old_state))) {
- new_state |= DLOCK_OWNER_MASK;
- } else if (enqueue) {
- new_state |= enqueue;
- } else if (unlikely(_dq_state_is_dirty(old_state))) {
- os_atomic_rmw_loop_give_up({
- // just renew the drain lock with an acquire barrier, to see
- // what the enqueuer that set DIRTY has done.
- // the xor generates better assembly as DISPATCH_QUEUE_DIRTY
- // is already in a register
- os_atomic_xor2o(dq, dq_state, DISPATCH_QUEUE_DIRTY, acquire);
- flags |= DISPATCH_WAKEUP_BARRIER_COMPLETE;
- return dx_wakeup(dq, qos, flags);
- });
- } else if (_dq_state_is_base_wlh(old_state)) {
- new_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
- new_state &= ~DISPATCH_QUEUE_ENQUEUED;
- } else {
- new_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK;
- }
- });
- old_state -= owned;
- dispatch_assert(_dq_state_drain_locked_by_self(old_state));
- dispatch_assert(!_dq_state_is_enqueued_on_manager(old_state));
-
-
- if (_dq_state_received_override(old_state)) {
- // Ensure that the root queue sees that this thread was overridden.
- _dispatch_set_basepri_override_qos(_dq_state_max_qos(old_state));
- }
-
- if (tq) {
- if (likely((old_state ^ new_state) & enqueue)) {
- dispatch_assert(_dq_state_is_enqueued(new_state));
- dispatch_assert(flags & DISPATCH_WAKEUP_CONSUME_2);
- return _dispatch_queue_push_queue(tq, dq, new_state);
- }
-#if HAVE_PTHREAD_WORKQUEUE_QOS
- // <rdar://problem/27694093> when doing sync to async handoff
- // if the queue received an override we have to forecefully redrive
- // the same override so that a new stealer is enqueued because
- // the previous one may be gone already
- if (_dq_state_should_override(new_state)) {
- return _dispatch_queue_class_wakeup_with_override(dq, new_state,
- flags);
- }
-#endif
- }
- if (flags & DISPATCH_WAKEUP_CONSUME_2) {
- return _dispatch_release_2_tailcall(dq);
- }
-}
-
-DISPATCH_NOINLINE
-static void
-_dispatch_queue_barrier_complete(dispatch_queue_t dq, dispatch_qos_t qos,
- dispatch_wakeup_flags_t flags)
-{
- dispatch_continuation_t dc_tmp, dc_start = NULL, dc_end = NULL;
- dispatch_queue_wakeup_target_t target = DISPATCH_QUEUE_WAKEUP_NONE;
- struct dispatch_object_s *dc = NULL;
- uint64_t owned = DISPATCH_QUEUE_IN_BARRIER +
- dq->dq_width * DISPATCH_QUEUE_WIDTH_INTERVAL;
- size_t count = 0;
-
- dispatch_assert(dx_metatype(dq) == _DISPATCH_QUEUE_TYPE);
-
- if (dq->dq_items_tail && !DISPATCH_QUEUE_IS_SUSPENDED(dq)) {
- dc = _dispatch_queue_head(dq);
- if (!_dispatch_object_is_sync_waiter(dc)) {
- // not a slow item, needs to wake up
- } else if (likely(dq->dq_width == 1) ||
- _dispatch_object_is_barrier(dc)) {
- // rdar://problem/8290662 "barrier/writer lock transfer"
- dc_start = dc_end = (dispatch_continuation_t)dc;
- owned = 0;
- count = 1;
- dc = _dispatch_queue_next(dq, dc);
- } else {
- // <rdar://problem/10164594> "reader lock transfer"
- // we must not wake waiters immediately because our right
- // for dequeuing is granted through holding the full "barrier" width
- // which a signaled work item could relinquish out from our feet
- dc_start = (dispatch_continuation_t)dc;
- do {
- // no check on width here because concurrent queues
- // do not respect width for blocked readers, the thread
- // is already spent anyway
- dc_end = (dispatch_continuation_t)dc;
- owned -= DISPATCH_QUEUE_WIDTH_INTERVAL;
- count++;
- dc = _dispatch_queue_next(dq, dc);
- } while (dc && _dispatch_object_is_sync_waiter_non_barrier(dc));
- }
-
- if (count) {
- do {
- dc_tmp = dc_start;
- dc_start = dc_start->do_next;
- _dispatch_sync_waiter_redirect_or_wake(dq, owned, dc_tmp);
- owned = DISPATCH_SYNC_WAITER_NO_UNLOCK;
- } while (dc_tmp != dc_end);
- if (flags & DISPATCH_WAKEUP_CONSUME_2) {
- return _dispatch_release_2_tailcall(dq);
+ if (unlikely(!_dispatch_queue_try_reserve_sync_width(dq))) {
+ _dispatch_introspection_queue_item_complete(dsc->_as_dc);
+ return _dispatch_queue_push_sync_waiter(dq, dsc);
}
- return;
}
- if (!(flags & DISPATCH_WAKEUP_CONSUME_2)) {
- _dispatch_retain_2(dq);
- flags |= DISPATCH_WAKEUP_CONSUME_2;
- }
- target = DISPATCH_QUEUE_WAKEUP_TARGET;
}
- return _dispatch_queue_class_barrier_complete(dq, qos, flags, target,owned);
+ return _dispatch_sync_waiter_wake(dq, dsc);
}
#if DISPATCH_COCOA_COMPAT
@@ -3818,90 +3477,27 @@
}
#endif
-DISPATCH_ALWAYS_INLINE
-static inline uint64_t
-_dispatch_sync_wait_prepare(dispatch_queue_t dq)
-{
- uint64_t old_state, new_state;
-
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, relaxed, {
- if (_dq_state_is_suspended(old_state) ||
- !_dq_state_is_base_wlh(old_state)) {
- os_atomic_rmw_loop_give_up(return old_state);
- }
- if (!_dq_state_drain_locked(old_state) ||
- _dq_state_in_sync_transfer(old_state)) {
- os_atomic_rmw_loop_give_up(return old_state);
- }
- new_state = old_state | DISPATCH_QUEUE_RECEIVED_SYNC_WAIT;
- });
- return new_state;
-}
-
-static void
-_dispatch_sync_waiter_compute_wlh(dispatch_queue_t dq,
- dispatch_sync_context_t dsc)
-{
- bool needs_locking = _dispatch_queue_is_legacy(dq);
-
- if (needs_locking) {
- dsc->dsc_release_storage = true;
- _dispatch_queue_sidelock_lock(dq);
- }
-
- dispatch_queue_t tq = dq->do_targetq;
- uint64_t dq_state = _dispatch_sync_wait_prepare(tq);
-
- if (_dq_state_is_suspended(dq_state) ||
- _dq_state_is_base_anon(dq_state)) {
- dsc->dsc_release_storage = false;
- dsc->dc_data = DISPATCH_WLH_ANON;
- } else if (_dq_state_is_base_wlh(dq_state)) {
- if (dsc->dsc_release_storage) {
- _dispatch_queue_retain_storage(tq);
- }
- dsc->dc_data = (dispatch_wlh_t)tq;
- } else {
- _dispatch_sync_waiter_compute_wlh(tq, dsc);
- }
- if (needs_locking) _dispatch_queue_sidelock_unlock(dq);
-}
-
DISPATCH_NOINLINE
static void
_dispatch_sync_wait(dispatch_queue_t top_dq, void *ctxt,
dispatch_function_t func, uintptr_t top_dc_flags,
dispatch_queue_t dq, uintptr_t dc_flags)
{
+ uint32_t tid = _dispatch_tid_self();
+ dispatch_qos_t oq_floor = _dispatch_get_basepri_override_qos_floor();
pthread_priority_t pp = _dispatch_get_priority();
- dispatch_tid tid = _dispatch_tid_self();
- dispatch_qos_t qos;
- uint64_t dq_state;
-
- dq_state = _dispatch_sync_wait_prepare(dq);
- if (unlikely(_dq_state_drain_locked_by(dq_state, tid))) {
- DISPATCH_CLIENT_CRASH((uintptr_t)dq_state,
- "dispatch_sync called on queue "
- "already owned by current thread");
- }
struct dispatch_sync_context_s dsc = {
.dc_flags = dc_flags | DISPATCH_OBJ_SYNC_WAITER_BIT,
+ .dc_data = (void *)(uintptr_t)tid,
.dc_other = top_dq,
.dc_priority = pp | _PTHREAD_PRIORITY_ENFORCE_FLAG,
.dc_voucher = DISPATCH_NO_VOUCHER,
.dsc_func = func,
.dsc_ctxt = ctxt,
- .dsc_waiter = tid,
+ .dsc_override_qos_floor = oq_floor,
+ .dsc_override_qos = oq_floor,
};
- if (_dq_state_is_suspended(dq_state) ||
- _dq_state_is_base_anon(dq_state)) {
- dsc.dc_data = DISPATCH_WLH_ANON;
- } else if (_dq_state_is_base_wlh(dq_state)) {
- dsc.dc_data = (dispatch_wlh_t)dq;
- } else {
- _dispatch_sync_waiter_compute_wlh(dq, &dsc);
- }
#if DISPATCH_COCOA_COMPAT
// It's preferred to execute synchronous blocks on the current thread
// due to thread-local side effects, etc. However, blocks submitted
@@ -3919,26 +3515,22 @@
dsc.dc_ctxt = &dsc;
#endif
- if (dsc.dc_data == DISPATCH_WLH_ANON) {
- dsc.dsc_override_qos_floor = dsc.dsc_override_qos =
- _dispatch_get_basepri_override_qos_floor();
- qos = _dispatch_qos_from_pp(pp);
- _dispatch_thread_event_init(&dsc.dsc_event);
- } else {
- qos = 0;
+ uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed);
+ if (unlikely(_dq_state_drain_locked_by(dq_state, tid))) {
+ DISPATCH_CLIENT_CRASH(dq, "dispatch_sync called on queue "
+ "already owned by current thread");
}
- _dispatch_queue_push_sync_waiter(dq, &dsc, qos);
- if (dsc.dc_data == DISPATCH_WLH_ANON) {
- _dispatch_thread_event_wait(&dsc.dsc_event); // acquire
- _dispatch_thread_event_destroy(&dsc.dsc_event);
- // If _dispatch_sync_waiter_wake() gave this thread an override,
- // ensure that the root queue sees it.
- if (dsc.dsc_override_qos > dsc.dsc_override_qos_floor) {
- _dispatch_set_basepri_override_qos(dsc.dsc_override_qos);
- }
- } else {
- _dispatch_event_loop_wait_for_ownership(&dsc);
+
+ _dispatch_thread_event_init(&dsc.dsc_event);
+ _dispatch_queue_push_sync_waiter(dq, &dsc);
+ _dispatch_thread_event_wait(&dsc.dsc_event); // acquire
+ _dispatch_thread_event_destroy(&dsc.dsc_event);
+ if (dsc.dsc_override_qos > dsc.dsc_override_qos_floor) {
+ // If we received an override from _dispatch_sync_waiter_wake(),
+ // ensure that the root queue sees that this thread was overridden.
+ _dispatch_set_basepri_override_qos(dsc.dsc_override_qos);
}
+
_dispatch_introspection_sync_begin(top_dq);
#if DISPATCH_COCOA_COMPAT
if (unlikely(dsc.dsc_func == NULL)) {
@@ -3970,7 +3562,7 @@
_dispatch_sync_recurse(dispatch_queue_t dq, void *ctxt,
dispatch_function_t func, uintptr_t dc_flags)
{
- dispatch_tid tid = _dispatch_tid_self();
+ uint32_t tid = _dispatch_tid_self();
dispatch_queue_t tq = dq->do_targetq;
do {
@@ -3995,7 +3587,7 @@
dispatch_barrier_sync_f(dispatch_queue_t dq, void *ctxt,
dispatch_function_t func)
{
- dispatch_tid tid = _dispatch_tid_self();
+ uint32_t tid = _dispatch_tid_self();
// The more correct thing to do would be to merge the qos of the thread
// that just acquired the barrier lock into the queue state.
@@ -4014,7 +3606,7 @@
if (unlikely(dq->do_targetq->do_targetq)) {
return _dispatch_sync_recurse(dq, ctxt, func, DISPATCH_OBJ_BARRIER_BIT);
}
- _dispatch_queue_barrier_sync_invoke_and_complete(dq, ctxt, func);
+ _dispatch_barrier_sync_invoke_and_complete(dq, ctxt, func);
}
DISPATCH_NOINLINE
@@ -4056,7 +3648,7 @@
}
// balanced in d_block_sync_invoke or d_block_wait
if (os_atomic_cmpxchg2o(dbpd, dbpd_queue, NULL, dq->_as_oq, relaxed)) {
- _dispatch_retain_2(dq);
+ _dispatch_retain(dq);
}
if (flags & DISPATCH_BLOCK_BARRIER) {
dispatch_barrier_sync_f(dq, work, _dispatch_block_sync_invoke);
@@ -4096,7 +3688,7 @@
_dispatch_barrier_trysync_or_async_f(dispatch_queue_t dq, void *ctxt,
dispatch_function_t func)
{
- dispatch_tid tid = _dispatch_tid_self();
+ uint32_t tid = _dispatch_tid_self();
if (unlikely(!_dispatch_queue_try_acquire_barrier_sync(dq, tid))) {
return _dispatch_barrier_async_detached_f(dq, ctxt, func);
}
@@ -4108,7 +3700,7 @@
_dispatch_trysync_recurse(dispatch_queue_t dq, void *ctxt,
dispatch_function_t f, uintptr_t dc_flags)
{
- dispatch_tid tid = _dispatch_tid_self();
+ uint32_t tid = _dispatch_tid_self();
dispatch_queue_t q, tq = dq->do_targetq;
for (;;) {
@@ -4143,7 +3735,7 @@
_dispatch_barrier_trysync_f(dispatch_queue_t dq, void *ctxt,
dispatch_function_t f)
{
- dispatch_tid tid = _dispatch_tid_self();
+ uint32_t tid = _dispatch_tid_self();
if (unlikely(!dq->do_targetq)) {
DISPATCH_CLIENT_CRASH(dq, "_dispatch_trsync called on a root queue");
}
@@ -4185,13 +3777,16 @@
{
dispatch_queue_wakeup_target_t target = DISPATCH_QUEUE_WAKEUP_NONE;
- if (unlikely(flags & DISPATCH_WAKEUP_BARRIER_COMPLETE)) {
- return _dispatch_queue_barrier_complete(dq, qos, flags);
- }
if (_dispatch_queue_class_probe(dq)) {
target = DISPATCH_QUEUE_WAKEUP_TARGET;
}
- return _dispatch_queue_class_wakeup(dq, qos, flags, target);
+ if (target) {
+ return _dispatch_queue_class_wakeup(dq, qos, flags, target);
+ } else if (qos) {
+ return _dispatch_queue_class_override_drainer(dq, qos, flags);
+ } else if (flags & DISPATCH_WAKEUP_CONSUME) {
+ return _dispatch_release_tailcall(dq);
+ }
}
#if DISPATCH_COCOA_COMPAT
@@ -4237,16 +3832,6 @@
}
#endif // DISPATCH_COCOA_COMPAT
-DISPATCH_ALWAYS_INLINE
-static inline dispatch_qos_t
-_dispatch_runloop_queue_reset_max_qos(dispatch_queue_class_t dqu)
-{
- uint64_t old_state, clear_bits = DISPATCH_QUEUE_MAX_QOS_MASK |
- DISPATCH_QUEUE_RECEIVED_OVERRIDE;
- old_state = os_atomic_and_orig2o(dqu._dq, dq_state, ~clear_bits, relaxed);
- return _dq_state_max_qos(old_state);
-}
-
void
_dispatch_runloop_queue_wakeup(dispatch_queue_t dq, dispatch_qos_t qos,
dispatch_wakeup_flags_t flags)
@@ -4257,14 +3842,14 @@
return _dispatch_queue_wakeup(dq, qos, flags);
}
- if (flags & DISPATCH_WAKEUP_MAKE_DIRTY) {
+ if (flags & DISPATCH_WAKEUP_FLUSH) {
os_atomic_or2o(dq, dq_state, DISPATCH_QUEUE_DIRTY, release);
}
if (_dispatch_queue_class_probe(dq)) {
return _dispatch_runloop_queue_poke(dq, qos, flags);
}
- qos = _dispatch_runloop_queue_reset_max_qos(dq);
+ qos = _dispatch_queue_reset_max_qos(dq);
if (qos) {
mach_port_t owner = DISPATCH_QUEUE_DRAIN_OWNER(dq);
if (_dispatch_queue_class_probe(dq)) {
@@ -4273,8 +3858,8 @@
_dispatch_thread_override_end(owner, dq);
return;
}
- if (flags & DISPATCH_WAKEUP_CONSUME_2) {
- return _dispatch_release_2_tailcall(dq);
+ if (flags & DISPATCH_WAKEUP_CONSUME) {
+ return _dispatch_release_tailcall(dq);
}
#else
return _dispatch_queue_wakeup(dq, qos, flags);
@@ -4333,7 +3918,7 @@
_dispatch_runloop_queue_poke(dispatch_queue_t dq, dispatch_qos_t qos,
dispatch_wakeup_flags_t flags)
{
- // it's not useful to handle WAKEUP_MAKE_DIRTY because mach_msg() will have
+ // it's not useful to handle WAKEUP_FLUSH because mach_msg() will have
// a release barrier and that when runloop queues stop being thread-bound
// they have a non optional wake-up to start being a "normal" queue
// either in _dispatch_runloop_queue_xref_dispose,
@@ -4363,8 +3948,8 @@
}
no_change:
_dispatch_runloop_queue_class_poke(dq);
- if (flags & DISPATCH_WAKEUP_CONSUME_2) {
- return _dispatch_release_2_tailcall(dq);
+ if (flags & DISPATCH_WAKEUP_CONSUME) {
+ return _dispatch_release_tailcall(dq);
}
}
#endif
@@ -4459,7 +4044,7 @@
}
#endif
do {
- _dispatch_retain(dq); // released in _dispatch_worker_thread
+ _dispatch_retain(dq);
while ((r = pthread_create(pthr, attr, _dispatch_worker_thread, dq))) {
if (r != EAGAIN) {
(void)dispatch_assume_zero(r);
@@ -4513,44 +4098,15 @@
static void
_dispatch_return_to_kernel(void)
{
- if (unlikely(_dispatch_get_wlh() == DISPATCH_WLH_ANON)) {
+ if (unlikely(_dispatch_get_wlh() == DISPATCH_WLH_GLOBAL)) {
_dispatch_clear_return_to_kernel();
} else {
_dispatch_event_loop_drain(KEVENT_FLAG_IMMEDIATE);
}
}
-void
-_dispatch_poll_for_events_4launchd(void)
-{
-#if DISPATCH_USE_KEVENT_WORKQUEUE
- if (_dispatch_get_wlh()) {
- dispatch_assert(_dispatch_deferred_items_get()->ddi_wlh_servicing);
- _dispatch_event_loop_drain(KEVENT_FLAG_IMMEDIATE);
- }
-#endif
-}
-
#if HAVE_PTHREAD_WORKQUEUE_NARROWING
-static os_atomic(uint64_t) _dispatch_narrowing_deadlines[DISPATCH_QOS_MAX];
-#if !DISPATCH_TIME_UNIT_USES_NANOSECONDS
-static uint64_t _dispatch_narrow_check_interval_cache;
-#endif
-
-DISPATCH_ALWAYS_INLINE
-static inline uint64_t
-_dispatch_narrow_check_interval(void)
-{
-#if DISPATCH_TIME_UNIT_USES_NANOSECONDS
- return 50 * NSEC_PER_MSEC;
-#else
- if (_dispatch_narrow_check_interval_cache == 0) {
- _dispatch_narrow_check_interval_cache =
- _dispatch_time_nano2mach(50 * NSEC_PER_MSEC);
- }
- return _dispatch_narrow_check_interval_cache;
-#endif
-}
+static os_atomic(uint64_t) _dispatch_narrowing_deadlines[DISPATCH_QOS_MAX - 1];
DISPATCH_ALWAYS_INLINE
static inline void
@@ -4560,7 +4116,7 @@
if (_dispatch_priority_qos(pri) &&
!(pri & DISPATCH_PRIORITY_FLAG_OVERCOMMIT)) {
dic->dic_next_narrow_check = _dispatch_approximate_time() +
- _dispatch_narrow_check_interval();
+ DISPATCH_NARROW_CHECK_INTERVAL;
}
}
@@ -4571,13 +4127,9 @@
{
if (dic->dic_next_narrow_check != DISPATCH_THREAD_IS_NARROWING) {
pthread_priority_t pp = _dispatch_get_priority();
- dispatch_qos_t qos = _dispatch_qos_from_pp(pp);
- if (unlikely(!qos || qos > countof(_dispatch_narrowing_deadlines))) {
- DISPATCH_CLIENT_CRASH(pp, "Thread QoS corruption");
- }
- size_t idx = qos - 1; // no entry needed for DISPATCH_QOS_UNSPECIFIED
+ size_t idx = _dispatch_qos_from_pp(pp) - 1;
os_atomic(uint64_t) *deadline = &_dispatch_narrowing_deadlines[idx];
- uint64_t oldval, newval = now + _dispatch_narrow_check_interval();
+ uint64_t oldval, newval = now + DISPATCH_NARROW_CHECK_INTERVAL;
dic->dic_next_narrow_check = newval;
os_atomic_rmw_loop(deadline, oldval, newval, relaxed, {
@@ -4657,8 +4209,6 @@
// but width can change while draining barrier work items, so we only
// convert to `dq->dq_width * WIDTH_INTERVAL` when we drop `IN_BARRIER`
owned = DISPATCH_QUEUE_IN_BARRIER;
- } else {
- owned &= DISPATCH_QUEUE_WIDTH_MASK;
}
dc = _dispatch_queue_head(dq);
@@ -4669,9 +4219,6 @@
if (unlikely(dic->dic_deferred)) {
goto out_with_deferred_compute_owned;
}
- if (unlikely(_dispatch_needs_to_return_to_kernel())) {
- _dispatch_return_to_kernel();
- }
if (unlikely(!dc)) {
if (!dq->dq_items_tail) {
break;
@@ -4684,6 +4231,9 @@
if (unlikely(_dispatch_queue_drain_should_narrow(dic))) {
break;
}
+ if (unlikely(_dispatch_needs_to_return_to_kernel())) {
+ _dispatch_return_to_kernel();
+ }
first_iteration:
dq_state = os_atomic_load(&dq->dq_state, relaxed);
@@ -4728,8 +4278,7 @@
next_dc = _dispatch_queue_next(dq, dc);
if (_dispatch_object_is_sync_waiter(dc)) {
owned -= DISPATCH_QUEUE_WIDTH_INTERVAL;
- _dispatch_sync_waiter_redirect_or_wake(dq,
- DISPATCH_SYNC_WAITER_NO_UNLOCK, dc);
+ _dispatch_sync_waiter_redirect_or_wake(dq, dc);
continue;
}
@@ -4750,13 +4299,12 @@
if (dc) {
owned = _dispatch_queue_adjust_owned(dq, owned, dc);
}
- *owned_ptr &= DISPATCH_QUEUE_ENQUEUED | DISPATCH_QUEUE_ENQUEUED_ON_MGR;
- *owned_ptr |= owned;
+ *owned_ptr = owned;
_dispatch_thread_frame_pop(&dtf);
return dc ? dq->do_targetq : NULL;
out_with_no_width:
- *owned_ptr &= DISPATCH_QUEUE_ENQUEUED | DISPATCH_QUEUE_ENQUEUED_ON_MGR;
+ *owned_ptr = 0;
_dispatch_thread_frame_pop(&dtf);
return DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT;
@@ -4773,8 +4321,7 @@
}
}
out_with_deferred:
- *owned_ptr &= DISPATCH_QUEUE_ENQUEUED | DISPATCH_QUEUE_ENQUEUED_ON_MGR;
- *owned_ptr |= owned;
+ *owned_ptr = owned;
if (unlikely(flags & DISPATCH_INVOKE_DISALLOW_SYNC_WAITERS)) {
DISPATCH_INTERNAL_CRASH(dc,
"Deferred continuation on source, mach channel or mgr");
@@ -4852,10 +4399,9 @@
DISPATCH_CLIENT_CRASH(0, "_dispatch_main_queue_callback_4CF called"
" after dispatch_main()");
}
- uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed);
- if (unlikely(!_dq_state_drain_locked_by_self(dq_state))) {
- DISPATCH_CLIENT_CRASH((uintptr_t)dq_state,
- "_dispatch_main_queue_callback_4CF called"
+ mach_port_t owner = DISPATCH_QUEUE_DRAIN_OWNER(dq);
+ if (slowpath(owner != _dispatch_tid_self())) {
+ DISPATCH_CLIENT_CRASH(owner, "_dispatch_main_queue_callback_4CF called"
" from the wrong thread");
}
@@ -4864,7 +4410,7 @@
// <rdar://problem/23256682> hide the frame chaining when CFRunLoop
// drains the main runloop, as this should not be observable that way
- _dispatch_adopt_wlh_anon();
+ _dispatch_set_wlh(dq->dq_wlh);
_dispatch_thread_frame_push_and_rebase(&dtf, dq, NULL);
pthread_priority_t pp = _dispatch_get_priority();
@@ -4886,6 +4432,8 @@
_dispatch_continuation_pop_inline(dc, &dic, DISPATCH_INVOKE_NONE, dq);
} while ((dc = next_dc));
+ // runloop based queues use their port for the queue PUBLISH pattern
+ // so this raw call to dx_wakeup(0) is valid
dx_wakeup(dq, 0, 0);
_dispatch_voucher_debug("main queue restore", voucher);
_dispatch_reset_basepri(old_dbp);
@@ -4905,7 +4453,7 @@
}
_dispatch_perfmon_start_notrace();
dispatch_thread_frame_s dtf;
- bool should_reset_wlh = _dispatch_adopt_wlh_anon_recurse();
+ _dispatch_set_wlh(dq->dq_wlh);
_dispatch_thread_frame_push(&dtf, dq);
pthread_priority_t pp = _dispatch_get_priority();
dispatch_priority_t pri = _dispatch_priority_from_pp(pp);
@@ -4920,6 +4468,8 @@
_dispatch_continuation_pop_inline(dc, &dic, DISPATCH_INVOKE_NONE, dq);
if (!next_dc) {
+ // runloop based queues use their port for the queue PUBLISH pattern
+ // so this raw call to dx_wakeup(0) is valid
dx_wakeup(dq, 0, 0);
}
@@ -4928,13 +4478,80 @@
_dispatch_reset_basepri_override();
_dispatch_reset_priority_and_voucher(pp, voucher);
_dispatch_thread_frame_pop(&dtf);
- if (should_reset_wlh) _dispatch_reset_wlh();
+ _dispatch_reset_wlh();
_dispatch_force_cache_cleanup();
_dispatch_perfmon_end_notrace();
return next_dc;
}
#endif
+DISPATCH_NOINLINE
+void
+_dispatch_try_lock_transfer_or_wakeup(dispatch_queue_t dq)
+{
+ dispatch_continuation_t dc_tmp, dc_start = NULL, dc_end = NULL;
+ struct dispatch_object_s *dc = NULL;
+ uint64_t owned;
+ size_t count = 0;
+
+ owned = DISPATCH_QUEUE_IN_BARRIER;
+ owned += dq->dq_width * DISPATCH_QUEUE_WIDTH_INTERVAL;
+attempt_running_slow_head:
+ if (dq->dq_items_tail && !DISPATCH_QUEUE_IS_SUSPENDED(dq)) {
+ dc = _dispatch_queue_head(dq);
+ if (!_dispatch_object_is_sync_waiter(dc)) {
+ // not a slow item, needs to wake up
+ } else if (likely(dq->dq_width == 1) ||
+ _dispatch_object_is_barrier(dc)) {
+ // rdar://problem/8290662 "barrier/writer lock transfer"
+ dc_start = dc_end = (dispatch_continuation_t)dc;
+ owned = 0;
+ count = 1;
+ dc = _dispatch_queue_next(dq, dc);
+ } else {
+ // <rdar://problem/10164594> "reader lock transfer"
+ // we must not wake waiters immediately because our right
+ // for dequeuing is granted through holding the full "barrier" width
+ // which a signaled work item could relinquish out from our feet
+ dc_start = (dispatch_continuation_t)dc;
+ do {
+ // no check on width here because concurrent queues
+ // do not respect width for blocked readers, the thread
+ // is already spent anyway
+ dc_end = (dispatch_continuation_t)dc;
+ owned -= DISPATCH_QUEUE_WIDTH_INTERVAL;
+ count++;
+ dc = _dispatch_queue_next(dq, dc);
+ } while (dc && _dispatch_object_is_sync_waiter_non_barrier(dc));
+ }
+
+ if (count) {
+ _dispatch_queue_drain_transfer_lock(dq, owned, dc_start);
+ do {
+ dc_tmp = dc_start;
+ dc_start = dc_start->do_next;
+ _dispatch_sync_waiter_redirect_or_wake(dq, dc_tmp);
+ } while (dc_tmp != dc_end);
+ return;
+ }
+ }
+
+ if (dc || dx_metatype(dq) != _DISPATCH_QUEUE_TYPE) {
+ // <rdar://problem/23336992> the following wakeup is needed for sources
+ // or mach channels: when ds_pending_data is set at the same time
+ // as a trysync_f happens, lock transfer code above doesn't know about
+ // ds_pending_data or the wakeup logic, but lock transfer is useless
+ // for sources and mach channels in the first place.
+ owned = _dispatch_queue_adjust_owned(dq, owned, dc);
+ _dispatch_queue_drain_unlock(dq, owned);
+ return dx_wakeup(dq, 0, DISPATCH_WAKEUP_WAITER_HANDOFF);
+ } else if (unlikely(!_dispatch_queue_drain_try_unlock(dq, owned, true))) {
+ // someone enqueued a slow item at the head
+ // looping may be its last chance
+ goto attempt_running_slow_head;
+ }
+}
+
void
_dispatch_mgr_queue_drain(void)
{
@@ -4967,37 +4584,96 @@
#pragma mark dispatch_queue_invoke
void
-_dispatch_queue_drain_sync_waiter(dispatch_queue_t dq,
+_dispatch_queue_drain_deferred_invoke(dispatch_queue_t dq,
dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags,
- uint64_t owned)
+ uint64_t to_unlock)
{
struct dispatch_object_s *dc = dic->dic_deferred;
- dispatch_assert(_dispatch_object_is_sync_waiter(dc));
- dic->dic_deferred = NULL;
- if (flags & DISPATCH_INVOKE_WLH) {
- // Leave the enqueued bit in place, completion of the last sync waiter
- // in the handoff chain is responsible for dequeuing
- //
- // We currently have a +2 to consume, but we need to keep a +1
- // for the thread request
- dispatch_assert(_dq_state_is_enqueued_on_target(owned));
- dispatch_assert(!_dq_state_is_enqueued_on_manager(owned));
- owned &= ~DISPATCH_QUEUE_ENQUEUED;
- _dispatch_release_no_dispose(dq);
- } else {
- // The sync waiter must own a reference
- _dispatch_release_2_no_dispose(dq);
+ if (_dispatch_object_is_sync_waiter(dc)) {
+ dispatch_assert(to_unlock == 0);
+ dic->dic_deferred = NULL;
+ _dispatch_queue_drain_transfer_lock(dq, to_unlock, dc);
+ _dispatch_sync_waiter_redirect_or_wake(dq, dc);
+ return _dispatch_release_tailcall(dq);
}
- return _dispatch_sync_waiter_redirect_or_wake(dq, owned, dc);
+
+ bool should_defer_again = false, should_pend_queue = true;
+ uint64_t old_state, new_state;
+
+ if (_dispatch_get_current_queue()->do_targetq) {
+ should_defer_again = true;
+ should_pend_queue = false;
+ }
+
+ if (dq->dq_width > 1) {
+ should_pend_queue = false;
+ } else if (should_pend_queue) {
+ dispatch_assert(to_unlock ==
+ DISPATCH_QUEUE_WIDTH_INTERVAL + DISPATCH_QUEUE_IN_BARRIER);
+ os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release,{
+ new_state = old_state;
+ if (_dq_state_has_waiters(old_state) ||
+ _dq_state_is_enqueued(old_state)) {
+ os_atomic_rmw_loop_give_up(break);
+ }
+ new_state += DISPATCH_QUEUE_DRAIN_PENDED;
+ new_state -= DISPATCH_QUEUE_IN_BARRIER;
+ new_state -= DISPATCH_QUEUE_WIDTH_INTERVAL;
+ });
+ should_pend_queue = (new_state & DISPATCH_QUEUE_DRAIN_PENDED);
+ }
+
+ if (!should_pend_queue) {
+ if (to_unlock & DISPATCH_QUEUE_IN_BARRIER) {
+ _dispatch_try_lock_transfer_or_wakeup(dq);
+ _dispatch_release(dq);
+ } else if (to_unlock) {
+ _dispatch_queue_drain_unlock(dq, to_unlock);
+ dx_wakeup(dq, 0, DISPATCH_WAKEUP_CONSUME);
+ } else {
+ _dispatch_release(dq);
+ }
+ dq = NULL;
+ }
+
+ if (!should_defer_again) {
+ dic->dic_deferred = NULL;
+ return dx_invoke(dc, dic, flags & _DISPATCH_INVOKE_PROPAGATE_MASK);
+ }
+
+ if (dq) {
+ uint32_t self = _dispatch_tid_self();
+ os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release,{
+ new_state = old_state;
+ if (!_dq_state_drain_pended(old_state) ||
+ _dq_state_drain_owner(old_state) != self) {
+ os_atomic_rmw_loop_give_up({
+ // We may have been overridden, so inform the root queue
+ _dispatch_set_basepri_override_qos(
+ _dq_state_max_qos(old_state));
+ return _dispatch_release_tailcall(dq);
+ });
+ }
+ new_state = DISPATCH_QUEUE_DRAIN_UNLOCK(new_state);
+ });
+ if (_dq_state_received_override(old_state)) {
+ // Ensure that the root queue sees that this thread was overridden.
+ _dispatch_set_basepri_override_qos(_dq_state_max_qos(old_state));
+ }
+ return dx_invoke(dq, dic, flags | DISPATCH_INVOKE_STEALING);
+ }
}
void
-_dispatch_queue_finalize_activation(dispatch_queue_t dq,
- DISPATCH_UNUSED bool *allow_resume)
+_dispatch_queue_finalize_activation(dispatch_queue_t dq)
{
dispatch_queue_t tq = dq->do_targetq;
_dispatch_queue_priority_inherit_from_target(dq, tq);
- _dispatch_queue_inherit_wlh_from_target(dq, tq);
+ _dispatch_queue_atomic_flags_set(tq, DQF_TARGETED);
+ if (!dq->dq_wlh) {
+ dispatch_wlh_t wlh = _dispatch_queue_class_compute_wlh(dq);
+ if (wlh) _dispatch_queue_class_record_wlh_hierarchy(dq, wlh);
+ }
}
DISPATCH_ALWAYS_INLINE
@@ -5024,7 +4700,7 @@
_dispatch_queue_invoke(dispatch_queue_t dq, dispatch_invoke_context_t dic,
dispatch_invoke_flags_t flags)
{
- _dispatch_queue_class_invoke(dq, dic, flags, 0, dispatch_queue_invoke2);
+ _dispatch_queue_class_invoke(dq, dic, flags, dispatch_queue_invoke2);
}
#pragma mark -
@@ -5043,6 +4719,7 @@
dou._do = dc->dc_data;
old_dp = _dispatch_root_queue_identity_assume(assumed_rq);
+ flags |= DISPATCH_INVOKE_OVERRIDING;
if (dc_type(dc) == DISPATCH_CONTINUATION_TYPE(OVERRIDE_STEALING)) {
flags |= DISPATCH_INVOKE_STEALING;
} else {
@@ -5061,6 +4738,19 @@
_dispatch_queue_set_current(old_rq);
}
+#if DISPATCH_USE_KEVENT_WORKQUEUE
+DISPATCH_ALWAYS_INLINE
+static inline dispatch_qos_t
+_dispatch_qos_root_queue_push_wlh(dispatch_queue_t rq, dispatch_qos_t qos)
+{
+ // for root queues, the override is the guaranteed minimum override level
+ if (qos > _dispatch_priority_override_qos(rq->dq_priority)) {
+ return qos;
+ }
+ return _dispatch_priority_qos(rq->dq_priority);
+}
+#endif // DISPATCH_USE_KEVENT_WORKQUEUE
+
DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_root_queue_push_needs_override(dispatch_queue_t rq,
@@ -5120,7 +4810,7 @@
dispatch_continuation_t dc = _dispatch_continuation_alloc();
dc->do_vtable = DC_VTABLE(OVERRIDE_STEALING);
- _dispatch_retain_2(dq);
+ _dispatch_retain(dq);
dc->dc_func = NULL;
dc->dc_ctxt = dc;
dc->dc_other = orig_rq;
@@ -5131,19 +4821,27 @@
}
DISPATCH_NOINLINE
-static void
-_dispatch_queue_class_wakeup_with_override_slow(dispatch_queue_t dq,
- uint64_t dq_state, dispatch_wakeup_flags_t flags)
+void
+_dispatch_queue_class_wakeup_with_override(dispatch_queue_t dq,
+ dispatch_qos_t qos, dispatch_wakeup_flags_t flags, uint64_t dq_state)
{
- dispatch_qos_t oqos, qos = _dq_state_max_qos(dq_state);
+ mach_port_t owner = _dq_state_drain_owner(dq_state);
dispatch_queue_t tq;
+ dispatch_qos_t oqos;
bool locked;
- if (_dq_state_is_base_anon(dq_state)) {
- mach_port_t owner = _dq_state_drain_owner(dq_state);
- if (owner) {
- (void)_dispatch_wqthread_override_start_check_owner(owner, qos,
+ if (_dq_state_is_suspended(dq_state)) {
+ goto out;
+ }
+
+ if (owner) {
+ int rc = _dispatch_wqthread_override_start_check_owner(owner, qos,
&dq->dq_state_lock);
+ // EPERM means the target of the override is not a work queue thread
+ // and could be a thread-bound queue such as the main queue.
+ // When that happens we must get to that queue and wake it up if we
+ // want the override to be appplied and take effect.
+ if (rc != EPERM) {
goto out;
}
}
@@ -5205,10 +4903,12 @@
apply_again:
if (dx_type(tq) == DISPATCH_QUEUE_GLOBAL_ROOT_TYPE) {
if (_dispatch_root_queue_push_queue_override_needed(tq, qos)) {
- _dispatch_root_queue_push_override_stealer(tq, dq, qos);
+ _dispatch_root_queue_push_queue_override(tq, dq, qos);
}
+ } else if (flags & DISPATCH_WAKEUP_WAITER_HANDOFF) {
+ dx_wakeup(tq, qos, flags);
} else if (_dispatch_queue_need_override(tq, qos)) {
- dx_wakeup(tq, qos, 0);
+ dx_wakeup(tq, qos, DISPATCH_WAKEUP_OVERRIDING);
}
while (unlikely(locked && !_dispatch_queue_sidelock_tryunlock(dq))) {
// rdar://problem/24081326
@@ -5228,62 +4928,145 @@
}
out:
- if (flags & DISPATCH_WAKEUP_CONSUME_2) {
- return _dispatch_release_2_tailcall(dq);
+ if (flags & DISPATCH_WAKEUP_CONSUME) {
+ return _dispatch_release_tailcall(dq);
}
}
-
-
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_queue_class_wakeup_with_override(dispatch_queue_t dq,
- uint64_t dq_state, dispatch_wakeup_flags_t flags)
-{
- dispatch_assert(_dq_state_should_override(dq_state));
-
- return _dispatch_queue_class_wakeup_with_override_slow(dq, dq_state, flags);
-}
#endif // HAVE_PTHREAD_WORKQUEUE_QOS
DISPATCH_NOINLINE
void
-_dispatch_root_queue_push(dispatch_queue_t rq, dispatch_object_t dou,
+_dispatch_queue_class_override_drainer(dispatch_queue_t dq,
+ dispatch_qos_t qos, dispatch_wakeup_flags_t flags)
+{
+#if HAVE_PTHREAD_WORKQUEUE_QOS
+ uint64_t old_state, new_state;
+
+ //
+ // Someone is trying to override the last work item of the queue.
+ //
+ os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, relaxed, {
+ if (!_dq_state_drain_locked(old_state) &&
+ !_dq_state_is_dirty(old_state)) {
+ os_atomic_rmw_loop_give_up(goto done);
+ }
+ new_state = _dq_state_merge_qos(old_state, qos);
+ if (new_state == old_state) {
+ os_atomic_rmw_loop_give_up(goto done);
+ }
+ });
+ if (_dq_state_drain_locked(new_state)) {
+ return _dispatch_queue_class_wakeup_with_override(dq, qos,
+ flags, new_state);
+ }
+
+done:
+#else
+ (void)qos;
+#endif // HAVE_PTHREAD_WORKQUEUE_QOS
+ if (flags & DISPATCH_WAKEUP_CONSUME) {
+ return _dispatch_release_tailcall(dq);
+ }
+}
+
+#if HAVE_PTHREAD_WORKQUEUE_QOS
+DISPATCH_NOINLINE
+static void
+_dispatch_root_queue_push_queue_override(dispatch_queue_t rq,
+ dispatch_queue_class_t dqu, dispatch_qos_t qos)
+{
+ // thread bound queues always have an owner set, so should never reach
+ // this codepath (see _dispatch_queue_class_wakeup_with_override).
+ dispatch_assert(!_dispatch_queue_is_thread_bound(dqu._dq));
+ _dispatch_root_queue_push_override_stealer(rq, dqu._dq, qos);
+}
+#endif // HAVE_PTHREAD_WORKQUEUE_QOS
+
+#if DISPATCH_USE_KEVENT_WORKQUEUE
+DISPATCH_ALWAYS_INLINE
+static inline void
+_dispatch_root_queue_push_queue(dispatch_queue_t rq, dispatch_queue_class_t dqu,
+ dispatch_qos_t qos)
+{
+ // thread bound queues aren't woken up on root queues
+ dispatch_assert(!_dispatch_queue_is_thread_bound(dqu._dq));
+ if (likely(_dispatch_root_queue_allows_wlh_for_queue(rq, dqu._dq))) {
+ dispatch_qos_t wlh_qos;
+ wlh_qos = _dispatch_qos_root_queue_push_wlh(rq, qos);
+ }
+#if HAVE_PTHREAD_WORKQUEUE_QOS
+ if (_dispatch_root_queue_push_needs_override(rq, qos)) {
+ return _dispatch_root_queue_push_override(rq, dqu._dq->_as_do, qos);
+ }
+#endif
+ _dispatch_root_queue_push_inline(rq, dqu._dq, dqu._dq, 1);
+}
+
+DISPATCH_NOINLINE
+static void
+_dispatch_root_queue_push_try_stash(dispatch_queue_t rq,
+ dispatch_queue_class_t dqu, dispatch_qos_t qos,
+ dispatch_deferred_items_t ddi)
+{
+ dispatch_wlh_t cur_wlh = _dispatch_get_wlh();
+ dispatch_wlh_t wlh = _dispatch_root_queue_wlh_for_queue(rq, dqu);
+ dispatch_queue_t old_dq = ddi->ddi_stashed_dq;
+ dispatch_priority_t rq_overcommit;
+ rq_overcommit = rq->dq_priority & DISPATCH_PRIORITY_FLAG_OVERCOMMIT;
+
+ if (cur_wlh != DISPATCH_WLH_GLOBAL) {
+ if (cur_wlh != (dispatch_wlh_t)dqu._dq) {
+ goto out;
+ }
+ dispatch_assert(old_dq == NULL);
+ }
+
+ if (likely(!old_dq || rq_overcommit)) {
+ dispatch_queue_t old_rq = ddi->ddi_stashed_rq;
+ dispatch_priority_t old_pri = ddi->ddi_stashed_pri;
+ ddi->ddi_stashed_rq = rq;
+ ddi->ddi_stashed_dq = dqu._dq;
+ ddi->ddi_stashed_pri = _dispatch_priority_make(qos, 0) | rq_overcommit;
+ _dispatch_debug("wlh[%p]: deferring item %p, rq %p, pri 0x%x",
+ cur_wlh, dqu._dq, rq, ddi->ddi_stashed_pri);
+ if (likely(!old_dq)) {
+ return;
+ }
+ // push the previously stashed item
+ qos = _dispatch_priority_qos(old_pri);
+ rq = old_rq;
+ dqu._dq = old_dq;
+ }
+
+out:
+ if (cur_wlh != DISPATCH_WLH_GLOBAL) {
+ _dispatch_debug("wlh[%p]: not deferring item %p with wlh %p, rq %p",
+ cur_wlh, dqu._dq, wlh, rq);
+ }
+ _dispatch_root_queue_push_queue(rq, dqu, qos);
+}
+#endif // DISPATCH_USE_KEVENT_WORKQUEUE
+
+DISPATCH_NOINLINE
+void
+_dispatch_root_queue_push(dispatch_queue_t dq, dispatch_object_t dou,
dispatch_qos_t qos)
{
#if DISPATCH_USE_KEVENT_WORKQUEUE
- dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
- if (unlikely(ddi && ddi->ddi_can_stash)) {
- dispatch_object_t old_dou = ddi->ddi_stashed_dou;
- dispatch_priority_t rq_overcommit;
- rq_overcommit = rq->dq_priority & DISPATCH_PRIORITY_FLAG_OVERCOMMIT;
-
- if (likely(!old_dou._do || rq_overcommit)) {
- dispatch_queue_t old_rq = ddi->ddi_stashed_rq;
- dispatch_qos_t old_qos = ddi->ddi_stashed_qos;
- ddi->ddi_stashed_rq = rq;
- ddi->ddi_stashed_dou = dou;
- ddi->ddi_stashed_qos = qos;
- _dispatch_debug("deferring item %p, rq %p, qos %d",
- dou._do, rq, qos);
- if (rq_overcommit) {
- ddi->ddi_can_stash = false;
- }
- if (likely(!old_dou._do)) {
- return;
- }
- // push the previously stashed item
- qos = old_qos;
- rq = old_rq;
- dou = old_dou;
+ if (_dispatch_object_has_vtable(dou) && dx_vtable(dou._do)->do_push) {
+ dispatch_deferred_items_t ddi = _dispatch_deferred_items_get();
+ if (unlikely(ddi && ddi->ddi_stashed_pri != DISPATCH_PRIORITY_NOSTASH)){
+ return _dispatch_root_queue_push_try_stash(dq, dou._dq, qos, ddi);
}
+ return _dispatch_root_queue_push_queue(dq, dou._dq, qos);
}
#endif
#if HAVE_PTHREAD_WORKQUEUE_QOS
- if (_dispatch_root_queue_push_needs_override(rq, qos)) {
- return _dispatch_root_queue_push_override(rq, dou, qos);
+ if (_dispatch_root_queue_push_needs_override(dq, qos)) {
+ return _dispatch_root_queue_push_override(dq, dou, qos);
}
#endif
- _dispatch_root_queue_push_inline(rq, dou, dou, 1);
+ _dispatch_root_queue_push_inline(dq, dou, dou, 1);
}
void
@@ -5292,10 +5075,10 @@
{
if (!(flags & DISPATCH_WAKEUP_BLOCK_WAIT)) {
DISPATCH_INTERNAL_CRASH(dq->dq_priority,
- "Don't try to wake up or override a root queue");
+ "Trying to wake up or override a root queue");
}
- if (flags & DISPATCH_WAKEUP_CONSUME_2) {
- return _dispatch_release_2_tailcall(dq);
+ if (flags & DISPATCH_WAKEUP_CONSUME) {
+ return _dispatch_release_tailcall(dq);
}
}
@@ -5309,179 +5092,150 @@
DISPATCH_NOINLINE
void
+_dispatch_queue_class_wakeup_enqueue(dispatch_queue_t dq, dispatch_qos_t qos,
+ dispatch_wakeup_flags_t flags, dispatch_queue_wakeup_target_t target)
+{
+ dispatch_queue_t tq;
+
+ if (!(flags & DISPATCH_WAKEUP_CONSUME)) {
+ _dispatch_retain(dq);
+ }
+ if (target == DISPATCH_QUEUE_WAKEUP_TARGET) {
+ // try_become_enqueuer has no acquire barrier, as the last block
+ // of a queue asyncing to that queue is not an uncommon pattern
+ // and in that case the acquire is completely useless
+ //
+ // so instead use depdendency ordering to read the targetq pointer.
+ os_atomic_thread_fence(dependency);
+ tq = os_atomic_load_with_dependency_on2o(dq, do_targetq, (long)qos);
+ } else {
+ tq = target;
+ }
+ return dx_push(tq, dq, qos);
+}
+
+DISPATCH_ALWAYS_INLINE
+static void
+_dispatch_queue_class_wakeup_finish(dispatch_queue_t dq, dispatch_qos_t qos,
+ dispatch_wakeup_flags_t flags, dispatch_queue_wakeup_target_t target,
+ uint64_t old_state, uint64_t new_state)
+{
+ dispatch_assert(target != DISPATCH_QUEUE_WAKEUP_NONE);
+ dispatch_assert(target != DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT);
+
+ if ((old_state ^ new_state) & DISPATCH_QUEUE_MAX_QOS_MASK) {
+ flags |= DISPATCH_WAKEUP_OVERRIDING;
+ } else {
+ flags &= ~(dispatch_wakeup_flags_t)DISPATCH_WAKEUP_OVERRIDING;
+ qos = _dq_state_max_qos(new_state);
+ }
+ if ((old_state ^ new_state) & DISPATCH_QUEUE_ENQUEUED) {
+ return _dispatch_queue_class_wakeup_enqueue(dq, qos, flags, target);
+ }
+
+#if HAVE_PTHREAD_WORKQUEUE_QOS
+ if ((flags & (DISPATCH_WAKEUP_OVERRIDING | DISPATCH_WAKEUP_WAITER_HANDOFF))
+ && target != DISPATCH_QUEUE_WAKEUP_MGR) {
+ return _dispatch_queue_class_wakeup_with_override(dq, qos,
+ flags, new_state);
+ }
+#endif
+
+ if (flags & DISPATCH_WAKEUP_CONSUME) {
+ return _dispatch_release_tailcall(dq);
+ }
+}
+
+DISPATCH_NOINLINE
+void
_dispatch_queue_class_wakeup(dispatch_queue_t dq, dispatch_qos_t qos,
dispatch_wakeup_flags_t flags, dispatch_queue_wakeup_target_t target)
{
- dispatch_assert(target != DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT);
+ uint64_t old_state, new_state;
- if (target && !(flags & DISPATCH_WAKEUP_CONSUME_2)) {
- _dispatch_retain_2(dq);
- flags |= DISPATCH_WAKEUP_CONSUME_2;
- }
+ qos = _dispatch_queue_override_qos(dq, qos);
+ os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
+ new_state = _dq_state_merge_qos(old_state, qos);
+ if (likely(_dq_state_should_wakeup(old_state))) {
+ new_state |= DISPATCH_QUEUE_ENQUEUED;
+ }
+ if (flags & DISPATCH_WAKEUP_FLUSH) {
+ new_state |= DISPATCH_QUEUE_DIRTY;
+ } else if (new_state == old_state) {
+ os_atomic_rmw_loop_give_up(break);
+ }
+ });
- if (unlikely(flags & DISPATCH_WAKEUP_BARRIER_COMPLETE)) {
- //
- // _dispatch_queue_class_barrier_complete() is about what both regular
- // queues and sources needs to evaluate, but the former can have sync
- // handoffs to perform which _dispatch_queue_class_barrier_complete()
- // doesn't handle, only _dispatch_queue_barrier_complete() does.
- //
- // _dispatch_queue_wakeup() is the one for plain queues that calls
- // _dispatch_queue_barrier_complete(), and this is only taken for non
- // queue types.
- //
- dispatch_assert(dx_metatype(dq) != _DISPATCH_QUEUE_TYPE);
- return _dispatch_queue_class_barrier_complete(dq, qos, flags, target,
- DISPATCH_QUEUE_SERIAL_DRAIN_OWNED);
- }
-
- if (target) {
- uint64_t old_state, new_state, enqueue = DISPATCH_QUEUE_ENQUEUED;
- if (target == DISPATCH_QUEUE_WAKEUP_MGR) {
- enqueue = DISPATCH_QUEUE_ENQUEUED_ON_MGR;
- }
- qos = _dispatch_queue_override_qos(dq, qos);
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
- new_state = _dq_state_merge_qos(old_state, qos);
- if (likely(!_dq_state_is_suspended(old_state) &&
- !_dq_state_is_enqueued(old_state) &&
- (!_dq_state_drain_locked(old_state) ||
- (enqueue != DISPATCH_QUEUE_ENQUEUED_ON_MGR &&
- _dq_state_is_base_wlh(old_state))))) {
- new_state |= enqueue;
- }
- if (flags & DISPATCH_WAKEUP_MAKE_DIRTY) {
- new_state |= DISPATCH_QUEUE_DIRTY;
- } else if (new_state == old_state) {
- os_atomic_rmw_loop_give_up(goto done);
- }
- });
-
- if (likely((old_state ^ new_state) & enqueue)) {
- dispatch_queue_t tq;
- if (target == DISPATCH_QUEUE_WAKEUP_TARGET) {
- // the rmw_loop above has no acquire barrier, as the last block
- // of a queue asyncing to that queue is not an uncommon pattern
- // and in that case the acquire would be completely useless
- //
- // so instead use depdendency ordering to read
- // the targetq pointer.
- os_atomic_thread_fence(dependency);
- tq = os_atomic_load_with_dependency_on2o(dq, do_targetq,
- (long)new_state);
- } else {
- tq = target;
- }
- dispatch_assert(_dq_state_is_enqueued(new_state));
- return _dispatch_queue_push_queue(tq, dq, new_state);
- }
-#if HAVE_PTHREAD_WORKQUEUE_QOS
- if (unlikely((old_state ^ new_state) & DISPATCH_QUEUE_MAX_QOS_MASK)) {
- if (_dq_state_should_override(new_state)) {
- return _dispatch_queue_class_wakeup_with_override(dq, new_state,
- flags);
- }
- }
- } else if (qos) {
- //
- // Someone is trying to override the last work item of the queue.
- //
- uint64_t old_state, new_state;
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, relaxed, {
- if (!_dq_state_drain_locked(old_state) ||
- !_dq_state_is_enqueued(old_state)) {
- os_atomic_rmw_loop_give_up(goto done);
- }
- new_state = _dq_state_merge_qos(old_state, qos);
- if (new_state == old_state) {
- os_atomic_rmw_loop_give_up(goto done);
- }
- });
- if (_dq_state_should_override(new_state)) {
- return _dispatch_queue_class_wakeup_with_override(dq, new_state,
- flags);
- }
-#endif // HAVE_PTHREAD_WORKQUEUE_QOS
- }
-done:
- if (likely(flags & DISPATCH_WAKEUP_CONSUME_2)) {
- return _dispatch_release_2_tailcall(dq);
- }
+ return _dispatch_queue_class_wakeup_finish(dq, qos, flags, target,
+ old_state, new_state);
}
DISPATCH_NOINLINE
static void
_dispatch_queue_push_sync_waiter(dispatch_queue_t dq,
- dispatch_sync_context_t dsc, dispatch_qos_t qos)
+ dispatch_sync_context_t dsc)
{
+ uint64_t pending_barrier_width =
+ (dq->dq_width - 1) * DISPATCH_QUEUE_WIDTH_INTERVAL;
+ uint64_t xor_owner_and_set_full_width_and_in_barrier =
+ _dispatch_tid_self() | DISPATCH_QUEUE_WIDTH_FULL_BIT |
+ DISPATCH_QUEUE_IN_BARRIER;
+ dispatch_qos_t qos = _dispatch_continuation_override_qos(dq, dsc->_as_dc);
uint64_t old_state, new_state;
-
- if (unlikely(dx_type(dq) == DISPATCH_QUEUE_NETWORK_EVENT_TYPE)) {
- DISPATCH_CLIENT_CRASH(0,
- "dispatch_sync onto a network event queue");
- }
+ dispatch_wakeup_flags_t flags = 0;
_dispatch_trace_continuation_push(dq, dsc->_as_dc);
-
if (unlikely(_dispatch_queue_push_update_tail(dq, dsc->_as_do))) {
// for slow waiters, we borrow the reference of the caller
// so we don't need to protect the wakeup with a temporary retain
_dispatch_queue_push_update_head(dq, dsc->_as_do);
+ flags = DISPATCH_WAKEUP_FLUSH;
if (unlikely(_dispatch_queue_is_thread_bound(dq))) {
- return dx_wakeup(dq, qos, DISPATCH_WAKEUP_MAKE_DIRTY);
+ return dx_wakeup(dq, qos, flags);
}
-
- uint64_t pending_barrier_width =
- (dq->dq_width - 1) * DISPATCH_QUEUE_WIDTH_INTERVAL;
- uint64_t set_owner_and_set_full_width_and_in_barrier =
- _dispatch_lock_value_for_self() |
- DISPATCH_QUEUE_WIDTH_FULL_BIT | DISPATCH_QUEUE_IN_BARRIER;
- // similar to _dispatch_queue_drain_try_unlock()
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
- new_state = _dq_state_merge_qos(old_state, qos);
- new_state |= DISPATCH_QUEUE_DIRTY;
- if (unlikely(_dq_state_drain_locked(old_state) ||
- !_dq_state_is_runnable(old_state))) {
- // not runnable, so we should just handle overrides
- } else if (_dq_state_is_base_wlh(old_state) &&
- _dq_state_is_enqueued(old_state)) {
- // 32123779 let the event thread redrive since it's out already
- } else if (_dq_state_has_pending_barrier(old_state) ||
- new_state + pending_barrier_width <
- DISPATCH_QUEUE_WIDTH_FULL_BIT) {
- // see _dispatch_queue_drain_try_lock
- new_state &= DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK;
- new_state |= set_owner_and_set_full_width_and_in_barrier;
- }
- });
-
- if (_dq_state_is_base_wlh(old_state) &&
- (dsc->dsc_waiter == _dispatch_tid_self())) {
- dsc->dsc_wlh_was_first = true;
- }
-
- if ((old_state ^ new_state) & DISPATCH_QUEUE_IN_BARRIER) {
- return _dispatch_queue_barrier_complete(dq, qos, 0);
- }
-#if HAVE_PTHREAD_WORKQUEUE_QOS
- if (unlikely((old_state ^ new_state) & DISPATCH_QUEUE_MAX_QOS_MASK)) {
- if (_dq_state_should_override(new_state)) {
- return _dispatch_queue_class_wakeup_with_override(dq,
- new_state, 0);
- }
- }
- } else if (unlikely(qos)) {
- os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, relaxed, {
- new_state = _dq_state_merge_qos(old_state, qos);
- if (old_state == new_state) {
- os_atomic_rmw_loop_give_up(return);
- }
- });
- if (_dq_state_should_override(new_state)) {
- return _dispatch_queue_class_wakeup_with_override(dq, new_state, 0);
- }
-#endif // HAVE_PTHREAD_WORKQUEUE_QOS
}
+
+ os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, release, {
+ new_state = _dq_state_merge_qos(old_state, qos);
+#ifdef DLOCK_NOWAITERS_BIT
+ new_state |= DLOCK_NOWAITERS_BIT;
+#else
+ new_state |= DLOCK_WAITERS_BIT;
+#endif
+ if (flags & DISPATCH_WAKEUP_FLUSH) {
+ new_state |= DISPATCH_QUEUE_DIRTY;
+ }
+ if (_dq_state_drain_pended(old_state)) {
+ // same as DISPATCH_QUEUE_DRAIN_UNLOCK
+ // but we want to be more efficient wrt the WAITERS_BIT
+ new_state &= ~DISPATCH_QUEUE_DRAIN_OWNER_MASK;
+ new_state &= ~DISPATCH_QUEUE_DRAIN_PENDED;
+ }
+ if (unlikely(_dq_state_drain_locked(new_state))) {
+#ifdef DLOCK_NOWAITERS_BIT
+ new_state &= ~(uint64_t)DLOCK_NOWAITERS_BIT;
+#endif
+ } else if (unlikely(!_dq_state_is_runnable(new_state) ||
+ !(flags & DISPATCH_WAKEUP_FLUSH))) {
+ // either not runnable, or was not for the first item (26700358)
+ // so we should not try to lock and handle overrides instead
+ } else if (_dq_state_has_pending_barrier(old_state) ||
+ new_state + pending_barrier_width <
+ DISPATCH_QUEUE_WIDTH_FULL_BIT) {
+ // see _dispatch_queue_drain_try_lock
+ new_state &= DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK;
+ new_state ^= xor_owner_and_set_full_width_and_in_barrier;
+ } else {
+ new_state |= DISPATCH_QUEUE_ENQUEUED;
+ }
+ });
+
+ if ((old_state ^ new_state) & DISPATCH_QUEUE_IN_BARRIER) {
+ return _dispatch_try_lock_transfer_or_wakeup(dq);
+ }
+
+ return _dispatch_queue_class_wakeup_finish(dq, qos, flags,
+ DISPATCH_QUEUE_WAKEUP_TARGET, old_state, new_state);
}
#pragma mark -
@@ -5597,87 +5351,36 @@
return head;
}
-#if DISPATCH_USE_KEVENT_WORKQUEUE
void
-_dispatch_root_queue_drain_deferred_wlh(dispatch_deferred_items_t ddi
- DISPATCH_PERF_MON_ARGS_PROTO)
+_dispatch_root_queue_drain_deferred_item(dispatch_queue_t rq,
+ dispatch_queue_t dq DISPATCH_PERF_MON_ARGS_PROTO)
{
- dispatch_queue_t rq = ddi->ddi_stashed_rq;
- dispatch_queue_t dq = ddi->ddi_stashed_dou._dq;
- _dispatch_queue_set_current(rq);
- dispatch_priority_t old_pri = _dispatch_set_basepri_wlh(rq->dq_priority);
- dispatch_invoke_context_s dic = { };
- dispatch_invoke_flags_t flags = DISPATCH_INVOKE_WORKER_DRAIN |
- DISPATCH_INVOKE_REDIRECTING_DRAIN | DISPATCH_INVOKE_WLH;
- _dispatch_queue_drain_init_narrowing_check_deadline(&dic, rq->dq_priority);
- uint64_t dq_state;
+ // fake that we queued `dq` on `rq` for introspection purposes
+ _dispatch_trace_continuation_push(rq, dq);
- ddi->ddi_wlh_servicing = true;
- if (unlikely(_dispatch_needs_to_return_to_kernel())) {
- _dispatch_return_to_kernel();
- }
-retry:
- dispatch_assert(ddi->ddi_wlh_needs_delete);
- _dispatch_trace_continuation_pop(rq, dq);
-
- if (_dispatch_queue_drain_try_lock_wlh(dq, &dq_state)) {
- dx_invoke(dq, &dic, flags);
- if (!ddi->ddi_wlh_needs_delete) {
- goto park;
- }
- dq_state = os_atomic_load2o(dq, dq_state, relaxed);
- if (unlikely(_dq_state_is_enqueued_on_target(dq_state))) {
- _dispatch_retain(dq);
- _dispatch_trace_continuation_push(dq->do_targetq, dq);
- goto retry;
- }
- } else {
- _dispatch_release_no_dispose(dq);
- }
-
- _dispatch_event_loop_leave_deferred((dispatch_wlh_t)dq, dq_state);
-
-park:
- // event thread that could steal
- _dispatch_perfmon_end(perfmon_thread_event_steal);
- _dispatch_reset_basepri(old_pri);
- _dispatch_reset_basepri_override();
- _dispatch_queue_set_current(NULL);
-
- _dispatch_voucher_debug("root queue clear", NULL);
- _dispatch_reset_voucher(NULL, DISPATCH_THREAD_PARK);
-}
-
-void
-_dispatch_root_queue_drain_deferred_item(dispatch_deferred_items_t ddi
- DISPATCH_PERF_MON_ARGS_PROTO)
-{
- dispatch_queue_t rq = ddi->ddi_stashed_rq;
_dispatch_queue_set_current(rq);
dispatch_priority_t old_pri = _dispatch_set_basepri(rq->dq_priority);
+#if DISPATCH_COCOA_COMPAT
+ void *pool = _dispatch_last_resort_autorelease_pool_push();
+#endif // DISPATCH_COCOA_COMPAT
dispatch_invoke_context_s dic = { };
dispatch_invoke_flags_t flags = DISPATCH_INVOKE_WORKER_DRAIN |
DISPATCH_INVOKE_REDIRECTING_DRAIN;
-#if DISPATCH_COCOA_COMPAT
- _dispatch_last_resort_autorelease_pool_push(&dic);
-#endif // DISPATCH_COCOA_COMPAT
_dispatch_queue_drain_init_narrowing_check_deadline(&dic, rq->dq_priority);
- _dispatch_continuation_pop_inline(ddi->ddi_stashed_dou, &dic, flags, rq);
-
+ _dispatch_continuation_pop_inline(dq, &dic, flags, rq);
// event thread that could steal
_dispatch_perfmon_end(perfmon_thread_event_steal);
+
#if DISPATCH_COCOA_COMPAT
- _dispatch_last_resort_autorelease_pool_pop(&dic);
+ _dispatch_last_resort_autorelease_pool_pop(pool);
#endif // DISPATCH_COCOA_COMPAT
_dispatch_reset_basepri(old_pri);
- _dispatch_reset_basepri_override();
_dispatch_queue_set_current(NULL);
_dispatch_voucher_debug("root queue clear", NULL);
_dispatch_reset_voucher(NULL, DISPATCH_THREAD_PARK);
}
-#endif
DISPATCH_NOT_TAIL_CALLED // prevent tailcall (for Instrument DTrace probe)
static void
@@ -5693,14 +5396,14 @@
dispatch_priority_t pri = dq->dq_priority;
if (!pri) pri = _dispatch_priority_from_pp(pp);
dispatch_priority_t old_dbp = _dispatch_set_basepri(pri);
- _dispatch_adopt_wlh_anon();
+ _dispatch_set_wlh(DISPATCH_WLH_GLOBAL);
+#if DISPATCH_COCOA_COMPAT
+ void *pool = _dispatch_last_resort_autorelease_pool_push();
+#endif // DISPATCH_COCOA_COMPAT
struct dispatch_object_s *item;
bool reset = false;
dispatch_invoke_context_s dic = { };
-#if DISPATCH_COCOA_COMPAT
- _dispatch_last_resort_autorelease_pool_push(&dic);
-#endif // DISPATCH_COCOA_COMPAT
dispatch_invoke_flags_t flags = DISPATCH_INVOKE_WORKER_DRAIN |
DISPATCH_INVOKE_REDIRECTING_DRAIN;
_dispatch_queue_drain_init_narrowing_check_deadline(&dic, pri);
@@ -5722,11 +5425,10 @@
}
#if DISPATCH_COCOA_COMPAT
- _dispatch_last_resort_autorelease_pool_pop(&dic);
+ _dispatch_last_resort_autorelease_pool_pop(pool);
#endif // DISPATCH_COCOA_COMPAT
_dispatch_reset_wlh();
_dispatch_reset_basepri(old_dbp);
- _dispatch_reset_basepri_override();
_dispatch_queue_set_current(NULL);
}
@@ -5828,25 +5530,13 @@
#endif
(void)os_atomic_inc2o(qc, dgq_thread_pool_size, release);
_dispatch_global_queue_poke(dq, 1, 0);
- _dispatch_release(dq); // retained in _dispatch_global_queue_poke_slow
+ _dispatch_release(dq);
+
return NULL;
}
#endif // DISPATCH_USE_PTHREAD_POOL
#pragma mark -
-#pragma mark dispatch_network_root_queue
-#if TARGET_OS_MAC
-
-dispatch_queue_t
-_dispatch_network_root_queue_create_4NW(const char *label,
- const pthread_attr_t *attrs, dispatch_block_t configure)
-{
- unsigned long flags = dispatch_pthread_root_queue_flags_pool_size(1);
- return dispatch_pthread_root_queue_create(label, flags, attrs, configure);
-}
-
-#endif // TARGET_OS_MAC
-#pragma mark -
#pragma mark dispatch_runloop_queue
static bool _dispatch_program_is_probably_callback_driven;
@@ -5863,11 +5553,11 @@
return DISPATCH_BAD_INPUT;
}
dqs = sizeof(struct dispatch_queue_s) - DISPATCH_QUEUE_CACHELINE_PAD;
- dq = _dispatch_object_alloc(DISPATCH_VTABLE(queue_runloop), dqs);
- _dispatch_queue_init(dq, DQF_THREAD_BOUND | DQF_CANNOT_TRYSYNC, 1,
- DISPATCH_QUEUE_ROLE_BASE_ANON);
+ dq = _dispatch_alloc(DISPATCH_VTABLE(queue_runloop), dqs);
+ _dispatch_queue_init(dq, DQF_THREAD_BOUND | DQF_CANNOT_TRYSYNC, 1, false);
dq->do_targetq = _dispatch_get_root_queue(DISPATCH_QOS_DEFAULT, true);
dq->dq_label = label ? label : "runloop-queue"; // no-copy contract
+ dq->dq_wlh = DISPATCH_WLH_GLOBAL;
_dispatch_runloop_queue_handle_init(dq);
_dispatch_queue_set_bound_thread(dq);
_dispatch_object_debug(dq, "%s", __func__);
@@ -5879,19 +5569,19 @@
{
_dispatch_object_debug(dq, "%s", __func__);
- dispatch_qos_t qos = _dispatch_runloop_queue_reset_max_qos(dq);
+ dispatch_qos_t qos = _dispatch_queue_reset_max_qos(dq);
_dispatch_queue_clear_bound_thread(dq);
- dx_wakeup(dq, qos, DISPATCH_WAKEUP_MAKE_DIRTY);
+ dx_wakeup(dq, qos, DISPATCH_WAKEUP_FLUSH);
if (qos) _dispatch_thread_override_end(DISPATCH_QUEUE_DRAIN_OWNER(dq), dq);
}
void
-_dispatch_runloop_queue_dispose(dispatch_queue_t dq, bool *allow_free)
+_dispatch_runloop_queue_dispose(dispatch_queue_t dq)
{
_dispatch_object_debug(dq, "%s", __func__);
_dispatch_introspection_queue_dispose(dq);
_dispatch_runloop_queue_handle_dispose(dq);
- _dispatch_queue_destroy(dq, allow_free);
+ _dispatch_queue_destroy(dq);
}
bool
@@ -6126,7 +5816,7 @@
new_state += DISPATCH_QUEUE_IN_BARRIER;
});
_dispatch_queue_atomic_flags_clear(dq, DQF_THREAD_BOUND|DQF_CANNOT_TRYSYNC);
- _dispatch_queue_barrier_complete(dq, 0, 0);
+ _dispatch_try_lock_transfer_or_wakeup(dq);
// overload the "probably" variable to mean that dispatch_main() or
// similar non-POSIX API was called
@@ -6158,13 +5848,12 @@
"Premature thread exit while a dispatch queue is running");
}
+DISPATCH_NORETURN
static void
_dispatch_wlh_cleanup(void *ctxt)
{
// POSIX defines that destructors are only called if 'ctxt' is non-null
- dispatch_queue_t wlh;
- wlh = (dispatch_queue_t)((uintptr_t)ctxt & ~DISPATCH_WLH_STORAGE_REF);
- _dispatch_queue_release_storage(wlh);
+ DISPATCH_INTERNAL_CRASH(ctxt, "Premature thread exit with active wlh");
}
DISPATCH_NORETURN
diff --git a/src/queue_internal.h b/src/queue_internal.h
index c1d0f6e..91a3186 100644
--- a/src/queue_internal.h
+++ b/src/queue_internal.h
@@ -44,9 +44,6 @@
#define DISPATCH_CACHELINE_ALIGN \
__attribute__((__aligned__(DISPATCH_CACHELINE_SIZE)))
-#define DISPATCH_CACHELINE_PAD_SIZE(type) \
- (roundup(sizeof(type), DISPATCH_CACHELINE_SIZE) - sizeof(type))
-
#pragma mark -
#pragma mark dispatch_queue_t
@@ -63,6 +60,7 @@
DQF_CANNOT_TRYSYNC = 0x00400000,
DQF_RELEASED = 0x00800000, // xref_cnt == -1
DQF_LEGACY = 0x01000000,
+ DQF_WLH_CHANGED = 0x02000000, // queue wlh changed from initial value
// only applies to sources
//
@@ -101,7 +99,6 @@
// `a` cannot do a cleared -> set transition anymore
// (see _dispatch_source_try_set_armed).
//
- DSF_WLH_CHANGED = 0x04000000,
DSF_CANCEL_WAITER = 0x08000000, // synchronous waiters for cancel
DSF_CANCELED = 0x10000000, // cancellation has been requested
DSF_ARMED = 0x20000000, // source is armed
@@ -118,6 +115,10 @@
struct os_mpsc_queue_s _as_oq[0]; \
DISPATCH_OBJECT_HEADER(x); \
_OS_MPSC_QUEUE_FIELDS(dq, dq_state); \
+ DISPATCH_UNION_LE(uint32_t volatile dq_atomic_flags, \
+ const uint16_t dq_width, \
+ const uint16_t __dq_opaque \
+ ); \
uint32_t dq_side_suspend_cnt; \
dispatch_unfair_lock_s dq_sidelock; \
union { \
@@ -126,26 +127,27 @@
struct dispatch_timer_source_refs_s *ds_timer_refs; \
struct dispatch_mach_recv_refs_s *dm_recv_refs; \
}; \
- DISPATCH_UNION_LE(uint32_t volatile dq_atomic_flags, \
- const uint16_t dq_width, \
- const uint16_t __dq_opaque \
- ); \
DISPATCH_INTROSPECTION_QUEUE_HEADER
- /* LP64: 32bit hole */
#define DISPATCH_QUEUE_HEADER(x) \
struct dispatch_queue_s _as_dq[0]; \
_DISPATCH_QUEUE_HEADER(x)
-struct _dispatch_unpadded_queue_s {
- _DISPATCH_QUEUE_HEADER(dummy);
-};
-
-#define DISPATCH_QUEUE_CACHELINE_PAD \
- DISPATCH_CACHELINE_PAD_SIZE(struct _dispatch_unpadded_queue_s)
-
#define DISPATCH_QUEUE_CACHELINE_PADDING \
char _dq_pad[DISPATCH_QUEUE_CACHELINE_PAD]
+#ifdef __LP64__
+#define DISPATCH_QUEUE_CACHELINE_PAD (( \
+ (sizeof(uint32_t) - DISPATCH_INTROSPECTION_QUEUE_HEADER_SIZE) \
+ + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE)
+#elif OS_OBJECT_HAVE_OBJC1
+#define DISPATCH_QUEUE_CACHELINE_PAD (( \
+ (11*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_HEADER_SIZE) \
+ + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE)
+#else
+#define DISPATCH_QUEUE_CACHELINE_PAD (( \
+ (12*sizeof(void*) - DISPATCH_INTROSPECTION_QUEUE_HEADER_SIZE) \
+ + DISPATCH_CACHELINE_SIZE) % DISPATCH_CACHELINE_SIZE)
+#endif
/*
* dispatch queues `dq_state` demystified
@@ -155,27 +157,27 @@
* Most Significant 32 bit Word
* ----------------------------
*
- * sc: suspend count (bits 63 - 58)
+ * sc: suspend count (bits 63 - 57)
* The suspend count unsurprisingly holds the suspend count of the queue
* Only 7 bits are stored inline. Extra counts are transfered in a side
* suspend count and when that has happened, the ssc: bit is set.
*/
-#define DISPATCH_QUEUE_SUSPEND_INTERVAL 0x0400000000000000ull
-#define DISPATCH_QUEUE_SUSPEND_HALF 0x20u
+#define DISPATCH_QUEUE_SUSPEND_INTERVAL 0x0200000000000000ull
+#define DISPATCH_QUEUE_SUSPEND_HALF 0x40u
/*
- * ssc: side suspend count (bit 57)
+ * ssc: side suspend count (bit 56)
* This bit means that the total suspend count didn't fit in the inline
* suspend count, and that there are additional suspend counts stored in the
* `dq_side_suspend_cnt` field.
*/
-#define DISPATCH_QUEUE_HAS_SIDE_SUSPEND_CNT 0x0200000000000000ull
+#define DISPATCH_QUEUE_HAS_SIDE_SUSPEND_CNT 0x0100000000000000ull
/*
- * i: inactive bit (bit 56)
+ * i: inactive bit (bit 55)
* This bit means that the object is inactive (see dispatch_activate)
*/
-#define DISPATCH_QUEUE_INACTIVE 0x0100000000000000ull
+#define DISPATCH_QUEUE_INACTIVE 0x0080000000000000ull
/*
- * na: needs activation (bit 55)
+ * na: needs activation (bit 54)
* This bit is set if the object is created inactive. It tells
* dispatch_queue_wakeup to perform various tasks at first wakeup.
*
@@ -183,24 +185,24 @@
* the object from being woken up (because _dq_state_should_wakeup will say
* no), except in the dispatch_activate/dispatch_resume codepath.
*/
-#define DISPATCH_QUEUE_NEEDS_ACTIVATION 0x0080000000000000ull
+#define DISPATCH_QUEUE_NEEDS_ACTIVATION 0x0040000000000000ull
/*
* This mask covers the suspend count (sc), side suspend count bit (ssc),
* inactive (i) and needs activation (na) bits
*/
-#define DISPATCH_QUEUE_SUSPEND_BITS_MASK 0xff80000000000000ull
+#define DISPATCH_QUEUE_SUSPEND_BITS_MASK 0xffc0000000000000ull
/*
- * ib: in barrier (bit 54)
+ * ib: in barrier (bit 53)
* This bit is set when the queue is currently executing a barrier
*/
-#define DISPATCH_QUEUE_IN_BARRIER 0x0040000000000000ull
+#define DISPATCH_QUEUE_IN_BARRIER 0x0020000000000000ull
/*
- * qf: queue full (bit 53)
+ * qf: queue full (bit 52)
* This bit is a subtle hack that allows to check for any queue width whether
* the full width of the queue is used or reserved (depending on the context)
* In other words that the queue has reached or overflown its capacity.
*/
-#define DISPATCH_QUEUE_WIDTH_FULL_BIT 0x0020000000000000ull
+#define DISPATCH_QUEUE_WIDTH_FULL_BIT 0x0010000000000000ull
#define DISPATCH_QUEUE_WIDTH_FULL 0x1000ull
#define DISPATCH_QUEUE_WIDTH_POOL (DISPATCH_QUEUE_WIDTH_FULL - 1)
#define DISPATCH_QUEUE_WIDTH_MAX (DISPATCH_QUEUE_WIDTH_FULL - 2)
@@ -208,7 +210,7 @@
({ uint16_t _width = (width); \
_width > 1 && _width < DISPATCH_QUEUE_WIDTH_POOL; })
/*
- * w: width (bits 52 - 41)
+ * w: width (bits 51 - 40)
* This encodes how many work items are in flight. Barriers hold `dq_width`
* of them while they run. This is encoded as a signed offset with respect,
* to full use, where the negative values represent how many available slots
@@ -217,19 +219,29 @@
*
* When this value is positive, then `wo` is always set to 1.
*/
-#define DISPATCH_QUEUE_WIDTH_INTERVAL 0x0000020000000000ull
-#define DISPATCH_QUEUE_WIDTH_MASK 0x003ffe0000000000ull
-#define DISPATCH_QUEUE_WIDTH_SHIFT 41
+#define DISPATCH_QUEUE_WIDTH_INTERVAL 0x0000010000000000ull
+#define DISPATCH_QUEUE_WIDTH_MASK 0x001fff0000000000ull
+#define DISPATCH_QUEUE_WIDTH_SHIFT 40
/*
- * pb: pending barrier (bit 40)
+ * pb: pending barrier (bit 39)
* Drainers set this bit when they couldn't run the next work item and it is
* a barrier. When this bit is set, `dq_width - 1` work item slots are
* reserved so that no wakeup happens until the last work item in flight
* completes.
*/
-#define DISPATCH_QUEUE_PENDING_BARRIER 0x0000010000000000ull
+#define DISPATCH_QUEUE_PENDING_BARRIER 0x0000008000000000ull
/*
- * d: dirty bit (bit 39)
+ * p: pended bit (bit 38)
+ * Set when a drain lock has been pended. When this bit is set,
+ * the drain lock is taken and ENQUEUED is never set.
+ *
+ * This bit marks a queue that needs further processing but was kept pended
+ * by an async drainer (not reenqueued) in the hope of being able to drain
+ * it further later.
+ */
+#define DISPATCH_QUEUE_DRAIN_PENDED 0x0000004000000000ull
+/*
+ * d: dirty bit (bit 37)
* This bit is set when a queue transitions from empty to not empty.
* This bit is set before dq_items_head is set, with appropriate barriers.
* Any thread looking at a queue head is responsible for unblocking any
@@ -341,40 +353,18 @@
*
* So on the async "acquire" side, there is no subtlety at all.
*/
-#define DISPATCH_QUEUE_DIRTY 0x0000008000000000ull
+#define DISPATCH_QUEUE_DIRTY 0x0000002000000000ull
/*
- * md: enqueued/draining on manager (bit 38)
- * Set when enqueued and draining on the manager hierarchy.
- *
- * Unlike the ENQUEUED bit, it is kept until the queue is unlocked from its
- * invoke call on the manager. This is used to prevent stealing, and
- * overrides to be applied down the target queue chain.
+ * e: enqueued bit (bit 36)
+ * Set when a queue is enqueued on its target queue
*/
-#define DISPATCH_QUEUE_ENQUEUED_ON_MGR 0x0000004000000000ull
+#define DISPATCH_QUEUE_ENQUEUED 0x0000001000000000ull
/*
- * r: queue graph role (bits 37 - 36)
- * Queue role in the target queue graph
- *
- * 11: unused
- * 10: WLH base
- * 01: non wlh base
- * 00: inner queue
- */
-#define DISPATCH_QUEUE_ROLE_MASK 0x0000003000000000ull
-#define DISPATCH_QUEUE_ROLE_BASE_WLH 0x0000002000000000ull
-#define DISPATCH_QUEUE_ROLE_BASE_ANON 0x0000001000000000ull
-#define DISPATCH_QUEUE_ROLE_INNER 0x0000000000000000ull
-/*
- * o: has override (bit 35, if role is DISPATCH_QUEUE_ROLE_BASE_ANON)
+ * o: has override (bits 34)
* Set when a queue has received a QOS override and needs to reset it.
* This bit is only cleared when the final drain_try_unlock() succeeds.
- *
- * sw: has received sync wait (bit 35, if role DISPATCH_QUEUE_ROLE_BASE_WLH)
- * Set when a queue owner has been exposed to the kernel because of
- * dispatch_sync() contention.
*/
#define DISPATCH_QUEUE_RECEIVED_OVERRIDE 0x0000000800000000ull
-#define DISPATCH_QUEUE_RECEIVED_SYNC_WAIT 0x0000000800000000ull
/*
* max_qos: max qos (bits 34 - 32)
* This is the maximum qos that has been enqueued on the queue
@@ -386,25 +376,27 @@
* This is used by the normal drain to drain exlusively relative to other
* drain stealers (like the QoS Override codepath). It holds the identity
* (thread port) of the current drainer.
- *
- * st: sync transfer (bit 1 or 30)
- * Set when a dispatch_sync() is transferred to
- *
- * e: enqueued bit (bit 0 or 31)
- * Set when a queue is enqueued on its target queue
*/
-#define DISPATCH_QUEUE_DRAIN_OWNER_MASK ((uint64_t)DLOCK_OWNER_MASK)
-#define DISPATCH_QUEUE_SYNC_TRANSFER ((uint64_t)DLOCK_FAILED_TRYLOCK_BIT)
-#define DISPATCH_QUEUE_ENQUEUED ((uint64_t)DLOCK_WAITERS_BIT)
-
+#define DISPATCH_QUEUE_DRAIN_UNLOCK_MASK (DISPATCH_QUEUE_DRAIN_PENDED | ~0u)
+#ifdef DLOCK_NOWAITERS_BIT
+#define DISPATCH_QUEUE_DRAIN_OWNER_MASK \
+ ((uint64_t)(DLOCK_OWNER_MASK | DLOCK_NOFAILED_TRYLOCK_BIT))
+#define DISPATCH_QUEUE_DRAIN_UNLOCK(v) \
+ (((v) & ~(DISPATCH_QUEUE_DIRTY | DISPATCH_QUEUE_DRAIN_PENDED \
+ | DISPATCH_QUEUE_DRAIN_OWNER_MASK)) ^ DLOCK_NOWAITERS_BIT)
#define DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK \
- (DISPATCH_QUEUE_ENQUEUED_ON_MGR | DISPATCH_QUEUE_ENQUEUED | \
- DISPATCH_QUEUE_ROLE_MASK | DISPATCH_QUEUE_MAX_QOS_MASK)
-
-#define DISPATCH_QUEUE_DRAIN_UNLOCK_MASK \
- (DISPATCH_QUEUE_DRAIN_OWNER_MASK | DISPATCH_QUEUE_RECEIVED_OVERRIDE | \
- DISPATCH_QUEUE_RECEIVED_SYNC_WAIT | DISPATCH_QUEUE_SYNC_TRANSFER)
-
+ (DISPATCH_QUEUE_ENQUEUED | DISPATCH_QUEUE_MAX_QOS_MASK | \
+ DLOCK_NOWAITERS_BIT)
+#else
+#define DISPATCH_QUEUE_DRAIN_OWNER_MASK \
+ ((uint64_t)(DLOCK_OWNER_MASK | DLOCK_FAILED_TRYLOCK_BIT))
+#define DISPATCH_QUEUE_DRAIN_UNLOCK(v) \
+ ((v) & ~(DISPATCH_QUEUE_DIRTY | DISPATCH_QUEUE_DRAIN_PENDED | \
+ DISPATCH_QUEUE_DRAIN_OWNER_MASK))
+#define DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK \
+ (DISPATCH_QUEUE_ENQUEUED | DISPATCH_QUEUE_MAX_QOS_MASK | \
+ DLOCK_WAITERS_BIT)
+#endif
/*
*******************************************************************************
*
@@ -426,6 +418,8 @@
* that right. To do so, prior to taking any decision, they also try to own
* the full "barrier" width on the given queue.
*
+ * see _dispatch_try_lock_transfer_or_wakeup
+ *
*******************************************************************************
*
* Enqueuing and wakeup rules
@@ -496,16 +490,11 @@
(DISPATCH_QUEUE_IN_BARRIER | DISPATCH_QUEUE_WIDTH_INTERVAL)
DISPATCH_CLASS_DECL(queue);
-
#if !defined(__cplusplus) || !DISPATCH_INTROSPECTION
struct dispatch_queue_s {
_DISPATCH_QUEUE_HEADER(queue);
DISPATCH_QUEUE_CACHELINE_PADDING; // for static queues only
} DISPATCH_ATOMIC64_ALIGN;
-
-#if __has_feature(c_static_assert) && !DISPATCH_INTROSPECTION
-_Static_assert(sizeof(struct dispatch_queue_s) <= 128, "dispatch queue size");
-#endif
#endif // !defined(__cplusplus) || !DISPATCH_INTROSPECTION
DISPATCH_INTERNAL_SUBCLASS_DECL(queue_serial, queue);
@@ -556,51 +545,51 @@
#define DISPATCH_QUEUE_WAKEUP_MGR (&_dispatch_mgr_q)
#define DISPATCH_QUEUE_WAKEUP_WAIT_FOR_EVENT ((dispatch_queue_wakeup_target_t)-1)
+void _dispatch_queue_class_wakeup_with_override(dispatch_queue_t dq,
+ dispatch_qos_t qos, dispatch_wakeup_flags_t flags, uint64_t dq_state);
+void _dispatch_queue_class_override_drainer(dispatch_queue_t dqu,
+ dispatch_qos_t qos, dispatch_wakeup_flags_t flags);
+void _dispatch_queue_class_wakeup_enqueue(dispatch_queue_t dq,
+ dispatch_qos_t qos, dispatch_wakeup_flags_t flags,
+ dispatch_queue_wakeup_target_t target);
void _dispatch_queue_class_wakeup(dispatch_queue_t dqu, dispatch_qos_t qos,
dispatch_wakeup_flags_t flags, dispatch_queue_wakeup_target_t target);
-dispatch_priority_t _dispatch_queue_compute_priority_and_wlh(
- dispatch_queue_t dq, dispatch_wlh_t *wlh_out);
-void _dispatch_queue_destroy(dispatch_queue_t dq, bool *allow_free);
-void _dispatch_queue_dispose(dispatch_queue_t dq, bool *allow_free);
-void _dispatch_queue_xref_dispose(struct dispatch_queue_s *dq);
+
+void _dispatch_queue_destroy(dispatch_queue_t dq);
+void _dispatch_queue_dispose(dispatch_queue_t dq);
void _dispatch_queue_set_target_queue(dispatch_queue_t dq, dispatch_queue_t tq);
void _dispatch_queue_suspend(dispatch_queue_t dq);
void _dispatch_queue_resume(dispatch_queue_t dq, bool activate);
-void _dispatch_queue_finalize_activation(dispatch_queue_t dq,
- bool *allow_resume);
+void _dispatch_queue_finalize_activation(dispatch_queue_t dq);
void _dispatch_queue_invoke(dispatch_queue_t dq,
dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags);
void _dispatch_global_queue_poke(dispatch_queue_t dq, int n, int floor);
void _dispatch_queue_push(dispatch_queue_t dq, dispatch_object_t dou,
dispatch_qos_t qos);
+void _dispatch_try_lock_transfer_or_wakeup(dispatch_queue_t dq);
void _dispatch_queue_wakeup(dispatch_queue_t dq, dispatch_qos_t qos,
dispatch_wakeup_flags_t flags);
dispatch_queue_wakeup_target_t _dispatch_queue_serial_drain(dispatch_queue_t dq,
dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags,
uint64_t *owned);
-void _dispatch_queue_drain_sync_waiter(dispatch_queue_t dq,
+void _dispatch_queue_drain_deferred_invoke(dispatch_queue_t dq,
dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags,
- uint64_t owned);
-void _dispatch_queue_specific_queue_dispose(
- dispatch_queue_specific_queue_t dqsq, bool *allow_free);
+ uint64_t to_unlock);
+void _dispatch_queue_specific_queue_dispose(dispatch_queue_specific_queue_t
+ dqsq);
void _dispatch_root_queue_wakeup(dispatch_queue_t dq, dispatch_qos_t qos,
dispatch_wakeup_flags_t flags);
void _dispatch_root_queue_push(dispatch_queue_t dq, dispatch_object_t dou,
dispatch_qos_t qos);
-#if DISPATCH_USE_KEVENT_WORKQUEUE
-void _dispatch_root_queue_drain_deferred_item(dispatch_deferred_items_t ddi
- DISPATCH_PERF_MON_ARGS_PROTO);
-void _dispatch_root_queue_drain_deferred_wlh(dispatch_deferred_items_t ddi
- DISPATCH_PERF_MON_ARGS_PROTO);
-#endif
-void _dispatch_pthread_root_queue_dispose(dispatch_queue_t dq,
- bool *allow_free);
+void _dispatch_root_queue_drain_deferred_item(dispatch_queue_t rq,
+ dispatch_queue_t dq DISPATCH_PERF_MON_ARGS_PROTO);
+void _dispatch_pthread_root_queue_dispose(dispatch_queue_t dq);
void _dispatch_main_queue_wakeup(dispatch_queue_t dq, dispatch_qos_t qos,
dispatch_wakeup_flags_t flags);
void _dispatch_runloop_queue_wakeup(dispatch_queue_t dq, dispatch_qos_t qos,
dispatch_wakeup_flags_t flags);
void _dispatch_runloop_queue_xref_dispose(dispatch_queue_t dq);
-void _dispatch_runloop_queue_dispose(dispatch_queue_t dq, bool *allow_free);
+void _dispatch_runloop_queue_dispose(dispatch_queue_t dq);
void _dispatch_mgr_queue_drain(void);
#if DISPATCH_USE_MGR_THREAD && DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES
void _dispatch_mgr_priority_init(void);
@@ -651,13 +640,6 @@
_DISPATCH_ROOT_QUEUE_IDX_COUNT,
};
-// skip zero
-// 1 - main_q
-// 2 - mgr_q
-// 3 - mgr_root_q
-// 4,5,6,7,8,9,10,11,12,13,14,15 - global queues
-// we use 'xadd' on Intel, so the initial value == next assigned
-#define DISPATCH_QUEUE_SERIAL_NUMBER_INIT 16
extern unsigned long volatile _dispatch_queue_serial_numbers;
extern struct dispatch_queue_s _dispatch_root_queues[];
extern struct dispatch_queue_s _dispatch_mgr_q;
@@ -848,11 +830,8 @@
dispatch_thread_frame_s dsc_dtf;
#endif
dispatch_thread_event_s dsc_event;
- dispatch_tid dsc_waiter;
dispatch_qos_t dsc_override_qos_floor;
dispatch_qos_t dsc_override_qos;
- bool dsc_wlh_was_first;
- bool dsc_release_storage;
} *dispatch_sync_context_t;
typedef struct dispatch_continuation_vtable_s {
diff --git a/src/semaphore.c b/src/semaphore.c
index 3fe94c6..fa6d21a 100644
--- a/src/semaphore.c
+++ b/src/semaphore.c
@@ -52,16 +52,15 @@
return DISPATCH_BAD_INPUT;
}
- dsema = (dispatch_semaphore_t)_dispatch_object_alloc(
- DISPATCH_VTABLE(semaphore), sizeof(struct dispatch_semaphore_s));
+ dsema = (dispatch_semaphore_t)_dispatch_alloc(DISPATCH_VTABLE(semaphore),
+ sizeof(struct dispatch_semaphore_s));
_dispatch_semaphore_class_init(value, dsema);
dsema->dsema_orig = value;
return dsema;
}
void
-_dispatch_semaphore_dispose(dispatch_object_t dou,
- DISPATCH_UNUSED bool *allow_free)
+_dispatch_semaphore_dispose(dispatch_object_t dou)
{
dispatch_semaphore_t dsema = dou._dsema;
@@ -163,7 +162,7 @@
static inline dispatch_group_t
_dispatch_group_create_with_count(long count)
{
- dispatch_group_t dg = (dispatch_group_t)_dispatch_object_alloc(
+ dispatch_group_t dg = (dispatch_group_t)_dispatch_alloc(
DISPATCH_VTABLE(group), sizeof(struct dispatch_group_s));
_dispatch_semaphore_class_init(count, dg);
if (count) {
@@ -217,7 +216,6 @@
_dispatch_sema4_create(&dg->dg_sema, _DSEMA4_POLICY_FIFO);
_dispatch_sema4_signal(&dg->dg_sema, rval);
}
- uint16_t refs = needs_release ? 1 : 0; // <rdar://problem/22318411>
if (head) {
// async group notify blocks
do {
@@ -226,9 +224,11 @@
_dispatch_continuation_async(dsn_queue, head);
_dispatch_release(dsn_queue);
} while ((head = next));
- refs++;
+ _dispatch_release(dg);
}
- if (refs) _dispatch_release_n(dg, refs);
+ if (needs_release) {
+ _dispatch_release(dg); // <rdar://problem/22318411>
+ }
return 0;
}
@@ -246,7 +246,7 @@
}
void
-_dispatch_group_dispose(dispatch_object_t dou, DISPATCH_UNUSED bool *allow_free)
+_dispatch_group_dispose(dispatch_object_t dou)
{
dispatch_group_t dg = dou._dg;
diff --git a/src/semaphore_internal.h b/src/semaphore_internal.h
index f9d0983..3a4ef6d 100644
--- a/src/semaphore_internal.h
+++ b/src/semaphore_internal.h
@@ -63,11 +63,11 @@
} dispatch_semaphore_class_t DISPATCH_TRANSPARENT_UNION;
dispatch_group_t _dispatch_group_create_and_enter(void);
-void _dispatch_group_dispose(dispatch_object_t dou, bool *allow_free);
+void _dispatch_group_dispose(dispatch_object_t dou);
size_t _dispatch_group_debug(dispatch_object_t dou, char *buf,
size_t bufsiz);
-void _dispatch_semaphore_dispose(dispatch_object_t dou, bool *allow_free);
+void _dispatch_semaphore_dispose(dispatch_object_t dou);
size_t _dispatch_semaphore_debug(dispatch_object_t dou, char *buf,
size_t bufsiz);
diff --git a/src/shims/lock.c b/src/shims/lock.c
index 617fa01..de90d60 100644
--- a/src/shims/lock.c
+++ b/src/shims/lock.c
@@ -34,7 +34,6 @@
_Static_assert(DLOCK_LOCK_DATA_CONTENTION == ULF_WAIT_WORKQ_DATA_CONTENTION,
"values should be the same");
-#if !HAVE_UL_UNFAIR_LOCK
DISPATCH_ALWAYS_INLINE
static inline void
_dispatch_thread_switch(dispatch_lock value, dispatch_lock_options_t flags,
@@ -48,7 +47,6 @@
}
thread_switch(_dispatch_lock_owner(value), option, timeout);
}
-#endif // HAVE_UL_UNFAIR_LOCK
#endif
#pragma mark - semaphores
@@ -317,13 +315,12 @@
_dispatch_ulock_wait(uint32_t *uaddr, uint32_t val, uint32_t timeout,
uint32_t flags)
{
+ dispatch_assert(!DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK);
int rc;
_dlock_syscall_switch(err,
rc = __ulock_wait(UL_COMPARE_AND_WAIT | flags, uaddr, val, timeout),
case 0: return rc > 0 ? ENOTEMPTY : 0;
case ETIMEDOUT: case EFAULT: return err;
- case EOWNERDEAD: DISPATCH_CLIENT_CRASH(*uaddr,
- "corruption of lock owner");
default: DISPATCH_INTERNAL_CRASH(err, "ulock_wait() failed");
);
}
@@ -331,6 +328,7 @@
static void
_dispatch_ulock_wake(uint32_t *uaddr, uint32_t flags)
{
+ dispatch_assert(!DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK);
_dlock_syscall_switch(err,
__ulock_wake(UL_COMPARE_AND_WAIT | flags, uaddr, 0),
case 0: case ENOENT: break;
@@ -346,13 +344,17 @@
_dispatch_unfair_lock_wait(uint32_t *uaddr, uint32_t val, uint32_t timeout,
dispatch_lock_options_t flags)
{
+ if (DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK) {
+ // <rdar://problem/25075359>
+ timeout = timeout < 1000 ? 1 : timeout / 1000;
+ _dispatch_thread_switch(val, flags, timeout);
+ return 0;
+ }
int rc;
_dlock_syscall_switch(err,
rc = __ulock_wait(UL_UNFAIR_LOCK | flags, uaddr, val, timeout),
case 0: return rc > 0 ? ENOTEMPTY : 0;
case ETIMEDOUT: case EFAULT: return err;
- case EOWNERDEAD: DISPATCH_CLIENT_CRASH(*uaddr,
- "corruption of lock owner");
default: DISPATCH_INTERNAL_CRASH(err, "ulock_wait() failed");
);
}
@@ -360,6 +362,10 @@
static void
_dispatch_unfair_lock_wake(uint32_t *uaddr, uint32_t flags)
{
+ if (DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK) {
+ // <rdar://problem/25075359>
+ return;
+ }
_dlock_syscall_switch(err, __ulock_wake(UL_UNFAIR_LOCK | flags, uaddr, 0),
case 0: case ENOENT: break;
default: DISPATCH_INTERNAL_CRASH(err, "ulock_wake() failed");
@@ -466,6 +472,13 @@
void
_dispatch_thread_event_signal_slow(dispatch_thread_event_t dte)
{
+#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK
+ if (DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK) {
+ kern_return_t kr = semaphore_signal(dte->dte_sema);
+ DISPATCH_SEMAPHORE_VERIFY_KR(kr);
+ return;
+ }
+#endif
#if HAVE_UL_COMPARE_AND_WAIT
_dispatch_ulock_wake(&dte->dte_value, 0);
#elif HAVE_FUTEX
@@ -478,6 +491,16 @@
void
_dispatch_thread_event_wait_slow(dispatch_thread_event_t dte)
{
+#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK
+ if (DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK) {
+ kern_return_t kr;
+ do {
+ kr = semaphore_wait(dte->dte_sema);
+ } while (unlikely(kr == KERN_ABORTED));
+ DISPATCH_SEMAPHORE_VERIFY_KR(kr);
+ return;
+ }
+#endif
#if HAVE_UL_COMPARE_AND_WAIT || HAVE_FUTEX
for (;;) {
uint32_t value = os_atomic_load(&dte->dte_value, acquire);
@@ -505,30 +528,30 @@
_dispatch_unfair_lock_lock_slow(dispatch_unfair_lock_t dul,
dispatch_lock_options_t flags)
{
- dispatch_lock value_self = _dispatch_lock_value_for_self();
- dispatch_lock old_value, new_value, next = value_self;
+ dispatch_lock tid_self = _dispatch_tid_self(), next = tid_self;
+ dispatch_lock tid_old, tid_new;
int rc;
for (;;) {
- os_atomic_rmw_loop(&dul->dul_lock, old_value, new_value, acquire, {
- if (likely(!_dispatch_lock_is_locked(old_value))) {
- new_value = next;
+ os_atomic_rmw_loop(&dul->dul_lock, tid_old, tid_new, acquire, {
+ if (likely(!_dispatch_lock_is_locked(tid_old))) {
+ tid_new = next;
} else {
- new_value = old_value | DLOCK_WAITERS_BIT;
- if (new_value == old_value) os_atomic_rmw_loop_give_up(break);
+ tid_new = tid_old & ~DLOCK_NOWAITERS_BIT;
+ if (tid_new == tid_old) os_atomic_rmw_loop_give_up(break);
}
});
- if (unlikely(_dispatch_lock_is_locked_by(old_value, value_self))) {
+ if (unlikely(_dispatch_lock_is_locked_by(tid_old, tid_self))) {
DISPATCH_CLIENT_CRASH(0, "trying to lock recursively");
}
- if (new_value == next) {
+ if (tid_new == next) {
return;
}
- rc = _dispatch_unfair_lock_wait(&dul->dul_lock, new_value, 0, flags);
+ rc = _dispatch_unfair_lock_wait(&dul->dul_lock, tid_new, 0, flags);
if (rc == ENOTEMPTY) {
- next = value_self | DLOCK_WAITERS_BIT;
+ next = tid_self & ~DLOCK_NOWAITERS_BIT;
} else {
- next = value_self;
+ next = tid_self;
}
}
}
@@ -545,28 +568,30 @@
_dispatch_unfair_lock_lock_slow(dispatch_unfair_lock_t dul,
dispatch_lock_options_t flags)
{
- dispatch_lock cur, value_self = _dispatch_lock_value_for_self();
+ dispatch_lock tid_cur, tid_self = _dispatch_tid_self();
uint32_t timeout = 1;
while (unlikely(!os_atomic_cmpxchgv(&dul->dul_lock,
- DLOCK_OWNER_NULL, value_self, &cur, acquire))) {
- if (unlikely(_dispatch_lock_is_locked_by(cur, self))) {
+ DLOCK_OWNER_NULL, tid_self, &tid_cur, acquire))) {
+ if (unlikely(_dispatch_lock_is_locked_by(tid_cur, tid_self))) {
DISPATCH_CLIENT_CRASH(0, "trying to lock recursively");
}
- _dispatch_thread_switch(cur, flags, timeout++);
+ _dispatch_thread_switch(tid_cur, flags, timeout++);
}
}
#endif
void
-_dispatch_unfair_lock_unlock_slow(dispatch_unfair_lock_t dul, dispatch_lock cur)
+_dispatch_unfair_lock_unlock_slow(dispatch_unfair_lock_t dul,
+ dispatch_lock tid_cur)
{
- if (unlikely(!_dispatch_lock_is_locked_by_self(cur))) {
- DISPATCH_CLIENT_CRASH(cur, "lock not owned by current thread");
+ dispatch_lock_owner tid_self = _dispatch_tid_self();
+ if (unlikely(!_dispatch_lock_is_locked_by(tid_cur, tid_self))) {
+ DISPATCH_CLIENT_CRASH(tid_cur, "lock not owned by current thread");
}
#if HAVE_UL_UNFAIR_LOCK
- if (_dispatch_lock_has_waiters(cur)) {
+ if (!(tid_cur & DLOCK_NOWAITERS_BIT)) {
_dispatch_unfair_lock_wake(&dul->dul_lock, 0);
}
#elif HAVE_FUTEX
@@ -583,37 +608,41 @@
_dispatch_gate_wait_slow(dispatch_gate_t dgl, dispatch_lock value,
dispatch_lock_options_t flags)
{
- dispatch_lock self = _dispatch_lock_value_for_self();
- dispatch_lock old_value, new_value;
+ dispatch_lock tid_self = _dispatch_tid_self(), tid_old, tid_new;
uint32_t timeout = 1;
for (;;) {
- os_atomic_rmw_loop(&dgl->dgl_lock, old_value, new_value, acquire, {
- if (likely(old_value == value)) {
+ os_atomic_rmw_loop(&dgl->dgl_lock, tid_old, tid_new, acquire, {
+ if (likely(tid_old == value)) {
os_atomic_rmw_loop_give_up_with_fence(acquire, return);
}
- new_value = old_value | DLOCK_WAITERS_BIT;
- if (new_value == old_value) os_atomic_rmw_loop_give_up(break);
+#ifdef DLOCK_NOWAITERS_BIT
+ tid_new = tid_old & ~DLOCK_NOWAITERS_BIT;
+#else
+ tid_new = tid_old | DLOCK_WAITERS_BIT;
+#endif
+ if (tid_new == tid_old) os_atomic_rmw_loop_give_up(break);
});
- if (unlikely(_dispatch_lock_is_locked_by(old_value, self))) {
+ if (unlikely(_dispatch_lock_is_locked_by(tid_old, tid_self))) {
DISPATCH_CLIENT_CRASH(0, "trying to lock recursively");
}
#if HAVE_UL_UNFAIR_LOCK
- _dispatch_unfair_lock_wait(&dgl->dgl_lock, new_value, 0, flags);
+ _dispatch_unfair_lock_wait(&dgl->dgl_lock, tid_new, 0, flags);
#elif HAVE_FUTEX
- _dispatch_futex_wait(&dgl->dgl_lock, new_value, NULL, FUTEX_PRIVATE_FLAG);
+ _dispatch_futex_wait(&dgl->dgl_lock, tid_new, NULL, FUTEX_PRIVATE_FLAG);
#else
- _dispatch_thread_switch(new_value, flags, timeout++);
+ _dispatch_thread_switch(tid_new, flags, timeout++);
#endif
(void)timeout;
}
}
void
-_dispatch_gate_broadcast_slow(dispatch_gate_t dgl, dispatch_lock cur)
+_dispatch_gate_broadcast_slow(dispatch_gate_t dgl, dispatch_lock tid_cur)
{
- if (unlikely(!_dispatch_lock_is_locked_by_self(cur))) {
- DISPATCH_CLIENT_CRASH(cur, "lock not owned by current thread");
+ dispatch_lock_owner tid_self = _dispatch_tid_self();
+ if (unlikely(!_dispatch_lock_is_locked_by(tid_cur, tid_self))) {
+ DISPATCH_CLIENT_CRASH(tid_cur, "lock not owned by current thread");
}
#if HAVE_UL_UNFAIR_LOCK
diff --git a/src/shims/lock.h b/src/shims/lock.h
index 99c5563..4bbbb42 100644
--- a/src/shims/lock.h
+++ b/src/shims/lock.h
@@ -30,87 +30,43 @@
#pragma mark - platform macros
DISPATCH_ENUM(dispatch_lock_options, uint32_t,
- DLOCK_LOCK_NONE = 0x00000000,
- DLOCK_LOCK_DATA_CONTENTION = 0x00010000,
+ DLOCK_LOCK_NONE = 0x00000000,
+ DLOCK_LOCK_DATA_CONTENTION = 0x00010000,
);
#if TARGET_OS_MAC
-typedef mach_port_t dispatch_tid;
+typedef mach_port_t dispatch_lock_owner;
typedef uint32_t dispatch_lock;
+#define DLOCK_OWNER_NULL ((dispatch_lock_owner)MACH_PORT_NULL)
#define DLOCK_OWNER_MASK ((dispatch_lock)0xfffffffc)
-#define DLOCK_WAITERS_BIT ((dispatch_lock)0x00000001)
-#define DLOCK_FAILED_TRYLOCK_BIT ((dispatch_lock)0x00000002)
-
-#define DLOCK_OWNER_NULL ((dispatch_tid)MACH_PORT_NULL)
-#define _dispatch_tid_self() ((dispatch_tid)_dispatch_thread_port())
-
-DISPATCH_ALWAYS_INLINE
-static inline dispatch_tid
-_dispatch_lock_owner(dispatch_lock lock_value)
-{
- if (lock_value & DLOCK_OWNER_MASK) {
- return lock_value | DLOCK_WAITERS_BIT | DLOCK_FAILED_TRYLOCK_BIT;
- }
- return DLOCK_OWNER_NULL;
-}
-
-#elif defined(__linux__)
-
-#include <linux/futex.h>
-#if !defined(__x86_64__) && !defined(__i386__) && !defined(__s390x__)
-#include <linux/membarrier.h>
-#endif
-#include <unistd.h>
-#include <sys/syscall.h> /* For SYS_xxx definitions */
-
-typedef pid_t dispatch_tid;
-typedef uint32_t dispatch_lock;
-
-#define DLOCK_OWNER_MASK ((dispatch_lock)FUTEX_TID_MASK)
-#define DLOCK_WAITERS_BIT ((dispatch_lock)FUTEX_WAITERS)
-#define DLOCK_FAILED_TRYLOCK_BIT ((dispatch_lock)FUTEX_OWNER_DIED)
-
-#define DLOCK_OWNER_NULL ((dispatch_tid)0)
-#define _dispatch_tid_self() ((dispatch_tid)(_dispatch_get_tsd_base()->tid))
-
-DISPATCH_ALWAYS_INLINE
-static inline dispatch_tid
-_dispatch_lock_owner(dispatch_lock lock_value)
-{
- return lock_value & DLOCK_OWNER_MASK;
-}
-
-#else
-# error define _dispatch_lock encoding scheme for your platform here
-#endif
-
-DISPATCH_ALWAYS_INLINE
-static inline dispatch_lock
-_dispatch_lock_value_from_tid(dispatch_tid tid)
-{
- return tid & DLOCK_OWNER_MASK;
-}
-
-DISPATCH_ALWAYS_INLINE
-static inline dispatch_lock
-_dispatch_lock_value_for_self(void)
-{
- return _dispatch_lock_value_from_tid(_dispatch_tid_self());
-}
+#define DLOCK_OWNER_INVALID ((dispatch_lock)0xffffffff)
+#define DLOCK_NOWAITERS_BIT ((dispatch_lock)0x00000001)
+#define DLOCK_NOFAILED_TRYLOCK_BIT ((dispatch_lock)0x00000002)
+#define _dispatch_tid_self() ((dispatch_lock_owner)_dispatch_thread_port())
DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_lock_is_locked(dispatch_lock lock_value)
{
- // equivalent to _dispatch_lock_owner(lock_value) == 0
return (lock_value & DLOCK_OWNER_MASK) != 0;
}
DISPATCH_ALWAYS_INLINE
+static inline dispatch_lock_owner
+_dispatch_lock_owner(dispatch_lock lock_value)
+{
+ lock_value &= DLOCK_OWNER_MASK;
+ if (lock_value) {
+ lock_value |= DLOCK_NOWAITERS_BIT | DLOCK_NOFAILED_TRYLOCK_BIT;
+ }
+ return lock_value;
+}
+
+DISPATCH_ALWAYS_INLINE
static inline bool
-_dispatch_lock_is_locked_by(dispatch_lock lock_value, dispatch_tid tid)
+_dispatch_lock_is_locked_by(dispatch_lock lock_value, dispatch_lock_owner tid)
{
// equivalent to _dispatch_lock_owner(lock_value) == tid
return ((lock_value ^ tid) & DLOCK_OWNER_MASK) == 0;
@@ -118,10 +74,57 @@
DISPATCH_ALWAYS_INLINE
static inline bool
-_dispatch_lock_is_locked_by_self(dispatch_lock lock_value)
+_dispatch_lock_has_waiters(dispatch_lock lock_value)
{
- // equivalent to _dispatch_lock_owner(lock_value) == tid
- return ((lock_value ^ _dispatch_tid_self()) & DLOCK_OWNER_MASK) == 0;
+ bool nowaiters_bit = (lock_value & DLOCK_NOWAITERS_BIT);
+ return _dispatch_lock_is_locked(lock_value) != nowaiters_bit;
+}
+
+DISPATCH_ALWAYS_INLINE
+static inline bool
+_dispatch_lock_has_failed_trylock(dispatch_lock lock_value)
+{
+ return !(lock_value & DLOCK_NOFAILED_TRYLOCK_BIT);
+}
+
+#elif defined(__linux__)
+#include <linux/futex.h>
+#if !defined(__x86_64__) && !defined(__i386__) && !defined(__s390x__)
+#include <linux/membarrier.h>
+#endif
+#include <unistd.h>
+#include <sys/syscall.h> /* For SYS_xxx definitions */
+
+typedef uint32_t dispatch_lock;
+typedef pid_t dispatch_lock_owner;
+
+#define DLOCK_OWNER_NULL ((dispatch_lock_owner)0)
+#define DLOCK_OWNER_MASK ((dispatch_lock)FUTEX_TID_MASK)
+#define DLOCK_OWNER_INVALID ((dispatch_lock)DLOCK_OWNER_MASK)
+#define DLOCK_WAITERS_BIT ((dispatch_lock)FUTEX_WAITERS)
+#define DLOCK_FAILED_TRYLOCK_BIT ((dispatch_lock)FUTEX_OWNER_DIED)
+#define _dispatch_tid_self() \
+ ((dispatch_lock_owner)(_dispatch_get_tsd_base()->tid))
+
+DISPATCH_ALWAYS_INLINE
+static inline bool
+_dispatch_lock_is_locked(dispatch_lock lock_value)
+{
+ return (lock_value & DLOCK_OWNER_MASK) != 0;
+}
+
+DISPATCH_ALWAYS_INLINE
+static inline dispatch_lock_owner
+_dispatch_lock_owner(dispatch_lock lock_value)
+{
+ return (lock_value & DLOCK_OWNER_MASK);
+}
+
+DISPATCH_ALWAYS_INLINE
+static inline bool
+_dispatch_lock_is_locked_by(dispatch_lock lock_value, dispatch_lock_owner tid)
+{
+ return _dispatch_lock_owner(lock_value) == tid;
}
DISPATCH_ALWAYS_INLINE
@@ -135,18 +138,32 @@
static inline bool
_dispatch_lock_has_failed_trylock(dispatch_lock lock_value)
{
- return (lock_value & DLOCK_FAILED_TRYLOCK_BIT);
+ return !(lock_value & DLOCK_FAILED_TRYLOCK_BIT);
}
+#else
+# error define _dispatch_lock encoding scheme for your platform here
+#endif
+
#if __has_include(<sys/ulock.h>)
#include <sys/ulock.h>
-#ifdef UL_COMPARE_AND_WAIT
-#define HAVE_UL_COMPARE_AND_WAIT 1
#endif
-#ifdef UL_UNFAIR_LOCK
-#define HAVE_UL_UNFAIR_LOCK 1
+
+#ifndef HAVE_UL_COMPARE_AND_WAIT
+#if defined(UL_COMPARE_AND_WAIT) && DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+# define HAVE_UL_COMPARE_AND_WAIT 1
+#else
+# define HAVE_UL_COMPARE_AND_WAIT 0
#endif
+#endif // HAVE_UL_COMPARE_AND_WAIT
+
+#ifndef HAVE_UL_UNFAIR_LOCK
+#if defined(UL_UNFAIR_LOCK) && DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
+# define HAVE_UL_UNFAIR_LOCK 1
+#else
+# define HAVE_UL_UNFAIR_LOCK 0
#endif
+#endif // HAVE_UL_UNFAIR_LOCK
#ifndef HAVE_FUTEX
#ifdef __linux__
@@ -158,6 +175,14 @@
#pragma mark - semaphores
+#ifndef DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK
+#if TARGET_OS_MAC
+#define DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK (!HAVE_UL_COMPARE_AND_WAIT)
+#else
+#define DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK 0
+#endif
+#endif
+
#if USE_MACH_SEM
typedef semaphore_t _dispatch_sema4_t;
@@ -245,7 +270,12 @@
* This locking primitive has no notion of ownership
*/
typedef struct dispatch_thread_event_s {
-#if HAVE_UL_COMPARE_AND_WAIT || HAVE_FUTEX
+#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK
+ union {
+ _dispatch_sema4_t dte_sema;
+ uint32_t dte_value;
+ };
+#elif HAVE_UL_COMPARE_AND_WAIT || HAVE_FUTEX
// 1 means signalled but not waited on yet
// UINT32_MAX means waited on, but not signalled yet
// 0 is the initial and final state
@@ -263,6 +293,13 @@
static inline void
_dispatch_thread_event_init(dispatch_thread_event_t dte)
{
+#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK
+ if (DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK) {
+ _dispatch_sema4_init(&dte->dte_sema, _DSEMA4_POLICY_FIFO);
+ _dispatch_sema4_create(&dte->dte_sema, _DSEMA4_POLICY_FIFO);
+ return;
+ }
+#endif
#if HAVE_UL_COMPARE_AND_WAIT || HAVE_FUTEX
dte->dte_value = 0;
#else
@@ -274,6 +311,12 @@
static inline void
_dispatch_thread_event_signal(dispatch_thread_event_t dte)
{
+#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK
+ if (DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK) {
+ _dispatch_thread_event_signal_slow(dte);
+ return;
+ }
+#endif
#if HAVE_UL_COMPARE_AND_WAIT || HAVE_FUTEX
if (os_atomic_inc_orig(&dte->dte_value, release) == 0) {
// 0 -> 1 transition doesn't need a signal
@@ -292,6 +335,12 @@
static inline void
_dispatch_thread_event_wait(dispatch_thread_event_t dte)
{
+#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK
+ if (DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK) {
+ _dispatch_thread_event_wait_slow(dte);
+ return;
+ }
+#endif
#if HAVE_UL_COMPARE_AND_WAIT || HAVE_FUTEX
if (os_atomic_dec(&dte->dte_value, acquire) == 0) {
// 1 -> 0 is always a valid transition, so we can return
@@ -308,6 +357,12 @@
static inline void
_dispatch_thread_event_destroy(dispatch_thread_event_t dte)
{
+#if DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK
+ if (DISPATCH_LOCK_USE_SEMAPHORE_FALLBACK) {
+ _dispatch_sema4_dispose(&dte->dte_sema, _DSEMA4_POLICY_FIFO);
+ return;
+ }
+#endif
#if HAVE_UL_COMPARE_AND_WAIT || HAVE_FUTEX
// nothing to do
dispatch_assert(dte->dte_value == 0);
@@ -332,9 +387,9 @@
static inline void
_dispatch_unfair_lock_lock(dispatch_unfair_lock_t l)
{
- dispatch_lock value_self = _dispatch_lock_value_for_self();
+ dispatch_lock tid_self = _dispatch_tid_self();
if (likely(os_atomic_cmpxchg(&l->dul_lock,
- DLOCK_OWNER_NULL, value_self, acquire))) {
+ DLOCK_OWNER_NULL, tid_self, acquire))) {
return;
}
return _dispatch_unfair_lock_lock_slow(l, DLOCK_LOCK_NONE);
@@ -342,42 +397,54 @@
DISPATCH_ALWAYS_INLINE
static inline bool
-_dispatch_unfair_lock_trylock(dispatch_unfair_lock_t l, dispatch_tid *owner)
+_dispatch_unfair_lock_trylock(dispatch_unfair_lock_t l,
+ dispatch_lock_owner *owner)
{
- dispatch_lock value_self = _dispatch_lock_value_for_self();
- dispatch_lock old_value, new_value;
+ dispatch_lock tid_old, tid_new, tid_self = _dispatch_tid_self();
- os_atomic_rmw_loop(&l->dul_lock, old_value, new_value, acquire, {
- if (likely(!_dispatch_lock_is_locked(old_value))) {
- new_value = value_self;
+ os_atomic_rmw_loop(&l->dul_lock, tid_old, tid_new, acquire, {
+ if (likely(!_dispatch_lock_is_locked(tid_old))) {
+ tid_new = tid_self;
} else {
- new_value = old_value | DLOCK_FAILED_TRYLOCK_BIT;
+#ifdef DLOCK_NOFAILED_TRYLOCK_BIT
+ tid_new = tid_old & ~DLOCK_NOFAILED_TRYLOCK_BIT;
+#else
+ tid_new = tid_old | DLOCK_FAILED_TRYLOCK_BIT;
+#endif
}
});
- if (owner) *owner = _dispatch_lock_owner(new_value);
- return !_dispatch_lock_is_locked(old_value);
+ if (owner) *owner = _dispatch_lock_owner(tid_new);
+ return !_dispatch_lock_is_locked(tid_old);
}
DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_unfair_lock_tryunlock(dispatch_unfair_lock_t l)
{
- dispatch_lock old_value, new_value;
+ dispatch_lock tid_old, tid_new;
- os_atomic_rmw_loop(&l->dul_lock, old_value, new_value, release, {
- if (unlikely(old_value & DLOCK_FAILED_TRYLOCK_BIT)) {
- new_value = old_value ^ DLOCK_FAILED_TRYLOCK_BIT;
+ os_atomic_rmw_loop(&l->dul_lock, tid_old, tid_new, release, {
+#ifdef DLOCK_NOFAILED_TRYLOCK_BIT
+ if (likely(tid_old & DLOCK_NOFAILED_TRYLOCK_BIT)) {
+ tid_new = DLOCK_OWNER_NULL;
} else {
- new_value = DLOCK_OWNER_NULL;
+ tid_new = tid_old | DLOCK_NOFAILED_TRYLOCK_BIT;
}
+#else
+ if (likely(!(tid_old & DLOCK_FAILED_TRYLOCK_BIT))) {
+ tid_new = DLOCK_OWNER_NULL;
+ } else {
+ tid_new = tid_old & ~DLOCK_FAILED_TRYLOCK_BIT;
+ }
+#endif
});
- if (unlikely(new_value)) {
+ if (unlikely(tid_new)) {
// unlock failed, renew the lock, which needs an acquire barrier
os_atomic_thread_fence(acquire);
return false;
}
- if (unlikely(_dispatch_lock_has_waiters(old_value))) {
- _dispatch_unfair_lock_unlock_slow(l, old_value);
+ if (unlikely(_dispatch_lock_has_waiters(tid_old))) {
+ _dispatch_unfair_lock_unlock_slow(l, tid_old);
}
return true;
}
@@ -386,18 +453,18 @@
static inline bool
_dispatch_unfair_lock_unlock_had_failed_trylock(dispatch_unfair_lock_t l)
{
- dispatch_lock cur, value_self = _dispatch_lock_value_for_self();
+ dispatch_lock tid_cur, tid_self = _dispatch_tid_self();
#if HAVE_FUTEX
if (likely(os_atomic_cmpxchgv(&l->dul_lock,
- value_self, DLOCK_OWNER_NULL, &cur, release))) {
+ tid_self, DLOCK_OWNER_NULL, &tid_cur, release))) {
return false;
}
#else
- cur = os_atomic_xchg(&l->dul_lock, DLOCK_OWNER_NULL, release);
- if (likely(cur == value_self)) return false;
+ tid_cur = os_atomic_xchg(&l->dul_lock, DLOCK_OWNER_NULL, release);
+ if (likely(tid_cur == tid_self)) return false;
#endif
- _dispatch_unfair_lock_unlock_slow(l, cur);
- return _dispatch_lock_has_failed_trylock(cur);
+ _dispatch_unfair_lock_unlock_slow(l, tid_cur);
+ return _dispatch_lock_has_failed_trylock(tid_cur);
}
DISPATCH_ALWAYS_INLINE
@@ -440,8 +507,9 @@
static inline bool
_dispatch_gate_tryenter(dispatch_gate_t l)
{
- return os_atomic_cmpxchg(&l->dgl_lock, DLOCK_GATE_UNLOCKED,
- _dispatch_lock_value_for_self(), acquire);
+ dispatch_lock tid_self = _dispatch_tid_self();
+ return likely(os_atomic_cmpxchg(&l->dgl_lock,
+ DLOCK_GATE_UNLOCKED, tid_self, acquire));
}
#define _dispatch_gate_wait(l, flags) \
@@ -451,18 +519,19 @@
static inline void
_dispatch_gate_broadcast(dispatch_gate_t l)
{
- dispatch_lock cur, value_self = _dispatch_lock_value_for_self();
- cur = os_atomic_xchg(&l->dgl_lock, DLOCK_GATE_UNLOCKED, release);
- if (likely(cur == value_self)) return;
- _dispatch_gate_broadcast_slow(l, cur);
+ dispatch_lock tid_cur, tid_self = _dispatch_tid_self();
+ tid_cur = os_atomic_xchg(&l->dgl_lock, DLOCK_GATE_UNLOCKED, release);
+ if (likely(tid_cur == tid_self)) return;
+ _dispatch_gate_broadcast_slow(l, tid_cur);
}
DISPATCH_ALWAYS_INLINE
static inline bool
_dispatch_once_gate_tryenter(dispatch_once_gate_t l)
{
- return os_atomic_cmpxchg(&l->dgo_once, DLOCK_ONCE_UNLOCKED,
- (dispatch_once_t)_dispatch_lock_value_for_self(), acquire);
+ dispatch_once_t tid_self = (dispatch_once_t)_dispatch_tid_self();
+ return likely(os_atomic_cmpxchg(&l->dgo_once,
+ DLOCK_ONCE_UNLOCKED, tid_self, acquire));
}
#define _dispatch_once_gate_wait(l) \
@@ -501,10 +570,11 @@
static inline void
_dispatch_once_gate_broadcast(dispatch_once_gate_t l)
{
- dispatch_lock value_self = _dispatch_lock_value_for_self();
- dispatch_once_t cur = _dispatch_once_xchg_done(&l->dgo_once);
- if (likely(cur == (dispatch_once_t)value_self)) return;
- _dispatch_gate_broadcast_slow(&l->dgo_gate, (dispatch_lock)cur);
+ dispatch_once_t tid_cur, tid_self = (dispatch_once_t)_dispatch_tid_self();
+
+ tid_cur = _dispatch_once_xchg_done(&l->dgo_once);
+ if (likely(tid_cur == tid_self)) return;
+ _dispatch_gate_broadcast_slow(&l->dgo_gate, (dispatch_lock)tid_cur);
}
#endif // __DISPATCH_SHIMS_LOCK__
diff --git a/src/shims/perfmon.h b/src/shims/perfmon.h
index be9327b..fe23a1d 100644
--- a/src/shims/perfmon.h
+++ b/src/shims/perfmon.h
@@ -63,7 +63,6 @@
#define DISPATCH_PERF_MON_ARGS_PROTO , uint64_t perfmon_start
#define DISPATCH_PERF_MON_ARGS , perfmon_start
#define DISPATCH_PERF_MON_VAR uint64_t perfmon_start;
-#define DISPATCH_PERF_MON_VAR_INIT uint64_t perfmon_start = 0;
#define _dispatch_perfmon_start_impl(trace) ({ \
if (trace) _dispatch_ktrace0(DISPATCH_PERF_MON_worker_thread_start); \
@@ -85,7 +84,6 @@
#define DISPATCH_PERF_MON_ARGS_PROTO
#define DISPATCH_PERF_MON_ARGS
#define DISPATCH_PERF_MON_VAR
-#define DISPATCH_PERF_MON_VAR_INIT
#define _dispatch_perfmon_workitem_inc()
#define _dispatch_perfmon_workitem_dec()
#define _dispatch_perfmon_start_impl(trace)
diff --git a/src/shims/time.h b/src/shims/time.h
index 0b8e926..3010f08 100644
--- a/src/shims/time.h
+++ b/src/shims/time.h
@@ -46,15 +46,7 @@
#define DISPATCH_CLOCK_COUNT (DISPATCH_CLOCK_MACH + 1)
} dispatch_clock_t;
-void _dispatch_time_init(void);
-
#if defined(__i386__) || defined(__x86_64__) || !HAVE_MACH_ABSOLUTE_TIME
-#define DISPATCH_TIME_UNIT_USES_NANOSECONDS 1
-#else
-#define DISPATCH_TIME_UNIT_USES_NANOSECONDS 0
-#endif
-
-#if DISPATCH_TIME_UNIT_USES_NANOSECONDS
// x86 currently implements mach time in nanoseconds
// this is NOT likely to change
DISPATCH_ALWAYS_INLINE
@@ -71,21 +63,52 @@
return nsec;
}
#else
-#define DISPATCH_USE_HOST_TIME 1
-extern uint64_t (*_dispatch_host_time_mach2nano)(uint64_t machtime);
-extern uint64_t (*_dispatch_host_time_nano2mach)(uint64_t nsec);
+typedef struct _dispatch_host_time_data_s {
+ dispatch_once_t pred;
+ long double frac;
+ bool ratio_1_to_1;
+} _dispatch_host_time_data_s;
+extern _dispatch_host_time_data_s _dispatch_host_time_data;
+void _dispatch_get_host_time_init(void *context);
+
static inline uint64_t
_dispatch_time_mach2nano(uint64_t machtime)
{
- return _dispatch_host_time_mach2nano(machtime);
+ _dispatch_host_time_data_s *const data = &_dispatch_host_time_data;
+ dispatch_once_f(&data->pred, NULL, _dispatch_get_host_time_init);
+
+ if (unlikely(!machtime || data->ratio_1_to_1)) {
+ return machtime;
+ }
+ if (machtime >= INT64_MAX) {
+ return INT64_MAX;
+ }
+ long double big_tmp = ((long double)machtime * data->frac) + .5L;
+ if (unlikely(big_tmp >= INT64_MAX)) {
+ return INT64_MAX;
+ }
+ return (uint64_t)big_tmp;
}
static inline uint64_t
_dispatch_time_nano2mach(uint64_t nsec)
{
- return _dispatch_host_time_nano2mach(nsec);
+ _dispatch_host_time_data_s *const data = &_dispatch_host_time_data;
+ dispatch_once_f(&data->pred, NULL, _dispatch_get_host_time_init);
+
+ if (unlikely(!nsec || data->ratio_1_to_1)) {
+ return nsec;
+ }
+ if (nsec >= INT64_MAX) {
+ return INT64_MAX;
+ }
+ long double big_tmp = ((long double)nsec / data->frac) + .5L;
+ if (unlikely(big_tmp >= INT64_MAX)) {
+ return INT64_MAX;
+ }
+ return (uint64_t)big_tmp;
}
-#endif // DISPATCH_USE_HOST_TIME
+#endif
/* XXXRW: Some kind of overflow detection needed? */
#define _dispatch_timespec_to_nano(ts) \
@@ -100,7 +123,7 @@
dispatch_static_assert(sizeof(NSEC_PER_SEC) == 8);
dispatch_static_assert(sizeof(USEC_PER_SEC) == 8);
-#if TARGET_OS_MAC
+#if TARGET_OS_MAC && DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(101200)
return clock_gettime_nsec_np(CLOCK_REALTIME);
#elif HAVE_DECL_CLOCK_REALTIME
struct timespec ts;
diff --git a/src/shims/tsd.h b/src/shims/tsd.h
index c119e4f..f3d3cea 100644
--- a/src/shims/tsd.h
+++ b/src/shims/tsd.h
@@ -65,9 +65,6 @@
#ifndef __TSD_RETURN_TO_KERNEL
#define __TSD_RETURN_TO_KERNEL 5
#endif
-#ifndef __TSD_MACH_SPECIAL_REPLY
-#define __TSD_MACH_SPECIAL_REPLY 8
-#endif
static const unsigned long dispatch_priority_key = __TSD_THREAD_QOS_CLASS;
static const unsigned long dispatch_r2k_key = __TSD_RETURN_TO_KERNEL;
@@ -313,11 +310,6 @@
#define _dispatch_set_thread_mig_reply_port(p) ( \
_dispatch_thread_setspecific(_PTHREAD_TSD_SLOT_MIG_REPLY, \
(void*)(uintptr_t)(p)))
-#define _dispatch_get_thread_special_reply_port() ((mach_port_t)(uintptr_t) \
- _dispatch_thread_getspecific(__TSD_MACH_SPECIAL_REPLY))
-#define _dispatch_set_thread_special_reply_port(p) ( \
- _dispatch_thread_setspecific(__TSD_MACH_SPECIAL_REPLY, \
- (void*)(uintptr_t)(p)))
#endif
DISPATCH_TSD_INLINE DISPATCH_CONST
diff --git a/src/source.c b/src/source.c
index fd337a9..7c85c74 100644
--- a/src/source.c
+++ b/src/source.c
@@ -23,9 +23,7 @@
static void _dispatch_source_handler_free(dispatch_source_t ds, long kind);
static void _dispatch_source_set_interval(dispatch_source_t ds, uint64_t interval);
-#define DISPATCH_TIMERS_UNREGISTER 0x1
-#define DISPATCH_TIMERS_RETAIN_2 0x2
-static void _dispatch_timers_update(dispatch_unote_t du, uint32_t flags);
+static void _dispatch_timers_update(dispatch_unote_t du);
static void _dispatch_timers_unregister(dispatch_timer_source_refs_t dt);
static void _dispatch_source_timer_configure(dispatch_source_t ds);
@@ -42,16 +40,18 @@
dispatch_source_refs_t dr;
dispatch_source_t ds;
+ // ensure _dispatch_evfilt_machport_direct_enabled is initialized
+ _dispatch_root_queues_init();
+
dr = dux_create(dst, handle, mask)._dr;
if (unlikely(!dr)) {
return DISPATCH_BAD_INPUT;
}
- ds = _dispatch_object_alloc(DISPATCH_VTABLE(source),
+ ds = _dispatch_alloc(DISPATCH_VTABLE(source),
sizeof(struct dispatch_source_s));
// Initialize as a queue first, then override some settings below.
- _dispatch_queue_init(ds->_as_dq, DQF_LEGACY, 1,
- DISPATCH_QUEUE_INACTIVE | DISPATCH_QUEUE_ROLE_INNER);
+ _dispatch_queue_init(ds->_as_dq, DQF_LEGACY, 1, true);
ds->dq_label = "source";
ds->do_ref_cnt++; // the reference the manager queue holds
ds->ds_refs = dr;
@@ -71,7 +71,7 @@
}
void
-_dispatch_source_dispose(dispatch_source_t ds, bool *allow_free)
+_dispatch_source_dispose(dispatch_source_t ds)
{
_dispatch_object_debug(ds, "%s", __func__);
_dispatch_source_handler_free(ds, DS_REGISTN_HANDLER);
@@ -79,7 +79,7 @@
_dispatch_source_handler_free(ds, DS_CANCEL_HANDLER);
_dispatch_unote_dispose(ds->ds_refs);
ds->ds_refs = NULL;
- _dispatch_queue_destroy(ds->_as_dq, allow_free);
+ _dispatch_queue_destroy(ds->_as_dq);
}
void
@@ -90,7 +90,7 @@
DISPATCH_CLIENT_CRASH(ds, "Release of a source that has not been "
"cancelled, but has a mandatory cancel handler");
}
- dx_wakeup(ds, 0, DISPATCH_WAKEUP_MAKE_DIRTY);
+ dx_wakeup(ds, 0, DISPATCH_WAKEUP_FLUSH);
}
long
@@ -210,7 +210,7 @@
DISPATCH_CLIENT_CRASH(filter, "Invalid source type");
}
- dx_wakeup(ds, _dispatch_qos_from_pp(pp), DISPATCH_WAKEUP_MAKE_DIRTY);
+ dx_wakeup(ds, _dispatch_qos_from_pp(pp), DISPATCH_WAKEUP_FLUSH);
}
void
@@ -534,7 +534,6 @@
// to tell the truth, it may not have happened yet
if (dqf & DSF_ARMED) {
_dispatch_timers_unregister(ds->ds_timer_refs);
- _dispatch_release_2(ds);
}
dr->du_ident = DISPATCH_TIMER_IDENT_CANCELED;
} else {
@@ -556,7 +555,7 @@
}
ds->ds_is_installed = true;
_dispatch_debug("kevent-source[%p]: disarmed kevent[%p]", ds, dr);
- _dispatch_release_tailcall(ds); // the retain is done at creation time
+ _dispatch_release(ds); // the retain is done at creation time
}
DISPATCH_ALWAYS_INLINE
@@ -580,7 +579,7 @@
{
dispatch_source_refs_t dr = ds->ds_refs;
if (dr->du_is_timer) {
- _dispatch_timers_update(dr, 0);
+ _dispatch_timers_update(dr);
return true;
}
if (unlikely(!_dispatch_source_tryarm(ds))) {
@@ -592,17 +591,14 @@
}
void
-_dispatch_source_refs_register(dispatch_source_t ds, dispatch_wlh_t wlh,
- dispatch_priority_t pri)
+_dispatch_source_refs_register(dispatch_source_t ds, dispatch_priority_t pri)
{
dispatch_source_refs_t dr = ds->ds_refs;
- dispatch_priority_t kbp;
dispatch_assert(!ds->ds_is_installed);
if (dr->du_is_timer) {
- dispatch_queue_t dq = ds->_as_dq;
- kbp = _dispatch_queue_compute_priority_and_wlh(dq, NULL);
+ dispatch_priority_t kbp = _dispatch_source_compute_kevent_priority(ds);
// aggressively coalesce background/maintenance QoS timers
// <rdar://problem/12200216&27342536>
if (_dispatch_qos_is_background(_dispatch_priority_qos(kbp))) {
@@ -613,12 +609,12 @@
dr->du_ident = _dispatch_source_timer_idx(dr);
}
}
- _dispatch_timers_update(dr, 0);
+ _dispatch_timers_update(dr);
return;
}
if (unlikely(!_dispatch_source_tryarm(ds) ||
- !_dispatch_unote_register(dr, wlh, pri))) {
+ !_dispatch_unote_register(dr, ds->dq_wlh, pri))) {
_dispatch_queue_atomic_flags_set_and_clear(ds->_as_dq, DSF_DELETED,
DSF_ARMED | DSF_DEFERRED_DELETE);
} else {
@@ -638,22 +634,65 @@
}
}
-DISPATCH_ALWAYS_INLINE
-static inline void
-_dispatch_source_install(dispatch_source_t ds, dispatch_wlh_t wlh,
- dispatch_priority_t pri)
+dispatch_priority_t
+_dispatch_source_compute_kevent_priority(dispatch_source_t ds)
{
- _dispatch_source_refs_register(ds, wlh, pri);
+ dispatch_priority_t p = ds->dq_priority & DISPATCH_PRIORITY_REQUESTED_MASK;
+ dispatch_queue_t tq = ds->do_targetq;
+ dispatch_priority_t tqp = tq->dq_priority & DISPATCH_PRIORITY_REQUESTED_MASK;
+
+ while (unlikely(!dx_hastypeflag(tq, QUEUE_ROOT))) {
+ if (unlikely(tq == &_dispatch_mgr_q)) {
+ return DISPATCH_PRIORITY_FLAG_MANAGER;
+ }
+ if (unlikely(_dispatch_queue_is_thread_bound(tq))) {
+ // thread-bound hierarchies are weird, we need to install
+ // from the context of the thread this hierarchy is bound to
+ return 0;
+ }
+ if (unlikely(DISPATCH_QUEUE_IS_SUSPENDED(tq))) {
+ // this queue may not be activated yet, so the queue graph may not
+ // have stabilized yet
+ _dispatch_ktrace1(DISPATCH_PERF_delayed_registration, ds);
+ return 0;
+ }
+ if (unlikely(_dispatch_queue_is_legacy(tq))) {
+ if (!_dispatch_is_in_root_queues_array(tq->do_targetq)) {
+ // we're not allowed to dereference tq->do_targetq
+ _dispatch_ktrace1(DISPATCH_PERF_delayed_registration, ds);
+ return 0;
+ }
+ }
+ if (!(tq->dq_priority & DISPATCH_PRIORITY_FLAG_INHERIT)) {
+ if (p < tqp) p = tqp;
+ }
+ tq = tq->do_targetq;
+ tqp = tq->dq_priority & DISPATCH_PRIORITY_REQUESTED_MASK;
+ }
+
+ if (unlikely(!tqp)) {
+ // pthread root queues opt out of QoS
+ return 0;
+ }
+ return _dispatch_priority_inherit_from_root_queue(p, tq);
+}
+
+static void
+_dispatch_source_install(dispatch_source_t ds, dispatch_priority_t pri,
+ dispatch_wlh_t wlh)
+{
+ if (!ds->dq_wlh && wlh) {
+ _dispatch_queue_class_record_wlh_hierarchy(ds, wlh);
+ }
+ _dispatch_source_refs_register(ds, pri);
ds->ds_is_installed = true;
}
void
-_dispatch_source_finalize_activation(dispatch_source_t ds, bool *allow_resume)
+_dispatch_source_finalize_activation(dispatch_source_t ds)
{
dispatch_continuation_t dc;
dispatch_source_refs_t dr = ds->ds_refs;
- dispatch_priority_t pri;
- dispatch_wlh_t wlh;
if (unlikely(dr->du_is_direct &&
(_dispatch_queue_atomic_flags(ds->_as_dq) & DSF_CANCELED))) {
@@ -673,12 +712,15 @@
}
// call "super"
- _dispatch_queue_finalize_activation(ds->_as_dq, allow_resume);
+ _dispatch_queue_finalize_activation(ds->_as_dq);
if (dr->du_is_direct && !ds->ds_is_installed) {
- dispatch_queue_t dq = ds->_as_dq;
- pri = _dispatch_queue_compute_priority_and_wlh(dq, &wlh);
- if (pri) _dispatch_source_install(ds, wlh, pri);
+ dispatch_priority_t pri = _dispatch_source_compute_kevent_priority(ds);
+ if (pri) {
+ dispatch_wlh_t wlh = ds->dq_wlh;
+ if (!wlh) wlh = _dispatch_queue_class_compute_wlh(ds);
+ _dispatch_source_install(ds, pri, wlh);
+ }
}
}
@@ -690,18 +732,8 @@
dispatch_source_t ds = dou._ds;
dispatch_queue_wakeup_target_t retq = DISPATCH_QUEUE_WAKEUP_NONE;
dispatch_queue_t dq = _dispatch_queue_get_current();
- dispatch_source_refs_t dr = ds->ds_refs;
- dispatch_queue_flags_t dqf;
- if (!(flags & DISPATCH_INVOKE_MANAGER_DRAIN) &&
- _dispatch_unote_wlh_changed(dr, _dispatch_get_wlh())) {
- dqf = _dispatch_queue_atomic_flags_set_orig(ds->_as_dq,
- DSF_WLH_CHANGED);
- if (!(dqf & DSF_WLH_CHANGED)) {
- _dispatch_bug_deprecated("Changing target queue "
- "hierarchy after source was activated");
- }
- }
+ flags |= DISPATCH_INVOKE_DISALLOW_SYNC_WAITERS;
if (_dispatch_queue_class_probe(ds)) {
// Intentionally always drain even when on the manager queue
@@ -719,7 +751,9 @@
// The order of tests here in invoke and in wakeup should be consistent.
+ dispatch_source_refs_t dr = ds->ds_refs;
dispatch_queue_t dkq = &_dispatch_mgr_q;
+ dispatch_queue_flags_t dqf;
bool prevent_starvation = false;
if (dr->du_is_direct) {
@@ -743,8 +777,8 @@
if (dq != dkq) {
return dkq;
}
- _dispatch_source_install(ds, _dispatch_get_wlh(),
- _dispatch_get_basepri());
+ _dispatch_source_install(ds, _dispatch_get_basepri(),
+ _dispatch_get_wlh());
}
if (unlikely(DISPATCH_QUEUE_IS_SUSPENDED(ds))) {
@@ -851,7 +885,7 @@
// from the source handler
return ds->do_targetq;
}
- if (prevent_starvation && dr->du_wlh == DISPATCH_WLH_ANON) {
+ if (prevent_starvation && dr->du_wlh == DISPATCH_WLH_GLOBAL) {
// keep the old behavior to force re-enqueue to our target queue
// for the rearm.
//
@@ -863,7 +897,7 @@
if (unlikely(!_dispatch_source_refs_resume(ds))) {
goto unregister_event;
}
- if (!prevent_starvation && _dispatch_wlh_should_poll_unote(dr)) {
+ if (!prevent_starvation && dr->du_wlh != DISPATCH_WLH_GLOBAL) {
// try to redrive the drain from under the lock for sources
// targeting an overcommit root queue to avoid parking
// when the next event has already fired
@@ -879,8 +913,7 @@
_dispatch_source_invoke(dispatch_source_t ds, dispatch_invoke_context_t dic,
dispatch_invoke_flags_t flags)
{
- _dispatch_queue_class_invoke(ds, dic, flags,
- DISPATCH_INVOKE_DISALLOW_SYNC_WAITERS, _dispatch_source_invoke2);
+ _dispatch_queue_class_invoke(ds, dic, flags, _dispatch_source_invoke2);
}
void
@@ -945,12 +978,13 @@
tq = DISPATCH_QUEUE_WAKEUP_TARGET;
}
- if ((tq == DISPATCH_QUEUE_WAKEUP_TARGET) &&
- ds->do_targetq == &_dispatch_mgr_q) {
- tq = DISPATCH_QUEUE_WAKEUP_MGR;
+ if (tq) {
+ return _dispatch_queue_class_wakeup(ds->_as_dq, qos, flags, tq);
+ } else if (qos) {
+ return _dispatch_queue_class_override_drainer(ds->_as_dq, qos, flags);
+ } else if (flags & DISPATCH_WAKEUP_CONSUME) {
+ return _dispatch_release_tailcall(ds);
}
-
- return _dispatch_queue_class_wakeup(ds->_as_dq, qos, flags, tq);
}
void
@@ -961,13 +995,13 @@
// could potentially invoke the source, do the cancellation,
// unregister the source, and deallocate it. We would
// need to therefore retain/release before setting the bit
- _dispatch_retain_2(ds);
+ _dispatch_retain(ds);
dispatch_queue_t q = ds->_as_dq;
if (_dispatch_queue_atomic_flags_set_orig(q, DSF_CANCELED) & DSF_CANCELED) {
- _dispatch_release_2_tailcall(ds);
+ _dispatch_release_tailcall(ds);
} else {
- dx_wakeup(ds, 0, DISPATCH_WAKEUP_MAKE_DIRTY | DISPATCH_WAKEUP_CONSUME_2);
+ dx_wakeup(ds, 0, DISPATCH_WAKEUP_FLUSH | DISPATCH_WAKEUP_CONSUME);
}
}
@@ -1002,12 +1036,13 @@
return;
}
if (dqf & DSF_CANCEL_WAITER) {
- goto wakeup;
+ goto override;
}
// simplified version of _dispatch_queue_drain_try_lock
// that also sets the DIRTY bit on failure to lock
- uint64_t set_owner_and_set_full_width = _dispatch_lock_value_for_self() |
+ dispatch_lock_owner tid_self = _dispatch_tid_self();
+ uint64_t xor_owner_and_set_full_width = tid_self |
DISPATCH_QUEUE_WIDTH_FULL_BIT | DISPATCH_QUEUE_IN_BARRIER;
uint64_t old_state, new_state;
@@ -1016,7 +1051,7 @@
if (likely(_dq_state_is_runnable(old_state) &&
!_dq_state_drain_locked(old_state))) {
new_state &= DISPATCH_QUEUE_DRAIN_PRESERVED_BITS_MASK;
- new_state |= set_owner_and_set_full_width;
+ new_state ^= xor_owner_and_set_full_width;
} else if (old_dqf & DSF_CANCELED) {
os_atomic_rmw_loop_give_up(break);
} else {
@@ -1046,15 +1081,15 @@
_dispatch_source_cancel_callout(ds, NULL, DISPATCH_INVOKE_NONE);
}
}
- dx_wakeup(ds, 0, DISPATCH_WAKEUP_BARRIER_COMPLETE);
- } else if (unlikely(_dq_state_drain_locked_by_self(old_state))) {
+ _dispatch_try_lock_transfer_or_wakeup(ds->_as_dq);
+ } else if (unlikely(_dq_state_drain_locked_by(old_state, tid_self))) {
DISPATCH_CLIENT_CRASH(ds, "dispatch_source_cancel_and_wait "
"called from a source handler");
} else {
dispatch_qos_t qos;
-wakeup:
+override:
qos = _dispatch_qos_from_pp(_dispatch_get_priority());
- dx_wakeup(ds, qos, DISPATCH_WAKEUP_MAKE_DIRTY);
+ dx_wakeup(ds, qos, DISPATCH_WAKEUP_OVERRIDING | DISPATCH_WAKEUP_FLUSH);
dispatch_activate(ds);
}
@@ -1086,8 +1121,8 @@
// threads running _dispatch_source_invoke2 to dispose of the source,
// so we can't safely borrow the reference we get from the muxnote udata
// anymore, and need our own
- wflags = DISPATCH_WAKEUP_CONSUME_2;
- _dispatch_retain_2(ds); // rdar://20382435
+ wflags = DISPATCH_WAKEUP_CONSUME;
+ _dispatch_retain(ds); // rdar://20382435
}
if ((flags & EV_UDATA_SPECIFIC) && (flags & EV_ONESHOT) &&
@@ -1153,7 +1188,7 @@
done:
_dispatch_object_debug(ds, "%s", __func__);
- dx_wakeup(ds, _dispatch_qos_from_pp(pp), wflags | DISPATCH_WAKEUP_MAKE_DIRTY);
+ dx_wakeup(ds, _dispatch_qos_from_pp(pp), wflags | DISPATCH_WAKEUP_FLUSH);
}
#pragma mark -
@@ -1232,7 +1267,7 @@
// Clear any pending data that might have accumulated on
// older timer params <rdar://problem/8574886>
os_atomic_store2o(ds, ds_pending_data, 0, relaxed);
- _dispatch_timers_update(dt, 0);
+ _dispatch_timers_update(dt);
}
}
@@ -1308,7 +1343,7 @@
_dispatch_source_timer_telemetry(ds, dtc->dtc_clock, &dtc->dtc_timer);
dtc = os_atomic_xchg2o(dt, dt_pending_config, dtc, release);
if (dtc) free(dtc);
- dx_wakeup(ds, 0, DISPATCH_WAKEUP_MAKE_DIRTY);
+ dx_wakeup(ds, 0, DISPATCH_WAKEUP_FLUSH);
}
static void
@@ -1761,9 +1796,6 @@
{
uint32_t idx = (dth->dth_count += DTH_ID_COUNT) - DTH_ID_COUNT;
- dispatch_assert(dt->dt_heap_entry[DTH_TARGET_ID] == DTH_INVALID_ID);
- dispatch_assert(dt->dt_heap_entry[DTH_DEADLINE_ID] == DTH_INVALID_ID);
-
if (idx == 0) {
dt->dt_heap_entry[DTH_TARGET_ID] = DTH_TARGET_ID;
dt->dt_heap_entry[DTH_DEADLINE_ID] = DTH_DEADLINE_ID;
@@ -1782,36 +1814,27 @@
DISPATCH_NOINLINE
static void
_dispatch_timer_heap_remove(dispatch_timer_heap_t dth,
- dispatch_timer_source_refs_t dt)
+ dispatch_timer_source_refs_t removed_dt)
{
uint32_t idx = (dth->dth_count -= DTH_ID_COUNT);
- dispatch_assert(dt->dt_heap_entry[DTH_TARGET_ID] != DTH_INVALID_ID);
- dispatch_assert(dt->dt_heap_entry[DTH_DEADLINE_ID] != DTH_INVALID_ID);
-
if (idx == 0) {
- dispatch_assert(dth->dth_min[DTH_TARGET_ID] == dt);
- dispatch_assert(dth->dth_min[DTH_DEADLINE_ID] == dt);
dth->dth_min[DTH_TARGET_ID] = dth->dth_min[DTH_DEADLINE_ID] = NULL;
- goto clear_heap_entry;
+ return;
}
for (uint32_t heap_id = 0; heap_id < DTH_ID_COUNT; heap_id++) {
- dispatch_timer_source_refs_t *slot, last_dt;
+ dispatch_timer_source_refs_t *slot, dt;
slot = _dispatch_timer_heap_get_slot(dth, idx + heap_id);
- last_dt = *slot; *slot = NULL;
- if (last_dt != dt) {
- uint32_t removed_idx = dt->dt_heap_entry[heap_id];
- _dispatch_timer_heap_resift(dth, last_dt, removed_idx);
+ dt = *slot; *slot = NULL;
+ if (dt != removed_dt) {
+ uint32_t removed_idx = removed_dt->dt_heap_entry[heap_id];
+ _dispatch_timer_heap_resift(dth, dt, removed_idx);
}
}
if (unlikely(idx <= _dispatch_timer_heap_capacity(dth->dth_segments - 1))) {
_dispatch_timer_heap_shrink(dth);
}
-
-clear_heap_entry:
- dt->dt_heap_entry[DTH_TARGET_ID] = DTH_INVALID_ID;
- dt->dt_heap_entry[DTH_DEADLINE_ID] = DTH_INVALID_ID;
}
DISPATCH_ALWAYS_INLINE
@@ -1819,9 +1842,6 @@
_dispatch_timer_heap_update(dispatch_timer_heap_t dth,
dispatch_timer_source_refs_t dt)
{
- dispatch_assert(dt->dt_heap_entry[DTH_TARGET_ID] != DTH_INVALID_ID);
- dispatch_assert(dt->dt_heap_entry[DTH_DEADLINE_ID] != DTH_INVALID_ID);
-
_dispatch_timer_heap_resift(dth, dt, dt->dt_heap_entry[DTH_TARGET_ID]);
_dispatch_timer_heap_resift(dth, dt, dt->dt_heap_entry[DTH_DEADLINE_ID]);
}
@@ -1866,7 +1886,6 @@
_dispatch_timer_heap_remove(heap, dt);
_dispatch_timers_reconfigure = true;
_dispatch_timers_processing_mask |= 1 << tidx;
- dispatch_assert(dt->du_wlh == NULL || dt->du_wlh == DISPATCH_WLH_ANON);
dt->du_wlh = NULL;
}
@@ -1883,8 +1902,7 @@
}
_dispatch_timers_reconfigure = true;
_dispatch_timers_processing_mask |= 1 << tidx;
- dispatch_assert(dt->du_wlh == NULL || dt->du_wlh == DISPATCH_WLH_ANON);
- dt->du_wlh = DISPATCH_WLH_ANON;
+ dt->du_wlh = DISPATCH_WLH_GLOBAL;
}
DISPATCH_ALWAYS_INLINE
@@ -1904,7 +1922,7 @@
// Updates the ordered list of timers based on next fire date for changes to ds.
// Should only be called from the context of _dispatch_mgr_q.
static void
-_dispatch_timers_update(dispatch_unote_t du, uint32_t flags)
+_dispatch_timers_update(dispatch_unote_t du)
{
dispatch_timer_source_refs_t dr = du._dt;
dispatch_source_t ds = _dispatch_source_from_refs(dr);
@@ -1914,47 +1932,26 @@
DISPATCH_ASSERT_ON_MANAGER_QUEUE();
if (unlikely(dr->du_ident == DISPATCH_TIMER_IDENT_CANCELED)) {
- dispatch_assert((flags & DISPATCH_TIMERS_RETAIN_2) == 0);
return;
}
// Unregister timers that are unconfigured, disabled, suspended or have
// missed intervals. Rearm after dispatch_set_timer(), resume or source
// invoke will reenable them
- will_register = !(flags & DISPATCH_TIMERS_UNREGISTER) &&
- dr->dt_timer.target < INT64_MAX &&
+ will_register = dr->dt_timer.target < INT64_MAX &&
!os_atomic_load2o(ds, ds_pending_data, relaxed) &&
!DISPATCH_QUEUE_IS_SUSPENDED(ds) &&
!os_atomic_load2o(dr, dt_pending_config, relaxed);
- if (likely(!_dispatch_unote_registered(dr))) {
- dispatch_assert((flags & DISPATCH_TIMERS_RETAIN_2) == 0);
- if (unlikely(!will_register || !_dispatch_source_timer_tryarm(ds))) {
+ if (!_dispatch_unote_registered(dr) && will_register) {
+ if (unlikely(!_dispatch_source_timer_tryarm(ds))) {
return;
}
verb = "armed";
- } else if (unlikely(!will_register)) {
+ } else if (unlikely(_dispatch_unote_registered(dr) && !will_register)) {
disarm = true;
verb = "disarmed";
}
- // The heap owns a +2 on dispatch sources it references
- //
- // _dispatch_timers_run2() also sometimes passes DISPATCH_TIMERS_RETAIN_2
- // when it wants to take over this +2 at the same time we are unregistering
- // the timer from the heap.
- //
- // Compute our refcount balance according to these rules, if our balance
- // would become negative we retain the source upfront, if it is positive, we
- // get rid of the extraneous refcounts after we're done touching the source.
- int refs = will_register ? -2 : 0;
- if (_dispatch_unote_registered(dr) && !(flags & DISPATCH_TIMERS_RETAIN_2)) {
- refs += 2;
- }
- if (refs < 0) {
- dispatch_assert(refs == -2);
- _dispatch_retain_2(ds);
- }
-
uint32_t tidx = _dispatch_source_timer_idx(dr);
if (unlikely(_dispatch_unote_registered(dr) &&
(!will_register || dr->du_ident != tidx))) {
@@ -1969,10 +1966,6 @@
}
_dispatch_debug("kevent-source[%p]: %s timer[%p]", ds, verb, dr);
_dispatch_object_debug(ds, "%s", __func__);
- if (refs > 0) {
- dispatch_assert(refs == 2);
- _dispatch_release_2_tailcall(ds);
- }
}
#define DISPATCH_TIMER_MISSED_MARKER 1ul
@@ -2065,19 +2058,21 @@
continue;
}
+ _dispatch_retain(ds);
data = os_atomic_load2o(ds, ds_pending_data, relaxed);
if (unlikely(data)) {
// the release barrier is required to make the changes
// to `ds_timer` visible to _dispatch_source_timer_data()
if (os_atomic_cmpxchg2o(ds, ds_pending_data, data,
data | DISPATCH_TIMER_MISSED_MARKER, release)) {
- _dispatch_timers_update(dr, DISPATCH_TIMERS_UNREGISTER);
+ _dispatch_timers_update(dr);
+ _dispatch_release(ds);
continue;
}
}
data = _dispatch_source_timer_compute_missed(dr, now, 0);
- _dispatch_timers_update(dr, DISPATCH_TIMERS_RETAIN_2);
+ _dispatch_timers_update(dr);
pending_data = data << 1;
if (!_dispatch_unote_registered(dr) && dr->dt_timer.target < INT64_MAX){
// if we unregistered because of suspension we have to fake we
@@ -2090,7 +2085,7 @@
_dispatch_trace_timer_fire(dr, data, data);
_dispatch_debug("kevent-source[%p]: fired timer[%p]", ds, dr);
_dispatch_object_debug(ds, "%s", __func__);
- dx_wakeup(ds, 0, DISPATCH_WAKEUP_MAKE_DIRTY | DISPATCH_WAKEUP_CONSUME_2);
+ dx_wakeup(ds, 0, DISPATCH_WAKEUP_FLUSH | DISPATCH_WAKEUP_CONSUME);
}
}
@@ -2259,46 +2254,55 @@
#pragma mark dispatch_mgr
void
-_dispatch_mgr_queue_push(dispatch_queue_t dq, dispatch_object_t dou,
- DISPATCH_UNUSED dispatch_qos_t qos)
+_dispatch_mgr_queue_wakeup(dispatch_queue_t dq,
+ dispatch_qos_t qos, dispatch_wakeup_flags_t flags)
{
- uint64_t dq_state;
- _dispatch_trace_continuation_push(dq, dou._do);
- if (unlikely(_dispatch_queue_push_update_tail(dq, dou._do))) {
- _dispatch_queue_push_update_head(dq, dou._do);
- dq_state = os_atomic_or2o(dq, dq_state, DISPATCH_QUEUE_DIRTY, release);
- if (!_dq_state_drain_locked_by_self(dq_state)) {
- _dispatch_event_loop_poke(DISPATCH_WLH_MANAGER, 0, 0);
- }
+ if (flags & DISPATCH_WAKEUP_FLUSH) {
+ os_atomic_or2o(dq, dq_state, DISPATCH_QUEUE_DIRTY, release);
}
-}
-DISPATCH_NORETURN
-void
-_dispatch_mgr_queue_wakeup(DISPATCH_UNUSED dispatch_queue_t dq,
- DISPATCH_UNUSED dispatch_qos_t qos,
- DISPATCH_UNUSED dispatch_wakeup_flags_t flags)
-{
- DISPATCH_INTERNAL_CRASH(0, "Don't try to wake up or override the manager");
+ if (_dispatch_queue_get_current() == &_dispatch_mgr_q) {
+ return;
+ }
+
+ if (!_dispatch_queue_class_probe(&_dispatch_mgr_q)) {
+ return;
+ }
+
+ _dispatch_event_loop_poke(DISPATCH_WLH_MANAGER, qos, 0);
}
#if DISPATCH_USE_MGR_THREAD
+DISPATCH_NOINLINE
+static void
+_dispatch_mgr_init(void)
+{
+ uint64_t owned = DISPATCH_QUEUE_SERIAL_DRAIN_OWNED;
+ _dispatch_queue_set_current(&_dispatch_mgr_q);
+ if (_dispatch_queue_drain_try_lock(&_dispatch_mgr_q,
+ DISPATCH_INVOKE_STEALING, NULL) != owned) {
+ DISPATCH_INTERNAL_CRASH(0, "Locking the manager should not fail");
+ }
+ _dispatch_mgr_priority_init();
+ _dispatch_event_loop_init();
+}
+
DISPATCH_NOINLINE DISPATCH_NORETURN
static void
_dispatch_mgr_invoke(void)
{
-#if DISPATCH_EVENT_BACKEND_KEVENT
- dispatch_kevent_s evbuf[DISPATCH_DEFERRED_ITEMS_EVENT_COUNT];
-#endif
- dispatch_deferred_items_s ddi = {
-#if DISPATCH_EVENT_BACKEND_KEVENT
- .ddi_maxevents = DISPATCH_DEFERRED_ITEMS_EVENT_COUNT,
- .ddi_eventlist = evbuf,
-#endif
- };
+ dispatch_deferred_items_s ddi;
bool poll;
+ ddi.ddi_stashed_pri = DISPATCH_PRIORITY_NOSTASH;
+ ddi.ddi_stashed_dq = NULL;
+ ddi.ddi_stashed_rq = NULL;
+#if DISPATCH_EVENT_BACKEND_KEVENT
+ ddi.ddi_nevents = 0;
+#endif
+ dispatch_assert(_dispatch_get_wlh() == DISPATCH_WLH_GLOBAL);
_dispatch_deferred_items_set(&ddi);
+
for (;;) {
_dispatch_mgr_queue_drain();
poll = _dispatch_mgr_timers();
@@ -2321,9 +2325,7 @@
}
#endif
#if DISPATCH_USE_MGR_THREAD
- _dispatch_queue_set_current(&_dispatch_mgr_q);
- _dispatch_mgr_priority_init();
- _dispatch_queue_mgr_lock(&_dispatch_mgr_q);
+ _dispatch_mgr_init();
// never returns, so burn bridges behind us & clear stack 2k ahead
_dispatch_clear_stack(2048);
_dispatch_mgr_invoke();
@@ -2344,8 +2346,14 @@
dispatch_deferred_items_t ddi)
{
dispatch_assert(wlh);
+ uint64_t owned = DISPATCH_QUEUE_SERIAL_DRAIN_OWNED;
dispatch_priority_t old_dbp;
+ ddi->ddi_stashed_pri = DISPATCH_PRIORITY_NOSTASH;
+ ddi->ddi_stashed_dq = NULL;
+ ddi->ddi_stashed_rq = NULL;
+ ddi->ddi_nevents = 0;
+
pthread_priority_t pp = _dispatch_get_priority();
if (!(pp & _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG)) {
// If this thread does not have the event manager flag set, don't setup
@@ -2355,7 +2363,7 @@
// Also add the NEEDS_UNBIND flag so that
// _dispatch_priority_compute_update knows it has to unbind
pp &= _PTHREAD_PRIORITY_OVERCOMMIT_FLAG | ~_PTHREAD_PRIORITY_FLAGS_MASK;
- if (wlh == DISPATCH_WLH_ANON) {
+ if (wlh == DISPATCH_WLH_GLOBAL) {
pp |= _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG;
} else {
// pthread sets the flag when it is an event delivery thread
@@ -2364,10 +2372,9 @@
}
_dispatch_thread_setspecific(dispatch_priority_key,
(void *)(uintptr_t)pp);
- if (wlh != DISPATCH_WLH_ANON) {
+ ddi->ddi_stashed_pri = 0;
+ if (wlh != DISPATCH_WLH_GLOBAL) {
_dispatch_debug("wlh[%p]: handling events", wlh);
- } else {
- ddi->ddi_can_stash = true;
}
return DISPATCH_KEVENT_WORKER_IS_NOT_MANAGER;
}
@@ -2397,7 +2404,15 @@
// ensure kevents registered from this thread are registered at manager QoS
old_dbp = _dispatch_set_basepri(DISPATCH_PRIORITY_FLAG_MANAGER);
_dispatch_queue_set_current(&_dispatch_mgr_q);
- _dispatch_queue_mgr_lock(&_dispatch_mgr_q);
+ if (_dispatch_queue_drain_try_lock(&_dispatch_mgr_q,
+ DISPATCH_INVOKE_STEALING, NULL) != owned) {
+ DISPATCH_INTERNAL_CRASH(0, "Locking the manager should not fail");
+ }
+ static int event_thread_init;
+ if (!event_thread_init) {
+ event_thread_init = 1;
+ _dispatch_event_loop_init();
+ }
return old_dbp;
}
@@ -2405,35 +2420,38 @@
static inline bool
_dispatch_wlh_worker_thread_reset(dispatch_priority_t old_dbp)
{
- bool needs_poll = _dispatch_queue_mgr_unlock(&_dispatch_mgr_q);
+ dispatch_queue_t dq = &_dispatch_mgr_q;
+ uint64_t orig_dq_state = DISPATCH_QUEUE_SERIAL_DRAIN_OWNED;
+
+ orig_dq_state = _dispatch_queue_drain_unlock(dq, orig_dq_state);
_dispatch_reset_basepri(old_dbp);
- _dispatch_reset_basepri_override();
_dispatch_queue_set_current(NULL);
- return needs_poll;
+ return _dq_state_is_dirty(orig_dq_state);
}
DISPATCH_ALWAYS_INLINE
static void
-_dispatch_wlh_worker_thread(dispatch_wlh_t wlh, dispatch_kevent_t events,
+_dispatch_wlh_worker_thread(dispatch_wlh_t wlh, dispatch_kevent_t *events,
int *nevents)
{
_dispatch_introspection_thread_add();
- DISPATCH_PERF_MON_VAR_INIT
- dispatch_deferred_items_s ddi = {
- .ddi_eventlist = events,
- };
- dispatch_priority_t old_dbp;
+ dispatch_kevent_t ke = *events;
+ DISPATCH_PERF_MON_VAR
+ int n = *nevents;
+ if (!dispatch_assume(n) || !dispatch_assume(*events)) return;
- old_dbp = _dispatch_wlh_worker_thread_init(wlh, &ddi);
+ dispatch_deferred_items_s ddi;
+ dispatch_priority_t old_dbp = _dispatch_wlh_worker_thread_init(wlh, &ddi);
if (old_dbp == DISPATCH_KEVENT_WORKER_IS_NOT_MANAGER) {
_dispatch_perfmon_start_impl(true);
} else {
- dispatch_assert(wlh == DISPATCH_WLH_ANON);
- wlh = DISPATCH_WLH_ANON;
+ dispatch_assert(wlh == DISPATCH_WLH_GLOBAL);
+ wlh = DISPATCH_WLH_GLOBAL;
}
+ _dispatch_set_wlh(wlh);
_dispatch_deferred_items_set(&ddi);
- _dispatch_event_loop_merge(events, *nevents);
+ _dispatch_event_loop_merge(ke, n);
if (old_dbp != DISPATCH_KEVENT_WORKER_IS_NOT_MANAGER) {
_dispatch_mgr_queue_drain();
@@ -2442,27 +2460,34 @@
poll = true;
}
if (poll) _dispatch_event_loop_poke(DISPATCH_WLH_MANAGER, 0, 0);
- } else if (ddi.ddi_stashed_dou._do) {
- _dispatch_debug("wlh[%p]: draining deferred item %p", wlh,
- ddi.ddi_stashed_dou._do);
- if (wlh == DISPATCH_WLH_ANON) {
- dispatch_assert(ddi.ddi_nevents == 0);
+ } else if (ddi.ddi_stashed_dq) {
+ if (wlh == DISPATCH_WLH_GLOBAL) {
+ if (ddi.ddi_nevents) _dispatch_event_loop_update();
_dispatch_deferred_items_set(NULL);
- _dispatch_root_queue_drain_deferred_item(&ddi
- DISPATCH_PERF_MON_ARGS);
} else {
- _dispatch_root_queue_drain_deferred_wlh(&ddi
- DISPATCH_PERF_MON_ARGS);
+ ddi.ddi_stashed_pri = DISPATCH_PRIORITY_NOSTASH;
}
+
+ _dispatch_debug("wlh[%p]: draining deferred item %p", wlh,
+ ddi.ddi_stashed_dq);
+ _dispatch_root_queue_drain_deferred_item(ddi.ddi_stashed_rq,
+ ddi.ddi_stashed_dq DISPATCH_PERF_MON_ARGS);
}
_dispatch_deferred_items_set(NULL);
- if (old_dbp == DISPATCH_KEVENT_WORKER_IS_NOT_MANAGER &&
- !ddi.ddi_stashed_dou._do) {
+ _dispatch_reset_wlh();
+
+ if (ddi.ddi_nevents) {
+ _dispatch_debug("flushing %d deferred kevents", ddi.ddi_nevents);
+ }
+ *nevents = ddi.ddi_nevents;
+ dispatch_static_assert(__builtin_types_compatible_p(typeof(**events),
+ typeof(*ddi.ddi_eventlist)));
+ memcpy(*events, ddi.ddi_eventlist,
+ (size_t)ddi.ddi_nevents * sizeof(*ddi.ddi_eventlist));
+ if (old_dbp == DISPATCH_KEVENT_WORKER_IS_NOT_MANAGER && !ddi.ddi_stashed_dq) {
_dispatch_perfmon_end(perfmon_thread_event_no_steal);
}
- _dispatch_debug("returning %d deferred kevents", ddi.ddi_nevents);
- *nevents = ddi.ddi_nevents;
}
DISPATCH_NOINLINE
@@ -2473,10 +2498,7 @@
// events for worker thread request have already been delivered earlier
return;
}
- if (!dispatch_assume(*nevents && *events)) return;
- _dispatch_adopt_wlh_anon();
- _dispatch_wlh_worker_thread(DISPATCH_WLH_ANON, *events, nevents);
- _dispatch_reset_wlh();
+ return _dispatch_wlh_worker_thread(DISPATCH_WLH_GLOBAL, events, nevents);
}
diff --git a/src/source_internal.h b/src/source_internal.h
index 55b81e7..2082274 100644
--- a/src/source_internal.h
+++ b/src/source_internal.h
@@ -98,13 +98,13 @@
#endif // __cplusplus
-void _dispatch_source_refs_register(dispatch_source_t ds,
- dispatch_wlh_t wlh, dispatch_priority_t bp);
+dispatch_priority_t
+_dispatch_source_compute_kevent_priority(dispatch_source_t ds);
+void _dispatch_source_refs_register(dispatch_source_t ds, dispatch_priority_t bp);
void _dispatch_source_refs_unregister(dispatch_source_t ds, uint32_t options);
void _dispatch_source_xref_dispose(dispatch_source_t ds);
-void _dispatch_source_dispose(dispatch_source_t ds, bool *allow_free);
-void _dispatch_source_finalize_activation(dispatch_source_t ds,
- bool *allow_resume);
+void _dispatch_source_dispose(dispatch_source_t ds);
+void _dispatch_source_finalize_activation(dispatch_source_t ds);
void _dispatch_source_invoke(dispatch_source_t ds,
dispatch_invoke_context_t dic, dispatch_invoke_flags_t flags);
void _dispatch_source_wakeup(dispatch_source_t ds, dispatch_qos_t qos,
@@ -117,8 +117,6 @@
void _dispatch_source_merge_data(dispatch_source_t ds, pthread_priority_t pp,
unsigned long val);
-void _dispatch_mgr_queue_push(dispatch_queue_t dq, dispatch_object_t dou,
- dispatch_qos_t qos);
void _dispatch_mgr_queue_wakeup(dispatch_queue_t dq, dispatch_qos_t qos,
dispatch_wakeup_flags_t flags);
void _dispatch_mgr_thread(dispatch_queue_t dq, dispatch_invoke_context_t dic,
diff --git a/src/swift/DispatchStubs.cc b/src/swift/DispatchStubs.cc
index de309c7..2c76b7b 100644
--- a/src/swift/DispatchStubs.cc
+++ b/src/swift/DispatchStubs.cc
@@ -173,6 +173,12 @@
dispatch_retain(obj);
}
+// DISPATCH_RUNTIME_STDLIB_INTERFACE
+// extern "C" dispatch_queue_t
+// _swift_apply_current_root_queue() {
+// return DISPATCH_APPLY_CURRENT_ROOT_QUEUE;
+// }
+
#define SOURCE(t) \
SWIFT_CC(swift) \
DISPATCH_RUNTIME_STDLIB_INTERFACE extern "C" dispatch_source_type_t \
diff --git a/src/swift/Queue.swift b/src/swift/Queue.swift
index 9075e97..b7628c9 100644
--- a/src/swift/Queue.swift
+++ b/src/swift/Queue.swift
@@ -344,5 +344,8 @@
@_silgen_name("_swift_dispatch_get_main_queue")
internal func _swift_dispatch_get_main_queue() -> dispatch_queue_t
+@_silgen_name("_swift_dispatch_apply_current_root_queue")
+internal func _swift_dispatch_apply_current_root_queue() -> dispatch_queue_t
+
@_silgen_name("_swift_dispatch_apply_current")
internal func _swift_dispatch_apply_current(_ iterations: Int, _ block: @convention(block) (Int) -> Void)
diff --git a/src/time.c b/src/time.c
index 5b0bab0..6db4880 100644
--- a/src/time.c
+++ b/src/time.c
@@ -20,74 +20,30 @@
#include "internal.h"
-#if DISPATCH_USE_HOST_TIME
-typedef struct _dispatch_host_time_data_s {
- long double frac;
- bool ratio_1_to_1;
-} _dispatch_host_time_data_s;
-
-DISPATCH_CACHELINE_ALIGN
-static _dispatch_host_time_data_s _dispatch_host_time_data;
-
-uint64_t (*_dispatch_host_time_mach2nano)(uint64_t machtime);
-uint64_t (*_dispatch_host_time_nano2mach)(uint64_t nsec);
-
-static uint64_t
-_dispatch_mach_host_time_mach2nano(uint64_t machtime)
-{
- _dispatch_host_time_data_s *const data = &_dispatch_host_time_data;
-
- if (unlikely(!machtime || data->ratio_1_to_1)) {
- return machtime;
- }
- if (machtime >= INT64_MAX) {
- return INT64_MAX;
- }
- long double big_tmp = ((long double)machtime * data->frac) + .5L;
- if (unlikely(big_tmp >= INT64_MAX)) {
- return INT64_MAX;
- }
- return (uint64_t)big_tmp;
-}
-
-static uint64_t
-_dispatch_mach_host_time_nano2mach(uint64_t nsec)
-{
- _dispatch_host_time_data_s *const data = &_dispatch_host_time_data;
-
- if (unlikely(!nsec || data->ratio_1_to_1)) {
- return nsec;
- }
- if (nsec >= INT64_MAX) {
- return INT64_MAX;
- }
- long double big_tmp = ((long double)nsec / data->frac) + .5L;
- if (unlikely(big_tmp >= INT64_MAX)) {
- return INT64_MAX;
- }
- return (uint64_t)big_tmp;
-}
-
-static void
-_dispatch_host_time_init(mach_timebase_info_data_t *tbi)
-{
- _dispatch_host_time_data.frac = tbi->numer;
- _dispatch_host_time_data.frac /= tbi->denom;
- _dispatch_host_time_data.ratio_1_to_1 = (tbi->numer == tbi->denom);
- _dispatch_host_time_mach2nano = _dispatch_mach_host_time_mach2nano;
- _dispatch_host_time_nano2mach = _dispatch_mach_host_time_nano2mach;
-}
-#endif // DISPATCH_USE_HOST_TIME
+#if !(defined(__i386__) || defined(__x86_64__) || !HAVE_MACH_ABSOLUTE_TIME) \
+ || TARGET_OS_WIN32
+DISPATCH_CACHELINE_ALIGN _dispatch_host_time_data_s _dispatch_host_time_data = {
+ .ratio_1_to_1 = true,
+};
void
-_dispatch_time_init(void)
+_dispatch_get_host_time_init(void *context DISPATCH_UNUSED)
{
-#if DISPATCH_USE_HOST_TIME
+#if !TARGET_OS_WIN32
mach_timebase_info_data_t tbi;
(void)dispatch_assume_zero(mach_timebase_info(&tbi));
- _dispatch_host_time_init(&tbi);
-#endif // DISPATCH_USE_HOST_TIME
+ _dispatch_host_time_data.frac = tbi.numer;
+ _dispatch_host_time_data.frac /= tbi.denom;
+ _dispatch_host_time_data.ratio_1_to_1 = (tbi.numer == tbi.denom);
+#else
+ LARGE_INTEGER freq;
+ dispatch_assume(QueryPerformanceFrequency(&freq));
+ _dispatch_host_time_data.frac = (long double)NSEC_PER_SEC /
+ (long double)freq.QuadPart;
+ _dispatch_host_time_data.ratio_1_to_1 = (freq.QuadPart == 1);
+#endif /* TARGET_OS_WIN32 */
}
+#endif
dispatch_time_t
dispatch_time(dispatch_time_t inval, int64_t delta)
diff --git a/src/voucher.c b/src/voucher.c
index 5beadf0..9f97b7a 100644
--- a/src/voucher.c
+++ b/src/voucher.c
@@ -359,11 +359,18 @@
#define _voucher_mach_recipe_size(payload_size) \
(sizeof(mach_voucher_attr_recipe_data_t) + (payload_size))
+#if VOUCHER_USE_MACH_VOUCHER_PRIORITY
#define _voucher_mach_recipe_alloca(v) ((mach_voucher_attr_recipe_t)alloca(\
_voucher_mach_recipe_size(0) + \
_voucher_mach_recipe_size(sizeof(ipc_pthread_priority_value_t)) + \
_voucher_mach_recipe_size(sizeof(_voucher_mach_udata_s)) + \
_voucher_extra_size(v)))
+#else
+#define _voucher_mach_recipe_alloca(v) ((mach_voucher_attr_recipe_t)alloca(\
+ _voucher_mach_recipe_size(0) + \
+ _voucher_mach_recipe_size(sizeof(_voucher_mach_udata_s)) + \
+ _voucher_extra_size(v)))
+#endif
DISPATCH_ALWAYS_INLINE
static inline mach_voucher_attr_recipe_size_t
@@ -384,6 +391,7 @@
};
size += _voucher_mach_recipe_size(0);
+#if VOUCHER_USE_MACH_VOUCHER_PRIORITY
if (pp) {
ipc_pthread_priority_value_t value = (ipc_pthread_priority_value_t)pp;
*mvar_buf++ = (mach_voucher_attr_recipe_data_t){
@@ -394,6 +402,7 @@
mvar_buf = _dispatch_memappend(mvar_buf, &value);
size += _voucher_mach_recipe_size(sizeof(value));
}
+#endif // VOUCHER_USE_MACH_VOUCHER_PRIORITY
if ((v && v->v_activity) || pp) {
_voucher_mach_udata_s *udata_buf;
@@ -508,6 +517,29 @@
mach_voucher_attr_recipe_size_t kvr_size = 0;
mach_voucher_attr_content_size_t udata_sz = 0;
_voucher_mach_udata_s *udata = NULL;
+#if !VOUCHER_USE_BANK_AUTOREDEEM
+ mach_voucher_t rkv;
+ const mach_voucher_attr_recipe_data_t redeem_recipe[] = {
+ [0] = {
+ .key = MACH_VOUCHER_ATTR_KEY_ALL,
+ .command = MACH_VOUCHER_ATTR_COPY,
+ .previous_voucher = kv,
+ },
+ [1] = {
+ .key = MACH_VOUCHER_ATTR_KEY_BANK,
+ .command = MACH_VOUCHER_ATTR_REDEEM,
+ },
+ };
+ kr = _voucher_create_mach_voucher(redeem_recipe, sizeof(redeem_recipe),
+ &rkv);
+ if (!dispatch_assume_zero(kr)) {
+ _voucher_dealloc_mach_voucher(kv);
+ _dispatch_kvoucher_debug("redeemed from 0x%08x", rkv, kv);
+ kv = rkv;
+ } else {
+ _dispatch_voucher_debug_machport(kv);
+ }
+#endif
voucher_t v = _voucher_find_and_retain(kv);
if (v) {
_dispatch_voucher_debug("kvoucher[0x%08x] found", v, kv);
@@ -562,12 +594,15 @@
.key = MACH_VOUCHER_ATTR_KEY_USER_DATA,
.command = MACH_VOUCHER_ATTR_REMOVE,
},
+#if VOUCHER_USE_MACH_VOUCHER_PRIORITY
[2] = {
.key = MACH_VOUCHER_ATTR_KEY_PTHPRIORITY,
.command = MACH_VOUCHER_ATTR_REMOVE,
},
+#endif
};
mach_voucher_attr_recipe_size_t size = sizeof(remove_userdata_recipe);
+
kr = _voucher_create_mach_voucher(remove_userdata_recipe, size, &nkv);
if (!dispatch_assume_zero(kr)) {
_dispatch_voucher_debug("kvoucher[0x%08x] udata removal "
@@ -768,7 +803,7 @@
{
_dispatch_voucher_debug("xref_dispose", voucher);
_voucher_remove(voucher);
- return _os_object_release_internal_n_inline((_os_object_t)voucher, 1);
+ return _os_object_release_internal_inline((_os_object_t)voucher);
}
void
@@ -834,7 +869,6 @@
if (dbgp) {
dm = dispatch_mach_create_f("com.apple.debug-channel",
DISPATCH_TARGET_QUEUE_DEFAULT, NULL, handler);
- dm->dm_recv_refs->du_can_be_wlh = false; // 29906118
dispatch_mach_connect(dm, dbgp, MACH_PORT_NULL, NULL);
// will force the DISPATCH_MACH_CONNECTED event
dispatch_mach_send_barrier_f(dm, NULL,
@@ -1091,13 +1125,7 @@
info_size = proc_pidinfo(getpid(), PROC_PIDUNIQIDENTIFIERINFO, 1,
&p_uniqinfo, PROC_PIDUNIQIDENTIFIERINFO_SIZE);
if (slowpath(info_size != PROC_PIDUNIQIDENTIFIERINFO_SIZE)) {
- if (info_size == 0) {
- DISPATCH_INTERNAL_CRASH(errno,
- "Unable to get the unique pid (error)");
- } else {
- DISPATCH_INTERNAL_CRASH(info_size,
- "Unable to get the unique pid (size)");
- }
+ DISPATCH_INTERNAL_CRASH(info_size, "Unable to get the unique pid");
}
_voucher_unique_pid = p_uniqinfo.p_uniqueid;
@@ -1429,7 +1457,7 @@
size_t offset = 0;
#define bufprintf(...) \
offset += dsnprintf(&buf[offset], bufsiz - offset, ##__VA_ARGS__)
- bufprintf("voucher[%p] = { xref = %d, ref = %d", v,
+ bufprintf("voucher[%p] = { xrefcnt = 0x%x, refcnt = 0x%x", v,
v->os_obj_xref_cnt + 1, v->os_obj_ref_cnt + 1);
if (v->v_kvbase) {
diff --git a/src/voucher_internal.h b/src/voucher_internal.h
index a0ddd4d..d16fc8a 100644
--- a/src/voucher_internal.h
+++ b/src/voucher_internal.h
@@ -123,7 +123,9 @@
#define DISPATCH_VOUCHER_ACTIVITY_DEBUG 1
#endif
+#if VOUCHER_USE_MACH_VOUCHER_PRIORITY
#include <voucher/ipc_pthread_priority_types.h>
+#endif
typedef uint32_t _voucher_magic_t;
typedef uint32_t _voucher_priority_t;
@@ -262,14 +264,6 @@
#define _dispatch_voucher_debug_machport(name) ((void)(name))
#endif
-#ifndef DISPATCH_VOUCHER_OBJC_DEBUG
-#if DISPATCH_INTROSPECTION || DISPATCH_DEBUG
-#define DISPATCH_VOUCHER_OBJC_DEBUG 1
-#else
-#define DISPATCH_VOUCHER_OBJC_DEBUG 0
-#endif
-#endif // DISPATCH_VOUCHER_OBJC_DEBUG
-
#if DISPATCH_PURE_C
DISPATCH_ALWAYS_INLINE
diff --git a/xcodeconfig/libdispatch-dyld-stub.xcconfig b/xcodeconfig/libdispatch-dyld-stub.xcconfig
index dd1814d..aabda62 100644
--- a/xcodeconfig/libdispatch-dyld-stub.xcconfig
+++ b/xcodeconfig/libdispatch-dyld-stub.xcconfig
@@ -18,11 +18,11 @@
// @APPLE_APACHE_LICENSE_HEADER_END@
//
+OTHER_LDFLAGS =
+BUILD_VARIANTS = normal
+GCC_PREPROCESSOR_DEFINITIONS = $(inherited) DISPATCH_VARIANT_STATIC=1 DISPATCH_VARIANT_DYLD_STUB=1 USE_OBJC=0 DISPATCH_USE_DTRACE=0
PRODUCT_NAME = libdispatch_dyld_stub
INSTALL_PATH = /usr/local/lib/dyld_stub
-BUILD_VARIANTS = normal
-GCC_PREPROCESSOR_DEFINITIONS = $(inherited) DISPATCH_VARIANT_DYLD_STUB=1 $(STATICLIB_PREPROCESSOR_DEFINITIONS)
-OTHER_LDFLAGS =
-VERSIONING_SYSTEM =
EXCLUDED_SOURCE_FILE_NAMES = *
-INCLUDED_SOURCE_FILE_NAMES = voucher.c // minimal with DISPATCH_VARIANT_DYLD_STUB
+INCLUDED_SOURCE_FILE_NAMES = voucher.c // it's minimal with DISPATCH_VARIANT_DYLD_STUB
+VERSIONING_SYSTEM =
diff --git a/xcodeconfig/libdispatch-mp-static.xcconfig b/xcodeconfig/libdispatch-mp-static.xcconfig
index af3715f..1f0eddc 100644
--- a/xcodeconfig/libdispatch-mp-static.xcconfig
+++ b/xcodeconfig/libdispatch-mp-static.xcconfig
@@ -18,12 +18,13 @@
// @APPLE_APACHE_LICENSE_HEADER_END@
//
-// skip simulator
-SUPPORTED_PLATFORMS = macosx iphoneos appletvos watchos
+OTHER_LDFLAGS =
+BUILD_VARIANTS = normal debug
+GCC_PREPROCESSOR_DEFINITIONS = $(inherited) DISPATCH_VARIANT_STATIC=1 USE_OBJC=0 DISPATCH_USE_DTRACE=0
PRODUCT_NAME = libdispatch
INSTALL_PATH = /usr/local/lib/system
-BUILD_VARIANTS = normal debug
-GCC_PREPROCESSOR_DEFINITIONS = $(inherited) $(STATICLIB_PREPROCESSOR_DEFINITIONS)
-OTHER_LDFLAGS =
+
+// skip simulator
+SUPPORTED_PLATFORMS = macosx iphoneos appletvos watchos
SKIP_INSTALL[sdk=*simulator*] = YES
EXCLUDED_SOURCE_FILE_NAMES[sdk=*simulator*] = *
diff --git a/xcodeconfig/libdispatch-resolved.xcconfig b/xcodeconfig/libdispatch-resolved.xcconfig
index 2f2e273..a42add8 100644
--- a/xcodeconfig/libdispatch-resolved.xcconfig
+++ b/xcodeconfig/libdispatch-resolved.xcconfig
@@ -23,4 +23,3 @@
OTHER_LDFLAGS =
SKIP_INSTALL = YES
VERSIONING_SYSTEM =
-EXCLUDED_SOURCE_FILE_NAMES = *
diff --git a/xcodeconfig/libdispatch-up-static.xcconfig b/xcodeconfig/libdispatch-up-static.xcconfig
index 170c5b3..0ece635 100644
--- a/xcodeconfig/libdispatch-up-static.xcconfig
+++ b/xcodeconfig/libdispatch-up-static.xcconfig
@@ -18,11 +18,8 @@
// @APPLE_APACHE_LICENSE_HEADER_END@
//
-// skip simulator
-SUPPORTED_PLATFORMS = macosx iphoneos appletvos watchos
-PRODUCT_NAME = libdispatch_up
-BUILD_VARIANTS = normal
-GCC_PREPROCESSOR_DEFINITIONS = $(inherited) DISPATCH_HW_CONFIG_UP=1 $(STATICLIB_PREPROCESSOR_DEFINITIONS)
OTHER_LDFLAGS =
+BUILD_VARIANTS = normal
SKIP_INSTALL = YES
-EXCLUDED_SOURCE_FILE_NAMES[sdk=*simulator*] = *
+EXCLUDED_SOURCE_FILE_NAMES = *
+GCC_PREPROCESSOR_DEFINITIONS = $(inherited) USE_OBJC=0 DISPATCH_USE_DTRACE=0
diff --git a/xcodeconfig/libdispatch.order b/xcodeconfig/libdispatch.order
index a25ecc9..9642ca4 100644
--- a/xcodeconfig/libdispatch.order
+++ b/xcodeconfig/libdispatch.order
@@ -71,6 +71,18 @@
_OBJC_METACLASS_$_OS_dispatch_queue_serial
_OBJC_METACLASS_$_OS_dispatch_queue_concurrent
_OBJC_METACLASS_$_OS_dispatch_queue_root
+_OBJC_METACLASS_$_OS_dispatch_queue_main
+_OBJC_METACLASS_$_OS_dispatch_queue_runloop
+_OBJC_METACLASS_$_OS_dispatch_queue_mgr
+_OBJC_METACLASS_$_OS_dispatch_queue_specific_queue
+_OBJC_METACLASS_$_OS_dispatch_queue_attr
+_OBJC_METACLASS_$_OS_dispatch_source
+_OBJC_METACLASS_$_OS_dispatch_mach
+_OBJC_METACLASS_$_OS_dispatch_mach_msg
+_OBJC_METACLASS_$_OS_dispatch_io
+_OBJC_METACLASS_$_OS_dispatch_operation
+_OBJC_METACLASS_$_OS_dispatch_disk
+_OBJC_METACLASS_$_OS_object
_OBJC_METACLASS_$_OS_voucher
#_OBJC_METACLASS_$_OS_voucher_recipe
_OBJC_METACLASS_$_OS_dispatch_data
diff --git a/xcodeconfig/libdispatch.xcconfig b/xcodeconfig/libdispatch.xcconfig
index 643e1d3..a2ea6d9 100644
--- a/xcodeconfig/libdispatch.xcconfig
+++ b/xcodeconfig/libdispatch.xcconfig
@@ -71,19 +71,16 @@
CLANG_WARN_OBJC_IMPLICIT_ATOMIC_PROPERTIES = YES
CLANG_WARN_OBJC_MISSING_PROPERTY_SYNTHESIS = YES
CLANG_WARN_SUSPICIOUS_IMPLICIT_CONVERSION = YES
-CLANG_WARN_SUSPICIOUS_MOVE = YES
-CLANG_WARN_UNREACHABLE_CODE = YES
GCC_TREAT_WARNINGS_AS_ERRORS = YES
GCC_OPTIMIZATION_LEVEL = s
-GCC_NO_COMMON_BLOCKS = YES
GCC_PREPROCESSOR_DEFINITIONS = __DARWIN_NON_CANCELABLE=1 $(DISPATCH_PREPROCESSOR_DEFINITIONS)
-STATICLIB_PREPROCESSOR_DEFINITIONS = DISPATCH_VARIANT_STATIC=1 USE_OBJC=0 DISPATCH_USE_DTRACE=0
+GCC_NO_COMMON_BLOCKS = YES
WARNING_CFLAGS = -Wall -Wextra -Warray-bounds-pointer-arithmetic -Watomic-properties -Wcomma -Wconditional-uninitialized -Wcovered-switch-default -Wdate-time -Wdeprecated -Wdouble-promotion -Wduplicate-enum -Wexpansion-to-defined -Wfloat-equal -Widiomatic-parentheses -Wignored-qualifiers -Wimplicit-fallthrough -Wnullable-to-nonnull-conversion -Wobjc-interface-ivars -Wover-aligned -Wpacked -Wpointer-arith -Wselector -Wstatic-in-inline -Wsuper-class-method-mismatch -Wswitch-enum -Wtautological-compare -Wunguarded-availability -Wunused -Wno-unknown-warning-option $(NO_WARNING_CFLAGS)
NO_WARNING_CFLAGS = -Wno-pedantic -Wno-bad-function-cast -Wno-c++-compat -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-cast-align -Wno-cast-qual -Wno-disabled-macro-expansion -Wno-documentation-unknown-command -Wno-format-nonliteral -Wno-missing-variable-declarations -Wno-old-style-cast -Wno-padded -Wno-reserved-id-macro -Wno-shift-sign-overflow -Wno-undef -Wno-unreachable-code-aggressive -Wno-unused-macros -Wno-used-but-marked-unused -Wno-vla
-OTHER_CFLAGS = -fverbose-asm -isystem $(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders $(PLATFORM_CFLAGS)
+OTHER_CFLAGS = -fverbose-asm -isystem $(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders
OTHER_CFLAGS[arch=i386][sdk=macosx*] = $(OTHER_CFLAGS) -fno-unwind-tables -fno-asynchronous-unwind-tables -fno-exceptions
OTHER_CFLAGS_normal = -momit-leaf-frame-pointer
-OTHER_CFLAGS_profile = $(OTHER_CFLAGS_normal) -DDISPATCH_PROFILE=1 -DDISPATCH_PERF_MON=1
+OTHER_CFLAGS_profile = $(OTHER_CFLAGS_normal) -DDISPATCH_PROFILE=1
OTHER_CFLAGS_debug = -fstack-protector -fno-inline -O0 -DDISPATCH_DEBUG=1 -DOS_DEBUG=1
GENERATE_PROFILING_CODE = NO
DYLIB_CURRENT_VERSION = $(CURRENT_PROJECT_VERSION)
diff --git a/xcodeconfig/libfirehose.xcconfig b/xcodeconfig/libfirehose.xcconfig
index 4c71199..07a8b9a 100644
--- a/xcodeconfig/libfirehose.xcconfig
+++ b/xcodeconfig/libfirehose.xcconfig
@@ -18,17 +18,18 @@
// @APPLE_APACHE_LICENSE_HEADER_END@
//
+OTHER_MIGFLAGS = -novouchers
+OTHER_LDFLAGS =
SUPPORTED_PLATFORMS = macosx iphoneos iphonesimulator appletvos appletvsimulator watchos watchsimulator
PRODUCT_NAME = $(TARGET_NAME)
INSTALL_PATH = /usr/local/lib/
-GCC_PREPROCESSOR_DEFINITIONS = $(inherited) FIREHOSE_SERVER=1 DISPATCH_USE_DTRACE=0
-OTHER_MIGFLAGS = -novouchers
-OTHER_LDFLAGS =
PUBLIC_HEADERS_FOLDER_PATH = /usr/include/os
PRIVATE_HEADERS_FOLDER_PATH = /usr/local/include/os
STRIP_INSTALLED_PRODUCT = NO
COPY_PHASE_STRIP = NO
SEPARATE_STRIP = NO
+GCC_PREPROCESSOR_DEFINITIONS = $(inherited) FIREHOSE_SERVER=1 DISPATCH_USE_DTRACE=0
+
VALID_ARCHS[sdk=macosx*] = $(NATIVE_ARCH_ACTUAL)
COPY_HEADERS_RUN_UNIFDEF = YES
diff --git a/xcodeconfig/libfirehose_kernel.xcconfig b/xcodeconfig/libfirehose_kernel.xcconfig
index c572f80..f6b2a99 100644
--- a/xcodeconfig/libfirehose_kernel.xcconfig
+++ b/xcodeconfig/libfirehose_kernel.xcconfig
@@ -20,14 +20,16 @@
#include "libfirehose.xcconfig"
-SUPPORTED_PLATFORMS = macosx iphoneos appletvos watchos
-PRODUCT_NAME = $(TARGET_NAME)
-INSTALL_PATH = /usr/local/lib/kernel/
-GCC_PREPROCESSOR_DEFINITIONS = $(inherited) KERNEL=1 DISPATCH_USE_DTRACE=0
OTHER_CFLAGS = -mkernel -nostdinc -Wno-packed
// LLVM_LTO = YES
+PRODUCT_NAME = $(TARGET_NAME)
+INSTALL_PATH = /usr/local/lib/kernel/
PRIVATE_HEADERS_FOLDER_PATH = /usr/local/include/kernel/os
+SUPPORTED_PLATFORMS = macosx iphoneos appletvos watchos
+
HEADER_SEARCH_PATHS = $(PROJECT_DIR) $(SDKROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders $(SDKROOT)/System/Library/Frameworks/Kernel.framework/Headers $(SDKROOT)/usr/local/include/os $(SDKROOT)/usr/local/include/firehose
+GCC_PREPROCESSOR_DEFINITIONS = $(inherited) KERNEL=1 DISPATCH_USE_DTRACE=0
+
COPY_HEADERS_RUN_UNIFDEF = YES
COPY_HEADERS_UNIFDEF_FLAGS = -DKERNEL=1 -DOS_FIREHOSE_SPI=1 -DOS_VOUCHER_ACTIVITY_SPI_TYPES=1 -UOS_VOUCHER_ACTIVITY_SPI
