blob: d63154ec1c0850feb19657675d2d9ae92e765631 [file] [log] [blame]
// Copyright 2017 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <atomic>
#include <stdio.h>
#include <string.h>
#include <utility>
#include <vector>
#include <zircon/assert.h>
#include <lib/async/cpp/task.h>
#include <lib/async/cpp/wait.h>
#include <lib/trace-engine/handler.h>
#include <lib/trace-engine/instrumentation.h>
#include <lib/zx/event.h>
#include "context_impl.h"
namespace {
// Amount of time to allow for other threads to release their references
// to the trace buffer during shutdown. See point of use for details.
constexpr zx::duration kSynchronousShutdownTimeout = zx::msec(1000);
// Trace engine lock.
// See rules below for how this is used.
std::mutex g_engine_mutex;
// Trace instrumentation state.
// Rules:
// - can only be modified while holding g_engine_mutex
// - can be read atomically at any time
std::atomic<int> g_state{TRACE_STOPPED};
// Trace disposition.
// This is the status that will be reported to the trace handler when the
// trace finishes.
// Rules:
// - can only be accessed or modified while holding g_engine_mutex
zx_status_t g_disposition __TA_GUARDED(g_engine_mutex){ZX_OK};
// Trace asynchronous dispatcher.
// Rules:
// - can only be modified while holding g_engine_mutex and engine is stopped
// - can be read outside the lock only while the engine is not stopped
async_dispatcher_t* g_dispatcher{nullptr};
// Trace handler.
// Rules:
// - can only be modified while holding g_engine_mutex and engine is stopped
// - can be read outside the lock only while the engine is not stopped
trace_handler_t* g_handler{nullptr};
// Set to true when a trace is terminated and writes are in flight.
// Rules:
// - can only be accessed or modified while holding g_engine_mutex
bool g_trace_terminated{false};
// Trace observer table.
// Rules:
// - can only be accessed or modified while holding g_engine_mutex
struct Observer {
// The event handle that we notify the observer through.
zx_handle_t event;
// Set to true when the engine starts to indicate we're waiting for this
// observer to call us back, via |trace_notify_observer_updated()|, that
// it has started. When it does call us back this is set back to false.
bool awaiting_update_after_start;
};
std::vector<Observer> g_observers __TA_GUARDED(g_engine_mutex);
// Trace context reference count.
// This functions as a non-exclusive lock for the engine's trace context.
// Rules:
// - acquiring a reference acts as an ACQUIRE fence
// - releasing a reference acts as a RELEASE fence
// - always 0 when engine stopped
// - transition from 0 to non-zero only happens when engine is started
// - the engine stops when the reference count goes to 0
// (in other words, holding a context reference prevents the engine from stopping)
//
// There are two separate counters here that collectively provide the full
// count: buffer acquisitions and prolonged acquisitions. Buffer acquisitions
// are for the purpose of writing to the trace buffer. Prolonged acquisitions
// are for things like adhoc trace providers where they want to maintain a
// reference to the context for the duration of the trace.
// Buffer acquisitions increment/decrement the count by
// |kBufferCounterIncrement|. Prolonged acquisitions increment/decrement the
// count by |kProlongedCounterIncrement|.
// To maintain the property that the full count only transitions from 0 to 1
// when the engine is started |kProlongedCounterIncrement| == 1.
std::atomic_uint32_t g_context_refs{0u};
// The uint32_t context ref count is split this way:
// |31 ... 8| = buffer acquisition count
// |7 ... 0| = prolonged acquisition count
// There are generally only a handful of prolonged acquisitions. The code will
// assert-fail if there are more. This allows for 2^24 buffer acquisitions
// which is basically 2^24 threads. The values are also chosen so that the
// full count is easily interpreted when printed in hex.
constexpr uint32_t kProlongedCounterShift = 0;
constexpr uint32_t kProlongedCounterIncrement = 1 << kProlongedCounterShift;
constexpr uint32_t kMaxProlongedCounter = 127;
constexpr uint32_t kProlongedCounterMask = 0xff;
constexpr uint32_t kBufferCounterShift = 8;
constexpr uint32_t kBufferCounterIncrement = 1 << kBufferCounterShift;
constexpr uint32_t kBufferCounterMask = 0xffffff00;
// Trace context.
// Rules:
// - can only be modified while holding g_engine_mutex and engine is stopped
// - can be accessed outside the lock while holding a context reference
trace_context_t* g_context{nullptr};
// Event for tracking:
// - when all observers has started
// (SIGNAL_ALL_OBSERVERS_STARTED)
// - when the trace context reference count has dropped to zero
// (SIGNAL_CONTEXT_RELEASED)
// Rules:
// - can only be modified while holding g_engine_mutex and engine is stopped
// - can be read outside the lock while the engine is not stopped
zx::event g_event;
constexpr zx_signals_t SIGNAL_ALL_OBSERVERS_STARTED = ZX_USER_SIGNAL_0;
constexpr zx_signals_t SIGNAL_CONTEXT_RELEASED = ZX_USER_SIGNAL_1;
// Asynchronous operations posted to the asynchronous dispatcher while the
// engine is running. Use of these structures is guarded by the engine lock.
async_wait_t g_event_wait;
inline uint32_t get_prolonged_context_refs(uint32_t raw) {
return (raw & kProlongedCounterMask) >> kProlongedCounterShift;
}
inline uint32_t get_buffer_context_refs(uint32_t raw) {
return (raw & kBufferCounterMask) >> kBufferCounterShift;
}
void handle_event(async_dispatcher_t* dispatcher, async_wait_t* wait, zx_status_t status,
const zx_packet_signal_t* signal);
// must hold g_engine_mutex
inline void update_disposition_locked(zx_status_t disposition) __TA_REQUIRES(g_engine_mutex) {
if (g_disposition == ZX_OK)
g_disposition = disposition;
}
void notify_observers_locked() __TA_REQUIRES(g_engine_mutex) {
for (auto& observer : g_observers) {
zx_status_t status = zx_object_signal(observer.event, 0u, ZX_EVENT_SIGNALED);
ZX_DEBUG_ASSERT(status == ZX_OK);
}
}
void notify_engine_all_observers_started_if_needed_locked() __TA_REQUIRES(g_engine_mutex) {
for (auto& item : g_observers) {
if (item.awaiting_update_after_start)
return;
}
g_event.signal(0u, SIGNAL_ALL_OBSERVERS_STARTED);
}
// Table of per-call-site cached category enabled/disabled flags.
// This is done by chaining all the
// |trace_acquire_context_for_category_cached()| call sites at runtime,
// and recording with the pointer the enabled/disabled flag.
//
// Operation:
// 1. When tracing starts each value is zero (kSiteStateUnknown).
// The value is generally a static local at the call site.
// Note that while tracing was off various call sites may have been cached,
// they are all reset to zero.
// 2. When a TRACE_*() macro is called, it calls
// trace_acquire_context_for_category_cached().
// 3. If the DISABLED bit is set, skip, we're done.
// 4. Call trace_acquire_context_for_category()
// 5. If the ENABLED bit is set, return, we're done.
// 6. Insert the call site to the head of the chain with the
// enabled/disabled bits set appropriately.
// 7. When tracing stops, empty the list. This includes resetting all chained
// values to "unknown". We know they're actually disabled, but the important
// part here is to flush the cache. A minor improvement would be to keep
// the current list.
// This is done both when the state transitions to STOPPING and again when
// the state transitions to STOPPED.
// 8. When tracing starts again, reset all chained values to "unknown" and
// flush the cache.
//
// The trick is doing this in as lock-free way as possible.
// Atomics are used for accessing the static local at the call site, and when
// the list needs to be traversed it is first atomically unchained from the
// main list and then operated on.
// Generally there aren't that many call sites, and we only need to traverse
// the list at trace start/stop time; so using a list isn't that much of a
// performance issue.
using trace_site_atomic_state_t = std::atomic<trace_site_state_t>;
// A sentinel is used so that there is no ambiguity between a null value
// being the end of the chain and a null value being the initial value of
// a chain slot.
trace_site_t g_site_cache_sentinel{};
std::atomic<trace_site_t*> g_site_cache{&g_site_cache_sentinel};
// Extra bits that are combined with the chain pointer to provide
// the full state.
constexpr trace_site_state_t kSiteStateUnknown = 0u;
constexpr trace_site_state_t kSiteStateDisabled = 1u;
constexpr trace_site_state_t kSiteStateEnabled = 2u;
constexpr trace_site_state_t kSiteStateFlagsMask = 3u;
// We don't export this value to the API, the API just says these values
// must be initialized to zero.
static_assert(kSiteStateUnknown == 0u);
// For clarity when reading the source.
using trace_site_flags_t = trace_site_state_t;
trace_site_state_t get_trace_site_raw_successor(trace_site_state_t state) {
return state & ~kSiteStateFlagsMask;
}
trace_site_t* get_trace_site_successor(trace_site_state_t state) {
return reinterpret_cast<trace_site_t*>(get_trace_site_raw_successor(state));
}
trace_site_flags_t get_trace_site_flags(trace_site_state_t state) {
return state & kSiteStateFlagsMask;
}
trace_site_atomic_state_t* get_trace_site_state_as_atomic(trace_site_t* site) {
return reinterpret_cast<trace_site_atomic_state_t*>(&site->state);
}
trace_site_state_t make_trace_site_state(trace_site_state_t successor, trace_site_flags_t flags) {
return successor | flags;
}
trace_site_state_t make_trace_site_state(trace_site_t* successor, trace_site_flags_t flags) {
return reinterpret_cast<trace_site_state_t>(successor) | flags;
}
trace_site_t* unchain_site_cache() {
trace_site_t* empty_cache = &g_site_cache_sentinel;
return g_site_cache.exchange(empty_cache, std::memory_order_relaxed);
}
void flush_site_cache() {
// Atomically swap in an empty cache with the current one.
trace_site_t* chain_head = unchain_site_cache();
trace_site_t* chain = chain_head;
while (chain != &g_site_cache_sentinel) {
trace_site_atomic_state_t* state_ptr = get_trace_site_state_as_atomic(chain);
trace_site_state_t curr_state = state_ptr->load(std::memory_order_relaxed);
trace_site_state_t new_state = kSiteStateUnknown;
state_ptr->store(new_state, std::memory_order_relaxed);
chain = get_trace_site_successor(curr_state);
}
}
// Update the state at |*site|.
// Note that multiple threads may race here for the same site.
void add_to_site_cache(trace_site_t* site, trace_site_state_t current_state, bool enabled) {
trace_site_atomic_state_t* state_ptr = get_trace_site_state_as_atomic(site);
// Even when tracing is on generally only a subset of categories
// are traced, so the test uses "unlikely".
trace_site_flags_t new_flags;
if (unlikely(enabled)) {
new_flags = kSiteStateEnabled;
} else {
new_flags = kSiteStateDisabled;
}
// At this point the recorded flags are zero. If we're the first to set
// them then we're good to add our entry to the cache (if not already in
// the cache). Otherwise punt. Note that this first setting of the flags
// won't be the last if we need to also chain this entry into the cache.
ZX_DEBUG_ASSERT(get_trace_site_flags(current_state) == kSiteStateUnknown);
trace_site_state_t new_state =
make_trace_site_state(get_trace_site_raw_successor(current_state), new_flags);
// If someone else changed our state punt. This can happen when another
// thread is tracing and gets there first.
if (unlikely(!state_ptr->compare_exchange_strong(
current_state, new_state, std::memory_order_acquire, std::memory_order_relaxed))) {
return;
}
if (get_trace_site_raw_successor(new_state)) {
// Already in chain.
return;
}
// Add to chain.
trace_site_t* old_cache_ptr = g_site_cache.load(std::memory_order_relaxed);
new_state = make_trace_site_state(old_cache_ptr, new_flags);
state_ptr->store(new_state, std::memory_order_relaxed);
// Atomically update both:
// - |g_site_cache| to point to |new_cache_ptr| (which is our entry)
// - |*state_ptr| (our entry) to point to the old |g_site_cache|
// This works because until our entry is live only its flag values
// matter to other threads. See the discussion in |trace_engine_stop()|.
trace_site_t* new_cache_ptr = site;
while (!g_site_cache.compare_exchange_weak(
old_cache_ptr, new_cache_ptr, std::memory_order_relaxed, std::memory_order_relaxed)) {
// Someone else updated |g_site_cache|. Reset our chain pointer
// and try again.
new_state = make_trace_site_state(old_cache_ptr, new_flags);
state_ptr->store(new_state, std::memory_order_relaxed);
}
}
} // namespace
/*** Trace engine functions ***/
// thread-safe
EXPORT_NO_DDK zx_status_t trace_engine_initialize(async_dispatcher_t* dispatcher,
trace_handler_t* handler,
trace_buffering_mode_t buffering_mode,
void* buffer, size_t buffer_num_bytes) {
ZX_DEBUG_ASSERT(dispatcher);
ZX_DEBUG_ASSERT(handler);
ZX_DEBUG_ASSERT(buffer);
switch (buffering_mode) {
case TRACE_BUFFERING_MODE_ONESHOT:
case TRACE_BUFFERING_MODE_CIRCULAR:
case TRACE_BUFFERING_MODE_STREAMING:
break;
default:
return ZX_ERR_INVALID_ARGS;
}
// The buffer size must be a multiple of 4096 (simplifies buffer size
// calcs).
if ((buffer_num_bytes & 0xfff) != 0) {
return ZX_ERR_INVALID_ARGS;
}
if (buffer_num_bytes < trace_context::min_buffer_size() ||
buffer_num_bytes > trace_context::max_buffer_size()) {
return ZX_ERR_INVALID_ARGS;
}
std::lock_guard<std::mutex> lock(g_engine_mutex);
// We must have fully terminated a prior tracing session before starting a new one.
if (g_handler) {
return ZX_ERR_BAD_STATE;
}
ZX_DEBUG_ASSERT(g_state.load(std::memory_order_relaxed) == TRACE_STOPPED);
ZX_DEBUG_ASSERT(g_context_refs.load(std::memory_order_relaxed) == 0u);
zx::event event;
zx_status_t status = zx::event::create(0u, &event);
if (status != ZX_OK) {
return status;
}
// Initialize the trace engine state and context.
// Note that we're still stopped at this point.
g_dispatcher = dispatcher;
g_handler = handler;
g_disposition = ZX_OK;
g_context = new trace_context(buffer, buffer_num_bytes, buffering_mode, handler);
g_event = std::move(event);
g_trace_terminated = false;
g_context->ClearEntireBuffer();
// Write the trace initialization record in case |trace_engine_start()| is
// called with |TRACE_START_RETAIN_BUFFER|.
trace_context_write_initialization_record(g_context, zx_ticks_per_second());
return ZX_OK;
}
// thread-safe
EXPORT_NO_DDK zx_status_t trace_engine_start(trace_start_mode_t start_mode) {
std::lock_guard<std::mutex> lock(g_engine_mutex);
switch (start_mode) {
case TRACE_START_CLEAR_ENTIRE_BUFFER:
case TRACE_START_CLEAR_NONDURABLE_BUFFER:
case TRACE_START_RETAIN_BUFFER:
break;
default:
return ZX_ERR_INVALID_ARGS;
}
// The engine must be initialized first.
if (g_handler == nullptr) {
// The provider library should have initialized us first.
// Use ZX_ERR_INTERNAL to distinguish this from the "not stopped"
// error: the FIDL provider protocol specifies that the response for
// the latter error is to ignore the error. We leave it to the caller
// to decide what to do with this error.
return ZX_ERR_INTERNAL;
}
// |g_handler,g_context| are set/reset together.
ZX_DEBUG_ASSERT(g_context != nullptr);
// We must have fully stopped a prior tracing session before starting a new one.
if (g_state.load(std::memory_order_relaxed) != TRACE_STOPPED) {
return ZX_ERR_BAD_STATE;
}
ZX_DEBUG_ASSERT(g_context_refs.load(std::memory_order_relaxed) == 0u);
// Schedule a waiter for |event|.
g_event_wait = {.state = {ASYNC_STATE_INIT},
.handler = &handle_event,
.object = g_event.get(),
.trigger = (SIGNAL_ALL_OBSERVERS_STARTED | SIGNAL_CONTEXT_RELEASED),
.options = 0};
zx_status_t status = async_begin_wait(g_dispatcher, &g_event_wait);
if (status != ZX_OK) {
return status;
}
// Initialize the trace engine state and context.
g_state.store(TRACE_STARTED, std::memory_order_relaxed);
switch (start_mode) {
case TRACE_START_CLEAR_ENTIRE_BUFFER:
g_context->ClearEntireBuffer();
trace_context_write_initialization_record(g_context, zx_ticks_per_second());
break;
case TRACE_START_CLEAR_NONDURABLE_BUFFER:
// Internal the "nondurable" buffer consists of the "rolling" buffers.
g_context->ClearRollingBuffers();
trace_context_write_initialization_record(g_context, zx_ticks_per_second());
break;
case TRACE_START_RETAIN_BUFFER:
// Nothing to do.
break;
default:
__UNREACHABLE;
}
// After this point clients can acquire references to the trace context.
g_context_refs.store(kProlongedCounterIncrement, std::memory_order_release);
// Flush the call-site cache.
// Do this after clients can acquire the trace context so that any cached
// values that got recorded prior to this are reset, and any new values
// from this point on will see that tracing is on.
flush_site_cache();
// Notify observers that the state changed.
if (g_observers.empty()) {
g_event.signal(0u, SIGNAL_ALL_OBSERVERS_STARTED);
} else {
for (auto& observer : g_observers)
observer.awaiting_update_after_start = true;
notify_observers_locked();
}
return ZX_OK;
}
namespace {
void trace_engine_stop_locked(zx_status_t disposition) __TA_REQUIRES(g_engine_mutex) {
// We must have an active trace in order to stop it.
int state = g_state.load(std::memory_order_relaxed);
if (state == TRACE_STOPPED) {
return;
}
update_disposition_locked(disposition);
if (state == TRACE_STOPPING) {
// already stopping
return;
}
ZX_DEBUG_ASSERT(state == TRACE_STARTED);
ZX_DEBUG_ASSERT(g_context_refs.load(std::memory_order_relaxed) != 0u);
// Begin stopping the trace.
g_state.store(TRACE_STOPPING, std::memory_order_relaxed);
// Flush the call-site cache.
// Do this after tracing is marked as stopping so that any cached
// values that got recorded prior to this are reset, and any new
// values from this point on will see that tracing is stopping.
// It's still possible that a cached value could be in the process of
// being recorded as being enabled. So we might reset the site's state
// and then it gets subsequently marked as enabled by another thread.
// This is perhaps clumsy but ok: If the site got marked as enabled then a
// trace context was acquired and engine state cannot change to STOPPED
// until that context is released after which we will reset the state back
// to disabled.
flush_site_cache();
// Notify observers that the state changed.
notify_observers_locked();
// Release the trace engine's own reference to the trace context.
// |handle_context_released()| will be called asynchronously when the last
// reference is released.
trace_release_prolonged_context(reinterpret_cast<trace_prolonged_context_t*>(g_context));
}
void trace_engine_terminate_locked() __TA_REQUIRES(g_engine_mutex) {
ZX_DEBUG_ASSERT(g_state.load(std::memory_order_relaxed) == TRACE_STOPPED);
ZX_DEBUG_ASSERT(g_context_refs.load(std::memory_order_relaxed) == 0u);
ZX_DEBUG_ASSERT(g_context != nullptr);
ZX_DEBUG_ASSERT(g_handler != nullptr);
delete g_context;
g_context = nullptr;
g_dispatcher = nullptr;
g_handler = nullptr;
g_event.reset();
}
} // namespace
// thread-safe
EXPORT_NO_DDK void trace_engine_stop(zx_status_t disposition) {
std::lock_guard<std::mutex> lock(g_engine_mutex);
trace_engine_stop_locked(disposition);
}
// thread-safe
EXPORT_NO_DDK void trace_engine_terminate() {
trace_handler_t* handler = nullptr;
{
std::lock_guard<std::mutex> lock(g_engine_mutex);
// If trace is still running, stop it.
// We must have an active trace in order to stop it.
if (g_state.load(std::memory_order_relaxed) == TRACE_STOPPED) {
if (g_handler == nullptr) {
// Already terminated.
return;
}
handler = g_handler;
trace_engine_terminate_locked();
} else {
// Final termination has to wait for completion of all pending writers.
g_trace_terminated = true;
trace_engine_stop_locked(ZX_OK);
}
}
if (handler) {
handler->ops->trace_terminated(handler);
}
}
// This is an internal function, only called from context.cpp.
// thread-safe
bool trace_engine_is_buffer_context_released() {
return (g_context_refs.load(std::memory_order_relaxed) & kBufferCounterMask) == 0;
}
// This is an internal function, only called from context.cpp.
// thread-safe
void trace_engine_request_save_buffer(uint32_t wrapped_count, uint64_t durable_data_end) {
// Handle the request on the engine's async loop. This may be get called
// while servicing a client trace request, and we don't want to handle it
// there.
async::PostTask(g_dispatcher, [wrapped_count, durable_data_end]() {
auto context = trace_acquire_prolonged_context();
if (context) {
auto tcontext = reinterpret_cast<trace_context_t*>(context);
tcontext->HandleSaveRollingBufferRequest(wrapped_count, durable_data_end);
trace_release_prolonged_context(context);
}
});
}
// This is called by the handler after it has saved a buffer.
// |wrapped_count| and |durable_end| are the values that were passed to it,
// and are passed back to us for sanity checking purposes.
// thread-safe
EXPORT_NO_DDK zx_status_t trace_engine_mark_buffer_saved(uint32_t wrapped_count,
uint64_t durable_data_end) {
auto context = trace_acquire_prolonged_context();
// No point in updating if there's no active trace.
if (!context) {
return ZX_ERR_BAD_STATE;
}
// Do this now, instead of as a separate iteration on the async loop.
// The concern is that we want to update buffer state ASAP to reduce the
// window where records might be dropped because the buffer is full.
auto tcontext = reinterpret_cast<trace_context_t*>(context);
tcontext->MarkRollingBufferSaved(wrapped_count, durable_data_end);
trace_release_prolonged_context(context);
return ZX_OK;
}
namespace {
void handle_all_observers_started() {
// TODO(TO-530): Allow indicating an observer failed to start.
// Clear the signal, otherwise we'll keep getting called.
g_event.signal(SIGNAL_ALL_OBSERVERS_STARTED, 0u);
// Note: There's no race in the use of |g_handler| here. If it will be set
// to NULL that will be done later (handle_context_released is called by
// handle_event after we are).
if (g_handler) {
g_handler->ops->trace_started(g_handler);
}
}
void handle_context_released() {
// All ready to clean up.
// Grab the mutex while modifying shared state.
zx_status_t disposition;
trace_handler_t* handler;
bool trace_terminated = false;
{
std::lock_guard<std::mutex> lock(g_engine_mutex);
ZX_DEBUG_ASSERT(g_state.load(std::memory_order_relaxed) == TRACE_STOPPING);
ZX_DEBUG_ASSERT(g_context_refs.load(std::memory_order_relaxed) == 0u);
ZX_DEBUG_ASSERT(g_context != nullptr);
// Update final buffer state.
g_context->UpdateBufferHeaderAfterStopped();
// Get final disposition.
if (g_context->WasRecordDropped())
update_disposition_locked(ZX_ERR_NO_MEMORY);
disposition = g_disposition;
// If we're also terminating |g_handler| will get reset.
handler = g_handler;
ZX_DEBUG_ASSERT(handler != nullptr);
// Tidy up.
g_disposition = ZX_OK;
// Clear the signal, otherwise we'll keep getting called.
g_event.signal(SIGNAL_CONTEXT_RELEASED, 0u);
// After this point, it's possible for the engine to be restarted
// (once we release the lock).
g_state.store(TRACE_STOPPED, std::memory_order_relaxed);
// Flush the call-site cache.
// Do this after tracing is marked as stopped so that any cached
// values that got recorded prior to this are reset, and any new
// values from this point on will see that tracing is stopped.
// Any call sites already chained are ok, the concern is with the
// timing of call sites about to be added to the chain. We're ok
// here because at this point it's impossible to acquire a trace
// context, and therefore it's impossible for a category to be
// cached as enabled.
flush_site_cache();
// If tracing has also terminated, finish processing that too.
if (g_trace_terminated) {
trace_terminated = true;
trace_engine_terminate_locked();
}
// Notify observers that the state changed.
notify_observers_locked();
}
// Handler operations are called outside the engine lock.
// Notify the handler about the final disposition.
handler->ops->trace_stopped(handler, disposition);
if (trace_terminated) {
handler->ops->trace_terminated(handler);
}
}
// Handles the case where the asynchronous dispatcher has encountered an error
// and will no longer be servicing the wait callback. Consequently, this is
// our last chance to stop the engine and await for all contexts to be released.
void handle_hard_shutdown(async_dispatcher_t* dispatcher) {
// Stop the engine, in case it hasn't noticed yet.
trace_engine_stop(ZX_ERR_CANCELED);
// And terminate it.
trace_engine_terminate();
// There may still be outstanding references to the trace context.
// We don't know when or whether they will be cleared but we can't complete
// shut down until they are gone since there might still be live pointers
// into the trace buffer so allow a brief timeout. If the release event
// hasn't been signaled by then, declare the trace engine dead in the water
// to prevent dangling pointers. This situations should be very rare as it
// only occurs when the asynchronous dispatcher is shutting down, typically
// just prior to process exit.
auto status = g_event.wait_one(SIGNAL_CONTEXT_RELEASED,
zx::deadline_after(kSynchronousShutdownTimeout), nullptr);
if (status == ZX_OK) {
handle_context_released();
return;
}
// Uh oh.
auto context_refs = g_context_refs.load(std::memory_order_relaxed);
fprintf(stderr,
"TraceEngine: Timed out waiting for %u buffer, %u prolonged trace context\n"
"references (raw 0x%x) to be released after %lu ns\n"
"while the asynchronous dispatcher was shutting down.\n"
"Tracing will no longer be available in this process.",
get_buffer_context_refs(context_refs), get_prolonged_context_refs(context_refs),
context_refs, kSynchronousShutdownTimeout.get());
}
void handle_event(async_dispatcher_t* dispatcher, async_wait_t* wait, zx_status_t status,
const zx_packet_signal_t* signal) {
// Note: This function may get all signals at the same time.
if (status == ZX_OK) {
if (signal->observed & SIGNAL_ALL_OBSERVERS_STARTED) {
handle_all_observers_started();
}
if (signal->observed & SIGNAL_CONTEXT_RELEASED) {
handle_context_released();
return; // trace engine is completely stopped now
}
status = async_begin_wait(dispatcher, &g_event_wait);
}
if (status != ZX_OK) {
handle_hard_shutdown(dispatcher);
}
}
} // namespace
/*** Trace instrumentation functions ***/
// thread-safe, lock-free
EXPORT trace_state_t trace_state() {
return static_cast<trace_state_t>(g_state.load(std::memory_order_relaxed));
}
// thread-safe
EXPORT bool trace_is_category_enabled(const char* category_literal) {
trace_context_t* context = trace_acquire_context();
if (likely(!context))
return false;
bool result = trace_context_is_category_enabled(context, category_literal);
trace_release_context(context);
return result;
}
// thread-safe, fail-fast, lock-free
EXPORT trace_context_t* trace_acquire_context() {
// Fail fast: Check whether we could possibly write into the trace buffer.
// The count must be at least 1 to indicate that the buffer is initialized.
// This is marked likely because tracing is usually disabled and we want
// to return as quickly as possible from this function.
uint32_t count = g_context_refs.load(std::memory_order_relaxed);
if (likely(count == 0u))
return nullptr;
// Attempt to increment the reference count.
// This also acts as a fence for future access to buffer state variables.
//
// Note the ACQUIRE fence here since the trace context may have changed
// from the perspective of this thread.
while (!g_context_refs.compare_exchange_weak(count, count + kBufferCounterIncrement,
std::memory_order_acquire,
std::memory_order_relaxed)) {
if (unlikely(count == 0u))
return nullptr;
}
return g_context;
}
// thread-safe, fail-fast, lock-free
EXPORT trace_context_t* trace_acquire_context_for_category(const char* category_literal,
trace_string_ref_t* out_ref) {
// This is marked likely because tracing is usually disabled and we want
// to return as quickly as possible from this function.
trace_context_t* context = trace_acquire_context();
if (likely(!context))
return nullptr;
if (!trace_context_register_category_literal(context, category_literal, out_ref)) {
trace_release_context(context);
return nullptr;
}
return context;
}
// TODO(PT-84): This function is split out from
// |trace_acquire_context_for_category_cached()| because gcc doesn't
// optimize the prologue as well as it could: It creates the stack frame
// for the entire function prior to the "is disabled?" early-exit test.
// Clang does fine, but for now to achieve optimum performance for the common
// case of tracing off, regardless of compiler, we employ this workaround.
// Both gcc and clang do the expected tail-call optimization, so all this
// costs is an extra branch when tracing is on.
//
// |current_state| is appended as an argument, violating the convention to
// put output parameters last to minimize the changes in the caller's tail
// call.
static __NO_INLINE trace_context_t* trace_acquire_context_for_category_cached_worker(
const char* category_literal, trace_site_t* site, trace_string_ref_t* out_ref,
trace_site_state_t current_state) {
trace_context_t* context = trace_acquire_context_for_category(category_literal, out_ref);
if (likely((current_state & kSiteStateFlagsMask) != kSiteStateUnknown)) {
return context;
}
// First time through for this trace run. Note that multiple threads may
// get to this point for the same call-site.
add_to_site_cache(site, current_state, context != nullptr);
return context;
}
// thread-safe, fail-fast, lock-free
EXPORT trace_context_t* trace_acquire_context_for_category_cached(const char* category_literal,
trace_site_t* site,
trace_string_ref_t* out_ref) {
trace_site_atomic_state_t* state_ptr = get_trace_site_state_as_atomic(site);
trace_site_state_t current_state = state_ptr->load(std::memory_order_relaxed);
if (likely(current_state & kSiteStateDisabled)) {
return nullptr;
}
return trace_acquire_context_for_category_cached_worker(category_literal, site, out_ref,
current_state);
}
// thread-safe
EXPORT zx_status_t trace_engine_flush_category_cache(void) {
std::lock_guard<std::mutex> lock(g_engine_mutex);
if (g_state.load(std::memory_order_relaxed) != TRACE_STOPPED)
return ZX_ERR_BAD_STATE;
// Empty the site cache. The next time the app tries to emit a trace event
// it will get re-added to the cache, but that's ok.
flush_site_cache();
return ZX_OK;
}
// thread-safe, never-fail, lock-free
EXPORT void trace_release_context(trace_context_t* context) {
ZX_DEBUG_ASSERT(context == g_context);
ZX_DEBUG_ASSERT(get_buffer_context_refs(g_context_refs.load(std::memory_order_relaxed)) != 0u);
// Note the RELEASE fence here since the trace context and trace buffer
// contents may have changes from the perspective of other threads.
auto previous = g_context_refs.fetch_sub(kBufferCounterIncrement, std::memory_order_release);
if (unlikely(previous == kBufferCounterIncrement)) {
// Notify the engine that the last reference was released.
zx_status_t status = g_event.signal(0u, SIGNAL_CONTEXT_RELEASED);
ZX_DEBUG_ASSERT(status == ZX_OK);
}
}
// thread-safe, fail-fast, lock-free
EXPORT_NO_DDK trace_prolonged_context_t* trace_acquire_prolonged_context() {
// There's no need for extreme efficiency here, but for consistency with
// |trace_acquire_context()| we copy what it does.
uint32_t count = g_context_refs.load(std::memory_order_relaxed);
if (likely(count == 0u))
return nullptr;
// Attempt to increment the reference count.
// This also acts as a fence for future access to buffer state variables.
//
// Note the ACQUIRE fence here since the trace context may have changed
// from the perspective of this thread.
while (!g_context_refs.compare_exchange_weak(count, count + kProlongedCounterIncrement,
std::memory_order_acquire,
std::memory_order_relaxed)) {
if (likely(count == 0u))
return nullptr;
}
ZX_DEBUG_ASSERT(get_prolonged_context_refs(g_context_refs.load(std::memory_order_relaxed)) <=
kMaxProlongedCounter);
return reinterpret_cast<trace_prolonged_context_t*>(g_context);
}
// thread-safe, never-fail, lock-free
EXPORT_NO_DDK void trace_release_prolonged_context(trace_prolonged_context_t* context) {
auto tcontext = reinterpret_cast<trace_context_t*>(context);
ZX_DEBUG_ASSERT(tcontext == g_context);
ZX_DEBUG_ASSERT(get_prolonged_context_refs(g_context_refs.load(std::memory_order_relaxed)) != 0u);
// Note the RELEASE fence here since the trace context and trace buffer
// contents may have changes from the perspective of other threads.
auto previous = g_context_refs.fetch_sub(kProlongedCounterIncrement, std::memory_order_release);
if (unlikely(previous == kProlongedCounterIncrement)) {
// Notify the engine that the last reference was released.
zx_status_t status = g_event.signal(0u, SIGNAL_CONTEXT_RELEASED);
ZX_DEBUG_ASSERT(status == ZX_OK);
}
}
/*** Asynchronous observers ***/
EXPORT zx_status_t trace_register_observer(zx_handle_t event) {
std::lock_guard<std::mutex> lock(g_engine_mutex);
for (const auto& item : g_observers) {
if (item.event == event)
return ZX_ERR_INVALID_ARGS;
}
g_observers.push_back(Observer{event, false});
return ZX_OK;
}
EXPORT zx_status_t trace_unregister_observer(zx_handle_t event) {
std::lock_guard<std::mutex> lock(g_engine_mutex);
for (auto it = g_observers.begin(); it != g_observers.end(); ++it) {
if (it->event == event) {
bool awaited = it->awaiting_update_after_start;
g_observers.erase(it);
if (awaited) {
notify_engine_all_observers_started_if_needed_locked();
}
return ZX_OK;
}
}
return ZX_ERR_NOT_FOUND;
}
EXPORT void trace_notify_observer_updated(zx_handle_t event) {
std::lock_guard<std::mutex> lock(g_engine_mutex);
for (auto& item : g_observers) {
if (item.event == event) {
if (item.awaiting_update_after_start) {
item.awaiting_update_after_start = false;
notify_engine_all_observers_started_if_needed_locked();
}
return;
}
}
}