| // Copyright 2019 The Fuchsia Authors |
| // |
| // Use of this source code is governed by a MIT-style |
| // license that can be found in the LICENSE file or at |
| // https://opensource.org/licenses/MIT |
| |
| #include "private.h" |
| |
| #ifndef HAVE_SANCOV |
| #error "build system regression" |
| #endif |
| |
| #if !HAVE_SANCOV |
| |
| InstrumentationDataVmo SancovGetPcVmo() { return {}; } |
| InstrumentationDataVmo SancovGetCountsVmo() { return {}; } |
| |
| #else // HAVE_SANCOV |
| |
| #include <stdint.h> |
| #include <zircon/assert.h> |
| |
| #include <ktl/algorithm.h> |
| #include <ktl/atomic.h> |
| #include <lk/init.h> |
| #include <vm/vm_object_paged.h> |
| |
| #include "kernel-mapped-vmo.h" |
| |
| #include <ktl/enforce.h> |
| |
| namespace { |
| |
| // The sancov file format is trivial: magic number and array of PCs. |
| // Each word after the first is either 0 or a PC that was hit. |
| |
| // TODO(mcgrathr): Move the constant into a header shared with other impls. |
| constexpr uint64_t kMagic64 = 0xC0BFFFFFFFFFFF64ULL; |
| |
| constexpr uint64_t kCountsMagic = 0x0023766f436e6153ULL; // "SanCov#" (LE) |
| |
| // The sancov tool matches "<binaryname>" to "<binaryname>.%u.sancov". |
| constexpr ktl::string_view kPcVmoName = "data/zircon.elf.1.sancov"; |
| |
| // This follows the sancov PCs file name just for consistency. |
| constexpr ktl::string_view kCountsVmoName = "data/zircon.elf.1.sancov-counts"; |
| |
| // Go back from the return address to the call site. |
| // Note this must exactly match the calculation in the sancov tool. |
| #ifdef __aarch64__ |
| // Fixed-size instructions, so go back to the previous instruction exactly. |
| constexpr uintptr_t kReturnAddressBias = 4; |
| #else |
| // Variable-sized instructions, so just go back one byte into the middle. |
| constexpr uintptr_t kReturnAddressBias = 1; |
| #endif |
| |
| // These are defined by the linker script. The __sancov_guards section is |
| // populated by the compiler with one slot corresponding to each instrumented |
| // PC location. |
| extern "C" uint32_t __start___sancov_guards[], __stop___sancov_guards[]; |
| |
| [[gnu::const]] size_t GuardsCount() { return __stop___sancov_guards - __start___sancov_guards; } |
| |
| [[gnu::const]] size_t DataSize() { return (GuardsCount() + 1) * sizeof(uint64_t); } |
| |
| // Instrumented code runs from the earliest point, before initialization. The |
| // memory for storing the PCs and counts hasn't been set up. However, code is |
| // running only on the boot CPU. So in the pre-initialization period, we |
| // accumulate 32-bit counts in the __sancov_guard slots. Then after the full |
| // buffers are set up, we copy those counts into the 64-bit counter slots and |
| // re-zero all the guard slots. Thereafter, each guard slot serves as an |
| // atomic flag indicating whether its corresponding PC has been stored yet. |
| // This way, no early PC hits are lost in the counts. However, for PCs whose |
| // only hits were before buffer setup, the nonzero counts will be paired with |
| // zero PC slots because the PC values are only saved in the real buffers. |
| |
| uint64_t* gSancovPcTable = nullptr; |
| uint64_t* gSancovPcCounts = nullptr; |
| |
| KernelMappedVmo gSancovPcVmo, gSancovCountsVmo; |
| |
| void InitSancov(uint level) { |
| fbl::RefPtr<VmObjectPaged> vmo; |
| zx_status_t status = VmObjectPaged::Create(PMM_ALLOC_FLAG_ANY, 0, DataSize(), |
| AttributionObject::GetKernelAttribution(), &vmo); |
| ZX_ASSERT(status == ZX_OK); |
| status = gSancovPcVmo.Init(ktl::move(vmo), 0, DataSize(), "sancov-pc-table"); |
| ZX_ASSERT(status == ZX_OK); |
| |
| status = VmObjectPaged::Create(PMM_ALLOC_FLAG_ANY, 0, DataSize(), |
| AttributionObject::GetKernelAttribution(), &vmo); |
| ZX_ASSERT(status == ZX_OK); |
| status = gSancovCountsVmo.Init(ktl::move(vmo), 0, DataSize(), "sancov-pc-counts-table"); |
| ZX_ASSERT(status == ZX_OK); |
| |
| gSancovPcTable = reinterpret_cast<uint64_t*>(gSancovPcVmo.base_locking()); |
| gSancovPcCounts = reinterpret_cast<uint64_t*>(gSancovCountsVmo.base_locking()); |
| |
| gSancovPcTable[0] = kMagic64; |
| gSancovPcCounts[0] = kCountsMagic; |
| |
| // Move the counts accumulated in the guard slots into their proper places, |
| // and reset the guards. |
| for (size_t i = 0; i < GuardsCount(); ++i) { |
| gSancovPcCounts[i + 1] = ktl::exchange(__start___sancov_guards[i], 0); |
| } |
| |
| // Just in case of LTO or whatnot, ensure that everything is in place before |
| // returning to run any instrumented code. |
| ktl::atomic_signal_fence(ktl::memory_order_seq_cst); |
| } |
| |
| // This needs to happen after the full VM system is available, but while the |
| // kernel is still running only in the initial thread on the boot CPU. |
| LK_INIT_HOOK(InitSancov, InitSancov, LK_INIT_LEVEL_KERNEL) |
| |
| } // namespace |
| |
| extern "C" { |
| |
| // This is run along with static constructors, pretty early in startup. |
| // It's always run on the boot CPU before secondary CPUs are started up. |
| void __sanitizer_cov_trace_pc_guard_init(uint32_t* start, uint32_t* end) { |
| // It's always called with the bounds of the section, which for the |
| // kernel are known statically anyway. |
| ZX_ASSERT(start == __start___sancov_guards); |
| ZX_ASSERT(end == __stop___sancov_guards); |
| } |
| |
| // This is called every time through a covered event. |
| // This might be run before __sanitizer_cov_trace_pc_guard_init has run. |
| void __sanitizer_cov_trace_pc_guard(uint32_t* guard_ptr) { |
| // Compute the table index based just on the address of the guard. |
| // The gSancovPcTable and gSancovPcCounts arrays parallel the guards, |
| // but the first slot in each of those is reserved for the magic number. |
| const size_t idx = guard_ptr - __start___sancov_guards + 1; |
| |
| if (!gSancovPcCounts) [[unlikely]] { |
| // Pre-initialization, just count the hit in the guard slot. See above. |
| ++*guard_ptr; |
| return; |
| } |
| |
| // Every time through, increment the counter. |
| ktl::atomic_ref count(gSancovPcCounts[idx]); |
| count.fetch_add(1, ktl::memory_order_relaxed); |
| |
| // Use the guard as a simple flag to indicate whether the PC has been stored. |
| ktl::atomic_ref guard(*guard_ptr); |
| if (unlikely(guard.load(ktl::memory_order_relaxed) == 0) && |
| likely(guard.exchange(1, ktl::memory_order_relaxed) == 0)) { |
| // This is really the first time through this PC on any CPU. |
| // This is now the only path that will ever use this slot in |
| // the table, so storing there doesn't need to be atomic. |
| gSancovPcTable[idx] = |
| // Take the raw return address. |
| reinterpret_cast<uintptr_t>(__builtin_return_address(0)) - |
| // Adjust it to point into the call instruction. |
| kReturnAddressBias - |
| // Adjust it from runtime to link-time addresses so no |
| // further adjustment is needed to decode the data. |
| reinterpret_cast<uintptr_t>(__executable_start) + KERNEL_BASE; |
| } |
| } |
| |
| } // extern "C" |
| |
| InstrumentationDataVmo SancovGetPcVmo() { |
| return { |
| .announce = "SanitizerCoverage", |
| .sink_name = "sancov", |
| .units = "PCs", |
| .scale = sizeof(gSancovPcTable[0]), |
| .handle = gSancovPcVmo.Publish(kPcVmoName, DataSize()), |
| }; |
| } |
| |
| InstrumentationDataVmo SancovGetCountsVmo() { |
| return { |
| .announce = "SanitizerCoverage Counts", |
| .sink_name = "sancov-counts", |
| .units = "counters", |
| .scale = sizeof(gSancovPcCounts[0]), |
| .handle = gSancovCountsVmo.Publish(kCountsVmoName, DataSize()), |
| }; |
| } |
| |
| #endif // HAVE_SANCOV |