blob: 34b14a2acc1d068bbdbc568c9210c63a3720ad8e [file] [log] [blame]
// Copyright 2020 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "memory_stress.h"
#include <endian.h>
#include <fuchsia/kernel/cpp/fidl.h>
#include <lib/fit/result.h>
#include <lib/stdcompat/span.h>
#include <lib/zx/clock.h>
#include <lib/zx/time.h>
#include <stdio.h>
#include <zircon/assert.h>
#include <zircon/status.h>
#include <zircon/syscalls.h>
#include <numeric>
#include <random>
#include <string>
#include <thread>
#include "compiler.h"
#include "memory_patterns.h"
#include "memory_range.h"
#include "memory_stats.h"
#include "profile_manager.h"
#include "src/lib/fxl/strings/string_printf.h"
#include "status.h"
#include "temperature_sensor.h"
#include "util.h"
namespace hwstress {
namespace {
// Create a std::default_random_engine PRNG pre-seeded with a small truly random value.
std::default_random_engine CreateRandomEngine() {
std::random_device rng;
return std::default_random_engine(rng());
}
// Writes a pattern to memory; verifies it is still the same, and writes out the
// complement; and finally verify the complement has been correctly written out.
template <typename PatternGenerator>
void TestPatternAndComplement(MemoryRange* memory, PatternGenerator pattern) {
PatternGenerator original_pattern = pattern;
// Write out the pattern.
WritePattern(memory->span(), pattern);
memory->CleanInvalidateCache();
// Verify the pattern, flipping each word as we progress.
//
// We perform a read/verify/write on each word at a time (instead of
// a VerifyPattern/WritePattern pair) to minimise the time between
// verifying the old value and writing the next test pattern.
{
uint64_t* start = memory->words();
size_t words = memory->size_words();
for (size_t i = 0; i < words; i++) {
uint64_t p = pattern();
if (unlikely(start[i] != p)) {
ZX_PANIC("Found memory error: expected 0x%16lx, got 0x%16lx at offset %ld.", p, start[i],
i);
}
start[i] = ~p;
}
memory->CleanInvalidateCache();
}
// Verify the pattern again.
VerifyPattern(memory->span(), InvertPattern(original_pattern));
}
// Make a |MemoryWorkload| consisting of writing a pattern to RAM
// and reading it again.
template <typename PatternGenerator>
MemoryWorkload MakePatternWorkload(std::string_view name, PatternGenerator pattern) {
MemoryWorkload result;
result.name = std::string(name);
result.exec = [pattern](StatusLine* /*unused*/, zx::duration /*unused*/, MemoryRange* memory) {
// Write and verify the pattern followed by its negation.
TestPatternAndComplement(memory, pattern);
};
result.memory_type = CacheMode::kCached;
return result;
}
// Repeatedly open/close individual rows on a RAM bank to try and trigger bit errors.
//
// See https://en.wikipedia.org/wiki/Row_hammer for background.
void RowHammer(StatusLine* status, MemoryRange* memory, zx::duration duration, uint64_t pattern) {
constexpr int kAddressesPerIteration = 4;
constexpr int kReadsPerIteration = 1'000'000;
// Set all memory to the desired pattern.
WritePattern(memory->span(), SimplePattern(pattern));
// Get random numbers returning a random page.
uint32_t num_pages = static_cast<uint32_t>(memory->size_bytes() / zx_system_get_page_size());
std::default_random_engine rng = CreateRandomEngine();
std::uniform_int_distribution<uint32_t> random_page(0, num_pages - 1);
// Perform several iterations on different addresses before spending time
// verifying memory.
status->Verbose("Performing RowHammer for %0.2fs with pattern %016lx...",
DurationToSecs(duration), pattern);
zx::time start = zx::clock::get_monotonic();
uint64_t iterations = 0;
while (zx::clock::get_monotonic() - start < duration) {
iterations++;
// Select addresses to hammer.
//
// Our goal is to force the DRAM to open and close a single row many
// times between a refresh cycle. We can do this by reading two
// different rows on the same bank of RAM in quick succession.
//
// Because we don't know the layout of RAM, we select N random
// pages, and read them quickly in succession. There is a good
// chance we'll get lucky and end up with at least two rows in the
// same bank.
volatile uint32_t* targets[kAddressesPerIteration];
uint8_t* data = memory->bytes();
for (auto& target : targets) {
target =
reinterpret_cast<volatile uint32_t*>(data + random_page(rng) * zx_system_get_page_size());
}
// Quickly activate the different rows.
UNROLL_LOOP_4
for (uint32_t j = 0; j < kReadsPerIteration; j++) {
UNROLL_LOOP
for (volatile uint32_t* target : targets) {
ForceEval(*target);
}
};
}
zx::time end = zx::clock::get_monotonic();
double seconds_per_iteration = DurationToSecs(end - start) / static_cast<double>(iterations);
status->Verbose("Done. Time per iteration = %0.2fs, row activations per 64ms refresh ~= %0.0f",
seconds_per_iteration,
(kReadsPerIteration / seconds_per_iteration) * (64. / 1000.));
// Ensure memory is still as expected.
VerifyPattern(memory->span(), SimplePattern(pattern));
}
MemoryWorkload MakeRowHammerWorkload(std::string_view name, uint64_t pattern) {
return MemoryWorkload{
.name = std::string(name),
// Execute the main RowHammer function.
.exec =
[pattern](StatusLine* status, zx::duration max_duration, MemoryRange* memory) {
RowHammer(status, memory, /*duration=*/std::min(max_duration, zx::sec(30)), pattern);
},
// Need to use uncached memory to ensure that each access hits main memory.
.memory_type = CacheMode::kUncached,
// We do not run in time proportional to the memory size, so don't
// attempt to report throughput numbers.
.report_throughput = false,
};
}
} // namespace
template <typename PatternGenerator>
void VerifyPatternOrDie(cpp20::span<uint8_t> range, PatternGenerator pattern) {
std::optional<std::string> result = VerifyPattern(range, pattern);
if (unlikely(result.has_value())) {
ZX_PANIC("Detected memory error: %s", result->c_str());
}
}
std::vector<MemoryWorkload> GenerateMemoryWorkloads() {
std::vector<MemoryWorkload> result;
// Simple patterns.
struct BitPattern {
std::string_view name;
uint64_t pattern;
};
for (const auto& pattern : std::initializer_list<BitPattern>{
{"All 0 bits", 0x0000'0000'0000'0000},
{"All 1 bits", 0xffff'ffff'ffff'fffful},
{"Alternating bits (1/2)", 0x5555'5555'5555'5555},
{"Alternating bits (2/2)", 0xaaaa'aaaa'aaaa'aaaa},
{"2 bits on / 2 bits off (1/2)", 0x3333'3333'3333'3333ul},
{"2 bits on / 2 bits off (2/2)", 0xcccc'cccc'cccc'ccccul},
{"4 bits on / 4 bits off (1/2)", 0xf0f0'f0f0'f0f0'f0f0ul},
{"4 bits on / 4 bits off (2/2)", 0x0f0f'0f0f'0f0f'0f0ful},
}) {
result.push_back(MakePatternWorkload(pattern.name, SimplePattern(pattern.pattern)));
}
// 1 in 6 bits set.
//
// Having every 6'th bit set results in rows of RAM not having bits to
// the north/south/east/west set, assuming that the rows are
// a power-of-two size.
auto every_sixth_bit = std::initializer_list<uint64_t>{
0b1000001000001000001000001000001000001000001000001000001000001000,
0b0010000010000010000010000010000010000010000010000010000010000010,
0b0000100000100000100000100000100000100000100000100000100000100000,
};
for (int i = 0; i < 6; i++) {
result.push_back(MakePatternWorkload(fxl::StringPrintf("Single bit set 6-bit (%d/6)", i + 1),
MultiWordPattern(RotatePattern(every_sixth_bit, i))));
}
for (int i = 0; i < 6; i++) {
result.push_back(
MakePatternWorkload(fxl::StringPrintf("Single bit clear 6-bit (%d/6)", i + 1),
MultiWordPattern(NegateWords((RotatePattern(every_sixth_bit, i))))));
}
// Random bits.
constexpr int kRandomBitIterations = 10;
for (int i = 0; i < kRandomBitIterations; i++) {
result.push_back(MakePatternWorkload(
fxl::StringPrintf("Random bits (%d/%d)", i + 1, kRandomBitIterations), RandomPattern()));
}
// Row hammer workloads.
result.push_back(MakeRowHammerWorkload("Row hammer, all bits clear", 0x0000'0000'0000'0000ul));
result.push_back(MakeRowHammerWorkload("Row hammer, all bits set", 0xffff'ffff'ffff'fffful));
result.push_back(
MakeRowHammerWorkload("Row hammer, alternating bits (1/2)", 0xaaaa'aaaa'aaaa'aaaaul));
result.push_back(
MakeRowHammerWorkload("Row hammer, alternating bits (1/2)", 0x5555'5555'5555'5555ul));
return result;
}
// Ensure that |result| contains at least |size| bytes of memory, mapped in as mode |mode|.
//
// Will deallocate and reallocate memory as required to achieve this.
zx::result<> ReallocateMemoryAsRequired(CacheMode mode, size_t size,
std::unique_ptr<MemoryRange>* result) {
// If we are already allocated and have the right cache mode, nothing to do.
if (*result != nullptr && result->get()->cache() == mode && result->get()->size_bytes() >= size) {
return zx::ok();
}
// Otherwise, allocate new memory.
result->reset();
zx::result<std::unique_ptr<MemoryRange>> maybe_memory = MemoryRange::Create(size, mode);
if (maybe_memory.is_error()) {
return zx::error(maybe_memory.status_value());
}
*result = std::move(maybe_memory).value();
return zx::ok();
}
fit::result<std::string, size_t> GetMemoryToTest(const CommandLineArgs& args) {
// Get amount of RAM and free memory in system.
zx::result<fuchsia::kernel::MemoryStats> maybe_stats = GetMemoryStats();
if (maybe_stats.is_error()) {
return fit::error("Could not fetch free memory.");
}
// If a value was specified, and doesn't exceed total system RAM, use that.
if (args.mem_to_test_megabytes.has_value()) {
size_t requested = MiB(args.mem_to_test_megabytes.value());
if (requested > maybe_stats->total_bytes()) {
return fit::error(fxl::StringPrintf(
"Specified memory size (%ld bytes) exceeds system memory size (%ld bytes).", requested,
maybe_stats->total_bytes()));
}
return fit::ok(requested);
}
// If user asked for a percent of total memory, calculate that.
if (args.ram_to_test_percent.has_value()) {
uint64_t total_bytes = maybe_stats->total_bytes();
auto test_bytes =
static_cast<uint64_t>(static_cast<double>(total_bytes) *
(static_cast<double>(args.ram_to_test_percent.value()) / 100.));
return fit::ok(RoundUp(test_bytes, zx_system_get_page_size()));
}
// Otherwise, try and calculate a reasonable value based on free memory.
//
// The default memory stress values for Fuchsia are:
// - 300MiB free => Warning
// - 150MiB free => Critical
// - 50MiB free => OOM
//
// We aim to hit just below the warning threshold.
uint64_t free_bytes = maybe_stats->free_bytes();
uint64_t slack = MiB(301);
if (free_bytes < slack + MiB(1)) {
// We don't have 300MiB free: just use 1MiB.
return zx::ok(MiB(1));
}
return zx::ok(RoundUp(free_bytes - slack, zx_system_get_page_size()));
}
MemoryWorkloadGenerator ::MemoryWorkloadGenerator(const std::vector<MemoryWorkload>& workloads,
uint32_t num_cpus)
: num_cpus_(num_cpus) {
ZX_ASSERT(!workloads.empty());
// Copy workloads into workloads_, converting into
// a std::optional<MemoryWorkload> in the process.
workloads_.reserve(workloads.size());
for (const MemoryWorkload& workload : workloads) {
workloads_.emplace_back(workload);
}
// We want to iterate through different workloads and different CPUs. One
// method would be to test 1 CPU through all workloads, or 1 workload
// through all CPUs. Neither is great: ideally, we would like to get
// good coverage of both CPUs and workloads relatively quickly.
//
// To try and quickly maximise coverage, we instead iterate through both
// simultaneously. If we have:
//
// gcd(num_cpus, num_workloads) == 1
//
// then the Chinese Remainder Theorem [1] ensures that after num_cpus
// * num_workloads iterations, we will have covered every combination of
// num_cpus * num_workloads.
//
// To ensure that gcd(num_cpus, num_workloads) == 1, we keep adding a number
// of dummy "null" workloads until this criteria is met.
//
// [1] https://en.wikipedia.org/wiki/Chinese_remainder_theorem
while (std::gcd(num_cpus_, workloads_.size()) != 1) {
workloads_.push_back(std::nullopt);
}
}
MemoryWorkloadGenerator::Workload MemoryWorkloadGenerator::Next() {
// Increment |n|, skipping over null workloads.
do {
n_++;
} while (!workloads_[n_ % workloads_.size()].has_value());
return Workload{
.cpu = static_cast<uint32_t>(n_ % num_cpus_),
.workload = workloads_[n_ % workloads_.size()].value(),
};
}
bool StressMemory(StatusLine* status, const CommandLineArgs& args, zx::duration duration,
TemperatureSensor* /*temperature_sensor*/) {
// Parse the amount of memory to test.
fit::result<std::string, size_t> bytes_to_test = GetMemoryToTest(args);
if (bytes_to_test.is_error()) {
status->Log(bytes_to_test.error_value());
return false;
}
status->Log("Testing %0.2fMiB of memory.",
static_cast<double>(bytes_to_test.value()) / static_cast<double>(MiB(1)));
// Create a profile manager.
std::unique_ptr<ProfileManager> profile_manager = ProfileManager::CreateFromEnvironment();
if (profile_manager == nullptr) {
status->Log("Error: could not create profile manager.");
return false;
}
// Get workloads.
std::vector<MemoryWorkload> workloads = GenerateMemoryWorkloads();
MemoryWorkloadGenerator workload_generator{workloads, zx_system_get_num_cpus()};
// Keep looping over the memory tests until we run out of time.
std::unique_ptr<MemoryRange> memory;
uint64_t num_tests = 1;
zx::time end_time = zx::deadline_after(duration);
while (zx::clock::get_monotonic() < end_time) {
MemoryWorkloadGenerator::Workload next = workload_generator.Next();
// Allocate memory for tests.
zx::result<> result =
ReallocateMemoryAsRequired(next.workload.memory_type, bytes_to_test.value(), &memory);
if (result.is_error()) {
status->Log("Failed to reallocate memory: %s", result.status_string());
return false;
}
// Log start of test.
status->Set("Test %4ld: CPU %2d : %s", num_tests, next.cpu, next.workload.name.c_str());
// Switch execution to the correct CPU.
profile_manager->SetThreadAffinity(*zx::thread::self(), 1u << next.cpu);
// Execute the workload.
zx::time test_start = zx::clock::get_monotonic();
next.workload.exec(status, /*max_duration=*/(end_time - zx::clock::get_monotonic()),
memory.get());
zx::duration test_duration = zx::clock::get_monotonic() - test_start;
// Calculate test time and throughput.
std::string throughput;
if (next.workload.report_throughput) {
throughput = fxl::StringPrintf(", throughput: %0.2f MiB/s",
static_cast<double>(memory->size_bytes()) /
DurationToSecs(test_duration) / 1024. / 1024.);
}
status->Log("Test %4ld: CPU %2d : %s: %0.3fs%s", num_tests, next.cpu,
next.workload.name.c_str(), DurationToSecs(test_duration), throughput.c_str());
num_tests++;
}
return true;
}
} // namespace hwstress