src/sys/fuzzing/common/runner-unittest.cc

// Copyright 2021 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/sys/fuzzing/common/runner-unittest.h"

#include <zircon/status.h>

#include "src/sys/fuzzing/common/testing/monitor.h"

namespace fuzzing {

// |Cleanse| tries to replace bytes with 0x20 or 0xff.
static constexpr size_t kNumReplacements = 2;

// Test fixtures.

void RunnerTest::SetUp() {
  AsyncTest::SetUp();
  handler_ = [](const Input& input) { return FuzzResult::NO_ERRORS; };
}

OptionsPtr RunnerTest::DefaultOptions() {
  auto options = MakeOptions();
  runner()->AddDefaults(options.get());
  return options;
}

void RunnerTest::Configure(const OptionsPtr& options) {
  options_ = options;
  options_->set_seed(1);
  FUZZING_EXPECT_OK(runner()->Configure(options_));
  RunUntilIdle();
}

void RunnerTest::SetCoverage(const Input& input, Coverage coverage) {
  coverage_[input.ToHex()] = std::move(coverage);
}

void RunnerTest::SetFuzzResultHandler(FuzzResultHandler handler) { handler_ = std::move(handler); }

void RunnerTest::SetLeak(bool has_leak) { has_leak_ = has_leak; }

Promise<Input> RunnerTest::RunOne() {
  return RunOne([this](const Input& input) {
    return SetFeedback(Coverage(coverage_[input.ToHex()]), handler_(input), has_leak_);
  });
}

Promise<Input> RunnerTest::RunOne(Coverage coverage) {
  return RunOne([this, coverage = std::move(coverage)](const Input& input) {
    return SetFeedback(std::move(coverage), handler_(input), has_leak_);
  });
}

Promise<Input> RunnerTest::RunOne(FuzzResult fuzz_result) {
  return RunOne([this, fuzz_result](const Input& input) {
    return SetFeedback(Coverage(coverage_[input.ToHex()]), fuzz_result, has_leak_);
  });
}

Promise<Input> RunnerTest::RunOne(bool has_leak) {
  return RunOne([this, has_leak](const Input& input) {
    return SetFeedback(Coverage(coverage_[input.ToHex()]), handler_(input), has_leak);
  });
}

Promise<Input> RunnerTest::RunOne(fit::function<ZxPromise<>(const Input&)> set_feedback) {
  Bridge<> bridge;
  auto task = GetTestInput()
                  .and_then([set_feedback = std::move(set_feedback)](Input& input) mutable {
                    return set_feedback(input).and_then([input = std::move(input)]() mutable {
                      return fpromise::ok(std::move(input));
                    });
                  })
                  .or_else([](const zx_status_t& status) {
                    // Target may close before returning test input.
                    EXPECT_EQ(status, ZX_ERR_PEER_CLOSED) << zx_status_get_string(status);
                    return fpromise::error();
                  })
                  .then([completer = std::move(bridge.completer)](Result<Input>& result) mutable {
                    if (result.is_ok()) {
                      completer.complete_ok();
                    } else {
                      completer.complete_error();
                    }
                    return std::move(result);
                  });
  auto consumer = std::move(previous_run_);
  previous_run_ = std::move(bridge.consumer);
  return consumer ? consumer.promise().and_then(std::move(task)).box() : task.box();
}

void RunnerTest::RunUntil(Promise<> promise) {
  RunUntil(
      std::move(promise),
      [this](const Result<Input>& result) -> Promise<Input> { return RunOne(); }, Input());
}

void RunnerTest::RunUntil(Promise<> promise, RunCallback run, Input input) {
  auto task =
      fpromise::make_promise([run = std::move(run),
                              result = Result<Input>(fpromise::ok(std::move(input))),
                              running = Future<Input>(), until = Future<>(std::move(promise))](
                                 Context& context) mutable -> Result<> {
        while (result.is_ok() && !until(context)) {
          if (!running) {
            running = run(result);
          }
          if (!running(context)) {
            return fpromise::pending();
          }
          result = running.take_result();
        }
        // Ignore errors; they were checked in |RunOne|.
        return fpromise::ok();
      }).wrap_with(scope_);
  FUZZING_EXPECT_OK(std::move(task));
  RunUntilIdle();
}

// Unit tests.

void RunnerTest::ExecuteNoError() {
  Configure(DefaultOptions());
  Input input({0x01});
  FUZZING_EXPECT_OK(runner()->Execute(input.Duplicate()), FuzzResult::NO_ERRORS);
  FUZZING_EXPECT_OK(RunOne(), std::move(input));
  RunUntilIdle();
}

void RunnerTest::ExecuteWithError() {
  Configure(DefaultOptions());
  Input input({0x02});
  FUZZING_EXPECT_OK(runner()->Execute(input.Duplicate()), FuzzResult::BAD_MALLOC);
  FUZZING_EXPECT_OK(RunOne(FuzzResult::BAD_MALLOC), std::move(input));
  RunUntilIdle();
}

void RunnerTest::ExecuteWithLeak() {
  auto options = DefaultOptions();
  options->set_detect_leaks(true);
  Configure(options);
  Input input({0x03});
  // Simulate a suspected leak, followed by an LSan exit. The leak detection heuristics only run
  // full leak detection when a leak is suspected based on mismatched allocations.
  SetLeak(true);
  FUZZING_EXPECT_OK(runner()->Execute(input.Duplicate()), FuzzResult::LEAK);
  FUZZING_EXPECT_OK(RunOne(), input.Duplicate());
  FUZZING_EXPECT_OK(RunOne(FuzzResult::LEAK), std::move(input));
  RunUntilIdle();
}
// Simulate no error on the original input.
void RunnerTest::MinimizeNoError() {
  Configure(DefaultOptions());
  Input input({0x04});
  FUZZING_EXPECT_ERROR(runner()->Minimize(input.Duplicate()), ZX_ERR_INVALID_ARGS);
  FUZZING_EXPECT_OK(RunOne(), std::move(input));
  RunUntilIdle();
}

// Empty input should exit immediately.
void RunnerTest::MinimizeEmpty() {
  Configure(DefaultOptions());
  Input input;
  FUZZING_EXPECT_OK(runner()->Minimize(input.Duplicate()), input.Duplicate());
  FUZZING_EXPECT_OK(RunOne(FuzzResult::CRASH), std::move(input));
  RunUntilIdle();
}

// 1-byte input should exit immediately.
void RunnerTest::MinimizeOneByte() {
  Configure(DefaultOptions());
  Input input({0x44});
  FUZZING_EXPECT_OK(runner()->Minimize(input.Duplicate()), input.Duplicate());
  FUZZING_EXPECT_OK(RunOne(FuzzResult::CRASH), std::move(input));
  RunUntilIdle();
}

void RunnerTest::MinimizeReduceByTwo() {
  auto options = DefaultOptions();
  constexpr size_t kRuns = 0x40;
  options->set_runs(kRuns);
  Configure(options);
  Input input({0x51, 0x52, 0x53, 0x54, 0x55, 0x56});
  Input minimized;
  Barrier barrier;
  auto task = runner()
                  ->Minimize(input.Duplicate())
                  .and_then([&minimized](Input& result) {
                    minimized = std::move(result);
                    return fpromise::ok();
                  })
                  .wrap_with(barrier);
  FUZZING_EXPECT_OK(std::move(task));

  // Crash until inputs are smaller than 4 bytes.
  RunUntil(
      barrier.sync(),
      [this](const Result<Input>& result) {
        return RunOne(result.value().size() > 3 ? FuzzResult::CRASH : FuzzResult::NO_ERRORS);
      },
      std::move(input));

  EXPECT_LE(minimized.size(), 3U);
}

void RunnerTest::MinimizeNewError() {
  auto options = DefaultOptions();
  options->set_run_limit(zx::msec(500).get());
  Configure(options);
  Input input({0x05, 0x15, 0x25, 0x35});
  Input minimized;

  // Simulate a crash on the original input, and a timeout on any smaller input.
  SetFuzzResultHandler([](const Input& input) {
    return input.size() > 3 ? FuzzResult::CRASH : FuzzResult::TIMEOUT;
  });
  Barrier barrier;
  auto task = runner()
                  ->Minimize(input.Duplicate())
                  .and_then([&minimized](Input& result) {
                    minimized = std::move(result);
                    return fpromise::ok();
                  })
                  .wrap_with(barrier);
  FUZZING_EXPECT_OK(std::move(task));
  RunUntil(barrier.sync());
  EXPECT_EQ(minimized, input);
}

void RunnerTest::CleanseNoReplacement() {
  Configure(DefaultOptions());
  Input input({0x07, 0x17, 0x27});
  Input cleansed;
  FUZZING_EXPECT_OK(runner()->Cleanse(input.Duplicate()), &cleansed);

  // Simulate no error after cleansing any byte.
  for (size_t i = 0; i < input.size(); ++i) {
    for (size_t j = 0; j < kNumReplacements; ++j) {
      FUZZING_EXPECT_OK(RunOne(FuzzResult::NO_ERRORS));
    }
  }

  RunUntilIdle();
  EXPECT_EQ(cleansed, input);
}

void RunnerTest::CleanseAlreadyClean() {
  Configure(DefaultOptions());
  Input input({' ', 0xff});
  Input cleansed;
  FUZZING_EXPECT_OK(runner()->Cleanse(input.Duplicate()), &cleansed);

  // All bytes match replacements, so this should be done.
  RunUntilIdle();
  EXPECT_EQ(cleansed, input);
}

void RunnerTest::CleanseTwoBytes() {
  Configure(DefaultOptions());

  Input input({0x08, 0x18, 0x28});
  Input inputs[] = {
      {0x20, 0x18, 0x28},  // 1st try.
      {0xff, 0x18, 0x28},  //
      {0x08, 0x20, 0x28},  //
      {0x08, 0xff, 0x28},  //
      {0x08, 0x18, 0x20},  //
      {0x08, 0x18, 0xff},  // Error on 2nd replacement of 3rd byte.
      {0x20, 0x18, 0xff},  // 2nd try. Error on 1st replacement of 1st byte.
      {0x20, 0x20, 0xff},  //
      {0x20, 0xff, 0xff},  //
      {0x20, 0x20, 0xff},  // 3rd try. No errors.
      {0x20, 0xff, 0xff},  //
  };
  SetFuzzResultHandler([](const Input& input) {
    auto hex = input.ToHex();
    return (hex == "081828" || hex == "0818ff" || hex == "2018ff") ? FuzzResult::DEATH
                                                                   : FuzzResult::NO_ERRORS;
  });

  Input cleansed;
  FUZZING_EXPECT_OK(runner()->Cleanse(std::move(input)), &cleansed);
  for (auto& input : inputs) {
    FUZZING_EXPECT_OK(RunOne(), std::move(input));
  }

  RunUntilIdle();
  EXPECT_EQ(cleansed, Input({0x20, 0x18, 0xff}));
}

void RunnerTest::FuzzUntilError() {
  auto runner = this->runner();
  auto options = DefaultOptions();
  options->set_detect_exits(true);
  options->set_mutation_depth(1);
  Configure(options);

  Artifact artifact;
  FUZZING_EXPECT_OK(runner->Fuzz(), &artifact);

  // Add some corpus elements.
  std::vector<Input> inputs;
  inputs.emplace_back("foo");
  inputs.emplace_back("bar");
  inputs.emplace_back("baz");
  inputs.emplace_back("qux");
  auto last = CorpusType::LIVE;
  auto next = CorpusType::SEED;
  for (const auto& input : inputs) {
    EXPECT_EQ(runner->AddToCorpus(next, input.Duplicate()), ZX_OK);
    std::swap(next, last);
  }

  // Set some coverage for the inputs above. Some runners (e.g. libFuzzer) won't mutate an input
  // that lacks any coverage. According to the AFL bucketing scheme used by libFuzzer and others,
  // the counter must be at least 2 to map to a coverage "feature".
  for (size_t i = 0; i < inputs.size(); ++i) {
    SetCoverage(inputs[i], {{i + 1, i + 1}});
  }

  std::vector<Input> actual;
  for (size_t i = 0; i < 100; ++i) {
    FUZZING_EXPECT_OK(RunOne().then([&](Result<Input>& result) {
      if (result.is_ok()) {
        actual.push_back(result.take_value());
      }
    }));
  }

  FUZZING_EXPECT_OK(RunOne(FuzzResult::EXIT));
  RunUntilIdle();

  // Helper lambda to check if the sequence of bytes given by |needle| appears in order, but not
  // necessarily contiguously, in the sequence of bytes given by |haystack|.
  auto contains = [](const Input& haystack, const Input& needle) -> bool {
    const auto* needle_data = needle.data();
    const auto* haystack_data = haystack.data();
    size_t i = 0;
    for (size_t j = 0; i < needle.size() && j < haystack.size(); ++j) {
      if (needle_data[i] == haystack_data[j]) {
        ++i;
      }
    }
    return i == needle.size();
  };

  // Verify that each corpus element is 1) used as-is, and 2) used as the basis for mutations.
  for (auto& orig : inputs) {
    bool exact_match_found = false;
    bool near_match_found = false;
    for (auto& input : actual) {
      if (orig == input) {
        exact_match_found = true;
      } else if (contains(input, orig)) {
        near_match_found = true;
      }
      if (exact_match_found && near_match_found) {
        break;
      }
    }
    EXPECT_TRUE(exact_match_found) << "input: " << orig.ToHex();
    EXPECT_TRUE(near_match_found) << "input: " << orig.ToHex();
  }

  EXPECT_EQ(artifact.fuzz_result(), FuzzResult::EXIT);
}

void RunnerTest::FuzzUntilRuns() {
  auto runner = this->runner();
  auto options = DefaultOptions();
  const size_t kNumRuns = 10;
  options->set_runs(kNumRuns);
  Configure(options);
  std::vector<std::string> expected({""});

  // Subscribe to status updates.
  FakeMonitor monitor(executor());
  runner->AddMonitor(monitor.NewBinding());

  // Fuzz for exactly |kNumRuns|.
  Artifact artifact;
  FUZZING_EXPECT_OK(runner->Fuzz(), &artifact);

  for (size_t i = 0; i < kNumRuns; ++i) {
    FUZZING_EXPECT_OK(RunOne({{i, i}}));
  }

  // Check that we get the expected status updates.
  FUZZING_EXPECT_OK(monitor.AwaitUpdate());
  RunUntilIdle();
  EXPECT_EQ(monitor.reason(), UpdateReason::INIT);
  auto status = monitor.take_status();
  ASSERT_TRUE(status.has_running());
  EXPECT_TRUE(status.running());
  ASSERT_TRUE(status.has_runs());
  auto runs = status.runs();
  ASSERT_TRUE(status.has_elapsed());
  EXPECT_GT(status.elapsed(), 0U);
  auto elapsed = status.elapsed();
  ASSERT_TRUE(status.has_covered_pcs());
  EXPECT_GE(status.covered_pcs(), 0U);
  auto covered_pcs = status.covered_pcs();

  monitor.pop_front();
  FUZZING_EXPECT_OK(monitor.AwaitUpdate());
  RunUntilIdle();
  EXPECT_EQ(size_t(monitor.reason()), size_t(UpdateReason::NEW));
  status = monitor.take_status();
  ASSERT_TRUE(status.has_running());
  EXPECT_TRUE(status.running());
  ASSERT_TRUE(status.has_runs());
  EXPECT_GT(status.runs(), runs);
  runs = status.runs();
  ASSERT_TRUE(status.has_elapsed());
  EXPECT_GT(status.elapsed(), elapsed);
  elapsed = status.elapsed();
  ASSERT_TRUE(status.has_covered_pcs());
  EXPECT_GT(status.covered_pcs(), covered_pcs);
  covered_pcs = status.covered_pcs();

  // Skip others up to DONE.
  while (monitor.reason() != UpdateReason::DONE) {
    monitor.pop_front();
    FUZZING_EXPECT_OK(monitor.AwaitUpdate());
    RunUntilIdle();
  }
  status = monitor.take_status();
  ASSERT_TRUE(status.has_running());
  EXPECT_FALSE(status.running());
  ASSERT_TRUE(status.has_runs());
  EXPECT_GE(status.runs(), runs);
  ASSERT_TRUE(status.has_elapsed());
  EXPECT_GT(status.elapsed(), elapsed);
  ASSERT_TRUE(status.has_covered_pcs());
  EXPECT_GE(status.covered_pcs(), covered_pcs);

  // All done.
  EXPECT_EQ(artifact.fuzz_result(), FuzzResult::NO_ERRORS);
}

void RunnerTest::FuzzUntilTime() {
  auto runner = this->runner();
  // Time is always tricky to test. As a result, this test verifies the bare minimum, namely that
  // the runner exits at least 100 ms after it started. All other verification is performed in more
  // controllable tests, such as |FuzzUntilRuns| above.
  auto options = DefaultOptions();
  options->set_max_total_time(zx::msec(100).get());
  Configure(options);
  auto start = zx::clock::get_monotonic();

  Artifact artifact;
  Barrier barrier;
  auto task = runner->Fuzz()
                  .and_then([&artifact](Artifact& result) {
                    artifact = std::move(result);
                    return fpromise::ok();
                  })
                  .wrap_with(barrier);
  FUZZING_EXPECT_OK(std::move(task));
  RunUntil(barrier.sync());

  EXPECT_EQ(artifact.fuzz_result(), FuzzResult::NO_ERRORS);
  auto elapsed = zx::clock::get_monotonic() - start;
  EXPECT_GE(elapsed, zx::msec(100));
}

void RunnerTest::MergeSeedError(zx_status_t expected, uint64_t oom_limit) {
  auto runner = this->runner();
  auto options = DefaultOptions();
  options->set_oom_limit(oom_limit);
  Configure(options);
  runner->AddToCorpus(CorpusType::SEED, Input({0x09}));
  if (expected == ZX_OK) {
    FUZZING_EXPECT_OK(runner->Merge());
  } else {
    FUZZING_EXPECT_ERROR(runner->Merge(), expected);
  }
  FUZZING_EXPECT_OK(RunOne(FuzzResult::OOM));
  RunUntilIdle();
}

void RunnerTest::Merge(bool keeps_errors, uint64_t oom_limit) {
  auto runner = this->runner();
  auto options = DefaultOptions();
  options->set_oom_limit(oom_limit);
  Configure(options);
  std::vector<std::string> expected_seed;
  std::vector<std::string> expected_live;

  // Empty input, implicitly included in all corpora.
  Input input0;
  expected_seed.push_back(input0.ToHex());
  expected_live.push_back(input0.ToHex());

  // Seed input => kept.
  Input input1({0x0a});
  SetCoverage(input1, {{0, 1}, {1, 2}, {2, 3}});
  runner->AddToCorpus(CorpusType::SEED, input1.Duplicate());
  expected_seed.push_back(input1.ToHex());

  // Triggers error => maybe kept.
  Input input2({0x0b});
  SetFuzzResultHandler([](const Input& input) {
    return input.ToHex() == "0b" ? FuzzResult::OOM : FuzzResult::NO_ERRORS;
  });
  runner->AddToCorpus(CorpusType::LIVE, input2.Duplicate());
  if (keeps_errors) {
    expected_live.push_back(input2.ToHex());
  }

  // Second-smallest and 2 non-seed features => kept.
  Input input5({0x0c, 0x0c});
  SetCoverage(input5, {{0, 2}, {2, 2}});
  runner->AddToCorpus(CorpusType::LIVE, input5.Duplicate());
  expected_live.push_back(input5.ToHex());

  // Larger and 1 feature not in any smaller inputs => kept.
  Input input4({0x0d, 0x0d, 0x0d});
  SetCoverage(input4, {{0, 2}, {1, 1}});
  runner->AddToCorpus(CorpusType::LIVE, input4.Duplicate());
  expected_live.push_back(input4.ToHex());

  // Second-smallest but only 1 non-seed feature above => skipped.
  Input input3({0x0e, 0x0e});
  SetCoverage(input3, {{0, 2}, {2, 3}});
  runner->AddToCorpus(CorpusType::LIVE, input3.Duplicate());

  // Smallest but features are subset of seed corpus => skipped.
  Input input6({0x0f});
  SetCoverage(input6, {{0, 1}, {2, 3}});
  runner->AddToCorpus(CorpusType::LIVE, input6.Duplicate());

  // Largest with all 3 of the new features => skipped.
  Input input7({0x10, 0x10, 0x10, 0x10});
  SetCoverage(input7, {{0, 2}, {1, 1}, {2, 2}});
  runner->AddToCorpus(CorpusType::LIVE, input7.Duplicate());

  Barrier barrier;
  FUZZING_EXPECT_OK(runner->Merge().wrap_with(barrier));
  RunUntil(barrier.sync());

  std::vector<std::string> actual_seed;
  for (size_t i = 0; i < expected_seed.size(); ++i) {
    actual_seed.push_back(runner->ReadFromCorpus(CorpusType::SEED, i).ToHex());
  }
  std::sort(expected_seed.begin(), expected_seed.end());
  std::sort(actual_seed.begin(), actual_seed.end());
  EXPECT_EQ(expected_seed, actual_seed);

  std::vector<std::string> actual_live;
  for (size_t i = 0; i < expected_live.size(); ++i) {
    actual_live.push_back(runner->ReadFromCorpus(CorpusType::LIVE, i).ToHex());
  }
  std::sort(actual_live.begin(), actual_live.end());
  EXPECT_EQ(expected_live, actual_live);
}

void RunnerTest::Stop() {
  auto runner = this->runner();
  Configure(DefaultOptions());
  Artifact artifact;
  Barrier barrier;
  auto task = runner->Fuzz()
                  .and_then([&artifact](Artifact& result) {
                    artifact = std::move(result);
                    return fpromise::ok();
                  })
                  .wrap_with(barrier);
  FUZZING_EXPECT_OK(std::move(task));
  FUZZING_EXPECT_OK(executor()
                        ->MakeDelayedPromise(zx::msec(100))
                        .then([runner](const Result<>& result) { return runner->Stop(); })
                        .then([runner](const ZxResult<>& result) {
                          // Should be idempotent.
                          return runner->Stop();
                        }));
  RunUntil(barrier.sync());
  EXPECT_EQ(artifact.fuzz_result(), FuzzResult::NO_ERRORS);
}

}  // namespace fuzzing