encoder/send_retryer_test.cc - cobalt - Git at Google

 // Copyright 2017 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 #include "encoder/send_retryer.h"

 #include <chrono>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>

 #include "./encrypted_message.pb.h"
 #include "./gtest.h"
 #include "./logging.h"
 #include "encoder/shuffler_client.h"
 #include "third_party/gflags/include/gflags/gflags.h"
 #include "util/clock.h"

 namespace cobalt {
 namespace encoder {
 namespace send_retryer {

 using util::IncrementingClock;

 namespace {

 // An implementation of ShufflerClientInterface that returns the sequence
 // of statuses it is told to return and records the number of times that
 // SendToShuffler() was invoked and the gRPC deadlines in each invocation.
 // Additionally, the client will optionally invoke cancel_handle->TryCancel() on
 // a specified invocation count number.
 class FakeShufflerClient : public ShufflerClientInterface {
  public:
   FakeShufflerClient(IncrementingClock* incrementing_clock,
                      CancelHandle* cancel_handle)
       : incrementing_clock(incrementing_clock), cancel_handle(cancel_handle) {}

   ~FakeShufflerClient() {}

   grpc::Status SendToShuffler(const EncryptedMessage& encrypted_message,
                               grpc::ClientContext* context = nullptr) override {
     call_count++;
     CHECK(context);
     // The gRPC deadline embedded in |context| is expressed as an absolute
     // deadline. We recover the value of |rpc_deadline| set by the
     // Retryer by subtracting the clock's current time using |peek_now()|.
     // This is a bit fragile: It works only because we know that there have
     // been no invocations of clock_->now() between the time that the the
     // Retryer computed the deadline and the call to this function. We
     // record the deadline as a number of milliseconds.
     deadlines.push_back(
         std::chrono::duration_cast<std::chrono::milliseconds>(
             context->deadline() - incrementing_clock->peek_now())
             .count());

     CHECK(call_count <= statuses_to_return.size());
     if (call_count == cancel_on_this_call_count) {
       cancel_handle->TryCancel();
     }
     return statuses_to_return[call_count - 1];
   }

   IncrementingClock* incrementing_clock;  // not owned.
   std::vector<grpc::Status> statuses_to_return = {grpc::Status::OK};
   size_t call_count = 0;
   std::vector<int64_t> deadlines;
   size_t cancel_on_this_call_count = -1;
   CancelHandle* cancel_handle;
 };

 }  // namespace

 class SendRetryerTest : public ::testing::Test {
  public:
   SendRetryerTest() {
     std::unique_ptr<IncrementingClock> clock(new IncrementingClock());
     incrementing_clock_ = clock.get();
     cancel_handle_.reset(new CancelHandle());
     cancel_handle_->sleep_notification_function_ = [this](int sleep_millis) {
       sleep_millis_used_.push_back(sleep_millis);
     };
     shuffler_client_.reset(
         new FakeShufflerClient(incrementing_clock_, cancel_handle_.get()));
     retryer_.reset(new SendRetryer(shuffler_client_.get()));
     retryer_->clock_ = std::move(clock);
     retryer_->initial_sleep_ = std::chrono::milliseconds(1);
   }

  protected:
   // Invokes SendToShuffler() with an initial_rpc_deadline of 10 seconds,
   // the given overall_deadline (also defaulting to 10 seconds), and our fixed
   // cancel_handle and encrypted_message.
   grpc::Status SendToShuffler(
       std::chrono::seconds overerall_deadline = std::chrono::seconds(10)) {
     return retryer_->SendToShuffler(std::chrono::seconds(10),
                                     overerall_deadline, cancel_handle_.get(),
                                     encrypted_message_);
   }

   // Checks that the status, call_count and deadlines are as expected.
   void CheckResults(grpc::Status status, grpc::StatusCode expected_code,
                     uint32_t expected_call_count,
                     std::vector<int> expected_deadline_seconds) {
     EXPECT_EQ(expected_code, status.error_code());
     EXPECT_EQ(expected_call_count, shuffler_client_->call_count);
     ASSERT_EQ(expected_call_count, shuffler_client_->deadlines.size());
     ASSERT_EQ(expected_call_count, expected_deadline_seconds.size());
     for (size_t i = 0; i < expected_call_count; i++) {
       EXPECT_EQ(1000 * expected_deadline_seconds[i],
                 shuffler_client_->deadlines[i])
           << "i=" << i << ", expected_seconds=" << expected_deadline_seconds[i]
           << ", actual_millis=" << shuffler_client_->deadlines[i];
     }
     // Check that the sleep times between send attempts started at the
     // expected initial value and doubled each time.
     ASSERT_EQ(expected_call_count - 1, sleep_millis_used_.size());
     // expected_sleep_millis is initialized to 1 because in the constructor for
     // SendRetryierTest we invoked
     // retryer_->initial_sleep_ = std::chrono::milliseconds(1);
     int expected_sleep_millis = 1;
     for (size_t i = 0; i < expected_call_count - 1; i++) {
       EXPECT_EQ(expected_sleep_millis, sleep_millis_used_[i]);
       expected_sleep_millis *= 2;
     }
   }

   std::unique_ptr<FakeShufflerClient> shuffler_client_;
   std::unique_ptr<SendRetryer> retryer_;
   EncryptedMessage encrypted_message_;
   IncrementingClock* incrementing_clock_;  // not owned.
   std::unique_ptr<CancelHandle> cancel_handle_;
   std::vector<int> sleep_millis_used_;
 };

 // Tests that when the ShufflerClient returns OK the first time then the
 // Retryer returns OK and does not retry.
 TEST_F(SendRetryerTest, ReturnsOkIn1) {
   auto status = SendToShuffler();
   // Expect 1 call with a deadline of 10 seconds to return OK.
   CheckResults(status, grpc::OK, 1u, {10u});
 }

 // Tests that when the ShufflerClient retruns a non-retryable status code
 // the first time then the Retryer return it and does not retry.
 TEST_F(SendRetryerTest, ReturnsInvalidArgIn1) {
   shuffler_client_->statuses_to_return = {
       grpc::Status(grpc::INVALID_ARGUMENT, "Invalid Argument")};
   auto status = SendToShuffler();

   // Expect 1 call with a deadline of 10 seconds to return INVALID_ARGUMENT.
   CheckResults(status, grpc::INVALID_ARGUMENT, 1u, {10u});
 }

 // Tests that when the ShufflerClient returns ABORTED the first time and OK
 // the second time then the Retryer tries a total of 2 times and returns OK.
 TEST_F(SendRetryerTest, ReturnsAbortedThenOk) {
   shuffler_client_->statuses_to_return = {
       grpc::Status(grpc::ABORTED, "Aborted"), grpc::Status::OK};
   auto status = SendToShuffler(std::chrono::seconds::max());

   // Expect 2 call with deadlines seconds {10, 10} to return OK.
   CheckResults(status, grpc::OK, 2u, {10u, 10u});
 }

 // Tests that when the ShufflerClient returns UNAVAILABLE the first time and
 // INVALID_ARGUMENT the second time then the Retryer tries a total of 2 times
 // and returns INVALID_ARGUMENT.
 TEST_F(SendRetryerTest, ReturnsUnavailableThenInvalidArgument) {
   shuffler_client_->statuses_to_return = {
       grpc::Status(grpc::UNAVAILABLE, "UNAVAILABLE"),
       grpc::Status(grpc::INVALID_ARGUMENT, "Invalid Argument")};
   auto status = SendToShuffler(std::chrono::seconds::max());

   // Expect 2 call with deadlines seconds {10, 10} to return INVALID_ARGUMENT.
   CheckResults(status, grpc::INVALID_ARGUMENT, 2u, {10u, 10u});
 }

 // Tests that when the ShufflerClient returns ABORTED the first time and
 // then INTERNAL the second time and then OK the third time then the Retryer
 // tries a total of 3 times and returns OK.
 TEST_F(SendRetryerTest, ReturnsAbortedThenInternalThenOk) {
   shuffler_client_->statuses_to_return = {
       grpc::Status(grpc::ABORTED, "Aborted"),
       grpc::Status(grpc::INTERNAL, "Internal"), grpc::Status::OK};
   auto status = SendToShuffler(std::chrono::seconds::max());

   // Expect 3 call with deadlines seconds {10, 10, 10} to return OK.
   CheckResults(status, grpc::OK, 3u, {10u, 10u, 10u});
 }

 // Tests that when the ShufflerClient returns DEADLINE_EXCEEDED multiple times
 // then OK, the Retryer increases the RPC deadline by a factor of 1.5 each time.
 TEST_F(SendRetryerTest, ReturnsDeadlineExceededTwiceThenOk) {
   shuffler_client_->statuses_to_return = {
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status::OK};
   auto status = SendToShuffler(std::chrono::seconds::max());

   CheckResults(status, grpc::OK, 5u, {10u, 15u, 23u, 35u, 53u});
 }

 // Tests that the Retryer quits when the overall deadline is reached.
 TEST_F(SendRetryerTest, DeadlineExceededAfterOne) {
   // Each time clock_->now() is invoked it will be 10 seconds later than
   // the previous time.
   incrementing_clock_->set_increment(std::chrono::seconds(10));

   // Instruct the FakeShufflerClient to return firt DEADLINE_EXCEEDED and
   // then OK. But it will never get a chance to return OK because it will
   // only be invoked once.
   shuffler_client_->statuses_to_return = {
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status::OK};

   // Invoke SendToShuffler() with an overall deadline of 21s.
   // Note that SendToShuffler() invokes clock_->now() twice in the while loop:
   // once before the send and once after the send. So after the first send it
   // will be 20 (simulated) seconds later than the start time and the overall
   // deadline will be within one second of expring and so there won't be a
   // second send.
   auto status = SendToShuffler(std::chrono::seconds(21));

   // After the first DEADLINE_EXCEEDED the Retryer will give up and
   // return DEADLINE_EXCEEDED. We expect only one attempt with a gRPC
   // deadline of 10s.
   CheckResults(status, grpc::DEADLINE_EXCEEDED, 1u, {10u});
 }

 // Tests that the Retryer quits when the overall deadline is reached.
 TEST_F(SendRetryerTest, DeadlineExceededAfterTwo) {
   // Each time clock_->now() is invoked it will be 10 seconds later than
   // the previous time.
   incrementing_clock_->set_increment(std::chrono::seconds(10));

   // Instruct the FakeShufflerClient to return DEADLINE_EXCEEDED twice and
   // then OK. But it will never get a chance to return OK because it will
   // only be invoked twice.
   shuffler_client_->statuses_to_return = {
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status::OK};

   // Invoke SendToShuffler() with an overall deadline of 25s.
   // Note that SendToShuffler() invokes clock_->now() twice in the while loop:
   // once before the send and once after the send. So after the first send it
   // will be 20 (simulated) seconds later than the start time and there will
   // be 5 seconds left for the overall deadline. The sleep times we are using
   // are negligable so the expected rpc timeout for the second send is 5s.
   auto status = SendToShuffler(std::chrono::seconds(25));

   // After the second DEADLINE_EXCEEDED the Retryer will give up and return
   // DEADLINE_EXCEEDED. We expecte two attempts with gRPC deadlines of 10s
   // and 5s respectively.
   CheckResults(status, grpc::DEADLINE_EXCEEDED, 2u, {10u, 5u});
 }

 // Tests that when the ShufflerClient returns DEADLINE_EXCEEDED multiple times
 // then OK, the Retryer increases the RPC deadline by a factor of 1.5 each time.
 TEST_F(SendRetryerTest, TestCancel) {
   // Instruct the FakeShufflerClient to return DEADLINE_EXCEEDED 4 times and
   // then return OK.
   shuffler_client_->statuses_to_return = {
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
       grpc::Status::OK};
   // But also instruct it to invoke TryCancel() on the CancelHandle after
   // the second call to Send().
   shuffler_client_->cancel_on_this_call_count = 2;

   auto status = SendToShuffler(std::chrono::seconds::max());

   // We expect Send() to have been invoked twice with deadlines of 10s and 15,
   // and then for the Retryer to notice the cancellation and return CANCELLED.
   CheckResults(status, grpc::CANCELLED, 2u, {10u, 15u});
 }

 }  // namespace send_retryer
 }  // namespace encoder
 }  // namespace cobalt
	// Copyright 2017 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	#include "encoder/send_retryer.h"

	#include <chrono>
	#include <memory>
	#include <string>
	#include <utility>
	#include <vector>

	#include "./encrypted_message.pb.h"
	#include "./gtest.h"
	#include "./logging.h"
	#include "encoder/shuffler_client.h"
	#include "third_party/gflags/include/gflags/gflags.h"
	#include "util/clock.h"

	namespace cobalt {
	namespace encoder {
	namespace send_retryer {

	using util::IncrementingClock;

	namespace {

	// An implementation of ShufflerClientInterface that returns the sequence
	// of statuses it is told to return and records the number of times that
	// SendToShuffler() was invoked and the gRPC deadlines in each invocation.
	// Additionally, the client will optionally invoke cancel_handle->TryCancel() on
	// a specified invocation count number.
	class FakeShufflerClient : public ShufflerClientInterface {
	public:
	FakeShufflerClient(IncrementingClock* incrementing_clock,
	CancelHandle* cancel_handle)
	: incrementing_clock(incrementing_clock), cancel_handle(cancel_handle) {}

	~FakeShufflerClient() {}

	grpc::Status SendToShuffler(const EncryptedMessage& encrypted_message,
	grpc::ClientContext* context = nullptr) override {
	call_count++;
	CHECK(context);
	// The gRPC deadline embedded in \|context\| is expressed as an absolute
	// deadline. We recover the value of \|rpc_deadline\| set by the
	// Retryer by subtracting the clock's current time using \|peek_now()\|.
	// This is a bit fragile: It works only because we know that there have
	// been no invocations of clock_->now() between the time that the the
	// Retryer computed the deadline and the call to this function. We
	// record the deadline as a number of milliseconds.
	deadlines.push_back(
	std::chrono::duration_cast<std::chrono::milliseconds>(
	context->deadline() - incrementing_clock->peek_now())
	.count());

	CHECK(call_count <= statuses_to_return.size());
	if (call_count == cancel_on_this_call_count) {
	cancel_handle->TryCancel();
	}
	return statuses_to_return[call_count - 1];
	}

	IncrementingClock* incrementing_clock; // not owned.
	std::vector<grpc::Status> statuses_to_return = {grpc::Status::OK};
	size_t call_count = 0;
	std::vector<int64_t> deadlines;
	size_t cancel_on_this_call_count = -1;
	CancelHandle* cancel_handle;
	};

	} // namespace

	class SendRetryerTest : public ::testing::Test {
	public:
	SendRetryerTest() {
	std::unique_ptr<IncrementingClock> clock(new IncrementingClock());
	incrementing_clock_ = clock.get();
	cancel_handle_.reset(new CancelHandle());
	cancel_handle_->sleep_notification_function_ = [this](int sleep_millis) {
	sleep_millis_used_.push_back(sleep_millis);
	};
	shuffler_client_.reset(
	new FakeShufflerClient(incrementing_clock_, cancel_handle_.get()));
	retryer_.reset(new SendRetryer(shuffler_client_.get()));
	retryer_->clock_ = std::move(clock);
	retryer_->initial_sleep_ = std::chrono::milliseconds(1);
	}

	protected:
	// Invokes SendToShuffler() with an initial_rpc_deadline of 10 seconds,
	// the given overall_deadline (also defaulting to 10 seconds), and our fixed
	// cancel_handle and encrypted_message.
	grpc::Status SendToShuffler(
	std::chrono::seconds overerall_deadline = std::chrono::seconds(10)) {
	return retryer_->SendToShuffler(std::chrono::seconds(10),
	overerall_deadline, cancel_handle_.get(),
	encrypted_message_);
	}

	// Checks that the status, call_count and deadlines are as expected.
	void CheckResults(grpc::Status status, grpc::StatusCode expected_code,
	uint32_t expected_call_count,
	std::vector<int> expected_deadline_seconds) {
	EXPECT_EQ(expected_code, status.error_code());
	EXPECT_EQ(expected_call_count, shuffler_client_->call_count);
	ASSERT_EQ(expected_call_count, shuffler_client_->deadlines.size());
	ASSERT_EQ(expected_call_count, expected_deadline_seconds.size());
	for (size_t i = 0; i < expected_call_count; i++) {
	EXPECT_EQ(1000 * expected_deadline_seconds[i],
	shuffler_client_->deadlines[i])
	<< "i=" << i << ", expected_seconds=" << expected_deadline_seconds[i]
	<< ", actual_millis=" << shuffler_client_->deadlines[i];
	}
	// Check that the sleep times between send attempts started at the
	// expected initial value and doubled each time.
	ASSERT_EQ(expected_call_count - 1, sleep_millis_used_.size());
	// expected_sleep_millis is initialized to 1 because in the constructor for
	// SendRetryierTest we invoked
	// retryer_->initial_sleep_ = std::chrono::milliseconds(1);
	int expected_sleep_millis = 1;
	for (size_t i = 0; i < expected_call_count - 1; i++) {
	EXPECT_EQ(expected_sleep_millis, sleep_millis_used_[i]);
	expected_sleep_millis *= 2;
	}
	}

	std::unique_ptr<FakeShufflerClient> shuffler_client_;
	std::unique_ptr<SendRetryer> retryer_;
	EncryptedMessage encrypted_message_;
	IncrementingClock* incrementing_clock_; // not owned.
	std::unique_ptr<CancelHandle> cancel_handle_;
	std::vector<int> sleep_millis_used_;
	};

	// Tests that when the ShufflerClient returns OK the first time then the
	// Retryer returns OK and does not retry.
	TEST_F(SendRetryerTest, ReturnsOkIn1) {
	auto status = SendToShuffler();
	// Expect 1 call with a deadline of 10 seconds to return OK.
	CheckResults(status, grpc::OK, 1u, {10u});
	}

	// Tests that when the ShufflerClient retruns a non-retryable status code
	// the first time then the Retryer return it and does not retry.
	TEST_F(SendRetryerTest, ReturnsInvalidArgIn1) {
	shuffler_client_->statuses_to_return = {
	grpc::Status(grpc::INVALID_ARGUMENT, "Invalid Argument")};
	auto status = SendToShuffler();

	// Expect 1 call with a deadline of 10 seconds to return INVALID_ARGUMENT.
	CheckResults(status, grpc::INVALID_ARGUMENT, 1u, {10u});
	}

	// Tests that when the ShufflerClient returns ABORTED the first time and OK
	// the second time then the Retryer tries a total of 2 times and returns OK.
	TEST_F(SendRetryerTest, ReturnsAbortedThenOk) {
	shuffler_client_->statuses_to_return = {
	grpc::Status(grpc::ABORTED, "Aborted"), grpc::Status::OK};
	auto status = SendToShuffler(std::chrono::seconds::max());

	// Expect 2 call with deadlines seconds {10, 10} to return OK.
	CheckResults(status, grpc::OK, 2u, {10u, 10u});
	}

	// Tests that when the ShufflerClient returns UNAVAILABLE the first time and
	// INVALID_ARGUMENT the second time then the Retryer tries a total of 2 times
	// and returns INVALID_ARGUMENT.
	TEST_F(SendRetryerTest, ReturnsUnavailableThenInvalidArgument) {
	shuffler_client_->statuses_to_return = {
	grpc::Status(grpc::UNAVAILABLE, "UNAVAILABLE"),
	grpc::Status(grpc::INVALID_ARGUMENT, "Invalid Argument")};
	auto status = SendToShuffler(std::chrono::seconds::max());

	// Expect 2 call with deadlines seconds {10, 10} to return INVALID_ARGUMENT.
	CheckResults(status, grpc::INVALID_ARGUMENT, 2u, {10u, 10u});
	}

	// Tests that when the ShufflerClient returns ABORTED the first time and
	// then INTERNAL the second time and then OK the third time then the Retryer
	// tries a total of 3 times and returns OK.
	TEST_F(SendRetryerTest, ReturnsAbortedThenInternalThenOk) {
	shuffler_client_->statuses_to_return = {
	grpc::Status(grpc::ABORTED, "Aborted"),
	grpc::Status(grpc::INTERNAL, "Internal"), grpc::Status::OK};
	auto status = SendToShuffler(std::chrono::seconds::max());

	// Expect 3 call with deadlines seconds {10, 10, 10} to return OK.
	CheckResults(status, grpc::OK, 3u, {10u, 10u, 10u});
	}

	// Tests that when the ShufflerClient returns DEADLINE_EXCEEDED multiple times
	// then OK, the Retryer increases the RPC deadline by a factor of 1.5 each time.
	TEST_F(SendRetryerTest, ReturnsDeadlineExceededTwiceThenOk) {
	shuffler_client_->statuses_to_return = {
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status::OK};
	auto status = SendToShuffler(std::chrono::seconds::max());

	CheckResults(status, grpc::OK, 5u, {10u, 15u, 23u, 35u, 53u});
	}

	// Tests that the Retryer quits when the overall deadline is reached.
	TEST_F(SendRetryerTest, DeadlineExceededAfterOne) {
	// Each time clock_->now() is invoked it will be 10 seconds later than
	// the previous time.
	incrementing_clock_->set_increment(std::chrono::seconds(10));

	// Instruct the FakeShufflerClient to return firt DEADLINE_EXCEEDED and
	// then OK. But it will never get a chance to return OK because it will
	// only be invoked once.
	shuffler_client_->statuses_to_return = {
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status::OK};

	// Invoke SendToShuffler() with an overall deadline of 21s.
	// Note that SendToShuffler() invokes clock_->now() twice in the while loop:
	// once before the send and once after the send. So after the first send it
	// will be 20 (simulated) seconds later than the start time and the overall
	// deadline will be within one second of expring and so there won't be a
	// second send.
	auto status = SendToShuffler(std::chrono::seconds(21));

	// After the first DEADLINE_EXCEEDED the Retryer will give up and
	// return DEADLINE_EXCEEDED. We expect only one attempt with a gRPC
	// deadline of 10s.
	CheckResults(status, grpc::DEADLINE_EXCEEDED, 1u, {10u});
	}

	// Tests that the Retryer quits when the overall deadline is reached.
	TEST_F(SendRetryerTest, DeadlineExceededAfterTwo) {
	// Each time clock_->now() is invoked it will be 10 seconds later than
	// the previous time.
	incrementing_clock_->set_increment(std::chrono::seconds(10));

	// Instruct the FakeShufflerClient to return DEADLINE_EXCEEDED twice and
	// then OK. But it will never get a chance to return OK because it will
	// only be invoked twice.
	shuffler_client_->statuses_to_return = {
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status::OK};

	// Invoke SendToShuffler() with an overall deadline of 25s.
	// Note that SendToShuffler() invokes clock_->now() twice in the while loop:
	// once before the send and once after the send. So after the first send it
	// will be 20 (simulated) seconds later than the start time and there will
	// be 5 seconds left for the overall deadline. The sleep times we are using
	// are negligable so the expected rpc timeout for the second send is 5s.
	auto status = SendToShuffler(std::chrono::seconds(25));

	// After the second DEADLINE_EXCEEDED the Retryer will give up and return
	// DEADLINE_EXCEEDED. We expecte two attempts with gRPC deadlines of 10s
	// and 5s respectively.
	CheckResults(status, grpc::DEADLINE_EXCEEDED, 2u, {10u, 5u});
	}

	// Tests that when the ShufflerClient returns DEADLINE_EXCEEDED multiple times
	// then OK, the Retryer increases the RPC deadline by a factor of 1.5 each time.
	TEST_F(SendRetryerTest, TestCancel) {
	// Instruct the FakeShufflerClient to return DEADLINE_EXCEEDED 4 times and
	// then return OK.
	shuffler_client_->statuses_to_return = {
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status(grpc::DEADLINE_EXCEEDED, "DEADLINE_EXCEEDED"),
	grpc::Status::OK};
	// But also instruct it to invoke TryCancel() on the CancelHandle after
	// the second call to Send().
	shuffler_client_->cancel_on_this_call_count = 2;

	auto status = SendToShuffler(std::chrono::seconds::max());

	// We expect Send() to have been invoked twice with deadlines of 10s and 15,
	// and then for the Retryer to notice the cancellation and return CANCELLED.
	CheckResults(status, grpc::CANCELLED, 2u, {10u, 15u});
	}

	} // namespace send_retryer
	} // namespace encoder
	} // namespace cobalt