zircon/system/ulib/io-scheduler/include/io-scheduler/io-scheduler.h

// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#pragma once

#include <memory>
#include <stdint.h>
#include <vector>

#include <fbl/condition_variable.h>
#include <fbl/function.h>
#include <fbl/intrusive_double_list.h>
#include <fbl/mutex.h>
#include <fbl/vector.h>
#include <zircon/types.h>

#include <io-scheduler/stream.h>
#include <io-scheduler/stream-op.h>
#include <io-scheduler/worker.h>

namespace ioscheduler {

// Reordering rules for the scheduler.
// Allow reordering of Read class operations with respect to each other.
constexpr uint32_t kOptionReorderReads  = (1u << 0);

// Allow reordering of Write class operations with respect to each other.
constexpr uint32_t kOptionReorderWrites = (1u << 1);

// Allow reordering of Read class operations ahead of Write class operations.
constexpr uint32_t kOptionReorderReadsAheadOfWrites = (1u << 2);

// Allow reordering of Write class operations ahead of Read class operations.
constexpr uint32_t kOptionReorderWritesAheadOfReads = (1u << 3);

// Disallow any reordering.
constexpr uint32_t kOptionStrictlyOrdered = 0;

// Allow all reordering options.
constexpr uint32_t kOptionFullyOutOfOrder = (kOptionReorderReads |
                                             kOptionReorderWrites |
                                             kOptionReorderReadsAheadOfWrites |
                                             kOptionReorderWritesAheadOfReads);

// Maximum priority for a stream.
constexpr uint32_t kMaxPriority = 31;

// Suggested default priority for a stream.
constexpr uint32_t kDefaultPriority = 8;

// Callback interface from Scheduler to client. Callbacks are made from within
// the Scheduler library to the client implementation. All callbacks are made
// with no locks held and are allowed to block. Any callbacks may be invoked
// simultanously, and one may be called multiple times concurrently, but never
// with the same data. Notably, Acquire(), Issue(), and Release() may be called
// multiple times after CancelAcquire() has been called.
class SchedulerClient {
public:
    // CanReorder
    //   Compare if ops can be reordered with respect to each other. This
    // function is called for every pair of ops whose position in
    // the stream is being considered for reorder relative to each other.
    // Returns:
    //   true if it is safe to reorder |second| ahead of |first|.
    //   false otherwise.
    virtual bool CanReorder(StreamOp* first, StreamOp* second) = 0;

    // Acquire
    //   Read zero or more ops from the client for intake into the
    // Scheduler. Every op obtained through Acquire will be returned to the client
    // via the Release callback. The Scheduler will never attempt to free these pointers.
    // Args:
    //   sop_list - an empty array of op pointers to be filled.
    //   list_count - number of entries in sop_list
    //   actual_count - the number of entries filled in sop_list.
    //   wait - block until data is available if true.
    // Returns:
    //   ZX_OK if one or more ops have been added to the list.
    //   ZX_ERR_CANCELED if op source has been closed.
    //   ZX_ERR_SHOULD_WAIT if ops are currently unavailable and |wait| is
    //     false.
    virtual zx_status_t Acquire(StreamOp** sop_list, size_t list_count, size_t* actual_count,
                                bool wait) = 0;

    // Issue
    //   Deliver an op to the IO hardware for immediate execution. This
    // function may block until the op is completed. If it does not block,
    // it should return ZX_ERR_ASYNC.
    // Args:
    //   sop - op to be completed.
    // Returns:
    //   ZX_OK if the op has been completed synchronously or it has failed to
    // issue due to bad parameters in the operation. The callee should update
    // the op’s result field to reflect the success or failure status of the
    // op.
    //   ZX_ERR_ASYNC if the op has been issued for asynchronous completion.
    // Notification of completion should be delivered via the Scheduler’s
    // AsyncComplete() API.
    //   Other error status describing the internal failure that has caused
    // the issue to fail.
    virtual zx_status_t Issue(StreamOp* sop) = 0;

    // Release
    //   Yield ownership of the operation. The completion status of the op
    // is available in its |result| field. Once released, the Scheduler
    // maintains no references to the op and it can be safely deallocated or
    // reused.
    // Args:
    //   sop - op to be released.
    virtual void Release(StreamOp* sop) = 0;

    // CancelAcquire
    //   Cancels any pending blocking calls to Acquire. No further reading of
    // ops should be done. Blocked Acquire callers and any subsequent Acquire
    // calls should return ZX_ERR_CANCELED.
    virtual void CancelAcquire() = 0;

    // Fatal
    //   The Scheduler has encountered a fatal asynchronous error. All pending
    // ops have been aborted. The Scheduler should be shut down and destroyed.
    // The shutdown should be performed from a different context than that of
    // the Fatal() call or else it may deadlock.
    virtual void Fatal() = 0;
};

class Scheduler {
public:
    Scheduler() {}
    ~Scheduler();
    DISALLOW_COPY_ASSIGN_AND_MOVE(Scheduler);

    // Client API - synchronous calls.
    // -------------------------------

    // Initialize a Scheduler object to usable state. Initialize must be called on
    // a newly created Scheduler object or Scheduler that has been shut down
    // before it can be used.
    // The Scheduler holds a pointer to |client| until Shutdown() has returned. It does not
    // manage the lifetime of this pointer and does not free it.
    zx_status_t Init(SchedulerClient* client, uint32_t options) __TA_EXCLUDES(stream_lock_);

    // Open a new stream with the requested ID and priority. It is safe to invoke
    // this function from a Scheduler callback context, except from Fatal().
    // |id| may not be that of a currently open stream.
    // |priority| must be in the inclusive range 0 to kMaxPriority.
    // Returns:
    // ZX_OK on success.
    // ZX_ERR_ALREADY_EXISTS if stream with same |id| is already open.
    // ZX_ERR_INVALID_ARGS if |priority| is out of range.
    // Other error status for internal errors.
    zx_status_t StreamOpen(uint32_t id, uint32_t priority) __TA_EXCLUDES(stream_lock_);

    // Close an open stream. All ops in the stream will be issued before the stream
    // is closed. New incoming ops to the closed stream will be released with
    // an error.
    zx_status_t StreamClose(uint32_t id) __TA_EXCLUDES(stream_lock_);

    // Begin scheduler service. This creates the worker threads that will invoke
    // the callbacks in SchedulerCallbacks.
    zx_status_t Serve() __TA_EXCLUDES(stream_lock_);

    // End scheduler service. This function blocks until all outstanding ops in
    // all streams are completed and closes all streams. Shutdown should not be invoked from a
    // callback function. To reuse the scheduler, call Init() again.
    void Shutdown() __TA_EXCLUDES(stream_lock_);


    // Client API - asynchronous calls.
    // --------------------------------

    // Asynchronous completion. When an issued operation has completed
    // asynchronously, this function should be called. The status of the operation
    // should be set in |sop|’s result field. This function is non-blocking and
    // safe to call from an interrupt handler context.
    void AsyncComplete(StreamOp* sop) __TA_EXCLUDES(stream_lock_);

    // API invoked by worker threads.
    // --------------------------------
    SchedulerClient* client() { return client_; }

    // Insert a list of ops into the scheduler queue.
    //
    // Ownership:
    //    Ops are exclusively retained by the Scheduler if they were successfully enqueued. Ops that
    // encounter enqueueing errors will be added to |out_list| for caller to release.
    //
    // |in_list| and |out_list| may point to the same buffer.
    // |out_num_ready| is an optional parameter that returns how many ops are ready to be dequeued
    //    in all streams.
    zx_status_t Enqueue(UniqueOp* in_list, size_t in_count,
                        UniqueOp* out_list, size_t* out_actual,
                        size_t* out_num_ready = nullptr) __TA_EXCLUDES(stream_lock_);

    // Remove an op from the scheduler queue.
    //
    // Ownership:
    //    If successful, ownership of the op is transferred to the caller.
    //
    // If no ops are available:
    //      returns ZX_ERR_CANCELED if shutdown has started.
    //      returns ZX_ERR_SHOULD_WAIT if |wait| is false.
    //      otherwise returns ZX_ERR_SHOULD_WAIT.
    zx_status_t Dequeue(UniqueOp* op_out, bool wait) __TA_EXCLUDES(stream_lock_);

private:
    using StreamIdMap = Stream::WAVLTreeSortById;
    using StreamList = Stream::ListUnsorted;

    zx_status_t FindStreamLocked(uint32_t id, StreamRef* out) __TA_REQUIRES(stream_lock_);

    SchedulerClient* client_ = nullptr; // Client-supplied callback interface.
    uint32_t options_ = 0;              // Ordering options.

    fbl::Mutex stream_lock_;
    // Set when shutdown has been called and workers should exit.
    bool shutdown_initiated_ __TA_GUARDED(stream_lock_) = true;
    // Number of existing streams.
    uint32_t num_streams_ __TA_GUARDED(stream_lock_) = 0;
    // Number of streams that have ops that need to be issued or completed.
    uint32_t active_streams_ __TA_GUARDED(stream_lock_) = 0;
    // Total number of acquired ops in all streams.
    uint32_t acquired_ops_ __TA_GUARDED(stream_lock_) = 0;
    // Map of id to stream. Contains all streams.
    StreamIdMap stream_map_ __TA_GUARDED(stream_lock_);
    // List of streams that have ops ready to be scheduled.
    StreamList active_list_ __TA_GUARDED(stream_lock_);
    // Event notifying worker threads that active streams are available.
    fbl::ConditionVariable active_available_ __TA_GUARDED(stream_lock_);

    fbl::Vector<fbl::unique_ptr<Worker>> workers_;
};

} // namespace ioscheduler