sdk/fidl/fuchsia.media/stream_processor.fidl

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

library fuchsia.media;

// See stream_processor.md for detailed interface documentation. The comments
// here are a summary only.  Client implementers should see stream_processor.md
// for more detail on any message that doesn't seem sufficiently-described
// here. StreamProcessor server implementers should probably read
// stream_processor.md before implementing.

// Overview of operation:
//
// 1. Create
//   * create via CodecFactory - see CodecFactory
//   * create via LicenseSession - see LicenseSession
// 2. Get input constraints
//   * OnInputConstraints() - sent unsolicited by stream processor shortly after
//     stream processor creation.
// 3. Provide input buffers
//   * SetInputBufferSettings() / AddInputBuffer()
// 4. Deliver input data
//   * QueueInputPacket() + OnFreeInputPacket(), for as long as it takes,
//     possibly working through all input packets repeatedly before...
// 5. Get output constraints and format
//   * OnOutputConfig() - may be delivered as early as before
//     OnInputConstraints() by some stream processors, but a client must
//     tolerate as late as after substantial input data has been delivered
//     including lots of input packet recycling via OnFreeInputPacket().
//   * This message can arrive more than once before the first output data.
// 6. Provide output buffers
//   * SetOutputBufferSettings() / AddOutputBuffer()
// 7. Data flows, with optional EndOfStream
//   * OnOutputPacket() / RecycleOutputPacket() / QueueInputPacket() /
//     OnFreeInputPacket() / QueueInputEndOfStream() / OnOutputEndOfStream()
//
// It's possible to re-use a StreamProcessor instance for another stream, and
// doing so can sometimes skip over re-allocation of buffers. This can be a
// useful thing to do for cases like seeking to a new location - at the
// StreamProcessor interface that can look like switching to a new stream.

// StreamBufferConstraints
//
// This struct helps ensure that packet count and buffer space are sufficient
// to avoid major problems.  For example, a video decoder needs sufficient
// video frame buffers to hold all potential reference frames concurrently +
// one more video buffer to decode into.  Else, the whole video decode pipe can
// easily deadlock.
//
// The secondary purpose of this struct is to help ensure that packet count and
// buffer space are sufficient to achieve reasonably performant operation.
//

// There are separate instances of this struct for stream input and stream
// output.
table StreamBufferConstraints {
    // This is a version number the server sets on the constraints to allow the
    // server to determine when the client has caught up with the latest
    // constraints sent by the server.  The server won't emit output data until
    // the client has configured output settings and buffers with a
    // buffer_constraints_version_ordinal >= the latest
    // buffer_constraints_version_ordinal that had
    // buffer_constraints_action_required true.  See
    // buffer_constraints_action_required comments for more.
    //
    // A buffer_constraints_version_ordinal of 0 is not permitted, to simplify
    // initial state handling.  Other than 0, both odd and even version ordinals
    // are allowed (in contrast to the stream_lifetime_ordinal, neither the
    // client nor server ever has a reason to consider the latest version to be
    // stale, so there would be no benefit to disallowing even values).
    1: uint64 buffer_constraints_version_ordinal;

    // default_settings
    //
    // These settings are "default" settings, not "recommended" settings.
    //
    // These "default" settings can be passed to SetInputBufferSettings() /
    // SetOutputBufferSettings() as-is without modification, but a client doing
    // that must still obey the semantics of packet_count_for_client, despite
    // the stream processor server not having any way to really know the proper
    // setting for that field.
    //
    // For StreamBufferConstraints fields whose names end in "recommended", the
    // default_settings will have the corresponding setting field set to that
    // recommended value.
    //
    // The stream processor promises that these default settings as-is (except
    // for buffer_lifetime_ordinal) are guaranteed to satisfy the constraints
    // indicated by the other fields of StreamBufferConstraints.  While
    // client-side checking that these settings are within the constraints is
    // likely unnecessary in the client, the client should still check that
    // these values are within client-side reasonable-ness bounds before using
    // these values, to avoid letting a stream processor server cause problems
    // for the client.
    //
    // This structure will always have single_buffer_mode false.  See
    // single_buffer_mode_allowed for whether single_buffer_mode true is
    // allowed.
    //
    // The client must set the buffer_lifetime_ordinal field to a proper value
    // before sending back to the server.  The 0 initially in this field will
    // be rejected by the server if sent back as-is.  See comments on
    // StreamBufferSettings.buffer_lifetime_ordinal.
    2: StreamBufferSettings default_settings;

    // For uncompressed video, separate and complete frames, each in its own
    // separate buffer (buffer-per-packet mode), is always a requirement.

    // per_packet_buffer_bytes.*:
    //
    // These per-packet buffer bytes constraints apply to both buffer-per-packet
    // mode and single-buffer mode (see single_buffer_mode).  If
    // buffer-per-packet mode, the constraints apply to each buffer separately.
    // If single-buffer mode, the constraints need to be multiplied by the number
    // of packets to determine the constraints on the single buffer.

    // per_packet_buffer_bytes_min:
    //
    // If a client is using buffer per packet mode, each buffer must be at least
    // this large.  If a client is using single-buffer mode, the one buffer must
    // be at least per_packet_buffer_bytes_min * packet_count_for_server_min in
    // size.
    3: uint32 per_packet_buffer_bytes_min;
    // Must be >= per_packet_buffer_bytes_min.  Delivering more than
    // this per input packet might not perform any better, and in fact might
    // perform worse.
    4: uint32 per_packet_buffer_bytes_recommended;
    // Must be >= per_packet_buffer_bytes_recommended.  Can be 0xFFFFFFFF if
    // there is no explicitly-enforced limit.
    5: uint32 per_packet_buffer_bytes_max;

    // Minimum number of packet_count_for_server.
    //
    // Re. input and output:
    //
    // This is a strict min for packet_count_for_server, but a client can use
    // more packets overall if the client wants to, by using a larger value for
    // packet_count_for_server and/or using a non-zero packets_for_client.  A
    // good reason to do the former would be if the client might tend to
    // deliver a few not-very-full buffers occasionally - or to have a few
    // extra packets within which to satisfy stream_input_bytes_min.  A good
    // reason to do the latter would be if a client needs to hold onto some
    // packets for any "extra" duration.
    //
    // If a client specifies a larger packet_count_for_server value than
    // packet_count_for_server_min, a server is permitted (but not encouraged)
    // to not make progress until packet_count_for_server are with the server,
    // not merely packet_count_for_server_min.
    //
    // For decoder input and audio encoder input: The
    // packet_count_for_server_min may or may not contain enough data to allow
    // the stream processor to make progress without copying into an internal
    // side buffer.  If there isn't enough data delivered in
    // packet_count_for_server_min packets to permit progress, the stream
    // processor must copy into its own side buffer internally to make
    // progress.
    //
    // If a client intends to use extra packets for client-side purposes, the
    // client should specify the extra packets in packets_for_client instead of
    // packet_count_for_server, but packet_count_for_server must still be >=
    // packet_count_for_server_min.
    //
    // See stream_processor.md for more on packet_count_for_server_min.
    6: uint32 packet_count_for_server_min;

    // This must be at least packet_count_for_server_min and at most
    // packet_count_for_server_recommended_max.
    //
    // This value is likely to be used as-is by most clients, so if having one
    // additional packet is a big performance win in a large percentage of
    // scenarios, it can be good for the server to include that additional
    // packet in this value.
    7: uint32 packet_count_for_server_recommended;

    // This can be the same as packet_count_for_server_max or can be lower.
    // Values above this value and <= packet_count_for_server_max are not
    // recommended by the stream processor, but should still work given
    // sufficient resources available to both the client and the stream
    // processor.
    8: uint32 packet_count_for_server_recommended_max;

    // This can be 0xFFFFFFFF if there's no stream processor-enforced max, but
    // stream processors are encouraged to set a large but still
    // plausibly-workable max, and clients are encouraged to request a number
    // of packets that isn't excessively large for the client's scenario.
    9: uint32 packet_count_for_server_max;

    // Normally this would be an implicit 0, but for now we have a min so we can
    // force the total number of packets to be a specific number that we know
    // works for the moment.
    10: uint32 packet_count_for_client_min;

    // packet_count_for_client_max
    //
    // The client must set packet_count_for_client to be <=
    // packet_count_for_client_max.
    //
    // This value must be at least 1.  This can be 0xFFFFFFFF if there's no
    // stream-processor-enforced max.  Clients are encouraged to request a
    // number of packets that isn't excessively large for the client's
    // scenario.
    11: uint32 packet_count_for_client_max;

    // single_buffer_mode_allowed false allows a stream processor that's not
    // required to support single-buffer mode for a given input or output the
    // ability to decline to support single-buffer mode on that input/output.
    //
    // All encoder output, regardless of audio or video: server support for
    // single-buffer mode is optional.  Please see stream_processor.md for all
    // the rules regarding single-buffer mode on output before using
    // single-buffer mode on an output.
    //
    // Audio decoder output: server support for single-buffer mode is required.
    //
    // Video decoder output: There is little reason for a video decoder to
    // support single-buffer mode on output.  Nearly all video decoders will set
    // this to false for their output.
    //
    // All decoder inputs: Servers must support single-buffer mode on input.
    // The client is responsible for managing the input buffer space such that
    // filling an input packet doesn't over-write any portion of an input
    // packet already in flight to the stream processor.
    //
    // Encoder inputs: Server support for single-buffer mode on encoder input is
    // optional.  This is more often useful for audio than for video.
    //
    // Support for buffer-per-packet mode is always required on both input and
    // output, regardless of stream processor type.
    12: bool single_buffer_mode_allowed;

    // If true, the buffers need to be physically contiguous pages, such as can
    // be allocated using zx_vmo_create_contiguous().
    13: bool is_physically_contiguous_required;
    // VERY TEMPORARY HACK / KLUDGE - we want the BufferAllocator (or one of
    // the participant drivers that needs physically contiguous buffers) to
    // call zx_vmo_create_contiguous(), definitely not the StreamProcessor
    // client, but until the BufferAllocator can be reached from a driver, this
    // is to grant the client special powers it really shouldn't have, very
    // temporarily until BufferAllocator is hooked up properly at which point
    // this can be removed. Strictly speaking we could reverse which end
    // allocates buffers in the StreamProcessor interface to avoid this hack
    // even before BufferAllocator, but the overall path seems shorter if we
    // jump directly from this to using BufferAllocator.
    14: handle very_temp_kludge_bti_handle;
};

// StreamOutputConfig
//
// The stream-processor-controlled output configuration, including both
// StreamBufferConstraints for the output and FormatDetails for the output.
//

// TODO(dustingreen): Need a better name for this struct, but still short
// hopefully.  It's stuff the stream processor gets to control, not the client.
// It's different than output buffer settings, which the client does get to
// control to some extent.  It's different than any configurable output
// settings the client might specify for output of an encoder.
table StreamOutputConfig {
    // A client which always immediately re-configures output buffers on
    // receipt of OnOutputConfig() with buffer_constraints_action_required true
    // can safely ignore this field.

    // A client is permitted to ignore an OnOutputConfig() message even with
    // buffer_constraints_action_required true if the client knows the server
    // has already been told to discard the remainder of the stream with the
    // same stream_lifetime_ordinal or if this stream_lifetime_ordinal field is
    // set to 0.  The server is required to re-send needed output config via
    // OnOutputConfig() with new stream_lifetime_ordinal and
    // buffer_constraints_action_required true, if the most recent completed
    // server-side output config isn't what the server wants/needs yet for the
    // new stream.
    1: uint64 stream_lifetime_ordinal;

    // buffer_constraints_action_required
    //
    // When the buffer constraints are delivered, they indicate whether action
    // is required.  A false value here permits delivery of constraints which
    // are fresher without forcing a buffer reconfiguration.  If this is false,
    // a client cannot assume that it's safe to immediately re-configure output
    // buffers.  If this is true, the client can assume it's safe to
    // immediately configure output buffers once.
    //
    // A client is permitted to ignore buffer constraint versions which have
    // buffer_constraints_action_required false.  The server is not permitted
    // to change buffer_constraints_action_required from false to true for the
    // same buffer_constraints_version_ordinal.
    //
    // For each configuration, a client must use new buffers, never buffers
    // that were previously used for anything else, and never buffers
    // previously used for any other StreamProcessor purposes.  This rule
    // exists for multiple good reasons, relevant to both mid-stream changes,
    // and changes on stream boundaries. A client should just use new buffers
    // each time.
    //
    // When this is true, the server has already de-refed as many low-level
    // output buffers as the server can while still performing efficient
    // transition to the new buffers and will de-ref the rest asap.  A Sync()
    // is not necessary to achieve non-overlap of resource usage to the extent
    // efficiently permitted by the formats involved.
    //
    // If buffer_constraints_action_required is true, the server _must_ not
    // deliver more output data until after output buffers have been configured
    // (or re-configured) by the client.
    //
    // See stream_processor.md for more on buffer_constraints_action_required.
    2: bool buffer_constraints_action_required;
    3: StreamBufferConstraints buffer_constraints;

    // format_details
    //
    // It's up to the client to determine if a change in
    // format_details.format_details_version_ordinal implies any client action
    // is required, based on particular fields in format_details vs. any old
    // value. The server guarantees that if the format has changed, then
    // format_details.format_details_version_ordinal will change, but a change
    // to format_details.format_details_version_ordinal does not guarantee that
    // the format details actually changed.  Servers are strongly encouraged to
    // not change format_details.format_details_version_ordinal other than
    // before the first output data of a stream unless there is a real
    // mid-stream format change in the stream.  Unnecessary mid-stream format
    // changes can cause simpler clients that have no need to handle mid-stream
    // format changes to just close the channel.  Format changes before the
    // first output data of a stream are not "mid-stream" in this context -
    // those can be useful for stream format detection / setup reasons.
    //
    // Note that in case output buffers don't really need to be re-configured
    // despite a format change, a server is encouraged, but not required, to
    // set buffer_constraints_action_required false on the message that conveys
    // the new format details.  Simpler servers may just treat the whole output
    // situation as one big thing and demand output buffer reconfiguration on
    // any change in the output situation.
    //
    // A client may or may not actually handle a new buffer_constraints with
    // buffer_constraints_action_required false, but the client should always
    // track the latest format_details.
    //
    // An updated format_details is ordered with respect to emitted output
    // packets, and applies to all subsequent packets until the next
    // format_details with larger version_ordinal.  A simple client that does
    // not intend to handle mid-stream format changes should still keep track
    // of the most recently received format_details until the first output
    // packet arrives, then lock down the format details, handle those format
    // details, and verify that any
    // format_details.format_details_version_ordinal received from the server
    // is the same as the locked-down format_details, until the client is done
    // with the stream.  Even such a simple client must tolerate
    // format_details.format_details_version_ordinal changing multiple times
    // before the start of data output from a stream (any stream - the first
    // stream or a subsequent stream).  This allows a stream processor to
    // request that output buffers and output format be configured
    // speculatively, and for the output config to be optionally adjusted by
    // the server before the first data output from a stream after the server
    // knows everything it needs to know to fully establish the initial output
    // format details.  This simplifies stream processor server implementation,
    // and allows a clever stream processor server to guess it's output config
    // for lower latency before any input data, while still being able to fix
    // the output config (including format details) if the guess turns out to
    // be wrong.
    //
    // Whether the format_details.format_details_version_ordinal will actually
    // change mid-stream is a per-stream-processor and per-stream detail that
    // is not specified in comments here, and in most cases also depends on
    // whether the format changes on the input to the stream processor.
    // Probably it'll be fairly common for a client to use a format which
    // technically supports mid-stream format change, but the client happens to
    // know that none of the streams the client intends to process will ever
    // have a mid-stream format change.
    4: FormatDetails format_details;
};

// Default values for input and output
// StreamBufferConstraints.default_settings.packet_count_for_server.
//
// These are defined as "const" in FIDL to avoid all server implementations
// needing to separately define their own values, and these should be
// reasonable as default values, but strictly speaking this is not intended to
// promise that this value won't change from build to build.  If a client cares
// about a specific number, the client should separately define what that
// number is and ensure that StreamBufferSettings.packet_count_for_client is
// at least large enough.
//
// In contrast to packet_count_for_client, the packet_count_for_server is much
// more stream-processor-specific, so this file has no numbers for that - each
// stream processor will set those as appropriate for the specific stream
// processor.
//
// These are not "recommended" values, only "default" values, in the sense that
// the stream processor doesn't really know what the correct setting for these
// values is for a given client, and if the default is not appropriate for a
// client, large problems could result such as deadlock.  See the comments on
// packet_count_for_client.
//
// Despite these defaults, every client should ideally care about the
// packet_count_for_client setting and should ensure that the setting is at
// least large enough to cover the number of packets the client might ever need
// to camp on for any non-transient duration concurrently.  The defaults are
// only intended to be plausible for some clients, not all clients.
//
// One for the client to be filling and one in transit.
const uint32 kDefaultInputPacketCountForClient = 2;
// One for the client to be rendering, and one in transit.
const uint32 kDefaultOutputPacketCountForClient = 2;

// For input, this is the default on a fairly arbitrary basis.
//
// TODO(dustingreen): Do we want the default for audio encoding to be
// single_buffer_mode true instead?  If so, we may split this up by audio/video
// encoder/decoder.
const bool kDefaultInputIsSingleBufferMode = false;
const bool kDefaultOutputIsSingleBufferMode = false;

// StreamBufferSettings
//
// See relevant corresponding constraints in StreamBufferConstraints.  The
// settings must satisfy the constraints.
//

// The client informs the stream processor of these settings and then
// separately informs the stream processor of each buffer.
table StreamBufferSettings {
    // buffer_lifetime_ordinal
    //
    // The containing message starts a new buffer_lifetime_ordinal.
    //
    // There is a separate buffer_lifetime_ordinal for input vs. output.
    //
    // Re-use of the same value is not allowed.  Values must be odd.  Values
    // must only increase (increasing by more than 2 is permitted).
    //
    // A buffer_lifetime_ordinal lifetime starts at SetInputBufferSettings() or
    // SetOutputBufferSettings(), and ends at the earlier of
    // CloseCurrentStream() with release_input_buffers/release_output_buffers
    // set or SetOutputBufferSettings() with new buffer_lifetime_ordinal in the
    // case of mid-stream output config change.
    //
    // See stream_processor.md for more on buffer_lifetime_ordinal.
    1: uint64 buffer_lifetime_ordinal;

    // buffer_constraints_version_ordinal
    //
    // This value indicates which version of constraints the client is/was aware
    // of so far.
    //
    // For input, this must always be 0 because constraints don't change for
    // input (settings can change, but there's no settings vs current
    // constraints synchronization issue on input).
    //
    // For output, this allows the server to know when the client is
    // sufficiently caught up before the server will generate any more output.
    //
    // When there is no active stream, a client is permitted to re-configure
    // buffers again using the same buffer_constraints_version_ordinal.
    //
    // See stream_processor.md for more on buffer_constraints_version_ordinal.
    2: uint64 buffer_constraints_version_ordinal;

    // The total packet count is split into two pieces to disambiguate how many
    // packets are allocated for the client to hold onto for whatever reason,
    // vs. how many packets are allocated for the server to hold onto for
    // whatever reason.
    //
    // Extra packets to provide slack for performance reasons can be in either
    // category, but typically packet_count_for_server_recommended will already
    // include any performance-relevant slack for the server's benefit.

    // packet_count_for_server
    //
    // How many packets the client is allocating for the stream processor
    // server's use. This must be >=
    // StreamBufferConstraints.packet_count_for_server_min.  If constraints
    // change such that this would no longer be true, the server will send an
    // OnOutputConfig() event.
    //
    // The stream processor server is allowed to demand that all of
    // packet_count_for_server become free before making further progress, even
    // if packet_count_for_server is > packet_count_for_server_min.
    //
    // A reasonable value for this is
    // StreamBufferConstraints.packet_count_for_server_recommended.
    //
    // See stream_processor.md for more on packet_count_for_server.
    3: uint32 packet_count_for_server;

    // packet_count_for_client
    //
    // This must be at least 1.  The server will close the channel if this is 0.
    //
    // How many packets the client is allocating for the client's use.  The
    // client may hold onto this many packets for arbitrarily-long duration
    // without handing these packets to the stream processor, and despite doing
    // so, the stream processor will continue to make progress and function
    // normally without getting stuck.  The client holding onto additional
    // packets transiently is ok, but the client needs to hand those additional
    // packets back to the stream processor eventually if the client wants the
    // stream processor to make further progress.
    //
    // In addition to this value needing to include at least as many packets as
    // the client ever intends to concurrently camp on indefinitely, any extra
    // slack to benefit client-side performance should also be included here.
    //
    // A typical value for this could be at least 2, but it depends strongly on
    // client implementation and overall client buffering goals.  It is up to
    // the client to determine how many packets are needed in this category by
    // any parts of the overall system that will be holding onto packets for
    // any reason.  Those parts of the system should have a documented and
    // possibly queryable defined value to help determine this number.  Setting
    // this value lower than it actually needs to be can result in the stream
    // processor not making progress as it sits waiting for packets, with the
    // client unable to recycle any more packets to the stream processor.  That
    // situation can be difficult to diagnose, while excessively-large values
    // here are wasteful, so care is warranted to set this value properly.
    4: uint32 packet_count_for_client;

    // per_packet_buffer_bytes
    //
    // In buffer-per-packet mode, we require that each buffer have usable bytes
    // equal to per_packet_buffer_bytes.  Use of differently-sized low-level
    // buffers is possible, but the size of the portion used via the
    // StreamProcessor interface per StreamBuffer must be the same for all the
    // buffers.
    //
    // In single-buffer mode, we require the portion of the low-level buffer
    // used via the StreamProcessor interface to be size
    // (packet_count_for_server + packet_count_for_client) *
    // per_packet_buffer_bytes.
    //
    // TODO(dustingreen): determine if we need to relax these restrictions a bit
    // for convenience when using gralloc video buffers.
    5: uint32 per_packet_buffer_bytes;

    // If true, there is only one buffer with index 0 which all packets
    // implicitly refer to.  If false, the packet_index and buffer_index are
    // 1:1, and each packet refers to its corresponding buffer.
    //
    // While it's possible to set up single_buffer_mode false with each buffer
    // referring to the same underlying VMO, single_buffer_mode true is more
    // efficient for that case since only one mapping is created.
    6: bool single_buffer_mode;
};

// StreamBuffer
//
// The StreamBuffer struct represents a pre-configured buffer.
//
// Both input and output uses StreamBuffer(s), but the two sets of buffers are
// separate.
//
// The client uses SetInputBufferSettings() + AddInputBuffer() * N to inform
// the stream processor about all the input buffers.
//
// The client uses SetOutputBufferSettings() + AddOutputBuffer() * N to inform
// the stream processor about all the output buffers.
//

// When single_buffer_mode is true, there is only buffer_index 0 shared by all
// Packet(s) of the relevant input or output.  When single_buffer_mode is
// false, the buffer_index equals the packet_index.
table StreamBuffer {
    // When using AddOutputBuffer()/AddInputBuffer(), this must match the
    // buffer_lifetime_ordinal of the most recent
    // SetOutputBufferSettings()/SetInputBufferSettings().
    1: uint64 buffer_lifetime_ordinal;

    // Buffers must be added via AddOutputBuffer() / AddInputBuffer() in order
    // by buffer_index, and the buffer_index is always equal to 0 or equal to
    // the packet_index (depending on single_buffer_mode true or false), but
    // it's still nice to have StreamBuffer include the buffer_index if only
    // for easier debugging.
    2: uint32 buffer_index;

    // For each new buffer_lifetime_ordinal, a client must use new low-level
    // buffers.  This rule exists for multiple very good reasons, and is
    // relevant to mid-stream changes, changes on stream boundaries, and both
    // input and output buffers.  A new buffer_lifetime_ordinal needs new
    // low-level buffers, not just new StreamBuffer(s).  If you find yourself
    // copying compressed input data into new low-level input buffers solely
    // due to this rule, consider asking the source of the data for the ability
    // to directly fill new VMOs.  The rule exists for good reasons, even for
    // input buffers.
    //
    // The previous paragraph does not prohibit carving up VMOs into sub-pieces
    // and using different sub-pieces as different StreamBuffer(s), with some
    // VMOs used for more than one StreamBuffer and possibly others used for
    // only one StreamBuffer.  While this is permitted and enables some
    // optimizations, it's not expected to be particularly common.
    //
    // See stream_processor.md for more on StreamBufferData, and more on why we
    // never re-use the same low-level buffers for different
    // buffer_lifetime_ordinal(s).
    3: StreamBufferData data;
};

// StreamBufferData
//
// For the moment, a VMO per buffer is the only type of buffer.
//

// This is extremely likely to change significantly when adding gralloc stuff,
// but the idea with this union is to have a struct per logical way of storing
// the data.  Any multi-domain storage within a gralloc buffer will likely be
// only indirectly represented here.
union StreamBufferData {
    StreamBufferDataVmo vmo;

    // TODO(dustingreen): add the gralloc way
};

// StreamBufferDataVmo
//

// Details for a buffer backed by a VMO.
table StreamBufferDataVmo {
    // The same VMO can be used by more than one StreamBuffer (only of the same
    // buffer_lifetime_ordinal), but each vmo_handle must be a separate handle.
    1: handle<vmo> vmo_handle;

    // Offset within the VMO of the first usable byte.  Must be < the VMO's size
    // in bytes.
    2: uint64 vmo_usable_start;

    // VMO-relative offset that's one past the last usable byte.  This can point
    // one byte beyond the end of the VMO if desired.  In other words, this can
    // be equal to the VMO's size, to indicate that the last byte of the VMO is
    // usable (and possibly many byte before that, depending on
    // vmo_usable_start).
    3: uint64 vmo_usable_size;
};

// PacketHeader
//
// When referring to a free packet, we use PacketHeader alone instead of
// Packet, since while a packet is free it doesn't really have meaningful
// offset or length etc.
//

// A populated Packet also has a PacketHeader.
table PacketHeader {
    // This is which buffer configuration lifetime this header is referring to.
    //
    // A packet_index is only really meaningful with respect to a particular
    // buffer_lifetime_ordinal.
    //
    // See StreamBufferSettings.buffer_lifetime_ordinal.
    //
    // For QueueInputPacket(), a server receiving a buffer_lifetime_ordinal that
    // isn't the current input buffer_lifetime_ordinal will close the channel.
    //
    // For OnFreeInputPacket() and RecycleOutputPacket(), the receiver (client
    // or server) must ignore a message with stale buffer_lifetime_ordinal.
    1: uint64 buffer_lifetime_ordinal;

    // The overall set of packet_index values is densely packed from 0..count-1
    // for input and output separately.  They can be queued in any order.
    //
    // Both the client and server should validate the packet_index against the
    // known bound and disconnect if it's out of bounds.
    //
    // When running in buffer-per-packet mode, the packet_index is also the
    // buffer index.  When running in single-buffer mode, the buffer index is
    // always 0 referring to the single buffer.
    //
    // The packet_index values don't imply anything about order of use of
    // packets. The client should not expect the ordering to remain the same
    // over time - the stream processor is free to hold on to an input or
    // output packet for a while during which other packet_index values may be
    // used multiple times.
    //
    // For a given properly-functioning StreamProcessor instance, packet_index
    // values will be unique among concurrently-outstanding packets.  Servers
    // should validate that a client isn't double-using a packet and clients
    // should validate as necessary to avoid undefined or unexpected client
    // behavior.
    2: uint32 packet_index;
};

// Packet
//
// A Packet represents a chunk of input or output data to or from a stream
// processor.
//
// stream processor output:
//
// While the Packet is outstanding with the client via OnOutputPacket(), the
// stream processor will avoid modifying the referenced output data.  After the
// client calls RecycleOutputPacket(packet_index), the stream processor is
// notified that the client is again ok with the referenced data changing.
//
// stream processor input:
//

// The client initially has all packet_index(es) available to fill, and later
// gets packet_index(s) that are again ready to fill via OnFreeInputPacket().
// The client must not modify the referenced data in between QueueInputPacket()
// and OnFreeInputPacket().
table Packet {
    1: PacketHeader header;

    // Which buffer this packet refers to.  For single-buffer mode this will
    // always be 0, but for multi-buffer mode, a given in-flight interval of a
    // packet can refer to any buffer.  The packet has an associated buffer only
    // while the packet is in-flight, not while the packet is free.
    //
    // The default value makes accidental inappropriate use of index 0 less
    // likely (will tend to complain in an obvious way if not filled out
    // instead of a non-obvious data corruption when decoding buffer 0
    // repeatedly instead of the correct buffers).
    2: uint32 buffer_index = 0x80000000;

    // stream_lifetime_ordinal
    //
    // The value 1 is the lowest permitted value after stream processor
    // creation.  Values sent by the client must be odd.  Values must only
    // increase.
    //
    // A stream_lifetime_ordinal represents the lifetime of a stream.  All
    // messages that are specific to a stream have the stream_lifetime_ordinal
    // value and the value is the same for all messages relating to a given
    // stream.
    //
    // See stream_processor.md for more on stream_lifetime_ordinal.
    3: uint64 stream_lifetime_ordinal;

    // start_offset and valid_length_bytes
    //
    // Which part of the relevant buffer is this packet using.  These are valid
    // for input data that's in-flight to the stream processor, and are valid
    // for output data from the stream processor.
    //
    // For compressed formats and uncompressed audio, the data in
    // [start_offset, start_offset + valid_length_bytes) is the contiguously
    // valid data referred to by this packet.
    //
    // For uncompressed video frames, FormatDetails is the primary means of
    // determining which bytes are relevant.  The offsets in FormatDetails
    // are relative to the start_offset here.  The valid_length_bytes must be
    // large enough to include the full last line of pixel data, including the
    // full line stride of the last line (not just the width in pixels of the
    // last line).
    //
    // Despite these being filled out, some uncompressed video buffers are of
    // types that are not readable by the CPU.  These fields being here don't
    // imply there's any way for the CPU to read an uncompressed frame.
    //
    // TODO(dustingreen): Do we have any reason to require that these be filled
    // out for opaque uncompressed video frames that the CPU can't read?  In
    // that case do we want to require them just so they can be potentially
    // passed on to a HW renderer in case the HW renderer has any use for them?
    // Or more likely, it may just be that these tend to refer to the whole
    // size of an uncompressed buffer, with format_details taking care of
    // specifying which bytes are actually relevant.
    4: uint32 start_offset;

    // valid_length_bytes
    //
    // This must be > 0.
    //
    // The semantics for valid data per packet vary depending on data type as
    // follows.
    //
    // uncompressed video - A video frame can't be split across packets.  Each
    // packet is one video frame.
    //
    // uncompressed audio - Regardless of float or int, linear or uLaw, or
    // number of channels, a packet must contain an non-negative number of
    // complete audio frames, where a single audio frame consists of data for
    // all the channels for the same single point in time.  Any
    // stream-processor-specific internal details re. lower rate sampling for
    // LFE channel or the like should be hidden by the StreamProcessor server
    // implementation.
    //
    // compressed data input - A packet must contain at least one byte of data.
    // See also stream_input_bytes_min.  Splitting AUs at arbitrary byte
    // boundaries is permitted, including at boundaries that are in AU headers.
    //
    // compressed data output - The stream processor is not required to fully
    // fill each output packet's buffer.
    5: uint32 valid_length_bytes;

    // This value is not strictly speaking a timestamp.  It is an arbitrary
    // unsigned 64-bit number that, under some circumstances, will be passed by
    // a stream processor unmodified from an input packet to the
    // exactly-corresponding output packet.
    //
    // For timestamp_ish values to be propagated from input to output the
    // following conditions must be true:
    //  * promise_separate_access_units_on_input must be true
    //  * has_timestamp_ish must be true for a given input packet, to have that
    //    timestamp_ish value (potentially) propagate through to an output
    //  * the StreamProcessor instance itself decides (async) that the input
    //    packet generates an output packet - if a given input never generates
    //    an output packet then the timestamp_ish value on the input will never
    //    show up on any output packet - depending on the characteristics of the
    //    input and output formats, and whether a decoder is willing to join
    //    mid-stream, etc this can be more or less likely to occur, but clients
    //    should be written to accommodate timestamp_ish values that are fed on
    //    input but never show up on output, at least to a reasonable degree
    //    (not crashing, not treating as an error).
    //
    // See stream_processor.md for more on timestamp_ish.
    6: uint64 timestamp_ish;

    // start_access_unit
    //
    // If promise_separate_access_units_on_input (TODO(dustingreen): or any
    // similar mode for output) is true, this bool must be set appropriately
    // depending on whether byte 0 _is_ or _is not_ the start of an access
    // unit. The client is required to know, and required to set this boolean
    // properly. The server is allowed to infer that when this boolean is
    // false, byte 0 is the first byte of a continuation of a
    // previously-started AU.  (The byte at start_offset is "byte 0".)
    //
    // If promise_separate_access_units_on_input is false, this boolean is
    // ignored.
    7: bool start_access_unit;

    // known_end_access_unit
    //
    // A client is never required to set this boolean to true.
    //
    // If promise_separate_access_units_on_input is true, for input data, this
    // boolean must be false if the last byte of this packet is not the last
    // byte of an AU, and this boolean _may_ be true if the last byte of this
    // packet is the last byte of an AU.  A client delivering one AU at a time
    // that's interested in the lowest possible latency via the decoder should
    // set this boolean to true when it can be set to true.
    //
    // If promise_separate_access_units_on_input is false, this boolean is
    // ignored.
    8: bool known_end_access_unit;
};

// StreamProcessor
//
// The StreamProcessor interface exists to anchor the configuration of input
// and output buffers, and depending on StreamProcessor server hosting
// strategy, the StreamProcessor interface can, in some configurations, be 1:1
// with a stream processor isolate (process), especially when using SW codecs.
// The StreamProcessor can be used to process up to one stream at a time.
//
// Descriptions of actions taken by methods of this interface and the states of
// things are given as if the methods are synchronously executed by the stream
// processor server, but in reality, as is typical of FIDL interfaces, the
// message processing is async.  The states described are to be read as the
// state from the client's point of view unless otherwise stated.  Events
// coming back from the server are of course delivered async, and a client that
// processes more than one stream per StreamProcessor instance needs to care
// whether a given event is from the current stream vs. some older
// soon-to-be-gone stream.
//
// The Sync() method's main purpose is to enable the client to robustly prevent
// having both old and new buffers allocated in the system at the same time,
// since media buffers can be significantly large, depending. The Sync() method
// achieves this by only delivering it's response when all previous calls to
// the StreamProcessor interface have actually taken effect in the
// StreamControl ordering domain. Sync() can also be used to wait for the
// stream processor server to catch up if there's a possibility that a client
// might otherwise get too far ahead of the StreamProcessor server, by for
// example requesting creation of a large number of streams in a row.  It can
// also be used during debugging to ensure that a stream processor server
// hasn't gotten stuck.  Calling Sync() is entirely optional and never required
// for correctness - only potentially required to de-overlap resource usage.
//

// Semi-trusted StreamProcessor server - SW decoders run in an isolate (with
// very few capabilities) just in case the decoding SW has a vulnerability
// which could be used to take over the StreamProcessor server.  Clients of the
// stream processor interface using decoders and processing streams of separate
// security contexts, to a greater extent than some other interfaces, need to
// protect themselves against invalid server behavior, such as double-free of a
// packet_index and any other invalid server behavior.  Having fed in
// compressed data of one security context, don't place too much trust in a
// single StreamProcessor instance to not mix data among any buffers that
// StreamProcessor server has ever been told about.  Instead, create separate
// StreamProcessor instances for use by security-separate client-side contexts.
// While the picture for HW-based decoders looks somewhat different and is out
// of scope of this paragraph, the client should always use separate
// StreamProcessor instances for security-separate client-side contexts.
protocol StreamProcessor {
    // EnableOnStreamFailed()
    //
    // Permit the server to use OnStreamFailed() instead of the server just
    // closing the whole StreamProcessor channel on stream failure.
    //
    // If the server hasn't seen this message by the time a stream fails, the
    // server will close the StreamProcessor channel instead of sending
    // OnStreamFailed().
    EnableOnStreamFailed();

    // OnStreamFailed()
    //
    // The stream has failed, but the StreamProcessor instance is still usable
    // for a new stream.
    //
    // This message is only ever sent by the server if the client previously
    // sent EnableOnStreamFailed().  If the client didn't send
    // EnableOnStreamFailed() then the server closes the StreamProcessor
    // channel instead.
    //
    // StreamProcessor server implementations are encouraged to handle stream
    // errors (and ideally to also report them via error_ bools of
    // OnOutputPacket() and OnOutputEndOfStream()) without failing the whole
    // stream, but if a stream processor server is unable to do that, but still
    // can cleanly contain the failure to the stream, the stream processor
    // server can (assuming EnableOnStreamFailed() was called) use
    // OnStreamFailed() to indicate the stream failure to the client without
    // closing the StreamProcessor channel.
    //
    // An ideal StreamProcessor server handles problems with input data without
    // sending this message, but sending this message is preferred vs. closing
    // the server end of the StreamProcessor channel if the StreamProcessor
    // server can 100% reliably contain the stream failure to the stream,
    // without any adverse impact to any later stream.
    //
    // No further messages will arrive from the server regarding the failed
    // stream.  This includes any OnOutputEndOfStream() that the client would
    // have otherwise expected.
    //
    // TODO(dustingreen): Add at least an error_message string and _maybe_ a
    // zx_status_t, though that might tend to encourage mis-use of zx_status_t
    // so maybe just error_message for quicker debugging on the client side.
    // Also plumb from CodecAdapterH264 and similar.
    -> OnStreamFailed(uint64 stream_lifetime_ordinal);

    // OnInputConstraints()
    //
    // The server sends this shortly after StreamProcessor creation to indicate
    // input buffer constraints.  The "min" and "max" input constraints don't
    // change for the life of the StreamProcessor.
    //
    // The "max" values for buffer size and count are large enough to support
    // the most demanding format the server supports on input.  The
    // "recommended" values should be workable for use with the input
    // FormatDetails conveyed during StreamProcessor creation.  The
    // "recommended" values are not necessarily suitable if the client uses
    // QueueInputFormatDetails() to change the input format.  In that case it's
    // up to the client to determine suitable values, either by creating a new
    // StreamProcessor instance instead, or knowing suitable values outside the
    // scope of this protocol.
    //
    // See comments on StreamBufferConstraints.
    //
    // This message is guaranteed to be sent unsolicited to the StreamProcessor
    // client during or shortly after StreamProcessor creation.  Clients should
    // not depend on this being the very first message to arrive at the client.
    // OnOutputConfig() may be sent first by some stream processors that
    // already know their initial output config without any input data, to
    // encourage (but not strictly require) the client to configure output
    // buffers before feeding the first input, to avoid a wasteful
    // OnOutputConfig() being generated for that first stream if the client has
    // started configuring output but isn't done configuring output before the
    // client sends the first input data for the first stream. A client is free
    // to ignore OnOutputConfig() with a stale stream_lifetime_ordinal, but
    // handling OnOutputConfig() with stream_lifetime_ordinal 0 (if any are
    // sent) can help reduce latency to first output.  See OnOutputConfig() for
    // more details.
    //
    // The "min" and "max" input constraints are guaranteed not to change for a
    // given StreamProcessor instance.  The "recommended" values may
    // effectively change when the server processes QueueInputFormatDetails().
    // There is not any way in the protocol short of creating a new
    // StreamProcessor instance for the client to get those new "recommended"
    // values.
    //
    // TODO(dustingreen): Maybe provide a way for the client to get updated
    // "recommended" values for input, maybe only on request rather than via
    // this event, to keep things simpler for simpler clients.  Maybe separate
    // the recommendations from the constraints.
    -> OnInputConstraints(StreamBufferConstraints input_constraints);

    // SetInputBufferSettings() and AddInputBuffer()
    //
    // Configuring input buffers consists of calling SetInputBufferSettings()
    // followed by a number of calls to AddInputBuffer() equal to the number of
    // buffers set via SetInputBufferSettings().  In buffer-per-packet mode,
    // this is the same as the number of packets.  In single-buffer mode, this
    // is 1.
    //
    // After OnInputConstraints(), the client uses these two methods to set up
    // input buffers and packets.
    //
    // Configuring input buffers is required before QueueInputPacket().
    //
    // The client can also re-set-up input buffers any time there is no current
    // stream.  The client need not wait until all previously-set-up input
    // buffers are with the client via OnFreeInputPacket().  The old
    // buffer_lifetime_ordinal just ends.  See stream_processor.md for more
    // info on "buffer lifetime".
    //
    // The recommended way to de-overlap resource usage (when/if the client
    // wants to) is to send CloseCurrentStream() with release_input_buffers
    // true then send Sync() and wait for its response before allocating any
    // new buffers. How to cause other parts of the system to release their
    // references on low-level buffers is outside the scope of this interface.
    //
    // This call ends any previous buffer_lifetime_ordinal, and starts a new
    // one.
    SetInputBufferSettings(StreamBufferSettings input_settings);

    // The client is required to add all the input buffers before sending any
    // message that starts a new stream else the stream processor will close
    // the StreamProcessor channel.
    //
    // When the last buffer is added with this message, all the input packets
    // effectively jump from non-existent to free with the client.  The
    // StreamProcessor will not generate an OnFreeInputPacket() for each new
    // input packet.  The client can immediately start sending
    // QueueInputPacket() after sending the last AddInputBuffer().
    AddInputBuffer(StreamBuffer buffer);

    // OnOutputConfig()
    //
    // This event informs the client of new output config.  The server will
    // send at least one of these messages before the first output packet of a
    // stream, but that message might not have
    // buffer_constraints_action_required true.
    //
    // If buffer_constraints_action_required is true and the
    // stream_lifetime_ordinal matches the current stream, the client must react
    // by configuring or re-configuring output buffers.
    //
    // Some clients may prefer not to support mid-stream output config changes,
    // but even those clients are required to process OnOutputConfig() messages
    // up to the first output packet of each stream, as OnOutputConfig() is used
    // for stream format detection as well as for potential mid-stream output
    // config changes.
    //
    // For more on OnOutputConfig(), see cocec.md.
    -> OnOutputConfig(StreamOutputConfig output_config);

    // SetOutputBufferSettings() and AddOutputBuffer()
    //
    // These are not permitted until after the first OnOutputConfig().
    //
    // Roughly speaking, these messages are sent in response to OnOutputConfig()
    // with buffer_constraints_action_required true.
    //
    // Configuring output buffers consists of calling SetOutputBufferSettings()
    // followed by a number of calls to AddOutputBuffer() equal to the number
    // of buffers set via SetOutputBufferSettings().  In buffer-per-packet
    // mode, this is the same as the number of packets.  In single-buffer mode,
    // this is 1.
    //
    // Configuring output buffers is _required_ after OnOutputConfig() is
    // received by the client with buffer_constraints_action_required true and
    // stream_lifetime_ordinal equal to the client's current
    // stream_lifetime_ordinal (even if there is an active stream), and is
    // _permitted_ any time there is no current stream.
    //
    // For more on SetOutputBufferSettings() and AddOutputBuffer(), see
    // stream_processor.md.
    SetOutputBufferSettings(StreamBufferSettings output_settings);
    AddOutputBuffer(StreamBuffer buffer);

    // FlushEndOfStreamAndCloseStream()
    //
    // This message is optional.
    //
    // This message is only valid after QueueInputEndOfStream() for this stream.
    // The stream_lifetime_ordinal input parameter must match the
    // stream_lifetime_ordinal of the QueueInputEndOfStream(), else the server
    // will close the channel.
    //
    // A client can use this message to flush through (not discard) the last
    // input data of a stream so that the stream processor server generates
    // corresponding output data for all the input data before the server moves
    // on to the next stream, without forcing the client to wait for
    // OnOutputEndOfStream() before queueing data of another stream.
    //
    // The difference between QueueInputEndOfStream() and
    // FlushEndOfStreamAndCloseStream():  QueueInputEndOfStream() is a promise
    // from the client that there will not be any more input data for the
    // stream (and this info is needed by some stream processors for the stream
    // processor to ever emit the very last output data).  The
    // QueueInputEndOfStream() having been sent doesn't prevent the client from
    // later completely discarding the rest of the current stream by closing
    // the current stream (with or without a stream switch).  In contrast,
    // FlushEndOfStreamAndCloseStream() is a request from the client that all
    // the previously-queued input data be processed including the logical
    // "EndOfStream" showing up as OnOutputEndOfStream() (in success case)
    // before moving on to any newer stream - this essentially changes the
    // close-stream handling from discard to flush-through for this stream
    // only.
    //
    // A client using this message can start providing input data for a new
    // stream without that causing discard of old stream data.  That's the
    // purpose of this message - to allow a client to flush through (not
    // discard) the old stream's last data (instead of the default when closing
    // or switching streams which is discard).
    //
    // Because the old stream is not done processing yet and the old stream's
    // data is not being discarded, the client must be prepared to continue to
    // process OnOutputConfig() messages until the stream_lifetime_ordinal is
    // done.  The client will know the stream_lifetime_ordinal is done when
    // OnOutputEndOfStream(), OnStreamFailed(), or the StreamProcessor channel
    // closes.
    //
    // For more on FlushEndOfStreamAndCloseStream(), see stream_processor.md.
    FlushEndOfStreamAndCloseStream(uint64 stream_lifetime_ordinal);

    // CloseCurrentStream()
    //
    // This "closes" the current stream, leaving no current stream.  In
    // addition, this message can optionally release input buffers or output
    // buffers.
    //
    // If there has never been any active stream, the stream_lifetime_ordinal
    // must be zero or the server will close the channel.  If there has been an
    // active stream, the stream_lifetime_ordinal must be the most recent
    // active stream whether that stream is still active or not.  Else the
    // server will close the channel.
    //
    // Multiple of this message without any new active stream in between is not
    // to be considered an error, which allows a client to use this message to
    // close the current stream to stop wasting processing power on a stream the
    // user no longer cares about, then later decide that buffers should be
    // released and send this message again with release_input_buffers and/or
    // release_output_buffers true to get the buffers released, if the client is
    // interested in trying to avoid overlap in resource usage between old
    // buffers and new buffers (not all clients are).
    //
    // See also Sync().
    //
    // For more on CloseCurrentStream(), see stream_processor.md.
    CloseCurrentStream(
        uint64 stream_lifetime_ordinal,
        bool release_input_buffers,
        bool release_output_buffers);

    // Sync() -> ()
    //
    // On completion, all previous StreamProcessor calls have done what they're
    // going to do server-side, _except_ for processing of data queued using
    // QueueInputPacket().
    //
    // The main purpose of this call is to enable the client to wait until
    // CloseCurrentStream() with release_input_buffers and/or
    // release_output_buffers set to true to take effect, before the client
    // allocates new buffers and re-sets-up input and/or output buffers.  This
    // de-overlapping of resource usage can be worthwhile for media buffers
    // which can consume resource types whose overall pools aren't necessarily
    // vast in comparison to resources consumed.  Especially if a client is
    // reconfiguring buffers multiple times.
    //
    // Note that Sync() prior to allocating new media buffers is not alone
    // sufficient to achieve non-overlap of media buffer resource usage system
    // wide, but it can be a useful part of achieving that.
    //
    // The Sync() transits the Output ordering domain and the StreamControl
    // ordering domain, but not the InputData ordering domain.  For more on
    // ordering domains see stream_processor.md.
    //
    // This request can be used to avoid hitting kMaxInFlightStreams which is
    // presently 10.  A client that stays <= 8 in-flight streams will
    // comfortably stay under the limit of 10.  While the protocol permits
    // repeated SetInputBufferSettings() and the like, a client that spams the
    // channel can expect that the channel will just close if the server or the
    // channel itself gets too far behind.
    Sync() -> ();

    // OnOutputPacket()
    //
    // This is how the stream processor emits an output packet to the stream
    // processor client.
    //
    // Order is significant.
    //
    // The client should eventually call RecycleOutputPacket() (possibly after
    // switching streams multiple times), unless the buffer_lifetime_ordinal
    // has moved on.  A stream change doesn't change which packets are busy
    // with the client vs. free with the server.  See "packet lifetime" in
    // stream_processor.md for more.
    //
    // The relevant buffer is buffer 0 if running in single-buffer mode, or the
    // buffer index is the same as packet_index if running in buffer-per-packet
    // mode.
    //
    // For low-level buffer types that support it, a StreamProcessor is free to
    // emit an output packet before the low-level buffer actually has any
    // usable data in the buffer, with the mechanism for signalling the
    // presence of data separate from the OnOutputPacket() message.  For such
    // low-level buffer types, downstream consumers of data from the emitted
    // packet must participate in the low-level buffer signalling mechanism to
    // know when it's safe to consume the data.  This is most likely to be
    // relevant when using a video decoder and gralloc-style buffers.
    //
    // The error_ bool(s) allow (but do not require) a StreamProcessor server
    // to report errors that happen during an AU or between AUs.
    //
    // The scope of error_detected_before starts at the end of the last
    // delivered output packet on this stream, or the start of stream if there
    // were no previous output packets on this stream.  The scope ends at the
    // start of the output_packet.
    //
    // The error_detected_before bool is separate so that discontinuities can be
    // indicated separately from whether the current packet is damaged.
    //
    // The scope of error_detected_during is from the start to the end of this
    // output_packet.
    -> OnOutputPacket(
           Packet output_packet,
           bool error_detected_before,
           bool error_detected_during);

    // RecycleOutputPacket()
    //
    // After the client is done with an output packet, the client needs to tell
    // the stream processor that the output packet can be re-used for more
    // output, via this method.
    //
    // It's not permitted to recycle an output packet that's already free with
    // the stream processor server.  It's permitted but discouraged for a
    // client to recycle an output packet that has been deallocated by an
    // explicit or implicit output buffer de-configuration().  See
    // buffer_lifetime_ordinal for more on that. A server must ignore any such
    // stale RecycleOutputPacket() calls.
    //
    // For more on RecycleOutputPacket(), see stream_processor.md.
    RecycleOutputPacket(PacketHeader available_output_packet);

    // OnOutputEndOfStream()
    //
    // After QueueInputEndOfStream() is sent by the StreamProcessor client,
    // within a reasonable duration the corresponding OnOutputEndOfStream()
    // will be sent by the StreamProcessor server.  Similar to
    // QueueInputEndOfStream(), OnOutputEndOfStream() is sent a maximum of once
    // per stream.
    //
    // No more stream data for this stream will be sent after this message.  All
    // input data for this stream was processed.
    //
    // While a StreamProcessor client is not required to
    // QueueInputEndOfStream() (unless the client wants to use
    // FlushEndOfStreamAndCloseStream()), if a StreamProcessor server receives
    // QueueInputEndOfStream(), and the client hasn't closed the stream, the
    // StreamProcessor server must generate a corresponding
    // OnOutputEndOfStream() if nothing went wrong, or must send
    // OnStreamFailed(), or must close the server end of the StreamProcessor
    // channel.  An ideal StreamProcessor server would handle and report stream
    // errors via the error_ flags and complete stream processing without
    // sending OnStreamFailed(), but in any case, the above-listed options are
    // the only ways that an OnOutputEndOfStream() won't happen after
    // QueueInputEndOfStream().
    //
    // There will be no more OnOutputPacket() or OnOutputConfig() messages for
    // this stream_lifetime_ordinal after this message - if a server doesn't
    // follow this rule, a client should close the StreamProcessor channel.
    //
    // The error_detected_before bool has the same semantics as the
    // error_detected_before bool in OnOutputPacket().
    -> OnOutputEndOfStream(
           uint64 stream_lifetime_ordinal,
           bool error_detected_before);

    //
    // Stream specific messages:
    //

    // QueueInputFormatDetails()
    //
    // If the input format details are still the same as specified during
    // StreamProcessor creation, this message is unnecessary and does not need
    // to be sent.
    //
    // If the stream doesn't exist yet, this message creates the stream.
    //
    // All servers must permit QueueInputFormatDetails() at the start of a
    // stream without failing, as long as the new format is supported by the
    // StreamProcessor instance.  Technically this allows for a server to only
    // support the exact input format set during StreamProcessor creation, and
    // that is by design.  A client that tries to switch formats and gets a
    // StreamProcessor channel failure should try again one more time with a
    // fresh StreamProcessor instance created with CodecFactory using the new
    // input format during creation, before giving up.
    //
    // These format details override the format details specified during stream
    // processor creation for this stream only.  The next stream will default
    // back to the format details set during stream processor creation.
    //
    // For now, QueueInputFormatDetails() sent mid-stream will fail the
    // StreamProcessor channel.  Clients shouldn't do this for now.
    //
    // This message is permitted at the start of the first stream (just like at
    // the start of any stream).  The format specified need not match what was
    // specified during stream processor creation, but if it doesn't match, the
    // StreamProcessor channel might close as described above.
    QueueInputFormatDetails(
        uint64 stream_lifetime_ordinal, FormatDetails format_details);

    // QueueInputPacket()
    //
    // This message queues input data to the stream processor for processing.
    //
    // If the stream doesn't exist yet, this message creates the new stream.
    //
    // The client is required to be willing to send QueueInputPacket() prior to
    // the server's first OnOutputConfig(), and is permitted to start a new
    // stream without output buffers configured yet.
    //
    // The client must continue to deliver input data via this message even if
    // the stream processor has not yet generated the first OnOutputConfig(),
    // and even if the StreamProcessor is generating OnFreeInputPacket() for
    // previously-queued input packets.  The input data must continue as long
    // as there are free packets to be assured that the server will ever
    // generate the first OnOutputConfig().
    //
    // For more on QueueInputPacket(), see stream_processor.md.
    QueueInputPacket(Packet packet);

    // OnFreeInputPacket()
    //
    // The server sends this message when the stream processor is done
    // consuming this packet and the packet can be re-filled by the client.
    //
    // This is not sent for all packets when a new buffer_lifetime_ordinal
    // starts as in that case all the packets are initially free with the
    // client.
    //
    // See comments on QueueInputBuffer() and "packet lifetime" in
    // stream_processor.md for for description of buffer lifetime and packet
    // lifetime.
    //
    // After receiving the available input buffer via this event, the stream
    // processor client can call later call QueueInputBuffer with appropriate
    // offset and length set.
    //
    // TODO(dustingreen): At the moment, there is no guarantee re. the order of
    // these messages with respect to the order of QueueInputPacket(), but at
    // least for decoders, it might be worthwhile to require that servers
    // preserve the order vs. QueueInputPacket(), to make it easier to feed
    // input from a ring buffer or similar.  For audio encoders it might still
    // make sense.  For video encoders probably not.
    -> OnFreeInputPacket(PacketHeader free_input_packet);

    // Inform the server that all QueueInputPacket() messages for this stream
    // have been sent.
    //
    // If the stream isn't closed first (by the client, or by OnStreamFailed(),
    // or StreamProcessor channel closing), there will later be a corresponding
    // OnOutputEndOfStream().
    //
    // The corresponding OnOutputEndOfStream() message will be generated only if
    // the server finishes processing the stream before the server sees the
    // client close the stream (such as by starting a new stream).  A way to
    // force the server to finish the stream before closing is to use
    // FlushEndOfStreamAndCloseStream() after QueueInputEndOfStream() before any
    // new stream.  Another way to force the server to finish the stream before
    // closing is to wait for the OnOutputEndOfStream() before taking any action
    // that closes the stream.
    //
    // In addition to serving as an "EndOfStream" marker to make it obvious
    // client-side when all input data has been processed, if a client never
    // sends QueueInputEndOfStream(), no amount of waiting will necessarily
    // result in all input data getting processed through to the output.  Some
    // stream processors have some internally-delayed data which only gets
    // pushed through by additional input data _or_ by this EndOfStream marker.
    // In that sense, this message can be viewed as a flush-through at
    // InputData domain level, but the flush-through only takes effect if the
    // stream processor even gets that far before the stream is just closed at
    // StreamControl domain level.  This message is not alone sufficient to act
    // as an overall flush-through at StreamControl level. For that, send this
    // message first and then send FlushEndOfStreamAndCloseStream() (at which
    // point it becomes possible to queue input data for a new stream without
    // causing discard of this older stream's data), or wait for the
    // OnOutputEndOfStream() before closing the current stream.
    //
    // If a client sends QueueInputPacket(), QueueInputFormatDetails(),
    // QueueInputEndOfStream() for this stream after the first
    // QueueInputEndOfStream() for this stream, a server should close the
    // StreamProcessor channel.
    QueueInputEndOfStream(uint64 stream_lifetime_ordinal);
};