Google Git
Sign in
fuchsia / fuchsia / 099dfaa57f8e77a17e57e89b07604f2a4d648410 / . / sdk / fidl / fuchsia.media / stream_processor.fidl
blob: 4511000282a38ed5890af1eefb0fa1784ee84e15 [file] [log] [blame]
// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
library fuchsia.media;
using fuchsia.sysmem;
/// This struct conveys the buffer_constraints_version_ordinal.
///
/// Historically this table conveyed more fields than it currently does, but
/// those fields are all deprecated in favor of using sysmem instead.
///
/// There are separate instances of this struct for stream input and stream
/// output.
///
/// Notes about fields:
///
/// For uncompressed video, separate and complete frames in their
/// separate buffers (buffer-per-packet mode) are always a requirement.
type StreamBufferConstraints = table {
/// This is a version number the server sets on the constraints to allow the
/// server to determine when the client has caught up with the latest
/// constraints sent by the server. The server won't emit output data until
/// the client has configured output settings and buffers with a
/// buffer_constraints_version_ordinal >= the latest
/// buffer_constraints_version_ordinal that had
/// buffer_constraints_action_required true. See
/// buffer_constraints_action_required comments for more.
///
/// A buffer_constraints_version_ordinal of 0 is not permitted, to simplify
/// initial state handling. Other than 0, both odd and even version ordinals
/// are allowed (in contrast to the stream_lifetime_ordinal, neither the
/// client nor server ever has a reason to consider the latest version to be
/// stale, so there would be no benefit to disallowing even values).
1: buffer_constraints_version_ordinal uint64;
@deprecated("Ignore. Obsolete.")
12: single_buffer_mode_allowed bool;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
2: default_settings StreamBufferSettings;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
3: per_packet_buffer_bytes_min uint32;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
4: per_packet_buffer_bytes_recommended uint32;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
5: per_packet_buffer_bytes_max uint32;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
6: packet_count_for_server_min uint32;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
7: packet_count_for_server_recommended uint32;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
8: packet_count_for_server_recommended_max uint32;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
9: packet_count_for_server_max uint32;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
10: packet_count_for_client_min uint32;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
11: packet_count_for_client_max uint32;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
13: is_physically_contiguous_required bool;
};
/// The stream-processor-controlled output configuration, including both
/// StreamBufferConstraints for the output and FormatDetails for the output.
//
// TODO(dustingreen): Need a better name for this struct, but still short
// hopefully. It's stuff the stream processor gets to control, not the client.
// It's different than output buffer settings, which the client does get to
// control to some extent. It's different than any configurable output
// settings the client might specify for output of an encoder.
type StreamOutputConstraints = table {
/// A client which always immediately re-configures output buffers on
/// receipt of OnOutputConstraints() with buffer_constraints_action_required
/// true can safely ignore this field.
///
/// A client is permitted to ignore an OnOutputConstraints() message even with
/// buffer_constraints_action_required true if the client knows the server
/// has already been told to discard the remainder of the stream with the
/// same stream_lifetime_ordinal or if this stream_lifetime_ordinal field is
/// set to 0. The server is required to re-send needed output config via
/// OnOutputConstraints() with new stream_lifetime_ordinal and
/// buffer_constraints_action_required true, if the most recent completed
/// server-side output config isn't what the server wants/needs yet for the
/// new stream.
1: stream_lifetime_ordinal uint64;
/// When the buffer constraints are delivered, they indicate whether action
/// is required. A false value here permits delivery of constraints which
/// are fresher without forcing a buffer reconfiguration. If this is false,
/// a client cannot assume that it's safe to immediately re-configure output
/// buffers. If this is true, the client can assume it's safe to
/// immediately configure output buffers once.
///
/// A client is permitted to ignore buffer constraint versions which have
/// buffer_constraints_action_required false. The server is not permitted
/// to change buffer_constraints_action_required from false to true for the
/// same buffer_constraints_version_ordinal.
///
/// For each configuration, a client must use new buffers, never buffers
/// that were previously used for anything else, and never buffers
/// previously used for any other StreamProcessor purposes. This rule
/// exists for multiple good reasons, relevant to both mid-stream changes,
/// and changes on stream boundaries. A client should just use new buffers
/// each time.
///
/// When this is true, the server has already de-refed as many low-level
/// output buffers as the server can while still performing efficient
/// transition to the new buffers and will de-ref the rest asap. A Sync()
/// is not necessary to achieve non-overlap of resource usage to the extent
/// efficiently permitted by the formats involved.
///
/// If buffer_constraints_action_required is true, the server _must_ not
/// deliver more output data until after output buffers have been configured
/// (or re-configured) by the client.
2: buffer_constraints_action_required bool;
3: buffer_constraints StreamBufferConstraints;
};
type StreamOutputFormat = table {
/// A client is permitted to ignore an OnOutputFormat() message even with
/// buffer_constraints_action_required true if the client knows the server
/// has already been told to discard the remainder of the stream with the
/// same stream_lifetime_ordinal or if this stream_lifetime_ordinal field is
/// set to 0. The server is required to re-send needed output config via
/// OnOutputConstraints() with new stream_lifetime_ordinal and
/// buffer_constraints_action_required true, if the most recent completed
/// server-side output config isn't what the server wants/needs yet for the
/// new stream.
///
/// The server is required to send an OnOutputFormat() before the first
/// output packet of a stream.
1: stream_lifetime_ordinal uint64;
/// If format_details.format_details_version_ordinal changes, the client
/// should inspect the new format details and determine if it must adjust to
/// the new format. The server guarantees that if the format has changed, then
/// format_details.format_details_version_ordinal will change, but a change
/// to format_details.format_details_version_ordinal does not guarantee that
/// the format details actually changed. Servers are strongly encouraged to
/// not change format_details.format_details_version_ordinal other than
/// before the first output data of a stream unless there is a real
/// mid-stream format change in the stream. Unnecessary mid-stream format
/// changes can cause simpler clients that have no need to handle mid-stream
/// format changes to just close the channel. Format changes before the
/// first output data of a stream are not "mid-stream" in this context -
/// those can be useful for stream format detection / setup reasons.
///
/// Note that in case output buffers don't really need to be re-configured
/// despite a format change, a server is encouraged, but not required, to
/// set buffer_constraints_action_required false on the message that conveys
/// the new format details. Simpler servers may just treat the whole output
/// situation as one big thing and demand output buffer reconfiguration on
/// any change in the output situation.
///
/// A client may or may not actually handle a new buffer_constraints with
/// buffer_constraints_action_required false, but the client should always
/// track the latest format_details.
///
/// An updated format_details is ordered with respect to emitted output
/// packets, and applies to all subsequent packets until the next
/// format_details with larger version_ordinal. A simple client that does
/// not intend to handle mid-stream format changes should still keep track
/// of the most recently received format_details until the first output
/// packet arrives, then lock down the format details, handle those format
/// details, and verify that any
/// format_details.format_details_version_ordinal received from the server
/// is the same as the locked-down format_details, until the client is done
/// with the stream. Even such a simple client must tolerate
/// format_details.format_details_version_ordinal changing multiple times
/// before the start of data output from a stream (any stream - the first
/// stream or a subsequent stream). This allows a stream processor to
/// request that output buffers and output format be configured
/// speculatively, and for the output config to be optionally adjusted by
/// the server before the first data output from a stream after the server
/// knows everything it needs to know to fully establish the initial output
/// format details. This simplifies stream processor server implementation,
/// and allows a clever stream processor server to guess it's output config
/// for lower latency before any input data, while still being able to fix
/// the output config (including format details) if the guess turns out to
/// be wrong.
///
/// Whether the format_details.format_details_version_ordinal will actually
/// change mid-stream is a per-stream-processor and per-stream detail that
/// is not specified in comments here, and in most cases also depends on
/// whether the format changes on the input to the stream processor.
/// Probably it'll be fairly common for a client to use a format which
/// technically supports mid-stream format change, but the client happens to
/// know that none of the streams the client intends to process will ever
/// have a mid-stream format change.
2: format_details FormatDetails;
};
// TODO(dustingreen): Consider renaming this to StreamBufferSettings once that's
// an option.
type StreamBufferPartialSettings = resource table {
/// The containing message starts a new buffer_lifetime_ordinal.
///
/// There is a separate buffer_lifetime_ordinal for input vs. output.
///
/// Re-use of the same value is not allowed. Values must be odd. Values
/// must only increase (increasing by more than 2 is permitted).
///
/// A buffer_lifetime_ordinal lifetime starts at SetInputBufferSettings() or
/// SetOutputBufferSettings(), and ends at the earlier of
/// CloseCurrentStream() with release_input_buffers/release_output_buffers
/// set or SetOutputBufferSettings() with new buffer_lifetime_ordinal in the
/// case of mid-stream output config change.
1: buffer_lifetime_ordinal uint64;
/// This value indicates which version of constraints the client is/was aware
/// of so far.
///
/// For input, this must always be 0 because constraints don't change for
/// input (settings can change, but there's no settings vs current
/// constraints synchronization issue on input).
///
/// For output, this allows the server to know when the client is
/// sufficiently caught up before the server will generate any more output.
///
/// When there is no active stream, a client is permitted to re-configure
/// buffers again using the same buffer_constraints_version_ordinal.
2: buffer_constraints_version_ordinal uint64;
/// The client end of a BufferCollectionToken channel, which the
/// StreamProcessor will use to deliver constraints to sysmem and learn of
/// buffers allocated by sysmem.
///
/// The client guarantees that the token is already known to sysmem (via
/// BufferCollectionToken.Sync(), BufferCollection.Sync(), or
/// BufferCollectionEvents.OnDuplicatedTokensKnownByServer()).
6: sysmem_token client_end:fuchsia.sysmem.BufferCollectionToken;
@deprecated("Ignore. Obsolete.")
3: single_buffer_mode bool;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
4: packet_count_for_server uint32;
@deprecated("Ignore. Use fuchsia.sysmem.BufferCollection.SetConstraints() ")
5: packet_count_for_client uint32;
};
/// PacketHeader
///
/// When referring to a free packet, we use PacketHeader alone instead of
/// Packet, since while a packet is free it doesn't really have meaningful
/// offset or length etc.
///
/// A populated Packet also has a PacketHeader.
type PacketHeader = table {
/// This is which buffer configuration lifetime this header is referring to.
///
/// A packet_index is only really meaningful with respect to a particular
/// buffer_lifetime_ordinal.
///
/// See StreamBufferPartialSettings.buffer_lifetime_ordinal.
///
/// For QueueInputPacket(), a server receiving a buffer_lifetime_ordinal that
/// isn't the current input buffer_lifetime_ordinal will close the channel.
///
/// For OnFreeInputPacket() and RecycleOutputPacket(), the receiver (client
/// or server) must ignore a message with stale buffer_lifetime_ordinal.
1: buffer_lifetime_ordinal uint64;
/// The overall set of packet_index values is densely packed from 0..count-1
/// for input and output separately. They can be queued in any order.
///
/// Both the client and server should validate the packet_index against the
/// known bound and disconnect if it's out of bounds.
///
/// When running in single-buffer mode, the buffer index is always 0.
///
/// The packet_index values don't imply anything about order of use of
/// packets. The client should not expect the ordering to remain the same
/// over time - the stream processor is free to hold on to an input or
/// output packet for a while during which other packet_index values may be
/// used multiple times.
///
/// For a given properly-functioning StreamProcessor instance, packet_index
/// values will be unique among concurrently-outstanding packets. Servers
/// should validate that a client isn't double-using a packet and clients
/// should validate as necessary to avoid undefined or unexpected client
/// behavior.
2: packet_index uint32;
};
/// A Packet represents a chunk of input or output data to or from a stream
/// processor.
///
/// stream processor output:
///
/// While the Packet is outstanding with the client via OnOutputPacket(), the
/// stream processor will avoid modifying the referenced output data. After the
/// client calls RecycleOutputPacket(packet_index), the stream processor is
/// notified that the client is again ok with the referenced data changing.
///
/// stream processor input:
///
/// The client initially has all packet_index(es) available to fill, and later
/// gets packet_index(s) that are again ready to fill via OnFreeInputPacket().
/// The client must not modify the referenced data in between QueueInputPacket()
/// and OnFreeInputPacket().
type Packet = table {
1: header PacketHeader;
/// Which buffer this packet refers to. For single-buffer mode this will
/// always be 0, but for multi-buffer mode, a given in-flight interval of a
/// packet can refer to any buffer. The packet has an associated buffer only
/// while the packet is in-flight, not while the packet is free.
///
/// The default value makes accidental inappropriate use of index 0 less
/// likely (will tend to complain in an obvious way if not filled out
/// instead of a non-obvious data corruption when decoding buffer 0
/// repeatedly instead of the correct buffers).
///
/// TODO(dustingreen): Try to make FIDL table defaults have meaning, and not
/// complain about !has when accessing the field. For now the default
/// specified here does nothing.
// TODO(fxbug.dev/7932): Default to 0x80000000.
2: buffer_index uint32;
/// The value 1 is the lowest permitted value after stream processor
/// creation. Values sent by the client must be odd. Values must only
/// increase.
///
/// A stream_lifetime_ordinal represents the lifetime of a stream. All
/// messages that are specific to a stream have the stream_lifetime_ordinal
/// value and the value is the same for all messages relating to a given
/// stream.
3: stream_lifetime_ordinal uint64;
/// Which part of the relevant buffer is this packet using. These are valid
/// for input data that's in-flight to the stream processor, and are valid
/// for output data from the stream processor.
///
/// For compressed formats and uncompressed audio, the data in
/// [start_offset, start_offset + valid_length_bytes) is the contiguously
/// valid data referred to by this packet.
///
/// For uncompressed video frames, FormatDetails is the primary means of
/// determining which bytes are relevant. The offsets in FormatDetails
/// are relative to the start_offset here. The valid_length_bytes must be
/// large enough to include the full last line of pixel data, including the
/// full line stride of the last line (not just the width in pixels of the
/// last line).
///
/// Despite these being filled out, some uncompressed video buffers are of
/// types that are not readable by the CPU. These fields being here don't
/// imply there's any way for the CPU to read an uncompressed frame.
//
// TODO(dustingreen): Do we have any reason to require that these be filled
// out for opaque uncompressed video frames that the CPU can't read? In
// that case do we want to require them just so they can be potentially
// passed on to a HW renderer in case the HW renderer has any use for them?
// Or more likely, it may just be that these tend to refer to the whole
// size of an uncompressed buffer, with format_details taking care of
// specifying which bytes are actually relevant.
4: start_offset uint32;
/// This must be > 0.
///
/// The semantics for valid data per packet vary depending on data type as
/// follows.
///
/// uncompressed video - A video frame can't be split across packets. Each
/// packet is one video frame.
///
/// uncompressed audio - Regardless of float or int, linear or uLaw, or
/// number of channels, a packet must contain an non-negative number of
/// complete audio frames, where a single audio frame consists of data for
/// all the channels for the same single point in time. Any
/// stream-processor-specific internal details re. lower rate sampling for
/// LFE channel or the like should be hidden by the StreamProcessor server
/// implementation.
///
/// compressed data input - A packet must contain at least one byte of data.
/// See also stream_input_bytes_min. Splitting AUs at arbitrary byte
/// boundaries is permitted, including at boundaries that are in AU headers.
///
/// compressed data output - The stream processor is not required to fully
/// fill each output packet's buffer.
5: valid_length_bytes uint32;
/// This value is not strictly speaking a timestamp. It is an arbitrary
/// unsigned 64-bit number that, under some circumstances, will be passed by
/// a stream processor unmodified from an input packet to the
/// exactly-corresponding output packet.
///
/// For timestamp_ish values to be propagated from input to output the
/// following conditions must be true:
/// * promise_separate_access_units_on_input must be true
/// * has_timestamp_ish must be true for a given input packet, to have that
/// timestamp_ish value (potentially) propagate through to an output
/// * the StreamProcessor instance itself decides (async) that the input
/// packet generates an output packet - if a given input never generates
/// an output packet then the timestamp_ish value on the input will never
/// show up on any output packet - depending on the characteristics of the
/// input and output formats, and whether a decoder is willing to join
/// mid-stream, etc this can be more or less likely to occur, but clients
/// should be written to accommodate timestamp_ish values that are fed on
/// input but never show up on output, at least to a reasonable degree
/// (not crashing, not treating as an error).
6: timestamp_ish uint64;
/// If promise_separate_access_units_on_input (TODO(dustingreen): or any
/// similar mode for output) is true, this bool must be set appropriately
/// depending on whether byte 0 _is_ or _is not_ the start of an access
/// unit. The client is required to know, and required to set this boolean
/// properly. The server is allowed to infer that when this boolean is
/// false, byte 0 is the first byte of a continuation of a
/// previously-started AU. (The byte at start_offset is "byte 0".)
///
/// If promise_separate_access_units_on_input is false, this boolean is
/// ignored.
7: start_access_unit bool;
/// A client is never required to set this boolean to true.
///
/// If promise_separate_access_units_on_input is true, for input data, this
/// boolean must be false if the last byte of this packet is not the last
/// byte of an AU, and this boolean _may_ be true if the last byte of this
/// packet is the last byte of an AU. A client delivering one AU at a time
/// that's interested in the lowest possible latency via the decoder should
/// set this boolean to true when it can be set to true.
///
/// If promise_separate_access_units_on_input is false, this boolean is
/// ignored.
8: known_end_access_unit bool;
/// Used for compressed video packets. If not present should be assumed to
/// be unknown. If false, indicates the packet is not part of a key frame. If
/// true, indicates the packet is part of a key frame.
9: key_frame bool;
};
/// Overview of operation:
///
/// 1. Create
/// * create via CodecFactory - see CodecFactory
/// * create via LicenseSession - see LicenseSession
/// 2. Get input constraints
/// * OnInputConstraints() - sent unsolicited by stream processor shortly after
/// stream processor creation.
/// 3. Provide input buffers
/// * SetInputBufferPartialSettings()
/// 4. Deliver input data
/// * QueueInputPacket() + OnFreeInputPacket(), for as long as it takes,
/// possibly working through all input packets repeatedly before...
/// 5. Get output constraints and format
/// * OnOutputConstraints()
/// * This is not sent until after at least one QueueInput* message is sent by
/// the client, even if the underlying processor behind the StreamProcessor
/// doesn't fundamentally need any input data to determine its output
/// constraints. This server behavior prevents clients taking an incorrect
/// dependency on the output constraints showing up before input is
/// delivered.
/// * A client must tolerate this arriving as late as after substantial input
/// data has been delivered, including lots of input packet recycling via
/// OnFreeInputPacket().
/// * This message can arrive more than once before the first output data.
/// 6. Provide output buffers
/// * SetOutputBufferPartialSettings() / CompleteOutputBufferPartialSettings()
/// 7. Data flows, with optional EndOfStream
/// * OnOutputPacket() / RecycleOutputPacket() / QueueInputPacket() /
/// OnFreeInputPacket() / QueueInputEndOfStream() / OnOutputEndOfStream()
///
/// Semi-trusted StreamProcessor server - SW decoders run in an isolate (with
/// very few capabilities) just in case the decoding SW has a vulnerability
/// which could be used to take over the StreamProcessor server. Clients of the
/// stream processor interface using decoders and processing streams of separate
/// security contexts, to a greater extent than some other interfaces, need to
/// protect themselves against invalid server behavior, such as double-free of a
/// packet_index and any other invalid server behavior. Having fed in
/// compressed data of one security context, don't place too much trust in a
/// single StreamProcessor instance to not mix data among any buffers that
/// StreamProcessor server has ever been told about. Instead, create separate
/// StreamProcessor instances for use by security-separate client-side contexts.
/// While the picture for HW-based decoders looks somewhat different and is out
/// of scope of this paragraph, the client should always use separate
/// StreamProcessor instances for security-separate client-side contexts.
///
/// Descriptions of actions taken by methods of this protocol and the states of
/// things are given as if the methods are synchronously executed by the stream
/// processor server, but in reality, as is typical of FIDL interfaces, the
/// message processing is async. The states described are to be read as the
/// state from the client's point of view unless otherwise stated. Events
/// coming back from the server are of course delivered async, and a client that
/// processes more than one stream per StreamProcessor instance needs to care
/// whether a given event is from the current stream vs. some older
/// soon-to-be-gone stream.
///
/// The Sync() method's main purpose is to enable the client to robustly prevent
/// having both old and new buffers allocated in the system at the same time,
/// since media buffers can be significantly large, depending. The Sync() method
/// achieves this by only delivering it's response when all previous calls to
/// the StreamProcessor protocol have actually taken effect in the
/// StreamControl ordering domain. Sync() can also be used to wait for the
/// stream processor server to catch up if there's a possibility that a client
/// might otherwise get too far ahead of the StreamProcessor server, by for
/// example requesting creation of a large number of streams in a row. It can
/// also be used during debugging to ensure that a stream processor server
/// hasn't gotten stuck. Calling Sync() is entirely optional and never required
/// for correctness - only potentially required to de-overlap resource usage.
///
/// It's possible to re-use a StreamProcessor instance for another stream, and
/// doing so can sometimes skip over re-allocation of buffers. This can be a
/// useful thing to do for cases like seeking to a new location - at the
/// StreamProcessor interface that can look like switching to a new stream.
protocol StreamProcessor {
/// Permit the server to use OnStreamFailed() instead of the server just
/// closing the whole StreamProcessor channel on stream failure.
///
/// If the server hasn't seen this message by the time a stream fails, the
/// server will close the StreamProcessor channel instead of sending
/// OnStreamFailed().
EnableOnStreamFailed();
/// The stream has failed, but the StreamProcessor instance is still usable
/// for a new stream.
///
/// This message is only ever sent by the server if the client previously
/// sent EnableOnStreamFailed(). If the client didn't send
/// EnableOnStreamFailed() then the server closes the StreamProcessor
/// channel instead.
///
/// StreamProcessor server implementations are encouraged to handle stream
/// errors (and ideally to also report them via error_ bools of
/// OnOutputPacket() and OnOutputEndOfStream()) without failing the whole
/// stream, but if a stream processor server is unable to do that, but still
/// can cleanly contain the failure to the stream, the stream processor
/// server can (assuming EnableOnStreamFailed() was called) use
/// OnStreamFailed() to indicate the stream failure to the client without
/// closing the StreamProcessor channel.
///
/// An ideal StreamProcessor server handles problems with input data without
/// sending this message, but sending this message is preferred vs. closing
/// the server end of the StreamProcessor channel if the StreamProcessor
/// server can 100% reliably contain the stream failure to the stream,
/// without any adverse impact to any later stream.
///
/// No further messages will arrive from the server regarding the failed
/// stream. This includes any OnOutputEndOfStream() that the client would
/// have otherwise expected.
-> OnStreamFailed(struct {
stream_lifetime_ordinal uint64;
error StreamError;
});
/// The server sends this shortly after StreamProcessor creation to indicate
/// input buffer constraints. The "min" and "max" input constraints don't
/// change for the life of the StreamProcessor.
///
/// The "max" values for buffer size and count are large enough to support
/// the most demanding format the server supports on input. The
/// "recommended" values should be workable for use with the input
/// FormatDetails conveyed during StreamProcessor creation. The
/// "recommended" values are not necessarily suitable if the client uses
/// QueueInputFormatDetails() to change the input format. In that case it's
/// up to the client to determine suitable values, either by creating a new
/// StreamProcessor instance instead, or knowing suitable values outside the
/// scope of this protocol.
///
/// See comments on StreamBufferConstraints.
///
/// This message is guaranteed to be sent unsolicited to the StreamProcessor
/// client during or shortly after StreamProcessor creation. Clients should
/// not depend on this being the very first message to arrive at the client.
///
/// The "min" and "max" input constraints are guaranteed not to change for a
/// given StreamProcessor instance. The "recommended" values may
/// effectively change when the server processes QueueInputFormatDetails().
/// There is not any way in the protocol short of creating a new
/// StreamProcessor instance for the client to get those new "recommended"
/// values.
//
// TODO(dustingreen): Maybe provide a way for the client to get updated
// "recommended" values for input, maybe only on request rather than via
// this event, to keep things simpler for simpler clients. Maybe separate
// the recommendations from the constraints.
-> OnInputConstraints(struct {
input_constraints StreamBufferConstraints;
});
/// This is the replacement for SetInputBufferSettings().
///
/// When the client is using sysmem to allocate buffers, this message is
/// used instead of SetInputBufferSettings()+AddInputBuffer(). Instead, a
/// single SetInputBufferPartialSettings() provides the StreamProcessor with
/// the client-specified input settings and a BufferCollectionToken which
/// the StreamProcessor will use to convey constraints to sysmem. Both the
/// client and the StreamProcessor will be informed of the allocated buffers
/// directly by sysmem via their BufferCollection channel (not via the
/// StreamProcessor channel).
///
/// The client must not QueueInput...() until after sysmem informs the client
/// that buffer allocation has completed and was successful.
///
/// The server should be prepared to see QueueInput...() before the server
/// has necessarily heard from sysmem that the buffers are allocated - the
/// server must tolerate either ordering, as the QueueInput...() and
/// notification of sysmem allocation completion arrive on different
/// channels, so the client having heard that allocation is complete doesn't
/// mean the server knows that allocation is complete yet. However, the
/// server can expect that allocation is in fact complete and can expect to
/// get the allocation information from sysmem immediately upon requesting
/// the information from sysmem.
SetInputBufferPartialSettings(resource struct {
input_settings StreamBufferPartialSettings;
});
/// This event informs the client of new output constraints.
///
/// This message is ordered with respect to other output (such as output
/// packets, output format, output end-of-stream).
///
/// Before the first OnOutputPacket() of a stream, the server guarantees that
/// at least one OnOutputConstraints() and exactly one OnOutputFormat() will
/// be sent. The server may not set buffer_constraints_action_required true
/// in OnOutputConstraints() if the buffer config is already suitable for the
/// stream (buffer_constraints_action_required false means the buffer config
/// is already fine). The client must tolerate multiple
/// OnOutputConstraints() (and 1 OnOutputFormat() message) before the first
/// output packet. As long as the client hasn't moved to a new stream, the
/// server won't send another OnOutputConstraints() until after the client
/// has configured output buffers.
///
/// This message can be sent mid-stream by a server. If
/// buffer_constraints_action_required false, the message is safe to
/// ignore, but a client may choose to stash the new constraints for
/// later use the next time the client wants to unilaterally re-configure
/// buffers (when allowed). If later the server needs the output config to
/// change, the server may send a new OnOutputConstraints() with
/// buffer_constraints_action_required true.
///
/// On buffer_constraints_action_required true, a client that does not wish
/// to fully handle mid-stream output buffer config changes should either
/// give up completely on the processing, or at least re-config the output
/// as specified before starting a new stream (and possibly re-delivering
/// input data, if the client wants). This avoids useless retry with a new
/// stream starting from just before the output buffer config change which
/// would hit the same mid-stream output config change again.
///
/// Similarly, some servers may only partly support mid-stream format
/// changes, or only support a mid-stream format change if the buffers are
/// already large enough to handle both before and after the format change.
/// Such servers should still indicate buffer_constraints_action_required
/// true, but then send OnStreamFailed() after the client has re-configured
/// output buffers (seamlessly dealing with the mid-stream output config
/// change is even better of course, but is not always feasible depending on
/// format). When the client retries with a new stream starting from a
/// nearby location in the client's logical overall media timeline, the
/// output buffers will already be suitable for the larger size output, so
/// the new stream will not need any mid-stream output buffer re-config,
/// only a mid-stream OnOutputFormat(). This strategy avoids the problem
/// that would otherwise occur if a client were to retry with a new stream
/// starting just before the mid-stream output buffer config change (the
/// retry wouldn't be effective since the same need for an output buffer
/// config change would be hit again). Servers are discouraged from sending
/// OnStreamFailed() solely due to a mid-stream need for different output
/// buffer config without first sending OnOutputConstraints() with
/// buffer_constraints_action_required true and waiting for the client to
/// re-configure output buffers (to avoid the useless client retry with a
/// new stream from a logical location before the config change).
///
/// When buffer_constraints_action_required true, the server will not send
/// any OnOutputPacket() for this stream until after the client has
/// configured/re-configured output buffers.
///
/// A client that gives up on processing a stream on any mid-stream
/// OnOutputConstraints() or mid-stream OnOutputFormat() should completely
/// ignore any OnOutputConstraints() with buffer_constraints_action_required
/// false. Otherwise the client may needlessly fail processing, or server
/// implementations might not be able to use
/// buffer_constraints_action_required false for fear of simpler clients
/// just disconnecting.
///
/// All clients, even those which don't want to support any mid-stream
/// output buffer re-config or mid-stream OnOutputFormat() are required to
/// deal with 1..multiple OnOutputConstraints() messages before the first
/// output packet, and 1 OnOutputFormat() messages before the first output
/// packet.
///
/// This message is ordered with respect to output packets, and with respect
/// to OnOutputFormat().
-> OnOutputConstraints(struct {
output_config StreamOutputConstraints;
});
/// This message is sent by the server before the first output packet of any
/// stream, and potentially mid-stream between output packets of the stream,
/// ordered with respect to output packets, and ordered with respect to
/// OnOutputConstraints().
///
/// The server guarantees that the first packet of every stream will be
/// preceeded by an OnOutputFormat().
///
/// The server guarantees that there will be an OnOutputFormat() between an
/// OnOutputConstraints() with buffer_constraints_action_required true and an
/// OnOutputPacket(). In other words, the client is essentially allowed to
/// forget what the output format is on any OnOutputConstraints() with
/// buffer_constraints_action_required true, because the server promises a
/// subsequent OnOutputFormat() before any OnOutputPacket().
///
/// If the server sets buffer_constraints_action_required true in
/// OnOutputConstraints(), the server won't send OnOutputFormat() (and
/// therefore also won't send OnOutputPacket()) until the client has
/// re-configured output buffers.
///
/// The server is allowed to send an OnOutputFormat() mid-stream between two
/// output packets.
///
/// A server won't send two adjacent OnOutputFormat() messages without any
/// output packet in between. However an OnOutputFormat() message doesn't
/// guarantee a subsequent packet, because for example the server could send
/// OnOutputEndOfStream() or OnStreamFailed() instead.
///
/// A client that does not wish to seamlessly handle mid-stream output format
/// changes should either ensure that no stream processed by the client
/// ever has any mid-stream format change, or the client should ensure that
/// any retry of processing starts the new attempt at a point logically at or
/// after the point where the old format has ended and the new format starts,
/// else the client could just hit the same mid-stream format change again.
///
/// An example of this message being sent mid-stream is mid-stream change
/// of dimensions of video frames output from a video decoder.
///
/// Not all servers will support seamless handling of format change. Those
/// that do support seamless handling of format change may require that the
/// format change not also require output buffer re-config, in order for the
/// handling to be seamless. See the comment block for OnOutputConstraints()
/// for more discussion of how servers and clients should behave - in
/// particular when they don't seamlessly handle output constraint change
/// and/or output format change.
///
/// If this message isn't being sent by the server when expected at the
/// start of a stream, the most common reason is that a OnOutputConstraints()
/// with buffer_constraints_action_required true hasn't been processed by the
/// client (by configuring output buffers using
/// SetOutputBufferPartialSettings() etc).
-> OnOutputFormat(struct {
output_format StreamOutputFormat;
});
/// This is the replacement for SetOutputBufferSettings().
///
/// When the client is using sysmem to allocate buffers, this message is
/// used instead of SetOutputBufferSettings()+AddOutputBuffer(). Instead, a
/// single SetOutputBufferPartialSettings() provides the StreamProcessor
/// with the client-specified output settings and a BufferCollectionToken
/// which the StreamProcessor will use to convey constraints to sysmem.
/// Both the client and the StreamProcessor will be informed of the
/// allocated buffers directly by sysmem via their BufferCollection channel
/// (not via the StreamProcessor channel).
///
/// Configuring output buffers is _required_ after OnOutputConstraints() is
/// received by the client with buffer_constraints_action_required true and
/// stream_lifetime_ordinal equal to the client's current
/// stream_lifetime_ordinal (even if there is an active stream), and is
/// _permitted_ any time there is no current stream.
///
/// Closing the current stream occurs on the StreamControl ordering domain,
/// so after a CloseCurrentStream() or FlushEndOfStreamAndCloseStream(), a
/// subsequent Sync() completion must be received by the client before the
/// client knows that there's no longer a current stream.
///
/// See also CompleteOutputBufferPartialSettings().
SetOutputBufferPartialSettings(resource struct {
output_settings StreamBufferPartialSettings;
});
/// After SetOutputBufferPartialSettings(), the server won't send
/// OnOutputConstraints(), OnOutputFormat(), OnOutputPacket(), or
/// OnOutputEndOfStream() until after the client sends
/// CompleteOutputBufferPartialSettings().
///
/// Some clients may be able to send
/// CompleteOutputBufferPartialSettings() immediately after
/// SetOutputBufferPartialSettings() - in that case the client needs to be
/// prepared to receive output without knowing the buffer count or packet
/// count yet - such clients may internally delay processing the received
/// output until the client has heard from sysmem (which is when the client
/// will learn the buffer count and packet count).
///
/// Other clients may first wait for sysmem to allocate, prepare to receive
/// output, and then send CompleteOutputBufferPartialSettings().
CompleteOutputBufferPartialSettings(struct {
buffer_lifetime_ordinal uint64;
});
/// This message is optional.
///
/// This message is only valid after QueueInputEndOfStream() for this stream.
/// The stream_lifetime_ordinal input parameter must match the
/// stream_lifetime_ordinal of the QueueInputEndOfStream(), else the server
/// will close the channel.
///
/// A client can use this message to flush through (not discard) the last
/// input data of a stream so that the stream processor server generates
/// corresponding output data for all the input data before the server moves
/// on to the next stream, without forcing the client to wait for
/// OnOutputEndOfStream() before queueing data of another stream.
///
/// The difference between QueueInputEndOfStream() and
/// FlushEndOfStreamAndCloseStream(): QueueInputEndOfStream() is a promise
/// from the client that there will not be any more input data for the
/// stream (and this info is needed by some stream processors for the stream
/// processor to ever emit the very last output data). The
/// QueueInputEndOfStream() having been sent doesn't prevent the client from
/// later completely discarding the rest of the current stream by closing
/// the current stream (with or without a stream switch). In contrast,
/// FlushEndOfStreamAndCloseStream() is a request from the client that all
/// the previously-queued input data be processed including the logical
/// "EndOfStream" showing up as OnOutputEndOfStream() (in success case)
/// before moving on to any newer stream - this essentially changes the
/// close-stream handling from discard to flush-through for this stream
/// only.
///
/// A client using this message can start providing input data for a new
/// stream without that causing discard of old stream data. That's the
/// purpose of this message - to allow a client to flush through (not
/// discard) the old stream's last data (instead of the default when closing
/// or switching streams which is discard).
///
/// Because the old stream is not done processing yet and the old stream's
/// data is not being discarded, the client must be prepared to continue to
/// process OnOutputConstraints() messages until the stream_lifetime_ordinal
/// is done. The client will know the stream_lifetime_ordinal is done when
/// OnOutputEndOfStream(), OnStreamFailed(), or the StreamProcessor channel
/// closes.
FlushEndOfStreamAndCloseStream(struct {
stream_lifetime_ordinal uint64;
});
/// This "closes" the current stream, leaving no current stream. In
/// addition, this message can optionally release input buffers or output
/// buffers.
///
/// If there has never been any active stream, the stream_lifetime_ordinal
/// must be zero or the server will close the channel. If there has been an
/// active stream, the stream_lifetime_ordinal must be the most recent
/// active stream whether that stream is still active or not. Else the
/// server will close the channel.
///
/// Multiple of this message without any new active stream in between is not
/// to be considered an error, which allows a client to use this message to
/// close the current stream to stop wasting processing power on a stream the
/// user no longer cares about, then later decide that buffers should be
/// released and send this message again with release_input_buffers and/or
/// release_output_buffers true to get the buffers released, if the client is
/// interested in trying to avoid overlap in resource usage between old
/// buffers and new buffers (not all clients are).
///
/// See also Sync().
CloseCurrentStream(struct {
stream_lifetime_ordinal uint64;
release_input_buffers bool;
release_output_buffers bool;
});
/// On completion, all previous StreamProcessor calls have done what they're
/// going to do server-side, _except_ for processing of data queued using
/// QueueInputPacket().
///
/// The main purpose of this call is to enable the client to wait until
/// CloseCurrentStream() with release_input_buffers and/or
/// release_output_buffers set to true to take effect, before the client
/// allocates new buffers and re-sets-up input and/or output buffers. This
/// de-overlapping of resource usage can be worthwhile for media buffers
/// which can consume resource types whose overall pools aren't necessarily
/// vast in comparison to resources consumed. Especially if a client is
/// reconfiguring buffers multiple times.
///
/// Note that Sync() prior to allocating new media buffers is not alone
/// sufficient to achieve non-overlap of media buffer resource usage system
/// wide, but it can be a useful part of achieving that.
///
/// The Sync() transits the Output ordering domain and the StreamControl
/// ordering domain, but not the InputData ordering domain.
///
/// This request can be used to avoid hitting kMaxInFlightStreams which is
/// presently 10. A client that stays <= 8 in-flight streams will
/// comfortably stay under the limit of 10. While the protocol permits
/// repeated SetInputBufferSettings() and the like, a client that spams the
/// channel can expect that the channel will just close if the server or the
/// channel itself gets too far behind.
Sync() -> ();
/// This is how the stream processor emits an output packet to the stream
/// processor client.
///
/// Order is significant.
///
/// The client should eventually call RecycleOutputPacket() (possibly after
/// switching streams multiple times), unless the buffer_lifetime_ordinal
/// has moved on. A stream change doesn't change which packets are busy
/// with the client vs. free with the server.
///
/// The relevant buffer is always the one specified in the packet's buffer_index field.
///
/// For low-level buffer types that support it, a StreamProcessor is free to
/// emit an output packet before the low-level buffer actually has any
/// usable data in the buffer, with the mechanism for signalling the
/// presence of data separate from the OnOutputPacket() message. For such
/// low-level buffer types, downstream consumers of data from the emitted
/// packet must participate in the low-level buffer signalling mechanism to
/// know when it's safe to consume the data. This is most likely to be
/// relevant when using a video decoder and gralloc-style buffers.
///
/// The error_ bool(s) allow (but do not require) a StreamProcessor server
/// to report errors that happen during an AU or between AUs.
///
/// The scope of error_detected_before starts at the end of the last
/// delivered output packet on this stream, or the start of stream if there
/// were no previous output packets on this stream. The scope ends at the
/// start of the output_packet.
///
/// The error_detected_before bool is separate so that discontinuities can be
/// indicated separately from whether the current packet is damaged.
///
/// The scope of error_detected_during is from the start to the end of this
/// output_packet.
-> OnOutputPacket(struct {
output_packet Packet;
error_detected_before bool;
error_detected_during bool;
});
/// After the client is done with an output packet, the client needs to tell
/// the stream processor that the output packet can be re-used for more
/// output, via this method.
///
/// It's not permitted to recycle an output packet that's already free with
/// the stream processor server. It's permitted but discouraged for a
/// client to recycle an output packet that has been deallocated by an
/// explicit or implicit output buffer de-configuration(). See
/// buffer_lifetime_ordinal for more on that. A server must ignore any such
/// stale RecycleOutputPacket() calls.
RecycleOutputPacket(struct {
available_output_packet PacketHeader;
});
/// After QueueInputEndOfStream() is sent by the StreamProcessor client,
/// within a reasonable duration the corresponding OnOutputEndOfStream()
/// will be sent by the StreamProcessor server. Similar to
/// QueueInputEndOfStream(), OnOutputEndOfStream() is sent a maximum of once
/// per stream.
///
/// No more stream data for this stream will be sent after this message. All
/// input data for this stream was processed.
///
/// While a StreamProcessor client is not required to
/// QueueInputEndOfStream() (unless the client wants to use
/// FlushEndOfStreamAndCloseStream()), if a StreamProcessor server receives
/// QueueInputEndOfStream(), and the client hasn't closed the stream, the
/// StreamProcessor server must generate a corresponding
/// OnOutputEndOfStream() if nothing went wrong, or must send
/// OnStreamFailed(), or must close the server end of the StreamProcessor
/// channel. An ideal StreamProcessor server would handle and report stream
/// errors via the error_ flags and complete stream processing without
/// sending OnStreamFailed(), but in any case, the above-listed options are
/// the only ways that an OnOutputEndOfStream() won't happen after
/// QueueInputEndOfStream().
///
/// There will be no more OnOutputPacket() or OnOutputConstraints() messages
/// for this stream_lifetime_ordinal after this message - if a server doesn't
/// follow this rule, a client should close the StreamProcessor channel.
///
/// The error_detected_before bool has the same semantics as the
/// error_detected_before bool in OnOutputPacket().
-> OnOutputEndOfStream(struct {
stream_lifetime_ordinal uint64;
error_detected_before bool;
});
// TODO(dustingreen): Rename from QueueInputFormatDetails() to
// QueueInputFormat().
//
/// If the input format details are still the same as specified during
/// StreamProcessor creation, this message is unnecessary and does not need
/// to be sent.
///
/// If the stream doesn't exist yet, this message creates the stream.
///
/// The server won't send OnOutputConstraints() until after the client has
/// sent at least one QueueInput* message.
///
/// All servers must permit QueueInputFormatDetails() at the start of a
/// stream without failing, as long as the new format is supported by the
/// StreamProcessor instance. Technically this allows for a server to only
/// support the exact input format set during StreamProcessor creation, and
/// that is by design. A client that tries to switch formats and gets a
/// StreamProcessor channel failure should try again one more time with a
/// fresh StreamProcessor instance created with CodecFactory using the new
/// input format during creation, before giving up.
///
/// These format details override the format details specified during stream
/// processor creation for this stream only. The next stream will default
/// back to the format details set during stream processor creation.
///
/// This message is permitted at the start of the first stream (just like at
/// the start of any stream). The format specified need not match what was
/// specified during stream processor creation, but if it doesn't match, the
/// StreamProcessor channel might close as described above.
QueueInputFormatDetails(struct {
stream_lifetime_ordinal uint64;
format_details FormatDetails;
});
/// This message queues input data to the stream processor for processing.
///
/// If the stream doesn't exist yet, this message creates the new stream.
///
/// The server won't send OnOutputConstraints() until after the client has
/// sent at least one QueueInput* message.
///
/// The client must continue to deliver input data via this message even if
/// the stream processor has not yet generated the first OnOutputConstraints(),
/// and even if the StreamProcessor is generating OnFreeInputPacket() for
/// previously-queued input packets. The input data must continue as long
/// as there are free packets to be assured that the server will ever
/// generate the first OnOutputConstraints().
QueueInputPacket(struct {
packet Packet;
});
/// The server sends this message when the stream processor is done
/// consuming the data in this packet (but not necessarily done processing
/// the data) and the packet can be re-filled by the client.
///
/// This is not sent for all packets when a new buffer_lifetime_ordinal
/// starts as in that case all the packets are initially free with the
/// client.
///
/// After receiving the available input buffer via this event, the stream
/// processor client can call later call QueueInputBuffer with appropriate
/// offset and length set, with the same packet_index, to re-use the
/// packet_index.
///
/// OnFreeInputPacket() does _not_ imply that the data in the input packet
/// has been processed successfully, only that the input data is no longer
/// needed by the StreamProcessor. If a client needs to know which input
/// data has generated corresponding output, using timestamp_ish values for
/// that is recommended.
///
/// Any reliance on the relative order of OnFreeInputPacket() and
/// OnStreamFailed() is discouraged and deprecated. Instead, use
/// timstamp_ish values to establish which input packets generated
/// corresponding output packets. Note that even using timestamp_ish values
/// doesn't necessarily imply that the processing of input data with a given
/// timestamp_ish value is fully complete, as in some StreamProcessor(s) the
/// data derived from an input packet can be kept for reference purposes for
/// a long time (in general indefinitely) after the input data has generated
/// its primary output data (the output data to which the timestamp_ish
/// value is attached). The StreamProcessor interface currently does not
/// provide any way to determine when all data derived from an input packet
/// has been discarded by the StreamProcessor, and if such a mechanism is
/// ever added to the StreamProcessor protocol, it would be an optional
/// StreamProcessor capability, since it would be infeasible to implement
/// for some StreamProcessor implementations that rely on external means to
/// process data, where the external means won't necessarily provide info
/// regarding when an input packet's derived data is fully discarded. An
/// input packet's derived data will never generate or contribute to any
/// output data for a different stream.
///
/// The order of OnFreeInputPacket() is not guaranteed to be the same as the
/// order of QueueInputPacket(). Any reliance on the order being the same
/// is strongly discouraged and deprecated. Clients are expected to work
/// properly even if the order of OnFreeInputPacket() messages is
/// intentionally scrambled with respect to each other (but not scrambled
/// across OnStreamFailed(), for now).
-> OnFreeInputPacket(struct {
free_input_packet PacketHeader;
});
/// Inform the server that all QueueInputPacket() messages for this stream
/// have been sent.
///
/// If the stream isn't closed first (by the client, or by OnStreamFailed(),
/// or StreamProcessor channel closing), there will later be a corresponding
/// OnOutputEndOfStream().
///
/// The corresponding OnOutputEndOfStream() message will be generated only if
/// the server finishes processing the stream before the server sees the
/// client close the stream (such as by starting a new stream). A way to
/// force the server to finish the stream before closing is to use
/// FlushEndOfStreamAndCloseStream() after QueueInputEndOfStream() before any
/// new stream. Another way to force the server to finish the stream before
/// closing is to wait for the OnOutputEndOfStream() before taking any action
/// that closes the stream.
///
/// In addition to serving as an "EndOfStream" marker to make it obvious
/// client-side when all input data has been processed, if a client never
/// sends QueueInputEndOfStream(), no amount of waiting will necessarily
/// result in all input data getting processed through to the output. Some
/// stream processors have some internally-delayed data which only gets
/// pushed through by additional input data _or_ by this EndOfStream marker.
/// In that sense, this message can be viewed as a flush-through at
/// InputData domain level, but the flush-through only takes effect if the
/// stream processor even gets that far before the stream is just closed at
/// StreamControl domain level. This message is not alone sufficient to act
/// as an overall flush-through at StreamControl level. For that, send this
/// message first and then send FlushEndOfStreamAndCloseStream() (at which
/// point it becomes possible to queue input data for a new stream without
/// causing discard of this older stream's data), or wait for the
/// OnOutputEndOfStream() before closing the current stream.
///
/// If a client sends QueueInputPacket(), QueueInputFormatDetails(),
/// QueueInputEndOfStream() for this stream after the first
/// QueueInputEndOfStream() for this stream, a server should close the
/// StreamProcessor channel.
QueueInputEndOfStream(struct {
stream_lifetime_ordinal uint64;
});
};
/// Deprecated. Use SetStreamBufferPartialSettings() and
/// StreamBufferPartialSettings instead.
@deprecated("Ignore. Use SetStreamBufferPartialSettings instead.")
type StreamBufferSettings = table {
@deprecated("Ignore. Use SetStreamBufferPartialSettings instead.")
1: buffer_lifetime_ordinal uint64;
@deprecated("Ignore. Use SetStreamBufferPartialSettings instead.")
2: buffer_constraints_version_ordinal uint64;
@deprecated("Ignore. Use SetStreamBufferPartialSettings instead.")
3: packet_count_for_server uint32;
@deprecated("Ignore. Use SetStreamBufferPartialSettings instead.")
4: packet_count_for_client uint32;
@deprecated("Ignore. Use SetStreamBufferPartialSettings instead.")
5: per_packet_buffer_bytes uint32;
@deprecated("Ignore. Use SetStreamBufferPartialSettings instead.")
6: single_buffer_mode bool;
};