blob: f67d1844b708b9f4a11e77051cb881461545f626 [file] [log] [blame]
// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
using fuchsia.sysmem;
// See for detailed interface documentation. The comments
// here are a summary only. Client implementers should see
// for more detail on any message that doesn't seem sufficiently-described
// here. StreamProcessor server implementers should probably read
// before implementing.
// Overview of operation:
// 1. Create
// * create via CodecFactory - see CodecFactory
// * create via LicenseSession - see LicenseSession
// 2. Get input constraints
// * OnInputConstraints() - sent unsolicited by stream processor shortly after
// stream processor creation.
// 3. Provide input buffers
// * SetInputBufferSettings() / AddInputBuffer()
// 4. Deliver input data
// * QueueInputPacket() + OnFreeInputPacket(), for as long as it takes,
// possibly working through all input packets repeatedly before...
// 5. Get output constraints and format
// * OnOutputConstraints() - may be delivered as early as before
// OnInputConstraints() by some stream processors, but a client must
// tolerate as late as after substantial input data has been delivered
// including lots of input packet recycling via OnFreeInputPacket().
// * This message can arrive more than once before the first output data.
// 6. Provide output buffers
// * SetOutputBufferSettings() / AddOutputBuffer()
// 7. Data flows, with optional EndOfStream
// * OnOutputPacket() / RecycleOutputPacket() / QueueInputPacket() /
// OnFreeInputPacket() / QueueInputEndOfStream() / OnOutputEndOfStream()
// It's possible to re-use a StreamProcessor instance for another stream, and
// doing so can sometimes skip over re-allocation of buffers. This can be a
// useful thing to do for cases like seeking to a new location - at the
// StreamProcessor interface that can look like switching to a new stream.
// StreamBufferConstraints
// This struct helps ensure that packet count and buffer space are sufficient
// to avoid major problems. For example, a video decoder needs sufficient
// video frame buffers to hold all potential reference frames concurrently +
// one more video buffer to decode into. Else, the whole video decode pipe can
// easily deadlock.
// The secondary purpose of this struct is to help ensure that packet count and
// buffer space are sufficient to achieve reasonably performant operation.
// There are separate instances of this struct for stream input and stream
// output.
// TODO(dustingreen): Some of the buffer-focused fields in this structure will
// go away in favor of using sysmem for those aspects. The packet count fields
// will stay. The single-buffer fields will stay.
table StreamBufferConstraints {
// This is a version number the server sets on the constraints to allow the
// server to determine when the client has caught up with the latest
// constraints sent by the server. The server won't emit output data until
// the client has configured output settings and buffers with a
// buffer_constraints_version_ordinal >= the latest
// buffer_constraints_version_ordinal that had
// buffer_constraints_action_required true. See
// buffer_constraints_action_required comments for more.
// A buffer_constraints_version_ordinal of 0 is not permitted, to simplify
// initial state handling. Other than 0, both odd and even version ordinals
// are allowed (in contrast to the stream_lifetime_ordinal, neither the
// client nor server ever has a reason to consider the latest version to be
// stale, so there would be no benefit to disallowing even values).
1: uint64 buffer_constraints_version_ordinal;
// default_settings
// These settings are "default" settings, not "recommended" settings.
// These "default" settings can be passed to SetInputBufferSettings() /
// SetOutputBufferSettings() as-is without modification, but a client doing
// that must still obey the semantics of packet_count_for_client, despite
// the stream processor server not having any way to really know the proper
// setting for that field.
// For StreamBufferConstraints fields whose names end in "recommended", the
// default_settings will have the corresponding setting field set to that
// recommended value.
// The stream processor promises that these default settings as-is (except
// for buffer_lifetime_ordinal) are guaranteed to satisfy the constraints
// indicated by the other fields of StreamBufferConstraints. While
// client-side checking that these settings are within the constraints is
// likely unnecessary in the client, the client should still check that
// these values are within client-side reasonable-ness bounds before using
// these values, to avoid letting a stream processor server cause problems
// for the client.
// This structure will always have single_buffer_mode false. See
// single_buffer_mode_allowed for whether single_buffer_mode true is
// allowed.
// The client must set the buffer_lifetime_ordinal field to a proper value
// before sending back to the server. The 0 initially in this field will
// be rejected by the server if sent back as-is. See comments on
// StreamBufferSettings.buffer_lifetime_ordinal.
2: StreamBufferSettings default_settings;
// For uncompressed video, separate and complete frames, each in its own
// separate buffer (buffer-per-packet mode), is always a requirement.
// per_packet_buffer_bytes.*:
// These per-packet buffer bytes constraints apply to both buffer-per-packet
// mode and single-buffer mode (see single_buffer_mode). If
// buffer-per-packet mode, the constraints apply to each buffer separately.
// If single-buffer mode, the constraints need to be multiplied by the number
// of packets to determine the constraints on the single buffer.
// per_packet_buffer_bytes_min:
// If a client is using buffer per packet mode, each buffer must be at least
// this large. If a client is using single-buffer mode, the one buffer must
// be at least per_packet_buffer_bytes_min * packet_count_for_server_min in
// size.
3: uint32 per_packet_buffer_bytes_min;
// Must be >= per_packet_buffer_bytes_min. Delivering more than
// this per input packet might not perform any better, and in fact might
// perform worse.
4: uint32 per_packet_buffer_bytes_recommended;
// Must be >= per_packet_buffer_bytes_recommended. Can be 0xFFFFFFFF if
// there is no explicitly-enforced limit.
5: uint32 per_packet_buffer_bytes_max;
// Minimum number of packet_count_for_server.
// Re. input and output:
// This is a strict min for packet_count_for_server, but a client can use
// more packets overall if the client wants to, by using a larger value for
// packet_count_for_server and/or using a non-zero packets_for_client. A
// good reason to do the former would be if the client might tend to
// deliver a few not-very-full buffers occasionally - or to have a few
// extra packets within which to satisfy stream_input_bytes_min. A good
// reason to do the latter would be if a client needs to hold onto some
// packets for any "extra" duration.
// If a client specifies a larger packet_count_for_server value than
// packet_count_for_server_min, a server is permitted (but not encouraged)
// to not make progress until packet_count_for_server are with the server,
// not merely packet_count_for_server_min.
// For decoder input and audio encoder input: The
// packet_count_for_server_min may or may not contain enough data to allow
// the stream processor to make progress without copying into an internal
// side buffer. If there isn't enough data delivered in
// packet_count_for_server_min packets to permit progress, the stream
// processor must copy into its own side buffer internally to make
// progress.
// If a client intends to use extra packets for client-side purposes, the
// client should specify the extra packets in packets_for_client instead of
// packet_count_for_server, but packet_count_for_server must still be >=
// packet_count_for_server_min.
// See for more on packet_count_for_server_min.
6: uint32 packet_count_for_server_min;
// This must be at least packet_count_for_server_min and at most
// packet_count_for_server_recommended_max.
// This value is likely to be used as-is by most clients, so if having one
// additional packet is a big performance win in a large percentage of
// scenarios, it can be good for the server to include that additional
// packet in this value.
7: uint32 packet_count_for_server_recommended;
// This can be the same as packet_count_for_server_max or can be lower.
// Values above this value and <= packet_count_for_server_max are not
// recommended by the stream processor, but should still work given
// sufficient resources available to both the client and the stream
// processor.
8: uint32 packet_count_for_server_recommended_max;
// This can be 0xFFFFFFFF if there's no stream processor-enforced max, but
// stream processors are encouraged to set a large but still
// plausibly-workable max, and clients are encouraged to request a number
// of packets that isn't excessively large for the client's scenario.
9: uint32 packet_count_for_server_max;
// Normally this would be an implicit 0, but for now we have a min so we can
// force the total number of packets to be a specific number that we know
// works for the moment.
10: uint32 packet_count_for_client_min;
// packet_count_for_client_max
// The client must set packet_count_for_client to be <=
// packet_count_for_client_max.
// This value must be at least 1. This can be 0xFFFFFFFF if there's no
// stream-processor-enforced max. Clients are encouraged to request a
// number of packets that isn't excessively large for the client's
// scenario.
11: uint32 packet_count_for_client_max;
// single_buffer_mode_allowed false allows a stream processor that's not
// required to support single-buffer mode for a given input or output the
// ability to decline to support single-buffer mode on that input/output.
// All encoder output, regardless of audio or video: server support for
// single-buffer mode is optional. Please see for all
// the rules regarding single-buffer mode on output before using
// single-buffer mode on an output.
// Audio decoder output: server support for single-buffer mode is required.
// Video decoder output: There is little reason for a video decoder to
// support single-buffer mode on output. Nearly all video decoders will set
// this to false for their output.
// All decoder inputs: Servers must support single-buffer mode on input.
// The client is responsible for managing the input buffer space such that
// filling an input packet doesn't over-write any portion of an input
// packet already in flight to the stream processor.
// Encoder inputs: Server support for single-buffer mode on encoder input is
// optional. This is more often useful for audio than for video.
// Support for buffer-per-packet mode is always required on both input and
// output, regardless of stream processor type.
12: bool single_buffer_mode_allowed;
// If true, the buffers need to be physically contiguous pages, such as can
// be allocated using zx_vmo_create_contiguous() (this syscall requires a
// bti handle which many clients won't have - the client will most likely
// want to use sysmem to allocate buffers).
13: bool is_physically_contiguous_required;
// VERY TEMPORARY HACK / KLUDGE - we want the BufferAllocator (or one of
// the participant drivers that needs physically contiguous buffers) to
// call zx_vmo_create_contiguous(), definitely not the StreamProcessor
// client, but until the BufferAllocator can be reached from a driver, this
// is to grant the client special powers it really shouldn't have, very
// temporarily until BufferAllocator is hooked up properly at which point
// this can be removed. Strictly speaking we could reverse which end
// allocates buffers in the StreamProcessor interface to avoid this hack
// even before BufferAllocator, but the overall path seems shorter if we
// jump directly from this to using BufferAllocator.
// TODO(dustingreen): remove once zero clients need this (zero clients that
// need to allocate physically contiguous buffers directly / all relevant
// clients using sysmem).
14: handle very_temp_kludge_bti_handle;
// StreamOutputConstraints
// The stream-processor-controlled output configuration, including both
// StreamBufferConstraints for the output and FormatDetails for the output.
// TODO(dustingreen): Need a better name for this struct, but still short
// hopefully. It's stuff the stream processor gets to control, not the client.
// It's different than output buffer settings, which the client does get to
// control to some extent. It's different than any configurable output
// settings the client might specify for output of an encoder.
table StreamOutputConstraints {
// A client which always immediately re-configures output buffers on
// receipt of OnOutputConstraints() with buffer_constraints_action_required true
// can safely ignore this field.
// A client is permitted to ignore an OnOutputConstraints() message even with
// buffer_constraints_action_required true if the client knows the server
// has already been told to discard the remainder of the stream with the
// same stream_lifetime_ordinal or if this stream_lifetime_ordinal field is
// set to 0. The server is required to re-send needed output config via
// OnOutputConstraints() with new stream_lifetime_ordinal and
// buffer_constraints_action_required true, if the most recent completed
// server-side output config isn't what the server wants/needs yet for the
// new stream.
1: uint64 stream_lifetime_ordinal;
// buffer_constraints_action_required
// When the buffer constraints are delivered, they indicate whether action
// is required. A false value here permits delivery of constraints which
// are fresher without forcing a buffer reconfiguration. If this is false,
// a client cannot assume that it's safe to immediately re-configure output
// buffers. If this is true, the client can assume it's safe to
// immediately configure output buffers once.
// A client is permitted to ignore buffer constraint versions which have
// buffer_constraints_action_required false. The server is not permitted
// to change buffer_constraints_action_required from false to true for the
// same buffer_constraints_version_ordinal.
// For each configuration, a client must use new buffers, never buffers
// that were previously used for anything else, and never buffers
// previously used for any other StreamProcessor purposes. This rule
// exists for multiple good reasons, relevant to both mid-stream changes,
// and changes on stream boundaries. A client should just use new buffers
// each time.
// When this is true, the server has already de-refed as many low-level
// output buffers as the server can while still performing efficient
// transition to the new buffers and will de-ref the rest asap. A Sync()
// is not necessary to achieve non-overlap of resource usage to the extent
// efficiently permitted by the formats involved.
// If buffer_constraints_action_required is true, the server _must_ not
// deliver more output data until after output buffers have been configured
// (or re-configured) by the client.
// See for more on buffer_constraints_action_required.
2: bool buffer_constraints_action_required;
3: StreamBufferConstraints buffer_constraints;
table StreamOutputFormat {
// A client is permitted to ignore an OnOutputFormat() message even with
// buffer_constraints_action_required true if the client knows the server
// has already been told to discard the remainder of the stream with the
// same stream_lifetime_ordinal or if this stream_lifetime_ordinal field is
// set to 0. The server is required to re-send needed output config via
// OnOutputConstraints() with new stream_lifetime_ordinal and
// buffer_constraints_action_required true, if the most recent completed
// server-side output config isn't what the server wants/needs yet for the
// new stream.
// The server is required to send an OnOutputFormat() before the first
// output packet of a stream.
1: uint64 stream_lifetime_ordinal;
// format_details
// It's up to the client to determine if a change in
// format_details.format_details_version_ordinal implies any client action
// is required, based on particular fields in format_details vs. any old
// value. The server guarantees that if the format has changed, then
// format_details.format_details_version_ordinal will change, but a change
// to format_details.format_details_version_ordinal does not guarantee that
// the format details actually changed. Servers are strongly encouraged to
// not change format_details.format_details_version_ordinal other than
// before the first output data of a stream unless there is a real
// mid-stream format change in the stream. Unnecessary mid-stream format
// changes can cause simpler clients that have no need to handle mid-stream
// format changes to just close the channel. Format changes before the
// first output data of a stream are not "mid-stream" in this context -
// those can be useful for stream format detection / setup reasons.
// Note that in case output buffers don't really need to be re-configured
// despite a format change, a server is encouraged, but not required, to
// set buffer_constraints_action_required false on the message that conveys
// the new format details. Simpler servers may just treat the whole output
// situation as one big thing and demand output buffer reconfiguration on
// any change in the output situation.
// A client may or may not actually handle a new buffer_constraints with
// buffer_constraints_action_required false, but the client should always
// track the latest format_details.
// An updated format_details is ordered with respect to emitted output
// packets, and applies to all subsequent packets until the next
// format_details with larger version_ordinal. A simple client that does
// not intend to handle mid-stream format changes should still keep track
// of the most recently received format_details until the first output
// packet arrives, then lock down the format details, handle those format
// details, and verify that any
// format_details.format_details_version_ordinal received from the server
// is the same as the locked-down format_details, until the client is done
// with the stream. Even such a simple client must tolerate
// format_details.format_details_version_ordinal changing multiple times
// before the start of data output from a stream (any stream - the first
// stream or a subsequent stream). This allows a stream processor to
// request that output buffers and output format be configured
// speculatively, and for the output config to be optionally adjusted by
// the server before the first data output from a stream after the server
// knows everything it needs to know to fully establish the initial output
// format details. This simplifies stream processor server implementation,
// and allows a clever stream processor server to guess it's output config
// for lower latency before any input data, while still being able to fix
// the output config (including format details) if the guess turns out to
// be wrong.
// Whether the format_details.format_details_version_ordinal will actually
// change mid-stream is a per-stream-processor and per-stream detail that
// is not specified in comments here, and in most cases also depends on
// whether the format changes on the input to the stream processor.
// Probably it'll be fairly common for a client to use a format which
// technically supports mid-stream format change, but the client happens to
// know that none of the streams the client intends to process will ever
// have a mid-stream format change.
2: FormatDetails format_details;
// DEPRECATED - this is splitting into StreamOutputConstraints and
// StreamOutputFormat instead. See those tables.
table StreamOutputConfig {
1: uint64 stream_lifetime_ordinal;
2: bool buffer_constraints_action_required;
3: StreamBufferConstraints buffer_constraints;
4: FormatDetails format_details;
// Default values for input and output
// StreamBufferConstraints.default_settings.packet_count_for_server.
// These are defined as "const" in FIDL to avoid all server implementations
// needing to separately define their own values, and these should be
// reasonable as default values, but strictly speaking this is not intended to
// promise that this value won't change from build to build. If a client cares
// about a specific number, the client should separately define what that
// number is and ensure that StreamBufferSettings.packet_count_for_client is
// at least large enough.
// In contrast to packet_count_for_client, the packet_count_for_server is much
// more stream-processor-specific, so this file has no numbers for that - each
// stream processor will set those as appropriate for the specific stream
// processor.
// These are not "recommended" values, only "default" values, in the sense that
// the stream processor doesn't really know what the correct setting for these
// values is for a given client, and if the default is not appropriate for a
// client, large problems could result such as deadlock. See the comments on
// packet_count_for_client.
// Despite these defaults, every client should ideally care about the
// packet_count_for_client setting and should ensure that the setting is at
// least large enough to cover the number of packets the client might ever need
// to camp on for any non-transient duration concurrently. The defaults are
// only intended to be plausible for some clients, not all clients.
// One for the client to be filling and one in transit.
const uint32 kDefaultInputPacketCountForClient = 2;
// One for the client to be rendering, and one in transit.
const uint32 kDefaultOutputPacketCountForClient = 2;
// For input, this is the default on a fairly arbitrary basis.
// TODO(dustingreen): Do we want the default for audio encoding to be
// single_buffer_mode true instead? If so, we may split this up by audio/video
// encoder/decoder.
const bool kDefaultInputIsSingleBufferMode = false;
const bool kDefaultOutputIsSingleBufferMode = false;
// StreamBufferSettings
// See relevant corresponding constraints in StreamBufferConstraints. The
// settings must satisfy the constraints.
// The client informs the stream processor of these settings and then
// separately informs the stream processor of each buffer.
table StreamBufferSettings {
// buffer_lifetime_ordinal
// The containing message starts a new buffer_lifetime_ordinal.
// There is a separate buffer_lifetime_ordinal for input vs. output.
// Re-use of the same value is not allowed. Values must be odd. Values
// must only increase (increasing by more than 2 is permitted).
// A buffer_lifetime_ordinal lifetime starts at SetInputBufferSettings() or
// SetOutputBufferSettings(), and ends at the earlier of
// CloseCurrentStream() with release_input_buffers/release_output_buffers
// set or SetOutputBufferSettings() with new buffer_lifetime_ordinal in the
// case of mid-stream output config change.
// See for more on buffer_lifetime_ordinal.
1: uint64 buffer_lifetime_ordinal;
// buffer_constraints_version_ordinal
// This value indicates which version of constraints the client is/was aware
// of so far.
// For input, this must always be 0 because constraints don't change for
// input (settings can change, but there's no settings vs current
// constraints synchronization issue on input).
// For output, this allows the server to know when the client is
// sufficiently caught up before the server will generate any more output.
// When there is no active stream, a client is permitted to re-configure
// buffers again using the same buffer_constraints_version_ordinal.
// See for more on buffer_constraints_version_ordinal.
2: uint64 buffer_constraints_version_ordinal;
// The total packet count is split into two pieces to disambiguate how many
// packets are allocated for the client to hold onto for whatever reason,
// vs. how many packets are allocated for the server to hold onto for
// whatever reason.
// Extra packets to provide slack for performance reasons can be in either
// category, but typically packet_count_for_server_recommended will already
// include any performance-relevant slack for the server's benefit.
// packet_count_for_server
// How many packets the client is allocating for the stream processor
// server's use. This must be >=
// StreamBufferConstraints.packet_count_for_server_min. If constraints
// change such that this would no longer be true, the server will send an
// OnOutputConstraints() event.
// The stream processor server is allowed to demand that all of
// packet_count_for_server become free before making further progress, even
// if packet_count_for_server is > packet_count_for_server_min.
// A reasonable value for this is
// StreamBufferConstraints.packet_count_for_server_recommended.
// See for more on packet_count_for_server.
3: uint32 packet_count_for_server;
// packet_count_for_client
// This must be at least 1. The server will close the channel if this is 0.
// How many packets the client is allocating for the client's use. The
// client may hold onto this many packets for arbitrarily-long duration
// without handing these packets to the stream processor, and despite doing
// so, the stream processor will continue to make progress and function
// normally without getting stuck. The client holding onto additional
// packets transiently is ok, but the client needs to hand those additional
// packets back to the stream processor eventually if the client wants the
// stream processor to make further progress.
// In addition to this value needing to include at least as many packets as
// the client ever intends to concurrently camp on indefinitely, any extra
// slack to benefit client-side performance should also be included here.
// A typical value for this could be at least 2, but it depends strongly on
// client implementation and overall client buffering goals. It is up to
// the client to determine how many packets are needed in this category by
// any parts of the overall system that will be holding onto packets for
// any reason. Those parts of the system should have a documented and
// possibly queryable defined value to help determine this number. Setting
// this value lower than it actually needs to be can result in the stream
// processor not making progress as it sits waiting for packets, with the
// client unable to recycle any more packets to the stream processor. That
// situation can be difficult to diagnose, while excessively-large values
// here are wasteful, so care is warranted to set this value properly.
4: uint32 packet_count_for_client;
// per_packet_buffer_bytes
// In buffer-per-packet mode, we require that each buffer have usable bytes
// equal to per_packet_buffer_bytes. Use of differently-sized low-level
// buffers is possible, but the size of the portion used via the
// StreamProcessor interface per StreamBuffer must be the same for all the
// buffers.
// In single-buffer mode, we require the portion of the low-level buffer
// used via the StreamProcessor interface to be size
// (packet_count_for_server + packet_count_for_client) *
// per_packet_buffer_bytes.
// TODO(dustingreen): determine if we need to relax these restrictions a bit
// for convenience when using gralloc video buffers.
5: uint32 per_packet_buffer_bytes;
// If true, there is only one buffer with index 0 which all packets
// implicitly refer to. If false, the packet_index and buffer_index are
// 1:1, and each packet refers to its corresponding buffer.
// While it's possible to set up single_buffer_mode false with each buffer
// referring to the same underlying VMO, single_buffer_mode true is more
// efficient for that case since only one mapping is created.
6: bool single_buffer_mode;
// StreamBufferPartialSettings
// This struct is used instead of StreamBufferSettings when sysmem is used to
// allocate buffers. The settings in StreamBufferSettings that are missing from
// StreamBufferPartialSettings can be conveyed from the client directly to
// sysmem.
// TODO(dustingreen): Consider renaming this to StreamBufferSettings once that's
// an option.
table StreamBufferPartialSettings {
// buffer_lifetime_ordinal
// The containing message starts a new buffer_lifetime_ordinal.
// There is a separate buffer_lifetime_ordinal for input vs. output.
// Re-use of the same value is not allowed. Values must be odd. Values
// must only increase (increasing by more than 2 is permitted).
// A buffer_lifetime_ordinal lifetime starts at SetInputBufferSettings() or
// SetOutputBufferSettings(), and ends at the earlier of
// CloseCurrentStream() with release_input_buffers/release_output_buffers
// set or SetOutputBufferSettings() with new buffer_lifetime_ordinal in the
// case of mid-stream output config change.
// See for more on buffer_lifetime_ordinal.
1: uint64 buffer_lifetime_ordinal;
// buffer_constraints_version_ordinal
// This value indicates which version of constraints the client is/was aware
// of so far.
// For input, this must always be 0 because constraints don't change for
// input (settings can change, but there's no settings vs current
// constraints synchronization issue on input).
// For output, this allows the server to know when the client is
// sufficiently caught up before the server will generate any more output.
// When there is no active stream, a client is permitted to re-configure
// buffers again using the same buffer_constraints_version_ordinal.
// See for more on buffer_constraints_version_ordinal.
2: uint64 buffer_constraints_version_ordinal;
// If true, there is only one buffer, but still typically more than one
// packet. If false, the # of packets == the number of buffers.
// While it's possible to set up single_buffer_mode false with each buffer
// referring to the same underlying VMO, single_buffer_mode true is more
// efficient for that case since only one mapping is created.
// This setting is specified by the client, and influences the constraints
// delivered from the StreamProcessor to sysmem (whether there's more than
// one buffer allocated overall or not). For single_buffer_mode true, the
// StreamProcessor is the one to ask sysmem for a buffer - the client should
// refrain from doing so or the StreamProcessor will just fail when more
// than one buffer gets allocated by sysmem.
3: bool single_buffer_mode;
/// When single_buffer_mode is false:
/// The actual packet count will be
/// max(packet_count_for_server + packet_count_for_client, sysmem_buffers).
/// The sysmem_buffers is BufferCollectionInfo.buffer_count from sysmem if
/// using sysmem, or 0 if not using sysmem.
/// When single_buffer_mode is true:
/// The actual packet count is packet_count_for_server +
/// packet_count_for_client.
/// If not using sysmem, or if using single_buffer_mode, these fields must be
/// set and consistent with correpsonding fields in StreamBufferConstraints.
/// If single_buffer_mode false and using sysmem, these fields can both be
/// non-set, or can both be set and consistent with correpsonding fields in
/// StreamBufferConstraints. If not set, the value used for the fields in
/// the "max" expression above is 0, so buffer_count.
// TODO(FIDL-609): Default to 0.
4: uint32 packet_count_for_server;
// TODO(FIDL-609): Default to 0.
5: uint32 packet_count_for_client;
// The client end of a BufferCollectionToken channel, which the
// StreamProcessor will use to deliver constraints to sysmem and learn of
// buffers allocated by sysmem.
// The client guarantees that the token is already known to sysmem (via
// BufferCollectionToken.Sync(), BufferCollection.Sync(), or
// BufferCollectionEvents.OnDuplicatedTokensKnownByServer()).
6: fuchsia.sysmem.BufferCollectionToken sysmem_token;
// StreamBuffer
// The StreamBuffer struct represents a pre-configured buffer.
// Both input and output uses StreamBuffer(s), but the two sets of buffers are
// separate.
// The client uses SetInputBufferSettings() + AddInputBuffer() * N to inform
// the stream processor about all the input buffers.
// The client uses SetOutputBufferSettings() + AddOutputBuffer() * N to inform
// the stream processor about all the output buffers.
// When single_buffer_mode is true, there is only buffer_index 0 shared by all
// Packet(s) of the relevant input or output. When single_buffer_mode is
// false, the buffer_index equals the packet_index.
table StreamBuffer {
// When using AddOutputBuffer()/AddInputBuffer(), this must match the
// buffer_lifetime_ordinal of the most recent
// SetOutputBufferSettings()/SetInputBufferSettings().
1: uint64 buffer_lifetime_ordinal;
// Buffers must be added via AddOutputBuffer() / AddInputBuffer() in order
// by buffer_index, and the buffer_index is always equal to 0 or equal to
// the packet_index (depending on single_buffer_mode true or false), but
// it's still nice to have StreamBuffer include the buffer_index if only
// for easier debugging.
2: uint32 buffer_index;
// For each new buffer_lifetime_ordinal, a client must use new low-level
// buffers. This rule exists for multiple very good reasons, and is
// relevant to mid-stream changes, changes on stream boundaries, and both
// input and output buffers. A new buffer_lifetime_ordinal needs new
// low-level buffers, not just new StreamBuffer(s). If you find yourself
// copying compressed input data into new low-level input buffers solely
// due to this rule, consider asking the source of the data for the ability
// to directly fill new VMOs. The rule exists for good reasons, even for
// input buffers.
// The previous paragraph does not prohibit carving up VMOs into sub-pieces
// and using different sub-pieces as different StreamBuffer(s), with some
// VMOs used for more than one StreamBuffer and possibly others used for
// only one StreamBuffer. While this is permitted and enables some
// optimizations, it's not expected to be particularly common.
// See for more on StreamBufferData, and more on why we
// never re-use the same low-level buffers for different
// buffer_lifetime_ordinal(s).
3: StreamBufferData data;
/// StreamBufferData
/// For the moment, a VMO per buffer is the only type of buffer.
/// This is extremely likely to change significantly when adding gralloc stuff,
/// but the idea with this union is to have a struct per logical way of storing
/// the data. Any multi-domain storage within a gralloc buffer will likely be
/// only indirectly represented here.
union StreamBufferData {
StreamBufferDataVmo vmo;
// TODO(dustingreen): add the gralloc way
/// StreamBufferDataVmo
/// Details for a buffer backed by a VMO.
table StreamBufferDataVmo {
// The same VMO can be used by more than one StreamBuffer (only of the same
// buffer_lifetime_ordinal), but each vmo_handle must be a separate handle.
1: handle<vmo> vmo_handle;
// Offset within the VMO of the first usable byte. Must be < the VMO's size
// in bytes.
2: uint64 vmo_usable_start;
// VMO-relative offset that's one past the last usable byte. This can point
// one byte beyond the end of the VMO if desired. In other words, this can
// be equal to the VMO's size, to indicate that the last byte of the VMO is
// usable (and possibly many byte before that, depending on
// vmo_usable_start).
3: uint64 vmo_usable_size;
/// PacketHeader
/// When referring to a free packet, we use PacketHeader alone instead of
/// Packet, since while a packet is free it doesn't really have meaningful
/// offset or length etc.
/// A populated Packet also has a PacketHeader.
table PacketHeader {
// This is which buffer configuration lifetime this header is referring to.
// A packet_index is only really meaningful with respect to a particular
// buffer_lifetime_ordinal.
// See StreamBufferSettings.buffer_lifetime_ordinal.
// For QueueInputPacket(), a server receiving a buffer_lifetime_ordinal that
// isn't the current input buffer_lifetime_ordinal will close the channel.
// For OnFreeInputPacket() and RecycleOutputPacket(), the receiver (client
// or server) must ignore a message with stale buffer_lifetime_ordinal.
1: uint64 buffer_lifetime_ordinal;
// The overall set of packet_index values is densely packed from 0..count-1
// for input and output separately. They can be queued in any order.
// Both the client and server should validate the packet_index against the
// known bound and disconnect if it's out of bounds.
// When running in buffer-per-packet mode, the packet_index is also the
// buffer index. When running in single-buffer mode, the buffer index is
// always 0 referring to the single buffer.
// The packet_index values don't imply anything about order of use of
// packets. The client should not expect the ordering to remain the same
// over time - the stream processor is free to hold on to an input or
// output packet for a while during which other packet_index values may be
// used multiple times.
// For a given properly-functioning StreamProcessor instance, packet_index
// values will be unique among concurrently-outstanding packets. Servers
// should validate that a client isn't double-using a packet and clients
// should validate as necessary to avoid undefined or unexpected client
// behavior.
2: uint32 packet_index;
// Packet
// A Packet represents a chunk of input or output data to or from a stream
// processor.
// stream processor output:
// While the Packet is outstanding with the client via OnOutputPacket(), the
// stream processor will avoid modifying the referenced output data. After the
// client calls RecycleOutputPacket(packet_index), the stream processor is
// notified that the client is again ok with the referenced data changing.
// stream processor input:
// The client initially has all packet_index(es) available to fill, and later
// gets packet_index(s) that are again ready to fill via OnFreeInputPacket().
// The client must not modify the referenced data in between QueueInputPacket()
// and OnFreeInputPacket().
table Packet {
1: PacketHeader header;
/// Which buffer this packet refers to. For single-buffer mode this will
/// always be 0, but for multi-buffer mode, a given in-flight interval of a
/// packet can refer to any buffer. The packet has an associated buffer only
/// while the packet is in-flight, not while the packet is free.
/// The default value makes accidental inappropriate use of index 0 less
/// likely (will tend to complain in an obvious way if not filled out
/// instead of a non-obvious data corruption when decoding buffer 0
/// repeatedly instead of the correct buffers).
/// TODO(dustingreen): Try to make FIDL table defaults have meaning, and not
/// complain about !has when accessing the field. For now the default
/// specified here does nothing.
// TODO(FIDL-609): Default to 0x80000000.
2: uint32 buffer_index;
// stream_lifetime_ordinal
// The value 1 is the lowest permitted value after stream processor
// creation. Values sent by the client must be odd. Values must only
// increase.
// A stream_lifetime_ordinal represents the lifetime of a stream. All
// messages that are specific to a stream have the stream_lifetime_ordinal
// value and the value is the same for all messages relating to a given
// stream.
// See for more on stream_lifetime_ordinal.
3: uint64 stream_lifetime_ordinal;
// start_offset and valid_length_bytes
// Which part of the relevant buffer is this packet using. These are valid
// for input data that's in-flight to the stream processor, and are valid
// for output data from the stream processor.
// For compressed formats and uncompressed audio, the data in
// [start_offset, start_offset + valid_length_bytes) is the contiguously
// valid data referred to by this packet.
// For uncompressed video frames, FormatDetails is the primary means of
// determining which bytes are relevant. The offsets in FormatDetails
// are relative to the start_offset here. The valid_length_bytes must be
// large enough to include the full last line of pixel data, including the
// full line stride of the last line (not just the width in pixels of the
// last line).
// Despite these being filled out, some uncompressed video buffers are of
// types that are not readable by the CPU. These fields being here don't
// imply there's any way for the CPU to read an uncompressed frame.
// TODO(dustingreen): Do we have any reason to require that these be filled
// out for opaque uncompressed video frames that the CPU can't read? In
// that case do we want to require them just so they can be potentially
// passed on to a HW renderer in case the HW renderer has any use for them?
// Or more likely, it may just be that these tend to refer to the whole
// size of an uncompressed buffer, with format_details taking care of
// specifying which bytes are actually relevant.
4: uint32 start_offset;
// valid_length_bytes
// This must be > 0.
// The semantics for valid data per packet vary depending on data type as
// follows.
// uncompressed video - A video frame can't be split across packets. Each
// packet is one video frame.
// uncompressed audio - Regardless of float or int, linear or uLaw, or
// number of channels, a packet must contain an non-negative number of
// complete audio frames, where a single audio frame consists of data for
// all the channels for the same single point in time. Any
// stream-processor-specific internal details re. lower rate sampling for
// LFE channel or the like should be hidden by the StreamProcessor server
// implementation.
// compressed data input - A packet must contain at least one byte of data.
// See also stream_input_bytes_min. Splitting AUs at arbitrary byte
// boundaries is permitted, including at boundaries that are in AU headers.
// compressed data output - The stream processor is not required to fully
// fill each output packet's buffer.
5: uint32 valid_length_bytes;
// This value is not strictly speaking a timestamp. It is an arbitrary
// unsigned 64-bit number that, under some circumstances, will be passed by
// a stream processor unmodified from an input packet to the
// exactly-corresponding output packet.
// For timestamp_ish values to be propagated from input to output the
// following conditions must be true:
// * promise_separate_access_units_on_input must be true
// * has_timestamp_ish must be true for a given input packet, to have that
// timestamp_ish value (potentially) propagate through to an output
// * the StreamProcessor instance itself decides (async) that the input
// packet generates an output packet - if a given input never generates
// an output packet then the timestamp_ish value on the input will never
// show up on any output packet - depending on the characteristics of the
// input and output formats, and whether a decoder is willing to join
// mid-stream, etc this can be more or less likely to occur, but clients
// should be written to accommodate timestamp_ish values that are fed on
// input but never show up on output, at least to a reasonable degree
// (not crashing, not treating as an error).
// See for more on timestamp_ish.
6: uint64 timestamp_ish;
// start_access_unit
// If promise_separate_access_units_on_input (TODO(dustingreen): or any
// similar mode for output) is true, this bool must be set appropriately
// depending on whether byte 0 _is_ or _is not_ the start of an access
// unit. The client is required to know, and required to set this boolean
// properly. The server is allowed to infer that when this boolean is
// false, byte 0 is the first byte of a continuation of a
// previously-started AU. (The byte at start_offset is "byte 0".)
// If promise_separate_access_units_on_input is false, this boolean is
// ignored.
7: bool start_access_unit;
// known_end_access_unit
// A client is never required to set this boolean to true.
// If promise_separate_access_units_on_input is true, for input data, this
// boolean must be false if the last byte of this packet is not the last
// byte of an AU, and this boolean _may_ be true if the last byte of this
// packet is the last byte of an AU. A client delivering one AU at a time
// that's interested in the lowest possible latency via the decoder should
// set this boolean to true when it can be set to true.
// If promise_separate_access_units_on_input is false, this boolean is
// ignored.
8: bool known_end_access_unit;
// StreamProcessor
// The StreamProcessor interface exists to anchor the configuration of input
// and output buffers, and depending on StreamProcessor server hosting
// strategy, the StreamProcessor interface can, in some configurations, be 1:1
// with a stream processor isolate (process), especially when using SW codecs.
// The StreamProcessor can be used to process up to one stream at a time.
// Descriptions of actions taken by methods of this interface and the states of
// things are given as if the methods are synchronously executed by the stream
// processor server, but in reality, as is typical of FIDL interfaces, the
// message processing is async. The states described are to be read as the
// state from the client's point of view unless otherwise stated. Events
// coming back from the server are of course delivered async, and a client that
// processes more than one stream per StreamProcessor instance needs to care
// whether a given event is from the current stream vs. some older
// soon-to-be-gone stream.
// The Sync() method's main purpose is to enable the client to robustly prevent
// having both old and new buffers allocated in the system at the same time,
// since media buffers can be significantly large, depending. The Sync() method
// achieves this by only delivering it's response when all previous calls to
// the StreamProcessor interface have actually taken effect in the
// StreamControl ordering domain. Sync() can also be used to wait for the
// stream processor server to catch up if there's a possibility that a client
// might otherwise get too far ahead of the StreamProcessor server, by for
// example requesting creation of a large number of streams in a row. It can
// also be used during debugging to ensure that a stream processor server
// hasn't gotten stuck. Calling Sync() is entirely optional and never required
// for correctness - only potentially required to de-overlap resource usage.
// Semi-trusted StreamProcessor server - SW decoders run in an isolate (with
// very few capabilities) just in case the decoding SW has a vulnerability
// which could be used to take over the StreamProcessor server. Clients of the
// stream processor interface using decoders and processing streams of separate
// security contexts, to a greater extent than some other interfaces, need to
// protect themselves against invalid server behavior, such as double-free of a
// packet_index and any other invalid server behavior. Having fed in
// compressed data of one security context, don't place too much trust in a
// single StreamProcessor instance to not mix data among any buffers that
// StreamProcessor server has ever been told about. Instead, create separate
// StreamProcessor instances for use by security-separate client-side contexts.
// While the picture for HW-based decoders looks somewhat different and is out
// of scope of this paragraph, the client should always use separate
// StreamProcessor instances for security-separate client-side contexts.
protocol StreamProcessor {
// EnableOnStreamFailed()
// Permit the server to use OnStreamFailed() instead of the server just
// closing the whole StreamProcessor channel on stream failure.
// If the server hasn't seen this message by the time a stream fails, the
// server will close the StreamProcessor channel instead of sending
// OnStreamFailed().
// OnStreamFailed()
// The stream has failed, but the StreamProcessor instance is still usable
// for a new stream.
// This message is only ever sent by the server if the client previously
// sent EnableOnStreamFailed(). If the client didn't send
// EnableOnStreamFailed() then the server closes the StreamProcessor
// channel instead.
// StreamProcessor server implementations are encouraged to handle stream
// errors (and ideally to also report them via error_ bools of
// OnOutputPacket() and OnOutputEndOfStream()) without failing the whole
// stream, but if a stream processor server is unable to do that, but still
// can cleanly contain the failure to the stream, the stream processor
// server can (assuming EnableOnStreamFailed() was called) use
// OnStreamFailed() to indicate the stream failure to the client without
// closing the StreamProcessor channel.
// An ideal StreamProcessor server handles problems with input data without
// sending this message, but sending this message is preferred vs. closing
// the server end of the StreamProcessor channel if the StreamProcessor
// server can 100% reliably contain the stream failure to the stream,
// without any adverse impact to any later stream.
// No further messages will arrive from the server regarding the failed
// stream. This includes any OnOutputEndOfStream() that the client would
// have otherwise expected.
// TODO(dustingreen): Add at least an error_message string and _maybe_ a
// zx_status_t, though that might tend to encourage mis-use of zx_status_t
// so maybe just error_message for quicker debugging on the client side.
// Also plumb from CodecAdapterH264 and similar.
-> OnStreamFailed(uint64 stream_lifetime_ordinal);
// OnInputConstraints()
// The server sends this shortly after StreamProcessor creation to indicate
// input buffer constraints. The "min" and "max" input constraints don't
// change for the life of the StreamProcessor.
// The "max" values for buffer size and count are large enough to support
// the most demanding format the server supports on input. The
// "recommended" values should be workable for use with the input
// FormatDetails conveyed during StreamProcessor creation. The
// "recommended" values are not necessarily suitable if the client uses
// QueueInputFormatDetails() to change the input format. In that case it's
// up to the client to determine suitable values, either by creating a new
// StreamProcessor instance instead, or knowing suitable values outside the
// scope of this protocol.
// See comments on StreamBufferConstraints.
// This message is guaranteed to be sent unsolicited to the StreamProcessor
// client during or shortly after StreamProcessor creation. Clients should
// not depend on this being the very first message to arrive at the client.
// OnOutputConstraints() may be sent first by some stream processors that
// already know their initial output config without any input data, to
// encourage (but not strictly require) the client to configure output
// buffers before feeding the first input, to avoid a wasteful
// OnOutputConstraints() being generated for that first stream if the client has
// started configuring output but isn't done configuring output before the
// client sends the first input data for the first stream. A client is free
// to ignore OnOutputConstraints() with a stale stream_lifetime_ordinal, but
// handling OnOutputConstraints() with stream_lifetime_ordinal 0 (if any are
// sent) can help reduce latency to first output. See OnOutputConstraints() for
// more details.
// The "min" and "max" input constraints are guaranteed not to change for a
// given StreamProcessor instance. The "recommended" values may
// effectively change when the server processes QueueInputFormatDetails().
// There is not any way in the protocol short of creating a new
// StreamProcessor instance for the client to get those new "recommended"
// values.
// TODO(dustingreen): Maybe provide a way for the client to get updated
// "recommended" values for input, maybe only on request rather than via
// this event, to keep things simpler for simpler clients. Maybe separate
// the recommendations from the constraints.
-> OnInputConstraints(StreamBufferConstraints input_constraints);
// SetInputBufferSettings() and AddInputBuffer()
// Configuring input buffers consists of calling SetInputBufferSettings()
// followed by a number of calls to AddInputBuffer() equal to the number of
// buffers set via SetInputBufferSettings(). In buffer-per-packet mode,
// this is the same as the number of packets. In single-buffer mode, this
// is 1.
// After OnInputConstraints(), the client uses these two methods to set up
// input buffers and packets.
// Configuring input buffers is required before QueueInputPacket().
// The client can also re-set-up input buffers any time there is no current
// stream. The client need not wait until all previously-set-up input
// buffers are with the client via OnFreeInputPacket(). The old
// buffer_lifetime_ordinal just ends. See for more
// info on "buffer lifetime".
// The recommended way to de-overlap resource usage (when/if the client
// wants to) is to send CloseCurrentStream() with release_input_buffers
// true then send Sync() and wait for its response before allocating any
// new buffers. How to cause other parts of the system to release their
// references on low-level buffers is outside the scope of this interface.
// This call ends any previous buffer_lifetime_ordinal, and starts a new
// one.
SetInputBufferSettings(StreamBufferSettings input_settings);
// The client is required to add all the input buffers before sending any
// message that starts a new stream else the stream processor will close
// the StreamProcessor channel.
// When the last buffer is added with this message, all the input packets
// effectively jump from non-existent to free with the client. The
// StreamProcessor will not generate an OnFreeInputPacket() for each new
// input packet. The client can immediately start sending
// QueueInputPacket() after sending the last AddInputBuffer().
AddInputBuffer(StreamBuffer buffer);
// SetInputBufferPartialSettings()
// When the client is using sysmem to allocate buffers, this message is
// used instead of SetInputBufferSettings()+AddInputBuffer(). Instead, a
// single SetInputBufferPartialSettings() provides the StreamProcessor with
// the client-specified input settings and a BufferCollectionToken which
// the StreamProcessor will use to convey constraints to sysmem. Both the
// client and the StreamProcessor will be informed of the allocated buffers
// directly by sysmem via their BufferCollection channel (not via the
// StreamProcessor channel).
// The client must not QueueInput...() until after sysmem informs the client
// that buffer allocation has completed and was successful.
// The server should be prepared to see QueueInput...() before the server
// has necessarily heard from sysmem that the buffers are allocated - the
// server must tolerate either ordering, as the QueueInput...() and
// notification of sysmem allocation completion arrive on different
// channels, so the client having heard that allocation is complete doesn't
// mean the server knows that allocation is complete yet. However, the
// server can expect that allocation is in fact complete and can expect to
// get the allocation information from sysmem immediately upon requesting
// the information from sysmem.
SetInputBufferPartialSettings(StreamBufferPartialSettings input_settings);
// OnOutputConstraints()
// This event informs the client of new output constraints.
// This message is ordered with respect to other output (such as output
// packets, output format, output end-of-stream).
// Before the first OnOutputPacket() of a stream, the server guarantees that
// at least one OnOutputConstraints() and exactly one OnOutputFormat() will
// be sent. The server may not set buffer_constraints_action_required true
// in OnOutputConstraints() if the buffer config is already suitable for the
// stream (buffer_constraints_action_required false means the buffer config
// is already fine). The client must tolerate multiple
// OnOutputConstraints() (and 1 OnOutputFormat() message) before the first
// output packet. As long as the client hasn't moved to a new stream, the
// server won't send another OnOutputConstraints() until after the client
// has configured output buffers.
// This message can be sent mid-stream by a server. If
// buffer_constraints_action_required false, the message is safe to
// ignore, but a client may choose to stash the new constraints for
// later use the next time the client wants to unilaterally re-configure
// buffers (when allowed). If later the server needs the output config to
// change, the server may send a new OnOutputConstraints() with
// buffer_constraints_action_required true.
// On buffer_constraints_action_required true, a client that does not wish
// to fully handle mid-stream output buffer config changes should either
// give up completely on the processing, or at least re-config the output
// as specified before starting a new stream (and possibly re-delivering
// input data, if the client wants). This avoids useless retry with a new
// stream starting from just before the output buffer config change which
// would hit the same mid-stream output config change again.
// Similarly, some servers may only partly support mid-stream format
// changes, or only support a mid-stream format change if the buffers are
// already large enough to handle both before and after the format change.
// Such servers should still indicate buffer_constraints_action_required
// true, but then send OnStreamFailed() after the client has re-configured
// output buffers (seamlessly dealing with the mid-stream output config
// change is even better of course, but is not always feasible depending on
// format). When the client retries with a new stream starting from a
// nearby location in the client's logical overall media timeline, the
// output buffers will already be suitable for the larger size output, so
// the new stream will not need any mid-stream output buffer re-config,
// only a mid-stream OnOutputFormat(). This strategy avoids the problem
// that would otherwise occur if a client were to retry with a new stream
// starting just before the mid-stream output buffer config change (the
// retry wouldn't be effective since the same need for an output buffer
// config change would be hit again). Servers are discouraged from sending
// OnStreamFailed() solely due to a mid-stream need for different output
// buffer config without first sending OnOutputConstraints() with
// buffer_constraints_action_required true and waiting for the client to
// re-configure output buffers (to avoid the useless client retry with a
// new stream from a logical location before the config change).
// When buffer_constraints_action_required true, the server will not send
// any OnOutputPacket() for this stream until after the client has
// configured/re-configured output buffers.
// A client that gives up on processing on any mid-stream
// OnOutputConstraints() or mid-stream OnOuptutFormat() should completely
// ignore any OnOutputConstraints() with buffer_constraints_action_required
// false. Otherwise the client may needlessly fail processing, or server
// implementations might not be able to use
// buffer_constraints_action_required false for fear of simpler clients
// just disconnecting.
// All clients, even those which don't want to support any mid-stream
// output buffer re-config or mid-stream OnOutputFormat() are required to
// deal with 1..multiple OnOutputConstraints() messages before the first
// output packet, and 1 OnOutputFormat() messages before the first output
// packet.
// This message is ordered with respect to output packets, and with respect
// to OnOutputFormat().
// For more on OnOutputConstraints(), see
-> OnOutputConstraints(StreamOutputConstraints output_config);
// OnOutputFormat()
// This message is sent by the server before the first output packet of any
// stream, and potentially mid-stream between output packets of the stream,
// ordered with respect to output packets, and ordered with respect to
// OnOutputConstraints().
// The server guarantees that the first packet of every stream will be
// preceeded by an OnOutputFormat().
// The server guarantees that there will be an OnOutputFormat() between an
// OnOutputConstraints() with buffer_constraints_action_required true and an
// OnOutputPacket(). In other words, the client is essentially allowed to
// forget what the output format is on any OnOutputConstraints() with
// buffer_constraints_action_required true, because the server promises a
// subsequent OnOutputFormat() before any OnOutputPacket().
// If the server sets buffer_constraints_action_required true in
// OnOutputConstraints(), the server won't send OnOutputFormat() (and therefore
// also won't send OnOutputPacket()) until the client has re-configured
// output buffers.
// The server is allowed to send an OnOutputFormat() mid-stream between two
// output packets.
// A server won't send two adjacent OnOutputFormat() messages without any
// output packet in between. However an OnOutputFormat() message doesn't
// guarantee a subsequent packet, because for example the server could send
// OnOutputEndOfStream() or OnStreamFailed() instead.
// A client that does not wish to seamlessly handle mid-stream output format
// changes should either ensure that no stream processed by the client
// ever has any mid-stream format change, or the client should ensure that
// any retry of processing starts the new attempt at a point logically at or
// after the ponit where the old format has ended and the new format starts,
// else the client could just hit the same mid-stream format change again.
// An example of this message being sent mid-stream is mid-stream change
// of dimensions of video frames output from a video decoder.
// Not all servers will support seamless handling of format change. Those
// that do support seamless handling of format change may require that the
// format change not also require output buffer re-config, in order for the
// handling to be seamless. See the comment block for OnOutputConstraints() for
// more discussion of how servers and clients should behave - in particular
// when they don't seamlessly handle output constraint change and/or output
// format change.
// If this message isn't being sent by the server when expected at the
// start of a stream, the most common reason is that a previous
// OnOutputConstraints() with buffer_constraints_action_required true hasn't
// been processed by the client (by configuring output buffers using
// SetOutputBufferPartialSettings() etc).
-> OnOutputFormat(StreamOutputFormat output_format);
// SetOutputBufferSettings() and AddOutputBuffer()
// These are not permitted until after the first OnOutputConstraints().
// Roughly speaking, these messages are sent in response to OnOutputConstraints()
// with buffer_constraints_action_required true.
// Configuring output buffers consists of calling SetOutputBufferSettings()
// followed by a number of calls to AddOutputBuffer() equal to the number
// of buffers set via SetOutputBufferSettings(). In buffer-per-packet
// mode, this is the same as the number of packets. In single-buffer mode,
// this is 1.
// Configuring output buffers is _required_ after OnOutputConstraints() is
// received by the client with buffer_constraints_action_required true and
// stream_lifetime_ordinal equal to the client's current
// stream_lifetime_ordinal (even if there is an active stream), and is
// _permitted_ any time there is no current stream.
// For more on SetOutputBufferSettings() and AddOutputBuffer(), see
SetOutputBufferSettings(StreamBufferSettings output_settings);
AddOutputBuffer(StreamBuffer buffer);
// SetOutputBufferPartialSettings()
// When the client is using sysmem to allocate buffers, this message is
// used instead of SetOutputBufferSettings()+AddOutputBuffer(). Instead, a
// single SetOutputBufferPartialSettings() provides the StreamProcessor
// with the client-specified output settings and a BufferCollectionToken
// which the StreamProcessor will use to convey constraints to sysmem.
// Both the client and the StreamProcessor will be informed of the
// allocated buffers directly by sysmem via their BufferCollection channel
// (not via the StreamProcessor channel).
// See also ClientReadyForOutput().
SetOutputBufferPartialSettings(StreamBufferPartialSettings output_settings);
// After SetOutputBufferPartialSettings(), the server won't send
// OnOutputConstraints(), OnOutputFormat(), OnOutputPacket(), or
// OnOutputEndOfStream() until after the client sends
// CompleteOutputBufferPartialSettings().
// Some clients may be able to send
// CompleteOutputBufferPartialSettings() immediately after
// SetOutputBufferPartialSettings() - in that case the client needs to be
// prepared to receive output without knowing the buffer count or packet
// count yet - such clients may internally delay processing the received
// output until the client has heard from sysmem (which is when the client
// will learn the buffer count and packet count).
// Other clients may first wait for sysmem to allocate, prepare to receive
// output, and then send CompleteOutputBufferPartialSettings().
CompleteOutputBufferPartialSettings(uint64 buffer_lifetime_ordinal);
// FlushEndOfStreamAndCloseStream()
// This message is optional.
// This message is only valid after QueueInputEndOfStream() for this stream.
// The stream_lifetime_ordinal input parameter must match the
// stream_lifetime_ordinal of the QueueInputEndOfStream(), else the server
// will close the channel.
// A client can use this message to flush through (not discard) the last
// input data of a stream so that the stream processor server generates
// corresponding output data for all the input data before the server moves
// on to the next stream, without forcing the client to wait for
// OnOutputEndOfStream() before queueing data of another stream.
// The difference between QueueInputEndOfStream() and
// FlushEndOfStreamAndCloseStream(): QueueInputEndOfStream() is a promise
// from the client that there will not be any more input data for the
// stream (and this info is needed by some stream processors for the stream
// processor to ever emit the very last output data). The
// QueueInputEndOfStream() having been sent doesn't prevent the client from
// later completely discarding the rest of the current stream by closing
// the current stream (with or without a stream switch). In contrast,
// FlushEndOfStreamAndCloseStream() is a request from the client that all
// the previously-queued input data be processed including the logical
// "EndOfStream" showing up as OnOutputEndOfStream() (in success case)
// before moving on to any newer stream - this essentially changes the
// close-stream handling from discard to flush-through for this stream
// only.
// A client using this message can start providing input data for a new
// stream without that causing discard of old stream data. That's the
// purpose of this message - to allow a client to flush through (not
// discard) the old stream's last data (instead of the default when closing
// or switching streams which is discard).
// Because the old stream is not done processing yet and the old stream's
// data is not being discarded, the client must be prepared to continue to
// process OnOutputConstraints() messages until the stream_lifetime_ordinal is
// done. The client will know the stream_lifetime_ordinal is done when
// OnOutputEndOfStream(), OnStreamFailed(), or the StreamProcessor channel
// closes.
// For more on FlushEndOfStreamAndCloseStream(), see
FlushEndOfStreamAndCloseStream(uint64 stream_lifetime_ordinal);
// CloseCurrentStream()
// This "closes" the current stream, leaving no current stream. In
// addition, this message can optionally release input buffers or output
// buffers.
// If there has never been any active stream, the stream_lifetime_ordinal
// must be zero or the server will close the channel. If there has been an
// active stream, the stream_lifetime_ordinal must be the most recent
// active stream whether that stream is still active or not. Else the
// server will close the channel.
// Multiple of this message without any new active stream in between is not
// to be considered an error, which allows a client to use this message to
// close the current stream to stop wasting processing power on a stream the
// user no longer cares about, then later decide that buffers should be
// released and send this message again with release_input_buffers and/or
// release_output_buffers true to get the buffers released, if the client is
// interested in trying to avoid overlap in resource usage between old
// buffers and new buffers (not all clients are).
// See also Sync().
// For more on CloseCurrentStream(), see
uint64 stream_lifetime_ordinal,
bool release_input_buffers,
bool release_output_buffers);
// Sync() -> ()
// On completion, all previous StreamProcessor calls have done what they're
// going to do server-side, _except_ for processing of data queued using
// QueueInputPacket().
// The main purpose of this call is to enable the client to wait until
// CloseCurrentStream() with release_input_buffers and/or
// release_output_buffers set to true to take effect, before the client
// allocates new buffers and re-sets-up input and/or output buffers. This
// de-overlapping of resource usage can be worthwhile for media buffers
// which can consume resource types whose overall pools aren't necessarily
// vast in comparison to resources consumed. Especially if a client is
// reconfiguring buffers multiple times.
// Note that Sync() prior to allocating new media buffers is not alone
// sufficient to achieve non-overlap of media buffer resource usage system
// wide, but it can be a useful part of achieving that.
// The Sync() transits the Output ordering domain and the StreamControl
// ordering domain, but not the InputData ordering domain. For more on
// ordering domains see
// This request can be used to avoid hitting kMaxInFlightStreams which is
// presently 10. A client that stays <= 8 in-flight streams will
// comfortably stay under the limit of 10. While the protocol permits
// repeated SetInputBufferSettings() and the like, a client that spams the
// channel can expect that the channel will just close if the server or the
// channel itself gets too far behind.
Sync() -> ();
// OnOutputPacket()
// This is how the stream processor emits an output packet to the stream
// processor client.
// Order is significant.
// The client should eventually call RecycleOutputPacket() (possibly after
// switching streams multiple times), unless the buffer_lifetime_ordinal
// has moved on. A stream change doesn't change which packets are busy
// with the client vs. free with the server. See "packet lifetime" in
// for more.
// The relevant buffer is buffer 0 if running in single-buffer mode, or the
// buffer index is the same as packet_index if running in buffer-per-packet
// mode.
// For low-level buffer types that support it, a StreamProcessor is free to
// emit an output packet before the low-level buffer actually has any
// usable data in the buffer, with the mechanism for signalling the
// presence of data separate from the OnOutputPacket() message. For such
// low-level buffer types, downstream consumers of data from the emitted
// packet must participate in the low-level buffer signalling mechanism to
// know when it's safe to consume the data. This is most likely to be
// relevant when using a video decoder and gralloc-style buffers.
// The error_ bool(s) allow (but do not require) a StreamProcessor server
// to report errors that happen during an AU or between AUs.
// The scope of error_detected_before starts at the end of the last
// delivered output packet on this stream, or the start of stream if there
// were no previous output packets on this stream. The scope ends at the
// start of the output_packet.
// The error_detected_before bool is separate so that discontinuities can be
// indicated separately from whether the current packet is damaged.
// The scope of error_detected_during is from the start to the end of this
// output_packet.
-> OnOutputPacket(
Packet output_packet,
bool error_detected_before,
bool error_detected_during);
// RecycleOutputPacket()
// After the client is done with an output packet, the client needs to tell
// the stream processor that the output packet can be re-used for more
// output, via this method.
// It's not permitted to recycle an output packet that's already free with
// the stream processor server. It's permitted but discouraged for a
// client to recycle an output packet that has been deallocated by an
// explicit or implicit output buffer de-configuration(). See
// buffer_lifetime_ordinal for more on that. A server must ignore any such
// stale RecycleOutputPacket() calls.
// For more on RecycleOutputPacket(), see
RecycleOutputPacket(PacketHeader available_output_packet);
// OnOutputEndOfStream()
// After QueueInputEndOfStream() is sent by the StreamProcessor client,
// within a reasonable duration the corresponding OnOutputEndOfStream()
// will be sent by the StreamProcessor server. Similar to
// QueueInputEndOfStream(), OnOutputEndOfStream() is sent a maximum of once
// per stream.
// No more stream data for this stream will be sent after this message. All
// input data for this stream was processed.
// While a StreamProcessor client is not required to
// QueueInputEndOfStream() (unless the client wants to use
// FlushEndOfStreamAndCloseStream()), if a StreamProcessor server receives
// QueueInputEndOfStream(), and the client hasn't closed the stream, the
// StreamProcessor server must generate a corresponding
// OnOutputEndOfStream() if nothing went wrong, or must send
// OnStreamFailed(), or must close the server end of the StreamProcessor
// channel. An ideal StreamProcessor server would handle and report stream
// errors via the error_ flags and complete stream processing without
// sending OnStreamFailed(), but in any case, the above-listed options are
// the only ways that an OnOutputEndOfStream() won't happen after
// QueueInputEndOfStream().
// There will be no more OnOutputPacket() or OnOutputConstraints() messages for
// this stream_lifetime_ordinal after this message - if a server doesn't
// follow this rule, a client should close the StreamProcessor channel.
// The error_detected_before bool has the same semantics as the
// error_detected_before bool in OnOutputPacket().
-> OnOutputEndOfStream(
uint64 stream_lifetime_ordinal,
bool error_detected_before);
// Stream specific messages:
// QueueInputFormatDetails()
// TODO(dustingreen): Rename from QueueInputFormatDetails() to
// QueueInputFormat().
// If the input format details are still the same as specified during
// StreamProcessor creation, this message is unnecessary and does not need
// to be sent.
// If the stream doesn't exist yet, this message creates the stream.
// All servers must permit QueueInputFormatDetails() at the start of a
// stream without failing, as long as the new format is supported by the
// StreamProcessor instance. Technically this allows for a server to only
// support the exact input format set during StreamProcessor creation, and
// that is by design. A client that tries to switch formats and gets a
// StreamProcessor channel failure should try again one more time with a
// fresh StreamProcessor instance created with CodecFactory using the new
// input format during creation, before giving up.
// These format details override the format details specified during stream
// processor creation for this stream only. The next stream will default
// back to the format details set during stream processor creation.
// For now, QueueInputFormatDetails() sent mid-stream will fail the
// StreamProcessor channel. Clients shouldn't do this for now.
// This message is permitted at the start of the first stream (just like at
// the start of any stream). The format specified need not match what was
// specified during stream processor creation, but if it doesn't match, the
// StreamProcessor channel might close as described above.
uint64 stream_lifetime_ordinal, FormatDetails format_details);
// QueueInputPacket()
// This message queues input data to the stream processor for processing.
// If the stream doesn't exist yet, this message creates the new stream.
// The client is required to be willing to send QueueInputPacket() prior to
// the server's first OnOutputConstraints(), and is permitted to start a new
// stream without output buffers configured yet.
// The client must continue to deliver input data via this message even if
// the stream processor has not yet generated the first OnOutputConstraints(),
// and even if the StreamProcessor is generating OnFreeInputPacket() for
// previously-queued input packets. The input data must continue as long
// as there are free packets to be assured that the server will ever
// generate the first OnOutputConstraints().
// For more on QueueInputPacket(), see
QueueInputPacket(Packet packet);
// OnFreeInputPacket()
// The server sends this message when the stream processor is done
// consuming this packet and the packet can be re-filled by the client.
// This is not sent for all packets when a new buffer_lifetime_ordinal
// starts as in that case all the packets are initially free with the
// client.
// See comments on QueueInputBuffer() and "packet lifetime" in
// for for description of buffer lifetime and packet
// lifetime.
// After receiving the available input buffer via this event, the stream
// processor client can call later call QueueInputBuffer with appropriate
// offset and length set.
// TODO(dustingreen): At the moment, there is no guarantee re. the order of
// these messages with respect to the order of QueueInputPacket(), but at
// least for decoders, it might be worthwhile to require that servers
// preserve the order vs. QueueInputPacket(), to make it easier to feed
// input from a ring buffer or similar. For audio encoders it might still
// make sense. For video encoders probably not.
-> OnFreeInputPacket(PacketHeader free_input_packet);
// Inform the server that all QueueInputPacket() messages for this stream
// have been sent.
// If the stream isn't closed first (by the client, or by OnStreamFailed(),
// or StreamProcessor channel closing), there will later be a corresponding
// OnOutputEndOfStream().
// The corresponding OnOutputEndOfStream() message will be generated only if
// the server finishes processing the stream before the server sees the
// client close the stream (such as by starting a new stream). A way to
// force the server to finish the stream before closing is to use
// FlushEndOfStreamAndCloseStream() after QueueInputEndOfStream() before any
// new stream. Another way to force the server to finish the stream before
// closing is to wait for the OnOutputEndOfStream() before taking any action
// that closes the stream.
// In addition to serving as an "EndOfStream" marker to make it obvious
// client-side when all input data has been processed, if a client never
// sends QueueInputEndOfStream(), no amount of waiting will necessarily
// result in all input data getting processed through to the output. Some
// stream processors have some internally-delayed data which only gets
// pushed through by additional input data _or_ by this EndOfStream marker.
// In that sense, this message can be viewed as a flush-through at
// InputData domain level, but the flush-through only takes effect if the
// stream processor even gets that far before the stream is just closed at
// StreamControl domain level. This message is not alone sufficient to act
// as an overall flush-through at StreamControl level. For that, send this
// message first and then send FlushEndOfStreamAndCloseStream() (at which
// point it becomes possible to queue input data for a new stream without
// causing discard of this older stream's data), or wait for the
// OnOutputEndOfStream() before closing the current stream.
// If a client sends QueueInputPacket(), QueueInputFormatDetails(),
// QueueInputEndOfStream() for this stream after the first
// QueueInputEndOfStream() for this stream, a server should close the
// StreamProcessor channel.
QueueInputEndOfStream(uint64 stream_lifetime_ordinal);