| // Copyright 2018 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| library fuchsia.mediacodec; |
| |
| // See codec.md for detailed interface documentation. The comments here are a |
| // summary only. Client implementers should see codec.md for more detail on any |
| // message that doesn't seem sufficiently-described here. Codec server |
| // implementers should probably read codec.md before implementing. |
| |
| // Overview of operation: |
| // |
| // 1. Create |
| // * create via CodecFactory - see CodecFactory |
| // 2. Get input constraints |
| // * OnInputConstraints() - sent unsolicited by codec shortly after codec |
| // creation. |
| // 3. Provide input buffers |
| // * SetInputBufferSettings() / AddInputBuffer() |
| // 4. Deliver input data |
| // * QueueInputPacket() + OnFreeInputPacket(), for as long as it takes, |
| // possibly working through all input packets repeatedly before... |
| // 5. Get output constraints and format |
| // * OnOutputConfig() - may be delivered as early as before |
| // OnInputConstraints() by some codecs, but a client must tolerate as late |
| // as after substantial input data has been delivered including lots of |
| // input packet recycling via OnFreeInputPacket(). |
| // * This message can arrive more than once before the first output data. |
| // 6. Provide output buffers |
| // * SetOutputBufferSettings() / AddOutputBuffer() |
| // 7. Data flows, with optional EndOfStream |
| // * OnOutputPacket() / RecycleOutputPacket() / QueueInputPacket() / |
| // OnFreeInputPacket() / QueueInputEndOfStream() / OnOutputEndOfStream() |
| // |
| // It's possible to re-use a Codec instance for another stream, and doing so |
| // can sometimes skip over re-allocation of buffers. This can be a useful thing |
| // to do for cases like seeking to a new location - at the Codec interface that |
| // can look like switching to a new stream. |
| |
| // CodecBufferConstraints |
| // |
| // This struct helps ensure that packet count and buffer space are sufficient |
| // to avoid major problems. For example, a video decoder needs sufficient |
| // video frame buffers to hold all potential reference frames concurrently + |
| // one more video buffer to decode into. Else, the whole video decode pipe can |
| // easily deadlock. |
| // |
| // The secondary purpose of this struct is to help ensure that packet count and |
| // buffer space are sufficient to achieve reasonably performant operation. |
| // |
| // There are separate instances of this struct for codec input and codec output. |
| struct CodecBufferConstraints { |
| // This is a version number the server sets on the constraints to allow the |
| // server to determine when the client has caught up with the latest |
| // constraints sent by the server. The server won't emit output data until |
| // the client has configured output settings and buffers with a |
| // buffer_constraints_version_ordinal >= the latest |
| // buffer_constraints_version_ordinal that had |
| // buffer_constraints_action_required true. See |
| // buffer_constraints_action_required comments for more. |
| // |
| // A buffer_constraints_version_ordinal of 0 is not permitted, to simplify |
| // initial state handling. Other than 0, both odd and even version ordinals |
| // are allowed (in constrast to the stream_lifetime_ordinal, neither the |
| // client nor server ever has a reason to consider the latest version to be |
| // stale, so there would be no benefit to disallowing even values). |
| uint64 buffer_constraints_version_ordinal; |
| |
| // default_settings |
| // |
| // These settings are "default" settings, not "recommended" settings. |
| // |
| // These "default" settings can be passed to SetInputBufferSettings() / |
| // SetOutputBufferSettings() as-is without modification, but a client doing |
| // that must still obey the semantics of packet_count_for_client, despite the |
| // codec server not having any way to really know the proper setting for |
| // that field. |
| // |
| // For CodecBufferConstraints fields whose names end in "recommended", the |
| // default_settings will have the corresponding setting field set to that |
| // recommended value. |
| // |
| // The codec promises that these default settings as-is (except for |
| // buffer_lifetime_ordinal) are guaranteed to |
| // satisfy the constraints indicated by the other fields of |
| // CodecBufferConstraints. While client-side checking that these |
| // settings are within the constraints is likely unnecessary in the client, |
| // the client should still check that these values are within client-side |
| // reasonable-ness bounds before using these values, to avoid letting a codec |
| // server cause problems for the client. |
| // |
| // This structure will always have single_buffer_mode false. See |
| // single_buffer_mode_allowed for whether single_buffer_mode true is allowed. |
| // |
| // The client must set the buffer_lifetime_ordinal field to a proper value |
| // before sending back to the server. The 0 initially in this field will be |
| // rejected by the server if sent back as-is. See comments on |
| // CodecPortBufferSettings.buffer_lifetime_ordinal. |
| CodecPortBufferSettings default_settings; |
| |
| // For uncompresesd video, separate and complete frames, each in its own |
| // separate buffer (buffer-per-packet mode), is always a requirement. |
| |
| // per_packet_buffer_bytes.*: |
| // |
| // These per-packet buffer bytes constraints apply to both buffer-per-packet |
| // mode and single-buffer mode (see single_buffer_mode). If |
| // buffer-per-packet mode, the constraints apply to each buffer separately. |
| // If single-buffer mode, the constraints need to be multiplied by the number |
| // of packets to determine the constraints on the single buffer. |
| |
| // per_packet_buffer_bytes_min: |
| // |
| // If a client is using buffer per packet mode, each buffer must be at least |
| // this large. If a client is using single-buffer mode, the one buffer must |
| // be at least per_packet_buffer_bytes_min * packet_count_for_codec_min in |
| // size. |
| uint32 per_packet_buffer_bytes_min; |
| // Must be >= per_packet_buffer_bytes_min. Delivering more than |
| // this per input packet might not perform any better, and in fact might |
| // perform worse. |
| uint32 per_packet_buffer_bytes_recommended; |
| // Must be >= per_packet_buffer_bytes_recommended. Can be 0xFFFFFFFF if there |
| // is no explicitly-enforced limit. |
| uint32 per_packet_buffer_bytes_max; |
| |
| // Minimum number of packet_count_for_codec. |
| // |
| // Re. input and output: |
| // |
| // This is a strict min for packet_count_for_codec, but a client can use more |
| // packets overall if the client wants to, by using a larger value for |
| // packet_count_for_codec and/or using a non-zero packets_for_client. A good |
| // reason to do the former would be if the client might tend to deliver a few |
| // not-very-full buffers occasionally - or to have a few extra packets within |
| // which to satisfy codec_input_bytes_min. A good reason to do the latter |
| // would be if a client needs to hold onto some packets for any "extra" |
| // duration. |
| // |
| // If a client specifies a larger packet_count_for_codec value than |
| // packet_count_for_codec_min, a server is permitted (but not encouraged) to |
| // not make progress until packet_count_for_codec are with the server, |
| // not merely packet_count_for_codec_min. |
| // |
| // For decoder input and audio encoder input: The packet_count_for_codec_min |
| // may or may not contain enough data to allow the codec to make progress |
| // without copying into an internal side buffer. If there isn't enough data |
| // delivered in packet_count_for_codec_min packets to permit progress, the |
| // codec must copy into its own side buffer internally to make progress. |
| // |
| // If a client intends to use extra packets for client-side purposes, the |
| // client should specify the extra packets in packets_for_client instead of |
| // packet_count_for_codec, but packet_count_for_codec must still be >= |
| // packet_count_for_codec_min. |
| // |
| // See codec.md for more on packet_count_for_codec_min. |
| uint32 packet_count_for_codec_min; |
| |
| // This must be at least packet_count_for_codec_min and at most |
| // packet_count_for_codec_recommended_max. |
| // |
| // This value is likely to be used as-is by most clients, so if having one |
| // additional packet is a big performance win in a large percentage of |
| // scenarios, it can be good for the server to include that additional packet |
| // in this value. |
| uint32 packet_count_for_codec_recommended; |
| |
| // This can be the same as packet_count_for_codec_max or can be lower. |
| // Values above this value and <= packet_count_for_codec_max are not |
| // recommended by the codec, but should still work given sufficient resoures |
| // available to both the client and the codec. |
| uint32 packet_count_for_codec_recommended_max; |
| |
| // This can be 0xFFFFFFFF if there's no codec-enforced max, but codecs are |
| // encouraged to set a large but still plausibly-workable max, and clients |
| // are encouraged to request a number of packets that isn't excessively large |
| // for the client's scenario. |
| uint32 packet_count_for_codec_max; |
| |
| // Normally this would be an implicit 0, but for now we have a min so we can |
| // force the total number of packets to be a specific number that we know |
| // works for the moment. |
| uint32 packet_count_for_client_min; |
| |
| // packet_count_for_client_max |
| // |
| // The client must set packet_count_for_client to be <= |
| // packet_count_for_client_max. |
| // |
| // This value must be at least 1. This can be 0xFFFFFFFF if there's no |
| // codec-enforced max. Clients are encouraged to request a number of |
| // packets that isn't excessively large for the client's scenario. |
| uint32 packet_count_for_client_max; |
| |
| // single_buffer_mode_allowed false allows a codec that's not required to |
| // support single-buffer mode for a given input or output the ability to |
| // decline to support single-buffer mode on that input/output. |
| // |
| // All encoder output, regardless of audio or video: server support for |
| // single-buffer mode is optional. Please see codec.md for all the rules |
| // regarding single-buffer mode on output before using single-buffer mode on |
| // an output. |
| // |
| // Audio decoder output: server support for single-buffer mode is required. |
| // |
| // Video decoder output: There is little reason for a video decoder to |
| // support single-buffer mode on output. Nearly all video decoders will set |
| // this to false for their output. |
| // |
| // All decoder inputs: Servers must support single-buffer mode on input. The |
| // client is responsible for managing the input buffer space such that |
| // filling an input packet doesn't over-write any portion of an input packet |
| // already in flight to the codec. |
| // |
| // Encoder inputs: Server support for single-buffer mode on encoder input is |
| // optional. This is more often useful for audio than for video. |
| // |
| // Support for buffer-per-packet mode is always required on both input and |
| // output, regardless of codec type. |
| bool single_buffer_mode_allowed; |
| |
| // If true, the buffers need to be physically contiguous pages, such as can be |
| // allocated using zx_vmo_create_contiguous(). |
| bool is_physically_contiguous_required; |
| // VERY TEMPORARY HACK / KLUDGE - we want the BufferAllocator (or one of the |
| // participant drivers that needs physically contiguous buffers) to call |
| // zx_vmo_create_contiguous(), definitely not the Codec client, but until the |
| // BufferAllocator can be reached from a driver, this is to grant the client |
| // special powers it really shouldn't have, very temporarily until |
| // BufferAllocator is hooked up properly at which point this can be removed. |
| // Strictly speaking we could reverse which end allocates buffers in the Codec |
| // interface to avoid this hack even before BufferAllocator, but the overall |
| // path seems shorter if we jump directly from this to using BufferAllocator. |
| handle? very_temp_kludge_bti_handle; |
| }; |
| |
| // CodecOutputConfig |
| // |
| // The codec-controlled output configuration, including both |
| // CodecBufferConstraints for the output and CodecFormatDetails for the output. |
| // |
| // TODO(dustingreen): Need a better name for this struct, but still short |
| // hopefully. It's stuff the codec gets to control, not the client. It's |
| // different than output buffer settings, which the client does get to control |
| // to some extent. It's different than any configurable output settings the |
| // client might specifiy for output of an encoder. |
| struct CodecOutputConfig { |
| // A client which always immediately re-configures output buffers on reciept |
| // of OnOutputConfig() with buffer_constraints_action_required true can safely |
| // ignore this field. |
| // |
| // A client is permitted to ignore an OnOutputConfig() message even with |
| // buffer_constraints_action_required true if the client knows the server has |
| // already been told to discard the remainder of the stream with the same |
| // stream_lifetime_ordinal or if this stream_lifetime_ordinal field is set to |
| // 0. The server is required to re-send needed output config via |
| // OnOutputConfig() with new stream_lifetime_ordinal and |
| // buffer_constraints_action_required true, if the the most recent completed |
| // server-side output config isn't what the server wants/needs yet for the |
| // new stream. |
| uint64 stream_lifetime_ordinal; |
| |
| // buffer_constraints_action_required |
| // |
| // When the buffer constraints are delivered, they indicate whether action is |
| // required. A false value here permits delivery of constraints which are |
| // fresher without forcing a buffer reconfiguration. If this is false, a |
| // client cannot assume that it's safe to immediately re-configure output |
| // buffers. If this is true, the client can assume it's safe to immediately |
| // configure output buffers once. |
| // |
| // A client is permitted to ignore buffer constraint versions which have |
| // buffer_constraints_action_required false. The server is not permitted to |
| // change buffer_constraints_action_required from false to true for the same |
| // buffer_constraints_version_ordinal. |
| // |
| // For each configuration, a client must use new buffers, never buffers that |
| // were prevoiusly used for anything else, and never buffers previously used |
| // for any other Codec purposes. This rule exists for multiple good reasons, |
| // relevant to both mid-stream changes, and changes on stream boundaries. |
| // A client should just use new buffers each time. |
| // |
| // When this is true, the server has already de-refed as many low-level |
| // output buffers as the server can while still performing efficient |
| // transition to the new buffers and will de-ref the rest asap. A Sync() is |
| // not necessary to achieve non-overlap of resource usage to the extent |
| // efficiently permitted by the formats involved. |
| // |
| // If buffer_constraints_action_required is true, the server _must_ not |
| // deliver more output data until after output buffers have been configured |
| // (or re-configured) by the client. |
| // |
| // See codec.md for more on buffer_constraints_action_required. |
| bool buffer_constraints_action_required; |
| CodecBufferConstraints buffer_constraints; |
| |
| // format_details |
| // |
| // It's up to the client to determine if a change in |
| // format_details.format_details_version_ordinal implies any client action is |
| // required, based on particular fields in format_details vs. any old value. |
| // The server guarantees that if the format has changed, then |
| // format_details.format_details_version_ordinal will change, but a change to |
| // format_details.format_details_version_ordinal does not guarantee that the |
| // format details actually changed. Servers are strongly encouraged to not |
| // change format_details.format_details_version_ordinal other than before the |
| // first output data of a stream unless there is a real mid-stream format |
| // change in the stream. Unnecessary mid-stream format changes can cause |
| // simpler clients that have no need to handle mid-stream format changes to |
| // just close the channel. Format changes before the first output data of a |
| // stream are not "mid-stream" in this context - those can be useful for |
| // stream format detection / setup reasons. |
| // |
| // Note that in case output buffers don't really need to be re-configured |
| // despite a format change, a server is encouraged, but not required, to |
| // set buffer_constraints_action_required false on the message that conveys |
| // the new format details. Simpler servers may just treat the whole output |
| // situation as one big thing and demand output buffer reconfiguration on any |
| // change in the output situation. |
| // |
| // A client may or may not actually handle a new buffer_constraints with |
| // buffer_constraints_action_required false, but the client should always |
| // track the latest format_details. |
| // |
| // An updated format_details is ordered with respect to emitted output |
| // packets, and applies to all subsequent packets until the next |
| // format_details with larger version_ordinal. A simple client that does not |
| // intend to handle mid-stream format changes should still keep track of the |
| // most recently received format_details until the first output packet |
| // arrives, then lock down the format details, handle those format details, |
| // and verify that any format_details.format_details_version_ordinal received |
| // from the server is the same as the locked-down format_details, until the |
| // client is done with the stream. Even such a simple client must tolerate |
| // format_details.format_details_version_ordinal changing multiple times |
| // before the start of data output from a stream (any stream - the first |
| // stream or a subsequent stream). This allows a codec to request that |
| // output buffers and output format be configured speculatively, and for the |
| // output config to be optionally adjusted by the server before the first |
| // data output from a stream after the server knows everything it needs to |
| // know to fully establish the initial output format details. This |
| // simplifies codec server implementation, and allows a clever codec server |
| // to guess it's output config for lower latency before any input data, while |
| // still being able to fix the output config (including format details) if |
| // the guess turns out to be wrong. |
| // |
| // Whether the format_details.format_details_version_ordinal will actually |
| // change mid-stream is a per-codec and per-stream detail that is not |
| // specified in comments here, and in most cases also depends on whether the |
| // format changes on the input to the codec. Probably it'll be fairly common |
| // for a client to use a format which technically supports mid-stream format |
| // change, but the client happens to know that none of the streams the client |
| // intends to process will ever have a mid-stream format change. |
| CodecFormatDetails format_details; |
| }; |
| |
| // Default values for input and output |
| // CodecBufferConstraints.default_settings.packet_count_for_codec. |
| // |
| // These are defined as "const" in FIDL to avoid all server implementations |
| // needing to separately define their own values, and these should be |
| // reasonable as default values, but strictly speaking this is not intended to |
| // promise that this value won't change from build to build. If a client cares |
| // about a specific number, the client should separately define what that |
| // number is and ensure that CodecPortBufferSettings.packet_count_for_client is |
| // at least large enough. |
| // |
| // In contrast to packet_count_for_client, the packet_count_for_codec is much |
| // more codec-specific, so this file has no numbers for that - each codec will |
| // set those as appropriate for the specific codec. |
| // |
| // These are not "recommended" values, only "default" values, in the sense that |
| // the codec doesn't really know what the correct setting for these values is |
| // for a given client, and if the default is not appropriate for a client, |
| // large problems could result such as deadlock. See the comments on |
| // packet_count_for_client. |
| // |
| // Despite these defaults, every client should ideally care about the |
| // packet_count_for_client setting and should ensure that the setting is at |
| // least large enough to cover the number of packets the client might ever need |
| // to camp on for any non-transient duration concurrently. The defaults are |
| // only intended to be plausible for some clients, not all clients. |
| // |
| // One for the client to be filling and one in transit. |
| const uint32 kDefaultInputPacketCountForClient = 2; |
| // One for the client to be rendering, and one in transit. |
| const uint32 kDefaultOutputPacketCountForClient = 2; |
| |
| // For input, this is the default on a fairly arbitrary basis. |
| // |
| // TODO(dustingreen): Do we want the default for audio encoding to be |
| // single_buffer_mode true instead? If so, we may split this up by audio/video |
| // encoder/decoder. |
| const bool kDefaultInputIsSingleBufferMode = false; |
| const bool kDefaultOutputIsSingleBufferMode = false; |
| |
| // CodecPortBufferSettings |
| // |
| // See relevant corresponding constraints in CodecBufferConstraints. The |
| // settings must satisfy the constraints. |
| // |
| // The client informs the codec of these settings and then separately informs |
| // the codec of each buffer. |
| // |
| // TODO(dustingreen): Rename from CodecPortBufferSettings to CodecBufferSettings |
| // in a separate CL (after some other stuff is out of the way first). |
| struct CodecPortBufferSettings { |
| // buffer_lifetime_ordinal |
| // |
| // The containing message starts a new buffer_lifetime_ordinal. |
| // |
| // There is a separate buffer_lifetime_ordinal for input vs. output. |
| // |
| // Re-use of the same value is not allowed. Values must be odd. Values must |
| // only increase (increasing by more than 2 is permitted). |
| // |
| // A buffer_lifetime_ordinal lifetime starts at SetInputBufferSettings() or |
| // SetOutputBufferSettings(), and ends at the the earlier of |
| // CloseCurrentStream() with release_input_buffers/release_output_buffers set |
| // or SetOutputBufferSettings() with new buffer_lifetime_ordinal in the case |
| // of mid-stream output config change. |
| // |
| // See codec.md for more on buffer_lifetime_ordinal. |
| uint64 buffer_lifetime_ordinal; |
| |
| // buffer_constraints_version_ordinal |
| // |
| // This value indicates which version of constraints the client is/was aware |
| // of so far. |
| // |
| // For input, this must always be 0 because constraints don't change for |
| // input (settings can change, but there's no settings vs current constraints |
| // synchronization issue on input). |
| // |
| // For output, this allows the server to know when the client is sufficiently |
| // caught up before the server will generate any more output. |
| // |
| // When there is no active stream, a client is permitted to re-configure |
| // buffers again using the same buffer_constraints_version_ordinal. |
| // |
| // See codec.md for more on buffer_constraints_version_ordinal. |
| uint64 buffer_constraints_version_ordinal; |
| |
| // The total packet count is split into two pieces to disambiguate how many |
| // packets are allocated for the client to hold onto for whatever reason, vs. |
| // how many packets are allocated for the server to hold onto for whatever |
| // reason. |
| // |
| // Extra packets to provide slack for peformance reasons can be in either |
| // category, but typically packet_count_for_codec_recommended will already |
| // include any performance-relevant slack for the server's benefit. |
| |
| // packet_count_for_codec |
| // |
| // How many packets the client is allocating for the codec server's use. |
| // This must be >= CodecBufferConstraints.packet_count_for_codec_min. If |
| // constraints change such that this would no longer be true, the server will |
| // send an OnOutputConfig() event. |
| // |
| // The codec server is allowed to demand that all of packet_count_for_codec |
| // become free before making further progress, even if |
| // packet_count_for_codec is > packet_count_for_codec_min. |
| // |
| // A reasonable value for this is |
| // CodecBufferConstraints.packet_count_for_codec_recommended. |
| // |
| // See codec.md for more on packet_count_for_codec. |
| uint32 packet_count_for_codec; |
| |
| // packet_count_for_client |
| // |
| // This must be at least 1. The server will close the channel if this is 0. |
| // |
| // How many packets the client is allocating for the client's use. The |
| // client may hold onto this many packets for arbitrarily-long duration |
| // without handing these packets to the codec, and despite doing so, the |
| // codec will continue to make progress and function normally without getting |
| // stuck. The client holding onto additional packets transiently is ok, but |
| // the client needs to hand those additional packets back to the codec |
| // eventually if the client wants the codec to make further progress. |
| // |
| // In addition to this value needing to include at least as many packets as |
| // the client ever intends to concurrently camp on indefinitely, any extra |
| // slack to benefit client-side performance should also be included here. |
| // |
| // A typical value for this could be at least 2, but it depends strongly on |
| // client implementation and overall client buffering goals. It is up to the |
| // client to determine how many packets are needed in this category by any |
| // parts of the overall system that will be holding onto packets for any |
| // reason. Those parts of the system should have a documented and possibly |
| // queryable defined value to help determine this number. Setting this value |
| // lower than it actaully needs to be can result in the codec not making |
| // progress as it sits waiting for packets, with the client unable to recycle |
| // any more packets to the codec. That situation can be difficult to |
| // diagnose, while excessively-large values here are wasteful, so care is |
| // warranted to set this value properly. |
| uint32 packet_count_for_client; |
| |
| // per_packet_buffer_bytes |
| // |
| // In buffer-per-packet mode, we require that each buffer have usable bytes |
| // equal to per_packet_buffer_bytes. Use of differently-sized low-level |
| // buffers is possible, but the size of the portion used via the Codec |
| // interface per CodecBuffer must be the same for all the buffers. |
| // |
| // In single-buffer mode, we require the portion of the low-level buffer used |
| // via the Codec interface to be size (packet_count_for_codec + |
| // packet_count_for_client) * per_packet_buffer_bytes. |
| // |
| // TODO(dustingreen): determine if we need to relax these restrictions a bit |
| // for convenience when using gralloc video buffers. |
| uint32 per_packet_buffer_bytes; |
| |
| // If true, there is only one buffer with index 0 which all packets |
| // implicitly refer to. If false, the packet_index and buffer_index are 1:1, |
| // and each packet refers to its corresponding buffer. |
| // |
| // While it's possible to set up single_buffer_mode false with each buffer |
| // referring to the same underlying VMO, single_buffer_mode true is more |
| // efficient for that case since only one mapping is created. |
| bool single_buffer_mode; |
| }; |
| |
| // CodecBuffer |
| // |
| // The CodecBuffer struct represents a pre-configured buffer. |
| // |
| // Both input and output uses CodecBuffer(s), but the two sets of buffers are |
| // separate. |
| // |
| // The client uses SetInputBufferSettings() + AddInputBuffer() * N to inform |
| // the codec about all the input buffers. |
| // |
| // The client uses SetOutputBufferSettings() + AddOutputBuffer() * N to inform |
| // the codec about all the output buffers. |
| // |
| // When single_buffer_mode is true, there is only buffer_index 0 shared by all |
| // CodecPacket(s) of the relevant input or output. When single_buffer_mode is |
| // false, the buffer_index equals the packet_index. |
| struct CodecBuffer { |
| // When using AddOutputBuffer()/AddInputBuffer(), this must match the |
| // buffer_lifetime_ordinal of the most recent |
| // SetOutputBufferSettings()/SetInputBufferSettings(). |
| uint64 buffer_lifetime_ordinal; |
| |
| // Buffers must be added via AddOutputBuffer() / AddInputBuffer() in order by |
| // buffer_index, and the buffer_index is always equal to 0 or equal to the |
| // packet_index (depending on single_buffer_mode true or false), but it's |
| // still nice to have CodecBuffer include the buffer_index if only for easier |
| // debugging. |
| uint32 buffer_index; |
| |
| // For each new buffer_lifetime_ordinal, a client must use new low-level |
| // buffers. This rule exists for multiple very good reasons, and is relevant |
| // to mid-stream changes, changes on stream boundaries, and both input and |
| // output buffers. A new buffer_lifetime_ordinal needs new low-level |
| // buffers, not just new CodecBuffer(s). If you find yourself copying |
| // compressed input data into new low-level input buffers solely due to this |
| // rule, consider asking the source of the data for the ability to directly |
| // fill new VMOs. The rule exists for good reasons, even for input buffers. |
| // |
| // The previous paragraph does not prohibit carving up VMOs into sub-pieces |
| // and using different sub-pieces as different CodecBuffer(s), with some VMOs |
| // used for more than one CodecBuffer and possibly others used for only one |
| // CodecBuffer. While this is permitted and enables some optimizations, it's |
| // not expected to be particularly common. |
| // |
| // See codec.md for more on CodecBufferData, and more on why we never re-use |
| // the same low-level buffers for different buffer_lifetime_ordinal(s). |
| CodecBufferData data; |
| }; |
| |
| // CodecBufferData |
| // |
| // For the moment, a VMO per buffer is the only type of buffer. |
| // |
| // This is extremely likely to change significantly when adding gralloc stuff, |
| // but the idea with this union is to have a struct per logical way of storing |
| // the data. Any multi-domain storage within a gralloc buffer will likely be |
| // only indirectly represented here. |
| union CodecBufferData { |
| CodecBufferDataVmo vmo; |
| |
| // TODO(dustingreen): add the gralloc way |
| }; |
| |
| // CodecBufferDataVmo |
| // |
| // Details for a buffer backed by a VMO. |
| struct CodecBufferDataVmo { |
| // The same VMO can be used by more than one CodecBuffer (only of the same |
| // buffer_lifetime_ordinal), but each vmo_handle must be a separate handle. |
| handle<vmo> vmo_handle; |
| |
| // Offset within the VMO of the first usable byte. Must be < the VMO's size |
| // in bytes. |
| uint64 vmo_usable_start; |
| |
| // VMO-relative offset that's one past the last usable byte. This can point |
| // one byte beyond the end of the VMO if desired. In other words, this can |
| // be equal to the VMO's size, to indicate that the last byte of the VMO is |
| // usable (and possibly many byte before that, depending on |
| // vmo_usable_start). |
| uint64 vmo_usable_size; |
| }; |
| |
| // CodecPacketHeader |
| // |
| // When referring to a free packet, we use CodecPacketHeader alone instead of |
| // CodecPacket, since while a packet is free it doesn't really have meaningful |
| // offset or length etc. |
| // |
| // A populated CodecPacket also has a CodecPacketHeader. |
| struct CodecPacketHeader { |
| // This is which buffer configuration lifetime this header is referring to. |
| // |
| // A packet_index is only really meaningful with respect to a particular |
| // buffer_lifetime_ordinal. |
| // |
| // See CodecPortBufferSettings.buffer_lifetime_ordinal. |
| // |
| // For QueueInputPacket(), a server receiving a buffer_lifetime_ordinal that |
| // isn't the current input buffer_lifetime_ordinal will close the channel. |
| // |
| // For OnFreeInputPacket() and RecycleOutputPacket(), the reciever (client or |
| // server) must ignore a message with stale buffer_lifetime_ordinal. |
| uint64 buffer_lifetime_ordinal; |
| |
| // The overall set of packet_index values is densely packed from 0..count-1 |
| // for input and output separately. They can be queued in any order. |
| // |
| // Both the client and server should validate the packet_index against the |
| // known bound and disconnect if it's out of bounds. |
| // |
| // When running in buffer-per-packet mode, the packet_index is also the |
| // buffer index. When running in single-buffer mode, the buffer index is |
| // always 0 referring to the single buffer. |
| // |
| // The packet_index values don't imply anything about order of use of |
| // packets. The client should not expect the ordering to remain the same over |
| // time - the codec is free to hold on to an input or output packet for a |
| // while during which other packet_index values may be used multiple times. |
| // |
| // For a given properly-functioning Codec instance, packet_index values will |
| // be unique among concurrently-outstanding packets. Servers should validate |
| // that a client isn't double-using a packet and clients should validate as |
| // necessary to avoid undefined or unexpected client behavior. |
| uint32 packet_index; |
| }; |
| |
| // CodecPacket |
| // |
| // A CodecPacket represents a chunk of input or output data to or from a codec. |
| // |
| // codec output: |
| // While the CodecPacket is outstanding with the client via OnOutputPacket(), |
| // the codec will avoid modifying the referenced output data. After the client |
| // calls RecycleOutputPacket(packet_index), the codec is notified that the |
| // client is again ok with the referenced data changing. |
| // |
| // codec input: |
| // The client initially has all packet_index(es) available to fill, and later |
| // gets packet_index(s) that are again ready to fill via OnFreeInputPacket(). |
| // The client must not modify the referenced data in between QueueInputPacket() |
| // and OnFreeInputPacket(). |
| struct CodecPacket { |
| CodecPacketHeader header; |
| |
| // Which buffer this packet refers to. For single-buffer mode this will |
| // always be 0, but for multi-buffer mode, a given in-flight interval of a |
| // packet can refer to any buffer. The packet has an associated buffer only |
| // while the packet is in-flight, not while the packet is free. |
| // |
| // The default value makes accidental inappropriate use of index 0 less |
| // likely (will tend to complain in an obvious way if not filled out instead |
| // of a non-obvious data corruption when decoding buffer 0 repeatedly instead |
| // of the correct buffers). |
| uint32 buffer_index = 0x80000000; |
| |
| // stream_lifetime_ordinal |
| // |
| // The value 1 is the lowest permitted value after codec creation. Values |
| // sent by the client must be odd. Values must only increase. |
| // |
| // A stream_lifetime_ordinal represents the lifetime of a stream. All |
| // messages that are specific to a stream have the stream_lifetime_ordinal |
| // value and the value is the same for all messages relating to a given |
| // stream. |
| // |
| // See codec.md for more on stream_lifetime_ordinal. |
| uint64 stream_lifetime_ordinal; |
| |
| // start_offset and valid_length_bytes |
| // |
| // Which part of the relevant buffer is this packet using. These are valid |
| // for input data that's in-flight to the codec, and are valid for output data |
| // from the codec. |
| // |
| // For compressed formats and uncompressed audio, the data in |
| // [start_offset, start_offset + valid_length_bytes) is the contiguously valid |
| // data refered to by this packet. |
| // |
| // For uncompressed video frames, CodecFormatDetails is the primary means of |
| // determining which bytes are relevant. The offsets in CodecFormatDetails |
| // are relative to the start_offset here. The valid_length_bytes must be |
| // large enough to include the full last line of pixel data, including the |
| // full line stride of the last line (not just the width in pixels of the |
| // last line). |
| // |
| // Despite these being filled out, some uncompressed video buffers are of |
| // types that are not readable by the CPU. These fields being here don't |
| // imply there's any way for the CPU to read an uncompressed frame. |
| // |
| // TODO(dustingreen): Do we have any reason to require that these be filled |
| // out for opaque uncompressed video frames that the CPU can't read? In that |
| // case do we want to require them just so they can be potentially passed on |
| // to a HW renderer in case the HW renderer has any use for them? Or more |
| // likely, it may just be that these tend to refer to the whole size of an |
| // uncompressed buffer, with format_details taking care of specifying which |
| // bytes are actually relevant. |
| uint32 start_offset; |
| |
| // valid_length_bytes |
| // |
| // This must be > 0. |
| // |
| // The semantics for valid data per packet vary depending on data type as |
| // follows. |
| // |
| // uncompressed video - A video frame can't be split across packets. Each |
| // packet is one video frame. |
| // |
| // uncompressed audio - Regardless of float or int, linear or uLaw, or number |
| // of channels, a packet must contain an non-negative number of complete |
| // audio frames, where a single audio frame consists of data for all the |
| // channels for the same single point in time. Any codec-specific internal |
| // details re. lower rate sampling for LFE channel or the like should be |
| // hidden by the Codec server implementation. |
| // |
| // compressed data input - A packet must contain at least one byte of data. |
| // See also codec_input_bytes_min. Splitting AUs at arbitrary byte |
| // boundaries is permitted, including at boundaries that are in AU headers. |
| // |
| // compressed data output - The codec is not required to fully fill each |
| // output packet's buffer. |
| uint32 valid_length_bytes; |
| |
| // This value is not strictly speaking a timestamp. It is an arbitrary |
| // unsigned 64-bit number that, under some circumstances, will be passed by a |
| // codec unmodified from an input packet to the exactly-corresponding output |
| // packet. |
| // |
| // For timestamp_ish values to be propagated from input to output the |
| // following conditions must be true: |
| // * promise_separate_access_units_on_input must be true |
| // * has_timestamp_ish must be true for a given input packet, to have that |
| // timestamp_ish value (potentially) propagate through to an output |
| // * the Codec instance itself decides (async) that the input packet |
| // generates an output packet - if a given input never generates an output |
| // packet then the timestamp_ish value on the input will never show up on |
| // any output packet - depending on the characteristics of the input and |
| // output formats, and whether a decoder is willing to join mid-stream, etc |
| // this can be more or less likely to occur, but clients should be written |
| // to accomodate timestamp_ish values that are fed on input but never show |
| // up on output, at least to a reasonable degree (not crashing, not |
| // treating as an error). |
| // |
| // See codec.md for more on timestamp_ish. |
| bool has_timestamp_ish; |
| uint64 timestamp_ish; |
| |
| // start_access_unit |
| // |
| // If promise_separate_access_units_on_input (TODO(dustingreen): or any |
| // similar mode for output) is true, this bool must be set appropriately |
| // depending on whether byte 0 _is_ or _is not_ the start of an access unit. |
| // The client is required to know, and required to set this boolean properly. |
| // The server is allowed to infer that when this boolean is false, byte 0 is |
| // the first byte of a continuation of a previously-started AU. (The byte at |
| // start_offset is "byte 0".) |
| // |
| // If promise_separate_access_units_on_input is false, this boolean is |
| // ignored. |
| bool start_access_unit; |
| |
| // known_end_access_unit |
| // |
| // A client is never required to set this boolean to true. |
| // |
| // If promise_separate_access_units_on_input is true, for input data, this |
| // boolean must be false if the last byte of this packet is not the last byte |
| // of an AU, and this boolean _may_ be true if the last byte of this packet |
| // is the last byte of an AU. A client delivering one AU at a time that's |
| // interested in the lowest possible latency via the decoder should set this |
| // boolean to true when it can be set to true. |
| // |
| // If promise_separate_access_units_on_input is false, this boolean is |
| // ignored. |
| bool known_end_access_unit; |
| }; |
| |
| // Codec |
| // |
| // The Codec interface exists to anchor the configuration of input and output |
| // buffers, and depending on Codec server hosting strategy, the Codec inteface |
| // can, in some configurations, be 1:1 with a codec isolate (process), |
| // especially when using SW codecs. The Codec can be used to process up to one |
| // stream at a time. |
| // |
| // Descriptions of actions taken by methods of this interface and the states of |
| // things are given as if the methods are synchronously executed by the codec |
| // server, but in reality, as is typical of FIDL interfaces, the message |
| // processing is async. The states described are to be read as the state from |
| // the client's point of view unless otherwise stated. Events coming back from |
| // the server are of course delivered async, and a client that processes more |
| // than one stream per Codec instance needs to care whether a given event is |
| // from the current stream vs. some older soon-to-be-gone stream. |
| // |
| // The Sync() method's main purpose is to enable the client to robustly prevent |
| // having both old and new buffers allocated in the system at the same time, |
| // since media buffers can be significantly large, depending. The Sync() method |
| // achieves this by only delivering it's response when all previous calls to |
| // the Codec interface have actually taken effect in the StreamControl ordering |
| // domain. Sync() can also be used to wait for the codec server to catch up if |
| // there's a possibility that a client might otherwise get too far ahead of the |
| // Codec server, by for example requesting creation of a large number of |
| // streams in a row. It can also be used during debugging to ensure that a |
| // codec server hasn't gotten stuck. Calling Sync() is entirely optional and |
| // never required for correctness - only potentially required to de-overlap |
| // resource usage. |
| // |
| // Semi-trusted Codec server - SW decoders run in an isolate (with very few |
| // capabilities) just in case the decoding SW has a vulnerability which could |
| // be used to take over the Codec server. Clients of the codec interface using |
| // decoders and processing streams of separate security contexts, to a greater |
| // extent than some other interfaces, need to protect themselves against |
| // invalid server behavior, such as double-free of a packet_index and any other |
| // invalid server behavior. Having fed in compressed data of one security |
| // context, don't place too much trust in a single Codec instance to not mix |
| // data among any buffers that Codec server has ever been told about. Instead, |
| // create separate Codec instances for use by security-separate client-side |
| // contexts. While the picture for HW-based decoders looks somewhat different |
| // and is out of scope of this paragraph, the client should always use separate |
| // Codec instances for security-separate client-side contexts. |
| interface Codec { |
| // EnableOnStreamFailed() |
| // |
| // Permit the server to use OnStreamFailed() instead of the server just |
| // closing the whole Codec channel on stream failure. |
| // |
| // If the server hasn't seen this message by the time a stream fails, the |
| // server will close the Codec channel instead of sending OnStreamFailed(). |
| EnableOnStreamFailed(); |
| |
| // OnStreamFailed() |
| // |
| // The stream has failed, but the Codec instance is still usable for a new |
| // stream. |
| // |
| // This message is only ever sent by the server if the client previously sent |
| // EnableOnStreamFailed(). If the client didn't send EnableOnStreamFailed() |
| // then the server closes the Codec channel instead. |
| // |
| // Codec server implementations are encouraged to handle stream errors (and |
| // ideally to also report them via error_ bools of OnOutputPacket() and |
| // OnOutputEndOfStream()) without failing the whole stream, but if a codec |
| // server is unable to do that, but still can cleanly contain the failure to |
| // the stream, the codec server can (assuming EnableOnStreamFailed() was |
| // called) use OnStreamFailed() to indicate the stream failure to the client |
| // without closing the Codec channel. |
| // |
| // An ideal Codec server handles problems with input data without sending |
| // this message, but sending this message is preferred vs. closing the server |
| // end of the Codec channel if the Codec server can 100% reliably contain the |
| // stream failure to the stream, without any adverse impact to any later |
| // stream. |
| // |
| // No further messages will arrive from the server regarding the failed |
| // stream. This includes any OnOutputEndOfStream() that the client would |
| // have otherwise expected. |
| // |
| // TODO(dustingreen): Add at least an error_message string and _maybe_ a |
| // zx_status_t, though that might tend to encourage mis-use of zx_status_t so |
| // maybe just error_message for quicker debugging on the client side. Also |
| // plumb from CodecAdapterH264 and similar. |
| -> OnStreamFailed(uint64 stream_lifetime_ordinal); |
| |
| // OnInputConstraints() |
| // |
| // The server sends this shortly after Codec creation to indicate input |
| // buffer constraints. The "min" and "max" input constraints don't change |
| // for the life of the Codec. |
| // |
| // The "max" values for buffer size and count are large enough to support the |
| // most demanding format the server supports on input. The "recommended" |
| // values should be workable for use with the input CodecFormatDetails |
| // conveyed during Codec creation. The "recommended" values are not |
| // necessarily suitable if the client uses QueueInputFormatDetails() to |
| // change the input format. In that case it's up to the client to determine |
| // suitable values, either by creating a new Codec instance instead, or |
| // knowing suitable values outside the scope of this protocol. |
| // |
| // See comments on CodecBufferConstraints. |
| // |
| // This message is guaranteed to be sent unsolicited to the Codec client |
| // during or shortly after Codec creation. Clients should not depend on this |
| // being the very first message to arrive at the client. OnOutputConfig() |
| // may be sent first by some codecs that already know their initial output |
| // config without any input data, to encourage (but not stictly require) the |
| // client to configure output buffers before feeding the first input, to |
| // avoid a wasteful OnOutputConfig() being generated for that first stream if |
| // the client has started configuring output but isn't done configuring |
| // output before the client sends the first input data for the first stream. |
| // A client is free to ignore OnOutputConfig() with a stale |
| // stream_lifetime_ordinal, but handling OnOutputConfig() with |
| // stream_lifetime_ordinal 0 (if any are sent) can help reduce latency to |
| // first output. See OnOutputConfig() for more details. |
| // |
| // The "min" and "max" input constraints are guaranteed not to change for a |
| // given Codec instance. The "recommended" values may effectively change |
| // when the server processes QueueInputFormatDetails(). There is not any way |
| // in the protocol short of creating a new Codec instance for the client to |
| // get those new "recommended" values. |
| // |
| // TODO(dustingreen): Maybe provide a way for the client to get updated |
| // "recommended" values for input, maybe only on request rather than via this |
| // event, to keep things simpler for simpler clients. Maybe separate the |
| // recommendations from the constraints. |
| -> OnInputConstraints(CodecBufferConstraints input_constraints); |
| |
| // SetInputBufferSettings() and AddInputBuffer() |
| // |
| // Configuring input buffers consists of calling SetInputBufferSettings() |
| // followed by a number of calls to AddInputBuffer() equal to the number of |
| // buffers set via SetInputBufferSettings(). In buffer-per-packet mode, this |
| // is the same as the number of packets. In single-buffer mode, this is 1. |
| // |
| // After OnInputConstraints(), the client uses these two methods to set up |
| // input buffers and packets. |
| // |
| // Configuring input buffers is required before QueueInputPacket(). |
| // |
| // The client can also re-set-up input buffers any time there is no current |
| // stream. The client need not wait until all previously-set-up input |
| // buffers are with the client via OnFreeInputPacket(). The old |
| // buffer_lifetime_ordinal just ends. See codec.md for more info on "buffer |
| // lifetime". |
| // |
| // The recommended way to de-overlap resource usage (when/if the client wants |
| // to) is to send CloseCurrentStream() with release_input_buffers true then |
| // send Sync() and wait for its response before allocating any new buffers. |
| // How to cause other parts of the system to release their references on |
| // low-level buffers is outside the scope of this interface. |
| // |
| // This call ends any previous buffer_lifetime_ordinal, and starts a new one. |
| SetInputBufferSettings(CodecPortBufferSettings input_settings); |
| |
| // The client is required to add all the input buffers before sending any |
| // message that starts a new stream else the codec will close the Codec |
| // channel. |
| // |
| // When the last buffer is added with this message, all the input packets |
| // effectively jump from non-existent to free with the client. The Codec |
| // will not generate an OnFreeInputPacket() for each new input packet. The |
| // client can immediately start sending QueueInputPacket() after sending the |
| // last AddInputBuffer(). |
| AddInputBuffer(CodecBuffer buffer); |
| |
| // OnOutputConfig() |
| // |
| // This event informs the client of new output config. The server will send |
| // at least one of these messages before the first output packet of a stream, |
| // but that message might not have buffer_constraints_action_required true. |
| // |
| // If buffer_constraints_action_required is true and the |
| // stream_lifetime_ordinal matches the current stream, the client must react |
| // by configuring or re-configuring output buffers. |
| // |
| // Some clients may prefer not to support mid-stream output config changes, |
| // but even those clients are required to process OnOutputConfig() messages |
| // up to the first output packet of each stream, as OnOutputConfig() is used |
| // for stream format detection as well as for potential mid-stream output |
| // config changes. |
| // |
| // For more on OnOutputConfig(), see cocec.md. |
| -> OnOutputConfig(CodecOutputConfig output_config); |
| |
| // SetOutputBufferSettings() and AddOutputBuffer() |
| // |
| // These are not permitted until after the first OnOutputConfig(). |
| // |
| // Roughly speaking, these messages are sent in response to OnOutputConfig() |
| // with buffer_constraints_action_required true. |
| // |
| // Configuring output buffers consists of calling SetOutputBufferSettings() |
| // followed by a number of calls to AddOutputBuffer() equal to the number of |
| // buffers set via SetOutputBufferSettings(). In buffer-per-packet mode, this |
| // is the same as the number of packets. In single-buffer mode, this is 1. |
| // |
| // Configuring output buffers is _required_ after OnOutputConfig() is |
| // received by the client with buffer_constraints_action_required true and |
| // stream_lifetime_ordinal equal to the client's current |
| // stream_lifetime_ordinal (even if there is an active stream), and is |
| // _permitted_ any time there is no current stream. |
| // |
| // For more on SetOutputBufferSettings() and AddOutputBuffer(), see codec.md. |
| SetOutputBufferSettings(CodecPortBufferSettings output_settings); |
| AddOutputBuffer(CodecBuffer buffer); |
| |
| // FlushEndOfStreamAndCloseStream() |
| // |
| // This message is optional. |
| // |
| // This message is only valid after QueueInputEndOfStream() for this stream. |
| // The stream_lifetime_ordinal input parameter must match the |
| // stream_lifetime_ordinal of the QueueInputEndOfStream(), else the server |
| // will close the channel. |
| // |
| // A client can use this message to flush through (not discard) the last |
| // input data of a stream so that the codec server generates corresponding |
| // output data for all the input data before the server moves on to the next |
| // stream, without forcing the client to wait for OnOutputEndOfStream() |
| // before queueing data of another stream. |
| // |
| // The difference between QueueInputEndOfStream() and |
| // FlushEndOfStreamAndCloseStream(): QueueInputEndOfStream() is a promise |
| // from the client that there will not be any more input data for the stream |
| // (and this info is needed by some codecs for the codec to ever emit the very |
| // last output data). The QueueInputEndOfStream() having been sent doesn't |
| // prevent the client from later completely discarding the rest of the |
| // current stream by closing the current stream (with or without a stream |
| // switch). In contrast, FlushEndOfStreamAndCloseStream() is a request from |
| // the client that all the previously-queued input data be processed |
| // including the logical "EndOfStream" showing up as OnOutputEndOfStream() |
| // (in success case) before moving on to any newer stream - this essentially |
| // changes the close-stream handling from discard to flush-through for this |
| // stream only. |
| // |
| // A client using this message can start providing input data for a new |
| // stream without that causing discard of old stream data. That's the purpose |
| // of this message - to allow a client to flush through (not discard) the old |
| // stream's last data (instead of the default when closing or switching |
| // streams which is discard). |
| // |
| // Because the old stream is not done processing yet and the old stream's |
| // data is not being discarded, the client must be prepared to continue to |
| // process OnOutputConfig() messages until the stream_lifetime_ordinal is |
| // done. The client will know the stream_lifetime_ordinal is done when |
| // OnOutputEndOfStream(), OnStreamFailed(), or the Codec channel closes. |
| // |
| // For more on FlushEndOfStreamAndCloseStream(), see codec.md. |
| FlushEndOfStreamAndCloseStream(uint64 stream_lifetime_ordinal); |
| |
| // CloseCurrentStream() |
| // |
| // This "closes" the current stream, leaving no current stream. In addition, |
| // this message can optionally release input buffers or output buffers. |
| // |
| // If there has never been any active stream, the stream_lifetime_ordinal must |
| // be zero or the server will close the channel. If there has been an active |
| // stream, the stream_lifetime_ordinal must be the most recent active stream |
| // whether that stream is still active or not. Else the server will close the |
| // channel. |
| // |
| // Multiple of this message without any new active stream in between is not |
| // to be considered an error, which allows a client to use this message to |
| // close the current stream to stop wasting processing power on a stream the |
| // user no longer cares about, then later decide that buffers should be |
| // released and send this message again with release_input_buffers and/or |
| // release_output_buffers true to get the buffers released, if the client is |
| // interested in trying to avoid overlap in resource usage between old |
| // buffers and new buffers (not all clients are). |
| // |
| // See also Sync(). |
| // |
| // For more on CloseCurrentStream(), see codec.md. |
| CloseCurrentStream( |
| uint64 stream_lifetime_ordinal, |
| bool release_input_buffers, |
| bool release_output_buffers); |
| |
| // Sync() -> () |
| // |
| // On completion, all previous Codec calls have done what they're going to do |
| // server-side, _except_ for processing of data queued using |
| // QueueInputPacket(). |
| // |
| // The main purpose of this call is to enable the client to wait until |
| // CloseCurrentStream() with release_input_buffers and/or |
| // release_output_buffers set to true to take effect, before the client |
| // allocates new buffers and re-sets-up input and/or output buffers. This |
| // de-overlapping of resource usage can be worthwhile for media buffers which |
| // can consume resource types whose overall pools aren't necessarily vast in |
| // comparsion to resources consumed. Especially if a client is reconfiguring |
| // buffers multiple times. |
| // |
| // Note that Sync() prior to allocating new media buffers is not alone |
| // sufficient to achieve non-overlap of media buffer resource usage system |
| // wide, but it can be a useful part of achieving that. |
| // |
| // The Sync() transits the Output ordering domain and the StreamControl |
| // ordering domain, but not the InputData ordering domain. For more on |
| // ordering domains see codec.md. |
| // |
| // This request can be used to avoid hitting kMaxInFlightStreams which is |
| // presently 10. A client that stays <= 8 in-flight streams will comfortably |
| // stay under the limit of 10. While the protocol permits repeated |
| // SetInputBufferSettings() and the like, a client that spams the channel |
| // can expect that the channel will just close if the server or the channel |
| // itself gets too far behind. |
| Sync() -> (); |
| |
| // OnOutputPacket() |
| // |
| // This is how the codec emits an output packet to the codec client. |
| // |
| // Order is significant. |
| // |
| // The client should eventually call RecycleOutputPacket() (possibly after |
| // switching streams multiple times), unless the buffer_lifetime_ordinal has |
| // moved on. A stream change doesn't change which packets are busy with |
| // the client vs. free with the server. See "packet lifetime" in codec.md for |
| // more. |
| // |
| // The relevant buffer is buffer 0 if running in single-buffer mode, or the |
| // buffer index is the same as packet_index if running in buffer-per-packet |
| // mode. |
| // |
| // For low-level buffer types that support it, a Codec is free to emit an |
| // output packet before the low-level buffer actually has any usable data in |
| // the buffer, with the mechanism for signalling the presence of data |
| // separate from the OnOutputPacket() message. For such low-level buffer |
| // types, downstream consumers of data from the emitted packet must |
| // participate in the low-level buffer signalling mechanism to know when it's |
| // safe to consume the data. This is most likely to be relevant when using a |
| // video decoder and gralloc-style buffers. |
| // |
| // The error_ bool(s) allow (but do not require) a Codec server to report |
| // errors that happen during an AU or between AUs. |
| // |
| // The scope of error_detected_before starts at the end of the last delivered |
| // output packet on this stream, or the start of stream if there were no |
| // previous output packets on this stream. The scope ends at the start of |
| // the output_packet. |
| // |
| // The error_detected_before bool is separate so that discontinuities can be |
| // indicated separately from whether the current packet is damaged. |
| // |
| // The scope of error_detected_during is from the start to the end of this |
| // output_packet. |
| -> OnOutputPacket( |
| CodecPacket output_packet, |
| bool error_detected_before, |
| bool error_detected_during); |
| |
| // RecycleOutputPacket() |
| // |
| // After the client is done with an output packet, the client needs to tell |
| // the codec that the output packet can be re-used for more output, via this |
| // method. |
| // |
| // It's not permitted to recycle an output packet that's already free with the |
| // codec server. It's permitted but discouraged for a client to recycle an |
| // output packet that has been deallocated by an explicit or implicit output |
| // buffer de-configuration(). See buffer_lifetime_ordinal for more on that. |
| // A server must ignore any such stale RecycleOutputPacket() calls. |
| // |
| // For more on RecycleOutputPacket(), see codec.md. |
| RecycleOutputPacket(CodecPacketHeader available_output_packet); |
| |
| // OnOutputEndOfStream() |
| // |
| // After QueueInputEndOfStream() is sent by the Codec client, within a |
| // reasonable duration the corresponding OnOutputEndOfStream() will be sent by |
| // the Codec server. Similar to QueueInputEndOfStream(), |
| // OnOutputEndOfStream() is sent a maximum of once per stream. |
| // |
| // No more stream data for this stream will be sent after this message. All |
| // input data for this stream was processed. |
| // |
| // While a Codec client is not required to QueueInputEndOfStream() (unless |
| // the client wants to use FlushEndOfStreamAndCloseStream()), if a Codec |
| // server receives QueueInputEndOfStream(), and the client hasn't closed the |
| // stream, the Codec server must generate a corresponding |
| // OnOutputEndOfStream() if nothing went wrong, or must send |
| // OnStreamFailed(), or must close the server end of the Codec channel. An |
| // ideal Codec server would handle and report stream errors via the error_ |
| // flags and complete stream processing without sending OnStreamFailed(), but |
| // in any case, the above-listed options are the only ways that an |
| // OnOutputEndOfStream() won't happen after QueueInputEndOfStream(). |
| // |
| // There will be no more OnOutputPacket() or OnOutputConfig() messages for |
| // this stream_lifetime_ordinal after this message - if a server doesn't |
| // follow this rule, a client should close the Codec channel. |
| // |
| // The error_detected_before bool has the same semantics as the |
| // error_detected_before bool in OnOutputPacket(). |
| -> OnOutputEndOfStream( |
| uint64 stream_lifetime_ordinal, |
| bool error_detected_before); |
| |
| // |
| // Stream specific messages: |
| // |
| |
| // QueueInputFormatDetails() |
| // |
| // If the input format details are still the same as specified during Codec |
| // creation, this message is unnecessary and does not need to be sent. |
| // |
| // If the stream doesn't exist yet, this message creates the stream. |
| // |
| // All servers must permit QueueInputFormatDetails() at the start of a stream |
| // without failing, as long as the new format is supported by the Codec |
| // instance. Technically this allows for a server to only support the exact |
| // input format set during Codec creation, and that is by design. A client |
| // that tries to switch formats and gets a Codec channel failure should try |
| // again one more time with a fresh Codec instance created with CodecFactory |
| // using the new input format during creation, before giving up. |
| // |
| // These format details override the format details |
| // specified during codec creation for this stream only. The next stream will |
| // default back to the format details set during codec creation. |
| // |
| // For now, QueueInputFormatDetails() sent mid-stream will fail the Codec |
| // channel. Clients shouldn't do this for now. |
| // |
| // This message is permitted at the start of the first stream (just like at |
| // the start of any stream). The format specified need not match what was |
| // specified during codec creation, but if it doesn't match, the Codec channel |
| // might close as described above. |
| QueueInputFormatDetails( |
| uint64 stream_lifetime_ordinal, CodecFormatDetails format_details); |
| |
| // QueueInputPacket() |
| // |
| // This message queues input data to the codec for processing. |
| // |
| // If the stream doesn't exist yet, this message creates the new stream. |
| // |
| // The client is required to be willing to send QueueInputPacket() prior to |
| // the server's first OnOutputConfig(), and is permitted to start a new stream |
| // without output buffers configured yet. |
| // |
| // The client must continue to deliver input data via this message even if the |
| // codec has not yet generated the first OnOutputConfig(), and even if the |
| // Codec is generating OnFreeInputPacket() for previously-queued input |
| // packets. The input data must continue as long as there are free packets |
| // to be assured that the server will ever generate the first |
| // OnOutputConfig(). |
| // |
| // For more on QueueInputPacket(), see codec.md. |
| QueueInputPacket(CodecPacket packet); |
| |
| // OnFreeInputPacket() |
| // |
| // The server sends this message when the codec is done consuming this packet |
| // and the packet can be re-filled by the client. |
| // |
| // This is not sent for all packets when a new buffer_lifetime_ordinal starts |
| // as in that case all the packets are initially free with the client. |
| // |
| // See comments on QueueInputBuffer() and "packet lifetime" in codec.md for |
| // for description of buffer lifetime and packet lifetime. |
| // |
| // After receiving the available input buffer via this event, the codec |
| // client can call later call QueueInputBuffer with appropriate offset and |
| // length set. |
| // |
| // TODO(dustingreen): At the moment, there is no guarantee re. the order of |
| // these messages with respect to the order of QueueInputPacket(), but at |
| // least for decoders, it might be worthwhile to require that servers preserve |
| // the order vs. QueueInputPacket(), to make it easier to feed input from a |
| // ring buffer or similar. For audio encoders it might still make sense. For |
| // video encoders probably not. |
| -> OnFreeInputPacket(CodecPacketHeader free_input_packet); |
| |
| // Inform the server that all QueueInputPacket() messages for this stream |
| // have been sent. |
| // |
| // If the stream isn't closed first (by the client, or by OnStreamFailed(), or |
| // Codec channel closing), there will later be a corresponding |
| // OnOutputEndOfStream(). |
| // |
| // The corresponding OnOutputEndOfStream() message will be generated only if |
| // the server finishes processing the stream before the server sees the |
| // client close the stream (such as by starting a new stream). A way to |
| // force the server to finish the stream before closing is to use |
| // FlushEndOfStreamAndCloseStream() after QueueInputEndOfStream() before any |
| // new stream. Another way to force the server to finish the stream before |
| // closing is to wait for the OnOutputEndOfStream() before taking any action |
| // that closes the stream. |
| // |
| // In addition to serving as an "EndOfStream" marker to make it obvious |
| // client-side when all input data has been processed, if a client never |
| // sends QueueInputEndOfStream(), no amount of waiting will necessarily |
| // result in all input data getting processed through to the output. Some |
| // codecs have some internally-delayed data which only gets pushed through by |
| // additional input data _or_ by this EndOfStream marker. In that sense, |
| // this message can be viewed as a flush-through at InputData domain level, |
| // but the flush-through only takes effect if the codec even gets that far |
| // before the stream is just closed at StreamControl domain level. This |
| // message is not alone sufficient to act as an overall flush-through at |
| // StreamControl level. For that, send this message first and then send |
| // FlushEndOfStreamAndCloseStream() (at which point it becomes possible to |
| // queue input data for a new stream without causing discard of this older |
| // stream's data), or wait for the OnOutputEndOfStream() before closing the |
| // current stream. |
| // |
| // If a client sends QueueInputPacket(), QueueInputFormatDetails(), |
| // QueueInputEndOfStream() for this stream after the first |
| // QueueInputEndOfStream() for this stream, a server should close the Codec |
| // channel. |
| QueueInputEndOfStream(uint64 stream_lifetime_ordinal); |
| }; |
