blob: c5f26fc7986ddb7f97caf13de28f22bf16f543d2 [file] [log] [blame]
// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <fuchsia/media/cpp/fidl.h>
#include <fuchsia/mediacodec/cpp/fidl.h>
#include <list>
#include <random>
#include <fbl/macros.h>
#include "codec_adapter_events.h"
#include "codec_input_item.h"
#include "codec_port.h"
class CodecBuffer;
// The CodecAdapter abstract base class is used by CodecImpl to interface with
// a particular SW or HW codec. At the layer of this interface, there's only
// ever up to one active stream to worry about, and Codec FIDL protocol
// enforcement has already been handled above.
// For HW-based codecs that need to share the HW, a CodecAdapter represents up
// to one active stream, and does not directly participate in sharing the HW;
// that's further down.
// The intent of this interface is to be as narrow an in-process codec interface
// as feasible between FIDL protocol aspects above, and codec-specific details
// below.
class CodecAdapter {
// At least for now, the CodecImpl and CodecAdapter share their main lock.
// The CodecImpl won't call CodecAdapter methods with the lock_ held, mainly
// to avoid building up dependencies on the lock sharing, and also to avoid
// situations where the core codec code would just have to release the lock_
// in order to acquire video_decoder_lock_ (which is "before" lock_, due to
// calls from interrupt handlers that already have video_decoder_lock_ held).
// The CodecAdapter should never call CodecAdapterEvents methods with lock_
// held.
CodecAdapter(std::mutex& lock, CodecAdapterEvents* codec_adapter_events);
virtual ~CodecAdapter();
// Core codec.
// For the moment, these methods are placeholders for calls to the core codec.
// TODO(dustingreen): Implement these. Maybe switch them to
// core_codec_->Method() calls at the call sites, if there's nothing relevant
// to do in the wrapper methods.
// During format detection, a codec may be ok with null output config (false),
// or may require an output config (true).
virtual bool IsCoreCodecRequiringOutputConfigForFormatDetection() = 0;
// If true, the codec can make use of VMOs that are mapped for direct access
// by the CPU.
// If true, the CodecImpl will map the buffer VMOs unless buffers are secure
// memory, and CodecBuffer::base() is usable for direct CPU access.
// If a codec doesn't support secure memory operation, then buffers won't be
// secure memory and will be mapped if IsCoreCodecMappedBufferUseful().
// If buffers are secure, then CodecImpl won't actually map the buffer VMOs,
// and CodecBuffer::base() isn't usable for direct CPU access. Instead
// CodecBuffer::base() will be a vaddr that will fault if accessed as if it
// were buffer data, and preserves the low-order vmo_usable_start %
// ZX_PAGE_SIZE bits.
virtual bool IsCoreCodecMappedBufferUseful(CodecPort port) = 0;
// If true, the codec is HW-based, in the sense that at least some of the
// processing is performed by specialized processing HW running separately
// from any CPU execution context.
virtual bool IsCoreCodecHwBased(CodecPort port) = 0;
// Any core codec that performs DMA that will potentially continue beyond the
// lifetime of the process that holds open the VMO handles being DMA(ed)
// should override this method to provide CodecImpl with the driver's BTI so
// VMOs can be properly pinned for DMA. If a core codec returns true from
// IsCoreCodecHwBased(), the core codec should also override this method.
// TODO(38650): At least the VP9 decoder isn't overriding this method yet.
// Also we should enforce that this method be overridden when
// IsCoreCodecHwBased() true.
virtual zx::unowned_bti CoreCodecBti() { return zx::unowned_bti(); }
// The initial input format details and later input format details will
// _often_ remain the same overall format, and only differ in ways that are
// reasonable on a format-specific basis. However, not always. A core codec
// should check that any new input format details is still fully compatible
// with the core codec's initialized configuration (as set up during
// CoreCodecInit()), and if not, fail the CodecImpl using
// onCoreCodecFailCodec(). Core codecs may re-configure themselves based
// on new input FormatDetails to the degree that's reasonable for the
// input format and the core codec's capabilities, but there's no particular
// degree to which this is required (for now at least). Core codecs are
// discouraged from attempting to reconfigure themselves to process completely
// different input formats that are better to think of as a completely
// different Codec.
// A client that's using different FormatDetails than the initial
// FormatDetails (to any degree) should try one more time with a fresh
// Codec before giving up (giving up immediately only if the format details at
// time of failure match the initial format details specified during Codec
// creation).
// The core codec can copy the initial_input_format_details during this call
// (using fidl::Clone() or similar), but as is the custom with references,
// should not stash the passed-in reference.
// TODO(dustingreen): Re-visit the lifetime rule and required copy here, once
// more is nailed down re. exactly how the core codec relates to CodecImpl.
virtual void CoreCodecInit(const fuchsia::media::FormatDetails& initial_input_format_details) = 0;
// The default implementation silently accepts OFF. On any other value, the
// default implementation fails the codec.
// CodecAdapter sub-classes can accept ON if they support ON. Same for
// DYNAMIC if/when we add that. The CodecFactory implementation should
// already have information on which codecs support ON (or DYNAMIC) for output
// or input, and CodecImpl will enforce consistency of BufferCollection
// constraints and BufferCollectionInfo_2 with the SecureMemoryMode specified
// during codec creation.
virtual void CoreCodecSetSecureMemoryMode(
CodecPort port, fuchsia::mediacodec::SecureMemoryMode secure_memory_mode);
// All codecs must implement this for both ports. The returned structure will
// be sent to sysmem in a SetConstraints() call. This method can be called
// on the FIDL thread or the StreamControl domain (thread for now).
// Input:
// For now, a core codec has no way to trigger being asked for new input
// constraints, so the input constraints (for now) need to be generally
// applicable to any potential setting/property of the input.
// A decoder should permit a fairly wide range of buffer space, without
// worrying whether the min is enough to efficiently handle a high bitrate.
// The CodecImpl will own bumping up the min based on approximate bitrate
// provided in the initial decoder creation parameters and/or per-stream input
// format details (this logic is shared because it can reasonably be shared).
// A core codec that has special requirements for extra input buffer space
// given a particular bitrate can take it upon itself to set the input min
// buffer space, but the idea is that typically it won't be necessary for a
// decoder to increase the min based on input bitrate beyond what CodecImpl
// does, so needing to do this should be fairly rare.
// A video encoder which needs to vary it's input BufferCollectionConstraints
// based on encoder settings can do so, using the data provided to
// CoreCodecInit(). If later use of QueueInputFormatDetails() (per-stream)
// results in an input packet that conforms to the old
// BufferCollectionConstraints but does not conform to the effective new
// BufferCollectionConstraints, the core codec can use
// CodecAdapterEvents::onCoreCodecFailCodec() (the core codec should fail the
// codec instance in this case rather than attempt to handle input data that
// is outside the bounds that would have been indicated by the core codec had
// the current input format details been used as the initial format details).
// Clients that change the input format details on the fly should be willing
// to re-request a new codec instance at least once starting with the new
// input format details via the CodecFactory. This is true for additional
// reasons beyond this paragraph involving the possibility of accelerated but
// partial codec implementations. If a client needs to change input format
// details but doesn't want to concern itself with tracking whether the
// current codec was created with the current input format details, a client
// can instead choose to always create a new codec via CodecFactory on any
// change to the input format details.
// Output:
// A core codec can trigger this method to get called again by indicating an
// output format detection/change with action_required true via
// CoreCodecEvents::onCoreCodecMidStreamOutputConstraintsChange().
// Filling out the usage bits is optional. If the usage bits are not filled
// out (all still 0), the caller will fill them out based on
// IsCoreCodecMappedBufferUseful() and IsCoreCodecHwBased(). The core codec
// must either leave usage set to all 0, or completely fill them out.
virtual fuchsia::sysmem::BufferCollectionConstraints CoreCodecGetBufferCollectionConstraints(
CodecPort port, const fuchsia::media::StreamBufferConstraints& stream_buffer_constraints,
const fuchsia::media::StreamBufferPartialSettings& partial_settings) = 0;
// There are no VMO handles in the buffer_collection_info. Those are instead
// provided via calls to CoreCodecAddBuffer(), as CodecImpl handles allocation
// of CodecBuffer instances (each of which has a VMO).
// This method allows a core codec to know things like buffer_count, whether
// sysmem selected CPU domain or RAM domain for sharing of buffers, whether
// protected buffers were allocated, etc.
// This call occurs regardless of whether "settings" or "partial settings" are
// set (regardless of whether the client is using sysmem), after the client
// sets input or output settings, and before the first buffer is added.
virtual void CoreCodecSetBufferCollectionInfo(
CodecPort port, const fuchsia::sysmem::BufferCollectionInfo_2& buffer_collection_info) = 0;
// Stream lifetime:
// The CoreCodecStartStream() and CoreCodecStopStream() calls bracket the
// lifetime of the current stream. The CoreCodecQueue.* calls are
// stream-specific and apply to the current stream. There is only up to one
// current stream, and CoreCodecQueue.* calls will only occur when there is a
// current stream.
// At least for now, we don't use a separate object instance for the current
// stream, for the following reasons:
// * This interface is the validated+de-async-ed version of the Codec FIDL
// interface and the Codec FIDL interface doesn't have a separate Stream
// object/channel, so not having a separate stream object here makes the
// correspondence closer.
// * While the stream is fairly separate, there are also aspects of stream
// behavior such as mid-stream output format change which can cause a
// stream to essentially re-configure codec-wide output buffers, so the
// separate-ness of a stream from the codec isn't complete (regardless of
// separate stream object or not).
// All that said, it certainly can be useful to think of the stream as a
// logical lifetime of a thing, despite it not being a separate object (at
// least for now). Some implementations of CodecAdapter may find it
// convenient to create their own up-to-one-at-a-time-per-CodecAdapter stream
// object to model the current stream, and that's totally fine.
// The "Queue" methods will only be called in between CoreCodecStartStream()
// and CoreCodecStopStream().
virtual void CoreCodecStartStream() = 0;
// The parameter includes the oob_bytes. The core codec is free to call
// onCoreCodecFailCodec() (immediately on this stack or async) if the
// override input format details can't be accommodated (even in situations
// where the override input format details would be ok as initial input format
// details, such as when new input buffer config is needed).
// That said, the core codec should try to accommodate the change, especially
// if the client has configured adequate input buffers, and the basic type of
// the input data hasn't changed.
// TODO(dustingreen): Nail down the above sorta-vaguely-described rules
// better.
// Only permitted between CoreCodecStartStream() and CoreCodecStopStream().
virtual void CoreCodecQueueInputFormatDetails(
const fuchsia::media::FormatDetails& per_stream_override_format_details) = 0;
// Only permitted between CoreCodecStartStream() and CoreCodecStopStream().
virtual void CoreCodecQueueInputPacket(CodecPacket* packet) = 0;
// Only permitted between CoreCodecStartStream() and CoreCodecStopStream().
virtual void CoreCodecQueueInputEndOfStream() = 0;
// Stop the core codec from processing any more data for the stream that was
// active and is now stopping.
virtual void CoreCodecStopStream() = 0;
// Reset the stream. Used in processing a watchdog. If an adapter never generates a watchdog, it
// doesn't need to override this method.
// Do not discard any input data or output data beyond what was being worked on at the time the
// watchdog fired.
virtual void CoreCodecResetStreamAfterCurrentFrame();
// Add input or output buffer.
// The buffers added via this method correspond to the buffers of the buffer
// collection - these are buffers of the collection most recently indicated
// via a call to CoreCodecSetBufferCollectionInfo(). While the VMOs are
// intentionally not included in that call, the VMOs are indicated here (this
// lets the CodecImpl own allocation of CodecBuffer instances).
// A core codec may be able to fully configure a buffer during this call and
// later ignore CoreCodecConfigureBuffers(), or a core codec may use
// CoreCodecConfigureBuffers() to finish configuring buffers.
virtual void CoreCodecAddBuffer(CodecPort port, const CodecBuffer* buffer) = 0;
// Finish setting up input or output buffer(s).
// Consider doing as much as feasible in CoreCodecAddBuffer() instead, to be
// _slightly_ nicer to shared_fidl_thread().
// TODO(dustingreen): Assuming a well-behaved client but time-consuming call
// to this method, potentially another Codec instance could be disrupted due
// to sharing the shared_fidl_thread(). If we see an example of that
// happening, we could switch to not sharing any FIDL threads across Codec
// instances.
virtual void CoreCodecConfigureBuffers(
CodecPort port, const std::vector<std::unique_ptr<CodecPacket>>& packets) = 0;
// This method can be called at any time while output buffers are (fully)
// configured, including while there's no active stream.
// This will also be called on each of the output packets shortly after
// CoreCodecConfigureBuffers() is called. This is implicit in the Codec
// interface, but explicit (via calls to this method) in the CodecAdapter
// interface.
virtual void CoreCodecRecycleOutputPacket(CodecPacket* packet) = 0;
// De-configure input or output buffers. This will never occur at a time
// when the core codec is expected to be processing data. For input, this
// can only be called while there's no active stream. For output, this can
// be called while there's no active stream, or after a stream is started but
// before any input data is queued, or during processing shortly after the
// core codec calling
// onCoreCodecMidStreamOutputConstraintsChange(true), after
// CoreCodecMidStreamOutputBufferReConfigPrepare() and before
// CoreCodecMidStreamOutputBufferReConfigFinish().
// The "ensure" part of the name is because this needs to ensure that buffers
// are fully de-configured, regardless of whether buffers are presently fully
// de-configured already, or if CoreCodecAddBuffer() has been called 1-N times
// but CoreCodecConfigureBuffers() hasn't been called yet (and won't be, if
// this method is called instead), or if CoreCodecAddBuffer() has been called
// N times and CoreCodecConfigureBuffers() has also been called.
virtual void CoreCodecEnsureBuffersNotConfigured(CodecPort port) = 0;
// The core codec may need to specify what input constraints to be used.
// This is called on the StreamControl ordering domain after CoreCodecInit and
// will not be called again after that as input constraints are static. Unlike
// most CodecAdapter functions, the CodecAdapter provides a default
// implementation that will work for most codecs. A codec-specific
// CodecAdapter may override this if it has different constraints.
virtual std::unique_ptr<const fuchsia::media::StreamBufferConstraints>
// The core codec needs to specify what output config is needed.
// output_re_config_required true:
// This is called on StreamControl ordering domain - this can happen very soon
// if CoreCodecStopStream() hasn't happened yet, or can happen much later when
// the next stream is starting. Or may not happen at all if CodecImpl fails
// due to channel closure or any other reason.
// output_re_config_required false:
// This is called on the same thread and same stack as
// onCoreCodecMidStreamOutputConstraintsChange() (and with same stream still
// active).
virtual std::unique_ptr<const fuchsia::media::StreamOutputConstraints>
CoreCodecBuildNewOutputConstraints(uint64_t stream_lifetime_ordinal,
uint64_t new_output_buffer_constraints_version_ordinal,
bool buffer_constraints_action_required) = 0;
// This will be called on the InputData domain, during the core codec's call
// to onCoreCodecOutputPacket(), so that the format will be delivered at most
// once before any packet which needs a new format to be indicated. The core
// codec can trigger this to occur during the next onCoreCodecOutputPacket()
// by calling onCoreCodecOutputFormatChange(). The tracking of pending
// output format is per-stream, and all streams start with a pending output
// format, so a core codec need not call onCoreCodecOutputFormatChange()
// unless the format change is mid-stream (but calling before the first packet
// is allowed and not harmful).
virtual fuchsia::media::StreamOutputFormat CoreCodecGetOutputFormat(
uint64_t stream_lifetime_ordinal, uint64_t new_output_format_details_version_ordinal) = 0;
// CoreCodecMidStreamOutputBufferReConfigPrepare()
// For a mid-stream format change where output buffer re-configuration is
// needed (as initiated async by the core codec calling
// CodecAdapterEvents::onCoreCodecMidStreamOutputConstraintsChange(true)),
// this method is called on the StreamControl thread before the client is
// notified of the need for output buffer re-config (via OnOutputConstraints()
// with buffer_constraints_action_required true).
// The core codec should do whatever is necessary to ensure that output
// buffers are done de-configuring to the extent feasible by the time this
// method returns. See next paragraph for the only cases where retaining old
// low-level buffers _might_ be justified (but for the most part, those
// reasons aren't really pragmatic reasons to be retaining old low-level
// buffers, at least for now). If a core codec keeps old low-level buffer
// handles/references around for a while to be more seamless (entirely
// optional and not recommended per next paragraph), the core codec must drop
// those handles/references as soon as they're no longer needed in trying to
// achieve more seamless-ness.
// A core codec need only support seamless resolution/format changes if the
// output buffers (considering separately, width, height, and any other
// similar parameter like color depth) are already large enough for both the
// before format and after format. If this is not the case, a codec is
// permitted, but not encouraged, to discard some output frames. A codec is
// also permitted to achieve a more seamless format switch despite output
// buffer re-config by retaining references to old-format low-level buffers,
// copying into temporary buffers and back out, or similar. However, core
// codec implementers should note that the process of re-configuring output
// buffers is not likely to be super-quick, and other parts of the system may
// not go to so much effort to achieve seamless-ness across an output buffer
// re-config, so ... it's probably best not to spend time trying to achieve
// seamless-ness for a situation which for other reasons might end up being
// non-seamless at least in terms of timing consistency in any case.
// As always, calls to CodecAdapterEvents must not be made while holding
// lock_.
virtual void CoreCodecMidStreamOutputBufferReConfigPrepare() = 0;
// This method is called when the mid-stream output buffer re-configuration
// has completed. This is called after all the calls to CoreCodecAddBuffer()
// and the call to CoreCodecConfigureBuffers() are done.
// The core codec should do whatever is necessary to get back into normal
// steady-state operation in this method.
// The core codec must not onCoreCodecOutputPacket() or
// onCoreCodecOutputEndOfStream() until this method has been called, or until
// CoreCodecStartStream() is called and some input is available, should the
// current stream be stopped before completing mid-stream output buffer
// re-config. This works partly because the CodecImpl guarantees that if a
// mid-stream re-config didn't finish, there will be a complete output
// re-config before the CoreCodecStartStream() - in other words this re-config
// is abandoned and a new one takes its place and is fully complete prior to
// the new stream starting.
virtual void CoreCodecMidStreamOutputBufferReConfigFinish() = 0;
// See comment on the constructor re. sharing this lock with the caller of
// CodecAdapter methods, at least for now.
std::mutex& lock_;
CodecAdapterEvents* events_ = nullptr;
// For now all the sub-classes queue input here, so may as well be in the base
// class for now.
std::list<CodecInputItem> input_queue_;
// A core codec will also want to track free output packets, but how best to
// do that is sub-class-specific.
// It's useful to have a source of random numbers that's compatible with std::
// for purposes such as scrambling the order of free packets. These are
// instance-specific only because of thread-safety considerations, not because
// of generated sequence considerations.
std::random_device random_device_;
std::mt19937 not_for_security_prng_;
CodecAdapter() = delete;