blob: 22b8657de763f2501ba5bdb78bbab39ac2873efc [file] [log] [blame]
* Copyright (C) 2010 The Android Open Source Project
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* See the License for the specific language governing permissions and
* limitations under the License.
#ifndef A_CODEC_H_
#define A_CODEC_H_
#include <stdint.h>
#include <android/native_window.h>
#include <media/hardware/MetadataBufferType.h>
#include <media/IOMX.h>
#include <media/stagefright/foundation/AHierarchicalStateMachine.h>
#include <media/stagefright/CodecBase.h>
#include <media/stagefright/FrameRenderTracker.h>
#include <media/stagefright/MediaDefs.h>
#include <media/stagefright/SkipCutBuffer.h>
#include <utils/NativeHandle.h>
#include <OMX_Audio.h>
#include <hardware/gralloc.h>
namespace android {
struct ABuffer;
class ACodecBufferChannel;
class MediaCodecBuffer;
class MemoryDealer;
struct DescribeColorFormat2Params;
struct DataConverter;
// Treble shared memory
namespace hidl {
namespace allocator {
namespace V1_0 {
struct IAllocator;
} // V1_0
} // allocator
namespace memory {
namespace V1_0 {
struct IMemory;
} // V1_0
} // memory
} // hidl
typedef hidl::allocator::V1_0::IAllocator TAllocator;
typedef hidl::memory::V1_0::IMemory TMemory;
struct ACodec : public AHierarchicalStateMachine, public CodecBase {
void initiateSetup(const sp<AMessage> &msg);
virtual std::shared_ptr<BufferChannelBase> getBufferChannel() override;
virtual void initiateAllocateComponent(const sp<AMessage> &msg);
virtual void initiateConfigureComponent(const sp<AMessage> &msg);
virtual void initiateCreateInputSurface();
virtual void initiateSetInputSurface(const sp<PersistentSurface> &surface);
virtual void initiateStart();
virtual void initiateShutdown(bool keepComponentAllocated = false);
virtual status_t queryCapabilities(
const AString &name, const AString &mime, bool isEncoder,
sp<MediaCodecInfo::Capabilities> *caps);
virtual status_t setSurface(const sp<Surface> &surface);
virtual void signalFlush();
virtual void signalResume();
virtual void signalSetParameters(const sp<AMessage> &msg);
virtual void signalEndOfInputStream();
virtual void signalRequestIDRFrame();
// AHierarchicalStateMachine implements the message handling
virtual void onMessageReceived(const sp<AMessage> &msg) {
// Returns 0 if configuration is not supported. NOTE: this is treated by
// some OMX components as auto level, and by others as invalid level.
static int /* OMX_VIDEO_AVCLEVELTYPE */ getAVCLevelFor(
int width, int height, int rate, int bitrate,
// Quirk still supported, even though deprecated
enum Quirks {
kRequiresAllocateBufferOnInputPorts = 1,
kRequiresAllocateBufferOnOutputPorts = 2,
static status_t getOMXChannelMapping(size_t numChannels, OMX_AUDIO_CHANNELTYPE map[]);
// Save the flag.
void setTrebleFlag(bool trebleFlag);
// Return the saved flag.
bool getTrebleFlag() const;
virtual ~ACodec();
struct BaseState;
struct UninitializedState;
struct LoadedState;
struct LoadedToIdleState;
struct IdleToExecutingState;
struct ExecutingState;
struct OutputPortSettingsChangedState;
struct ExecutingToIdleState;
struct IdleToLoadedState;
struct FlushingState;
struct DeathNotifier;
enum {
kWhatSetup = 'setu',
kWhatOMXMessage = 'omx ',
// same as kWhatOMXMessage - but only used with
// handleMessage during OMX message-list handling
kWhatOMXMessageItem = 'omxI',
kWhatOMXMessageList = 'omxL',
kWhatInputBufferFilled = 'inpF',
kWhatOutputBufferDrained = 'outD',
kWhatShutdown = 'shut',
kWhatFlush = 'flus',
kWhatResume = 'resm',
kWhatDrainDeferredMessages = 'drai',
kWhatAllocateComponent = 'allo',
kWhatConfigureComponent = 'conf',
kWhatSetSurface = 'setS',
kWhatCreateInputSurface = 'cisf',
kWhatSetInputSurface = 'sisf',
kWhatSignalEndOfInputStream = 'eois',
kWhatStart = 'star',
kWhatRequestIDRFrame = 'ridr',
kWhatSetParameters = 'setP',
kWhatSubmitOutputMetadataBufferIfEOS = 'subm',
kWhatOMXDied = 'OMXd',
kWhatReleaseCodecInstance = 'relC',
kWhatForceStateTransition = 'fstt',
enum {
kPortIndexInput = 0,
kPortIndexOutput = 1
enum {
kFlagIsSecure = 1,
kFlagPushBlankBuffersToNativeWindowOnShutdown = 2,
kFlagIsGrallocUsageProtected = 4,
enum {
struct BufferInfo {
enum Status {
UNRECOGNIZED, // not a tracked buffer
static inline Status getSafeStatus(BufferInfo *info) {
return info == NULL ? UNRECOGNIZED : info->mStatus;
IOMX::buffer_id mBufferID;
Status mStatus;
unsigned mDequeuedAt;
sp<MediaCodecBuffer> mData; // the client's buffer; if not using data conversion, this is
// the codec buffer; otherwise, it is allocated separately
sp<RefBase> mMemRef; // and a reference to the IMemory, so it does not go away
sp<MediaCodecBuffer> mCodecData; // the codec's buffer
sp<RefBase> mCodecRef; // and a reference to the IMemory
sp<GraphicBuffer> mGraphicBuffer;
bool mNewGraphicBuffer;
int mFenceFd;
FrameRenderTracker::Info *mRenderInfo;
// The following field and 4 methods are used for debugging only
bool mIsReadFence;
// Store |fenceFd| and set read/write flag. Log error, if there is already a fence stored.
void setReadFence(int fenceFd, const char *dbg);
void setWriteFence(int fenceFd, const char *dbg);
// Log error, if the current fence is not a read/write fence.
void checkReadFence(const char *dbg);
void checkWriteFence(const char *dbg);
static const char *_asString(BufferInfo::Status s);
void dumpBuffers(OMX_U32 portIndex);
// If |fd| is non-negative, waits for fence with |fd| and logs an error if it fails. Returns
// the error code or OK on success. If |fd| is negative, it returns OK
status_t waitForFence(int fd, const char *dbg);
struct BufferStats {
int64_t mEmptyBufferTimeUs;
int64_t mFillBufferDoneTimeUs;
KeyedVector<int64_t, BufferStats> mBufferStats;
sp<UninitializedState> mUninitializedState;
sp<LoadedState> mLoadedState;
sp<LoadedToIdleState> mLoadedToIdleState;
sp<IdleToExecutingState> mIdleToExecutingState;
sp<ExecutingState> mExecutingState;
sp<OutputPortSettingsChangedState> mOutputPortSettingsChangedState;
sp<ExecutingToIdleState> mExecutingToIdleState;
sp<IdleToLoadedState> mIdleToLoadedState;
sp<FlushingState> mFlushingState;
sp<SkipCutBuffer> mSkipCutBuffer;
int32_t mSampleRate;
AString mComponentName;
uint32_t mFlags;
sp<IOMX> mOMX;
sp<IOMXNode> mOMXNode;
int32_t mNodeGeneration;
bool mTrebleFlag;
sp<TAllocator> mAllocator[2];
sp<MemoryDealer> mDealer[2];
bool mUsingNativeWindow;
sp<ANativeWindow> mNativeWindow;
int mNativeWindowUsageBits;
android_native_rect_t mLastNativeWindowCrop;
int32_t mLastNativeWindowDataSpace;
sp<AMessage> mConfigFormat;
sp<AMessage> mInputFormat;
sp<AMessage> mOutputFormat;
// Initial output format + configuration params that is reused as the base for all subsequent
// format updates. This will equal to mOutputFormat until the first actual frame is received.
sp<AMessage> mBaseOutputFormat;
FrameRenderTracker mRenderTracker; // render information for buffers rendered by ACodec
Vector<BufferInfo> mBuffers[2];
bool mPortEOS[2];
status_t mInputEOSResult;
List<sp<AMessage> > mDeferredQueue;
sp<AMessage> mLastOutputFormat;
bool mIsVideo;
bool mIsEncoder;
bool mFatalError;
bool mShutdownInProgress;
bool mExplicitShutdown;
bool mIsLegacyVP9Decoder;
// If "mKeepComponentAllocated" we only transition back to Loaded state
// and do not release the component instance.
bool mKeepComponentAllocated;
int32_t mEncoderDelay;
int32_t mEncoderPadding;
int32_t mRotationDegrees;
bool mChannelMaskPresent;
int32_t mChannelMask;
unsigned mDequeueCounter;
IOMX::PortMode mPortMode[2];
int32_t mMetadataBuffersToSubmit;
size_t mNumUndequeuedBuffers;
sp<DataConverter> mConverter[2];
sp<IGraphicBufferSource> mGraphicBufferSource;
int64_t mRepeatFrameDelayUs;
int64_t mMaxPtsGapUs;
float mMaxFps;
double mFps;
double mCaptureFps;
bool mCreateInputBuffersSuspended;
uint32_t mLatency;
bool mTunneled;
OMX_INDEXTYPE mDescribeColorAspectsIndex;
OMX_INDEXTYPE mDescribeHDRStaticInfoIndex;
std::shared_ptr<ACodecBufferChannel> mBufferChannel;
int32_t mStateGeneration;
enum {
} mVendorExtensionsStatus;
status_t setCyclicIntraMacroblockRefresh(const sp<AMessage> &msg, int32_t mode);
status_t allocateBuffersOnPort(OMX_U32 portIndex);
status_t freeBuffersOnPort(OMX_U32 portIndex);
status_t freeBuffer(OMX_U32 portIndex, size_t i);
status_t handleSetSurface(const sp<Surface> &surface);
status_t setPortMode(int32_t portIndex, IOMX::PortMode mode);
status_t setupNativeWindowSizeFormatAndUsage(
ANativeWindow *nativeWindow /* nonnull */, int *finalUsage /* nonnull */,
bool reconnect);
status_t configureOutputBuffersFromNativeWindow(
OMX_U32 *nBufferCount, OMX_U32 *nBufferSize,
OMX_U32 *nMinUndequeuedBuffers, bool preregister);
status_t allocateOutputMetadataBuffers();
status_t submitOutputMetadataBuffer();
void signalSubmitOutputMetadataBufferIfEOS_workaround();
status_t allocateOutputBuffersFromNativeWindow();
status_t cancelBufferToNativeWindow(BufferInfo *info);
status_t freeOutputBuffersNotOwnedByComponent();
BufferInfo *dequeueBufferFromNativeWindow();
inline bool storingMetadataInDecodedBuffers() {
return (mPortMode[kPortIndexOutput] == IOMX::kPortModeDynamicANWBuffer) && !mIsEncoder;
inline bool usingSecureBufferOnEncoderOutput() {
return (mPortMode[kPortIndexOutput] == IOMX::kPortModePresetSecureBuffer) && mIsEncoder;
BufferInfo *findBufferByID(
uint32_t portIndex, IOMX::buffer_id bufferID,
ssize_t *index = NULL);
status_t fillBuffer(BufferInfo *info);
status_t setComponentRole(bool isEncoder, const char *mime);
status_t configureCodec(const char *mime, const sp<AMessage> &msg);
status_t configureTunneledVideoPlayback(int32_t audioHwSync,
const sp<ANativeWindow> &nativeWindow);
status_t setVideoPortFormatType(
OMX_U32 portIndex,
OMX_VIDEO_CODINGTYPE compressionFormat,
bool usingNativeBuffers = false);
status_t setSupportedOutputFormat(bool getLegacyFlexibleFormat);
status_t setupVideoDecoder(
const char *mime, const sp<AMessage> &msg, bool usingNativeBuffers, bool haveSwRenderer,
sp<AMessage> &outputformat);
status_t setupVideoEncoder(
const char *mime, const sp<AMessage> &msg,
sp<AMessage> &outputformat, sp<AMessage> &inputformat);
status_t setVideoFormatOnPort(
OMX_U32 portIndex,
int32_t width, int32_t height,
OMX_VIDEO_CODINGTYPE compressionFormat, float frameRate = -1.0);
// sets |portIndex| port buffer numbers to be |bufferNum|. NOTE: Component could reject
// this setting if the |bufferNum| is less than the minimum buffer num of the port.
status_t setPortBufferNum(OMX_U32 portIndex, int bufferNum);
// gets index or sets it to 0 on error. Returns error from codec.
status_t initDescribeColorAspectsIndex();
// sets |params|. If |readBack| is true, it re-gets them afterwards if set succeeded.
// returns the codec error.
status_t setCodecColorAspects(DescribeColorAspectsParams &params, bool readBack = false);
// gets |params|; returns the codec error. |param| should not change on error.
status_t getCodecColorAspects(DescribeColorAspectsParams &params);
// gets dataspace guidance from codec and platform. |params| should be set up with the color
// aspects to use. If |tryCodec| is true, the codec is queried first. If it succeeds, we
// return OK. Otherwise, we fall back to the platform guidance and return the codec error;
// though, we return OK if the codec failed with UNSUPPORTED, as codec guidance is optional.
status_t getDataSpace(
DescribeColorAspectsParams &params, android_dataspace *dataSpace /* nonnull */,
bool tryCodec);
// sets color aspects for the encoder for certain |width/height| based on |configFormat|, and
// set resulting color config into |outputFormat|. If |usingNativeWindow| is true, we use
// video defaults if config is unspecified. Returns error from the codec.
status_t setColorAspectsForVideoDecoder(
int32_t width, int32_t height, bool usingNativeWindow,
const sp<AMessage> &configFormat, sp<AMessage> &outputFormat);
// gets color aspects for the encoder for certain |width/height| based on |configFormat|, and
// set resulting color config into |outputFormat|. If |dataSpace| is non-null, it requests
// dataspace guidance from the codec and platform and sets it into |dataSpace|. Returns the
// error from the codec.
status_t getColorAspectsAndDataSpaceForVideoDecoder(
int32_t width, int32_t height, const sp<AMessage> &configFormat,
sp<AMessage> &outputFormat, android_dataspace *dataSpace);
// sets color aspects for the video encoder assuming bytebuffer mode for certain |configFormat|
// and sets resulting color config into |outputFormat|. For mediarecorder, also set dataspace
// into |inputFormat|. Returns the error from the codec.
status_t setColorAspectsForVideoEncoder(
const sp<AMessage> &configFormat,
sp<AMessage> &outputFormat, sp<AMessage> &inputFormat);
// sets color aspects for the video encoder in surface mode. This basically sets the default
// video values for unspecified aspects and sets the dataspace to use in the input format.
// Also sets the dataspace into |dataSpace|.
// Returns any codec errors during this configuration, except for optional steps.
status_t setInitialColorAspectsForVideoEncoderSurfaceAndGetDataSpace(
android_dataspace *dataSpace /* nonnull */);
// gets color aspects for the video encoder input port and sets them into the |format|.
// Returns any codec errors.
status_t getInputColorAspectsForVideoEncoder(sp<AMessage> &format);
// updates the encoder output format with |aspects| defaulting to |dataSpace| for
// unspecified values.
void onDataSpaceChanged(android_dataspace dataSpace, const ColorAspects &aspects);
// gets index or sets it to 0 on error. Returns error from codec.
status_t initDescribeHDRStaticInfoIndex();
// sets HDR static metadata for the video encoder/decoder based on |configFormat|, and
// sets resulting HDRStaticInfo config into |outputFormat|. Returns error from the codec.
status_t setHDRStaticInfoForVideoCodec(
OMX_U32 portIndex, const sp<AMessage> &configFormat, sp<AMessage> &outputFormat);
// sets |params|. Returns the codec error.
status_t setHDRStaticInfo(const DescribeHDRStaticInfoParams &params);
// gets |params|. Returns the codec error.
status_t getHDRStaticInfo(DescribeHDRStaticInfoParams &params);
// gets HDR static information for the video encoder/decoder port and sets them into |format|.
status_t getHDRStaticInfoForVideoCodec(OMX_U32 portIndex, sp<AMessage> &format);
typedef struct drcParams {
int32_t drcCut;
int32_t drcBoost;
int32_t heavyCompression;
int32_t targetRefLevel;
int32_t encodedTargetLevel;
} drcParams_t;
status_t setupAACCodec(
bool encoder,
int32_t numChannels, int32_t sampleRate, int32_t bitRate,
int32_t aacProfile, bool isADTS, int32_t sbrMode,
int32_t maxOutputChannelCount, const drcParams_t& drc,
int32_t pcmLimiterEnable);
status_t setupAC3Codec(bool encoder, int32_t numChannels, int32_t sampleRate);
status_t setupEAC3Codec(bool encoder, int32_t numChannels, int32_t sampleRate);
status_t selectAudioPortFormat(
OMX_U32 portIndex, OMX_AUDIO_CODINGTYPE desiredFormat);
status_t setupAMRCodec(bool encoder, bool isWAMR, int32_t bitRate);
status_t setupG711Codec(bool encoder, int32_t sampleRate, int32_t numChannels);
status_t setupFlacCodec(
bool encoder, int32_t numChannels, int32_t sampleRate, int32_t compressionLevel);
status_t setupRawAudioFormat(
OMX_U32 portIndex, int32_t sampleRate, int32_t numChannels,
AudioEncoding encoding = kAudioEncodingPcm16bit);
status_t setPriority(int32_t priority);
status_t setLatency(uint32_t latency);
status_t getLatency(uint32_t *latency);
status_t setOperatingRate(float rateFloat, bool isVideo);
status_t getIntraRefreshPeriod(uint32_t *intraRefreshPeriod);
status_t setIntraRefreshPeriod(uint32_t intraRefreshPeriod, bool inConfigure);
// Configures temporal layering based on |msg|. |inConfigure| shall be true iff this is called
// during configure() call. on success the configured layering is set in |outputFormat|. If
// |outputFormat| is mOutputFormat, it is copied to trigger an output format changed event.
status_t configureTemporalLayers(
const sp<AMessage> &msg, bool inConfigure, sp<AMessage> &outputFormat);
status_t setMinBufferSize(OMX_U32 portIndex, size_t size);
status_t setupMPEG4EncoderParameters(const sp<AMessage> &msg);
status_t setupH263EncoderParameters(const sp<AMessage> &msg);
status_t setupAVCEncoderParameters(const sp<AMessage> &msg);
status_t setupHEVCEncoderParameters(const sp<AMessage> &msg);
status_t setupVPXEncoderParameters(const sp<AMessage> &msg, sp<AMessage> &outputFormat);
status_t verifySupportForProfileAndLevel(int32_t profile, int32_t level);
status_t configureBitrate(
int32_t bitrate, OMX_VIDEO_CONTROLRATETYPE bitrateMode);
void configureEncoderLatency(const sp<AMessage> &msg);
status_t setupErrorCorrectionParameters();
// Returns true iff all buffers on the given port have status
bool allYourBuffersAreBelongToUs(OMX_U32 portIndex);
bool allYourBuffersAreBelongToUs();
void waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs();
size_t countBuffersOwnedByComponent(OMX_U32 portIndex) const;
size_t countBuffersOwnedByNativeWindow() const;
void deferMessage(const sp<AMessage> &msg);
void processDeferredMessages();
void onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
// called when we have dequeued a buffer |buf| from the native window to track render info.
// |fenceFd| is the dequeue fence, and |info| points to the buffer info where this buffer is
// stored.
void updateRenderInfoForDequeuedBuffer(
ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info);
// Checks to see if any frames have rendered up until |until|, and to notify client
// (MediaCodec) of rendered frames up-until the frame pointed to by |until| or the first
// unrendered frame. These frames are removed from the render queue.
// If |dropIncomplete| is true, unrendered frames up-until |until| will be dropped from the
// queue, allowing all rendered framed up till then to be notified of.
// (This will effectively clear the render queue up-until (and including) |until|.)
// If |until| is NULL, or is not in the rendered queue, this method will check all frames.
void notifyOfRenderedFrames(
bool dropIncomplete = false, FrameRenderTracker::Info *until = NULL);
// Pass |expectedFormat| to print a warning if the format differs from it.
// Using sp<> instead of const sp<>& because expectedFormat is likely the current mOutputFormat
// which will get updated inside.
void onOutputFormatChanged(sp<const AMessage> expectedFormat = NULL);
void addKeyFormatChangesToRenderBufferNotification(sp<AMessage> &notify);
void sendFormatChange();
status_t getPortFormat(OMX_U32 portIndex, sp<AMessage> &notify);
void signalError(
OMX_ERRORTYPE error = OMX_ErrorUndefined,
status_t internalError = UNKNOWN_ERROR);
status_t requestIDRFrame();
status_t setParameters(const sp<AMessage> &params);
// set vendor extension parameters specified in params that are supported by the codec
status_t setVendorParameters(const sp<AMessage> &params);
// get vendor extension parameters supported by the codec for a specific port and add it to
// |format|
status_t getVendorParameters(OMX_U32 portIndex, sp<AMessage> &format);
// Send EOS on input stream.
void onSignalEndOfInputStream();
// Force EXEC->IDLE->LOADED shutdown sequence if not stale.
void forceStateTransition(int generation);
} // namespace android
#endif // A_CODEC_H_