| // Copyright 2018 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| #include "codec_adapter_ffmpeg_decoder.h" | |
| #include <limits> | |
| extern "C" { | |
| #include "libavutil/imgutils.h" | |
| } | |
| #include <lib/async/cpp/task.h> | |
| #include <lib/fit/defer.h> | |
| #include <lib/media/codec_impl/codec_buffer.h> | |
| namespace { | |
| AVPixelFormat Images2PixelFormatToAvPixelFormat(fuchsia::images2::PixelFormat pixel_format) { | |
| switch (pixel_format) { | |
| case fuchsia::images2::PixelFormat::YV12: | |
| return AV_PIX_FMT_YUV420P; | |
| default: | |
| return AV_PIX_FMT_NONE; | |
| } | |
| } | |
| } // namespace | |
| namespace { | |
| // A client using the min shouldn't necessarily expect performance to be | |
| // acceptable when running higher bit-rates. | |
| constexpr uint32_t kInputPerPacketBufferBytesMin = 8 * 1024; | |
| // This is an arbitrary cap for now. | |
| constexpr uint32_t kInputPerPacketBufferBytesMax = 4 * 1024 * 1024; | |
| // Arbitrary limit; specific value is historical. | |
| static constexpr uint32_t kMaxOutputBufferCount = 34; | |
| // Arbitrary limit. | |
| static constexpr uint32_t kMaxInputBufferCount = 256; | |
| // FFMPEG requires at least this many padding bytes after each output frame, to allow for optimized | |
| // code to do less bounds checking (an understandable tradeoff). Without this, FFMPEG can sometimes | |
| // read (and maybe sometimes write though I haven't personally observed that) beyond the end of the | |
| // output buffer which can crash the isolate process. | |
| static constexpr uint32_t kFfmpegOutputFramePaddingBytes = 16; | |
| } // namespace | |
| CodecAdapterFfmpegDecoder::CodecAdapterFfmpegDecoder(std::mutex& lock, | |
| CodecAdapterEvents* codec_adapter_events) | |
| : CodecAdapterSW(lock, codec_adapter_events) {} | |
| CodecAdapterFfmpegDecoder::~CodecAdapterFfmpegDecoder() = default; | |
| void CodecAdapterFfmpegDecoder::ProcessInputLoop() { | |
| std::optional<CodecInputItem> maybe_input_item; | |
| while ((maybe_input_item = input_queue_.WaitForElement())) { | |
| CodecInputItem input_item = std::move(maybe_input_item.value()); | |
| if (input_item.is_format_details()) { | |
| if (avcodec_context_) { | |
| events_->onCoreCodecFailCodec("Midstream input format change is not supported."); | |
| return; | |
| } | |
| auto maybe_avcodec_context = AvCodecContext::CreateDecoder( | |
| input_item.format_details(), | |
| [this](const AvCodecContext::FrameBufferRequest& frame_buffer_request, AVFrame* frame, | |
| int flags) { return GetBuffer(frame_buffer_request, frame, flags); }); | |
| if (!maybe_avcodec_context) { | |
| events_->onCoreCodecFailCodec("Failed to create ffmpeg decoder."); | |
| return; | |
| } | |
| avcodec_context_ = std::move(maybe_avcodec_context.value()); | |
| } else if (input_item.is_end_of_stream()) { | |
| if (!avcodec_context_) { | |
| events_->onCoreCodecOutputEndOfStream(/*error_detected_before=*/false); | |
| continue; | |
| } | |
| avcodec_context_->EndStream(); | |
| DecodeFrames(); | |
| } else if (input_item.is_packet()) { | |
| ZX_DEBUG_ASSERT(avcodec_context_); | |
| auto return_packet = fit::defer( | |
| [this, &input_item] { events_->onCoreCodecInputPacketDone(input_item.packet()); }); | |
| const int result = avcodec_context_->SendPacket(input_item.packet()); | |
| if (result < 0) { | |
| if (IsStreamActive()) { | |
| events_->onCoreCodecFailCodec("Failed to decode input packet with ffmpeg error: %s", | |
| av_err2str(result)); | |
| } | |
| return; | |
| } | |
| return_packet.call(); | |
| DecodeFrames(); | |
| } | |
| } | |
| } | |
| void CodecAdapterFfmpegDecoder::CleanUpAfterStream() { avcodec_context_ = nullptr; } | |
| std::pair<fuchsia::media::FormatDetails, size_t> CodecAdapterFfmpegDecoder::OutputFormatDetails() { | |
| std::lock_guard<std::mutex> lock(lock_); | |
| ZX_ASSERT(decoded_output_info_.has_value()); | |
| const auto& [image_format, per_packet_buffer_bytes] = decoded_output_info_.value(); | |
| fuchsia::media::FormatDetails format_details; | |
| format_details.set_mime_type("video/raw"); | |
| format_details.set_image_format(fidl::Clone(image_format)); | |
| return {std::move(format_details), per_packet_buffer_bytes}; | |
| } | |
| void CodecAdapterFfmpegDecoder::FfmpegFreeBufferCallback(void* ctx, uint8_t* base) { | |
| auto* self = reinterpret_cast<CodecAdapterFfmpegDecoder*>(ctx); | |
| self->output_buffer_pool_.FreeBuffer(base); | |
| } | |
| int CodecAdapterFfmpegDecoder::GetBuffer( | |
| const AvCodecContext::FrameBufferRequest& decoded_output_info, AVFrame* frame, int flags) { | |
| bool should_config_output = false; | |
| bool output_increased_in_size = false; | |
| bool need_new_buffers = false; | |
| { | |
| std::lock_guard<std::mutex> lock(lock_); | |
| if (!IsStreamActiveLocked()) { | |
| return AVERROR(ENOMEM); | |
| } | |
| need_new_buffers = !decoded_output_info_ || !output_buffer_pool_.has_buffers_configured(); | |
| if (need_new_buffers || | |
| !fidl::Equals((*decoded_output_info_).format, decoded_output_info.format)) { | |
| output_increased_in_size = | |
| !need_new_buffers && | |
| decoded_output_info.buffer_bytes_needed > (*decoded_output_info_).buffer_bytes_needed; | |
| if (!output_increased_in_size) { | |
| // When !need_new_buffers, keep the buffer_bytes_needed that the | |
| // currently configured output buffers were allocated for, so a later | |
| // switch back to dimensions that still fit in the configured buffers | |
| // does not fail the output_increased_in_size check above. | |
| decoded_output_info_ = { | |
| .format = fidl::Clone(decoded_output_info.format), | |
| .buffer_bytes_needed = need_new_buffers ? decoded_output_info.buffer_bytes_needed | |
| : (*decoded_output_info_).buffer_bytes_needed}; | |
| should_config_output = true; | |
| if (need_new_buffers) { | |
| SetOutputReconfigPendingLocked(); | |
| } | |
| } | |
| } | |
| } | |
| if (output_increased_in_size) { | |
| events_->onCoreCodecFailCodec( | |
| "Midstream output config change to larger format is not supported."); | |
| return AVERROR(ENOMEM); | |
| } | |
| if (should_config_output) { | |
| if (need_new_buffers) { | |
| events_->onCoreCodecMidStreamOutputConstraintsChange( | |
| /*output_re_config_required=*/true); | |
| const bool is_active = WaitForOutputReconfigToComplete(); | |
| if (!is_active) { | |
| return AVERROR(ENOMEM); | |
| } | |
| } else { | |
| events_->onCoreCodecOutputFormatChange(); | |
| } | |
| } | |
| auto buffer = output_buffer_pool_.AllocateBuffer(decoded_output_info.buffer_bytes_needed); | |
| if (!buffer) { | |
| return AVERROR(ENOMEM); | |
| } | |
| ZX_ASSERT(decoded_output_info.buffer_bytes_needed <= buffer->size()); | |
| ZX_ASSERT(decoded_output_info.buffer_bytes_needed + kFfmpegOutputFramePaddingBytes <= | |
| buffer->raw_vmo_size() - buffer->vmo_offset()); | |
| AVPixelFormat pix_fmt = | |
| Images2PixelFormatToAvPixelFormat(decoded_output_info.format.pixel_format()); | |
| if (pix_fmt == AV_PIX_FMT_NONE) { | |
| events_->onCoreCodecFailCodec("Unsupported format: %d", pix_fmt); | |
| return -1; | |
| } | |
| ZX_ASSERT(frame->format == pix_fmt); | |
| AVBufferRef* buffer_ref = av_buffer_create(buffer->base(), static_cast<int>(buffer->size()), | |
| FfmpegFreeBufferCallback, this, flags); | |
| int fill_arrays_status = av_image_fill_arrays(frame->data, frame->linesize, buffer_ref->data, | |
| pix_fmt, decoded_output_info.format.size().width, | |
| decoded_output_info.format.size().height, 1); | |
| if (fill_arrays_status < 0) { | |
| events_->onCoreCodecFailCodec("Ffmpeg fill arrays failed: %d", fill_arrays_status); | |
| return -1; | |
| } | |
| // IYUV is not YV12. Ffmpeg only decodes into IYUV. The difference between | |
| // YV12 and IYUV is the order of the U and V planes. Here we trick Ffmpeg | |
| // into writing them in YV12 order relative to one another. | |
| std::swap(frame->data[1], frame->data[2]); | |
| frame->buf[0] = buffer_ref; | |
| // ffmpeg says to set extended_data to data if we're not using extended_data | |
| frame->extended_data = frame->data; | |
| return 0; | |
| } | |
| void CodecAdapterFfmpegDecoder::DecodeFrames() { | |
| ZX_DEBUG_ASSERT(thrd_current() == input_processing_thread_); | |
| ZX_DEBUG_ASSERT(avcodec_context_); | |
| while (true) { | |
| auto [error, frame] = avcodec_context_->ReceiveFrame(); | |
| if (error == AVERROR(EAGAIN)) { | |
| return; | |
| } else if (error == AVERROR_EOF) { | |
| events_->onCoreCodecOutputEndOfStream(/*error_detected_before=*/false); | |
| return; | |
| } else if (error < 0) { | |
| if (IsStreamActive()) { | |
| events_->onCoreCodecFailCodec("DecodeFrames(): Failed to decode frame: %s", | |
| av_err2str(error)); | |
| } | |
| return; | |
| } | |
| std::optional<CodecPacket*> maybe_output_packet = free_output_packets_.WaitForElement(); | |
| if (!maybe_output_packet) { | |
| return; | |
| } | |
| auto output_packet = *maybe_output_packet; | |
| auto release_buffer = | |
| fit::defer([this, &output_packet]() { free_output_packets_.Push(output_packet); }); | |
| auto buffer_alloc = output_buffer_pool_.FindBufferByBase(frame->data[0]); | |
| ZX_ASSERT(buffer_alloc); | |
| if (buffer_alloc->bytes_used > std::numeric_limits<uint32_t>::max()) { | |
| events_->onCoreCodecFailCodec("Could not represent bytes_used as uint32_t"); | |
| return; | |
| } | |
| output_packet->SetBuffer(buffer_alloc->buffer); | |
| output_packet->SetStartOffset(0); | |
| output_packet->SetValidLengthBytes(static_cast<uint32_t>(buffer_alloc->bytes_used)); | |
| output_packet->SetTimstampIsh(frame->pts); | |
| { | |
| std::lock_guard<std::mutex> lock(lock_); | |
| ZX_DEBUG_ASSERT(in_use_by_client_.find(output_packet) == in_use_by_client_.end()); | |
| in_use_by_client_.emplace(output_packet, std::move(frame)); | |
| } | |
| release_buffer.cancel(); | |
| events_->onCoreCodecOutputPacket(output_packet, | |
| /*error_detected_before=*/false, | |
| /*error_detected_during=*/false); | |
| } | |
| } | |
| fuchsia_sysmem2::BufferCollectionConstraints | |
| CodecAdapterFfmpegDecoder::CoreCodecGetBufferCollectionConstraints2( | |
| CodecPort port, const fuchsia::media::StreamBufferConstraints& stream_buffer_constraints, | |
| const fuchsia::media::StreamBufferPartialSettings& partial_settings) { | |
| std::lock_guard<std::mutex> lock(lock_); | |
| fuchsia_sysmem2::BufferCollectionConstraints result; | |
| // The CodecImpl won't hand us the sysmem token, so we shouldn't expect to | |
| // have the token here. | |
| ZX_DEBUG_ASSERT(!partial_settings.has_sysmem_token()); | |
| // TODO(https://fxbug.dev/42084949): plumb/permit range of buffer count from further down, | |
| // instead of single number frame_count, and set this to the actual | |
| // stream-required # of reference frames + # that can concurrently decode. | |
| // Packets and buffers are not the same thing, and we should permit the # of | |
| // packets to be >= the # of buffers. We shouldn't be | |
| // allocating buffers on behalf of the client here, but until we plumb the | |
| // range of frame_count and are more flexible on # of allocated buffers, we | |
| // have to make sure there are at least as many buffers as packets. We | |
| // categorize the buffers as for camping and for slack. This should change to | |
| // be just the buffers needed for camping and maybe 1 for shared slack. If | |
| // the client wants more buffers the client can demand buffers in its own | |
| // fuchsia::sysmem::BufferCollection::SetConstraints(). | |
| if (port == kOutputPort) { | |
| result.min_buffer_count_for_camping() = kMinOutputBufferCountForCamping; | |
| } else { | |
| result.min_buffer_count_for_camping() = kMinInputBufferCountForCamping; | |
| } | |
| ZX_DEBUG_ASSERT(!result.min_buffer_count_for_dedicated_slack().has_value()); | |
| ZX_DEBUG_ASSERT(!result.min_buffer_count_for_shared_slack().has_value()); | |
| if (port == kOutputPort) { | |
| result.max_buffer_count() = kMaxOutputBufferCount; | |
| } else { | |
| result.max_buffer_count() = kMaxInputBufferCount; | |
| } | |
| uint32_t per_packet_buffer_bytes_min; | |
| uint32_t per_packet_buffer_bytes_max; | |
| if (port == kInputPort) { | |
| per_packet_buffer_bytes_min = kInputPerPacketBufferBytesMin; | |
| per_packet_buffer_bytes_max = kInputPerPacketBufferBytesMax; | |
| } else { | |
| ZX_ASSERT(decoded_output_info_.has_value()); | |
| const auto& image_format = decoded_output_info_->format; | |
| ZX_DEBUG_ASSERT(port == kOutputPort); | |
| // YV12, based on min stride. | |
| per_packet_buffer_bytes_min = image_format.bytes_per_row() * image_format.size().height * 3 / 2; | |
| // At least for now, don't cap the per-packet buffer size for output. The | |
| // HW only cares about the portion we set up for output anyway, and the | |
| // client has no way to force output to occur into portions of the output | |
| // buffer beyond what's implied by the max supported image dimensions. | |
| per_packet_buffer_bytes_max = 0xFFFFFFFF; | |
| } | |
| auto& bmc = result.buffer_memory_constraints().emplace(); | |
| bmc.min_size_bytes() = per_packet_buffer_bytes_min; | |
| bmc.max_size_bytes() = per_packet_buffer_bytes_max; | |
| // These are all false because SW decode. | |
| bmc.physically_contiguous_required() = false; | |
| bmc.secure_required() = false; | |
| if (port == kOutputPort) { | |
| ZX_ASSERT(decoded_output_info_.has_value()); | |
| const auto& image_format = decoded_output_info_->format; | |
| auto& image_constraints = result.image_format_constraints().emplace().emplace_back(); | |
| image_constraints.pixel_format() = fuchsia_images2::PixelFormat::kYv12; | |
| // TODO(https://fxbug.dev/42084950): confirm that REC709 is always what we want here, or plumb | |
| // actual YUV color space if it can ever be REC601_*. Since 2020 and 2100 | |
| // are minimum 10 bits per Y sample and we're outputting YV12, 601 is the | |
| // only other potential possibility here. | |
| image_constraints.color_spaces() = {fuchsia_images2::ColorSpace::kRec709}; | |
| // The non-"required_" fields indicate the decoder's ability to potentially | |
| // output frames at various dimensions as coded in the stream. Aside from | |
| // the current stream being somewhere in these bounds, these have nothing to | |
| // do with the current stream in particular. | |
| image_constraints.min_size() = {16, 16}; | |
| // This intentionally isn't the height of a 4k frame. See | |
| // max_coded_width_times_coded_height. We intentionally constrain the max | |
| // dimension in width or height to the width of a 4k frame. While the HW | |
| // might be able to go bigger than that as long as the other dimension is | |
| // smaller to compensate, we don't really need to enable any larger than | |
| // 4k's width in either dimension, so we don't. | |
| image_constraints.max_size() = {3840, 3840}; | |
| image_constraints.min_bytes_per_row() = 16; | |
| // no hard-coded max stride, at least for now | |
| ZX_DEBUG_ASSERT(!image_constraints.max_bytes_per_row().has_value()); | |
| image_constraints.max_width_times_height() = 3840 * 2160; | |
| image_constraints.size_alignment() = {16, 16}; | |
| image_constraints.bytes_per_row_divisor() = 16; | |
| image_constraints.pad_beyond_image_size_bytes() = kFfmpegOutputFramePaddingBytes; | |
| // TODO(dustingreen): Since this is a producer that will always produce at | |
| // offset 0 of a physical page, we don't really care if this field is | |
| // consistent with any constraints re. what the HW can do. | |
| image_constraints.start_offset_divisor() = 1; | |
| // Odd display dimensions are permitted, but these don't imply odd YV12 | |
| // dimensions - those are constrainted by coded_width_divisor and | |
| // coded_height_divisor which are both 16. | |
| image_constraints.display_rect_alignment() = {1, 1}; | |
| // The decoder is producing frames and the decoder has no choice but to | |
| // produce frames at their coded size. The decoder wants to potentially be | |
| // able to support a stream with dynamic resolution, potentially including | |
| // dimensions both less than and greater than the dimensions that led to the | |
| // current need to allocate a BufferCollection. For this reason, the | |
| // required_ fields are set to the exact current dimensions, and the | |
| // permitted (non-required_) fields is set to the full potential range that | |
| // the decoder could potentially output. If an initiator wants to require a | |
| // larger range of dimensions that includes the required range indicated | |
| // here (via a-priori knowledge of the potential stream dimensions), an | |
| // initiator is free to do so. | |
| image_constraints.required_min_size() = {image_format.size().width, image_format.size().height}; | |
| image_constraints.required_max_size() = {image_format.size().width, image_format.size().height}; | |
| // Sysmem2 doesn't have required_min_bytes_per_row or | |
| // required_max_bytes_per_row (at least for now). If those later prove to be | |
| // worth adding to sysmem2, we'd set those to | |
| // image_format.bytes_per_row() here. | |
| // | |
| // The way we'd know is CoreCodecSetBufferCollectionInfo would fail the | |
| // codec when bytes_per_row() isn't within | |
| // [min_bytes_per_row..max_bytes_per_row]. | |
| // | |
| // The decoder is the producer. We're implicitly relying on the consumer(s) | |
| // to not overly constrain the bytes_per_row using min_bytes_per_row and | |
| // max_bytes_per_row, in order for decode of the current stream dimensions | |
| // to be possible. | |
| } else { | |
| ZX_DEBUG_ASSERT(!result.image_format_constraints().has_value()); | |
| } | |
| // We don't have to fill out usage - CodecImpl takes care of that. | |
| ZX_DEBUG_ASSERT(!result.usage().has_value()); | |
| return result; | |
| } | |
| void CodecAdapterFfmpegDecoder::CoreCodecSetBufferCollectionInfo( | |
| CodecPort port, const fuchsia_sysmem2::BufferCollectionInfo& buffer_collection_info) { | |
| std::optional<uint32_t> bytes_per_row; | |
| uint32_t min_bytes_per_row; | |
| uint32_t max_bytes_per_row; | |
| { // scope lock | |
| std::lock_guard<std::mutex> lock(lock_); | |
| if (port == kInputPort) { | |
| ZX_DEBUG_ASSERT(buffer_collection_info.buffers()->size() >= kMinInputBufferCountForCamping); | |
| } else { | |
| ZX_DEBUG_ASSERT(buffer_collection_info.buffers()->size() >= kMinOutputBufferCountForCamping); | |
| } | |
| if (decoded_output_info_.has_value()) { | |
| const auto& image_format = decoded_output_info_->format; | |
| bytes_per_row = image_format.bytes_per_row(); | |
| } | |
| } | |
| if (bytes_per_row.has_value()) { | |
| auto& ifc = *buffer_collection_info.settings()->image_format_constraints(); | |
| min_bytes_per_row = *ifc.min_bytes_per_row(); | |
| max_bytes_per_row = *ifc.max_bytes_per_row(); | |
| if (bytes_per_row < min_bytes_per_row) { | |
| events_->onCoreCodecFailCodec("bytes_per_row < *ifc.min_bytes_per_row()"); | |
| return; | |
| } | |
| if (bytes_per_row > max_bytes_per_row) { | |
| events_->onCoreCodecFailCodec("bytes_per_row > *ifc.max_bytes_per_row()"); | |
| return; | |
| } | |
| } | |
| } |