| // Copyright 2018 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| #include "use_video_decoder.h" | |
| #include <fuchsia/sysmem/cpp/fidl.h> | |
| #include <inttypes.h> | |
| #include <lib/async-loop/cpp/loop.h> | |
| #include <lib/async-loop/default.h> | |
| #include <lib/fdio/directory.h> | |
| #include <lib/fidl/cpp/clone.h> | |
| #include <lib/fit/defer.h> | |
| #include <lib/fit/function.h> | |
| #include <lib/media/test/codec_client.h> | |
| #include <lib/media/test/one_shot_event.h> | |
| #include <lib/stdcompat/span.h> | |
| #include <lib/syslog/cpp/macros.h> | |
| #include <lib/zx/time.h> | |
| #include <stdint.h> | |
| #include <string.h> | |
| #include <zircon/assert.h> | |
| #include <zircon/time.h> | |
| #include <algorithm> | |
| #include <cstring> | |
| #include <iterator> | |
| #include <optional> | |
| #include <thread> | |
| #include <vector> | |
| #include <fbl/algorithm.h> | |
| #include <src/media/lib/raw_video_writer/raw_video_writer.h> | |
| #include <tee-client-api/tee-client-types.h> | |
| #include "in_stream_peeker.h" | |
| #include "input_copier.h" | |
| #include "util.h" | |
| namespace { | |
| // Most cases secure output can't be read to be verified, but under some testing | |
| // circumstances it can be possible. | |
| constexpr bool kVerifySecureOutput = false; | |
| // Queue SPS and PPS separately from the subsequent picture slice. | |
| constexpr bool kH264SeparateSpsPps = true; | |
| // Force some splitting of frames across packet boundaries. The remainder of the frame data will go | |
| // in subsequent packets. | |
| constexpr size_t kMaxFrameBytesPerPacket = 4ul * 1024; | |
| constexpr zx::duration kInStreamDeadlineDuration = zx::sec(30); | |
| // This example only has one stream_lifetime_ordinal which is 1. | |
| // | |
| // TODO(dustingreen): actually re-use the Codec instance for at least one more | |
| // stream, even if it's just to decode the same data again. | |
| constexpr uint64_t kStreamLifetimeOrdinal = 1; | |
| constexpr uint8_t kLongStartCodeArray[] = {0x00, 0x00, 0x00, 0x01}; | |
| constexpr uint8_t kShortStartCodeArray[] = {0x00, 0x00, 0x01}; | |
| // For now, we need to ensure we can process frames up to this large for h264. This doesn't add in | |
| // the 128 * 1024 that we reserve for SEI/SPS/PPS that can be in the same buffer as a frame. | |
| constexpr uint32_t kInputLargeFrameSizeH264 = 1920 * 1080 * 3 / 2 / 2; | |
| constexpr uint32_t kInputReservedForBigHeadersSizeH264 = 128 * 1024; | |
| constexpr uint32_t kInputMinBufferSizeH264 = | |
| kInputLargeFrameSizeH264 + kInputReservedForBigHeadersSizeH264; | |
| // We need to ensure we can process frames/superframes up to this large for VP9. This is the size | |
| // Chromium currently sets for input buffers (for both h264 and vp9). | |
| constexpr uint32_t kInputLargeFrameSizeVp9 = 1920 * 1080 * 3 / 2 / 2 + 128 * 1024; | |
| constexpr uint32_t kInputMinBufferSizeVp9 = kInputLargeFrameSizeVp9; | |
| constexpr uint32_t kInputMinBufferSize = std::max(kInputMinBufferSizeH264, kInputMinBufferSizeVp9); | |
| constexpr uint32_t kFfmpegOutputFramePaddingBytes = 16; | |
| // Intel Legacy Y-tiled format has 128x32 byte tiles. | |
| constexpr uint32_t kTileWidthAlignment = 128; | |
| constexpr uint32_t kTileHeightAlignment = 32; | |
| // If readable_bytes is 0, that's considered a "start code", to allow the caller | |
| // to terminate a NAL the same way regardless of whether another start code is | |
| // found or the end of the buffer is found. | |
| // | |
| // ptr has readable_bytes of data - the function only evaluates whether there is | |
| // a start code at the beginning of the data at ptr. | |
| // | |
| // readable_bytes - the caller indicates how many bytes are readable starting at | |
| // ptr. | |
| // | |
| // *start_code_size_bytes will have length of the start code in bytes when the | |
| // function returns true - unchanged otherwise. Normally this would be 3 or 4, | |
| // but a 0 is possible if readable_bytes is 0. | |
| bool is_start_code(uint8_t* ptr, size_t readable_bytes, size_t* start_code_size_bytes_out) { | |
| if (readable_bytes == 0) { | |
| *start_code_size_bytes_out = 0; | |
| return true; | |
| } | |
| if (readable_bytes >= 4) { | |
| if (!memcmp(ptr, kLongStartCodeArray, sizeof(kLongStartCodeArray))) { | |
| *start_code_size_bytes_out = 4; | |
| return true; | |
| } | |
| } | |
| if (readable_bytes >= 3) { | |
| if (!memcmp(ptr, kShortStartCodeArray, sizeof(kShortStartCodeArray))) { | |
| *start_code_size_bytes_out = 3; | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| // Test-only. Not for production use. Caller must ensure there are at least 5 | |
| // bytes at nal_unit. | |
| uint8_t GetNalUnitType(const uint8_t* nal_unit) { | |
| // Also works with 4-byte startcodes. | |
| static const uint8_t start_code[3] = {0, 0, 1}; | |
| uint8_t* next_start = static_cast<uint8_t*>(memmem(nal_unit, 5, start_code, sizeof(start_code))) + | |
| sizeof(start_code); | |
| return *next_start & 0xf; | |
| } | |
| struct __attribute__((__packed__)) IvfHeader { | |
| uint32_t signature; | |
| uint16_t version; | |
| uint16_t header_length; | |
| uint32_t fourcc; | |
| uint16_t width; | |
| uint16_t height; | |
| uint32_t frame_rate; | |
| uint32_t time_scale; | |
| uint32_t frame_count; | |
| uint32_t unused; | |
| }; | |
| struct __attribute__((__packed__)) IvfFrameHeader { | |
| uint32_t size_bytes; | |
| uint64_t presentation_timestamp; | |
| }; | |
| // Tries to find a Jpeg image within the buffer. The buffer must start with SOI marker. If the image | |
| // is found within the buffer the size of the image will be returned. If it can not find the image | |
| // std::nullopt will be returned. | |
| std::optional<size_t> FindJpegImage(cpp20::span<const uint8_t> buffer) { | |
| static constexpr uint8_t MARKER_PREFIX = 0xFFu; | |
| static constexpr uint8_t SOI = 0xD8u; | |
| static constexpr uint8_t EOI = 0xD9u; | |
| size_t cursor = 0u; | |
| // Function that will read a byte from the given cursor position. If there is remaining data the | |
| // byte will be read, the cursor position advanced, and the byte returned. If there are no | |
| // remaining bytes left to read, std::nullopt will be returned instead. | |
| auto read_byte = | |
| fit::function<std::optional<uint8_t>()>([&cursor, &buffer]() -> std::optional<uint8_t> { | |
| if (cursor < buffer.size_bytes()) { | |
| uint8_t result = buffer[cursor]; | |
| cursor += 1u; | |
| return result; | |
| } else { | |
| return std::nullopt; | |
| } | |
| }); | |
| // Ensure that we are at the SOI | |
| auto marker1 = read_byte(); | |
| auto marker2 = read_byte(); | |
| if (!marker1.has_value() || !marker2.has_value()) { | |
| return std::nullopt; | |
| } | |
| if ((marker1.value() != MARKER_PREFIX) || (marker2.value() != SOI)) { | |
| return std::nullopt; | |
| } | |
| // Search for the EOI marker | |
| while (cursor < buffer.size_bytes()) { | |
| auto itr = std::find_if(buffer.begin() + cursor, buffer.end(), | |
| [](uint8_t data) { return (data == MARKER_PREFIX); }); | |
| // If at the the end, then there isn't an EOI marker in this buffer | |
| if (itr == buffer.end()) { | |
| return std::nullopt; | |
| } | |
| // Update the cursor to the current position of the marker (one past the marker prefix) | |
| cursor = std::distance(buffer.begin(), itr) + 1u; | |
| // Skip any fill bytes | |
| auto marker2 = read_byte(); | |
| while (marker2.has_value() && (marker2.value() == MARKER_PREFIX)) { | |
| marker2 = read_byte(); | |
| } | |
| if (!marker2.has_value()) { | |
| return std::nullopt; | |
| } | |
| if (marker2.value() == EOI) { | |
| return cursor; | |
| } | |
| } | |
| // If we exited, we ran out of bytes without find the EOI | |
| return std::nullopt; | |
| } | |
| enum class Format { | |
| kMjpeg, | |
| kH264, | |
| kVp9, | |
| }; | |
| const uint8_t kSliceNalUnitTypes[] = {1, 2, 3, 4, 5, 19, 20, 21}; | |
| bool IsSliceNalUnitType(uint8_t nal_unit_type) { | |
| for (uint8_t type : kSliceNalUnitTypes) { | |
| if (type == nal_unit_type) { | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| fuchsia::math::SizeU GetDisplayDimensions(const fuchsia::images2::ImageFormat& image_format) { | |
| ZX_ASSERT(image_format.has_size()); | |
| if (image_format.has_display_rect()) { | |
| return {.width = image_format.display_rect().width, | |
| .height = image_format.display_rect().height}; | |
| } | |
| if (image_format.has_valid_size()) { | |
| return image_format.valid_size(); | |
| } | |
| return image_format.size(); | |
| } | |
| // Exits unless `image_format` sets `pixel_format`, `size`, and `bytes_per_row`, any `display_rect` | |
| // starts at the origin, the display dimensions are non-zero and fit in `size`, and `packet` holds | |
| // all planes implied by `size` and `bytes_per_row`. Plane overlap cannot occur because plane | |
| // offsets are derived from `size` and `bytes_per_row`. | |
| void CheckFormatIntegrity(const fuchsia::images2::ImageFormat& image_format, | |
| const fuchsia::media::Packet& packet) { | |
| if (!image_format.has_pixel_format() || !image_format.has_size() || | |
| !image_format.has_bytes_per_row()) { | |
| Exit("image_format missing pixel_format, size, or bytes_per_row"); | |
| } | |
| size_t coded_width = image_format.size().width; | |
| size_t coded_height = image_format.size().height; | |
| size_t bytes_per_row = image_format.bytes_per_row(); | |
| if (coded_width == 0 || coded_height == 0 || coded_width > bytes_per_row) { | |
| Exit("invalid coded size or bytes_per_row"); | |
| } | |
| if (image_format.has_display_rect() && | |
| (image_format.display_rect().x != 0 || image_format.display_rect().y != 0)) { | |
| Exit("display_rect x/y != 0"); | |
| } | |
| if (image_format.has_valid_size() && (image_format.valid_size().width > coded_width || | |
| image_format.valid_size().height > coded_height)) { | |
| Exit("valid_size exceeds size"); | |
| } | |
| fuchsia::math::SizeU display_size = GetDisplayDimensions(image_format); | |
| size_t display_width = display_size.width; | |
| size_t display_height = display_size.height; | |
| if (display_width == 0 || display_height == 0 || display_width > coded_width || | |
| display_height > coded_height) { | |
| Exit("invalid display dimensions"); | |
| } | |
| switch (image_format.pixel_format()) { | |
| case fuchsia::images2::PixelFormat::NV12: { | |
| bool is_y_tiled = image_format.has_pixel_format_modifier() && | |
| image_format.pixel_format_modifier() == | |
| fuchsia::images2::PixelFormatModifier::INTEL_I915_Y_TILED; | |
| if (is_y_tiled && bytes_per_row % kTileWidthAlignment != 0) { | |
| Exit("Y-tiled NV12 bytes_per_row not aligned to kTileWidthAlignment"); | |
| } | |
| size_t primary_height = | |
| is_y_tiled ? fbl::round_up(coded_height, static_cast<size_t>(kTileHeightAlignment)) | |
| : coded_height; | |
| size_t uv_plane_height = | |
| is_y_tiled ? fbl::round_up(coded_height / 2, static_cast<size_t>(kTileHeightAlignment)) | |
| : (coded_height / 2); | |
| size_t y_size = primary_height * bytes_per_row; | |
| // NV12 requires UV be same line stride as Y; ensure odd display_width/height chroma samples | |
| // fit within the allocated UV plane. | |
| if (((display_width + 1) / 2) * 2 > bytes_per_row || | |
| (display_height + 1) / 2 > coded_height / 2) { | |
| Exit("NV12 chroma dimensions exceed coded plane dimensions"); | |
| } | |
| size_t total_size = y_size + uv_plane_height * bytes_per_row; | |
| if (packet.valid_length_bytes() < total_size) { | |
| Exit( | |
| "packet.valid_length_bytes < total_size (1) - valid_length_bytes: %u total_size: " | |
| "%zu", | |
| packet.valid_length_bytes(), total_size); | |
| } | |
| break; | |
| } | |
| case fuchsia::images2::PixelFormat::YV12: { | |
| if ((display_width + 1) / 2 > bytes_per_row / 2 || | |
| (display_height + 1) / 2 > coded_height / 2) { | |
| Exit("YV12 chroma dimensions exceed coded plane dimensions"); | |
| } | |
| size_t y_size = coded_height * bytes_per_row; | |
| size_t v_size = (coded_height / 2) * (bytes_per_row / 2); | |
| size_t u_size = v_size; | |
| size_t total_size = y_size + u_size + v_size; | |
| if (packet.valid_length_bytes() < total_size) { | |
| Exit("packet.valid_length_bytes < total_size (2)"); | |
| } | |
| break; | |
| } | |
| default: | |
| Exit("pixel_format != NV12 && pixel_format != YV12"); | |
| } | |
| } | |
| class VideoDecoderRunner { | |
| public: | |
| VideoDecoderRunner(Format format, UseVideoDecoderParams params); | |
| void Run(); | |
| private: | |
| uint64_t QueueMjpegFrames(uint64_t stream_lifetime_ordinal, uint64_t input_pts_counter_start); | |
| uint64_t QueueH264Frames(uint64_t stream_lifetime_ordinal, uint64_t input_pts_counter_start); | |
| uint64_t QueueVp9Frames(uint64_t stream_lifetime_ordinal, uint64_t input_pts_counter_start); | |
| Format format_; | |
| UseVideoDecoderParams params_; | |
| std::optional<CodecClient> codec_client_; | |
| // For testing purposes, we share some info from output to input. Normally this sort of sharing | |
| // woudln't tend to happen. | |
| // | |
| // Unlike the usual situation with video decoders, in this test, input PTS values are sequential, | |
| // while output PTS values are impacted by frame reordering. | |
| // | |
| // For h264, the degree of reordering is supposed to be bounded by max_num_reorder_frames. This | |
| // means that a given frame can be delayed by up to max_num_reorder_frames, requiring that many | |
| // additional frames on input before the delayed frame's PTS is seen on output. | |
| // | |
| // This value is only ever written by the output thread. It is read by both the output thread | |
| // and the input thread. | |
| std::atomic<int64_t> max_output_pts_seen_{-1}; | |
| }; | |
| VideoDecoderRunner::VideoDecoderRunner(Format format, UseVideoDecoderParams params) | |
| : format_(std::move(format)), params_(std::move(params)) {} | |
| uint64_t VideoDecoderRunner::QueueMjpegFrames(uint64_t stream_lifetime_ordinal, | |
| uint64_t input_pts_counter_start) { | |
| int64_t input_pts_counter = input_pts_counter_start; | |
| uint32_t frame_ordinal = 0; | |
| auto queue_jpeg_frame = [this, stream_lifetime_ordinal, &input_pts_counter, | |
| &frame_ordinal](size_t byte_count) { | |
| auto in_stream = params_.in_stream; | |
| const int64_t skip_frame_ordinal = params_.test_params->skip_frame_ordinal; | |
| std::unique_ptr<fuchsia::media::Packet> packet = codec_client_->BlockingGetFreeInputPacket(); | |
| if (!packet) { | |
| fprintf(stderr, "Returning because failed to get input packet\n"); | |
| return false; | |
| } | |
| auto increment_input_pts_counter = fit::defer([&input_pts_counter] { input_pts_counter++; }); | |
| ZX_ASSERT(packet->has_header()); | |
| ZX_ASSERT(packet->header().has_packet_index()); | |
| const CodecBuffer& buffer = codec_client_->BlockingGetFreeInputBufferForPacket(packet.get()); | |
| ZX_ASSERT(packet->buffer_index() == buffer.buffer_index()); | |
| // MJPEG decoder doesn't yet support splitting access units into multiple | |
| // packets. | |
| if (byte_count > buffer.size_bytes()) { | |
| Exit("buffer_count >= buffer.size_bytes() - byte_count: %lu buffer.size_bytes(): %lu\n", | |
| byte_count, buffer.size_bytes()); | |
| } | |
| // Setup packet | |
| packet->set_stream_lifetime_ordinal(stream_lifetime_ordinal); | |
| packet->set_start_offset(0); | |
| packet->set_valid_length_bytes(byte_count); | |
| packet->set_timestamp_ish(input_pts_counter); | |
| packet->set_start_access_unit(true); | |
| packet->set_known_end_access_unit(true); | |
| uint32_t actual_bytes_read; | |
| uint8_t* read_address = buffer.base(); | |
| zx_status_t status = | |
| in_stream->ReadBytesComplete(byte_count, &actual_bytes_read, read_address, | |
| zx::deadline_after(kInStreamDeadlineDuration)); | |
| ZX_ASSERT(status == ZX_OK); | |
| if (actual_bytes_read < byte_count) { | |
| Exit("Frame truncated."); | |
| } | |
| ZX_DEBUG_ASSERT(actual_bytes_read == byte_count); | |
| auto do_not_queue_input_packet_after_all = fit::defer( | |
| [this, &packet] { codec_client_->DoNotQueueInputPacketAfterAll(std::move(packet)); }); | |
| if (input_pts_counter == skip_frame_ordinal) { | |
| LOGF("skipping input frame: %" PRId64, input_pts_counter); | |
| // ~do_not_queue_input_packet_after_all, ~increment_input_pts_counter | |
| return true; | |
| } | |
| do_not_queue_input_packet_after_all.cancel(); | |
| codec_client_->QueueInputPacket(std::move(packet)); | |
| ++frame_ordinal; | |
| if (frame_ordinal == params_.test_params->frame_count) { | |
| return false; | |
| } | |
| // ~increment_input_pts_counter | |
| return true; | |
| }; | |
| auto in_stream = params_.in_stream; | |
| uint32_t max_peek_bytes = in_stream->max_peek_bytes(); | |
| int64_t input_stop_stream_after_frame_ordinal = | |
| params_.test_params->input_stop_stream_after_frame_ordinal; | |
| int64_t stream_frame_ordinal = 0; | |
| while (true) { | |
| uint32_t actual_peek_bytes; | |
| uint8_t* peek; | |
| VLOGF("PeekBytes()..."); | |
| zx_status_t status = in_stream->PeekBytes(max_peek_bytes, &actual_peek_bytes, &peek, | |
| zx::deadline_after(kInStreamDeadlineDuration)); | |
| ZX_ASSERT(status == ZX_OK); | |
| VLOGF("PeekBytes() done"); | |
| if (actual_peek_bytes == 0) { | |
| // Out of input. Not an error. No more input AUs. | |
| ZX_DEBUG_ASSERT(in_stream->eos_position_known() && | |
| in_stream->cursor_position() == in_stream->eos_position()); | |
| break; | |
| } | |
| auto maybe_found_image = FindJpegImage({peek, actual_peek_bytes}); | |
| if (!maybe_found_image.has_value()) { | |
| // If we could not find a JPEG image with the max allow peek bytes then we never will. Either | |
| // the stream is invalid or the encoded picture exceeds our peek limit and we will have to | |
| // increase it. | |
| if (actual_peek_bytes == max_peek_bytes) { | |
| Exit("Could not find JPEG image in stream with max peek bytes."); | |
| } | |
| continue; | |
| } | |
| // Queue the whole JPEG image | |
| if (!queue_jpeg_frame(maybe_found_image.value())) { | |
| break; | |
| } | |
| if (stream_frame_ordinal == input_stop_stream_after_frame_ordinal) { | |
| break; | |
| } | |
| stream_frame_ordinal++; | |
| } | |
| return input_pts_counter - input_pts_counter_start; | |
| } | |
| // Payload data for bear.h264 is 00 00 00 01 start code before each NAL, with | |
| // SPS / PPS NALs and also frame NALs. We deliver to Codec NAL-by-NAL without | |
| // the start code. | |
| // | |
| // Since the .h264 file has SPS + PPS NALs in addition to frame NALs, we don't | |
| // use oob_bytes for this stream. | |
| // | |
| // TODO(dustingreen): Determine for .mp4 or similar which don't have SPS / PPS | |
| // in band whether .mp4 provides ongoing OOB data, or just at the start, and | |
| // document in codec.fidl how that's to be handled. | |
| // | |
| // Returns how many input packets queued with a PTS. | |
| uint64_t VideoDecoderRunner::QueueH264Frames(uint64_t stream_lifetime_ordinal, | |
| uint64_t input_pts_counter_start) { | |
| // Raw .h264 has start code 00 00 01 or 00 00 00 01 before each NAL, and | |
| // the start codes don't alias in the middle of NALs, so we just scan | |
| // for NALs and send them in to the decoder. | |
| uint64_t input_pts_counter = input_pts_counter_start; | |
| uint64_t found_frame_ordinal = 0; | |
| uint64_t kept_frame_ordinal = 0; | |
| std::vector<uint8_t> accumulator; | |
| auto queue_access_unit = [this, stream_lifetime_ordinal, &input_pts_counter, &found_frame_ordinal, | |
| &kept_frame_ordinal, | |
| &accumulator](uint8_t* bytes, size_t byte_count) -> bool { | |
| auto tvp = params_.input_copier; | |
| size_t start_code_size_bytes = 0; | |
| ZX_ASSERT(is_start_code(bytes, byte_count, &start_code_size_bytes)); | |
| ZX_ASSERT(start_code_size_bytes < byte_count); | |
| uint8_t nal_unit_type = bytes[start_code_size_bytes] & 0x1f; | |
| auto increment_found_frame_ordinal = fit::defer([nal_unit_type, &found_frame_ordinal] { | |
| if (IsSliceNalUnitType(nal_unit_type)) { | |
| ++found_frame_ordinal; | |
| } | |
| }); | |
| constexpr uint32_t kFrameNumGapsModulus = 2; | |
| if (params_.test_params->frame_num_gaps && IsSliceNalUnitType(nal_unit_type) && | |
| ((found_frame_ordinal + (kFrameNumGapsModulus - 1)) % kFrameNumGapsModulus == 0)) { | |
| printf("frame_num_gaps skipping an input slice\n"); | |
| return true; | |
| } | |
| size_t insert_offset = accumulator.size(); | |
| size_t new_size = insert_offset + byte_count; | |
| if (accumulator.capacity() < new_size) { | |
| size_t new_capacity = std::max(accumulator.capacity() * 2, new_size); | |
| accumulator.reserve(new_capacity); | |
| } | |
| accumulator.resize(insert_offset + byte_count); | |
| // Zero pad first few frames a lot to verify large frames can decode. | |
| if (IsSliceNalUnitType(nal_unit_type) && kept_frame_ordinal < 5) { | |
| ZX_DEBUG_ASSERT(byte_count < kInputLargeFrameSizeH264); | |
| uint32_t zero_padding_bytes = kInputLargeFrameSizeH264 - byte_count; | |
| accumulator.resize(accumulator.size() + zero_padding_bytes); | |
| insert_offset += zero_padding_bytes; | |
| } | |
| memcpy(accumulator.data() + insert_offset, bytes, byte_count); | |
| if (!kH264SeparateSpsPps && !IsSliceNalUnitType(nal_unit_type)) { | |
| return true; | |
| } | |
| auto orig_bytes = bytes; | |
| bytes = accumulator.data(); | |
| byte_count = accumulator.size(); | |
| auto clear_accumulator = fit::defer([&accumulator] { accumulator.clear(); }); | |
| size_t bytes_so_far = 0; | |
| // printf("queuing offset: %ld byte_count: %zu\n", bytes - | |
| // input_bytes.get(), byte_count); | |
| while (bytes_so_far != byte_count) { | |
| VLOGF("BlockingGetFreeInputPacket()..."); | |
| std::unique_ptr<fuchsia::media::Packet> packet = codec_client_->BlockingGetFreeInputPacket(); | |
| if (!packet) { | |
| return false; | |
| } | |
| VLOGF("BlockingGetFreeInputPacket() done"); | |
| if (!packet->has_header()) { | |
| Exit("broken server sent packet without header"); | |
| } | |
| if (!packet->header().has_packet_index()) { | |
| Exit("broken server sent packet without packet index"); | |
| } | |
| // For input we do buffer_index == packet_index. | |
| const CodecBuffer& buffer = codec_client_->BlockingGetFreeInputBufferForPacket(packet.get()); | |
| ZX_ASSERT(packet->buffer_index() == buffer.buffer_index()); | |
| uint32_t padding_length = tvp ? tvp->PaddingLength() : 0; | |
| size_t bytes_to_copy = | |
| std::min(byte_count - bytes_so_far, buffer.size_bytes() - padding_length); | |
| // Force some frames to split across packet boundary. | |
| // | |
| // TODO(https://fxbug.dev/42084549): Also cover more than one frame in a packet, and split | |
| // headers. | |
| // | |
| // TODO(https://fxbug.dev/42084549): Enable testing frames split across packets once SW decode | |
| // can do that, or have this be gated on whether capability was requested of decoder and try | |
| // requesting this capability then fall back to not this capability. | |
| (void)kMaxFrameBytesPerPacket; | |
| // bytes_to_copy = std::min(bytes_to_copy, kMaxFrameBytesPerPacket); | |
| packet->set_stream_lifetime_ordinal(stream_lifetime_ordinal); | |
| packet->set_start_offset(0); | |
| packet->set_valid_length_bytes(bytes_to_copy); | |
| if (bytes_so_far == 0) { | |
| uint8_t nal_unit_type = GetNalUnitType(orig_bytes); | |
| if (IsSliceNalUnitType(nal_unit_type)) { | |
| constexpr zx::duration kComplainInterval = zx::sec(5); | |
| zx::time complain_time = zx::clock::get_monotonic() + kComplainInterval; | |
| // Wait until max_output_pts_seen_ increases to within the threshold, or time out while | |
| // complaining every 5 seconds. | |
| while (static_cast<int64_t>(input_pts_counter) > | |
| max_output_pts_seen_ + 1 + params_.test_params->max_num_reorder_frames_threshold) { | |
| zx::time now = zx::clock::get_monotonic(); | |
| if (now >= complain_time) { | |
| fprintf(stderr, | |
| "max_num_reorder_frames_threshold not satisfied? - keep waiting - may time " | |
| "out...\n"); | |
| complain_time = now + kComplainInterval; | |
| } | |
| zx::nanosleep(zx::deadline_after(zx::msec(1))); | |
| } | |
| packet->set_timestamp_ish(input_pts_counter++); | |
| } | |
| } | |
| packet->set_start_access_unit(bytes_so_far == 0); | |
| packet->set_known_end_access_unit(bytes_so_far + bytes_to_copy == byte_count); | |
| if (tvp) { | |
| TEEC_Result result = tvp->DecryptVideo(bytes + bytes_so_far, bytes_to_copy, buffer.vmo()); | |
| ZX_ASSERT(result == TEEC_SUCCESS); | |
| } else { | |
| memcpy(buffer.base(), bytes + bytes_so_far, bytes_to_copy); | |
| } | |
| codec_client_->QueueInputPacket(std::move(packet)); | |
| bytes_so_far += bytes_to_copy; | |
| } | |
| if (IsSliceNalUnitType(nal_unit_type)) { | |
| kept_frame_ordinal++; | |
| } | |
| if (kept_frame_ordinal == params_.test_params->frame_count) { | |
| return false; | |
| } | |
| return true; | |
| }; | |
| auto in_stream = params_.in_stream; | |
| // Let caller-provided in_stream drive how far ahead we peek. If it's not far | |
| // enough to find a start code or the EOS, then we'll error out. | |
| uint32_t max_peek_bytes = in_stream->max_peek_bytes(); | |
| // default -1 | |
| int64_t input_stop_stream_after_frame_ordinal = | |
| params_.test_params->input_stop_stream_after_frame_ordinal; | |
| int64_t stream_frame_ordinal = 0; | |
| while (true) { | |
| // Until clang-tidy correctly interprets Exit(), this "= 0" satisfies it. | |
| size_t start_code_size_bytes = 0; | |
| uint32_t actual_peek_bytes; | |
| uint8_t* peek; | |
| VLOGF("PeekBytes()..."); | |
| zx_status_t status = in_stream->PeekBytes(max_peek_bytes, &actual_peek_bytes, &peek, | |
| zx::deadline_after(kInStreamDeadlineDuration)); | |
| ZX_ASSERT(status == ZX_OK); | |
| VLOGF("PeekBytes() done"); | |
| if (actual_peek_bytes == 0) { | |
| // Out of input. Not an error. No more input AUs. | |
| ZX_DEBUG_ASSERT(in_stream->eos_position_known() && | |
| in_stream->cursor_position() == in_stream->eos_position()); | |
| break; | |
| } | |
| if (!is_start_code(&peek[0], actual_peek_bytes, &start_code_size_bytes)) { | |
| for (uint32_t i = 0; i < 64; ++i) { | |
| LOGF("peek[%u] == 0x%x", i, peek[i]); | |
| } | |
| char buf[65] = {}; | |
| memcpy(&buf[0], &peek[0], 64); | |
| buf[64] = 0; | |
| LOGF("peek[0..64]: %s", buf); | |
| if (in_stream->cursor_position() == 0) { | |
| Exit( | |
| "Didn't find a start code at the start of the file, and this " | |
| "example doesn't scan forward (for now)."); | |
| } else { | |
| Exit( | |
| "Fell out of sync somehow - previous NAL offset + previous " | |
| "NAL length not a start code."); | |
| } | |
| } | |
| if (in_stream->eos_position_known() && | |
| in_stream->cursor_position() + start_code_size_bytes == in_stream->eos_position()) { | |
| Exit("Start code at end of file unexpected"); | |
| } | |
| size_t nal_start_offset = start_code_size_bytes; | |
| // Scan for end of NAL. The end of NAL can be because we're out of peeked | |
| // data, or because we hit another start code. | |
| size_t find_end_iter = nal_start_offset; | |
| size_t ignore_start_code_size_bytes; | |
| while (find_end_iter <= actual_peek_bytes && | |
| !is_start_code(&peek[find_end_iter], actual_peek_bytes - find_end_iter, | |
| &ignore_start_code_size_bytes)) { | |
| find_end_iter++; | |
| } | |
| ZX_DEBUG_ASSERT(find_end_iter <= actual_peek_bytes); | |
| if (find_end_iter == nal_start_offset) { | |
| Exit("Two adjacent start codes unexpected."); | |
| } | |
| ZX_DEBUG_ASSERT(find_end_iter > nal_start_offset); | |
| size_t nal_length = find_end_iter - nal_start_offset; | |
| if (!queue_access_unit(&peek[0], start_code_size_bytes + nal_length)) { | |
| // only reached on error | |
| break; | |
| } | |
| // This is after queue_access_unit so it's more obvious in the test output | |
| // how the stream is behaving wrt max_num_reorder_frames_threshold. | |
| if (params_.test_params->per_frame_debug_output) { | |
| LOGF("H264 input stream: %" PRIu64 " stream_frame_ordinal: %" PRId64 | |
| " input_pts_counter: %" PRIu64 " frame_header.size_bytes: 0x%zx", | |
| stream_lifetime_ordinal, stream_frame_ordinal, input_pts_counter, | |
| start_code_size_bytes + nal_length); | |
| } | |
| // start code + NAL payload | |
| VLOGF("TossPeekedBytes()..."); | |
| in_stream->TossPeekedBytes(start_code_size_bytes + nal_length); | |
| VLOGF("TossPeekedBytes() done"); | |
| if (stream_frame_ordinal == input_stop_stream_after_frame_ordinal) { | |
| break; | |
| } | |
| stream_frame_ordinal++; | |
| } | |
| return input_pts_counter - input_pts_counter_start; | |
| } | |
| uint64_t VideoDecoderRunner::QueueVp9Frames(uint64_t stream_lifetime_ordinal, | |
| uint64_t input_pts_counter_start) { | |
| // default -1 | |
| int64_t input_pts_counter = input_pts_counter_start; | |
| uint32_t frame_ordinal = 0; | |
| auto queue_access_unit = [this, stream_lifetime_ordinal, &input_pts_counter, | |
| &frame_ordinal](size_t byte_count) { | |
| auto in_stream = params_.in_stream; | |
| auto tvp = params_.input_copier; | |
| const int64_t skip_frame_ordinal = params_.test_params->skip_frame_ordinal; | |
| // We want each compresed input frame being queued to have a unique (synthetic) PTS value. | |
| auto increment_input_pts_counter = fit::defer([&input_pts_counter] { input_pts_counter++; }); | |
| std::unique_ptr<fuchsia::media::Packet> packet = codec_client_->BlockingGetFreeInputPacket(); | |
| if (!packet) { | |
| // We still run increment_input_pts_counter here because each compressed input frame has a | |
| // unique (synthetic) PTS value; this value doesn't depend on whether we were able to find a | |
| // free input packet to deliver the compressed input frame to the decoder. | |
| fprintf(stderr, "Returning because failed to get input packet\n"); | |
| return false; | |
| } | |
| //////////////////////////////////////////////////////////////////////////////////////////////// | |
| // No more return false from here down. Before we return true, we must have consumed the input | |
| // data, and incremented the input_frame_ordinal, and returned the input packet to the | |
| // codec_client. The codec_client only wants the input packet back after its been filled out | |
| // completely. | |
| //////////////////////////////////////////////////////////////////////////////////////////////// | |
| auto do_not_return_early_interval = fit::defer( | |
| [] { ZX_PANIC("don't return early until packet is set up and returned to codec_client"); }); | |
| ZX_ASSERT(packet->has_header()); | |
| ZX_ASSERT(packet->header().has_packet_index()); | |
| const CodecBuffer& buffer = codec_client_->BlockingGetFreeInputBufferForPacket(packet.get()); | |
| ZX_ASSERT(packet->buffer_index() == buffer.buffer_index()); | |
| // VP9 decoder doesn't yet support splitting access units into multiple | |
| // packets. | |
| if (byte_count > buffer.size_bytes()) { | |
| fprintf(stderr, | |
| "buffer_count >= buffer.size_bytes() - byte_count: %lu buffer.size_bytes(): %lu\n", | |
| byte_count, buffer.size_bytes()); | |
| } | |
| ZX_ASSERT(byte_count <= buffer.size_bytes()); | |
| // Check that we don't waste contiguous space on non-secure VP9 input buffers. | |
| ZX_ASSERT(!buffer.is_physically_contiguous() || tvp); | |
| packet->set_stream_lifetime_ordinal(stream_lifetime_ordinal); | |
| packet->set_start_offset(0); | |
| packet->set_valid_length_bytes(byte_count); | |
| constexpr zx::duration kComplainInterval = zx::sec(5); | |
| zx::time complain_time = zx::clock::get_monotonic() + kComplainInterval; | |
| // Wait until max_output_pts_seen_ increases to within the threshold, or time out while | |
| // complaining every 5 seconds. | |
| while (static_cast<int64_t>(input_pts_counter) > | |
| max_output_pts_seen_ + params_.test_params->max_num_reorder_frames_threshold) { | |
| zx::time now = zx::clock::get_monotonic(); | |
| if (now >= complain_time) { | |
| fprintf(stderr, | |
| "max_num_reorder_frames_threshold not satisfied? - keep waiting - may time " | |
| "out...\n"); | |
| complain_time = now + kComplainInterval; | |
| } | |
| zx::nanosleep(zx::deadline_after(zx::msec(1))); | |
| } | |
| // We don't use frame_header->presentation_timestamp, because we want to | |
| // send through frame index in timestamp_ish field instead, for consistency | |
| // with .h264 files which don't have timestamps in them, and so tests can | |
| // assume frame index as timestamp_ish on output. | |
| packet->set_timestamp_ish(input_pts_counter); | |
| packet->set_start_access_unit(true); | |
| packet->set_known_end_access_unit(true); | |
| uint32_t actual_bytes_read; | |
| std::unique_ptr<uint8_t[]> bytes; | |
| uint8_t* read_address = nullptr; | |
| if (tvp) { | |
| bytes = std::make_unique<uint8_t[]>(byte_count); | |
| read_address = bytes.get(); | |
| } else { | |
| read_address = buffer.base(); | |
| } | |
| zx_status_t status = | |
| in_stream->ReadBytesComplete(byte_count, &actual_bytes_read, read_address, | |
| zx::deadline_after(kInStreamDeadlineDuration)); | |
| ZX_ASSERT(status == ZX_OK); | |
| if (actual_bytes_read < byte_count) { | |
| Exit("Frame truncated."); | |
| } | |
| ZX_DEBUG_ASSERT(actual_bytes_read == byte_count); | |
| ///////////////////////////////////////////////////////////////////////////////// | |
| // Switch from not being able to return early to being able to return true early. | |
| ///////////////////////////////////////////////////////////////////////////////// | |
| do_not_return_early_interval.cancel(); | |
| auto do_not_queue_input_packet_after_all = fit::defer( | |
| [this, &packet] { codec_client_->DoNotQueueInputPacketAfterAll(std::move(packet)); }); | |
| if (input_pts_counter == skip_frame_ordinal) { | |
| LOGF("skipping input frame: %" PRId64, input_pts_counter); | |
| // ~do_not_queue_input_packet_after_all, ~increment_input_pts_counter | |
| return true; | |
| } | |
| if (tvp) { | |
| VLOGF("before DecryptVideo..."); | |
| TEEC_Result result = tvp->DecryptVideo(bytes.get(), byte_count, buffer.vmo()); | |
| VLOGF("after DecryptVideo"); | |
| ZX_ASSERT(result == TEEC_SUCCESS); | |
| } | |
| do_not_queue_input_packet_after_all.cancel(); | |
| // Ideally we'd figure out why this padding doesn't work / how to pad VP9 frames, if possible. | |
| #if 0 | |
| if (frame_ordinal <= 5) { | |
| // Assert not a superframe. If we have some of these in input, we can skip padding them | |
| // instead, or find a way to pad a superframe. | |
| ZX_DEBUG_ASSERT((*(buffer.base() + packet->valid_length_bytes() - 1) & 0xE0) != 0xC0); | |
| ZX_DEBUG_ASSERT(buffer.size_bytes() >= kInputLargeFrameSizeVp9); | |
| if (byte_count < kInputLargeFrameSizeVp9) { | |
| uint32_t zero_bytes_count = kInputLargeFrameSizeVp9 - byte_count; | |
| memset(buffer.base() + byte_count, 0, zero_bytes_count); | |
| packet->set_valid_length_bytes(packet->valid_length_bytes() + zero_bytes_count); | |
| } | |
| } | |
| #endif | |
| codec_client_->QueueInputPacket(std::move(packet)); | |
| ++frame_ordinal; | |
| if (frame_ordinal == params_.test_params->frame_count) { | |
| return false; | |
| } | |
| // ~increment_input_pts_counter | |
| return true; | |
| }; | |
| auto in_stream = params_.in_stream; | |
| IvfHeader header; | |
| uint32_t actual_bytes_read; | |
| zx_status_t status = in_stream->ReadBytesComplete(sizeof(header), &actual_bytes_read, | |
| reinterpret_cast<uint8_t*>(&header), | |
| zx::deadline_after(kInStreamDeadlineDuration)); | |
| // This could fail if a remote-source stream breaks. | |
| ZX_ASSERT(status == ZX_OK); | |
| // This could fail if the input is too short. | |
| ZX_ASSERT(actual_bytes_read == sizeof(header)); | |
| size_t remaining_header_length = header.header_length - sizeof(header); | |
| // We're not interested in any remaining portion of the header, but we should | |
| // skip the rest of the header, if any. | |
| if (remaining_header_length) { | |
| uint8_t toss_buffer[1024]; | |
| while (remaining_header_length != 0) { | |
| uint32_t bytes_to_read = std::min(sizeof(toss_buffer), remaining_header_length); | |
| uint32_t actual_bytes_read; | |
| status = in_stream->ReadBytesComplete(bytes_to_read, &actual_bytes_read, &toss_buffer[0], | |
| zx::deadline_after(kInStreamDeadlineDuration)); | |
| ZX_ASSERT(status == ZX_OK); | |
| ZX_ASSERT(actual_bytes_read == bytes_to_read); | |
| remaining_header_length -= actual_bytes_read; | |
| } | |
| } | |
| ZX_DEBUG_ASSERT(!remaining_header_length); | |
| // default -1 | |
| int64_t input_stop_stream_after_frame_ordinal = | |
| params_.test_params->input_stop_stream_after_frame_ordinal; | |
| int64_t stream_frame_ordinal = 0; | |
| while (true) { | |
| IvfFrameHeader frame_header; | |
| status = in_stream->ReadBytesComplete(sizeof(frame_header), &actual_bytes_read, | |
| reinterpret_cast<uint8_t*>(&frame_header), | |
| zx::deadline_after(kInStreamDeadlineDuration)); | |
| ZX_ASSERT(status == ZX_OK); | |
| if (actual_bytes_read == 0) { | |
| // No more frames. That's fine. | |
| break; | |
| } | |
| if (actual_bytes_read < sizeof(frame_header)) { | |
| Exit("Frame header truncated."); | |
| } | |
| ZX_DEBUG_ASSERT(actual_bytes_read == sizeof(frame_header)); | |
| if (!queue_access_unit(frame_header.size_bytes)) { | |
| // can be fine in case of vp9 input fuzzing test | |
| break; | |
| } | |
| // This is after queue_access_unit to make max_num_reorder_frames_threshold behavior line up | |
| // with the test output. | |
| if (params_.test_params->per_frame_debug_output) { | |
| LOGF("VP9 input stream: %" PRIu64 " stream_frame_ordinal: %" PRId64 | |
| " input_pts_counter: %" PRIu64 " frame_header.size_bytes: 0x%x", | |
| stream_lifetime_ordinal, stream_frame_ordinal, input_pts_counter, | |
| frame_header.size_bytes); | |
| } | |
| if (stream_frame_ordinal == input_stop_stream_after_frame_ordinal) { | |
| break; | |
| } | |
| stream_frame_ordinal++; | |
| } | |
| return input_pts_counter - input_pts_counter_start; | |
| } | |
| void VideoDecoderRunner::Run() { | |
| const UseVideoDecoderTestParams default_test_params; | |
| if (!params_.test_params) { | |
| params_.test_params = &default_test_params; | |
| } | |
| params_.test_params->Validate(); | |
| VLOGF("before CodecClient::CodecClient()..."); | |
| codec_client_.emplace(params_.fidl_loop, params_.fidl_thread, std::move(params_.sysmem)); | |
| codec_client_->SetMinInputBufferSize(kInputMinBufferSize); | |
| // no effect if 0 | |
| codec_client_->SetMinOutputBufferSize(params_.min_output_buffer_size); | |
| // no effect if 0 | |
| codec_client_->SetMinOutputBufferCount(params_.min_output_buffer_count); | |
| codec_client_->set_is_output_secure(params_.is_secure_output); | |
| codec_client_->set_is_input_secure(params_.is_secure_input); | |
| codec_client_->set_in_lax_mode(params_.lax_mode); | |
| codec_client_->set_is_output_tiled(params_.test_params->is_output_y_tiled); | |
| codec_client_->set_is_dynamic_buffers(params_.test_params->is_dynamic_buffers); | |
| std::string mime_type; | |
| switch (format_) { | |
| case Format::kMjpeg: | |
| mime_type = "video/x-motion-jpeg"; | |
| break; | |
| case Format::kH264: | |
| mime_type = "video/h264"; | |
| break; | |
| case Format::kVp9: | |
| mime_type = "video/vp9"; | |
| break; | |
| } | |
| if (params_.test_params->mime_type) { | |
| mime_type = params_.test_params->mime_type.value(); | |
| } | |
| async::PostTask( | |
| params_.fidl_loop->dispatcher(), | |
| [this, codec_client_request = codec_client_->GetTheRequestOnce(), mime_type]() mutable { | |
| VLOGF("before codec_factory->CreateDecoder() (async)"); | |
| fuchsia::media::FormatDetails input_details; | |
| input_details.set_format_details_version_ordinal(0); | |
| input_details.set_mime_type(mime_type.c_str()); | |
| fuchsia::mediacodec::CreateDecoder_Params decoder_params; | |
| decoder_params.set_input_details(std::move(input_details)); | |
| // This is required for timestamp_ish values to transit the | |
| // Codec. | |
| // | |
| // TODO(https://fxbug.dev/42135582): We shouldn't need to promise this to have PTS(s) flow | |
| // through. | |
| decoder_params.set_promise_separate_access_units_on_input(true); | |
| if (params_.is_secure_output) { | |
| decoder_params.set_secure_output_mode(fuchsia::mediacodec::SecureMemoryMode::ON); | |
| } | |
| if (params_.is_secure_input) { | |
| decoder_params.set_secure_input_mode(fuchsia::mediacodec::SecureMemoryMode::ON); | |
| } | |
| if (params_.test_params->require_sw) { | |
| decoder_params.set_require_sw(true); | |
| } | |
| // Bind the fuchsia::media::CodecFactoryHandle to a CodecFactoryPtr so we can send a | |
| // CreateDecoder message. This unbinds params_.codec_factory. | |
| auto codec_factory_ptr = params_.codec_factory.Bind(); | |
| codec_factory_ptr.set_error_handler([](zx_status_t status) { | |
| FX_PLOGS(FATAL, status) << "codec_factory failed - unexpected"; | |
| }); | |
| codec_factory_ptr->CreateDecoder(std::move(decoder_params), | |
| std::move(codec_client_request)); | |
| // Now that the CreateDecoder message is sent, we no longer need to keep a channel open | |
| // to the CodecFactory so we can just let codec_factory_ptr fall out of scope. | |
| }); | |
| VLOGF("before codec_client.Start()..."); | |
| codec_client_->Start(); | |
| VLOGF("before starting in_thread..."); | |
| auto in_thread = std::make_unique<std::thread>([this]() { | |
| auto& in_stream = params_.in_stream; | |
| auto& test_params = params_.test_params; | |
| VLOGF("in_thread start"); | |
| // default 1 | |
| const uint32_t loop_stream_count = test_params->loop_stream_count; | |
| // default 2 | |
| const uint64_t keep_stream_modulo = test_params->keep_stream_modulo; | |
| uint64_t stream_lifetime_ordinal = kStreamLifetimeOrdinal; | |
| uint64_t input_frame_pts_counter = 0; | |
| uint32_t frames_queued = 0; | |
| for (uint32_t loop_ordinal = 0; loop_ordinal < loop_stream_count; | |
| ++loop_ordinal, stream_lifetime_ordinal += 2) { | |
| switch (format_) { | |
| case Format::kMjpeg: | |
| frames_queued = QueueMjpegFrames(stream_lifetime_ordinal, input_frame_pts_counter); | |
| break; | |
| case Format::kH264: | |
| frames_queued = QueueH264Frames(stream_lifetime_ordinal, input_frame_pts_counter); | |
| break; | |
| case Format::kVp9: | |
| frames_queued = QueueVp9Frames(stream_lifetime_ordinal, input_frame_pts_counter); | |
| break; | |
| } | |
| // Send through QueueInputEndOfStream(). | |
| VLOGF("QueueInputEndOfStream() - stream_lifetime_ordinal: %" PRIu64, stream_lifetime_ordinal); | |
| // For debugging a flake: | |
| if (test_params->loop_stream_count > 1) { | |
| LOGF("QueueInputEndOfStream() - stream_lifetime_ordinal: %" PRIu64, | |
| stream_lifetime_ordinal); | |
| } | |
| codec_client_->QueueInputEndOfStream(stream_lifetime_ordinal); | |
| if (stream_lifetime_ordinal % keep_stream_modulo == 1) { | |
| // We flush and close to run the handling code server-side. However, we don't | |
| // yet verify that this successfully achieves what it says. | |
| VLOGF("FlushEndOfStreamAndCloseStream() - stream_lifetime_ordinal: %" PRIu64, | |
| stream_lifetime_ordinal); | |
| // For debugging a flake: | |
| if (test_params->loop_stream_count > 1) { | |
| LOGF("FlushEndOfStreamAndCloseStream() - stream_lifetime_ordinal: %" PRIu64, | |
| stream_lifetime_ordinal); | |
| } | |
| codec_client_->FlushEndOfStreamAndCloseStream(stream_lifetime_ordinal); | |
| // Stitch together the PTS values of the streams which we're keeping. | |
| input_frame_pts_counter += frames_queued; | |
| } | |
| if (loop_ordinal + 1 != loop_stream_count) { | |
| zx_status_t status = in_stream->ResetToStart(zx::deadline_after(kInStreamDeadlineDuration)); | |
| ZX_ASSERT(status == ZX_OK); | |
| } | |
| } | |
| VLOGF("in_thread done"); | |
| }); | |
| // Separate thread to process the output. | |
| // | |
| // codec_client outlives the thread. | |
| auto out_thread = std::make_unique<std::thread>([this]() { | |
| VLOGF("out_thread start"); | |
| // We allow the server to send multiple output constraint updates if it | |
| // wants; see implementation of BlockingGetEmittedOutput() which will hide | |
| // multiple constraint updates before the first packet from this code. In | |
| // contrast we assert if the server sends multiple format updates with no | |
| // packets in between since that's not compliant with the protocol rules. | |
| std::shared_ptr<const fuchsia::media::StreamOutputFormat> prev_stream_format; | |
| const fuchsia::images2::ImageFormat* raw_image_format = nullptr; | |
| std::optional<zx::time> frame_zero_time; | |
| uint64_t frame_index = 0; | |
| std::unique_ptr<uint8_t[]> reusable_i420_bytes; | |
| size_t reusable_i420_capacity = 0; | |
| while (true) { | |
| VLOGF("BlockingGetEmittedOutput()..."); | |
| std::unique_ptr<CodecOutput> output = codec_client_->BlockingGetEmittedOutput(); | |
| VLOGF("BlockingGetEmittedOutput() done"); | |
| if (!output) { | |
| return; | |
| } | |
| if (output->stream_lifetime_ordinal() % 2 == 0) { | |
| Exit( | |
| "server emitted a stream_lifetime_ordinal that client didn't set " | |
| "on any input"); | |
| } | |
| if (output->end_of_stream() || output->stream_failed()) { | |
| if (output->end_of_stream()) { | |
| VLOGF("output end_of_stream() - stream_lifetime_ordinal: %" PRIu64, | |
| output->stream_lifetime_ordinal()); | |
| } else { | |
| ZX_ASSERT(output->stream_failed()); | |
| VLOGF("otuput stream_failed() - stream_lifetime_ordinal: %" PRIu64, | |
| output->stream_lifetime_ordinal()); | |
| } | |
| // For debugging a flake: | |
| if (params_.test_params->loop_stream_count > 1) { | |
| LOGF("output %s - stream_lifetime_ordinal: %" PRIu64, | |
| output->end_of_stream() ? "end_of_stream()" : "stream_failed()", | |
| output->stream_lifetime_ordinal()); | |
| } | |
| // default 1 | |
| const int64_t loop_stream_count = params_.test_params->loop_stream_count; | |
| const uint64_t max_stream_lifetime_ordinal = (loop_stream_count - 1) * 2 + 1; | |
| if (output->stream_lifetime_ordinal() != max_stream_lifetime_ordinal) { | |
| continue; | |
| } | |
| VLOGF("done with output - stream_lifetime_ordinal: %" PRIu64, | |
| output->stream_lifetime_ordinal()); | |
| // For debugging a flake: | |
| if (params_.test_params->loop_stream_count > 1) { | |
| LOGF("done with output - stream_lifetime_ordinal: %" PRIu64, | |
| output->stream_lifetime_ordinal()); | |
| } | |
| // Just "break;" would be more fragile under code modification. | |
| goto end_of_output; | |
| } | |
| const fuchsia::media::Packet& packet = output->packet(); | |
| auto increment_frame_index = fit::defer([&frame_index] { ++frame_index; }); | |
| if (!packet.has_header()) { | |
| // The server should not generate any empty packets. | |
| Exit("broken server sent packet without header"); | |
| } | |
| // cleanup can run on any thread, and codec_client.RecycleOutputPacket() is ok with that since | |
| // it switches to the dispatcher thread before sending a message. | |
| auto cleanup = fit::defer([this, packet_header = fidl::Clone(packet.header())]() mutable { | |
| codec_client_->RecycleOutputPacket(std::move(packet_header)); | |
| }); | |
| std::shared_ptr<const fuchsia::media::StreamOutputFormat> format = output->format(); | |
| if (!packet.has_buffer_index()) { | |
| // The server should not generate any empty packets. | |
| Exit("broken server sent packet without buffer index"); | |
| } | |
| // This will remain live long enough because this thread is the only | |
| // thread that re-allocates output buffers. | |
| const CodecBuffer& buffer = codec_client_->GetOutputBufferByIndex(packet.buffer_index()); | |
| ZX_ASSERT(!prev_stream_format || | |
| (prev_stream_format->has_format_details() && | |
| prev_stream_format->format_details().format_details_version_ordinal())); | |
| if (!format->has_format_details()) { | |
| Exit("!format->has_format_details()"); | |
| } | |
| if (!format->format_details().has_format_details_version_ordinal()) { | |
| Exit("!format->format_details().has_format_details_version_ordinal()"); | |
| } | |
| if (!packet.has_valid_length_bytes() || packet.valid_length_bytes() == 0) { | |
| // The server should not generate any empty packets. | |
| Exit("broken server sent empty packet"); | |
| } | |
| if (!packet.has_start_offset()) { | |
| // The server should not generate any empty packets. | |
| Exit("broken server sent packet without start offset"); | |
| } | |
| // We have a non-empty packet of the stream. | |
| if (params_.test_params->require_sw) { | |
| if ((buffer.vmo_offset() + packet.start_offset() + packet.valid_length_bytes()) % | |
| zx_system_get_page_size() == | |
| 0) { | |
| // If this doesn't print, then it means the check for padding below isn't really checking | |
| // whether we force an extra 16 bytes after the output frame, since buffers are always | |
| // aligned up to 4KiB anyway. Only when the frame size is 4KiB aligned will we see the | |
| // 16 bytes of padding really change the buffer size. When using ffmpeg sw decode on the | |
| // 1080p_10_frames.h264 test file, this does print. | |
| printf("test is really checking for padding\n"); | |
| } | |
| uint64_t vmo_size = 0; | |
| zx_status_t status = buffer.vmo().get_size(&vmo_size); | |
| ZX_ASSERT(status == ZX_OK); | |
| if (vmo_size - buffer.vmo_offset() - packet.valid_length_bytes() - packet.start_offset() < | |
| kFfmpegOutputFramePaddingBytes) { | |
| Exit("require_sw true decoder didn't pad as required by FFMPEG"); | |
| } | |
| } | |
| if (!prev_stream_format || prev_stream_format.get() != format.get()) { | |
| VLOGF("handling output format"); | |
| // Every output has a format. This happens exactly once. | |
| prev_stream_format = format; | |
| if (!format->has_format_details()) { | |
| Exit("!format_details"); | |
| } | |
| const fuchsia::media::FormatDetails& format_details = format->format_details(); | |
| if (!format_details.has_image_format()) { | |
| Exit("!format.image_format"); | |
| } | |
| raw_image_format = &format_details.image_format(); | |
| // Ensure that image_format doesn't exceed packet requirements | |
| CheckFormatIntegrity(*raw_image_format, packet); | |
| } | |
| if (frame_index == 0) { | |
| ZX_ASSERT(!frame_zero_time); | |
| frame_zero_time.emplace(zx::clock::get_monotonic()); | |
| } | |
| if (params_.test_params->print_fps) { | |
| zx::time now = zx::clock::get_monotonic(); | |
| zx::duration duration = now - frame_zero_time.value(); | |
| if (frame_index != 0 && frame_index % params_.test_params->print_fps_modulus == 0) { | |
| printf("frame_index: %" PRIu64 " fps: %g\n", frame_index, | |
| static_cast<double>(frame_index) * 1000000.0 / | |
| static_cast<double>(duration.to_usecs())); | |
| fflush(nullptr); | |
| } | |
| } | |
| if (params_.emit_frame) { | |
| uint32_t coded_height = raw_image_format->size().height; | |
| fuchsia::math::SizeU display_size = GetDisplayDimensions(*raw_image_format); | |
| uint32_t display_width = display_size.width; | |
| uint32_t display_height = display_size.height; | |
| uint32_t bytes_per_row = raw_image_format->bytes_per_row(); | |
| // i420_ptr points to an I420 buffer - Y plane first, then U plane, then | |
| // V plane. The U and V planes are half size in both directions. Each | |
| // plane is 8 bits per sample. | |
| uint32_t i420_stride = fbl::round_up(display_width, 2u); | |
| // When width is odd, we want a chroma sample for the right-most luma. | |
| uint32_t uv_width = (display_width + 1) / 2; | |
| // When height is odd, we want a chroma sample for the bottom-most luma. | |
| uint32_t uv_height = (display_height + 1) / 2; | |
| uint32_t uv_stride = i420_stride / 2; | |
| const size_t i420_size = static_cast<size_t>(i420_stride) * display_height + | |
| static_cast<size_t>(uv_stride) * uv_height * 2; | |
| uint8_t* i420_ptr = nullptr; | |
| if (!params_.test_params->skip_formatting_output_pixels && | |
| (kVerifySecureOutput || !params_.is_secure_output)) { | |
| // `reusable_i420_bytes` is reused across frames and not zeroed, so every pixel-format | |
| // conversion path below must write every byte up to `i420_size` (including the luma | |
| // right-edge pad byte when `display_width < i420_stride`). | |
| if (i420_size > reusable_i420_capacity) { | |
| reusable_i420_bytes = std::make_unique_for_overwrite<uint8_t[]>(i420_size); | |
| reusable_i420_capacity = i420_size; | |
| } | |
| i420_ptr = reusable_i420_bytes.get(); | |
| switch (raw_image_format->pixel_format()) { | |
| case fuchsia::images2::PixelFormat::NV12: { | |
| if (raw_image_format->has_pixel_format_modifier() && | |
| raw_image_format->pixel_format_modifier() == | |
| fuchsia::images2::PixelFormatModifier::INTEL_I915_Y_TILED) { | |
| size_t secondary_offset = static_cast<size_t>(bytes_per_row) * | |
| fbl::round_up(coded_height, kTileHeightAlignment); | |
| // Y | |
| uint8_t* y_plane_src = buffer.base() + packet.start_offset(); | |
| uint8_t* y_plane_dst = i420_ptr; | |
| for (uint32_t y_offset = 0u; y_offset < display_height; y_offset++) { | |
| for (uint32_t x_offset = 0u; x_offset < display_width; x_offset++) { | |
| uint32_t swizzled_offset = | |
| ConvertLinearToLegacyYTiled(y_offset, x_offset, bytes_per_row); | |
| y_plane_dst[x_offset] = y_plane_src[swizzled_offset]; | |
| } | |
| if (display_width < i420_stride) { | |
| y_plane_dst[display_width] = 0; | |
| } | |
| y_plane_dst += i420_stride; | |
| } | |
| // UV | |
| uint8_t* uv_src = buffer.base() + packet.start_offset() + secondary_offset; | |
| uint8_t* u_dst_line = y_plane_dst; | |
| uint8_t* v_dst_line = u_dst_line + static_cast<size_t>(uv_stride) * uv_height; | |
| for (uint32_t uv_iter = 0; uv_iter < uv_height; uv_iter++) { | |
| uint8_t* u_dst = u_dst_line; | |
| uint8_t* v_dst = v_dst_line; | |
| for (uint32_t uv_line_iter = 0; uv_line_iter < uv_width; ++uv_line_iter) { | |
| uint32_t u_offset = | |
| ConvertLinearToLegacyYTiled(uv_iter, uv_line_iter * 2, bytes_per_row); | |
| uint32_t v_offset = | |
| ConvertLinearToLegacyYTiled(uv_iter, uv_line_iter * 2 + 1, bytes_per_row); | |
| *u_dst++ = uv_src[u_offset]; | |
| *v_dst++ = uv_src[v_offset]; | |
| } | |
| u_dst_line += uv_stride; | |
| v_dst_line += uv_stride; | |
| } | |
| } else { | |
| size_t secondary_offset = static_cast<size_t>(bytes_per_row) * coded_height; | |
| // Y | |
| uint8_t* y_src = buffer.base() + packet.start_offset(); | |
| uint8_t* y_dst = i420_ptr; | |
| for (uint32_t y_iter = 0; y_iter < display_height; y_iter++) { | |
| memcpy(y_dst, y_src, display_width); | |
| if (display_width < i420_stride) { | |
| y_dst[display_width] = 0; | |
| } | |
| y_src += bytes_per_row; | |
| y_dst += i420_stride; | |
| } | |
| // UV | |
| uint8_t* uv_src = buffer.base() + packet.start_offset() + secondary_offset; | |
| uint8_t* u_dst_line = y_dst; | |
| uint8_t* v_dst_line = u_dst_line + static_cast<size_t>(uv_stride) * uv_height; | |
| for (uint32_t uv_iter = 0; uv_iter < uv_height; uv_iter++) { | |
| uint8_t* u_dst = u_dst_line; | |
| uint8_t* v_dst = v_dst_line; | |
| for (uint32_t uv_line_iter = 0; uv_line_iter < uv_width; ++uv_line_iter) { | |
| *u_dst++ = uv_src[uv_line_iter * 2]; | |
| *v_dst++ = uv_src[uv_line_iter * 2 + 1]; | |
| } | |
| uv_src += bytes_per_row; | |
| u_dst_line += uv_stride; | |
| v_dst_line += uv_stride; | |
| } | |
| } | |
| break; | |
| } | |
| case fuchsia::images2::PixelFormat::YV12: { | |
| // Y | |
| uint8_t* y_src = buffer.base() + packet.start_offset(); | |
| uint8_t* y_dst = i420_ptr; | |
| for (uint32_t y_iter = 0; y_iter < display_height; y_iter++) { | |
| memcpy(y_dst, y_src, display_width); | |
| if (display_width < i420_stride) { | |
| y_dst[display_width] = 0; | |
| } | |
| y_src += bytes_per_row; | |
| y_dst += i420_stride; | |
| } | |
| // UV | |
| uint8_t* v_src = buffer.base() + packet.start_offset() + | |
| static_cast<size_t>(bytes_per_row) * coded_height; | |
| uint8_t* u_src = v_src + static_cast<size_t>(bytes_per_row / 2) * (coded_height / 2); | |
| uint8_t* u_dst = y_dst; | |
| uint8_t* v_dst = u_dst + static_cast<size_t>(uv_stride) * uv_height; | |
| for (uint32_t uv_iter = 0; uv_iter < uv_height; uv_iter++) { | |
| memcpy(u_dst, u_src, uv_width); | |
| memcpy(v_dst, v_src, uv_width); | |
| u_dst += uv_stride; | |
| v_dst += uv_stride; | |
| u_src += bytes_per_row / 2; | |
| v_src += bytes_per_row / 2; | |
| } | |
| break; | |
| } | |
| default: | |
| Exit("Feeding EmitFrame not yet implemented for pixel_format: %u", | |
| static_cast<uint32_t>(raw_image_format->pixel_format())); | |
| } | |
| } | |
| params_.emit_frame(output->stream_lifetime_ordinal(), i420_ptr, display_width, | |
| display_height, i420_stride, packet.has_timestamp_ish(), | |
| packet.has_timestamp_ish() ? packet.timestamp_ish() : 0); | |
| } | |
| // We update this after emit_frame to make stdout info reflect how the stream is behaving wrt | |
| // max_num_reorder_frames_threshold. | |
| if (packet.has_timestamp_ish()) { | |
| uint64_t timestamp_ish = packet.timestamp_ish(); | |
| ZX_ASSERT(timestamp_ish < std::numeric_limits<int64_t>::max()); | |
| if (static_cast<int64_t>(timestamp_ish) > max_output_pts_seen_) { | |
| max_output_pts_seen_ = static_cast<int64_t>(timestamp_ish); | |
| } | |
| } | |
| // If we didn't std::move(cleanup) before here, then ~cleanup runs here. | |
| } | |
| end_of_output:; | |
| VLOGF("out_thread done"); | |
| // output thread done | |
| // ~raw_video_writer | |
| }); | |
| // decode for a bit... in_thread, loop, out_thread, and the codec itself are | |
| // taking care of it. | |
| // First wait for the input thread to be done feeding input data. Before the | |
| // in_thread terminates, it'll have sent in a last empty EOS input buffer. | |
| VLOGF("before in_thread->join()..."); | |
| in_thread->join(); | |
| VLOGF("after in_thread->join()"); | |
| // The EOS queued as an input buffer should cause the codec to output an EOS | |
| // output buffer, at which point out_thread should terminate, after it has | |
| // finalized the output file. | |
| VLOGF("before out_thread->join()..."); | |
| out_thread->join(); | |
| VLOGF("after out_thread->join()"); | |
| // Close the channels explicitly (just so we can more easily print messages | |
| // before and after vs. ~codec_client). | |
| VLOGF("before codec_client stop..."); | |
| codec_client_->Stop(); | |
| VLOGF("after codec_client stop."); | |
| codec_client_ = std::nullopt; | |
| // success | |
| return; | |
| } | |
| void use_video_decoder(Format format, UseVideoDecoderParams params) { | |
| VLOGF("use_video_decoder()"); | |
| auto video_decoder_runner = | |
| std::make_unique<VideoDecoderRunner>(std::move(format), std::move(params)); | |
| video_decoder_runner->Run(); | |
| // ~video_decoder_runner | |
| } | |
| } // namespace | |
| void use_h264_decoder(UseVideoDecoderParams params) { | |
| use_video_decoder(Format::kH264, std::move(params)); | |
| } | |
| void use_vp9_decoder(UseVideoDecoderParams params) { | |
| use_video_decoder(Format::kVp9, std::move(params)); | |
| } | |
| void use_mjpeg_decoder(UseVideoDecoderParams params) { | |
| use_video_decoder(Format::kMjpeg, std::move(params)); | |
| } |