blob: 4f85b69ac5e67ce82a6a72f4d1c4decec058bd20 [file] [log] [blame] [edit]
// Copyright 2023 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/camera/bin/camera-gym/buffer_collage_flatland.h"
#include <fuchsia/images/cpp/fidl.h>
#include <fuchsia/sysmem/cpp/fidl.h>
#include <fuchsia/ui/composition/cpp/fidl.h>
#include <fuchsia/ui/views/cpp/fidl.h>
#include <lib/async-loop/default.h>
#include <lib/async/cpp/task.h>
#include <lib/async/cpp/wait.h>
#include <lib/async/default.h>
#include <lib/async/dispatcher.h>
#include <lib/fpromise/bridge.h>
#include <lib/fpromise/promise.h>
#include <lib/syslog/cpp/macros.h>
#include <lib/trace/event.h>
#include <lib/ui/scenic/cpp/buffer_collection_import_export_tokens.h>
#include <lib/ui/scenic/cpp/view_creation_tokens.h>
#include <lib/ui/scenic/cpp/view_identity.h>
#include <lib/ui/scenic/cpp/view_token_pair.h>
#include <lib/zx/eventpair.h>
#include <zircon/errors.h>
#include <zircon/status.h>
#include <zircon/syscalls/object.h>
#include <zircon/types.h>
#include <cmath>
#include <memory>
#include <optional>
#include "src/camera/bin/camera-gym/screen_util.h"
namespace camera_flatland {
using Command = fuchsia::camera::gym::Command;
using SetDescriptionCommand = fuchsia::camera::gym::SetDescriptionCommand;
using CaptureFrameCommand = fuchsia::camera::gym::CaptureFrameCommand;
constexpr uint32_t kViewRequestTimeoutMs = 5000;
BufferCollageFlatland::BufferCollageFlatland() : loop_(&kAsyncLoopConfigNoAttachToCurrentThread) {
SetStopOnError(sysmem_allocator_);
}
BufferCollageFlatland::~BufferCollageFlatland() {
zx_status_t status =
async::PostTask(loop_.dispatcher(), fit::bind_member(this, &BufferCollageFlatland::Stop));
ZX_ASSERT(status == ZX_OK);
loop_.JoinThreads();
}
fpromise::result<std::unique_ptr<BufferCollageFlatland>, zx_status_t> BufferCollageFlatland::Create(
std::unique_ptr<simple_present::FlatlandConnection> flatland_connection,
fuchsia::ui::composition::AllocatorHandle flatland_allocator,
fuchsia::element::GraphicalPresenterHandle graphical_presenter,
fuchsia::sysmem2::AllocatorHandle sysmem_allocator, fit::closure stop_callback) {
auto collage = std::unique_ptr<BufferCollageFlatland>(new BufferCollageFlatland);
collage->start_time_ = zx::clock::get_monotonic();
collage->stop_callback_ = std::move(stop_callback);
collage->flatland_connection_ = std::move(flatland_connection);
zx_status_t status =
collage->flatland_allocator_.Bind(std::move(flatland_allocator), collage->loop_.dispatcher());
if (status != ZX_OK) {
FX_PLOGS(ERROR, status);
return fpromise::error(status);
}
status = collage->graphical_presenter_.Bind(std::move(graphical_presenter),
collage->loop_.dispatcher());
if (status != ZX_OK) {
FX_PLOGS(ERROR, status);
return fpromise::error(status);
}
status =
collage->sysmem_allocator_.Bind(std::move(sysmem_allocator), collage->loop_.dispatcher());
if (status != ZX_OK) {
FX_PLOGS(ERROR, status);
return fpromise::error(status);
}
collage->flatland_ = collage->flatland_connection_->flatland();
// Start a thread and begin processing messages.
status = collage->loop_.StartThread("BufferCollage Loop");
if (status != ZX_OK) {
FX_PLOGS(ERROR, status);
return fpromise::error(status);
}
return fpromise::ok(std::move(collage));
}
// Called by stream_cycler when a new camera stream is available.
fpromise::promise<uint32_t> BufferCollageFlatland::AddCollection(
fuchsia::sysmem2::BufferCollectionTokenHandle token, fuchsia::images2::ImageFormat image_format,
std::string description) {
TRACE_DURATION("camera", "BufferCollageFlatland::AddCollection");
ZX_ASSERT(image_format.has_size());
ZX_ASSERT(image_format.size().width > 0);
ZX_ASSERT(image_format.size().height > 0);
ZX_ASSERT(image_format.has_bytes_per_row());
ZX_ASSERT(image_format.bytes_per_row() > 0);
ZX_ASSERT(flatland_connection_);
fpromise::bridge<uint32_t> task_bridge;
fit::closure add_collection = [this, token = std::move(token),
image_format = std::move(image_format), description,
result = std::move(task_bridge.completer)]() mutable {
auto collection_id = next_collection_id_++;
FX_LOGS(DEBUG) << "Adding collection with ID " << collection_id << ".";
ZX_ASSERT(collection_views_.find(collection_id) == collection_views_.end());
auto& view = collection_views_[collection_id];
auto name = "Collection (" + std::to_string(collection_id) + ")";
SetRemoveCollectionViewOnError(view.buffer_collection, collection_id, name);
view.image_format = fidl::Clone(image_format);
// Bind token to |loop_| and duplicate the token.
fuchsia::sysmem2::BufferCollectionTokenPtr token_ptr;
SetStopOnError(token_ptr, "BufferCollectionToken");
zx_status_t status = token_ptr.Bind(std::move(token), loop_.dispatcher());
if (status != ZX_OK) {
FX_LOGS(ERROR) << "Failed to bind token. " << status;
Stop();
result.complete_error();
return;
}
fuchsia::sysmem2::BufferCollectionTokenHandle scenic_token;
fuchsia::sysmem2::BufferCollectionTokenDuplicateRequest dup_request;
dup_request.set_rights_attenuation_mask(ZX_RIGHT_SAME_RIGHTS);
dup_request.set_token_request(scenic_token.NewRequest());
token_ptr->Duplicate(std::move(dup_request));
fuchsia::sysmem2::AllocatorBindSharedCollectionRequest bind_shared_request;
bind_shared_request.set_token(std::move(token_ptr));
bind_shared_request.set_buffer_collection_request(
view.buffer_collection.NewRequest(loop_.dispatcher()));
sysmem_allocator_->BindSharedCollection(std::move(bind_shared_request));
// Sync the collection and create flatland image using token provided by camera stream.
view.buffer_collection->Sync([this, collection_id, token = std::move(scenic_token),
result = std::move(result)](
fuchsia::sysmem2::Node_Sync_Result sync_result) mutable {
ZX_ASSERT(collection_views_.find(collection_id) != collection_views_.end());
auto& view = collection_views_[collection_id];
// Set minimal constraints then wait for buffer allocation.
fuchsia::sysmem2::BufferCollectionSetConstraintsRequest set_constraints_request;
set_constraints_request.mutable_constraints()->mutable_usage()->set_none(
fuchsia::sysmem2::NONE_USAGE);
view.buffer_collection->SetConstraints(std::move(set_constraints_request));
view.ref_pair = allocation::BufferCollectionImportExportTokens::New();
fuchsia::ui::composition::RegisterBufferCollectionArgs args = {};
args.set_export_token(std::move(view.ref_pair.export_token));
args.set_buffer_collection_token2(std::move(token));
flatland_allocator_->RegisterBufferCollection(
std::move(args), [this, collection_id, result = std::move(result)](
fuchsia::ui::composition::Allocator_RegisterBufferCollection_Result
register_result) mutable {
if (register_result.is_err()) {
FX_LOGS(ERROR) << "Failed to register buffers.";
Stop();
result.complete_error();
return;
}
ZX_ASSERT(collection_views_.find(collection_id) != collection_views_.end());
auto& view = collection_views_[collection_id];
view.buffer_collection->WaitForAllBuffersAllocated(
[this, collection_id, result = std::move(result)](
fuchsia::sysmem2::BufferCollection_WaitForAllBuffersAllocated_Result
wait_result) mutable {
if (wait_result.is_framework_err()) {
FX_PLOGS(ERROR, fidl::ToUnderlying(wait_result.framework_err()))
<< "Failed to allocate buffers (framework err).";
Stop();
result.complete_error();
return;
}
if (wait_result.is_err()) {
FX_PLOGS(ERROR, static_cast<uint32_t>(wait_result.err()))
<< "Failed to allocate buffers (err).";
Stop();
result.complete_error();
return;
}
ZX_ASSERT(collection_views_.find(collection_id) != collection_views_.end());
auto& view = collection_views_[collection_id];
fuchsia::ui::composition::TransformId transform_id{.value = next_transform_id++};
view.transform_id = transform_id;
uint32_t buffer_count = static_cast<uint32_t>(
wait_result.response().buffer_collection_info().buffers().size());
view.buffer_count = buffer_count;
// Rearranges layout to add the new view. Content doesn't get updated until
// PostShowBuffer is called.
UpdateLayout();
for (uint32_t buffer_id = 0; buffer_id < buffer_count; ++buffer_id) {
fuchsia::ui::composition::ImageProperties image_properties = {};
image_properties.set_size(
{view.image_format.size().width, view.image_format.size().height});
fuchsia::ui::composition::BufferCollectionImportToken import_token_copy;
view.ref_pair.import_token.value.duplicate(ZX_RIGHT_SAME_RIGHTS,
&import_token_copy.value);
// TODO: Should we try to reuse content_id?
fuchsia::ui::composition::ContentId content_id{.value = next_content_id++};
view.buffer_id_to_content_id[buffer_id] = content_id;
flatland_->CreateImage({content_id}, std::move(import_token_copy), buffer_id,
std::move(image_properties));
}
FX_LOGS(DEBUG) << "Successfully added collection " << collection_id << ".";
result.complete_ok(collection_id);
});
});
});
};
async::PostTask(loop_.dispatcher(), std::move(add_collection));
return task_bridge.consumer.promise();
}
void BufferCollageFlatland::RemoveCollection(uint32_t collection_id) {
TRACE_DURATION("camera", "BufferCollage::RemoveCollection");
if (async_get_default_dispatcher() != loop_.dispatcher()) {
// Marshal the task to our own thread if called from elsewhere.
auto nonce = TRACE_NONCE();
TRACE_FLOW_BEGIN("camera", "post_remove_collection", nonce);
async::PostTask(loop_.dispatcher(), [this, collection_id, nonce] {
TRACE_DURATION("camera", "BufferCollage::RemoveCollection.task");
TRACE_FLOW_END("camera", "post_remove_collection", nonce);
RemoveCollection(collection_id);
});
return;
}
auto it = collection_views_.find(collection_id);
if (it == collection_views_.end()) {
FX_LOGS(INFO) << "Skipping RemoveCollection for already-removed collection ID "
<< collection_id;
return;
}
auto& view = it->second;
if (view.buffer_collection.is_bound()) {
view.buffer_collection->Release();
}
for (uint32_t buffer_id = 0; buffer_id < view.buffer_count; ++buffer_id) {
auto buffer_info_it = view.buffer_id_to_content_id.find(buffer_id);
if (buffer_info_it != view.buffer_id_to_content_id.end()) {
auto& content_id = view.buffer_id_to_content_id[buffer_id];
flatland_->ReleaseImage(content_id);
}
}
flatland_->RemoveChild(kRootTransformId, view.transform_id);
flatland_->ReleaseTransform(view.transform_id);
if (view.buffer_collection.is_bound()) {
view.buffer_collection->Release();
}
collection_views_.erase(it);
UpdateLayout();
}
void BufferCollageFlatland::PostShowBuffer(uint32_t collection_id, uint32_t buffer_index,
zx::eventpair* release_fence,
std::optional<fuchsia::math::RectF> subregion) {
auto nonce = TRACE_NONCE();
TRACE_DURATION("camera", "BufferCollage::PostShowBuffer");
TRACE_FLOW_BEGIN("camera", "post_show_buffer", nonce);
async::PostTask(loop_.dispatcher(), [=, release_fence = release_fence]() mutable {
TRACE_DURATION("camera", "BufferCollage::PostShowBuffer.task");
TRACE_FLOW_END("camera", "post_show_buffer", nonce);
ShowBuffer(collection_id, buffer_index, release_fence, subregion);
});
}
void BufferCollageFlatland::Stop() {
flatland_->Clear();
collection_views_.clear();
loop_.Quit();
if (stop_callback_) {
stop_callback_();
stop_callback_ = nullptr;
}
sysmem_allocator_ = nullptr;
}
template <typename T>
void BufferCollageFlatland::SetStopOnError(fidl::InterfacePtr<T>& p, std::string name) {
p.set_error_handler([this, name, &p](zx_status_t status) {
FX_PLOGS(ERROR, status) << name << " disconnected unexpectedly.";
p = nullptr;
Stop();
});
}
template <typename T>
void BufferCollageFlatland::SetRemoveCollectionViewOnError(fidl::InterfacePtr<T>& p,
uint32_t view_id, std::string name) {
p.set_error_handler([this, view_id, name](zx_status_t status) {
FX_PLOGS(WARNING, status) << name << " view_id=" << view_id << " disconnected unexpectedly.";
if (collection_views_.find(view_id) == collection_views_.end()) {
FX_LOGS(INFO) << name << " view_id=" << view_id << " already removed.";
return;
}
RemoveCollection(view_id);
});
}
void BufferCollageFlatland::ShowBuffer(uint32_t collection_id, uint32_t buffer_index,
zx::eventpair* release_fence,
std::optional<fuchsia::math::RectF> subregion) {
TRACE_DURATION("camera", "BufferCollage::ShowBuffer");
auto it = collection_views_.find(collection_id);
if (it == collection_views_.end()) {
FX_LOGS(ERROR) << "Invalid collection ID " << collection_id << " out of "
<< collection_views_.size();
Stop();
return;
}
auto& view = it->second;
if (buffer_index >= view.buffer_count) {
FX_LOGS(ERROR) << "Invalid buffer index " << buffer_index << ".";
Stop();
return;
}
auto caller_event = MakeEventBridge(loop_.dispatcher(), release_fence);
if (caller_event.is_error()) {
FX_PLOGS(ERROR, caller_event.error());
Stop();
return;
}
TRACE_FLOW_BEGIN("gfx", "flatlant_set_content", buffer_index);
flatland_->SetContent(view.transform_id, view.buffer_id_to_content_id[buffer_index]);
std::vector<zx::event> scenic_fences;
scenic_fences.push_back(caller_event.take_value());
fuchsia::ui::composition::PresentArgs present_args;
present_args.set_release_fences(std::move(scenic_fences));
present_args.set_unsquashable(false);
flatland_connection_->Present(std::move(present_args), [](auto) {});
}
void BufferCollageFlatland::UpdateLayout() {
FX_DCHECK(loop_.dispatcher() == async_get_default_dispatcher());
ZX_ASSERT(width_ > 0);
ZX_ASSERT(height_ > 0);
auto [rows, cols] = screen_util::GetGridSize(static_cast<uint32_t>(collection_views_.size()));
constexpr float kPadding = 4.0f;
// cell_width and cell_height is the maximum size allowed to fit a single camera stream view in
// the grid.
float cell_width = static_cast<float>(width_) / static_cast<float>(cols) - kPadding;
float cell_height = static_cast<float>(height_) / static_cast<float>(rows) - kPadding;
uint32_t index = 0;
for (auto& [id, view] : collection_views_) {
float display_width = static_cast<float>(view.image_format.display_rect().width);
float display_height = static_cast<float>(view.image_format.display_rect().height);
if (!view.view_created) {
flatland_->CreateTransform(view.transform_id);
flatland_->AddChild(kRootTransformId, view.transform_id);
view.view_created = true;
}
// scale display width and height to fix inside the cell boundary.
auto scale = screen_util::Scale(display_width, display_height, cell_width, cell_height);
auto [x_center, y_center] =
screen_util::GetCenter(index++, static_cast<uint32_t>(collection_views_.size()));
// Find center then shift up by half of scaled width.
auto translated_x =
static_cast<int32_t>(static_cast<float>(width_) * x_center) - (scale * display_width * 0.5);
// Find center then shift left by half of scaled height.
auto translated_y = static_cast<int32_t>(static_cast<float>(height_) * y_center) -
(scale * display_height * 0.5);
flatland_->SetScale(view.transform_id, {scale, scale});
flatland_->SetTranslation(view.transform_id, {static_cast<int32_t>(translated_x),
static_cast<int32_t>(translated_y)});
}
}
void BufferCollageFlatland::SetupBaseView() {
ZX_ASSERT(flatland_);
flatland_->CreateTransform(kRootTransformId);
flatland_->SetRootTransform(kRootTransformId);
}
void BufferCollageFlatland::PresentView() {
async::PostTask(loop_.dispatcher(), [this]() mutable {
parent_watcher_.set_error_handler([this](zx_status_t status) {
FX_LOGS(ERROR) << "Error from fuchsia::ui::composition::ParentViewportWatcher: "
<< zx_status_get_string(status);
Stop();
});
auto view_identity = scenic::NewViewIdentityOnCreation();
auto [view_token, parent_viewport_token] = scenic::ViewCreationTokenPair::New();
flatland_->CreateView2(std::move(view_token), std::move(view_identity),
/* protocols = */ {}, parent_watcher_.NewRequest());
parent_watcher_->GetLayout([this](auto layout_info) {
width_ = layout_info.logical_size().width;
height_ = layout_info.logical_size().height;
FX_LOGS(DEBUG) << "Received layout info: w=" << width_ << ", h=" << height_;
SetupBaseView();
flatland_connection_->Present({}, [](auto) {});
});
fuchsia::element::ViewSpec view_spec;
view_spec.set_viewport_creation_token(std::move(parent_viewport_token));
graphical_presenter_->PresentView(
std::move(view_spec), nullptr, nullptr,
[](fuchsia::element::GraphicalPresenter_PresentView_Result result) {
if (result.is_err())
FX_LOGS(ERROR) << "PresentView failed";
});
});
}
} // namespace camera_flatland