blob: 99262aef320275d91f696c03c4ee4677a6021aba [file] [log] [blame]
// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <fuchsia/accessibility/semantics/cpp/fidl.h>
#include <fuchsia/sys/cpp/fidl.h>
#include <fuchsia/ui/gfx/cpp/fidl.h>
#include <lib/fidl/cpp/binding_set.h>
#include <lib/sys/inspect/cpp/component.h>
#include <zircon/types.h>
#include <memory>
#include <optional>
#include <queue>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "src/embedder/engine/embedder.h"
namespace embedder {
/// Accessibility bridge.
/// This class intermediates accessibility-related calls between Fuchsia and
/// Flutter, translating and relaying requests between Flutter's
/// platform-agnostic accessibility APIs and Fuchsia's APIs and behaviour.
/// This bridge performs the following functions, among others:
/// * Translates Flutter's semantics node updates to events Fuchsia requires
/// (e.g. Flutter only sends updates for changed nodes, but Fuchsia requires
/// the entire flattened subtree to be sent when a node changes.
class AccessibilityBridge : public fuchsia::accessibility::semantics::SemanticListener {
using SetSemanticsEnabledCallback = std::function<void(bool)>;
using DispatchSemanticsActionCallback = std::function<void(int32_t, FlutterSemanticsAction)>;
// TODO(MI4-2531, FIDL-718): Remove this. We shouldn't be worried about
// batching messages at this level.
// FIDL may encode a C++ struct as larger than the sizeof the C++ struct.
// This is to make sure we don't send updates that are too large.
static constexpr uint32_t kMaxMessageSize = ZX_CHANNEL_MAX_MSG_BYTES / 2;
static_assert(fuchsia::accessibility::semantics::MAX_LABEL_SIZE < kMaxMessageSize - 1);
/// Flutter uses signed 32 bit integers for node IDs, while Fuchsia uses
/// unsigned 32 bit integers. A change in the size on either one would break
/// casts and size tracking logic in the implementation.
static constexpr size_t kNodeIdSize = sizeof(FlutterSemanticsNode::id);
static_assert(kNodeIdSize == sizeof(fuchsia::accessibility::semantics::Node().node_id()),
"FlutterSemanticsNode::id and "
"fuchsia::accessibility::semantics::Node::node_id differ in size.");
/// Maximum number of node ids to be deleted through the Semantics API.
static constexpr size_t kMaxDeletionsPerUpdate = kMaxMessageSize / kNodeIdSize;
/// set_semantics_enabled_callback is the callback to be run following an
/// invocation of the OnSemanticsModeChanged FIDL interface from Fuchsia.
/// dispatch_semantics_action_callback is the callback to be run following an
/// invocation of the OnAccessibilityActionRequested FIDL interface from Fuchsia.
/// semantics_manager is the Fuchsia semantics manager component
/// view_ref is the accessibility view reference initialized in
/// inspect_node is a child node of Fuchsia's ComponentInspector
AccessibilityBridge(SetSemanticsEnabledCallback set_semantics_enabled_callback,
DispatchSemanticsActionCallback dispatch_semantics_action_callback,
fuchsia::accessibility::semantics::SemanticsManagerHandle semantics_manager,
fuchsia::ui::views::ViewRef view_ref, inspect::Node inspect_node);
/// Disable Copy and Assignment
AccessibilityBridge(const AccessibilityBridge&) = delete;
AccessibilityBridge& operator=(const AccessibilityBridge&) = delete;
/// Returns true if accessible navigation is enabled.
bool GetSemanticsEnabled() const;
/// Enables Flutter accessibility navigation features.
/// Once enabled, any semantics updates in the Flutter application will
/// trigger |FuchsiaAccessibility::DispatchAccessibilityEvent| callbacks
/// to send events back to the Fuchsia SemanticsManager.
void SetSemanticsEnabled(bool enabled);
/// Handle FlutterSemanticsNode from embedder API
void AddSemanticsUpdate(const FlutterSemanticsUpdate* update);
/// Handle platform message from the flutter engine
void HandlePlatformMessage(const FlutterPlatformMessage* message);
/// Requests a message announcement from the accessibility TTS system.
void RequestAnnounce(const std::string message);
/// Notifies the bridge of a 'hover move' touch exploration event.
zx_status_t OnHoverMove(double x, double y);
/// |fuchsia::accessibility::semantics::SemanticListener|
void HitTest(
fuchsia::math::PointF local_point,
fuchsia::accessibility::semantics::SemanticListener::HitTestCallback callback) override;
/// |fuchsia::accessibility::semantics::SemanticListener|
void OnAccessibilityActionRequested(
uint32_t node_id, fuchsia::accessibility::semantics::Action action,
callback) override;
/// Set the internal pixel ratio
/// This is the factor used to convert between a view’s logical
/// coordinate space (assigned by scenic) and its allocated buffer size
void SetPixelRatio(float ratio);
/// Fuchsia's default root semantic node id.
static constexpr int32_t kRootNodeId = 0;
/// Represents an atomic semantic update to Fuchsia, which can contain deletes
/// and updates of semantic nodes.
/// An atomic update is a set of operations that take a semantic tree in a
/// valid state to another valid state. Please check the semantics FIDL
/// documentation for details.
struct FuchsiaAtomicUpdate {
FuchsiaAtomicUpdate() = default;
~FuchsiaAtomicUpdate() = default;
FuchsiaAtomicUpdate(FuchsiaAtomicUpdate&&) = default;
/// Adds a node to be updated. |size| contains the
/// size in bytes of the node to be updated.
void AddNodeUpdate(fuchsia::accessibility::semantics::Node node, size_t size);
/// Adds a node to be deleted.
void AddNodeDeletion(uint32_t id);
/// clear the updates and deletions vectors
void Clear();
std::vector<std::pair<fuchsia::accessibility::semantics::Node, size_t>> updates;
std::vector<uint32_t> deletions;
/// Holds a flutter semantic node and some extra info.
/// In particular, it adds a screen_rect field to FlutterSemanticsNode.
struct SemanticsNode {
FlutterSemanticsNode data;
FlutterRect screen_rect;
/// Adds a semantics node update to the buffer of node updates to apply.
void AddSemanticsNodeUpdates(const FlutterSemanticsNode* update, size_t count);
fuchsia::accessibility::semantics::Node GetRootNodeUpdate(size_t& node_size);
/// Derives the BoundingBox of a Flutter semantics node from its
/// rect and elevation.
fuchsia::ui::gfx::BoundingBox GetNodeLocation(const FlutterSemanticsNode& node) const;
/// Gets mat4 transformation from a Flutter semantics node.
fuchsia::ui::gfx::mat4 GetNodeTransform(const FlutterSemanticsNode& node) const;
/// Converts a Flutter semantics node's transformation to a mat4.
fuchsia::ui::gfx::mat4 ConvertFlutterTransformToMat4(const FlutterTransformation transform) const;
/// Derives the attributes for a Fuchsia semantics node from a Flutter
/// semantics node.
fuchsia::accessibility::semantics::Attributes GetNodeAttributes(const FlutterSemanticsNode& node,
size_t* added_size) const;
/// Derives the states for a Fuchsia semantics node from a Flutter semantics
/// node.
fuchsia::accessibility::semantics::States GetNodeStates(const FlutterSemanticsNode& node,
size_t* additional_size) const;
/// Derives the set of supported actions for a Fuchsia semantics node from
/// a Flutter semantics node.
std::vector<fuchsia::accessibility::semantics::Action> GetNodeActions(
const FlutterSemanticsNode& node, size_t* additional_size) const;
/// Derives the role for a Fuchsia semantics node from a Flutter
/// semantics node.
fuchsia::accessibility::semantics::Role GetNodeRole(const FlutterSemanticsNode& node) const;
/// Gets the set of reachable descendants from the given node id.
std::unordered_set<int32_t> GetDescendants(int32_t node_id) const;
/// Removes internal references to any dangling nodes from previous
/// updates, and adds the nodes to be deleted to the current |atomic_update|.
/// The node ids to be deleted are only collected at this point, and will be
/// committed in the next call to |Apply()|.
void PruneUnreachableNodes(FuchsiaAtomicUpdate* atomic_update);
/// Updates the on-screen positions of accessibility elements,
/// starting from the root element with an identity matrix.
/// This should be called from Update.
void UpdateScreenRects();
/// Updates the on-screen positions of accessibility elements, starting
/// from node_id and using the specified transform.
/// Update calls this via UpdateScreenRects().
void UpdateScreenRects(int32_t node_id, FlutterTransformation parent_transform,
std::unordered_set<int32_t>* visited_nodes);
/// Traverses the semantics tree to find the node_id hit by the given x,y
/// point.
/// Assumes that SemanticsNode::screen_rect is up to date.
std::optional<int32_t> GetHitNode(int32_t node_id, float x, float y);
/// Returns whether the node is considered focusable.
bool IsFocusable(const FlutterSemanticsNode& node) const;
/// Converts a fuchsia::accessibility::semantics::Action to a
/// flutterSemanticsAction.
/// The node_id parameter is used for printing warnings about unsupported
/// action types.
std::optional<FlutterSemanticsAction> GetFlutterSemanticsAction(
fuchsia::accessibility::semantics::Action fuchsia_action, uint32_t node_id);
/// Applies the updates and deletions in |atomic_update|, sending them via
/// |tree_ptr|.
void Apply(FuchsiaAtomicUpdate* atomic_update);
/// |fuchsia::accessibility::semantics::SemanticListener|
void OnSemanticsModeChanged(bool enabled, OnSemanticsModeChangedCallback callback) override;
// TODO(benbergkamp)
// #if DEBUG
/// Fills the inspect tree with debug information about the semantic tree.
void FillInspectTree(int32_t flutter_node_id, int32_t current_level, inspect::Node inspect_node,
inspect::Inspector* inspector) const;
// #endif // DEBUG
SetSemanticsEnabledCallback set_semantics_enabled_callback_;
DispatchSemanticsActionCallback dispatch_semantics_action_callback_;
FlutterSemanticsNode root_flutter_semantics_node_;
/// The pixel ratio used in the most recent AddSemanticsNodeUpdates function call
float last_seen_view_pixel_ratio_ = 1.f;
/// The pixel ratio that will be used in the next function call to AddSemanticsNodeUpdates
/// Need to differenciate from last_seen, because there is special logic for handling a
/// change in the pixel ratio
float next_pixel_ratio_ = 1.f;
fidl::Binding<fuchsia::accessibility::semantics::SemanticListener> binding_;
fuchsia::accessibility::semantics::SemanticsManagerPtr fuchsia_semantics_manager_;
fuchsia::accessibility::semantics::SemanticTreePtr tree_ptr_;
/// This is the cache of all nodes we've sent to Fuchsia's SemanticsManager.
/// Assists with pruning unreachable nodes and hit testing.
std::unordered_map<int32_t, SemanticsNode> nodes_;
bool semantics_enabled_;
/// Queue of atomic updates to be sent to Fuchsia.
std::shared_ptr<std::queue<FuchsiaAtomicUpdate>> atomic_updates_;
FuchsiaAtomicUpdate current_atomic_update_;
/// Node to publish inspect data.
inspect::Node inspect_node_;
// TODO(benbergkamp)
// #if DEBUG
/// Inspect node to store a dump of the semantic tree. Note that this only gets
/// computed if requested, so it does not use memory to store the dump unless
/// an explicit request is made.
inspect::LazyNode inspect_node_tree_dump_;
// #endif // DEBUG
} // namespace embedder