blob: f18ac0ee36fac0357085f5d777b9e13b5636c56a [file] [log] [blame]
 // Copyright 2022 The Fuchsia Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "src/embedder/accessibility_bridge.h" #include #include #include #include #include #include #include #include "src/embedder/fuchsia_logger.h" #include "src/embedder/logging.h" #include "src/embedder/root_inspect_node.h" #include "src/embedder/standard_message_codec/encodable_value.h" #include "src/embedder/standard_message_codec/standard_message_codec.h" namespace embedder { namespace accessibility_matrix_utility { /// The FlutterTransformation (defined in embedder.h) can be considered a 3x3 matrix /// /// scaleX | skewY | pers0 /// skewX | scaleY | pers1 /// transX | transY | pers2 /// /// Casting this object to a 3x3 array greatly simplifies the logic for multiplication /// operations between these types constexpr size_t kMatrixSize = 3; using FlutterMatrix33 = std::array, kMatrixSize>; /// Ensure there is no padding and the casting will work properly static_assert(sizeof(FlutterMatrix33) == sizeof(FlutterTransformation)); /// A 1x3 Matrix struct Scalar { const auto& x() const { return data[0]; } const auto& y() const { return data[1]; } const auto& z() const { return data[2]; } std::array data; }; /// Initialize a 3x3 matrix from the Scalar's x, y, z values: /// x, 0, 0 /// 0, y, 0 /// 0, 0, z /// /// Then, cast the 3x3 matrix back into a FlutterTransformation FlutterTransformation CreateTransformFromScalar(const Scalar& scalar) { FlutterTransformation transform; memset(&transform, 0, sizeof(FlutterTransformation)); transform.scaleX = scalar.x(); transform.scaleY = scalar.y(); transform.pers2 = scalar.z(); return transform; } /// Multiply two 3x3 matrices by each other /// Return the result casted back into a FlutterTransformation FlutterTransformation MultiplyTransformations(const FlutterTransformation& left, const FlutterTransformation& right) { const FlutterMatrix33& a = *reinterpret_cast(&left); const FlutterMatrix33& b = *reinterpret_cast(&right); FlutterMatrix33 product; auto compute_cell = [&](size_t row, size_t col) { double result = 0.0; for (size_t i = 0; i < kMatrixSize; i++) { result += a[row][i] * b[i][col]; } return result; }; for (size_t row = 0; row < kMatrixSize; row++) { for (size_t col = 0; col < kMatrixSize; col++) { product[row][col] = compute_cell(row, col); } } return *reinterpret_cast(&product); } /// Multipy a 1x3 matrix with a 3x3 matrix Scalar MapScalarOnTransformation(const Scalar& scalar, const FlutterTransformation& transformation_in) { const FlutterMatrix33& transformation = *reinterpret_cast(&transformation_in); Scalar result; for (size_t row = 0; row < kMatrixSize; row++) { for (size_t col = 0; col < kMatrixSize; col++) { result.data[row] += transformation[row][col] * scalar.data[col]; } } return result; } } // namespace accessibility_matrix_utility namespace amu = accessibility_matrix_utility; namespace { /// Re-arrange the rectangle values so bottom & right /// values are largest void SortRectangle(FlutterRect& rect) { if (rect.left > rect.right) { std::swap(rect.left, rect.right); } if (rect.top > rect.bottom) { std::swap(rect.top, rect.bottom); } } /// Check if a rectangle contains a given point bool RectangleContains(const FlutterRect& rect, int32_t x, int32_t y) { return x >= rect.left && x < rect.right && y >= rect.top && y < rect.bottom; } /// Utility to simplify checking whether enum bitfields backed by an integer contain a /// particular enum value /// (e.g. the 'flags' or 'actions' fields on the FlutterSemanticsNode type) template struct EnumFlagSet { int32_t flags = 0; EnumFlagSet(T _flags) : flags(static_cast(_flags)) {} bool HasFlag(T flag) const { return (flags & static_cast(flag)) != 0; } }; static constexpr char kTreeDumpInspectRootName[] = "semantic_tree_root"; // Converts flutter semantic node flags to a string representation. std::string NodeFlagsToString(const FlutterSemanticsNode& node) { std::string output; EnumFlagSet flag_set(node.flags); if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasCheckedState)) { output += "kHasCheckedState|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasEnabledState)) { output += "kHasEnabledState|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasImplicitScrolling)) { output += "kHasImplicitScrolling|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasToggledState)) { output += "kHasToggledState|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsButton)) { output += "kIsButton|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsChecked)) { output += "kIsChecked|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsEnabled)) { output += "kIsEnabled|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocusable)) { output += "kIsFocusable|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocused)) { output += "kIsFocused|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsHeader)) { output += "kIsHeader|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsHidden)) { output += "kIsHidden|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsImage)) { output += "kIsImage|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsInMutuallyExclusiveGroup)) { output += "kIsInMutuallyExclusiveGroup|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsKeyboardKey)) { output += "kIsKeyboardKey|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsLink)) { output += "kIsLink|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsLiveRegion)) { output += "kIsLiveRegion|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsObscured)) { output += "kIsObscured|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly)) { output += "kIsReadOnly|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsSelected)) { output += "kIsSelected|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsSlider)) { output += "kIsSlider|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField)) { output += "kIsTextField|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsToggled)) { output += "kIsToggled|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagNamesRoute)) { output += "kNamesRoute|"; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagScopesRoute)) { output += "kScopesRoute|"; } return output; } // Converts flutter semantic node actions to a string representation. std::string NodeActionsToString(const FlutterSemanticsNode& node) { std::string output; EnumFlagSet action_set(node.actions); if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionCopy)) { output += "kCopy|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionCustomAction)) { output += "kCustomAction|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionCut)) { output += "kCut|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionDecrease)) { output += "kDecrease|"; } if (action_set.HasFlag( FlutterSemanticsAction::kFlutterSemanticsActionDidGainAccessibilityFocus)) { output += "kDidGainAccessibilityFocus|"; } if (action_set.HasFlag( FlutterSemanticsAction::kFlutterSemanticsActionDidLoseAccessibilityFocus)) { output += "kDidLoseAccessibilityFocus|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionDismiss)) { output += "kDismiss|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionIncrease)) { output += "kIncrease|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionLongPress)) { output += "kLongPress|"; } if (action_set.HasFlag( FlutterSemanticsAction::kFlutterSemanticsActionMoveCursorBackwardByCharacter)) { output += "kMoveCursorBackwardByCharacter|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionMoveCursorBackwardByWord)) { output += "kMoveCursorBackwardByWord|"; } if (action_set.HasFlag( FlutterSemanticsAction::kFlutterSemanticsActionMoveCursorForwardByCharacter)) { output += "kMoveCursorForwardByCharacter|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionMoveCursorForwardByWord)) { output += "kMoveCursorForwardByWord|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionPaste)) { output += "kPaste|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionScrollDown)) { output += "kScrollDown|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionScrollLeft)) { output += "kScrollLeft|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionScrollRight)) { output += "kScrollRight|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionScrollUp)) { output += "kScrollUp|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionSetSelection)) { output += "kSetSelection|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionSetText)) { output += "kSetText|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionShowOnScreen)) { output += "kShowOnScreen|"; } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionTap)) { output += "kTap|"; } return output; } /// Returns a string representation of the flutter semantic node absolute /// location. std::string NodeLocationToString(const FlutterRect& rect) { auto min_x = rect.left; auto min_y = rect.top; auto max_x = rect.right; auto max_y = rect.bottom; std::string location = "min(" + std::to_string(min_x) + ", " + std::to_string(min_y) + ") max(" + std::to_string(max_x) + ", " + std::to_string(max_y) + ")"; return location; } /// Returns a string representation of the node's different types of children. std::string NodeChildrenToString(const FlutterSemanticsNode& node) { std::stringstream output; if (node.child_count != 0) { output << "children in traversal order:["; for (size_t child = 0; child < node.child_count; child++) { output << node.children_in_traversal_order[child] << ", "; } output << "]\n"; output << "children in hit test order:["; for (size_t child = 0; child < node.child_count; child++) { output << node.children_in_hit_test_order[child] << ", "; } output << ']'; } return output.str(); } } // namespace AccessibilityBridge::AccessibilityBridge( SetSemanticsEnabledCallback set_semantics_enabled_callback, DispatchSemanticsActionCallback dispatch_semantics_action_callback, fuchsia::accessibility::semantics::SemanticsManagerHandle semantics_manager, fuchsia::ui::views::ViewRef view_ref, inspect::Node inspect_node) : set_semantics_enabled_callback_(std::move(set_semantics_enabled_callback)), dispatch_semantics_action_callback_(std::move(dispatch_semantics_action_callback)), binding_(this), fuchsia_semantics_manager_(semantics_manager.Bind()), atomic_updates_(std::make_shared>()), inspect_node_(std::move(inspect_node)) { fuchsia_semantics_manager_.set_error_handler([](zx_status_t status) { FX_LOGF(ERROR, kLogTag, "Flutter cannot connect to SemanticsManager with status: %s.", zx_status_get_string(status)); }); fuchsia_semantics_manager_->RegisterViewForSemantics(std::move(view_ref), binding_.NewBinding(), tree_ptr_.NewRequest()); // TODO(benbergkamp) // #if DEBUG // The first argument to |CreateLazyValues| is the name of the lazy node, and // will only be displayed if the callback used to generate the node's content // fails. Therefore, we use an error message for this node name. inspect_node_tree_dump_ = inspect_node_.CreateLazyValues("dump_fail", [this]() { inspect::Inspector inspector; if (auto it = nodes_.find(kRootNodeId); it == nodes_.end()) { inspector.GetRoot().CreateString("empty_tree", "this semantic tree is empty", &inspector); } else { FillInspectTree(kRootNodeId, /*current_level=*/1, inspector.GetRoot().CreateChild(kTreeDumpInspectRootName), &inspector); } return fpromise::make_ok_promise(std::move(inspector)); }); // #endif DEBUG } bool AccessibilityBridge::GetSemanticsEnabled() const { return semantics_enabled_; } void AccessibilityBridge::SetSemanticsEnabled(bool enabled) { semantics_enabled_ = enabled; if (!enabled) { nodes_.clear(); } } fuchsia::ui::gfx::BoundingBox AccessibilityBridge::GetNodeLocation( const FlutterSemanticsNode& node) const { fuchsia::ui::gfx::BoundingBox box; box.min.x = node.rect.left; box.min.y = node.rect.top; box.min.z = static_cast(node.elevation); box.max.x = node.rect.right; box.max.y = node.rect.bottom; box.max.z = static_cast(node.thickness); return box; } fuchsia::ui::gfx::mat4 AccessibilityBridge::GetNodeTransform( const FlutterSemanticsNode& node) const { return ConvertFlutterTransformToMat4(node.transform); } fuchsia::ui::gfx::mat4 AccessibilityBridge::ConvertFlutterTransformToMat4( const FlutterTransformation transform) const { fuchsia::ui::gfx::mat4 value; float* m = value.matrix.data(); memcpy(m, &transform, sizeof(transform)); return value; } fuchsia::accessibility::semantics::Attributes AccessibilityBridge::GetNodeAttributes( const FlutterSemanticsNode& node, size_t* added_size) const { fuchsia::accessibility::semantics::Attributes attributes; // TODO(MI4-2531): Don't truncate. attributes.set_label(std::string( node.label, std::min(strlen(node.label), fuchsia::accessibility::semantics::MAX_LABEL_SIZE))); *added_size += attributes.label().size(); // Truncate attributes.set_secondary_label(std::string( node.tooltip, std::min(strlen(node.tooltip), fuchsia::accessibility::semantics::MAX_LABEL_SIZE))); *added_size += attributes.secondary_label().size(); const EnumFlagSet flag_set(node.flags); if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsKeyboardKey)) { attributes.set_is_keyboard_key(true); } return attributes; } fuchsia::accessibility::semantics::States AccessibilityBridge::GetNodeStates( const FlutterSemanticsNode& node, size_t* additional_size) const { fuchsia::accessibility::semantics::States states; (*additional_size) += sizeof(fuchsia::accessibility::semantics::States); const EnumFlagSet node_flag_set(node.flags); // Set checked state. if (!node_flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasCheckedState)) { states.set_checked_state(fuchsia::accessibility::semantics::CheckedState::NONE); } else { states.set_checked_state( node_flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsChecked) ? fuchsia::accessibility::semantics::CheckedState::CHECKED : fuchsia::accessibility::semantics::CheckedState::UNCHECKED); } // Set selected state. states.set_selected(node_flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsSelected)); // Flutter's definition of a hidden node is different from Fuchsia, so it must // not be set here. // Set value. states.set_value(std::string( node.value, std::min(strlen(node.value), fuchsia::accessibility::semantics::MAX_VALUE_SIZE))); *additional_size += states.value().size(); // Set toggled state. if (node_flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasToggledState)) { states.set_toggled_state( node_flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsToggled) ? fuchsia::accessibility::semantics::ToggledState::ON : fuchsia::accessibility::semantics::ToggledState::OFF); } return states; } std::vector AccessibilityBridge::GetNodeActions( const FlutterSemanticsNode& node, size_t* additional_size) const { std::vector node_actions; EnumFlagSet action_set(node.actions); if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionTap)) { node_actions.push_back(fuchsia::accessibility::semantics::Action::DEFAULT); } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionLongPress)) { node_actions.push_back(fuchsia::accessibility::semantics::Action::SECONDARY); } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionShowOnScreen)) { node_actions.push_back(fuchsia::accessibility::semantics::Action::SHOW_ON_SCREEN); } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionIncrease)) { node_actions.push_back(fuchsia::accessibility::semantics::Action::INCREMENT); } if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionDecrease)) { node_actions.push_back(fuchsia::accessibility::semantics::Action::DECREMENT); } *additional_size += node_actions.size() * sizeof(fuchsia::accessibility::semantics::Action); return node_actions; } fuchsia::accessibility::semantics::Role AccessibilityBridge::GetNodeRole( const FlutterSemanticsNode& node) const { EnumFlagSet flag_set(node.flags); if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsButton)) { return fuchsia::accessibility::semantics::Role::BUTTON; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField)) { return fuchsia::accessibility::semantics::Role::TEXT_FIELD; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsLink)) { return fuchsia::accessibility::semantics::Role::LINK; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsSlider)) { return fuchsia::accessibility::semantics::Role::SLIDER; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsHeader)) { return fuchsia::accessibility::semantics::Role::HEADER; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsImage)) { return fuchsia::accessibility::semantics::Role::IMAGE; } EnumFlagSet action_set(node.actions); // If a flutter node supports the kIncrease or kDecrease actions, it can be // treated as a slider control by assistive technology. This is important // because users have special gestures to deal with sliders, and Fuchsia API // requires nodes that can receive this kind of action to be a slider control. if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionIncrease) || action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionDecrease)) { return fuchsia::accessibility::semantics::Role::SLIDER; } // If a flutter node has a checked state, then we assume it is either a // checkbox or a radio button. We distinguish between checkboxes and // radio buttons based on membership in a mutually exclusive group. if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasCheckedState)) { if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsInMutuallyExclusiveGroup)) { return fuchsia::accessibility::semantics::Role::RADIO_BUTTON; } else { return fuchsia::accessibility::semantics::Role::CHECK_BOX; } } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasToggledState)) { return fuchsia::accessibility::semantics::Role::TOGGLE_SWITCH; } return fuchsia::accessibility::semantics::Role::UNKNOWN; } std::unordered_set AccessibilityBridge::GetDescendants(int32_t node_id) const { std::unordered_set descendents; std::deque to_process = {node_id}; while (!to_process.empty()) { int32_t id = to_process.front(); to_process.pop_front(); descendents.emplace(id); auto it = nodes_.find(id); if (it != nodes_.end()) { const auto& node = it->second.data; for (size_t child = 0; child < node.child_count; child++) { int32_t child_id = node.children_in_hit_test_order[child]; if (descendents.find(child_id) == descendents.end()) { to_process.push_back(child_id); } else { // This indicates either a cycle or a child with multiple parents. // Flutter should never let this happen, but the engine API does not // explicitly forbid it right now. // TODO(http://fxbug.dev/75905): Crash flutter accessibility bridge // when a cycle in the tree is found. FX_LOGF(ERROR, kLogTag, "Semantics Node %d has already been listed as" " a child of another node, ignoring for parent %d.", child_id, id); } } } } return descendents; } // The only known usage of a negative number for a node ID is in the embedder // API as a sentinel value, which is not expected here. No valid producer of // nodes should give us a negative ID. static uint32_t FlutterIdToFuchsiaId(int32_t flutter_node_id) { FX_DCHECK(flutter_node_id >= 0); return static_cast(flutter_node_id); } void AccessibilityBridge::PruneUnreachableNodes(FuchsiaAtomicUpdate* atomic_update) { const auto& reachable_nodes = GetDescendants(kRootNodeId); auto iter = nodes_.begin(); while (iter != nodes_.end()) { int32_t id = iter->first; if (reachable_nodes.find(id) == reachable_nodes.end()) { atomic_update->AddNodeDeletion(FlutterIdToFuchsiaId(id)); iter = nodes_.erase(iter); } else { iter++; } } } // TODO(FIDL-718) - remove this, handle the error instead in something like // set_error_handler. static void PrintNodeSizeError(uint32_t node_id) { FX_LOGF(ERROR, kLogTag, "Semantics node with ID %d exceeded the maximum " "FIDL message size and may not be delivered to the accessibility " "manager service.", node_id); } void AccessibilityBridge::AddSemanticsUpdate(const FlutterSemanticsUpdate* update) { AddSemanticsNodeUpdates(update->nodes, update->nodes_count); // Note: Not handling actions, as custom semantics actions are not yet supported } void AccessibilityBridge::AddSemanticsNodeUpdates(const FlutterSemanticsNode* update, size_t count) { bool seen_root = false; for (unsigned i = 0; i < count; i++) { auto& update_ref = update[i]; // We handle root update separately in GetRootNodeUpdate. // TODO(chunhtai): remove this special case after we remove the inverse // view pixel ratio transformation in scenic view. // TODO(http://fxbug.dev/75908): Investigate flutter a11y bridge refactor // after removal of the inverse view pixel ratio transformation in scenic // view). if (update_ref.id == kRootNodeId) { root_flutter_semantics_node_ = *update; // copy seen_root = true; continue; } // "Normal case" where it's neither the root nor the terminal entry size_t this_node_size = sizeof(fuchsia::accessibility::semantics::Node); // Store the nodes for later hit testing and logging. nodes_[update_ref.id].data = update_ref; // copy fuchsia::accessibility::semantics::Node fuchsia_node; std::vector child_ids; // Send the nodes in traversal order, so the manager can figure out // traversal. for (size_t child_id = 0; child_id < update_ref.child_count; child_id++) { child_ids.push_back(FlutterIdToFuchsiaId(update_ref.children_in_traversal_order[child_id])); } // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in // the flutter accessibility bridge. fuchsia_node.set_node_id(update_ref.id) .set_role(GetNodeRole(update_ref)) .set_location(GetNodeLocation(update_ref)) .set_transform(GetNodeTransform(update_ref)) .set_attributes(GetNodeAttributes(update_ref, &this_node_size)) .set_states(GetNodeStates(update_ref, &this_node_size)) .set_actions(GetNodeActions(update_ref, &this_node_size)) .set_child_ids(child_ids); this_node_size += kNodeIdSize * update_ref.child_count; current_atomic_update_.AddNodeUpdate(std::move(fuchsia_node), this_node_size); } // Handled a batch of updates without ever getting a root node. FX_DCHECK(nodes_.find(kRootNodeId) != nodes_.end() || seen_root); // Handles root node update. if (seen_root || last_seen_view_pixel_ratio_ != next_pixel_ratio_) { last_seen_view_pixel_ratio_ = next_pixel_ratio_; size_t root_node_size; fuchsia::accessibility::semantics::Node root_update = GetRootNodeUpdate(root_node_size); current_atomic_update_.AddNodeUpdate(std::move(root_update), root_node_size); } PruneUnreachableNodes(¤t_atomic_update_); UpdateScreenRects(); atomic_updates_->push(std::move(current_atomic_update_)); if (atomic_updates_->size() == 1) { // There were no commits in the queue, so send this one. Apply(&atomic_updates_->front()); } current_atomic_update_.Clear(); } fuchsia::accessibility::semantics::Node AccessibilityBridge::GetRootNodeUpdate(size_t& node_size) { fuchsia::accessibility::semantics::Node root_fuchsia_node; std::vector child_ids; node_size = sizeof(fuchsia::accessibility::semantics::Node); for (size_t child = 0; child < root_flutter_semantics_node_.child_count; child++) { int32_t flutter_child_id = root_flutter_semantics_node_.children_in_traversal_order[child]; child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id)); } float inverse_view_pixel_ratio = 1.f / last_seen_view_pixel_ratio_; amu::Scalar scalar{.data = {inverse_view_pixel_ratio, inverse_view_pixel_ratio, 1.0}}; FlutterTransformation inverse_view_pixel_ratio_transform(amu::CreateTransformFromScalar(scalar)); const auto& result = amu::MultiplyTransformations(root_flutter_semantics_node_.transform, inverse_view_pixel_ratio_transform); nodes_[root_flutter_semantics_node_.id].data = root_flutter_semantics_node_; // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in // the flutter accessibility bridge. root_fuchsia_node.set_node_id(root_flutter_semantics_node_.id) .set_role(GetNodeRole(root_flutter_semantics_node_)) .set_location(GetNodeLocation(root_flutter_semantics_node_)) .set_transform(ConvertFlutterTransformToMat4(result)) .set_attributes(GetNodeAttributes(root_flutter_semantics_node_, &node_size)) .set_states(GetNodeStates(root_flutter_semantics_node_, &node_size)) .set_actions(GetNodeActions(root_flutter_semantics_node_, &node_size)) .set_child_ids(child_ids); node_size += kNodeIdSize * root_flutter_semantics_node_.child_count; return root_fuchsia_node; } namespace smc = standard_message_codec; void AccessibilityBridge::HandlePlatformMessage(const FlutterPlatformMessage* message) { const smc::StandardMessageCodec& standard_message_codec = smc::StandardMessageCodec::GetInstance(nullptr); std::unique_ptr decoded = standard_message_codec.DecodeMessage(message->message, message->message_size); smc::EncodableMap map = std::get(*decoded); std::string type = std::get(map.at(smc::EncodableValue("type"))); if (type == "announce") { smc::EncodableMap data_map = std::get(map.at(smc::EncodableValue("data"))); std::string text = std::get(data_map.at(smc::EncodableValue("message"))); RequestAnnounce(text); } } void AccessibilityBridge::RequestAnnounce(const std::string message) { fuchsia::accessibility::semantics::SemanticEvent semantic_event; fuchsia::accessibility::semantics::AnnounceEvent announce_event; announce_event.set_message(message); semantic_event.set_announce(std::move(announce_event)); tree_ptr_->SendSemanticEvent(std::move(semantic_event), []() {}); } void AccessibilityBridge::UpdateScreenRects() { std::unordered_set visited_nodes; // The embedder applies a special pixel ratio transform to the root of the // view, and the accessibility bridge applies the inverse of this transform // to the root node. However, this transform is not persisted in the flutter // representation of the root node, so we need to account for it explicitly // here. float inverse_view_pixel_ratio = 1.f / last_seen_view_pixel_ratio_; amu::Scalar scalar{.data = {inverse_view_pixel_ratio, inverse_view_pixel_ratio, 1.0}}; FlutterTransformation inverse_view_pixel_ratio_transform(amu::CreateTransformFromScalar(scalar)); UpdateScreenRects(kRootNodeId, inverse_view_pixel_ratio_transform, &visited_nodes); } void AccessibilityBridge::UpdateScreenRects(int32_t node_id, FlutterTransformation parent_transform, std::unordered_set* visited_nodes) { auto it = nodes_.find(node_id); if (it == nodes_.end()) { FX_LOG(ERROR, kLogTag, "UpdateScreenRects called on unknown node"); return; } auto& node = it->second; const auto& current_transform = amu::MultiplyTransformations(parent_transform, node.data.transform); const auto& rect = node.data.rect; amu::Scalar dst[2]{amu::MapScalarOnTransformation(amu::Scalar{.data = {rect.left, rect.top, 0}}, current_transform), amu::MapScalarOnTransformation( amu::Scalar{.data = {rect.right, rect.bottom, 0}}, current_transform)}; FlutterRect new_rect{ .left = dst[0].x(), .top = dst[0].y(), .right = dst[1].x(), .bottom = dst[1].y()}; SortRectangle(new_rect); node.screen_rect = new_rect; visited_nodes->emplace(node_id); for (uint32_t child = 0; child < node.data.child_count; child++) { int32_t child_id = node.data.children_in_hit_test_order[child]; if (visited_nodes->find(child_id) == visited_nodes->end()) { UpdateScreenRects(child_id, current_transform, visited_nodes); } } } std::optional AccessibilityBridge::GetFlutterSemanticsAction( fuchsia::accessibility::semantics::Action fuchsia_action, uint32_t node_id) { switch (fuchsia_action) { // The default action associated with the element. case fuchsia::accessibility::semantics::Action::DEFAULT: return FlutterSemanticsAction::kFlutterSemanticsActionTap; // The secondary action associated with the element. This may correspond to // a long press (touchscreens) or right click (mouse). case fuchsia::accessibility::semantics::Action::SECONDARY: return FlutterSemanticsAction::kFlutterSemanticsActionLongPress; // Set (input/non-accessibility) focus on this element. case fuchsia::accessibility::semantics::Action::SET_FOCUS: FX_LOGF(WARNING, kLogTag, "Unsupported action SET_FOCUS sent for " "accessibility node %d", node_id); return {}; // Set the element's value. case fuchsia::accessibility::semantics::Action::SET_VALUE: FX_LOGF(WARNING, kLogTag, "Unsupported action SET_VALUE sent for " "accessibility node %d", node_id); return {}; // Scroll node to make it visible. case fuchsia::accessibility::semantics::Action::SHOW_ON_SCREEN: return FlutterSemanticsAction::kFlutterSemanticsActionShowOnScreen; case fuchsia::accessibility::semantics::Action::INCREMENT: return FlutterSemanticsAction::kFlutterSemanticsActionIncrease; case fuchsia::accessibility::semantics::Action::DECREMENT: return FlutterSemanticsAction::kFlutterSemanticsActionDecrease; default: FX_LOGF(WARNING, kLogTag, "Unexpected action %d sent for " "accessibility node %d", static_cast(fuchsia_action), node_id); return {}; } } // |fuchsia::accessibility::semantics::SemanticListener| void AccessibilityBridge::OnAccessibilityActionRequested( uint32_t node_id, fuchsia::accessibility::semantics::Action action, fuchsia::accessibility::semantics::SemanticListener::OnAccessibilityActionRequestedCallback callback) { // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in // the flutter accessibility bridge. if (nodes_.find(node_id) == nodes_.end()) { FX_LOGF(ERROR, kLogTag, "Attempted to send accessibility action " "%d to unknown node id: %d", static_cast(action), node_id); callback(false); return; } std::optional flutter_action = GetFlutterSemanticsAction(action, node_id); if (!flutter_action.has_value()) { callback(false); return; } dispatch_semantics_action_callback_(static_cast(node_id), flutter_action.value()); callback(true); } // |fuchsia::accessibility::semantics::SemanticListener| void AccessibilityBridge::HitTest( fuchsia::math::PointF local_point, fuchsia::accessibility::semantics::SemanticListener::HitTestCallback callback) { auto hit_node_id = GetHitNode(kRootNodeId, local_point.x, local_point.y); FX_DCHECK(hit_node_id.has_value()); fuchsia::accessibility::semantics::Hit hit; // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in // the flutter accessibility bridge. hit.set_node_id(hit_node_id.value_or(kRootNodeId)); callback(std::move(hit)); } std::optional AccessibilityBridge::GetHitNode(int32_t node_id, float x, float y) { auto it = nodes_.find(node_id); if (it == nodes_.end()) { FX_LOGF(ERROR, kLogTag, "Attempted to hit test unknown node id: %d", node_id); return {}; } auto const& node = it->second; EnumFlagSet flag_set(node.data.flags); if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsHidden) || // !RectangleContains(node.screen_rect, x, y)) { return {}; } for (size_t child = 0; child < node.data.child_count; child++) { int32_t child_id = node.data.children_in_hit_test_order[child]; auto candidate = GetHitNode(child_id, x, y); if (candidate) { return candidate; } } if (IsFocusable(node.data)) { return node_id; } return {}; } bool AccessibilityBridge::IsFocusable(const FlutterSemanticsNode& node) const { EnumFlagSet flag_set(node.flags); if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagScopesRoute)) { return false; } if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocusable)) { return true; } // Always consider platform views focusable. if (node.platform_view_id > -1 /*TODO this is a hack, need to get the -1 value from proper channel*/) { return true; } // Always consider actionable nodes focusable. if (node.actions != 0) { return true; } // Consider text nodes focusable. return strlen(node.label) > 0 || strlen(node.value) > 0 || strlen(node.hint) > 0; } // |fuchsia::accessibility::semantics::SemanticListener| void AccessibilityBridge::OnSemanticsModeChanged(bool enabled, OnSemanticsModeChangedCallback callback) { set_semantics_enabled_callback_(enabled); } void AccessibilityBridge::SetPixelRatio(float ratio) { next_pixel_ratio_ = ratio; } // TODO(benbergkamp) // #if DEBUG void AccessibilityBridge::FillInspectTree(int32_t flutter_node_id, int32_t current_level, inspect::Node inspect_node, inspect::Inspector* inspector) const { const auto it = nodes_.find(flutter_node_id); if (it == nodes_.end()) { inspect_node.CreateString( "missing_child", "This node has a parent in the semantic tree but has no value", inspector); inspector->emplace(std::move(inspect_node)); return; } const auto& semantic_node = it->second; const auto& data = semantic_node.data; inspect_node.CreateInt("id", data.id, inspector); // Even with an empty label, we still want to create the property to // explicetly show that it is empty. inspect_node.CreateString("label", data.label, inspector); if (strlen(data.hint) > 0) { inspect_node.CreateString("hint", data.hint, inspector); } if (strlen(data.value) > 0) { inspect_node.CreateString("value", data.value, inspector); } if (strlen(data.increased_value) > 0) { inspect_node.CreateString("increased_value", data.increased_value, inspector); } if (strlen(data.decreased_value) > 0) { inspect_node.CreateString("decreased_value", data.decreased_value, inspector); } if (data.text_direction) { inspect_node.CreateString("text_direction", data.text_direction == 1 ? "RTL" : "LTR", inspector); } if (data.flags) { inspect_node.CreateString("flags", NodeFlagsToString(data), inspector); } if (data.actions) { inspect_node.CreateString("actions", NodeActionsToString(data), inspector); } inspect_node.CreateString("location", NodeLocationToString(semantic_node.screen_rect), inspector); if (data.child_count > 0) { inspect_node.CreateString("children", NodeChildrenToString(data), inspector); } inspect_node.CreateInt("current_level", current_level, inspector); for (size_t child = 0; child < semantic_node.data.child_count; child++) { int32_t flutter_child_id = semantic_node.data.children_in_traversal_order[child]; const auto inspect_name = "node_" + std::to_string(flutter_child_id); FillInspectTree(flutter_child_id, current_level + 1, inspect_node.CreateChild(inspect_name), inspector); } inspector->emplace(std::move(inspect_node)); } // #endif DEBUG void AccessibilityBridge::Apply(FuchsiaAtomicUpdate* atomic_update) { size_t begin = 0; auto it = atomic_update->deletions.begin(); // Process up to kMaxDeletionsPerUpdate deletions at a time. while (it != atomic_update->deletions.end()) { std::vector to_delete; size_t end = std::min(atomic_update->deletions.size() - begin, kMaxDeletionsPerUpdate); std::copy(std::make_move_iterator(it), std::make_move_iterator(it + end), std::back_inserter(to_delete)); tree_ptr_->DeleteSemanticNodes(std::move(to_delete)); begin = end; it += end; } std::vector to_update; size_t current_size = 0; for (auto& node_and_size : atomic_update->updates) { if (current_size + node_and_size.second > kMaxMessageSize) { tree_ptr_->UpdateSemanticNodes(std::move(to_update)); current_size = 0; to_update.clear(); } current_size += node_and_size.second; to_update.push_back(std::move(node_and_size.first)); } if (!to_update.empty()) { tree_ptr_->UpdateSemanticNodes(std::move(to_update)); } // Commit this update and subsequent ones; for flow control wait for a // response between each commit. tree_ptr_->CommitUpdates( [this, atomic_updates = std::weak_ptr>(atomic_updates_)]() { auto atomic_updates_ptr = atomic_updates.lock(); if (!atomic_updates_ptr) { // The queue no longer exists, which means that is no longer // necessary. return; } // Removes the update that just went through. atomic_updates_ptr->pop(); if (!atomic_updates_ptr->empty()) { Apply(&atomic_updates_ptr->front()); } }); atomic_update->Clear(); } void AccessibilityBridge::FuchsiaAtomicUpdate::AddNodeUpdate( fuchsia::accessibility::semantics::Node node, size_t size) { if (size > kMaxMessageSize) { // TODO(MI4-2531, FIDL-718): Remove this // This is defensive. If, despite our best efforts, we ended up with a node // that is larger than the max fidl size, we send no updates. PrintNodeSizeError(node.node_id()); return; } updates.emplace_back(std::move(node), size); } void AccessibilityBridge::FuchsiaAtomicUpdate::AddNodeDeletion(uint32_t id) { deletions.push_back(id); } void AccessibilityBridge::FuchsiaAtomicUpdate::Clear() { updates.clear(); deletions.clear(); } } // namespace embedder