| // Copyright 2022 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "src/embedder/accessibility_bridge.h" |
| |
| #include <lib/inspect/cpp/inspector.h> |
| #include <lib/syslog/global.h> |
| #include <lib/zx/process.h> |
| #include <zircon/status.h> |
| #include <zircon/types.h> |
| |
| #include <algorithm> |
| #include <deque> |
| |
| #include "src/embedder/fuchsia_logger.h" |
| #include "src/embedder/logging.h" |
| #include "src/embedder/root_inspect_node.h" |
| #include "src/embedder/standard_message_codec/encodable_value.h" |
| #include "src/embedder/standard_message_codec/standard_message_codec.h" |
| |
| namespace embedder { |
| namespace accessibility_matrix_utility { |
| |
| /// The FlutterTransformation (defined in embedder.h) can be considered a 3x3 matrix |
| /// |
| /// scaleX | skewY | pers0 |
| /// skewX | scaleY | pers1 |
| /// transX | transY | pers2 |
| /// |
| /// Casting this object to a 3x3 array greatly simplifies the logic for multiplication |
| /// operations between these types |
| constexpr size_t kMatrixSize = 3; |
| using FlutterMatrix33 = std::array<std::array<double, kMatrixSize>, kMatrixSize>; |
| /// Ensure there is no padding and the casting will work properly |
| static_assert(sizeof(FlutterMatrix33) == sizeof(FlutterTransformation)); |
| |
| /// A 1x3 Matrix |
| struct Scalar { |
| const auto& x() const { return data[0]; } |
| const auto& y() const { return data[1]; } |
| const auto& z() const { return data[2]; } |
| std::array<double, kMatrixSize> data; |
| }; |
| |
| /// Initialize a 3x3 matrix from the Scalar's x, y, z values: |
| /// x, 0, 0 |
| /// 0, y, 0 |
| /// 0, 0, z |
| /// |
| /// Then, cast the 3x3 matrix back into a FlutterTransformation |
| FlutterTransformation CreateTransformFromScalar(const Scalar& scalar) { |
| FlutterTransformation transform; |
| memset(&transform, 0, sizeof(FlutterTransformation)); |
| transform.scaleX = scalar.x(); |
| transform.scaleY = scalar.y(); |
| transform.pers2 = scalar.z(); |
| return transform; |
| } |
| |
| /// Multiply two 3x3 matrices by each other |
| /// Return the result casted back into a FlutterTransformation |
| FlutterTransformation MultiplyTransformations(const FlutterTransformation& left, |
| const FlutterTransformation& right) { |
| const FlutterMatrix33& a = *reinterpret_cast<const FlutterMatrix33*>(&left); |
| const FlutterMatrix33& b = *reinterpret_cast<const FlutterMatrix33*>(&right); |
| |
| FlutterMatrix33 product; |
| |
| auto compute_cell = [&](size_t row, size_t col) { |
| double result = 0.0; |
| for (size_t i = 0; i < kMatrixSize; i++) { |
| result += a[row][i] * b[i][col]; |
| } |
| return result; |
| }; |
| |
| for (size_t row = 0; row < kMatrixSize; row++) { |
| for (size_t col = 0; col < kMatrixSize; col++) { |
| product[row][col] = compute_cell(row, col); |
| } |
| } |
| return *reinterpret_cast<FlutterTransformation*>(&product); |
| } |
| |
| /// Multipy a 1x3 matrix with a 3x3 matrix |
| Scalar MapScalarOnTransformation(const Scalar& scalar, |
| const FlutterTransformation& transformation_in) { |
| const FlutterMatrix33& transformation = |
| *reinterpret_cast<const FlutterMatrix33*>(&transformation_in); |
| |
| Scalar result; |
| |
| for (size_t row = 0; row < kMatrixSize; row++) { |
| for (size_t col = 0; col < kMatrixSize; col++) { |
| result.data[row] += transformation[row][col] * scalar.data[col]; |
| } |
| } |
| return result; |
| } |
| } // namespace accessibility_matrix_utility |
| namespace amu = accessibility_matrix_utility; |
| |
| namespace { |
| |
| /// Re-arrange the rectangle values so bottom & right |
| /// values are largest |
| void SortRectangle(FlutterRect& rect) { |
| if (rect.left > rect.right) { |
| std::swap(rect.left, rect.right); |
| } |
| if (rect.top > rect.bottom) { |
| std::swap(rect.top, rect.bottom); |
| } |
| } |
| |
| /// Check if a rectangle contains a given point |
| bool RectangleContains(const FlutterRect& rect, int32_t x, int32_t y) { |
| return x >= rect.left && x < rect.right && y >= rect.top && y < rect.bottom; |
| } |
| |
| /// Utility to simplify checking whether enum bitfields backed by an integer contain a |
| /// particular enum value |
| /// (e.g. the 'flags' or 'actions' fields on the FlutterSemanticsNode type) |
| template <typename T> |
| struct EnumFlagSet { |
| int32_t flags = 0; |
| EnumFlagSet(T _flags) : flags(static_cast<int32_t>(_flags)) {} |
| bool HasFlag(T flag) const { return (flags & static_cast<int32_t>(flag)) != 0; } |
| }; |
| |
| static constexpr char kTreeDumpInspectRootName[] = "semantic_tree_root"; |
| |
| // Converts flutter semantic node flags to a string representation. |
| std::string NodeFlagsToString(const FlutterSemanticsNode& node) { |
| std::string output; |
| EnumFlagSet<FlutterSemanticsFlag> flag_set(node.flags); |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasCheckedState)) { |
| output += "kHasCheckedState|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasEnabledState)) { |
| output += "kHasEnabledState|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasImplicitScrolling)) { |
| output += "kHasImplicitScrolling|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasToggledState)) { |
| output += "kHasToggledState|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsButton)) { |
| output += "kIsButton|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsChecked)) { |
| output += "kIsChecked|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsEnabled)) { |
| output += "kIsEnabled|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocusable)) { |
| output += "kIsFocusable|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocused)) { |
| output += "kIsFocused|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsHeader)) { |
| output += "kIsHeader|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsHidden)) { |
| output += "kIsHidden|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsImage)) { |
| output += "kIsImage|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsInMutuallyExclusiveGroup)) { |
| output += "kIsInMutuallyExclusiveGroup|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsKeyboardKey)) { |
| output += "kIsKeyboardKey|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsLink)) { |
| output += "kIsLink|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsLiveRegion)) { |
| output += "kIsLiveRegion|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsObscured)) { |
| output += "kIsObscured|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsReadOnly)) { |
| output += "kIsReadOnly|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsSelected)) { |
| output += "kIsSelected|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsSlider)) { |
| output += "kIsSlider|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField)) { |
| output += "kIsTextField|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsToggled)) { |
| output += "kIsToggled|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagNamesRoute)) { |
| output += "kNamesRoute|"; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagScopesRoute)) { |
| output += "kScopesRoute|"; |
| } |
| |
| return output; |
| } |
| |
| // Converts flutter semantic node actions to a string representation. |
| std::string NodeActionsToString(const FlutterSemanticsNode& node) { |
| std::string output; |
| |
| EnumFlagSet<FlutterSemanticsAction> action_set(node.actions); |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionCopy)) { |
| output += "kCopy|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionCustomAction)) { |
| output += "kCustomAction|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionCut)) { |
| output += "kCut|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionDecrease)) { |
| output += "kDecrease|"; |
| } |
| if (action_set.HasFlag( |
| FlutterSemanticsAction::kFlutterSemanticsActionDidGainAccessibilityFocus)) { |
| output += "kDidGainAccessibilityFocus|"; |
| } |
| if (action_set.HasFlag( |
| FlutterSemanticsAction::kFlutterSemanticsActionDidLoseAccessibilityFocus)) { |
| output += "kDidLoseAccessibilityFocus|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionDismiss)) { |
| output += "kDismiss|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionIncrease)) { |
| output += "kIncrease|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionLongPress)) { |
| output += "kLongPress|"; |
| } |
| if (action_set.HasFlag( |
| FlutterSemanticsAction::kFlutterSemanticsActionMoveCursorBackwardByCharacter)) { |
| output += "kMoveCursorBackwardByCharacter|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionMoveCursorBackwardByWord)) { |
| output += "kMoveCursorBackwardByWord|"; |
| } |
| if (action_set.HasFlag( |
| FlutterSemanticsAction::kFlutterSemanticsActionMoveCursorForwardByCharacter)) { |
| output += "kMoveCursorForwardByCharacter|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionMoveCursorForwardByWord)) { |
| output += "kMoveCursorForwardByWord|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionPaste)) { |
| output += "kPaste|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionScrollDown)) { |
| output += "kScrollDown|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionScrollLeft)) { |
| output += "kScrollLeft|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionScrollRight)) { |
| output += "kScrollRight|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionScrollUp)) { |
| output += "kScrollUp|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionSetSelection)) { |
| output += "kSetSelection|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionSetText)) { |
| output += "kSetText|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionShowOnScreen)) { |
| output += "kShowOnScreen|"; |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionTap)) { |
| output += "kTap|"; |
| } |
| |
| return output; |
| } |
| |
| /// Returns a string representation of the flutter semantic node absolute |
| /// location. |
| std::string NodeLocationToString(const FlutterRect& rect) { |
| auto min_x = rect.left; |
| auto min_y = rect.top; |
| auto max_x = rect.right; |
| auto max_y = rect.bottom; |
| std::string location = "min(" + std::to_string(min_x) + ", " + std::to_string(min_y) + ") max(" + |
| std::to_string(max_x) + ", " + std::to_string(max_y) + ")"; |
| |
| return location; |
| } |
| |
| /// Returns a string representation of the node's different types of children. |
| std::string NodeChildrenToString(const FlutterSemanticsNode& node) { |
| std::stringstream output; |
| if (node.child_count != 0) { |
| output << "children in traversal order:["; |
| for (size_t child = 0; child < node.child_count; child++) { |
| output << node.children_in_traversal_order[child] << ", "; |
| } |
| output << "]\n"; |
| output << "children in hit test order:["; |
| for (size_t child = 0; child < node.child_count; child++) { |
| output << node.children_in_hit_test_order[child] << ", "; |
| } |
| output << ']'; |
| } |
| return output.str(); |
| } |
| |
| } // namespace |
| |
| AccessibilityBridge::AccessibilityBridge( |
| SetSemanticsEnabledCallback set_semantics_enabled_callback, |
| DispatchSemanticsActionCallback dispatch_semantics_action_callback, |
| fuchsia::accessibility::semantics::SemanticsManagerHandle semantics_manager, |
| fuchsia::ui::views::ViewRef view_ref, inspect::Node inspect_node) |
| : set_semantics_enabled_callback_(std::move(set_semantics_enabled_callback)), |
| dispatch_semantics_action_callback_(std::move(dispatch_semantics_action_callback)), |
| binding_(this), |
| fuchsia_semantics_manager_(semantics_manager.Bind()), |
| atomic_updates_(std::make_shared<std::queue<FuchsiaAtomicUpdate>>()), |
| inspect_node_(std::move(inspect_node)) { |
| fuchsia_semantics_manager_.set_error_handler([](zx_status_t status) { |
| FX_LOGF(ERROR, kLogTag, "Flutter cannot connect to SemanticsManager with status: %s.", |
| zx_status_get_string(status)); |
| }); |
| |
| fuchsia_semantics_manager_->RegisterViewForSemantics(std::move(view_ref), binding_.NewBinding(), |
| tree_ptr_.NewRequest()); |
| // TODO(benbergkamp) |
| // #if DEBUG |
| // The first argument to |CreateLazyValues| is the name of the lazy node, and |
| // will only be displayed if the callback used to generate the node's content |
| // fails. Therefore, we use an error message for this node name. |
| inspect_node_tree_dump_ = inspect_node_.CreateLazyValues("dump_fail", [this]() { |
| inspect::Inspector inspector; |
| if (auto it = nodes_.find(kRootNodeId); it == nodes_.end()) { |
| inspector.GetRoot().CreateString("empty_tree", "this semantic tree is empty", &inspector); |
| } else { |
| FillInspectTree(kRootNodeId, /*current_level=*/1, |
| inspector.GetRoot().CreateChild(kTreeDumpInspectRootName), &inspector); |
| } |
| return fpromise::make_ok_promise(std::move(inspector)); |
| }); |
| // #endif DEBUG |
| } |
| |
| bool AccessibilityBridge::GetSemanticsEnabled() const { return semantics_enabled_; } |
| |
| void AccessibilityBridge::SetSemanticsEnabled(bool enabled) { |
| semantics_enabled_ = enabled; |
| if (!enabled) { |
| nodes_.clear(); |
| } |
| } |
| |
| fuchsia::ui::gfx::BoundingBox AccessibilityBridge::GetNodeLocation( |
| const FlutterSemanticsNode& node) const { |
| fuchsia::ui::gfx::BoundingBox box; |
| box.min.x = node.rect.left; |
| box.min.y = node.rect.top; |
| box.min.z = static_cast<float>(node.elevation); |
| box.max.x = node.rect.right; |
| box.max.y = node.rect.bottom; |
| box.max.z = static_cast<float>(node.thickness); |
| return box; |
| } |
| |
| fuchsia::ui::gfx::mat4 AccessibilityBridge::GetNodeTransform( |
| const FlutterSemanticsNode& node) const { |
| return ConvertFlutterTransformToMat4(node.transform); |
| } |
| |
| fuchsia::ui::gfx::mat4 AccessibilityBridge::ConvertFlutterTransformToMat4( |
| const FlutterTransformation transform) const { |
| fuchsia::ui::gfx::mat4 value; |
| float* m = value.matrix.data(); |
| memcpy(m, &transform, sizeof(transform)); |
| return value; |
| } |
| |
| fuchsia::accessibility::semantics::Attributes AccessibilityBridge::GetNodeAttributes( |
| const FlutterSemanticsNode& node, size_t* added_size) const { |
| fuchsia::accessibility::semantics::Attributes attributes; |
| |
| // TODO(MI4-2531): Don't truncate. |
| attributes.set_label(std::string( |
| node.label, std::min(strlen(node.label), fuchsia::accessibility::semantics::MAX_LABEL_SIZE))); |
| *added_size += attributes.label().size(); |
| |
| // Truncate |
| attributes.set_secondary_label(std::string( |
| node.tooltip, |
| std::min(strlen(node.tooltip), fuchsia::accessibility::semantics::MAX_LABEL_SIZE))); |
| *added_size += attributes.secondary_label().size(); |
| |
| const EnumFlagSet<FlutterSemanticsFlag> flag_set(node.flags); |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsKeyboardKey)) { |
| attributes.set_is_keyboard_key(true); |
| } |
| return attributes; |
| } |
| |
| fuchsia::accessibility::semantics::States AccessibilityBridge::GetNodeStates( |
| const FlutterSemanticsNode& node, size_t* additional_size) const { |
| fuchsia::accessibility::semantics::States states; |
| (*additional_size) += sizeof(fuchsia::accessibility::semantics::States); |
| |
| const EnumFlagSet<FlutterSemanticsFlag> node_flag_set(node.flags); |
| // Set checked state. |
| if (!node_flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasCheckedState)) { |
| states.set_checked_state(fuchsia::accessibility::semantics::CheckedState::NONE); |
| } else { |
| states.set_checked_state( |
| node_flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsChecked) |
| ? fuchsia::accessibility::semantics::CheckedState::CHECKED |
| : fuchsia::accessibility::semantics::CheckedState::UNCHECKED); |
| } |
| |
| // Set selected state. |
| states.set_selected(node_flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsSelected)); |
| |
| // Flutter's definition of a hidden node is different from Fuchsia, so it must |
| // not be set here. |
| |
| // Set value. |
| states.set_value(std::string( |
| node.value, std::min(strlen(node.value), fuchsia::accessibility::semantics::MAX_VALUE_SIZE))); |
| *additional_size += states.value().size(); |
| |
| // Set toggled state. |
| if (node_flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasToggledState)) { |
| states.set_toggled_state( |
| node_flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsToggled) |
| ? fuchsia::accessibility::semantics::ToggledState::ON |
| : fuchsia::accessibility::semantics::ToggledState::OFF); |
| } |
| |
| return states; |
| } |
| |
| std::vector<fuchsia::accessibility::semantics::Action> AccessibilityBridge::GetNodeActions( |
| const FlutterSemanticsNode& node, size_t* additional_size) const { |
| std::vector<fuchsia::accessibility::semantics::Action> node_actions; |
| |
| EnumFlagSet<FlutterSemanticsAction> action_set(node.actions); |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionTap)) { |
| node_actions.push_back(fuchsia::accessibility::semantics::Action::DEFAULT); |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionLongPress)) { |
| node_actions.push_back(fuchsia::accessibility::semantics::Action::SECONDARY); |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionShowOnScreen)) { |
| node_actions.push_back(fuchsia::accessibility::semantics::Action::SHOW_ON_SCREEN); |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionIncrease)) { |
| node_actions.push_back(fuchsia::accessibility::semantics::Action::INCREMENT); |
| } |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionDecrease)) { |
| node_actions.push_back(fuchsia::accessibility::semantics::Action::DECREMENT); |
| } |
| |
| *additional_size += node_actions.size() * sizeof(fuchsia::accessibility::semantics::Action); |
| return node_actions; |
| } |
| |
| fuchsia::accessibility::semantics::Role AccessibilityBridge::GetNodeRole( |
| const FlutterSemanticsNode& node) const { |
| EnumFlagSet<FlutterSemanticsFlag> flag_set(node.flags); |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsButton)) { |
| return fuchsia::accessibility::semantics::Role::BUTTON; |
| } |
| |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsTextField)) { |
| return fuchsia::accessibility::semantics::Role::TEXT_FIELD; |
| } |
| |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsLink)) { |
| return fuchsia::accessibility::semantics::Role::LINK; |
| } |
| |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsSlider)) { |
| return fuchsia::accessibility::semantics::Role::SLIDER; |
| } |
| |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsHeader)) { |
| return fuchsia::accessibility::semantics::Role::HEADER; |
| } |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsImage)) { |
| return fuchsia::accessibility::semantics::Role::IMAGE; |
| } |
| |
| EnumFlagSet<FlutterSemanticsAction> action_set(node.actions); |
| // If a flutter node supports the kIncrease or kDecrease actions, it can be |
| // treated as a slider control by assistive technology. This is important |
| // because users have special gestures to deal with sliders, and Fuchsia API |
| // requires nodes that can receive this kind of action to be a slider control. |
| if (action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionIncrease) || |
| action_set.HasFlag(FlutterSemanticsAction::kFlutterSemanticsActionDecrease)) { |
| return fuchsia::accessibility::semantics::Role::SLIDER; |
| } |
| |
| // If a flutter node has a checked state, then we assume it is either a |
| // checkbox or a radio button. We distinguish between checkboxes and |
| // radio buttons based on membership in a mutually exclusive group. |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasCheckedState)) { |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsInMutuallyExclusiveGroup)) { |
| return fuchsia::accessibility::semantics::Role::RADIO_BUTTON; |
| } else { |
| return fuchsia::accessibility::semantics::Role::CHECK_BOX; |
| } |
| } |
| |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagHasToggledState)) { |
| return fuchsia::accessibility::semantics::Role::TOGGLE_SWITCH; |
| } |
| return fuchsia::accessibility::semantics::Role::UNKNOWN; |
| } |
| |
| std::unordered_set<int32_t> AccessibilityBridge::GetDescendants(int32_t node_id) const { |
| std::unordered_set<int32_t> descendents; |
| std::deque<int32_t> to_process = {node_id}; |
| while (!to_process.empty()) { |
| int32_t id = to_process.front(); |
| to_process.pop_front(); |
| descendents.emplace(id); |
| |
| auto it = nodes_.find(id); |
| if (it != nodes_.end()) { |
| const auto& node = it->second.data; |
| for (size_t child = 0; child < node.child_count; child++) { |
| int32_t child_id = node.children_in_hit_test_order[child]; |
| if (descendents.find(child_id) == descendents.end()) { |
| to_process.push_back(child_id); |
| } else { |
| // This indicates either a cycle or a child with multiple parents. |
| // Flutter should never let this happen, but the engine API does not |
| // explicitly forbid it right now. |
| // TODO(http://fxbug.dev/75905): Crash flutter accessibility bridge |
| // when a cycle in the tree is found. |
| FX_LOGF(ERROR, kLogTag, |
| "Semantics Node %d has already been listed as" |
| " a child of another node, ignoring for parent %d.", |
| child_id, id); |
| } |
| } |
| } |
| } |
| return descendents; |
| } |
| |
| // The only known usage of a negative number for a node ID is in the embedder |
| // API as a sentinel value, which is not expected here. No valid producer of |
| // nodes should give us a negative ID. |
| static uint32_t FlutterIdToFuchsiaId(int32_t flutter_node_id) { |
| FX_DCHECK(flutter_node_id >= 0); |
| return static_cast<uint32_t>(flutter_node_id); |
| } |
| |
| void AccessibilityBridge::PruneUnreachableNodes(FuchsiaAtomicUpdate* atomic_update) { |
| const auto& reachable_nodes = GetDescendants(kRootNodeId); |
| auto iter = nodes_.begin(); |
| while (iter != nodes_.end()) { |
| int32_t id = iter->first; |
| if (reachable_nodes.find(id) == reachable_nodes.end()) { |
| atomic_update->AddNodeDeletion(FlutterIdToFuchsiaId(id)); |
| iter = nodes_.erase(iter); |
| } else { |
| iter++; |
| } |
| } |
| } |
| |
| // TODO(FIDL-718) - remove this, handle the error instead in something like |
| // set_error_handler. |
| static void PrintNodeSizeError(uint32_t node_id) { |
| FX_LOGF(ERROR, kLogTag, |
| "Semantics node with ID %d exceeded the maximum " |
| "FIDL message size and may not be delivered to the accessibility " |
| "manager service.", |
| node_id); |
| } |
| |
| void AccessibilityBridge::AddSemanticsUpdate(const FlutterSemanticsUpdate* update) { |
| AddSemanticsNodeUpdates(update->nodes, update->nodes_count); |
| // Note: Not handling actions, as custom semantics actions are not yet supported |
| } |
| |
| void AccessibilityBridge::AddSemanticsNodeUpdates(const FlutterSemanticsNode* update, |
| size_t count) { |
| bool seen_root = false; |
| for (unsigned i = 0; i < count; i++) { |
| auto& update_ref = update[i]; |
| |
| // We handle root update separately in GetRootNodeUpdate. |
| // TODO(chunhtai): remove this special case after we remove the inverse |
| // view pixel ratio transformation in scenic view. |
| // TODO(http://fxbug.dev/75908): Investigate flutter a11y bridge refactor |
| // after removal of the inverse view pixel ratio transformation in scenic |
| // view). |
| if (update_ref.id == kRootNodeId) { |
| root_flutter_semantics_node_ = *update; // copy |
| seen_root = true; |
| continue; |
| } |
| |
| // "Normal case" where it's neither the root nor the terminal entry |
| size_t this_node_size = sizeof(fuchsia::accessibility::semantics::Node); |
| |
| // Store the nodes for later hit testing and logging. |
| nodes_[update_ref.id].data = update_ref; // copy |
| fuchsia::accessibility::semantics::Node fuchsia_node; |
| std::vector<uint32_t> child_ids; |
| |
| // Send the nodes in traversal order, so the manager can figure out |
| // traversal. |
| for (size_t child_id = 0; child_id < update_ref.child_count; child_id++) { |
| child_ids.push_back(FlutterIdToFuchsiaId(update_ref.children_in_traversal_order[child_id])); |
| } |
| |
| // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in |
| // the flutter accessibility bridge. |
| fuchsia_node.set_node_id(update_ref.id) |
| .set_role(GetNodeRole(update_ref)) |
| .set_location(GetNodeLocation(update_ref)) |
| .set_transform(GetNodeTransform(update_ref)) |
| .set_attributes(GetNodeAttributes(update_ref, &this_node_size)) |
| .set_states(GetNodeStates(update_ref, &this_node_size)) |
| .set_actions(GetNodeActions(update_ref, &this_node_size)) |
| .set_child_ids(child_ids); |
| this_node_size += kNodeIdSize * update_ref.child_count; |
| current_atomic_update_.AddNodeUpdate(std::move(fuchsia_node), this_node_size); |
| } |
| |
| // Handled a batch of updates without ever getting a root node. |
| FX_DCHECK(nodes_.find(kRootNodeId) != nodes_.end() || seen_root); |
| |
| // Handles root node update. |
| if (seen_root || last_seen_view_pixel_ratio_ != next_pixel_ratio_) { |
| last_seen_view_pixel_ratio_ = next_pixel_ratio_; |
| size_t root_node_size; |
| fuchsia::accessibility::semantics::Node root_update = GetRootNodeUpdate(root_node_size); |
| current_atomic_update_.AddNodeUpdate(std::move(root_update), root_node_size); |
| } |
| |
| PruneUnreachableNodes(¤t_atomic_update_); |
| UpdateScreenRects(); |
| |
| atomic_updates_->push(std::move(current_atomic_update_)); |
| if (atomic_updates_->size() == 1) { |
| // There were no commits in the queue, so send this one. |
| Apply(&atomic_updates_->front()); |
| } |
| current_atomic_update_.Clear(); |
| } |
| |
| fuchsia::accessibility::semantics::Node AccessibilityBridge::GetRootNodeUpdate(size_t& node_size) { |
| fuchsia::accessibility::semantics::Node root_fuchsia_node; |
| std::vector<uint32_t> child_ids; |
| node_size = sizeof(fuchsia::accessibility::semantics::Node); |
| for (size_t child = 0; child < root_flutter_semantics_node_.child_count; child++) { |
| int32_t flutter_child_id = root_flutter_semantics_node_.children_in_traversal_order[child]; |
| child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id)); |
| } |
| |
| float inverse_view_pixel_ratio = 1.f / last_seen_view_pixel_ratio_; |
| amu::Scalar scalar{.data = {inverse_view_pixel_ratio, inverse_view_pixel_ratio, 1.0}}; |
| FlutterTransformation inverse_view_pixel_ratio_transform(amu::CreateTransformFromScalar(scalar)); |
| |
| const auto& result = amu::MultiplyTransformations(root_flutter_semantics_node_.transform, |
| inverse_view_pixel_ratio_transform); |
| nodes_[root_flutter_semantics_node_.id].data = root_flutter_semantics_node_; |
| |
| // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in |
| // the flutter accessibility bridge. |
| root_fuchsia_node.set_node_id(root_flutter_semantics_node_.id) |
| .set_role(GetNodeRole(root_flutter_semantics_node_)) |
| .set_location(GetNodeLocation(root_flutter_semantics_node_)) |
| .set_transform(ConvertFlutterTransformToMat4(result)) |
| .set_attributes(GetNodeAttributes(root_flutter_semantics_node_, &node_size)) |
| .set_states(GetNodeStates(root_flutter_semantics_node_, &node_size)) |
| .set_actions(GetNodeActions(root_flutter_semantics_node_, &node_size)) |
| .set_child_ids(child_ids); |
| node_size += kNodeIdSize * root_flutter_semantics_node_.child_count; |
| return root_fuchsia_node; |
| } |
| |
| namespace smc = standard_message_codec; |
| |
| void AccessibilityBridge::HandlePlatformMessage(const FlutterPlatformMessage* message) { |
| const smc::StandardMessageCodec& standard_message_codec = |
| smc::StandardMessageCodec::GetInstance(nullptr); |
| std::unique_ptr<smc::EncodableValue> decoded = |
| standard_message_codec.DecodeMessage(message->message, message->message_size); |
| |
| smc::EncodableMap map = std::get<smc::EncodableMap>(*decoded); |
| std::string type = std::get<std::string>(map.at(smc::EncodableValue("type"))); |
| if (type == "announce") { |
| smc::EncodableMap data_map = std::get<smc::EncodableMap>(map.at(smc::EncodableValue("data"))); |
| std::string text = std::get<std::string>(data_map.at(smc::EncodableValue("message"))); |
| RequestAnnounce(text); |
| } |
| } |
| |
| void AccessibilityBridge::RequestAnnounce(const std::string message) { |
| fuchsia::accessibility::semantics::SemanticEvent semantic_event; |
| fuchsia::accessibility::semantics::AnnounceEvent announce_event; |
| announce_event.set_message(message); |
| semantic_event.set_announce(std::move(announce_event)); |
| |
| tree_ptr_->SendSemanticEvent(std::move(semantic_event), []() {}); |
| } |
| |
| void AccessibilityBridge::UpdateScreenRects() { |
| std::unordered_set<int32_t> visited_nodes; |
| |
| // The embedder applies a special pixel ratio transform to the root of the |
| // view, and the accessibility bridge applies the inverse of this transform |
| // to the root node. However, this transform is not persisted in the flutter |
| // representation of the root node, so we need to account for it explicitly |
| // here. |
| float inverse_view_pixel_ratio = 1.f / last_seen_view_pixel_ratio_; |
| amu::Scalar scalar{.data = {inverse_view_pixel_ratio, inverse_view_pixel_ratio, 1.0}}; |
| FlutterTransformation inverse_view_pixel_ratio_transform(amu::CreateTransformFromScalar(scalar)); |
| |
| UpdateScreenRects(kRootNodeId, inverse_view_pixel_ratio_transform, &visited_nodes); |
| } |
| |
| void AccessibilityBridge::UpdateScreenRects(int32_t node_id, FlutterTransformation parent_transform, |
| std::unordered_set<int32_t>* visited_nodes) { |
| auto it = nodes_.find(node_id); |
| if (it == nodes_.end()) { |
| FX_LOG(ERROR, kLogTag, "UpdateScreenRects called on unknown node"); |
| return; |
| } |
| auto& node = it->second; |
| const auto& current_transform = |
| amu::MultiplyTransformations(parent_transform, node.data.transform); |
| |
| const auto& rect = node.data.rect; |
| amu::Scalar dst[2]{amu::MapScalarOnTransformation(amu::Scalar{.data = {rect.left, rect.top, 0}}, |
| current_transform), |
| amu::MapScalarOnTransformation( |
| amu::Scalar{.data = {rect.right, rect.bottom, 0}}, current_transform)}; |
| |
| FlutterRect new_rect{ |
| .left = dst[0].x(), .top = dst[0].y(), .right = dst[1].x(), .bottom = dst[1].y()}; |
| SortRectangle(new_rect); |
| node.screen_rect = new_rect; |
| |
| visited_nodes->emplace(node_id); |
| |
| for (uint32_t child = 0; child < node.data.child_count; child++) { |
| int32_t child_id = node.data.children_in_hit_test_order[child]; |
| if (visited_nodes->find(child_id) == visited_nodes->end()) { |
| UpdateScreenRects(child_id, current_transform, visited_nodes); |
| } |
| } |
| } |
| |
| std::optional<FlutterSemanticsAction> AccessibilityBridge::GetFlutterSemanticsAction( |
| fuchsia::accessibility::semantics::Action fuchsia_action, uint32_t node_id) { |
| switch (fuchsia_action) { |
| // The default action associated with the element. |
| case fuchsia::accessibility::semantics::Action::DEFAULT: |
| return FlutterSemanticsAction::kFlutterSemanticsActionTap; |
| // The secondary action associated with the element. This may correspond to |
| // a long press (touchscreens) or right click (mouse). |
| case fuchsia::accessibility::semantics::Action::SECONDARY: |
| return FlutterSemanticsAction::kFlutterSemanticsActionLongPress; |
| // Set (input/non-accessibility) focus on this element. |
| case fuchsia::accessibility::semantics::Action::SET_FOCUS: |
| FX_LOGF(WARNING, kLogTag, |
| "Unsupported action SET_FOCUS sent for " |
| "accessibility node %d", |
| node_id); |
| return {}; |
| // Set the element's value. |
| case fuchsia::accessibility::semantics::Action::SET_VALUE: |
| FX_LOGF(WARNING, kLogTag, |
| "Unsupported action SET_VALUE sent for " |
| "accessibility node %d", |
| node_id); |
| return {}; |
| // Scroll node to make it visible. |
| case fuchsia::accessibility::semantics::Action::SHOW_ON_SCREEN: |
| return FlutterSemanticsAction::kFlutterSemanticsActionShowOnScreen; |
| case fuchsia::accessibility::semantics::Action::INCREMENT: |
| return FlutterSemanticsAction::kFlutterSemanticsActionIncrease; |
| case fuchsia::accessibility::semantics::Action::DECREMENT: |
| return FlutterSemanticsAction::kFlutterSemanticsActionDecrease; |
| default: |
| FX_LOGF(WARNING, kLogTag, |
| "Unexpected action %d sent for " |
| "accessibility node %d", |
| static_cast<int32_t>(fuchsia_action), node_id); |
| return {}; |
| } |
| } |
| |
| // |fuchsia::accessibility::semantics::SemanticListener| |
| void AccessibilityBridge::OnAccessibilityActionRequested( |
| uint32_t node_id, fuchsia::accessibility::semantics::Action action, |
| fuchsia::accessibility::semantics::SemanticListener::OnAccessibilityActionRequestedCallback |
| callback) { |
| // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in |
| // the flutter accessibility bridge. |
| if (nodes_.find(node_id) == nodes_.end()) { |
| FX_LOGF(ERROR, kLogTag, |
| "Attempted to send accessibility action " |
| "%d to unknown node id: %d", |
| static_cast<int32_t>(action), node_id); |
| callback(false); |
| return; |
| } |
| |
| std::optional<FlutterSemanticsAction> flutter_action = GetFlutterSemanticsAction(action, node_id); |
| if (!flutter_action.has_value()) { |
| callback(false); |
| return; |
| } |
| dispatch_semantics_action_callback_(static_cast<int32_t>(node_id), flutter_action.value()); |
| callback(true); |
| } |
| |
| // |fuchsia::accessibility::semantics::SemanticListener| |
| void AccessibilityBridge::HitTest( |
| fuchsia::math::PointF local_point, |
| fuchsia::accessibility::semantics::SemanticListener::HitTestCallback callback) { |
| auto hit_node_id = GetHitNode(kRootNodeId, local_point.x, local_point.y); |
| FX_DCHECK(hit_node_id.has_value()); |
| fuchsia::accessibility::semantics::Hit hit; |
| // TODO(http://fxbug.dev/75910): check the usage of FlutterIdToFuchsiaId in |
| // the flutter accessibility bridge. |
| hit.set_node_id(hit_node_id.value_or(kRootNodeId)); |
| callback(std::move(hit)); |
| } |
| |
| std::optional<int32_t> AccessibilityBridge::GetHitNode(int32_t node_id, float x, float y) { |
| auto it = nodes_.find(node_id); |
| if (it == nodes_.end()) { |
| FX_LOGF(ERROR, kLogTag, "Attempted to hit test unknown node id: %d", node_id); |
| return {}; |
| } |
| auto const& node = it->second; |
| EnumFlagSet<FlutterSemanticsFlag> flag_set(node.data.flags); |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsHidden) || // |
| !RectangleContains(node.screen_rect, x, y)) { |
| return {}; |
| } |
| |
| for (size_t child = 0; child < node.data.child_count; child++) { |
| int32_t child_id = node.data.children_in_hit_test_order[child]; |
| auto candidate = GetHitNode(child_id, x, y); |
| if (candidate) { |
| return candidate; |
| } |
| } |
| |
| if (IsFocusable(node.data)) { |
| return node_id; |
| } |
| |
| return {}; |
| } |
| |
| bool AccessibilityBridge::IsFocusable(const FlutterSemanticsNode& node) const { |
| EnumFlagSet<FlutterSemanticsFlag> flag_set(node.flags); |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagScopesRoute)) { |
| return false; |
| } |
| |
| if (flag_set.HasFlag(FlutterSemanticsFlag::kFlutterSemanticsFlagIsFocusable)) { |
| return true; |
| } |
| |
| // Always consider platform views focusable. |
| if (node.platform_view_id > |
| -1 /*TODO this is a hack, need to get the -1 value from proper channel*/) { |
| return true; |
| } |
| |
| // Always consider actionable nodes focusable. |
| if (node.actions != 0) { |
| return true; |
| } |
| |
| // Consider text nodes focusable. |
| return strlen(node.label) > 0 || strlen(node.value) > 0 || strlen(node.hint) > 0; |
| } |
| |
| // |fuchsia::accessibility::semantics::SemanticListener| |
| void AccessibilityBridge::OnSemanticsModeChanged(bool enabled, |
| OnSemanticsModeChangedCallback callback) { |
| set_semantics_enabled_callback_(enabled); |
| } |
| |
| void AccessibilityBridge::SetPixelRatio(float ratio) { next_pixel_ratio_ = ratio; } |
| |
| // TODO(benbergkamp) |
| // #if DEBUG |
| void AccessibilityBridge::FillInspectTree(int32_t flutter_node_id, int32_t current_level, |
| inspect::Node inspect_node, |
| inspect::Inspector* inspector) const { |
| const auto it = nodes_.find(flutter_node_id); |
| if (it == nodes_.end()) { |
| inspect_node.CreateString( |
| "missing_child", "This node has a parent in the semantic tree but has no value", inspector); |
| inspector->emplace(std::move(inspect_node)); |
| return; |
| } |
| const auto& semantic_node = it->second; |
| const auto& data = semantic_node.data; |
| |
| inspect_node.CreateInt("id", data.id, inspector); |
| // Even with an empty label, we still want to create the property to |
| // explicetly show that it is empty. |
| inspect_node.CreateString("label", data.label, inspector); |
| if (strlen(data.hint) > 0) { |
| inspect_node.CreateString("hint", data.hint, inspector); |
| } |
| if (strlen(data.value) > 0) { |
| inspect_node.CreateString("value", data.value, inspector); |
| } |
| if (strlen(data.increased_value) > 0) { |
| inspect_node.CreateString("increased_value", data.increased_value, inspector); |
| } |
| if (strlen(data.decreased_value) > 0) { |
| inspect_node.CreateString("decreased_value", data.decreased_value, inspector); |
| } |
| |
| if (data.text_direction) { |
| inspect_node.CreateString("text_direction", data.text_direction == 1 ? "RTL" : "LTR", |
| inspector); |
| } |
| |
| if (data.flags) { |
| inspect_node.CreateString("flags", NodeFlagsToString(data), inspector); |
| } |
| if (data.actions) { |
| inspect_node.CreateString("actions", NodeActionsToString(data), inspector); |
| } |
| |
| inspect_node.CreateString("location", NodeLocationToString(semantic_node.screen_rect), inspector); |
| if (data.child_count > 0) { |
| inspect_node.CreateString("children", NodeChildrenToString(data), inspector); |
| } |
| |
| inspect_node.CreateInt("current_level", current_level, inspector); |
| |
| for (size_t child = 0; child < semantic_node.data.child_count; child++) { |
| int32_t flutter_child_id = semantic_node.data.children_in_traversal_order[child]; |
| const auto inspect_name = "node_" + std::to_string(flutter_child_id); |
| FillInspectTree(flutter_child_id, current_level + 1, inspect_node.CreateChild(inspect_name), |
| inspector); |
| } |
| inspector->emplace(std::move(inspect_node)); |
| } |
| // #endif DEBUG |
| |
| void AccessibilityBridge::Apply(FuchsiaAtomicUpdate* atomic_update) { |
| size_t begin = 0; |
| auto it = atomic_update->deletions.begin(); |
| |
| // Process up to kMaxDeletionsPerUpdate deletions at a time. |
| while (it != atomic_update->deletions.end()) { |
| std::vector<uint32_t> to_delete; |
| size_t end = std::min(atomic_update->deletions.size() - begin, kMaxDeletionsPerUpdate); |
| std::copy(std::make_move_iterator(it), std::make_move_iterator(it + end), |
| std::back_inserter(to_delete)); |
| tree_ptr_->DeleteSemanticNodes(std::move(to_delete)); |
| begin = end; |
| it += end; |
| } |
| |
| std::vector<fuchsia::accessibility::semantics::Node> to_update; |
| size_t current_size = 0; |
| for (auto& node_and_size : atomic_update->updates) { |
| if (current_size + node_and_size.second > kMaxMessageSize) { |
| tree_ptr_->UpdateSemanticNodes(std::move(to_update)); |
| current_size = 0; |
| to_update.clear(); |
| } |
| current_size += node_and_size.second; |
| to_update.push_back(std::move(node_and_size.first)); |
| } |
| if (!to_update.empty()) { |
| tree_ptr_->UpdateSemanticNodes(std::move(to_update)); |
| } |
| |
| // Commit this update and subsequent ones; for flow control wait for a |
| // response between each commit. |
| tree_ptr_->CommitUpdates( |
| [this, atomic_updates = std::weak_ptr<std::queue<FuchsiaAtomicUpdate>>(atomic_updates_)]() { |
| auto atomic_updates_ptr = atomic_updates.lock(); |
| if (!atomic_updates_ptr) { |
| // The queue no longer exists, which means that is no longer |
| // necessary. |
| return; |
| } |
| // Removes the update that just went through. |
| atomic_updates_ptr->pop(); |
| if (!atomic_updates_ptr->empty()) { |
| Apply(&atomic_updates_ptr->front()); |
| } |
| }); |
| |
| atomic_update->Clear(); |
| } |
| |
| void AccessibilityBridge::FuchsiaAtomicUpdate::AddNodeUpdate( |
| fuchsia::accessibility::semantics::Node node, size_t size) { |
| if (size > kMaxMessageSize) { |
| // TODO(MI4-2531, FIDL-718): Remove this |
| // This is defensive. If, despite our best efforts, we ended up with a node |
| // that is larger than the max fidl size, we send no updates. |
| PrintNodeSizeError(node.node_id()); |
| return; |
| } |
| updates.emplace_back(std::move(node), size); |
| } |
| |
| void AccessibilityBridge::FuchsiaAtomicUpdate::AddNodeDeletion(uint32_t id) { |
| deletions.push_back(id); |
| } |
| |
| void AccessibilityBridge::FuchsiaAtomicUpdate::Clear() { |
| updates.clear(); |
| deletions.clear(); |
| } |
| } // namespace embedder |
