[A11y] updates node definition with extra semantic information.
This change addes missing semantic information into the semantic node,
part of the accessibility infrastructure.
Previously, the node missed some important information to represent a
semantic node correctly.
For example: attributes, states and actions that can be performed on a
node were missing.
This change introduces the missing fields into the node data
structure. All the rest is left almost untouched, with some minor
naming improvements and the removal of some redundancies (for example,
removing the extra data field in the node itself).
Another important and noticeable changes is the removal of the label field.
Labels describe the name of a semantic element, but a label of a
semantic node can be defined by another semantic node (please see the
aria spec 1.1 for examples).
With this in mind, the best way of labeling an element is via an
attribute pair <label, actual node label>, when it produces its own,
or by the attribute pair <labeled_by, node_id>, when another node is
responsible for that.
Hit testing logic is also removed, as this is not how we are going to
do this in the future. the field children_hit_test_order was removed,
as we figure out how hit testing will be done for accessibility.
Tested:
* Installed on device.
* fx run-test a11y-tests
MI4-2322 #comment
Change-Id: I2be557e1772257a5e4673e70cc6582dc6af4d84f
diff --git a/garnet/bin/a11y/a11y_manager/semantics/semantic_tree_impl.cc b/garnet/bin/a11y/a11y_manager/semantics/semantic_tree_impl.cc
index f77c38d..8db7854 100644
--- a/garnet/bin/a11y/a11y_manager/semantics/semantic_tree_impl.cc
+++ b/garnet/bin/a11y/a11y_manager/semantics/semantic_tree_impl.cc
@@ -30,39 +30,6 @@
box.max.y >= point.y;
}
-const fuchsia::accessibility::semantics::NodePtr SemanticTreeImpl::HitTest(
- const std::unordered_map<uint32_t, fuchsia::accessibility::semantics::Node>&
- nodes,
- uint32_t starting_node_id, escher::vec4 coordinates) const {
- auto it = nodes.find(starting_node_id);
- if (it == nodes.end()) {
- return nullptr;
- }
- escher::mat4 transform =
- scenic_impl::gfx::Unwrap(it->second.data().transform());
- escher::vec4 local_coordinates = transform * coordinates;
- escher::vec2 point(local_coordinates[0], local_coordinates[1]);
-
- if (!BoxContainsPoint(it->second.data().location(), point)) {
- return nullptr;
- }
- for (const auto& child : it->second.children_hit_test_order()) {
- auto node = HitTest(nodes, child, local_coordinates);
- if (node != nullptr) {
- return node;
- }
- }
- auto node_ptr = fuchsia::accessibility::semantics::Node::New();
- it->second.Clone(node_ptr.get());
- return node_ptr;
-}
-
-fuchsia::accessibility::semantics::NodePtr
-SemanticTreeImpl::GetHitAccessibilityNode(fuchsia::math::PointF point) {
- escher::vec4 coordinate(point.x, point.y, 0, 1);
- return HitTest(nodes_, kRootNode, coordinate);
-}
-
fuchsia::accessibility::semantics::NodePtr
SemanticTreeImpl::GetAccessibilityNode(uint32_t node_id) {
auto node_it = nodes_.find(node_id);
@@ -147,14 +114,17 @@
tree_log->append(kIndentSize * current_level, ' ');
// Add logs for the current node.
- absl::StrAppend(tree_log, "Node_id: ", std::to_string(root_node->node_id()),
- ", Label:", root_node->data().label(), kNewLine);
+ absl::StrAppend(
+ tree_log, "Node_id: ", std::to_string(root_node->node_id()), ", Label:",
+ root_node->attributes().has_label() ? root_node->attributes().label()
+ : "_empty",
+ kNewLine);
// Iterate through all the children of the current node.
- if (!root_node->has_children_traversal_order()) {
+ if (!root_node->has_child_ids()) {
return;
}
- for (const auto& child : root_node->children_traversal_order()) {
+ for (const auto& child : root_node->child_ids()) {
fuchsia::accessibility::semantics::NodePtr node_ptr =
GetAccessibilityNode(child);
LogSemanticTreeHelper(std::move(node_ptr), current_level + 1, tree_log);
@@ -189,10 +159,10 @@
}
visited->insert(node->node_id());
- if (!node->has_children_traversal_order()) {
+ if (!node->has_child_ids()) {
return false;
}
- for (const auto& child : node->children_traversal_order()) {
+ for (const auto& child : node->child_ids()) {
fuchsia::accessibility::semantics::NodePtr child_ptr =
GetAccessibilityNode(child);
if (!child_ptr) {
@@ -215,8 +185,8 @@
return;
}
- if (node->has_children_traversal_order()) {
- for (const auto& child : node->children_traversal_order()) {
+ if (node->has_child_ids()) {
+ for (const auto& child : node->child_ids()) {
DeleteSubtree(child);
}
}
@@ -229,15 +199,14 @@
// Loop through all the nodes in the tree, since there can be trees not rooted
// at 0(root-node).
for (auto it = nodes_.begin(); it != nodes_.end(); ++it) {
- if (it->second.has_children_traversal_order()) {
+ if (it->second.has_child_ids()) {
// Loop through all the children of the node.
- for (auto child_it = it->second.children_traversal_order().begin();
- child_it != it->second.children_traversal_order().end();
- ++child_it) {
+ for (auto child_it = it->second.child_ids().begin();
+ child_it != it->second.child_ids().end(); ++child_it) {
// If a child node is same as node_id, then delete child node from the
// list.
if (*child_it == node_id) {
- it->second.mutable_children_traversal_order()->erase(child_it);
+ it->second.mutable_child_ids()->erase(child_it);
return;
}
}
diff --git a/garnet/bin/a11y/a11y_manager/semantics/semantic_tree_impl.h b/garnet/bin/a11y/a11y_manager/semantics/semantic_tree_impl.h
index 619745a..765b91e 100644
--- a/garnet/bin/a11y/a11y_manager/semantics/semantic_tree_impl.h
+++ b/garnet/bin/a11y/a11y_manager/semantics/semantic_tree_impl.h
@@ -47,12 +47,6 @@
~SemanticTreeImpl() override = default;
- // Provides a way to perform hit-testing for a front-end node with local view
- // hit coordinates from Scenic. Returns the deepest node that the input
- // touches. Currently, this only supports 2D hit-tests using bounding boxes.
- fuchsia::accessibility::semantics::NodePtr GetHitAccessibilityNode(
- fuchsia::math::PointF point);
-
// Provides a way to query a node with node id. This method returns
// a copy of the queried node. It may return a nullptr if no node is found.
fuchsia::accessibility::semantics::NodePtr GetAccessibilityNode(
@@ -112,16 +106,6 @@
// Helper function to delete pointer from parent node to given node.
void DeletePointerFromParent(uint32_t node_id);
- // Internal recursive hit-test function using the cached tree. Returns a
- // null pointer if no hit nodes were found. Returns a copy of the node
- // (but not the subtree), to prevent tree modification.
- // NOTE: This is a 2D hit test and only operates on bounding boxes of
- // semantics nodes.
- const fuchsia::accessibility::semantics::NodePtr HitTest(
- const std::unordered_map<uint32_t,
- fuchsia::accessibility::semantics::Node>& nodes,
- uint32_t starting_node_id, escher::vec4 coordinates) const;
-
// Internal helper function to check if a point is within a bounding box.
bool BoxContainsPoint(const fuchsia::ui::gfx::BoundingBox& box,
const escher::vec2& point) const;
diff --git a/garnet/bin/a11y/a11y_manager/semantics/semantics_manager_impl.cc b/garnet/bin/a11y/a11y_manager/semantics/semantics_manager_impl.cc
index 35e0e87..c026e6f 100644
--- a/garnet/bin/a11y/a11y_manager/semantics/semantics_manager_impl.cc
+++ b/garnet/bin/a11y/a11y_manager/semantics/semantics_manager_impl.cc
@@ -42,16 +42,6 @@
std::move(semantic_tree_request));
}
-fuchsia::accessibility::semantics::NodePtr
-SemanticsManagerImpl::GetHitAccessibilityNode(
- const fuchsia::ui::views::ViewRef& view_ref,
- const fuchsia::math::PointF point) {
- for (auto& binding : semantic_tree_bindings_.bindings()) {
- if (binding->impl()->IsSameView(view_ref))
- return binding->impl()->GetHitAccessibilityNode(point);
- }
- return nullptr;
-}
fuchsia::accessibility::semantics::NodePtr
SemanticsManagerImpl::GetAccessibilityNode(
diff --git a/garnet/bin/a11y/a11y_manager/semantics/semantics_manager_impl.h b/garnet/bin/a11y/a11y_manager/semantics/semantics_manager_impl.h
index ba8b0df..a3be2e3 100644
--- a/garnet/bin/a11y/a11y_manager/semantics/semantics_manager_impl.h
+++ b/garnet/bin/a11y/a11y_manager/semantics/semantics_manager_impl.h
@@ -25,14 +25,6 @@
void SetDebugDirectory(vfs::PseudoDir* debug_dir);
- // Provides the a11y manager with a way to perform hit-testing for a
- // front-end node when it has the view id and the local view hit
- // coordinates from Scenic. Currently, this only supports 2D hit-tests
- // using bounding boxes.
- fuchsia::accessibility::semantics::NodePtr GetHitAccessibilityNode(
- const fuchsia::ui::views::ViewRef& view_ref,
- const fuchsia::math::PointF point);
-
// Provides the manager a way to query a node if it already knows
// what view id and node id it wants to query for. This method returns
// a copy of the queried node. It may return a nullptr if no node is found.
diff --git a/sdk/fidl/fuchsia.accessibility.semantics/BUILD.gn b/sdk/fidl/fuchsia.accessibility.semantics/BUILD.gn
index 7559881..8d62122 100644
--- a/sdk/fidl/fuchsia.accessibility.semantics/BUILD.gn
+++ b/sdk/fidl/fuchsia.accessibility.semantics/BUILD.gn
@@ -8,7 +8,7 @@
sdk_category = "experimental"
sources = [
- "data.fidl",
+ "node.fidl",
"semantics_manager.fidl",
]
diff --git a/sdk/fidl/fuchsia.accessibility.semantics/data.fidl b/sdk/fidl/fuchsia.accessibility.semantics/data.fidl
deleted file mode 100644
index 4e8f04c..0000000
--- a/sdk/fidl/fuchsia.accessibility.semantics/data.fidl
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright 2018 The Fuchsia Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-library fuchsia.accessibility.semantics;
-
-using fuchsia.ui.gfx;
-
-// TODO(SCN-814) Figure out how we want to serialize data.
-
-/// Represents the purpose of a ui element; ex. button, text area.
-/// Currently not being used. Will be expanded in the future.
-enum Role {
- NONE = 0;
-};
-
-/// Represents actions that can be applied to Nodes.
-enum Action {
- GAIN_ACCESSIBILITY_FOCUS = 1;
- LOSE_ACCESSIBILITY_FOCUS = 2;
- TAP = 3;
-};
-
-/// Accessibility data for an UI element.
-table Data {
- 1: Role role;
- 2: string label;
- // Local bounding box of this element.
- 3: fuchsia.ui.gfx.BoundingBox location;
- // Transform from parent coordinate space to local space.
- 4: fuchsia.ui.gfx.mat4 transform;
-};
-
-// Wrapper for accessibility data to store structure of UI tree. In progress.
-table Node {
- // Node id identifying position in a front-end's local UI structure.
- // We assume 0 to be the root node.
- 1: uint32 node_id;
- 2: vector<uint32> children_traversal_order;
- 3: vector<uint32> children_hit_test_order;
- 4: Data data;
-};
diff --git a/sdk/fidl/fuchsia.accessibility.semantics/node.fidl b/sdk/fidl/fuchsia.accessibility.semantics/node.fidl
new file mode 100644
index 0000000..0f441c3
--- /dev/null
+++ b/sdk/fidl/fuchsia.accessibility.semantics/node.fidl
@@ -0,0 +1,75 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+library fuchsia.accessibility.semantics;
+
+using fuchsia.ui.gfx;
+
+/// Represents actions that can be applied to Nodes. In progress.
+enum Action {
+ // The default action associated with the role of an element.
+ DEFAULT = 1;
+};
+
+/// Represents a role of an element on an UI. In progress.
+enum Role {
+ // Role used to represent elements which role is not currently supported.
+ unknown = 1;
+};
+
+/// An attribute is an essential property to describe an element. Unlike states,
+/// attributes define the nature of an element, and a change in one of them
+/// changes the nature of the element itself.
+/// Example: a button with a label attribute 'ok', could have its label changed
+/// to 'cancel', which would alter drastically this element. In progress.
+table Attributes {
+ // The label for an element.
+ 1: string label;
+};
+
+/// A state is a dynamic property of an element that may change in response to
+/// user action or automated processes. Thus, they are different from attributes
+/// in an important point, which is frequency of change.
+/// Example: a checkbox can be checked / unchecked, and this state can be
+/// altered via user input. In progress.
+table States {
+ // Whether the element is checked.
+ 1: bool checked;
+};
+
+/// Node: data structure to represent semantic information about an UI element.
+///
+/// The Node represents a semantic element on an interface. This may
+/// be a button, a text field, a checkbox or any element that has a relevant
+/// semantic meaning so that assistive technology can understand the current UI.
+table Node {
+ // Unique ID that represents the node in a particular UI.
+ // Zero is assumed to be the root node and the only entry point to the tree.
+ // No forest is allowed.
+ 1: uint32 node_id;
+
+ // Role of this element, e.g. button, checkbox, etc.
+ 2: Role role;
+
+ // A list of states of this object, e.g. checked, editable, etc.
+ 3: States states;
+
+ // A list of attributes of this node.
+ 4: Attributes attributes;
+
+ // List of actions that can be performed on this node.
+ 5: vector<Action> actions;
+
+ // The ordered list of children IDs of this node.
+ // It is assumed that runtimes supplying semantic tree information will not
+ // create cycles inside the tree.
+ // It is also assumed that each node can have only one parent.
+ 6: vector<uint32> child_ids;
+
+ // Local bounding box of this element.
+ 7: fuchsia.ui.gfx.BoundingBox location;
+
+ // Transform from parent coordinate space to local space.
+ 8: fuchsia.ui.gfx.mat4 transform;
+};
