| // Copyright 2019 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "src/ui/a11y/lib/screen_reader/screen_reader.h" |
| |
| #include <lib/fpromise/promise.h> |
| #include <lib/syslog/cpp/macros.h> |
| |
| #include <memory> |
| |
| #include "fuchsia/accessibility/gesture/cpp/fidl.h" |
| #include "src/ui/a11y/lib/screen_reader/change_range_value_action.h" |
| #include "src/ui/a11y/lib/screen_reader/change_semantic_level_action.h" |
| #include "src/ui/a11y/lib/screen_reader/default_action.h" |
| #include "src/ui/a11y/lib/screen_reader/explore_action.h" |
| #include "src/ui/a11y/lib/screen_reader/inject_pointer_event_action.h" |
| #include "src/ui/a11y/lib/screen_reader/linear_navigation_action.h" |
| #include "src/ui/a11y/lib/screen_reader/process_update_action.h" |
| #include "src/ui/a11y/lib/screen_reader/recover_a11y_focus_action.h" |
| #include "src/ui/a11y/lib/screen_reader/three_finger_swipe_action.h" |
| |
| namespace a11y { |
| namespace { |
| |
| constexpr char kNextActionLabel[] = "Next Action"; |
| constexpr char kPreviousActionLabel[] = "Previous Action"; |
| constexpr char kExploreActionLabel[] = "Explore Action"; |
| constexpr char kDefaultActionLabel[] = "Default Action"; |
| constexpr char kThreeFingerUpSwipeActionLabel[] = "Three finger Up Swipe Action"; |
| constexpr char kThreeFingerDownSwipeActionLabel[] = "Three finger Down Swipe Action"; |
| constexpr char kThreeFingerLeftSwipeActionLabel[] = "Three finger Left Swipe Action"; |
| constexpr char kThreeFingerRightSwipeActionLabel[] = "Three finger Right Swipe Action"; |
| constexpr char kPreviousSemanticLevelActionLabel[] = "Previous Semantic Level Action"; |
| constexpr char kNextSemanticLevelActionLabel[] = "Next Semantic Level Action"; |
| constexpr char kIncrementRangeValueActionLabel[] = "Increment Range Value Action"; |
| constexpr char kDecrementRangeValueActionLabel[] = "Decrement Range Value Action"; |
| constexpr char kRecoverA11YFocusActionLabel[] = "Recover A11Y Focus Action"; |
| constexpr char kInjectPointerEventActionLabel[] = "Inject Pointer Event Action"; |
| constexpr char kProcessUpdateActionLabel[] = "Process Update Action"; |
| |
| // Returns the appropriate next action based on the semantic level. |
| std::string NextActionFromSemanticLevel(ScreenReaderContext::SemanticLevel semantic_level) { |
| switch (semantic_level) { |
| case ScreenReaderContext::SemanticLevel::kDefault: |
| return std::string(kNextActionLabel); |
| case ScreenReaderContext::SemanticLevel::kAdjustValue: |
| return std::string(kIncrementRangeValueActionLabel); |
| default: |
| // Other semantic levels are not implemented yet, so return an empty action name. |
| return std::string(""); |
| } |
| return std::string(""); |
| } |
| |
| // Returns the appropriate previous action based on the semantic level. |
| std::string PreviousActionFromSemanticLevel(ScreenReaderContext::SemanticLevel semantic_level) { |
| switch (semantic_level) { |
| case ScreenReaderContext::SemanticLevel::kDefault: |
| return std::string(kPreviousActionLabel); |
| case ScreenReaderContext::SemanticLevel::kAdjustValue: |
| return std::string(kDecrementRangeValueActionLabel); |
| default: |
| // Other semantic levels are not implemented yet, so return an empty action name. |
| return std::string(""); |
| } |
| return std::string(""); |
| } |
| |
| } // namespace |
| |
| // Private implementation of the registry for the Screen Reader use only. Note that only the Screen |
| // Reader will be able to access the methods implemented here. |
| class ScreenReader::ScreenReaderActionRegistryImpl : public ScreenReaderActionRegistry { |
| public: |
| ScreenReaderActionRegistryImpl() = default; |
| ~ScreenReaderActionRegistryImpl() override = default; |
| void AddAction(std::string name, std::unique_ptr<ScreenReaderAction> action) override { |
| actions_.insert({std::move(name), std::move(action)}); |
| } |
| |
| ScreenReaderAction* GetActionByName(const std::string& name) override { |
| auto action_it = actions_.find(name); |
| if (action_it == actions_.end()) { |
| FX_LOGS(ERROR) << "No Screen Reader action found with string :" << name; |
| return nullptr; |
| } |
| return action_it->second.get(); |
| } |
| |
| private: |
| std::unordered_map<std::string, std::unique_ptr<ScreenReaderAction>> actions_; |
| }; |
| |
| ScreenReader::ScreenReader(std::unique_ptr<ScreenReaderContext> context, |
| SemanticsSource* semantics_source, |
| InjectorManagerInterface* injector_manager, |
| GestureListenerRegistry* gesture_listener_registry, |
| TtsManager* tts_manager, bool announce_screen_reader_enabled) |
| : ScreenReader(std::move(context), semantics_source, injector_manager, |
| gesture_listener_registry, tts_manager, announce_screen_reader_enabled, |
| std::make_unique<ScreenReaderActionRegistryImpl>()) {} |
| |
| ScreenReader::ScreenReader(std::unique_ptr<ScreenReaderContext> context, |
| SemanticsSource* semantics_source, |
| InjectorManagerInterface* injector_manager, |
| GestureListenerRegistry* gesture_listener_registry, |
| TtsManager* tts_manager, bool announce_screen_reader_enabled, |
| std::unique_ptr<ScreenReaderActionRegistry> action_registry) |
| : context_(std::move(context)), |
| gesture_listener_registry_(gesture_listener_registry), |
| tts_manager_(tts_manager), |
| action_registry_(std::move(action_registry)), |
| weak_ptr_factory_(this) { |
| action_context_ = std::make_unique<ScreenReaderAction::ActionContext>(); |
| action_context_->semantics_source = semantics_source; |
| action_context_->injector_manager = injector_manager; |
| InitializeActions(); |
| FX_DCHECK(tts_manager_); |
| |
| if (announce_screen_reader_enabled) { |
| tts_manager->RegisterTTSEngineReadyCallback( |
| [this]() { SpeakMessage(fuchsia::intl::l10n::MessageIds::SCREEN_READER_ON_HINT); }); |
| } |
| |
| context_->speaker()->set_epitaph(fuchsia::intl::l10n::MessageIds::SCREEN_READER_OFF_HINT); |
| } |
| |
| ScreenReader::~ScreenReader() { tts_manager_->UnregisterTTSEngineReadyCallback(); } |
| |
| void ScreenReader::BindGestures(a11y::GestureHandlerV2* gesture_handler) { |
| // Add gestures with higher priority earlier than gestures with lower priority. |
| // Add a three finger Up swipe recognizer. This corresponds to a physical three finger Right |
| // swipe. |
| bool gesture_bind_status = gesture_handler->BindSwipeAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| ExecuteAction(kThreeFingerRightSwipeActionLabel, std::move(context)); |
| }, |
| GestureHandlerV2::kThreeFingerUpSwipe); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add a three finger Down swipe recognizer. This corresponds to a physical three finger Left |
| // swipe. |
| gesture_bind_status = gesture_handler->BindSwipeAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| ExecuteAction(kThreeFingerLeftSwipeActionLabel, std::move(context)); |
| }, |
| GestureHandlerV2::kThreeFingerDownSwipe); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add a three finger Left swipe recognizer. This corresponds to a physical three finger Up swipe. |
| gesture_bind_status = gesture_handler->BindSwipeAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| ExecuteAction(kThreeFingerUpSwipeActionLabel, std::move(context)); |
| }, |
| GestureHandlerV2::kThreeFingerLeftSwipe); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add a three finger Right swipe recognizer. This corresponds to a physical three finger Down |
| // swipe. |
| gesture_bind_status = gesture_handler->BindSwipeAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| ExecuteAction(kThreeFingerDownSwipeActionLabel, std::move(context)); |
| }, |
| GestureHandlerV2::kThreeFingerRightSwipe); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add one finger Down swipe recognizer. This corresponds to a physical one finger Left swipe. |
| gesture_bind_status = gesture_handler->BindSwipeAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| auto action_name = PreviousActionFromSemanticLevel(context_->semantic_level()); |
| ExecuteAction(action_name, std::move(context)); |
| }, |
| GestureHandlerV2::kOneFingerDownSwipe); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add one finger Up swipe recognizer. This corresponds to a physical one finger Right swipe. |
| gesture_bind_status = gesture_handler->BindSwipeAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| auto action_name = NextActionFromSemanticLevel(context_->semantic_level()); |
| ExecuteAction(action_name, std::move(context)); |
| }, |
| GestureHandlerV2::kOneFingerUpSwipe); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add one finger Left swipe recognizer. This corresponds to a physical one finger Up swipe. |
| gesture_bind_status = gesture_handler->BindSwipeAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| ExecuteAction(kPreviousSemanticLevelActionLabel, std::move(context)); |
| }, |
| GestureHandlerV2::kOneFingerLeftSwipe); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add one finger Right swipe recognizer. This corresponds to a physical one finger Down swipe. |
| gesture_bind_status = gesture_handler->BindSwipeAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| ExecuteAction(kNextSemanticLevelActionLabel, std::move(context)); |
| }, |
| GestureHandlerV2::kOneFingerRightSwipe); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add OneFingerDoubleTap recognizer. |
| gesture_bind_status = gesture_handler->BindOneFingerDoubleTapAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| // This simulated tap down / up event is necessary because some of the supported runtimes at |
| // the moment do not have an accessibility action to bring up a keyboard when interacting |
| // with a text field. |
| if (context_->IsTextFieldFocused()) { |
| SimulateTapDown(context); |
| SimulateTapUp(context); |
| } |
| // TODO(https://fxbug.dev/42160603): Default action should not be needed after a simulated tap down / |
| // up. |
| ExecuteAction(kDefaultActionLabel, std::move(context)); |
| }); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add MFingerNTapDragRecognizer (1 finger, 2 taps), recognizer. |
| gesture_bind_status = gesture_handler->BindMFingerNTapDragAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| SimulateTapDown(std::move(context)); |
| }, /*on_start*/ |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| ExecuteAction(kInjectPointerEventActionLabel, std::move(context)); |
| }, /*on_update*/ |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| SimulateTapUp(std::move(context)); |
| } /*on_complete*/, |
| 1u /*num_fingers*/, 2u /*num_taps*/); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add OneFingerSingleTap recognizer. |
| gesture_bind_status = gesture_handler->BindOneFingerSingleTapAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| context_->set_semantic_level(ScreenReaderContext::SemanticLevel::kDefault); |
| ExecuteAction(kExploreActionLabel, std::move(context)); |
| }); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add OneFingerDrag recognizer. |
| gesture_bind_status = gesture_handler->BindOneFingerDragAction( |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| context_->set_semantic_level(ScreenReaderContext::SemanticLevel::kDefault); |
| context_->set_mode(ScreenReaderContext::ScreenReaderMode::kContinuousExploration); |
| }, /*on_start*/ |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| FX_DCHECK(context_->mode() == |
| ScreenReaderContext::ScreenReaderMode::kContinuousExploration); |
| ExecuteAction(kExploreActionLabel, std::move(context)); |
| }, /*on_update*/ |
| [this](a11y::gesture_util_v2::GestureContext context) { |
| FX_DCHECK(context_->mode() == |
| ScreenReaderContext::ScreenReaderMode::kContinuousExploration); |
| context_->set_mode(ScreenReaderContext::ScreenReaderMode::kNormal); |
| // At the end of an explore action, if a virtual keyboard is in focus, activate the last |
| // touched key. |
| if (context_->IsVirtualKeyboardFocused()) { |
| ExecuteAction(kDefaultActionLabel, std::move(context)); |
| } |
| } /*on_complete*/); |
| FX_DCHECK(gesture_bind_status); |
| |
| // Add TwoFingerSingleTap recognizer. |
| gesture_bind_status = gesture_handler->BindTwoFingerSingleTapAction( |
| [this](a11y::gesture_util_v2::GestureContext /* unused */) { |
| // Cancel any outstanding speech. |
| auto promise = context_->speaker()->CancelTts(); |
| auto* executor = context_->executor(); |
| executor->schedule_task(std::move(promise)); |
| }); |
| FX_DCHECK(gesture_bind_status); |
| } |
| |
| void ScreenReader::InitializeActions() { |
| action_registry_->AddAction(kExploreActionLabel, std::make_unique<a11y::ExploreAction>( |
| action_context_.get(), context_.get())); |
| action_registry_->AddAction(kDefaultActionLabel, std::make_unique<a11y::DefaultAction>( |
| action_context_.get(), context_.get())); |
| action_registry_->AddAction( |
| kPreviousActionLabel, |
| std::make_unique<a11y::LinearNavigationAction>( |
| action_context_.get(), context_.get(), a11y::LinearNavigationAction::kPreviousAction)); |
| action_registry_->AddAction(kNextActionLabel, std::make_unique<a11y::LinearNavigationAction>( |
| action_context_.get(), context_.get(), |
| a11y::LinearNavigationAction::kNextAction)); |
| action_registry_->AddAction( |
| kNextSemanticLevelActionLabel, |
| std::make_unique<ChangeSemanticLevelAction>(ChangeSemanticLevelAction::Direction::kForward, |
| action_context_.get(), context_.get())); |
| action_registry_->AddAction( |
| kPreviousSemanticLevelActionLabel, |
| std::make_unique<ChangeSemanticLevelAction>(ChangeSemanticLevelAction::Direction::kBackward, |
| action_context_.get(), context_.get())); |
| action_registry_->AddAction( |
| kIncrementRangeValueActionLabel, |
| std::make_unique<ChangeRangeValueAction>( |
| action_context_.get(), context_.get(), |
| ChangeRangeValueAction::ChangeRangeValueActionType::kIncrementAction)); |
| action_registry_->AddAction( |
| kDecrementRangeValueActionLabel, |
| std::make_unique<ChangeRangeValueAction>( |
| action_context_.get(), context_.get(), |
| ChangeRangeValueAction::ChangeRangeValueActionType::kDecrementAction)); |
| |
| action_registry_->AddAction(kThreeFingerUpSwipeActionLabel, |
| std::make_unique<a11y::ThreeFingerSwipeAction>( |
| action_context_.get(), context_.get(), gesture_listener_registry_, |
| fuchsia::accessibility::gesture::Type::THREE_FINGER_SWIPE_UP)); |
| |
| action_registry_->AddAction(kThreeFingerDownSwipeActionLabel, |
| std::make_unique<a11y::ThreeFingerSwipeAction>( |
| action_context_.get(), context_.get(), gesture_listener_registry_, |
| fuchsia::accessibility::gesture::Type::THREE_FINGER_SWIPE_DOWN)); |
| |
| action_registry_->AddAction(kThreeFingerLeftSwipeActionLabel, |
| std::make_unique<a11y::ThreeFingerSwipeAction>( |
| action_context_.get(), context_.get(), gesture_listener_registry_, |
| fuchsia::accessibility::gesture::Type::THREE_FINGER_SWIPE_LEFT)); |
| |
| action_registry_->AddAction(kThreeFingerRightSwipeActionLabel, |
| std::make_unique<a11y::ThreeFingerSwipeAction>( |
| action_context_.get(), context_.get(), gesture_listener_registry_, |
| fuchsia::accessibility::gesture::Type::THREE_FINGER_SWIPE_RIGHT)); |
| |
| action_registry_->AddAction( |
| kRecoverA11YFocusActionLabel, |
| std::make_unique<RecoverA11YFocusAction>(action_context_.get(), context_.get())); |
| |
| action_registry_->AddAction( |
| kInjectPointerEventActionLabel, |
| std::make_unique<InjectPointerEventAction>(action_context_.get(), context_.get())); |
| |
| action_registry_->AddAction( |
| kProcessUpdateActionLabel, |
| std::make_unique<ProcessUpdateAction>(action_context_.get(), context_.get())); |
| } |
| |
| bool ScreenReader::ExecuteAction(const std::string& action_name, |
| a11y::gesture_util_v2::GestureContext gesture_context) { |
| auto* action = action_registry_->GetActionByName(action_name); |
| if (!action) { |
| return false; |
| } |
| action->Run(gesture_context); |
| return true; |
| } |
| |
| void ScreenReader::SpeakMessage(fuchsia::intl::l10n::MessageIds message_id) { |
| auto* speaker = context_->speaker(); |
| auto promise = |
| speaker->SpeakMessageByIdPromise(message_id, {.interrupt = true, .save_utterance = false}); |
| context_->executor()->schedule_task(std::move(promise)); |
| } |
| |
| void ScreenReader::SpeakMessage(const std::string& message) { |
| auto* speaker = context_->speaker(); |
| fuchsia::accessibility::tts::Utterance utterance; |
| utterance.set_message(message); |
| auto promise = speaker->SpeakMessagePromise(std::move(utterance), |
| {.interrupt = true, .save_utterance = false}); |
| context_->executor()->schedule_task(std::move(promise)); |
| } |
| |
| fxl::WeakPtr<SemanticsEventListener> ScreenReader::GetSemanticsEventListenerWeakPtr() { |
| return weak_ptr_factory_.GetWeakPtr(); |
| } |
| |
| void ScreenReader::OnEvent(SemanticsEventInfo event_info) { |
| // Process internal semantic events. |
| switch (event_info.event_type) { |
| case SemanticsEventType::kSemanticTreeUpdated: { |
| a11y::gesture_util_v2::GestureContext gesture_context; |
| if (event_info.view_ref_koid) { |
| gesture_context.view_ref_koid = *event_info.view_ref_koid; |
| } |
| |
| ExecuteAction(kRecoverA11YFocusActionLabel, gesture_context); |
| ExecuteAction(kProcessUpdateActionLabel, std::move(gesture_context)); |
| |
| break; |
| } |
| case SemanticsEventType::kUnknown: |
| break; |
| } |
| |
| // Process semantic events coming from semantic providers. |
| if (!event_info.semantic_event) { |
| return; |
| } |
| |
| if (event_info.semantic_event->is_announce()) { |
| const auto& announce = event_info.semantic_event->announce(); |
| if (announce.has_message()) { |
| SpeakMessage(announce.message()); |
| } |
| } |
| } |
| |
| void ScreenReader::SimulateTapDown(a11y::gesture_util_v2::GestureContext context) { |
| // Enable injector for the view that is receiving pointer events. |
| action_context_->injector_manager->MarkViewReadyForInjection(context.view_ref_koid, true); |
| // When the gesture detects, events are already under way. We need to inject an (ADD) event here |
| // to simulate the beginning of the stream that will be injected after this tap down. |
| context.last_event_phase = fuchsia::ui::pointer::EventPhase::ADD; |
| ExecuteAction(kInjectPointerEventActionLabel, context); |
| context.last_event_phase = |
| fuchsia::ui::pointer::EventPhase::CHANGE; // fuchsia::ui::input::PointerEventPhase::MOVE; |
| ExecuteAction(kInjectPointerEventActionLabel, context); |
| } |
| |
| void ScreenReader::SimulateTapUp(a11y::gesture_util_v2::GestureContext context) { |
| context.last_event_phase = fuchsia::ui::pointer::EventPhase::REMOVE; |
| ExecuteAction(kInjectPointerEventActionLabel, context); |
| |
| // End injection for the view. |
| action_context_->injector_manager->MarkViewReadyForInjection(context.view_ref_koid, false); |
| } |
| |
| } // namespace a11y |