| // Copyright 2019 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| #include "src/ui/a11y/lib/screen_reader/screen_reader.h" | |
| #include <lib/fpromise/promise.h> | |
| #include <lib/syslog/cpp/macros.h> | |
| #include <memory> | |
| #include "fuchsia/accessibility/gesture/cpp/fidl.h" | |
| #include "src/ui/a11y/lib/screen_reader/change_range_value_action.h" | |
| #include "src/ui/a11y/lib/screen_reader/change_semantic_level_action.h" | |
| #include "src/ui/a11y/lib/screen_reader/default_action.h" | |
| #include "src/ui/a11y/lib/screen_reader/explore_action.h" | |
| #include "src/ui/a11y/lib/screen_reader/inject_pointer_event_action.h" | |
| #include "src/ui/a11y/lib/screen_reader/linear_navigation_action.h" | |
| #include "src/ui/a11y/lib/screen_reader/process_update_action.h" | |
| #include "src/ui/a11y/lib/screen_reader/recover_a11y_focus_action.h" | |
| #include "src/ui/a11y/lib/screen_reader/three_finger_swipe_action.h" | |
| namespace a11y { | |
| namespace { | |
| constexpr char kNextActionLabel[] = "Next Action"; | |
| constexpr char kPreviousActionLabel[] = "Previous Action"; | |
| constexpr char kExploreActionLabel[] = "Explore Action"; | |
| constexpr char kDefaultActionLabel[] = "Default Action"; | |
| constexpr char kThreeFingerUpSwipeActionLabel[] = "Three finger Up Swipe Action"; | |
| constexpr char kThreeFingerDownSwipeActionLabel[] = "Three finger Down Swipe Action"; | |
| constexpr char kThreeFingerLeftSwipeActionLabel[] = "Three finger Left Swipe Action"; | |
| constexpr char kThreeFingerRightSwipeActionLabel[] = "Three finger Right Swipe Action"; | |
| constexpr char kPreviousSemanticLevelActionLabel[] = "Previous Semantic Level Action"; | |
| constexpr char kNextSemanticLevelActionLabel[] = "Next Semantic Level Action"; | |
| constexpr char kIncrementRangeValueActionLabel[] = "Increment Range Value Action"; | |
| constexpr char kDecrementRangeValueActionLabel[] = "Decrement Range Value Action"; | |
| constexpr char kRecoverA11YFocusActionLabel[] = "Recover A11Y Focus Action"; | |
| constexpr char kInjectPointerEventActionLabel[] = "Inject Pointer Event Action"; | |
| constexpr char kProcessUpdateActionLabel[] = "Process Update Action"; | |
| // Returns the appropriate next action based on the semantic level. | |
| std::string NextActionFromSemanticLevel(ScreenReaderContext::SemanticLevel semantic_level) { | |
| switch (semantic_level) { | |
| case ScreenReaderContext::SemanticLevel::kDefault: | |
| return std::string(kNextActionLabel); | |
| case ScreenReaderContext::SemanticLevel::kAdjustValue: | |
| return std::string(kIncrementRangeValueActionLabel); | |
| default: | |
| // Other semantic levels are not implemented yet, so return an empty action name. | |
| return std::string(""); | |
| } | |
| return std::string(""); | |
| } | |
| // Returns the appropriate previous action based on the semantic level. | |
| std::string PreviousActionFromSemanticLevel(ScreenReaderContext::SemanticLevel semantic_level) { | |
| switch (semantic_level) { | |
| case ScreenReaderContext::SemanticLevel::kDefault: | |
| return std::string(kPreviousActionLabel); | |
| case ScreenReaderContext::SemanticLevel::kAdjustValue: | |
| return std::string(kDecrementRangeValueActionLabel); | |
| default: | |
| // Other semantic levels are not implemented yet, so return an empty action name. | |
| return std::string(""); | |
| } | |
| return std::string(""); | |
| } | |
| } // namespace | |
| // Private implementation of the registry for the Screen Reader use only. Note that only the Screen | |
| // Reader will be able to access the methods implemented here. | |
| class ScreenReader::ScreenReaderActionRegistryImpl : public ScreenReaderActionRegistry { | |
| public: | |
| ScreenReaderActionRegistryImpl() = default; | |
| ~ScreenReaderActionRegistryImpl() override = default; | |
| void AddAction(std::string name, std::unique_ptr<ScreenReaderAction> action) override { | |
| actions_.insert({std::move(name), std::move(action)}); | |
| } | |
| ScreenReaderAction* GetActionByName(const std::string& name) override { | |
| auto action_it = actions_.find(name); | |
| if (action_it == actions_.end()) { | |
| FX_LOGS(ERROR) << "No Screen Reader action found with string :" << name; | |
| return nullptr; | |
| } | |
| return action_it->second.get(); | |
| } | |
| private: | |
| std::unordered_map<std::string, std::unique_ptr<ScreenReaderAction>> actions_; | |
| }; | |
| ScreenReader::ScreenReader(std::unique_ptr<ScreenReaderContext> context, | |
| SemanticsSource* semantics_source, | |
| InjectorManagerInterface* injector_manager, | |
| GestureListenerRegistry* gesture_listener_registry, | |
| TtsManager* tts_manager, bool announce_screen_reader_enabled) | |
| : ScreenReader(std::move(context), semantics_source, injector_manager, | |
| gesture_listener_registry, tts_manager, announce_screen_reader_enabled, | |
| std::make_unique<ScreenReaderActionRegistryImpl>()) {} | |
| ScreenReader::ScreenReader(std::unique_ptr<ScreenReaderContext> context, | |
| SemanticsSource* semantics_source, | |
| InjectorManagerInterface* injector_manager, | |
| GestureListenerRegistry* gesture_listener_registry, | |
| TtsManager* tts_manager, bool announce_screen_reader_enabled, | |
| std::unique_ptr<ScreenReaderActionRegistry> action_registry) | |
| : context_(std::move(context)), | |
| gesture_listener_registry_(gesture_listener_registry), | |
| tts_manager_(tts_manager), | |
| action_registry_(std::move(action_registry)), | |
| weak_ptr_factory_(this) { | |
| action_context_ = std::make_unique<ScreenReaderAction::ActionContext>(); | |
| action_context_->semantics_source = semantics_source; | |
| action_context_->injector_manager = injector_manager; | |
| InitializeActions(); | |
| FX_DCHECK(tts_manager_); | |
| if (announce_screen_reader_enabled) { | |
| tts_manager->RegisterTTSEngineReadyCallback( | |
| [this]() { SpeakMessage(fuchsia::intl::l10n::MessageIds::SCREEN_READER_ON_HINT); }); | |
| } | |
| context_->speaker()->set_epitaph(fuchsia::intl::l10n::MessageIds::SCREEN_READER_OFF_HINT); | |
| } | |
| ScreenReader::~ScreenReader() { | |
| FX_LOGS(INFO) << __func__; | |
| tts_manager_->UnregisterTTSEngineReadyCallback(); | |
| } | |
| void ScreenReader::BindGestures(a11y::GestureHandlerV2* gesture_handler) { | |
| // Add gestures with higher priority earlier than gestures with lower priority. | |
| // Add a three finger Up swipe recognizer. This corresponds to a physical three finger Right | |
| // swipe. | |
| bool gesture_bind_status = gesture_handler->BindSwipeAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| ExecuteAction(kThreeFingerRightSwipeActionLabel, std::move(context)); | |
| }, | |
| GestureHandlerV2::kThreeFingerUpSwipe); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add a three finger Down swipe recognizer. This corresponds to a physical three finger Left | |
| // swipe. | |
| gesture_bind_status = gesture_handler->BindSwipeAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| ExecuteAction(kThreeFingerLeftSwipeActionLabel, std::move(context)); | |
| }, | |
| GestureHandlerV2::kThreeFingerDownSwipe); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add a three finger Left swipe recognizer. This corresponds to a physical three finger Up swipe. | |
| gesture_bind_status = gesture_handler->BindSwipeAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| ExecuteAction(kThreeFingerUpSwipeActionLabel, std::move(context)); | |
| }, | |
| GestureHandlerV2::kThreeFingerLeftSwipe); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add a three finger Right swipe recognizer. This corresponds to a physical three finger Down | |
| // swipe. | |
| gesture_bind_status = gesture_handler->BindSwipeAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| ExecuteAction(kThreeFingerDownSwipeActionLabel, std::move(context)); | |
| }, | |
| GestureHandlerV2::kThreeFingerRightSwipe); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add one finger Down swipe recognizer. This corresponds to a physical one finger Left swipe. | |
| gesture_bind_status = gesture_handler->BindSwipeAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| auto action_name = PreviousActionFromSemanticLevel(context_->semantic_level()); | |
| ExecuteAction(action_name, std::move(context)); | |
| }, | |
| GestureHandlerV2::kOneFingerDownSwipe); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add one finger Up swipe recognizer. This corresponds to a physical one finger Right swipe. | |
| gesture_bind_status = gesture_handler->BindSwipeAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| auto action_name = NextActionFromSemanticLevel(context_->semantic_level()); | |
| ExecuteAction(action_name, std::move(context)); | |
| }, | |
| GestureHandlerV2::kOneFingerUpSwipe); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add one finger Left swipe recognizer. This corresponds to a physical one finger Up swipe. | |
| gesture_bind_status = gesture_handler->BindSwipeAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| ExecuteAction(kPreviousSemanticLevelActionLabel, std::move(context)); | |
| }, | |
| GestureHandlerV2::kOneFingerLeftSwipe); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add one finger Right swipe recognizer. This corresponds to a physical one finger Down swipe. | |
| gesture_bind_status = gesture_handler->BindSwipeAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| ExecuteAction(kNextSemanticLevelActionLabel, std::move(context)); | |
| }, | |
| GestureHandlerV2::kOneFingerRightSwipe); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add OneFingerDoubleTap recognizer. | |
| gesture_bind_status = gesture_handler->BindOneFingerDoubleTapAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| // This simulated tap down / up event is necessary because some of the supported runtimes at | |
| // the moment do not have an accessibility action to bring up a keyboard when interacting | |
| // with a text field. | |
| if (context_->IsTextFieldFocused()) { | |
| SimulateTapDown(context); | |
| SimulateTapUp(context); | |
| } | |
| // TODO(https://fxbug.dev/42160603): Default action should not be needed after a simulated | |
| // tap down / up. | |
| ExecuteAction(kDefaultActionLabel, std::move(context)); | |
| }); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add MFingerNTapDragRecognizer (1 finger, 2 taps), recognizer. | |
| gesture_bind_status = gesture_handler->BindMFingerNTapDragAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| SimulateTapDown(std::move(context)); | |
| }, /*on_start*/ | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| ExecuteAction(kInjectPointerEventActionLabel, std::move(context)); | |
| }, /*on_update*/ | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| SimulateTapUp(std::move(context)); | |
| } /*on_complete*/, | |
| 1u /*num_fingers*/, 2u /*num_taps*/); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add OneFingerSingleTap recognizer. | |
| gesture_bind_status = gesture_handler->BindOneFingerSingleTapAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| context_->set_semantic_level(ScreenReaderContext::SemanticLevel::kDefault); | |
| ExecuteAction(kExploreActionLabel, std::move(context)); | |
| }); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add OneFingerDrag recognizer. | |
| gesture_bind_status = gesture_handler->BindOneFingerDragAction( | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| context_->set_semantic_level(ScreenReaderContext::SemanticLevel::kDefault); | |
| context_->set_mode(ScreenReaderContext::ScreenReaderMode::kContinuousExploration); | |
| }, /*on_start*/ | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| FX_DCHECK(context_->mode() == | |
| ScreenReaderContext::ScreenReaderMode::kContinuousExploration); | |
| ExecuteAction(kExploreActionLabel, std::move(context)); | |
| }, /*on_update*/ | |
| [this](a11y::gesture_util_v2::GestureContext context) { | |
| FX_DCHECK(context_->mode() == | |
| ScreenReaderContext::ScreenReaderMode::kContinuousExploration); | |
| context_->set_mode(ScreenReaderContext::ScreenReaderMode::kNormal); | |
| // At the end of an explore action, if a virtual keyboard is in focus, activate the last | |
| // touched key. | |
| if (context_->IsVirtualKeyboardFocused()) { | |
| ExecuteAction(kDefaultActionLabel, std::move(context)); | |
| } | |
| } /*on_complete*/); | |
| FX_DCHECK(gesture_bind_status); | |
| // Add TwoFingerSingleTap recognizer. | |
| gesture_bind_status = gesture_handler->BindTwoFingerSingleTapAction( | |
| [this](a11y::gesture_util_v2::GestureContext /* unused */) { | |
| // Cancel any outstanding speech. | |
| auto promise = context_->speaker()->CancelTts(); | |
| auto* executor = context_->executor(); | |
| executor->schedule_task(std::move(promise)); | |
| }); | |
| FX_DCHECK(gesture_bind_status); | |
| } | |
| void ScreenReader::InitializeActions() { | |
| action_registry_->AddAction(kExploreActionLabel, std::make_unique<a11y::ExploreAction>( | |
| action_context_.get(), context_.get())); | |
| action_registry_->AddAction(kDefaultActionLabel, std::make_unique<a11y::DefaultAction>( | |
| action_context_.get(), context_.get())); | |
| action_registry_->AddAction( | |
| kPreviousActionLabel, | |
| std::make_unique<a11y::LinearNavigationAction>( | |
| action_context_.get(), context_.get(), a11y::LinearNavigationAction::kPreviousAction)); | |
| action_registry_->AddAction(kNextActionLabel, std::make_unique<a11y::LinearNavigationAction>( | |
| action_context_.get(), context_.get(), | |
| a11y::LinearNavigationAction::kNextAction)); | |
| action_registry_->AddAction( | |
| kNextSemanticLevelActionLabel, | |
| std::make_unique<ChangeSemanticLevelAction>(ChangeSemanticLevelAction::Direction::kForward, | |
| action_context_.get(), context_.get())); | |
| action_registry_->AddAction( | |
| kPreviousSemanticLevelActionLabel, | |
| std::make_unique<ChangeSemanticLevelAction>(ChangeSemanticLevelAction::Direction::kBackward, | |
| action_context_.get(), context_.get())); | |
| action_registry_->AddAction( | |
| kIncrementRangeValueActionLabel, | |
| std::make_unique<ChangeRangeValueAction>( | |
| action_context_.get(), context_.get(), | |
| ChangeRangeValueAction::ChangeRangeValueActionType::kIncrementAction)); | |
| action_registry_->AddAction( | |
| kDecrementRangeValueActionLabel, | |
| std::make_unique<ChangeRangeValueAction>( | |
| action_context_.get(), context_.get(), | |
| ChangeRangeValueAction::ChangeRangeValueActionType::kDecrementAction)); | |
| action_registry_->AddAction(kThreeFingerUpSwipeActionLabel, | |
| std::make_unique<a11y::ThreeFingerSwipeAction>( | |
| action_context_.get(), context_.get(), gesture_listener_registry_, | |
| fuchsia::accessibility::gesture::Type::THREE_FINGER_SWIPE_UP)); | |
| action_registry_->AddAction(kThreeFingerDownSwipeActionLabel, | |
| std::make_unique<a11y::ThreeFingerSwipeAction>( | |
| action_context_.get(), context_.get(), gesture_listener_registry_, | |
| fuchsia::accessibility::gesture::Type::THREE_FINGER_SWIPE_DOWN)); | |
| action_registry_->AddAction(kThreeFingerLeftSwipeActionLabel, | |
| std::make_unique<a11y::ThreeFingerSwipeAction>( | |
| action_context_.get(), context_.get(), gesture_listener_registry_, | |
| fuchsia::accessibility::gesture::Type::THREE_FINGER_SWIPE_LEFT)); | |
| action_registry_->AddAction(kThreeFingerRightSwipeActionLabel, | |
| std::make_unique<a11y::ThreeFingerSwipeAction>( | |
| action_context_.get(), context_.get(), gesture_listener_registry_, | |
| fuchsia::accessibility::gesture::Type::THREE_FINGER_SWIPE_RIGHT)); | |
| action_registry_->AddAction( | |
| kRecoverA11YFocusActionLabel, | |
| std::make_unique<RecoverA11YFocusAction>(action_context_.get(), context_.get())); | |
| action_registry_->AddAction( | |
| kInjectPointerEventActionLabel, | |
| std::make_unique<InjectPointerEventAction>(action_context_.get(), context_.get())); | |
| action_registry_->AddAction( | |
| kProcessUpdateActionLabel, | |
| std::make_unique<ProcessUpdateAction>(action_context_.get(), context_.get())); | |
| } | |
| bool ScreenReader::ExecuteAction(const std::string& action_name, | |
| a11y::gesture_util_v2::GestureContext gesture_context) { | |
| auto* action = action_registry_->GetActionByName(action_name); | |
| if (!action) { | |
| return false; | |
| } | |
| action->Run(gesture_context); | |
| return true; | |
| } | |
| void ScreenReader::SpeakMessage(fuchsia::intl::l10n::MessageIds message_id) { | |
| auto* speaker = context_->speaker(); | |
| auto promise = | |
| speaker->SpeakMessageByIdPromise(message_id, {.interrupt = true, .save_utterance = false}); | |
| context_->executor()->schedule_task(std::move(promise)); | |
| } | |
| void ScreenReader::SpeakMessage(const std::string& message) { | |
| auto* speaker = context_->speaker(); | |
| fuchsia::accessibility::tts::Utterance utterance; | |
| utterance.set_message(message); | |
| auto promise = speaker->SpeakMessagePromise(std::move(utterance), | |
| {.interrupt = true, .save_utterance = false}); | |
| context_->executor()->schedule_task(std::move(promise)); | |
| } | |
| fxl::WeakPtr<SemanticsEventListener> ScreenReader::GetSemanticsEventListenerWeakPtr() { | |
| return weak_ptr_factory_.GetWeakPtr(); | |
| } | |
| void ScreenReader::OnEvent(SemanticsEventInfo event_info) { | |
| // Process internal semantic events. | |
| switch (event_info.event_type) { | |
| case SemanticsEventType::kSemanticTreeUpdated: { | |
| a11y::gesture_util_v2::GestureContext gesture_context; | |
| if (event_info.view_ref_koid) { | |
| gesture_context.view_ref_koid = *event_info.view_ref_koid; | |
| } | |
| ExecuteAction(kRecoverA11YFocusActionLabel, gesture_context); | |
| ExecuteAction(kProcessUpdateActionLabel, std::move(gesture_context)); | |
| break; | |
| } | |
| case SemanticsEventType::kUnknown: | |
| break; | |
| } | |
| // Process semantic events coming from semantic providers. | |
| if (!event_info.semantic_event) { | |
| return; | |
| } | |
| if (event_info.semantic_event->is_announce()) { | |
| const auto& announce = event_info.semantic_event->announce(); | |
| if (announce.has_message()) { | |
| SpeakMessage(announce.message()); | |
| } | |
| } | |
| } | |
| void ScreenReader::SimulateTapDown(a11y::gesture_util_v2::GestureContext context) { | |
| // Enable injector for the view that is receiving pointer events. | |
| action_context_->injector_manager->MarkViewReadyForInjection(context.view_ref_koid, true); | |
| // When the gesture detects, events are already under way. We need to inject an (ADD) event here | |
| // to simulate the beginning of the stream that will be injected after this tap down. | |
| context.last_event_phase = fuchsia::ui::pointer::EventPhase::ADD; | |
| ExecuteAction(kInjectPointerEventActionLabel, context); | |
| context.last_event_phase = | |
| fuchsia::ui::pointer::EventPhase::CHANGE; // fuchsia::ui::input::PointerEventPhase::MOVE; | |
| ExecuteAction(kInjectPointerEventActionLabel, context); | |
| } | |
| void ScreenReader::SimulateTapUp(a11y::gesture_util_v2::GestureContext context) { | |
| context.last_event_phase = fuchsia::ui::pointer::EventPhase::REMOVE; | |
| ExecuteAction(kInjectPointerEventActionLabel, context); | |
| // End injection for the view. | |
| action_context_->injector_manager->MarkViewReadyForInjection(context.view_ref_koid, false); | |
| } | |
| } // namespace a11y |