[scenic] Add IME support to BaseView.
Testing: E2E test to ensure keyboard events are piped through IME.
Change-Id: I0d71601c73768326409676e81007ca3e06b416a1
diff --git a/garnet/public/lib/ui/base_view/cpp/base_view.cc b/garnet/public/lib/ui/base_view/cpp/base_view.cc
index ad341e0..e375211 100644
--- a/garnet/public/lib/ui/base_view/cpp/base_view.cc
+++ b/garnet/public/lib/ui/base_view/cpp/base_view.cc
@@ -8,6 +8,7 @@
#include <lib/ui/scenic/cpp/commands.h>
#include <lib/ui/scenic/cpp/view_token_pair.h>
#include <trace/event.h>
+#include <zircon/status.h>
#include "src/lib/fxl/logging.h"
@@ -21,13 +22,28 @@
std::move(context.session_and_listener_request.second)),
session_(std::move(context.session_and_listener_request.first)),
view_(&session_, std::move(context.view_token), debug_name),
- root_node_(&session_) {
+ root_node_(&session_),
+ ime_client_(this),
+ enable_ime_(context.enable_ime) {
session_.SetDebugName(debug_name);
// Listen for metrics events on our top node.
root_node_.SetEventMask(fuchsia::ui::gfx::kMetricsEventMask);
view_.AddChild(root_node_);
+ if (enable_ime_) {
+ startup_context_->ConnectToEnvironmentService(ime_manager_.NewRequest());
+
+ ime_.set_error_handler([](zx_status_t status) {
+ FXL_LOG(ERROR) << "Interface error on: Input Method Editor "
+ << zx_status_get_string(status);
+ });
+ ime_manager_.set_error_handler([](zx_status_t status) {
+ FXL_LOG(ERROR) << "Interface error on: Text Sync Service "
+ << zx_status_get_string(status);
+ });
+ }
+
// We must immediately invalidate the scene, otherwise we wouldn't ever hook
// the View up to the ViewHolder. An alternative would be to require
// subclasses to call an Init() method to set up the initial connection.
@@ -101,6 +117,11 @@
}
break;
case ::fuchsia::ui::scenic::Event::Tag::kInput: {
+ if (event.input().Which() ==
+ fuchsia::ui::input::InputEvent::Tag::kFocus &&
+ enable_ime_) {
+ OnHandleFocusEvent(event.input().focus());
+ }
OnInputEvent(std::move(event.input()));
break;
}
@@ -154,4 +175,50 @@
});
}
+// |fuchsia::ui::input::InputMethodEditorClient|
+void BaseView::DidUpdateState(
+ fuchsia::ui::input::TextInputState state,
+ std::unique_ptr<fuchsia::ui::input::InputEvent> input_event) {
+ if (input_event) {
+ const fuchsia::ui::input::InputEvent& input = *input_event;
+ fuchsia::ui::input::InputEvent input_event_copy;
+ fidl::Clone(input, &input_event_copy);
+ OnInputEvent(std::move(input_event_copy));
+ }
+}
+
+// |fuchsia::ui::input::InputMethodEditorClient|
+void BaseView::OnAction(fuchsia::ui::input::InputMethodAction action) {}
+
+bool BaseView::OnHandleFocusEvent(const fuchsia::ui::input::FocusEvent& focus) {
+ if (focus.focused) {
+ ActivateIme();
+ return true;
+ } else if (!focus.focused) {
+ DeactivateIme();
+ return true;
+ }
+ return false;
+}
+
+void BaseView::ActivateIme() {
+ ime_manager_->GetInputMethodEditor(
+ fuchsia::ui::input::KeyboardType::TEXT, // keyboard type
+ fuchsia::ui::input::InputMethodAction::DONE, // input method action
+ fuchsia::ui::input::TextInputState{}, // initial state
+ ime_client_.NewBinding(), // client
+ ime_.NewRequest() // editor
+ );
+}
+
+void BaseView::DeactivateIme() {
+ if (ime_) {
+ ime_manager_->HideKeyboard();
+ ime_ = nullptr;
+ }
+ if (ime_client_.is_bound()) {
+ ime_client_.Unbind();
+ }
+}
+
} // namespace scenic
diff --git a/garnet/public/lib/ui/base_view/cpp/base_view.h b/garnet/public/lib/ui/base_view/cpp/base_view.h
index 4b0578c..eb12057 100644
--- a/garnet/public/lib/ui/base_view/cpp/base_view.h
+++ b/garnet/public/lib/ui/base_view/cpp/base_view.h
@@ -23,6 +23,7 @@
fidl::InterfaceRequest<fuchsia::sys::ServiceProvider> incoming_services;
fidl::InterfaceHandle<fuchsia::sys::ServiceProvider> outgoing_services;
component::StartupContext* startup_context;
+ bool enable_ime = false;
};
// Abstract base implementation of a view for simple applications.
@@ -31,7 +32,8 @@
//
// It is not necessary to use this class to implement all Views.
// This class is merely intended to make the simple apps easier to write.
-class BaseView : private fuchsia::ui::scenic::SessionListener {
+class BaseView : private fuchsia::ui::scenic::SessionListener,
+ private fuchsia::ui::input::InputMethodEditorClient {
public:
// Subclasses are typically created by ViewProviderService::CreateView(),
// which provides the necessary args to pass down to this base class.
@@ -161,8 +163,27 @@
// Subclasses should not override this.
void OnScenicEvent(::std::vector<fuchsia::ui::scenic::Event> events) override;
+ // |fuchsia::ui::input::InputMethodEditorClient|
+ void DidUpdateState(
+ fuchsia::ui::input::TextInputState state,
+ std::unique_ptr<fuchsia::ui::input::InputEvent> event) override;
+
+ // |fuchsia::ui::input::InputMethodEditorClient|
+ void OnAction(fuchsia::ui::input::InputMethodAction action) override;
+
void PresentScene(zx_time_t presentation_time);
+ // Handles focus event when IME is enabled. This event is used to activate
+ // or deactivate the IME client.
+ bool OnHandleFocusEvent(const fuchsia::ui::input::FocusEvent& focus);
+
+ // Gets a new input method editor from the IME manager.
+ void ActivateIme();
+
+ // Detaches the input method editor connection, ending the edit session and
+ // closing the onscreen keyboard.
+ void DeactivateIme();
+
component::StartupContext* const startup_context_;
fuchsia::sys::ServiceProviderPtr incoming_services_;
component::ServiceNamespace outgoing_services_;
@@ -172,6 +193,10 @@
scenic::View view_;
scenic::EntityNode root_node_;
+ fidl::Binding<fuchsia::ui::input::InputMethodEditorClient> ime_client_;
+ fuchsia::ui::input::InputMethodEditorPtr ime_;
+ fuchsia::ui::input::ImeServicePtr ime_manager_;
+
fuchsia::ui::gfx::vec3 logical_size_;
fuchsia::ui::gfx::vec3 physical_size_;
fuchsia::ui::gfx::ViewProperties view_properties_;
@@ -181,6 +206,7 @@
size_t session_present_count_ = 0;
bool invalidate_pending_ = false;
bool present_pending_ = false;
+ bool enable_ime_ = false;
};
} // namespace scenic
diff --git a/garnet/tests/e2e_input_tests/BUILD.gn b/garnet/tests/e2e_input_tests/BUILD.gn
index 32bbc5d..f40fc73 100644
--- a/garnet/tests/e2e_input_tests/BUILD.gn
+++ b/garnet/tests/e2e_input_tests/BUILD.gn
@@ -7,12 +7,23 @@
test_package("e2e_input_tests") {
deps = [
+ ":base_view_ime_test",
":mediabuttons_listener_test",
":minimal_input_test",
]
tests = [
{
+ name = "base_view_ime_test"
+ environments = [
+ {
+ dimensions = {
+ device_type = "Intel NUC Kit NUC7i5DNHE"
+ }
+ },
+ ]
+ },
+ {
name = "minimal_input_test"
environments = [
{
@@ -61,6 +72,31 @@
}
# Each e2e test must run in its own executable.
+test("base_view_ime_test") {
+ sources = [
+ "base_view_ime_test.cc",
+ ]
+ output_name = "base_view_ime_test"
+ deps = [
+ "//garnet/public/lib/component/cpp",
+ "//garnet/public/lib/gtest",
+ "//garnet/public/lib/ui/base_view/cpp",
+ "//garnet/public/lib/ui/input/cpp",
+ "//sdk/fidl/fuchsia.ui.input",
+ "//sdk/fidl/fuchsia.ui.policy",
+ "//sdk/fidl/fuchsia.ui.scenic",
+ "//sdk/lib/ui/scenic/cpp",
+ "//src/lib/fxl",
+ "//src/lib/fxl/test:gtest_main",
+ "//third_party/googletest:gtest",
+ "//zircon/public/lib/async-loop-cpp",
+ "//zircon/public/lib/fdio",
+ "//zircon/public/lib/fit",
+ "//zircon/public/lib/zx",
+ ]
+}
+
+# Each e2e test must run in its own executable.
test("mediabuttons_listener_test") {
sources = [
"mediabuttons_listener_test.cc",
diff --git a/garnet/tests/e2e_input_tests/base_view_ime_test.cc b/garnet/tests/e2e_input_tests/base_view_ime_test.cc
new file mode 100644
index 0000000..1656d79
--- /dev/null
+++ b/garnet/tests/e2e_input_tests/base_view_ime_test.cc
@@ -0,0 +1,278 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <fuchsia/ui/input/cpp/fidl.h>
+#include <fuchsia/ui/policy/cpp/fidl.h>
+#include <fuchsia/ui/scenic/cpp/fidl.h>
+#include <gtest/gtest.h>
+#include <lib/async-loop/cpp/loop.h>
+#include <lib/component/cpp/startup_context.h>
+#include <lib/fdio/spawn.h>
+#include <lib/fit/function.h>
+#include <lib/gtest/real_loop_fixture.h>
+#include <lib/ui/base_view/cpp/base_view.h>
+#include <lib/ui/input/cpp/formatting.h>
+#include <lib/ui/scenic/cpp/session.h>
+#include <lib/ui/scenic/cpp/view_token_pair.h>
+#include <zircon/status.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "src/lib/fxl/logging.h"
+
+// NOTE WELL. Run each of these e2e tests in its own executable. They each
+// consume and maintain process-global context, so it's better to keep them
+// separate. Plus, separation means they start up components in a known good
+// state, instead of reusing component state possibly dirtied by other tests.
+
+namespace {
+
+using fuchsia::ui::input::InputEvent;
+using PointerPhase = fuchsia::ui::input::PointerEventPhase;
+using KeyboardPhase = fuchsia::ui::input::KeyboardEventPhase;
+
+// Shared context for all tests in this process.
+// Set it up once, never delete it.
+component::StartupContext* g_context = nullptr;
+
+// Max timeout in failure cases.
+// Set this as low as you can that still works across all test platforms.
+constexpr zx::duration kTimeout = zx::min(5);
+
+// A very small Scenic client. Puts up a fuchsia-colored rectangle, and stores
+// input events for examination.
+class ImeClientView : public scenic::BaseView {
+ public:
+ ImeClientView(scenic::ViewContext context, async_dispatcher_t* dispatcher)
+ : scenic::BaseView(std::move(context), "ImeClientView"),
+ dispatcher_(dispatcher) {
+ FXL_CHECK(dispatcher_);
+ }
+
+ void CreateScene(uint32_t width_in_px, uint32_t height_in_px) {
+ float width = static_cast<float>(width_in_px);
+ float height = static_cast<float>(height_in_px);
+
+ scenic::ShapeNode background(session());
+
+ scenic::Material material(session());
+ material.SetColor(255, 0, 255, 255); // Fuchsia
+ background.SetMaterial(material);
+
+ scenic::Rectangle rectangle(session(), width, height);
+ background.SetShape(rectangle);
+ background.SetTranslation(width / 2, height / 2, -10.f);
+
+ root_node().AddChild(background);
+ }
+
+ void Update(uint64_t present_time) {
+ session()->Present(
+ present_time, [this](fuchsia::images::PresentationInfo info) {
+ Update(info.presentation_time + info.presentation_interval);
+ });
+ }
+
+ // |scenic::BaseView|
+ void OnInputEvent(InputEvent event) override {
+ if (on_input_) {
+ on_input_(std::move(event));
+ }
+ }
+
+ void SetOnInputCallback(fit::function<void(InputEvent)> on_input) {
+ on_input_ = std::move(on_input);
+ }
+
+ private:
+ // |scenic::SessionListener|
+ void OnScenicError(std::string error) override { FXL_LOG(FATAL) << error; }
+
+ async_dispatcher_t* dispatcher_ = nullptr;
+ fit::function<void(InputEvent)> on_input_;
+};
+
+class ImeInputTest : public gtest::RealLoopFixture {
+ protected:
+ // Mildly complex ctor, but we don't throw and we don't call virtual methods.
+ ImeInputTest() {
+ // This fixture constructor may run multiple times, but we want the context
+ // to be set up just once per process.
+ if (g_context == nullptr) {
+ g_context = component::StartupContext::CreateFromStartupInfo().release();
+ }
+
+ auto [view_token, view_holder_token] = scenic::ViewTokenPair::New();
+
+ // Connect to Scenic, create a View.
+ scenic_ =
+ g_context->ConnectToEnvironmentService<fuchsia::ui::scenic::Scenic>();
+ scenic_.set_error_handler([](zx_status_t status) {
+ FXL_LOG(FATAL) << "Lost connection to Scenic: "
+ << zx_status_get_string(status);
+ });
+ scenic::ViewContext view_context = {
+ .enable_ime = true,
+ .session_and_listener_request =
+ scenic::CreateScenicSessionPtrAndListenerRequest(scenic_.get()),
+ .view_token = std::move(view_token),
+ .incoming_services = nullptr,
+ .outgoing_services = nullptr,
+ .startup_context = g_context,
+ };
+ view_ = std::make_unique<ImeClientView>(std::move(view_context),
+ dispatcher());
+
+ // Connect to RootPresenter, create a ViewHolder.
+ root_presenter_ =
+ g_context
+ ->ConnectToEnvironmentService<fuchsia::ui::policy::Presenter>();
+ root_presenter_.set_error_handler([](zx_status_t status) {
+ FXL_LOG(FATAL) << "Lost connection to RootPresenter: "
+ << zx_status_get_string(status);
+ });
+ root_presenter_->PresentView(std::move(view_holder_token), nullptr);
+
+ // When display is available, create content and drive input to touchscreen.
+ scenic_->GetDisplayInfo([this](fuchsia::ui::gfx::DisplayInfo display_info) {
+ display_width_ = display_info.width_in_px;
+ display_height_ = display_info.height_in_px;
+
+ FXL_CHECK(display_width_ > 0 && display_height_ > 0)
+ << "Display size unsuitable for this test: (" << display_width_
+ << ", " << display_height_ << ").";
+
+ view_->CreateScene(display_width_, display_height_);
+ view_->Update(zx_clock_get_monotonic());
+
+ inject_input_(); // Display up, content ready. Send in input.
+
+ test_was_run_ = true; // Actually did work for this test.
+ });
+
+ // Post a "just in case" quit task, if the test hangs.
+ async::PostDelayedTask(
+ dispatcher(),
+ [] {
+ FXL_LOG(FATAL)
+ << "\n\n>> Test did not complete in time, terminating. <<\n\n";
+ },
+ kTimeout);
+ }
+
+ ~ImeInputTest() override {
+ FXL_CHECK(test_was_run_) << "Oops, didn't actually do anything.";
+ }
+
+ void InjectInput(std::vector<const char*> args) {
+ // Start with process name, end with nullptr.
+ args.insert(args.begin(), "input");
+ args.push_back(nullptr);
+
+ // Start the /bin/input process.
+ zx_handle_t proc;
+ zx_status_t status = fdio_spawn(ZX_HANDLE_INVALID, FDIO_SPAWN_CLONE_ALL,
+ "/bin/input", args.data(), &proc);
+ FXL_CHECK(status == ZX_OK)
+ << "fdio_spawn: " << zx_status_get_string(status);
+
+ // Wait for termination.
+ status = zx_object_wait_one(proc, ZX_PROCESS_TERMINATED,
+ (zx::clock::get_monotonic() + kTimeout).get(),
+ nullptr);
+ FXL_CHECK(status == ZX_OK)
+ << "zx_object_wait_one: " << zx_status_get_string(status);
+
+ // Check termination status.
+ zx_info_process_t info;
+ status = zx_object_get_info(proc, ZX_INFO_PROCESS, &info, sizeof(info),
+ nullptr, nullptr);
+ FXL_CHECK(status == ZX_OK)
+ << "zx_object_get_info: " << zx_status_get_string(status);
+ FXL_CHECK(info.return_code == 0)
+ << "info.return_code: " << info.return_code;
+ }
+
+ void SetInjectInputCallback(fit::function<void()> inject_input) {
+ inject_input_ = std::move(inject_input);
+ }
+
+ void SetOnTerminateCallback(fit::function<void()> on_terminate) {
+ on_terminate_ = std::move(on_terminate);
+ }
+
+ fuchsia::ui::policy::PresenterPtr root_presenter_;
+ fuchsia::ui::scenic::ScenicPtr scenic_;
+
+ std::unique_ptr<ImeClientView> view_;
+ uint32_t display_width_ = 0;
+ uint32_t display_height_ = 0;
+
+ std::vector<InputEvent> observed_;
+ fit::function<void()> inject_input_;
+ fit::function<void()> on_terminate_;
+ bool test_was_run_ = false;
+};
+
+TEST_F(ImeInputTest, Keyboard) {
+ // Handle input. Fires for every input event received.
+ view_->SetOnInputCallback([this](InputEvent event) {
+ // Store inputs for checking later.
+ observed_.push_back(std::move(event));
+
+ // Inject text events after tap gesture is done and view has focus.
+ if (event.is_pointer() && event.pointer().phase == PointerPhase::REMOVE) {
+ async::PostTask(dispatcher(), [this] {
+ // Send the Esc key(hid usage code: 41)
+ InjectInput({"keyevent", "41", nullptr});
+ });
+ }
+
+ // Simple termination condition: when key up event is received.
+ if (event.is_keyboard() &&
+ event.keyboard().phase == KeyboardPhase::RELEASED) {
+ async::PostTask(dispatcher(), [this] {
+ FXL_CHECK(on_terminate_) << "on_terminate_ was not set!";
+ on_terminate_();
+ });
+ }
+ });
+
+ // Inject tap. Fires when display and content are available.
+ SetInjectInputCallback([this] {
+ InjectInput({"tap", // Tap at the center of the display
+ std::to_string(display_width_ / 2).c_str(),
+ std::to_string(display_height_ / 2).c_str(), nullptr});
+ });
+
+ // Set up expectations. Fires when we see the "quit" message.
+ SetOnTerminateCallback([this]() {
+ if (FXL_VLOG_IS_ON(2)) {
+ for (const auto& event : observed_) {
+ FXL_LOG(INFO) << "Input event observed: " << event;
+ }
+ }
+
+ EXPECT_EQ(observed_.size(), 7u);
+
+ EXPECT_EQ(observed_[0].pointer().phase, PointerPhase::ADD);
+ EXPECT_TRUE(observed_[1].focus().focused);
+ EXPECT_EQ(observed_[2].pointer().phase, PointerPhase::DOWN);
+ EXPECT_EQ(observed_[3].pointer().phase, PointerPhase::UP);
+ EXPECT_EQ(observed_[4].pointer().phase, PointerPhase::REMOVE);
+ EXPECT_EQ(observed_[5].keyboard().phase, KeyboardPhase::PRESSED);
+ EXPECT_EQ(observed_[6].keyboard().phase, KeyboardPhase::RELEASED);
+
+ QuitLoop();
+ // Today, we can't quietly break the View/ViewHolder connection.
+ });
+
+ RunLoop(); // Go!
+}
+
+} // namespace
+
+// NOTE: We link in FXL's gtest_main to enable proper logging.
diff --git a/garnet/tests/e2e_input_tests/meta/base_view_ime_test.cmx b/garnet/tests/e2e_input_tests/meta/base_view_ime_test.cmx
new file mode 100644
index 0000000..b09d864
--- /dev/null
+++ b/garnet/tests/e2e_input_tests/meta/base_view_ime_test.cmx
@@ -0,0 +1,34 @@
+{
+ "facets": {
+ "fuchsia.test": {
+ "injected-services": {
+ "fuchsia.sysmem.Allocator": "fuchsia-pkg://fuchsia.com/sysmem_connector#meta/sysmem_connector.cmx",
+ "fuchsia.ui.input.ImeService": "fuchsia-pkg://fuchsia.com/ime_service#meta/ime_service.cmx",
+ "fuchsia.ui.input.InputDeviceRegistry": "fuchsia-pkg://fuchsia.com/root_presenter#meta/root_presenter.cmx",
+ "fuchsia.ui.policy.Presenter": "fuchsia-pkg://fuchsia.com/root_presenter#meta/root_presenter.cmx",
+ "fuchsia.ui.scenic.Scenic": "fuchsia-pkg://fuchsia.com/scenic#meta/scenic.cmx",
+ "fuchsia.vulkan.loader.Loader": "fuchsia-pkg://fuchsia.com/vulkan_loader#meta/vulkan_loader.cmx"
+ }
+ }
+ },
+ "program": {
+ "binary": "test/base_view_ime_test"
+ },
+ "sandbox": {
+ "features": [
+ "shell-commands"
+ ],
+ "services": [
+ "fuchsia.tracelink.Registry",
+ "fuchsia.process.Launcher",
+ "fuchsia.process.Resolver",
+ "fuchsia.sys.Environment",
+ "fuchsia.sys.Launcher",
+ "fuchsia.sys.Loader",
+ "fuchsia.ui.input.ImeService",
+ "fuchsia.ui.input.InputDeviceRegistry",
+ "fuchsia.ui.policy.Presenter",
+ "fuchsia.ui.scenic.Scenic"
+ ]
+ }
+}
diff --git a/garnet/tests/e2e_input_tests/meta/minimal_input_test.cmx b/garnet/tests/e2e_input_tests/meta/minimal_input_test.cmx
index df915aa..176793b 100644
--- a/garnet/tests/e2e_input_tests/meta/minimal_input_test.cmx
+++ b/garnet/tests/e2e_input_tests/meta/minimal_input_test.cmx
@@ -25,6 +25,7 @@
"fuchsia.sys.Environment",
"fuchsia.sys.Launcher",
"fuchsia.sys.Loader",
+ "fuchsia.ui.input.ImeService",
"fuchsia.ui.input.InputDeviceRegistry",
"fuchsia.ui.policy.Presenter",
"fuchsia.ui.scenic.Scenic"
diff --git a/products/workstation.gni b/products/workstation.gni
index 9c6ac16..0eef5c1 100644
--- a/products/workstation.gni
+++ b/products/workstation.gni
@@ -10,11 +10,10 @@
"//src/modular/bundles:framework",
"//topaz/packages/config:ermine",
"//topaz/packages/prod:ermine",
+ "//topaz/packages/prod:term",
]
-cache_package_labels += [
- "//topaz/packages/prod:simple_browser",
-]
+cache_package_labels += [ "//topaz/packages/prod:simple_browser" ]
universe_package_labels += [
"//garnet/packages/prod:media_audio",
