[flutter-embedder][keyboard] Port keyboard input functionality from flutter/engine. - Key data can be sent to the engine, but doesn't appear as text input to textfield in flutter app. - flutter/keyevent channel must be taken care of in future prs. - main.dart - I added a textinput field, but doesn't show key data right now. - brought in rapidjson as a submodule, with anticipation of using in future keyboard work. Change-Id: If8c3e95c189cae1ba1c5d99b7e035cba0a646c0c Reviewed-on: https://fuchsia-review.googlesource.com/c/flutter-embedder/+/735084 Reviewed-by: Alexander Biggs <akbiggs@google.com>
diff --git a/.gitmodules b/.gitmodules index 5d0af1d..02517cc 100644 --- a/.gitmodules +++ b/.gitmodules
@@ -10,3 +10,6 @@ [submodule "third_party/depot_tools"] path = third_party/depot_tools url = https://chromium.googlesource.com/chromium/tools/depot_tools.git +[submodule "third_party/rapidjson"] + path = third_party/rapidjson + url = https://fuchsia.googlesource.com/third_party/rapidjson/
diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index a8858de..967fd6d 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel
@@ -58,3 +58,10 @@ name = "com_google_googletest", path = "third_party/googletest", ) + +# rapidjson. +new_local_repository( + name = "rapidjson", + build_file = "third_party/BUILD.bazel", + path = "third_party/rapidjson", +)
diff --git a/src/embedder/BUILD.bazel b/src/embedder/BUILD.bazel index 6d96afe..24d10ed 100644 --- a/src/embedder/BUILD.bazel +++ b/src/embedder/BUILD.bazel
@@ -14,31 +14,36 @@ # and runs the Flutter app defined by those assets. fuchsia_cc_binary( name = "embedder", - visibility = ["//visibility:public"], srcs = [ "embedder_state.h", "flatland_connection.cc", "flatland_connection.h", "flatland_ids.h", "flatland_view_provider.h", + "fuchsia_logger.cc", + "fuchsia_logger.h", + "keyboard.cc", + "keyboard.h", "logging.h", "main.cc", + "mouse_delegate.cc", + "mouse_delegate.h", + "pointer_utility.h", "software_surface.cc", "software_surface.h", - "mouse_delegate.h", - "mouse_delegate.cc", - "touch_delegate.h", + "text_delegate.cc", + "text_delegate.h", "touch_delegate.cc", - "pointer_utility.h", - "fuchsia_logger.h", - "fuchsia_logger.cc", + "touch_delegate.h", ], + visibility = ["//visibility:public"], deps = [ "//src/embedder/engine:embedder_header", # TODO(akbiggs): Enable switching between debug and profile # builds of libflutter_engine.so without manually editing this # file. "//src/embedder/engine/debug_x64:libflutter_engine", + "@rapidjson", "@fuchsia_sdk//fidl/fuchsia.fonts:fuchsia.fonts_cc", "@fuchsia_sdk//fidl/fuchsia.sysmem:fuchsia.sysmem_cc", "@fuchsia_sdk//fidl/fuchsia.ui.app:fuchsia.ui.app_cc", @@ -57,10 +62,10 @@ fuchsia_component_manifest( name = "embedder_manifest", - visibility = ["//visibility:public"], src = "meta/embedder.cml", includes = [ "@fuchsia_sdk//pkg/syslog:client", "@fuchsia_sdk//pkg/vulkan:client", ], + visibility = ["//visibility:public"], )
diff --git a/src/embedder/flatland_view_provider.h b/src/embedder/flatland_view_provider.h index 4152ea0..d5cdce5 100644 --- a/src/embedder/flatland_view_provider.h +++ b/src/embedder/flatland_view_provider.h
@@ -8,7 +8,12 @@ #include <fuchsia/ui/app/cpp/fidl.h> #include <lib/syslog/global.h> #include <lib/ui/scenic/cpp/view_identity.h> +#include <lib/ui/scenic/cpp/view_ref_pair.h> +#include <memory> + +#include "logging.h" +#include "src/embedder/engine/embedder.h" #include "src/embedder/flatland_connection.h" #include "src/embedder/flatland_ids.h" #include "src/embedder/logging.h" @@ -18,9 +23,13 @@ /// Implements ViewProvider by using Flatland to create a view. class FlatlandViewProvider final : public fuchsia::ui::app::ViewProvider { public: - FlatlandViewProvider(FlatlandConnection* flatland_connection, - fuchsia::ui::composition::ViewBoundProtocols protocols) - : flatland_connection_(flatland_connection), flatland_view_protocols_(std::move(protocols)) {} + FlatlandViewProvider(FlatlandConnection* flatland_connection, scenic::ViewRefPair view_ref_pair, + fuchsia::ui::composition::ViewBoundProtocols flatland_view_protocols) + : flatland_connection_(flatland_connection), + flatland_view_protocols_(std::move(flatland_view_protocols)) { + view_identity_ = {.view_ref = std::move(view_ref_pair.view_ref), + .view_ref_control = std::move(view_ref_pair.control_ref)}; + } ~FlatlandViewProvider() override {} FlatlandViewProvider(const FlatlandViewProvider&) = delete; @@ -40,9 +49,9 @@ void CreateView2(fuchsia::ui::app::CreateView2Args view_args) override { auto* flatland = flatland_connection_->flatland(); - flatland->CreateView2(std::move(*view_args.mutable_view_creation_token()), - scenic::NewViewIdentityOnCreation(), std::move(flatland_view_protocols_), - parent_viewport_watcher_.NewRequest()); + flatland->CreateView2( + std::move(*view_args.mutable_view_creation_token()), std::move(view_identity_), + std::move(flatland_view_protocols_) /* protocols */, parent_viewport_watcher_.NewRequest()); flatland->CreateTransform({kRootTransformId}); flatland->SetRootTransform({kRootTransformId}); @@ -51,6 +60,7 @@ private: FlatlandConnection* flatland_connection_ = nullptr; fuchsia::ui::composition::ViewBoundProtocols flatland_view_protocols_; + fuchsia::ui::views::ViewIdentityOnCreation view_identity_; fuchsia::ui::composition::ParentViewportWatcherPtr parent_viewport_watcher_; };
diff --git a/src/embedder/keyboard.cc b/src/embedder/keyboard.cc new file mode 100644 index 0000000..e2edd61 --- /dev/null +++ b/src/embedder/keyboard.cc
@@ -0,0 +1,352 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "src/embedder/keyboard.h" + +#include <fuchsia/input/cpp/fidl.h> +#include <fuchsia/ui/input/cpp/fidl.h> +#include <fuchsia/ui/input3/cpp/fidl.h> +#include <lib/syslog/global.h> + +#include <cstdint> +#include <iostream> + +#include "logging.h" + +namespace embedder { + +using fuchsia::input::Key; +using fuchsia::ui::input::kModifierCapsLock; +using fuchsia::ui::input::kModifierLeftAlt; +using fuchsia::ui::input::kModifierLeftControl; +using fuchsia::ui::input::kModifierLeftShift; +using fuchsia::ui::input::kModifierNone; +using fuchsia::ui::input::kModifierRightAlt; +using fuchsia::ui::input::kModifierRightControl; +using fuchsia::ui::input::kModifierRightShift; +using fuchsia::ui::input3::KeyEvent; +using fuchsia::ui::input3::KeyEventType; + +namespace { + +// A simple keymap from a QWERTY keyboard to code points. A value 0 means no +// code point has been assigned for the respective keypress. Column 0 is the +// code point without a level modifier active, and Column 1 is the code point +// with a level modifier (e.g. Shift key) active. +static const uint32_t kQwertyToCodePoints[][2] = { + // 0x00 + {}, + {}, + {}, + {}, + // 0x04, + {'a', 'A'}, + {'b', 'B'}, + {'c', 'C'}, + {'d', 'D'}, + // 0x08 + {'e', 'E'}, + {'f', 'F'}, + {'g', 'G'}, + {'h', 'H'}, + // 0x0c + {'i', 'I'}, + {'j', 'J'}, + {'k', 'K'}, + {'l', 'L'}, + // 0x10 + {'m', 'M'}, + {'n', 'N'}, + {'o', 'O'}, + {'p', 'P'}, + // 0x14 + {'q', 'Q'}, + {'r', 'R'}, + {'s', 'S'}, + {'t', 'T'}, + // 0x18 + {'u', 'U'}, + {'v', 'V'}, + {'w', 'W'}, + {'x', 'X'}, + // 0x1c + {'y', 'Y'}, + {'z', 'Z'}, + {'1', '!'}, + {'2', '@'}, + // 0x20 + {'3', '#'}, + {'4', '$'}, + {'5', '%'}, + {'6', '^'}, + // 0x24 + {'7', '&'}, + {'8', '*'}, + {'9', '('}, + {'0', ')'}, + // 0x28 + {}, + {}, + {}, + {}, + // 0x2c + {' ', ' '}, + {'-', '_'}, + {'=', '+'}, + {'[', '{'}, + // 0x30 + {']', '}'}, + {'\\', '|'}, + {}, + {';', ':'}, + // 0x34 + {'\'', '"'}, + {'`', '~'}, + {',', '<'}, + {'.', '>'}, + // 0x38 + {'/', '?'}, + {}, + {}, + {}, + // 0x3c + {}, + {}, + {}, + {}, + // 0x40 + {}, + {}, + {}, + {}, + // 0x44 + {}, + {}, + {}, + {}, + // 0x48 + {}, + {}, + {}, + {}, + // 0x4c + {}, + {}, + {}, + {}, + // 0x50 + {}, + {}, + {}, + {}, + // 0x54 + {'/', 0}, + {'*', 0}, + {'-', 0}, + {'+', 0}, + // 0x58 + {}, + {'1', 0}, + {'2', 0}, + {'3', 0}, + // 0x5c + {'4', 0}, + {'5', 0}, + {'6', 0}, + {'7', 0}, + // 0x60 + {'8', 0}, + {'9', 0}, + {'0', 0}, + {'.', 0}, +}; + +} // namespace + +Keyboard::Keyboard() + : any_events_received_(false), + stateful_caps_lock_(false), + left_shift_(false), + right_shift_(false), + left_alt_(false), + right_alt_(false), + left_ctrl_(false), + right_ctrl_(false), + last_event_() {} + +bool Keyboard::ConsumeEvent(KeyEvent event) { + if (!event.has_type()) { + return false; + } + if (!event.has_key() && !event.has_key_meaning()) { + return false; + } + // Check if the time sequence of the events is correct. + last_event_ = std::move(event); + any_events_received_ = true; + + if (!event.has_key()) { + // The key only has key meaning. Key meaning currently can not + // update the modifier state, so we just short-circuit the table + // below. + return true; + } + + const Key& key = last_event_.key(); + const KeyEventType& event_type = last_event_.type(); + switch (event_type) { + // For modifier keys, a SYNC is the same as a press. + case KeyEventType::SYNC: + switch (key) { + case Key::CAPS_LOCK: + stateful_caps_lock_ = true; + break; + case Key::LEFT_ALT: + left_alt_ = true; + break; + case Key::LEFT_CTRL: + left_ctrl_ = true; + break; + case Key::LEFT_SHIFT: + left_shift_ = true; + break; + case Key::RIGHT_ALT: + right_alt_ = true; + break; + case Key::RIGHT_CTRL: + right_ctrl_ = true; + break; + case Key::RIGHT_SHIFT: + right_shift_ = true; + break; + default: + // no-op + break; + } + break; + case KeyEventType::PRESSED: + switch (key) { + case Key::CAPS_LOCK: + stateful_caps_lock_ = !stateful_caps_lock_; + break; + case Key::LEFT_ALT: + left_alt_ = true; + break; + case Key::LEFT_CTRL: + left_ctrl_ = true; + break; + case Key::LEFT_SHIFT: + left_shift_ = true; + break; + case Key::RIGHT_ALT: + right_alt_ = true; + break; + case Key::RIGHT_CTRL: + right_ctrl_ = true; + break; + case Key::RIGHT_SHIFT: + right_shift_ = true; + break; + default: + // No-op + break; + } + break; + case KeyEventType::RELEASED: + switch (key) { + case Key::CAPS_LOCK: + // No-op. + break; + case Key::LEFT_ALT: + left_alt_ = false; + break; + case Key::LEFT_CTRL: + left_ctrl_ = false; + break; + case Key::LEFT_SHIFT: + left_shift_ = false; + break; + case Key::RIGHT_ALT: + right_alt_ = false; + break; + case Key::RIGHT_CTRL: + right_ctrl_ = false; + break; + case Key::RIGHT_SHIFT: + right_shift_ = false; + break; + default: + // No-op + break; + } + break; + case KeyEventType::CANCEL: + // No-op? + break; + default: + // No-op + break; + } + return true; +} + +bool Keyboard::IsShift() { return left_shift_ | right_shift_ | stateful_caps_lock_; } + +bool Keyboard::IsKeys() { return LastHIDUsagePage() == 0x7; } + +uint32_t Keyboard::Modifiers() { + return kModifierNone + (kModifierLeftShift * left_shift_) + (kModifierLeftAlt * left_alt_) + + (kModifierLeftControl * left_ctrl_) + (kModifierRightShift * right_shift_) + + (kModifierRightAlt * right_alt_) + (kModifierRightControl * right_ctrl_) + + (kModifierCapsLock * stateful_caps_lock_); +} + +uint32_t Keyboard::LastCodePoint() { + // If the key has a meaning, and if the meaning is a code point, always have + // that code point take precedence over any other keymap. + if (last_event_.has_key_meaning()) { + const auto& key_meaning = last_event_.key_meaning(); + if (key_meaning.is_codepoint()) { + return key_meaning.codepoint(); + } + } + + static const int qwerty_map_size = sizeof(kQwertyToCodePoints) / sizeof(kQwertyToCodePoints[0]); + if (!IsKeys()) { + return 0; + } + const auto usage = LastHIDUsageID(); + if (usage < qwerty_map_size) { + return kQwertyToCodePoints[usage][IsShift() & 1]; + } + // Any other keys don't have a code point. + return 0; +} + +char* Keyboard::LastCodePointAsChar() { + static char textToWrite[9]; + uint32_t codepoint = LastCodePoint(); + sprintf(textToWrite, "%c", codepoint); + return textToWrite; +} + +uint32_t Keyboard::GetLastKey() { + // For logical key determination, the physical key does not matter as long + // as code point is set. + // https://github.com/flutter/flutter/blob/570e39d38b799e91abe4f73f120ce494049c4ff0/packages/flutter/lib/src/services/raw_keyboard_fuchsia.dart#L71 + // It is not quite clear what happens to the physical key, though: + // https://github.com/flutter/flutter/blob/570e39d38b799e91abe4f73f120ce494049c4ff0/packages/flutter/lib/src/services/raw_keyboard_fuchsia.dart#L88 + if (!last_event_.has_key()) { + return 0; + } + return static_cast<uint32_t>(last_event_.key()); +} + +uint32_t Keyboard::LastHIDUsage() { return GetLastKey() & 0xFFFFFFFF; } + +uint16_t Keyboard::LastHIDUsageID() { return GetLastKey() & 0xFFFF; } + +uint16_t Keyboard::LastHIDUsagePage() { return (GetLastKey() >> 16) & 0xFFFF; } + +} // namespace embedder
diff --git a/src/embedder/keyboard.h b/src/embedder/keyboard.h new file mode 100644 index 0000000..9c60a25 --- /dev/null +++ b/src/embedder/keyboard.h
@@ -0,0 +1,87 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_SHELL_PLATFORM_FUCHSIA_KEYBOARD_H_ +#define FLUTTER_SHELL_PLATFORM_FUCHSIA_KEYBOARD_H_ + +#include <fuchsia/ui/input3/cpp/fidl.h> + +namespace embedder { + +/// Keyboard handles the keyboard signals from fuchsia.ui.input3. Specifically, +/// input3 has no notion of a code point, and does not track stateful versions +/// of the modifier keys. +class Keyboard final { + public: + explicit Keyboard(); + + /// Consumes the given keyboard event. Keyboard will adjust the modifier + /// state based on the info given in the event. Returns true if the event has + /// been integrated into the internal state successfully, or false otherwise. + bool ConsumeEvent(fuchsia::ui::input3::KeyEvent event); + + /// Gets the currently active modifier keys. + uint32_t Modifiers(); + + /// Gets the last encountered code point. The reported code point depends on + /// the state of the modifier keys. + uint32_t LastCodePoint(); + + char* LastCodePointAsChar(); + + /// Gets the last encountered HID usage. This is a 32-bit number, with the + /// upper 16 bits equal to `LastHidUsagePage()`, and the lower 16 bits equal + /// to `LastHIDUsageID()`. + /// + /// The key corresponding to A will have the usage 0x7004. This function will + /// return 0x7004 in that case. + uint32_t LastHIDUsage(); + + /// Gets the last encountered HID usage page. + /// + /// The key corresponding to A will have the usage 0x7004. This function will + /// return 0x7 in that case. + uint16_t LastHIDUsagePage(); + + /// Gets the last encountered HID usage ID. + /// + /// The key corresponding to A will have the usage 0x7004. This function will + /// return 0x4 in that case. + uint16_t LastHIDUsageID(); + + private: + /// Return true if any level shift is active. + bool IsShift(); + + /// Returns true if the last key event was about a key that may have a code + /// point associated. + bool IsKeys(); + + /// Returns the value of the last key as a uint32_t. + /// If there isn't such a value (as in the case of on-screen keyboards), this + /// will return a 0; + uint32_t GetLastKey(); + + /// Set to false until any event is received. + bool any_events_received_ : 1; + + /// The flags below show the state of the keyboard modifiers after the last + /// event has been processed. Stateful keys remain in the same state after + /// a release and require an additional press to toggle. + bool stateful_caps_lock_ : 1; + bool left_shift_ : 1; + bool right_shift_ : 1; + bool left_alt_ : 1; + bool right_alt_ : 1; + bool left_ctrl_ : 1; + bool right_ctrl_ : 1; + + /// The last received key event. If any_events_received_ is not set, this is + /// not valid. + fuchsia::ui::input3::KeyEvent last_event_; +}; + +} // namespace embedder + +#endif // FLUTTER_SHELL_PLATFORM_FUCHSIA_KEYBOARD_H_
diff --git a/src/embedder/main.cc b/src/embedder/main.cc index fd07f95..b6cd7d5 100644 --- a/src/embedder/main.cc +++ b/src/embedder/main.cc
@@ -20,6 +20,7 @@ #include <lib/sys/cpp/component_context.h> #include <lib/syslog/global.h> #include <lib/ui/scenic/cpp/view_identity.h> +#include <lib/ui/scenic/cpp/view_ref_pair.h> #include <zircon/status.h> #include <string> @@ -31,6 +32,7 @@ #include "src/embedder/logging.h" #include "src/embedder/mouse_delegate.h" #include "src/embedder/software_surface.h" +#include "src/embedder/text_delegate.h" #include "src/embedder/touch_delegate.h" namespace embedder { @@ -84,6 +86,13 @@ FX_LOG(INFO, tag, message); } +/// Callback for the embedder's FlutterEngineSendKeyEvent function. +void FuchsiaFlutterKeyEventCallback(bool handled, void* user_data) { + // TODO(naudzghebre): Currently this callback isn't being fired. Do something meaningful + // here. + FX_LOG(ERROR, embedder::kLogTag, "FlutterKeyEventCallback invoked!"); +} + /// Fuchsia implementation of the Flutter Engine's software rendering /// |surface_acquire_callback|. bool FuchsiaAcquireSoftwareSurface(void* user_data, size_t width, size_t height, @@ -263,6 +272,27 @@ }; FlutterEngineSendWindowMetricsEvent(embedder.engine, &window_metrics_event); + // Connect to Keyboard service. + fuchsia::ui::input3::KeyboardHandle keyboard; + zx_status_t keyboard_status = + embedder.component_context->svc()->Connect<fuchsia::ui::input3::Keyboard>( + keyboard.NewRequest()); + if (keyboard_status != ZX_OK) { + FX_LOGF(ERROR, embedder::kLogTag, "Failed to connect to fuchsia::ui::input3::Keyboard: %s", + zx_status_get_string(keyboard_status)); + return EXIT_FAILURE; + } + + // Connect to ImeService service. + fuchsia::ui::input::ImeServiceHandle ime_service; + zx_status_t ime_status = + embedder.component_context->svc()->Connect<fuchsia::ui::input::ImeService>( + ime_service.NewRequest()); + if (ime_status != ZX_OK) { + FX_LOGF(ERROR, embedder::kLogTag, "Failed to connect to fuchsia::ui::input::ImeService: %s", + zx_status_get_string(ime_status)); + } + // Connect to Flatland. fuchsia::ui::composition::FlatlandHandle flatland_handle; zx_status_t status = @@ -288,14 +318,39 @@ // TODO(akbiggs): What should we do after a frame finishes presenting? }); + // The protocol endpoints bound to a Flatland ViewCreationToken. + fuchsia::ui::composition::ViewBoundProtocols flatland_view_protocols; + // Mouse & Touch source fuchsia::ui::pointer::MouseSourceHandle mouse_source; fuchsia::ui::pointer::TouchSourceHandle touch_source; - fuchsia::ui::composition::ViewBoundProtocols protocols; - protocols.set_touch_source(touch_source.NewRequest()); - protocols.set_mouse_source(mouse_source.NewRequest()); + flatland_view_protocols.set_touch_source(touch_source.NewRequest()); + flatland_view_protocols.set_mouse_source(mouse_source.NewRequest()); + + auto view_ref_pair = scenic::ViewRefPair::New(); + + fuchsia::ui::views::ViewRef platform_view_ref; + view_ref_pair.view_ref.Clone(&platform_view_ref); + + // Keyboard/Text input. + auto text_delegate = std::make_unique<embedder::TextDelegate>( + std::move(platform_view_ref), std::move(ime_service), std::move(keyboard), + [&embedder](const FlutterKeyEvent* event) { + // Send key event to the engine. + FlutterEngineResult result = FlutterEngineSendKeyEvent( + embedder.engine, event, embedder::FuchsiaFlutterKeyEventCallback, &embedder); + + // This is here for logging success/failure. Eventually remove. + if (result == kSuccess) { + FX_LOG(INFO, embedder::kLogTag, "FlutterEngineResult: SUCCESS"); + } else { + FX_LOG(INFO, embedder::kLogTag, "FlutterEngineResult: FAIL"); + } + }); + embedder::FlatlandViewProvider view_provider(embedder.flatland_connection.get(), - std::move(protocols)); + std::move(view_ref_pair), + std::move(flatland_view_protocols)); auto send_pointer_events_to_engine = [&embedder](std::vector<FlutterPointerEvent> events) { if (events.empty())
diff --git a/src/embedder/meta/embedder.cml b/src/embedder/meta/embedder.cml index 1a14109..97c2788 100644 --- a/src/embedder/meta/embedder.cml +++ b/src/embedder/meta/embedder.cml
@@ -57,6 +57,11 @@ "fuchsia.ui.composition.Allocator", // For compositing/presenting views. "fuchsia.ui.composition.Flatland", + // For connecting to the Ime service. + "fuchsia.ui.input.ImeService", + // For connecting to the Keyboard service. + "fuchsia.ui.input3.Keyboard", + "fuchsia.ui.scenic.Scenic", ] }, ],
diff --git a/src/embedder/software_surface.cc b/src/embedder/software_surface.cc index 6f72aaf..9ee30ed 100644 --- a/src/embedder/software_surface.cc +++ b/src/embedder/software_surface.cc
@@ -311,10 +311,11 @@ } if (read_finished_callback_ != nullptr) { - FX_LOG(ERROR, kLogTag, - "Attempted to signal a write on the surface when the " - "previous write has not yet been acknowledged by the " - "compositor."); + // Commented out until we figure out why this is being triggered. It's spamming the logs. + // FX_LOG(ERROR, kLogTag, + // "Attempted to signal a write on the surface when the " + // "previous write has not yet been acknowledged by the " + // "compositor."); return; } read_finished_callback_ = on_read_finished;
diff --git a/src/embedder/text_delegate.cc b/src/embedder/text_delegate.cc new file mode 100644 index 0000000..e211b5b --- /dev/null +++ b/src/embedder/text_delegate.cc
@@ -0,0 +1,235 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "src/embedder/text_delegate.h" + +#include <fuchsia/ui/input/cpp/fidl.h> +#include <fuchsia/ui/input3/cpp/fidl.h> +#include <fuchsia/ui/views/cpp/fidl.h> +#include <lib/fidl/cpp/binding.h> +#include <lib/syslog/global.h> +#include <zircon/status.h> + +#include "logging.h" +#include "src/embedder/engine/embedder.h" +#include "src/embedder/fuchsia_logger.h" +#include "src/embedder/keyboard.h" +#include "src/embedder/logging.h" + +namespace embedder { + +// static constexpr char kInputActionKey[] = "inputAction"; + +// See: https://api.flutter.dev/flutter/services/TextInputAction.html +// Only the actions relevant for Fuchsia are listed here. +static constexpr char kTextInputActionDone[] = "TextInputAction.done"; +static constexpr char kTextInputActionNewline[] = "TextInputAction.newline"; +static constexpr char kTextInputActionGo[] = "TextInputAction.go"; +static constexpr char kTextInputActionNext[] = "TextInputAction.next"; +static constexpr char kTextInputActionPrevious[] = "TextInputAction.previous"; +static constexpr char kTextInputActionNone[] = "TextInputAction.none"; +static constexpr char kTextInputActionSearch[] = "TextInputAction.search"; +static constexpr char kTextInputActionSend[] = "TextInputAction.send"; +static constexpr char kTextInputActionUnspecified[] = "TextInputAction.unspecified"; + +[[maybe_unused]] +// Converts Flutter TextInputAction to Fuchsia action enum. +static fuchsia::ui::input::InputMethodAction +IntoInputMethodAction(const std::string action_string) { + if (action_string == kTextInputActionNewline) { + return fuchsia::ui::input::InputMethodAction::NEWLINE; + } else if (action_string == kTextInputActionDone) { + return fuchsia::ui::input::InputMethodAction::DONE; + } else if (action_string == kTextInputActionGo) { + return fuchsia::ui::input::InputMethodAction::GO; + } else if (action_string == kTextInputActionNext) { + return fuchsia::ui::input::InputMethodAction::NEXT; + } else if (action_string == kTextInputActionPrevious) { + return fuchsia::ui::input::InputMethodAction::PREVIOUS; + } else if (action_string == kTextInputActionNone) { + return fuchsia::ui::input::InputMethodAction::NONE; + } else if (action_string == kTextInputActionSearch) { + return fuchsia::ui::input::InputMethodAction::SEARCH; + } else if (action_string == kTextInputActionSend) { + return fuchsia::ui::input::InputMethodAction::SEND; + } else if (action_string == kTextInputActionUnspecified) { + return fuchsia::ui::input::InputMethodAction::UNSPECIFIED; + } + // If this message comes along it means we should really add the missing 'if' + // above. + FX_LOGF(INFO, kLogTag, "unexpected action_string: %s", action_string.c_str()); + // Substituting DONE for an unexpected action string will probably be OK. + return fuchsia::ui::input::InputMethodAction::DONE; +} + +[[maybe_unused]] +// Converts the Fuchsia action enum into Flutter TextInputAction. +static const std::string +IntoTextInputAction(fuchsia::ui::input::InputMethodAction action) { + if (action == fuchsia::ui::input::InputMethodAction::NEWLINE) { + return kTextInputActionNewline; + } else if (action == fuchsia::ui::input::InputMethodAction::DONE) { + return kTextInputActionDone; + } else if (action == fuchsia::ui::input::InputMethodAction::GO) { + return kTextInputActionGo; + } else if (action == fuchsia::ui::input::InputMethodAction::NEXT) { + return kTextInputActionNext; + } else if (action == fuchsia::ui::input::InputMethodAction::PREVIOUS) { + return kTextInputActionPrevious; + } else if (action == fuchsia::ui::input::InputMethodAction::NONE) { + return kTextInputActionNone; + } else if (action == fuchsia::ui::input::InputMethodAction::SEARCH) { + return kTextInputActionSearch; + } else if (action == fuchsia::ui::input::InputMethodAction::SEND) { + return kTextInputActionSend; + } else if (action == fuchsia::ui::input::InputMethodAction::UNSPECIFIED) { + return kTextInputActionUnspecified; + } + // If this message comes along it means we should really add the missing 'if' + // above. + FX_LOGF(INFO, kLogTag, "unexpected action: %d", static_cast<uint32_t>(action)); + + // Substituting "done" for an unexpected text input action will probably + // be OK. + return kTextInputActionDone; +} + +// TODO(fxbug.dev/8868): Terminate embedder if Fuchsia system FIDL connections +// have error. +template <class T> +void SetInterfaceErrorHandler(fidl::InterfacePtr<T>& interface, std::string name) { + interface.set_error_handler([name](zx_status_t status) { + FX_LOGF(ERROR, kLogTag, "Interface error on: %s, status: %s", name.c_str(), + zx_status_get_string(status)); + }); +} + +template <class T> +void SetInterfaceErrorHandler(fidl::Binding<T>& binding, std::string name) { + binding.set_error_handler([name](zx_status_t status) { + FX_LOGF(ERROR, kLogTag, "Binding error on: %s, status: %s", name.c_str(), + zx_status_get_string(status)); + }); +} + +TextDelegate::TextDelegate(fuchsia::ui::views::ViewRef view_ref, + fuchsia::ui::input::ImeServiceHandle ime_service, + fuchsia::ui::input3::KeyboardHandle keyboard, + std::function<void(const FlutterKeyEvent* event)> dispatch_callback) + : dispatch_callback_(dispatch_callback), + ime_client_(this), + text_sync_service_(ime_service.Bind()), + keyboard_listener_binding_(this), + keyboard_(keyboard.Bind()) { + // Register all error handlers. + SetInterfaceErrorHandler(ime_, "Input Method Editor"); + SetInterfaceErrorHandler(ime_client_, "IME Client"); + SetInterfaceErrorHandler(text_sync_service_, "Text Sync Service"); + SetInterfaceErrorHandler(keyboard_listener_binding_, "Keyboard Listener"); + SetInterfaceErrorHandler(keyboard_, "Keyboard"); + + // Configure keyboard listener. + keyboard_->AddListener(std::move(view_ref), keyboard_listener_binding_.NewBinding(), [] {}); +} + +void TextDelegate::ActivateIme() { + ActivateIme(requested_text_action_.value_or(fuchsia::ui::input::InputMethodAction::DONE)); +} + +void TextDelegate::ActivateIme(fuchsia::ui::input::InputMethodAction action) { + FX_DCHECK(last_text_state_.has_value()); + + requested_text_action_ = action; + text_sync_service_->GetInputMethodEditor(fuchsia::ui::input::KeyboardType::TEXT, // keyboard type + action, // input method action + last_text_state_.value(), // initial state + ime_client_.NewBinding(), // client + ime_.NewRequest() // editor + ); +} + +void TextDelegate::DeactivateIme() { + if (ime_) { + text_sync_service_->HideKeyboard(); + ime_ = nullptr; + } + if (ime_client_.is_bound()) { + ime_client_.Unbind(); + } +} + +// |fuchsia::ui::input::InputMethodEditorClient| +void TextDelegate::DidUpdateState(fuchsia::ui::input::TextInputState state, + std::unique_ptr<fuchsia::ui::input::InputEvent> input_event) { + FX_LOG(ERROR, kLogTag, "TextDelegate::DidUpdateState not implemented."); +} + +// |fuchsia::ui::input::InputMethodEditorClient| +void TextDelegate::OnAction(fuchsia::ui::input::InputMethodAction action) { + FX_LOG(ERROR, kLogTag, "TextDelegate::OnAction not implemented."); +} + +// |fuchsia::ui:input3::KeyboardListener| +void TextDelegate::OnKeyEvent( + fuchsia::ui::input3::KeyEvent key_event, + fuchsia::ui::input3::KeyboardListener::OnKeyEventCallback on_key_event_callback) { + const char* type = nullptr; + FlutterKeyEventType flutter_key_type; + switch (key_event.type()) { + case fuchsia::ui::input3::KeyEventType::PRESSED: + type = "keydown"; + flutter_key_type = FlutterKeyEventType::kFlutterKeyEventTypeDown; + break; + case fuchsia::ui::input3::KeyEventType::RELEASED: + type = "keyup"; + flutter_key_type = FlutterKeyEventType::kFlutterKeyEventTypeUp; + break; + case fuchsia::ui::input3::KeyEventType::SYNC: + // SYNC means the key was pressed while focus was not on this application. + // This should possibly behave like PRESSED in the future, though it + // doesn't hurt to ignore it today. + case fuchsia::ui::input3::KeyEventType::CANCEL: + // CANCEL means the key was released while focus was not on this + // application. + // This should possibly behave like RELEASED in the future to ensure that + // a key is not repeated forever if it is pressed while focus is lost. + default: + break; + } + + if (type == nullptr) { + FX_LOGF(INFO, kLogTag, "Unknown key event phase."); + + // Notify the key event wasn't handled by this keyboard listener. + on_key_event_callback(fuchsia::ui::input3::KeyEventStatus::NOT_HANDLED); + return; + } + + keyboard_translator_.ConsumeEvent(std::move(key_event)); + + // Encourage the Engine to vsync for a 60Hz display. + const uint64_t now_nanos = FlutterEngineGetCurrentTime(); + const uint64_t now_micro = now_nanos * 1e3; + + FlutterKeyEvent flutter_key_event = { + .struct_size = sizeof(FlutterKeyEvent), + .timestamp = static_cast<double>(now_micro), + .type = flutter_key_type, + .physical = static_cast<uint64_t>(keyboard_translator_.LastHIDUsage()), + .logical = static_cast<uint64_t>(keyboard_translator_.LastCodePoint()), + .character = strcmp(type, "keydown") == 0 + ? keyboard_translator_.LastCodePointAsChar() + : nullptr, // Only assign the character on keydown strokes. + .synthesized = + true}; // For now we want the hardware keyboard to handle this event, and therefore the + // synthesized field is set to true and needs to take this codepath: + // https://github.com/flutter/flutter/blob/master/packages/flutter/lib/src/services/hardware_keyboard.dart#L873 + + // Send key event to engine. + dispatch_callback_(&flutter_key_event); + + // Notify the key event was handled by this keyboard listener. + on_key_event_callback(fuchsia::ui::input3::KeyEventStatus::HANDLED); +} +} // namespace embedder
diff --git a/src/embedder/text_delegate.h b/src/embedder/text_delegate.h new file mode 100644 index 0000000..d688e1c --- /dev/null +++ b/src/embedder/text_delegate.h
@@ -0,0 +1,143 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef SRC_EMBEDDER_TEXT_DELEGATE_H_ +#define SRC_EMBEDDER_TEXT_DELEGATE_H_ + +#include <fuchsia/ui/input/cpp/fidl.h> +#include <fuchsia/ui/input3/cpp/fidl.h> +#include <fuchsia/ui/views/cpp/fidl.h> +#include <lib/fidl/cpp/binding.h> + +#include <memory> + +#include "src/embedder/engine/embedder.h" +#include "src/embedder/keyboard.h" +#include "src/embedder/logging.h" + +namespace embedder { + +/// The channel name used for text editing platofrm messages. +constexpr char kTextInputChannel[] = "flutter/textinput"; + +/// The channel name used for key event platform messages. +constexpr char kKeyEventChannel[] = "flutter/keyevent"; + +/// TextDelegate handles keyboard input and text editing. +/// +/// It mediates between Fuchsia's input and Flutter's platform messages. When it +/// is initialized, it contacts `fuchsia.ui.input.Keyboard` to register itself +/// as listener of key events. +/// +/// Whenever a text editing request comes from the +/// Flutter app, it will activate Fuchsia's input method editor, and will send +/// text edit actions coming from the Fuchsia platform over to the Flutter app, +/// by converting FIDL messages (`fuchsia.ui.input.InputMethodEditorClient` +/// calls) to appropriate text editing Flutter platform messages. +/// +/// For details refer to: +/// * Flutter side: +/// https://api.flutter.dev/javadoc/io/flutter/embedding/engine/systemchannels/TextInputChannel.html +/// * Fuchsia side: https://fuchsia.dev/reference/fidl/fuchsia.ui.input +/// TODO(naudzghebre): +/// 1. Handle 'flutter/keyevent' platform messages +/// 2. Handle 'flutter/textinput' platform messages +class TextDelegate : public fuchsia::ui::input3::KeyboardListener, + public fuchsia::ui::input::InputMethodEditorClient { + public: + /// Creates a new TextDelegate. + /// + /// Args: + /// view_ref: the reference to the app's view. Required for registration + /// with Fuchsia. + /// ime_service: a handle to Fuchsia's input method service. + /// keyboard: the keyboard listener, gets notified of key presses and + /// releases. + /// dispatch_callback: a function used to send a Flutter platform message. + TextDelegate(fuchsia::ui::views::ViewRef view_ref, + fuchsia::ui::input::ImeServiceHandle ime_service, + fuchsia::ui::input3::KeyboardHandle keyboard, + std::function<void(const FlutterKeyEvent*)> dispatch_callback); + TextDelegate(const TextDelegate&) = delete; + TextDelegate& operator=(const TextDelegate&) = delete; + + /// |fuchsia.ui.input3.KeyboardListener| + /// Called by the embedder every time there is a key event to process. + void OnKeyEvent(fuchsia::ui::input3::KeyEvent key_event, + fuchsia::ui::input3::KeyboardListener::OnKeyEventCallback callback) override; + + /// |fuchsia::ui::input::InputMethodEditorClient| + /// Called by the embedder every time the edit state is updated. + void DidUpdateState(fuchsia::ui::input::TextInputState state, + std::unique_ptr<fuchsia::ui::input::InputEvent> event) override; + + /// |fuchsia::ui::input::InputMethodEditorClient| + /// Called by the embedder when the action key is pressed, and the requested + /// action is supplied to Flutter. + void OnAction(fuchsia::ui::input::InputMethodAction action) override; + + /// Gets a new input method editor from the input connection. Run when both + /// Scenic has focus and Flutter has requested input with setClient. + void ActivateIme(); + + /// Detaches the input method editor connection, ending the edit session and + /// closing the onscreen keyboard. Call when input is no longer desired, + /// either because Scenic says we lost focus or when Flutter no longer has a + /// text field focused. + void DeactivateIme(); + + /// Returns true if there is a text state (i.e. if some text editing is in + /// progress). + bool HasTextState() { return last_text_state_.has_value(); } + + private: + /// Activates the input method editor, assigning |action| to the "enter" key. + /// This action will be reported by |OnAction| above when the "enter" key is + /// pressed. Note that in the case of multi-line text editors, |OnAction| will + /// never be called: instead, the text editor will insert a newline into the + /// edited text. + void ActivateIme(fuchsia::ui::input::InputMethodAction action); + + /// Converts Fuchsia platform key codes into Flutter key codes. + Keyboard keyboard_translator_; + + /// A callback for sending a single Flutter platform message. + std::function<void(const FlutterKeyEvent* event)> dispatch_callback_; + + /// TextDelegate server-side binding. Methods called when the text edit state + /// is updated. + fidl::Binding<fuchsia::ui::input::InputMethodEditorClient> ime_client_; + + /// An interface for interacting with a text input control. + fuchsia::ui::input::InputMethodEditorPtr ime_; + + /// An interface for requesting the InputMethodEditor. + fuchsia::ui::input::ImeServicePtr text_sync_service_; + + /// The locally-unique identifier of the text input currently in use. Flutter + /// usually uses only one at a time. + /// int current_text_input_client_ = 0; + + /// TextDelegate server side binding. Methods called when a key is pressed. + fidl::Binding<fuchsia::ui::input3::KeyboardListener> keyboard_listener_binding_; + + /// The client-side stub for calling the Keyboard protocol. + fuchsia::ui::input3::KeyboardPtr keyboard_; + + /// last_text_state_ is the last state of the text input as reported by the IME + /// or initialized by Flutter. We set it to null if Flutter doesn't want any + /// input, since then there is no text input state at all. + /// If nullptr, then no editing is in progress. + std::optional<fuchsia::ui::input::TextInputState> last_text_state_; + + /// The action that Flutter expects to happen when the user presses the "enter" + /// key. For example, it could be `InputMethodAction::DONE` which would cause + /// text editing to stop and the current text to be accepted. + /// If set to std::nullopt, then no editing is in progress. + std::optional<fuchsia::ui::input::InputMethodAction> requested_text_action_; +}; + +} // namespace embedder + +#endif // SRC_EMBEDDER_TEXT_DELEGATE_H_
diff --git a/src/examples/hello_flutter/lib/main.dart b/src/examples/hello_flutter/lib/main.dart index 6e82b47..e66ea45 100644 --- a/src/examples/hello_flutter/lib/main.dart +++ b/src/examples/hello_flutter/lib/main.dart
@@ -1,4 +1,5 @@ import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; void main() { print("Hello from Dart!"); @@ -51,6 +52,25 @@ class _MyHomePageState extends State<MyHomePage> { int _counter = 0; + late TextEditingController _controller; + + @override + void initState() { + super.initState(); + _controller = TextEditingController(); + + // Temporary hack to display key events to the textfield. + HardwareKeyboard.instance.addHandler((event) { + _controller.text = _controller.text + (event.character ?? ""); + return true; + }); + } + + @override + void dispose() { + _controller.dispose(); + super.dispose(); + } void _incrementCounter() { setState(() { @@ -104,6 +124,19 @@ '$_counter', style: Theme.of(context).textTheme.headline4, ), + Padding( + padding: EdgeInsets.all(15), + child: SizedBox( + width: 200, + child: TextField( + controller: _controller, + decoration: InputDecoration( + border: OutlineInputBorder(), + labelText: 'Input Text', + ), + ), + ), + ), ], ), ),
diff --git a/third_party/BUILD.bazel b/third_party/BUILD.bazel new file mode 100644 index 0000000..04f2c62 --- /dev/null +++ b/third_party/BUILD.bazel
@@ -0,0 +1,9 @@ +# Copyright 2022 The Fuchsia Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +cc_library( + name = "rapidjson", + srcs = glob(["rapidjson/include/**/*.h"]), + visibility = ["//visibility:public"], +)
diff --git a/third_party/rapidjson b/third_party/rapidjson new file mode 160000 index 0000000..ef8fd47 --- /dev/null +++ b/third_party/rapidjson
@@ -0,0 +1 @@ +Subproject commit ef8fd47794c25c265d978c66ab99ada5a3343f00