LibWeb+LibWebView: Migrate Browser's input event handling to LibWebView

The Serenity chrome is the only chrome thus far that sends all input key
and mouse events to WebContent, including shortcut activations. This is
necessary for all chromes - we must give web pages a chance to intercept
input events before handling them ourselves.

To make this easier for other chromes, this patch moves Serenity's input
event handling to LibWebView. To do so, we add the Web::InputEvent type,
which models the event data we need within LibWeb. Chromes will then be
responsible for converting between this type and their native events.

This class lives in LibWeb (rather than LibWebView) because the plan is
to use it wholesale throughout the Page's event handler and across IPC.
Right now, we still send the individual fields of the event over IPC,
but it will be an easy refactor to send the event itself. We just can't
do this until all chromes have been ported to this event queueing.

Also note that we now only handle key input events back in the chrome.
WebContent handles all mouse events that it possibly can. If it was not
able to handle a mouse event, there's nothing for the chrome to do (i.e.
there is no clicking, scrolling, etc. the chrome is able to do if the
WebContent couldn't).

Meta/gn/secondary/Ladybird/BUILD.gn

@@ -40,6 +40,7 @@ link_qt("ladybird_qt_components") {
 
 config("ladybird_config") {
   include_dirs = [
+    "//Userland",
     "//Userland/Applications",
     "//Userland/Services",
   ]

Userland/Libraries/LibWeb/Page/InputEvent.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2024, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/OwnPtr.h>
+#include <AK/Variant.h>
+#include <LibGfx/Point.h>
+#include <LibWeb/PixelUnits.h>
+
+// FIXME: These should not be included outside of Serenity. This FIXME appears in several locations across the Ladybird
+//        chromes. The classes in this file provide a good opportunity to remove LibGUI and Kernel types from LibWeb.
+#include <Kernel/API/KeyCode.h>
+#include <LibGUI/Event.h>
+
+namespace Web {
+
+struct ChromeInputData {
+    virtual ~ChromeInputData() = default;
+};
+
+struct KeyEvent {
+public:
+    enum class Type {
+        KeyDown,
+        KeyUp,
+    };
+
+    Type type;
+    KeyCode key { KeyCode::Key_Invalid };
+    KeyModifier modifiers { KeyModifier::Mod_None };
+    u32 code_point { 0 };
+
+    OwnPtr<ChromeInputData> chrome_data;
+};
+
+struct MouseEvent {
+public:
+    enum class Type {
+        MouseDown,
+        MouseUp,
+        MouseMove,
+        MouseWheel,
+        DoubleClick,
+    };
+
+    Type type;
+    Web::DevicePixelPoint position;
+    Web::DevicePixelPoint screen_position;
+    GUI::MouseButton button { GUI::MouseButton::None };
+    GUI::MouseButton buttons { GUI::MouseButton::None };
+    KeyModifier modifiers { KeyModifier::Mod_None };
+    int wheel_delta_x { 0 };
+    int wheel_delta_y { 0 };
+
+    OwnPtr<ChromeInputData> chrome_data;
+};
+
+using InputEvent = Variant<KeyEvent, MouseEvent>;
+
+}

Userland/Libraries/LibWebView/OutOfProcessWebView.cpp

@@ -69,8 +69,8 @@ OutOfProcessWebView::OutOfProcessWebView()
         GUI::Application::the()->hide_tooltip();
     };
 
-    on_finish_handling_input_event = [this](auto event_was_accepted) {
+    on_finish_handling_key_event = [this](auto const& event) {
-        did_finish_handling_input_event(event_was_accepted);
+        finish_handling_key_event(event);
     };
 
     on_request_worker_agent = []() {
@@ -169,22 +169,22 @@ void OutOfProcessWebView::update_zoom()
 
 void OutOfProcessWebView::keydown_event(GUI::KeyEvent& event)
 {
-    enqueue_input_event(event);
+    enqueue_native_event(Web::KeyEvent::Type::KeyDown, event);
 }
 
 void OutOfProcessWebView::keyup_event(GUI::KeyEvent& event)
 {
-    enqueue_input_event(event);
+    enqueue_native_event(Web::KeyEvent::Type::KeyUp, event);
 }
 
 void OutOfProcessWebView::mousedown_event(GUI::MouseEvent& event)
 {
-    enqueue_input_event(event);
+    enqueue_native_event(Web::MouseEvent::Type::MouseDown, event);
 }
 
 void OutOfProcessWebView::mouseup_event(GUI::MouseEvent& event)
 {
-    enqueue_input_event(event);
+    enqueue_native_event(Web::MouseEvent::Type::MouseUp, event);
 
     if (event.button() == GUI::MouseButton::Backward) {
         if (on_navigate_back)
@@ -197,17 +197,17 @@ void OutOfProcessWebView::mouseup_event(GUI::MouseEvent& event)
 
 void OutOfProcessWebView::mousemove_event(GUI::MouseEvent& event)
 {
-    enqueue_input_event(event);
+    enqueue_native_event(Web::MouseEvent::Type::MouseMove, event);
 }
 
 void OutOfProcessWebView::mousewheel_event(GUI::MouseEvent& event)
 {
-    enqueue_input_event(event);
+    enqueue_native_event(Web::MouseEvent::Type::MouseWheel, event);
 }
 
 void OutOfProcessWebView::doubleclick_event(GUI::MouseEvent& event)
 {
-    enqueue_input_event(event);
+    enqueue_native_event(Web::MouseEvent::Type::DoubleClick, event);
 }
 
 void OutOfProcessWebView::theme_change_event(GUI::ThemeChangeEvent& event)
@@ -305,99 +305,52 @@ void OutOfProcessWebView::hide_event(GUI::HideEvent&)
     set_system_visibility_state(false);
 }
 
-void OutOfProcessWebView::enqueue_input_event(InputEvent const& event)
+void OutOfProcessWebView::enqueue_native_event(Web::MouseEvent::Type type, GUI::MouseEvent const& event)
 {
-    m_pending_input_events.enqueue(event);
-    process_next_input_event();
-}
-
-void OutOfProcessWebView::process_next_input_event()
-{
-    if (m_pending_input_events.is_empty())
-        return;
-
-    if (m_is_awaiting_response_for_input_event)
-        return;
-    m_is_awaiting_response_for_input_event = true;
-
-    // Send the next event over to the web content to be handled by JS.
-    // We'll later get a message to say whether JS prevented the default event behavior,
-    // at which point we either discard or handle that event, then try and process the next one.
-    auto event = m_pending_input_events.head();
-    event.visit(
-        [this](GUI::KeyEvent const& event) {
-            switch (event.type()) {
-            case GUI::Event::Type::KeyDown:
-                client().async_key_down(m_client_state.page_index, event.key(), event.modifiers(), event.code_point());
-                break;
-            case GUI::Event::Type::KeyUp:
-                client().async_key_up(m_client_state.page_index, event.key(), event.modifiers(), event.code_point());
-                break;
-            default:
-                dbgln("Unrecognized key event type in OOPWV input event queue: {}", event.type());
-                VERIFY_NOT_REACHED();
-            }
-        },
-        [this](GUI::MouseEvent const& event) {
     auto position = to_content_position(event.position()).to_type<Web::DevicePixels>();
     auto screen_position = (event.position() + (window()->position() + relative_position())).to_type<Web::DevicePixels>();
-            switch (event.type()) {
+
-            case GUI::Event::Type::MouseDown:
-                client().async_mouse_down(m_client_state.page_index, position, screen_position, event.button(), event.buttons(), event.modifiers());
-                break;
-            case GUI::Event::Type::MouseUp:
-                client().async_mouse_up(m_client_state.page_index, position, screen_position, event.button(), event.buttons(), event.modifiers());
-                break;
-            case GUI::Event::Type::MouseMove:
-                client().async_mouse_move(m_client_state.page_index, position, screen_position, event.button(), event.buttons(), event.modifiers());
-                break;
-            case GUI::Event::Type::MouseWheel: {
     // FIXME: This wheel delta step size multiplier is used to remain the old scroll behaviour, in future use system step size.
-                constexpr int scroll_step_size = 24;
+    static constexpr int SCROLL_STEP_SIZE = 24;
-                client().async_mouse_wheel(m_client_state.page_index, position, screen_position, event.button(), event.buttons(), event.modifiers(), event.wheel_delta_x() * scroll_step_size, event.wheel_delta_y() * scroll_step_size);
+    auto wheel_delta_x = event.wheel_delta_x() * SCROLL_STEP_SIZE;
-                break;
+    auto wheel_delta_y = event.wheel_delta_y() * SCROLL_STEP_SIZE;
-            }
+
-            case GUI::Event::Type::MouseDoubleClick:
+    enqueue_input_event(Web::MouseEvent { type, position, screen_position, event.button(), static_cast<GUI::MouseButton>(event.buttons()), static_cast<KeyModifier>(event.modifiers()), wheel_delta_x, wheel_delta_y, nullptr });
-                client().async_doubleclick(m_client_state.page_index, position, screen_position, event.button(), event.buttons(), event.modifiers());
-                break;
-            default:
-                dbgln("Unrecognized mouse event type in OOPWV input event queue: {}", event.type());
-                VERIFY_NOT_REACHED();
-            }
-        });
 }
 
-void OutOfProcessWebView::did_finish_handling_input_event(bool event_was_accepted)
+struct KeyData : Web::ChromeInputData {
+    explicit KeyData(GUI::KeyEvent const& event)
+        : event(make<GUI::KeyEvent>(event))
     {
-    VERIFY(m_is_awaiting_response_for_input_event);
+    }
 
-    auto event = m_pending_input_events.dequeue();
+    NonnullOwnPtr<GUI::KeyEvent> event;
-    m_is_awaiting_response_for_input_event = false;
+};
 
-    if (!event_was_accepted) {
+void OutOfProcessWebView::enqueue_native_event(Web::KeyEvent::Type type, GUI::KeyEvent const& event)
-        // Here we handle events that were not consumed or cancelled by web content.
+{
-        // That is, we manually implement the steps that would have happened if the original
+    enqueue_input_event(Web::KeyEvent { type, event.key(), static_cast<KeyModifier>(event.modifiers()), event.code_point(), make<KeyData>(event) });
-        // OutOfProcessWebView::foo_event() had called event.ignore().
+}
+
+void OutOfProcessWebView::finish_handling_key_event(Web::KeyEvent const& key_event)
+{
+    // First, we give our superclass a chance to handle the event.
     //
-        // The first step is to give our superclass a chance to handle the event.
+    // If it does not, we dispatch the event to our parent widget, but limited such that it will never bubble up to the
+    // Window. (Otherwise, it would then dispatch the event to us since we are the focused widget, and it would go around
+    // indefinitely.)
     //
-        // Then, if it does not, we dispatch the event to our parent widget, but limited so
+    // Finally, any unhandled KeyDown events are propagated to trigger any shortcut Actions.
-        // that it will never bubble up to the Window. (Otherwise, it would then dispatch the
+    auto& chrome_data = verify_cast<KeyData>(*key_event.chrome_data);
-        // event to us since we are the focused widget, and it would go round indefinitely.)
+    auto& event = *chrome_data.event;
-        //
+
-        // Finally, any unhandled KeyDown events are propagated to trigger any Actions.
+    switch (key_event.type) {
-        event.visit(
+    case Web::KeyEvent::Type::KeyDown:
-            [this](GUI::KeyEvent& event) {
-                switch (event.type()) {
-                case GUI::Event::Type::KeyDown:
         Super::keydown_event(event);
         break;
-                case GUI::Event::Type::KeyUp:
+    case Web::KeyEvent::Type::KeyUp:
         Super::keyup_event(event);
         break;
-                default:
-                    dbgln("Unhandled key event type in OOPWV input event queue: {}", event.type());
-                    VERIFY_NOT_REACHED();
     }
 
     if (!event.is_accepted()) {
@@ -405,38 +358,8 @@ void OutOfProcessWebView::did_finish_handling_input_event(bool event_was_accepte
 
         // NOTE: If other events can ever trigger shortcuts, propagate those here.
         if (!event.is_accepted() && event.type() == GUI::Event::Type::KeyDown)
-                        window()->propagate_shortcuts(event, this);
+            window()->propagate_shortcuts(static_cast<GUI::KeyEvent&>(event), this);
     }
-            },
-            [this](GUI::MouseEvent& event) {
-                switch (event.type()) {
-                case GUI::Event::Type::MouseDown:
-                    Super::mousedown_event(event);
-                    break;
-                case GUI::Event::Type::MouseUp:
-                    Super::mouseup_event(event);
-                    break;
-                case GUI::Event::Type::MouseMove:
-                    Super::mousemove_event(event);
-                    break;
-                case GUI::Event::Type::MouseWheel:
-                    Super::mousewheel_event(event);
-                    break;
-                case GUI::Event::Type::MouseDoubleClick:
-                    Super::doubleclick_event(event);
-                    break;
-                default:
-                    dbgln("Unhandled mouse event type in OOPWV input event queue: {}", event.type());
-                    VERIFY_NOT_REACHED();
-                }
-
-                if (!event.is_accepted())
-                    parent_widget()->dispatch_event(event, window());
-                // FIXME: Propagate event for mouse-button shortcuts once that is implemented.
-            });
-    }
-
-    process_next_input_event();
 }
 
 void OutOfProcessWebView::set_content_scales_to_viewport(bool b)

Userland/Libraries/LibWebView/OutOfProcessWebView.h

@@ -7,7 +7,6 @@
 
 #pragma once
 
-#include <AK/Queue.h>
 #include <AK/URL.h>
 #include <LibGUI/AbstractScrollableWidget.h>
 #include <LibGUI/Widget.h>
@@ -85,13 +84,9 @@ private:
     virtual Gfx::IntPoint to_content_position(Gfx::IntPoint widget_position) const override;
     virtual Gfx::IntPoint to_widget_position(Gfx::IntPoint content_position) const override;
 
-    using InputEvent = Variant<GUI::KeyEvent, GUI::MouseEvent>;
+    void enqueue_native_event(Web::MouseEvent::Type, GUI::MouseEvent const& event);
-    void enqueue_input_event(InputEvent const&);
+    void enqueue_native_event(Web::KeyEvent::Type, GUI::KeyEvent const& event);
-    void process_next_input_event();
+    void finish_handling_key_event(Web::KeyEvent const&);
-    void did_finish_handling_input_event(bool event_was_accepted);
-
-    bool m_is_awaiting_response_for_input_event { false };
-    Queue<InputEvent> m_pending_input_events;
 
     bool m_content_scales_to_viewport { false };
 };

Userland/Libraries/LibWebView/ViewImplementation.cpp

@@ -112,6 +112,60 @@ void ViewImplementation::reset_zoom()
     update_zoom();
 }
 
+void ViewImplementation::enqueue_input_event(Web::InputEvent event)
+{
+    // Send the next event over to the WebContent to be handled by JS. We'll later get a message to say whether JS
+    // prevented the default event behavior, at which point we either discard or handle that event, and then try to
+    // process the next one.
+    m_pending_input_events.enqueue(move(event));
+
+    // FIXME: Replace these IPCs with a singular "async_input_event".
+    m_pending_input_events.tail().visit(
+        [this](Web::KeyEvent const& event) {
+            switch (event.type) {
+            case Web::KeyEvent::Type::KeyDown:
+                client().async_key_down(m_client_state.page_index, event.key, event.modifiers, event.code_point);
+                break;
+            case Web::KeyEvent::Type::KeyUp:
+                client().async_key_up(m_client_state.page_index, event.key, event.modifiers, event.code_point);
+                break;
+            }
+        },
+        [this](Web::MouseEvent const& event) {
+            switch (event.type) {
+            case Web::MouseEvent::Type::MouseDown:
+                client().async_mouse_down(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
+                break;
+            case Web::MouseEvent::Type::MouseUp:
+                client().async_mouse_up(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
+                break;
+            case Web::MouseEvent::Type::MouseMove:
+                client().async_mouse_move(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
+                break;
+            case Web::MouseEvent::Type::MouseWheel:
+                client().async_mouse_wheel(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers, event.wheel_delta_x, event.wheel_delta_y);
+                break;
+            case Web::MouseEvent::Type::DoubleClick:
+                client().async_doubleclick(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
+                break;
+            }
+        });
+}
+
+void ViewImplementation::did_finish_handling_input_event(Badge<WebContentClient>, bool event_was_accepted)
+{
+    auto event = m_pending_input_events.dequeue();
+
+    if (!event_was_accepted && event.has<Web::KeyEvent>()) {
+        auto const& key_event = event.get<Web::KeyEvent>();
+
+        // Here we handle events that were not consumed or cancelled by the WebContent. Propagate the event back
+        // to the concrete view implementation.
+        if (on_finish_handling_key_event)
+            on_finish_handling_key_event(key_event);
+    }
+}
+
 void ViewImplementation::set_preferred_color_scheme(Web::CSS::PreferredColorScheme color_scheme)
 {
     client().async_set_preferred_color_scheme(page_id(), color_scheme);

Userland/Libraries/LibWebView/ViewImplementation.h

@@ -10,6 +10,7 @@
 #include <AK/Forward.h>
 #include <AK/Function.h>
 #include <AK/LexicalPath.h>
+#include <AK/Queue.h>
 #include <AK/String.h>
 #include <LibCore/Promise.h>
 #include <LibGfx/Forward.h>
@@ -18,6 +19,7 @@
 #include <LibWeb/HTML/ActivateTab.h>
 #include <LibWeb/HTML/ColorPickerUpdateState.h>
 #include <LibWeb/HTML/SelectItem.h>
+#include <LibWeb/Page/InputEvent.h>
 #include <LibWebView/Forward.h>
 #include <LibWebView/WebContentClient.h>
 
@@ -52,6 +54,9 @@ public:
     float zoom_level() const { return m_zoom_level; }
     float device_pixel_ratio() const { return m_device_pixel_ratio; }
 
+    void enqueue_input_event(Web::InputEvent);
+    void did_finish_handling_input_event(Badge<WebContentClient>, bool event_was_accepted);
+
     void set_preferred_color_scheme(Web::CSS::PreferredColorScheme);
 
     ByteString selected_text();
@@ -167,7 +172,7 @@ public:
     Function<void(Color current_color)> on_request_color_picker;
     Function<void(Web::HTML::AllowMultipleFiles)> on_request_file_picker;
     Function<void(Gfx::IntPoint content_position, i32 minimum_width, Vector<Web::HTML::SelectItem> items)> on_request_select_dropdown;
-    Function<void(bool)> on_finish_handling_input_event;
+    Function<void(Web::KeyEvent const&)> on_finish_handling_key_event;
     Function<void()> on_text_test_finish;
     Function<void(Gfx::Color)> on_theme_color_change;
     Function<void(String const&, String const&, String const&)> on_insert_clipboard_entry;
@@ -234,6 +239,8 @@ protected:
     float m_zoom_level { 1.0 };
     float m_device_pixel_ratio { 1.0 };
 
+    Queue<Web::InputEvent> m_pending_input_events;
+
     RefPtr<Core::Timer> m_backing_store_shrink_timer;
 
     RefPtr<Gfx::Bitmap> m_backup_bitmap;

Userland/Libraries/LibWebView/WebContentClient.cpp

@@ -839,10 +839,9 @@ void WebContentClient::did_finish_handling_input_event(u64 page_id, bool event_w
         dbgln("Received finish handling input event for unknown page ID {}", page_id);
         return;
     }
-    auto& view = *maybe_view.value();
 
-    if (view.on_finish_handling_input_event)
+    auto& view = *maybe_view.value();
-        view.on_finish_handling_input_event(event_was_accepted);
+    view.did_finish_handling_input_event({}, event_was_accepted);
 }
 
 void WebContentClient::did_change_theme_color(u64 page_id, Gfx::Color color)