LibWebView+WebContent: Use Web::InputEvent for WebContent input IPC

Now that all input events are handled by LibWebView, replace the IPCs
which send the fields of Web::KeyEvent / Web::MouseEvent individually
with one IPC per event type (key or mouse).

We can also replace the ad-hoc queued input structure with a smaller
struct that simply holds the tranferred Web::KeyEvent / Web::MouseEvent.

In the future, we can also adapt Web::EventHandler to use these structs.

Meta/gn/secondary/Userland/Libraries/LibWeb/Page/BUILD.gn

@@ -7,6 +7,7 @@ source_set("Page") {
   sources = [
     "EditEventHandler.cpp",
     "EventHandler.cpp",
+    "InputEvent.cpp",
     "Page.cpp",
   ]
 }

Userland/Libraries/LibWeb/CMakeLists.txt

@@ -492,6 +492,7 @@ set(SOURCES
     NavigationTiming/PerformanceTiming.cpp
     Page/EditEventHandler.cpp
     Page/EventHandler.cpp
+    Page/InputEvent.cpp
     Page/Page.cpp
     Painting/AudioPaintable.cpp
     Painting/BackgroundPainting.cpp

Userland/Libraries/LibWeb/Page/InputEvent.cpp

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2024, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibIPC/Decoder.h>
+#include <LibIPC/Encoder.h>
+#include <LibWeb/Page/InputEvent.h>
+
+namespace Web {
+
+KeyEvent KeyEvent::clone_without_chrome_data() const
+{
+    return { type, key, modifiers, code_point, nullptr };
+}
+
+MouseEvent MouseEvent::clone_without_chrome_data() const
+{
+    return { type, position, screen_position, button, buttons, modifiers, wheel_delta_x, wheel_delta_y, nullptr };
+}
+
+}
+
+template<>
+ErrorOr<void> IPC::encode(Encoder& encoder, Web::KeyEvent const& event)
+{
+    TRY(encoder.encode(event.type));
+    TRY(encoder.encode(event.key));
+    TRY(encoder.encode(event.modifiers));
+    TRY(encoder.encode(event.code_point));
+    return {};
+}
+
+template<>
+ErrorOr<Web::KeyEvent> IPC::decode(Decoder& decoder)
+{
+    auto type = TRY(decoder.decode<Web::KeyEvent::Type>());
+    auto key = TRY(decoder.decode<KeyCode>());
+    auto modifiers = TRY(decoder.decode<KeyModifier>());
+    auto code_point = TRY(decoder.decode<u32>());
+
+    return Web::KeyEvent { type, key, modifiers, code_point, nullptr };
+}
+
+template<>
+ErrorOr<void> IPC::encode(Encoder& encoder, Web::MouseEvent const& event)
+{
+    TRY(encoder.encode(event.type));
+    TRY(encoder.encode(event.position));
+    TRY(encoder.encode(event.screen_position));
+    TRY(encoder.encode(event.button));
+    TRY(encoder.encode(event.buttons));
+    TRY(encoder.encode(event.modifiers));
+    TRY(encoder.encode(event.wheel_delta_x));
+    TRY(encoder.encode(event.wheel_delta_y));
+    return {};
+}
+
+template<>
+ErrorOr<Web::MouseEvent> IPC::decode(Decoder& decoder)
+{
+    auto type = TRY(decoder.decode<Web::MouseEvent::Type>());
+    auto position = TRY(decoder.decode<Web::DevicePixelPoint>());
+    auto screen_position = TRY(decoder.decode<Web::DevicePixelPoint>());
+    auto button = TRY(decoder.decode<GUI::MouseButton>());
+    auto buttons = TRY(decoder.decode<GUI::MouseButton>());
+    auto modifiers = TRY(decoder.decode<KeyModifier>());
+    auto wheel_delta_x = TRY(decoder.decode<int>());
+    auto wheel_delta_y = TRY(decoder.decode<int>());
+
+    return Web::MouseEvent { type, position, screen_position, button, buttons, modifiers, wheel_delta_x, wheel_delta_y, nullptr };
+}

Userland/Libraries/LibWeb/Page/InputEvent.h

@@ -9,6 +9,7 @@
 #include <AK/OwnPtr.h>
 #include <AK/Variant.h>
 #include <LibGfx/Point.h>
+#include <LibIPC/Forward.h>
 #include <LibWeb/PixelUnits.h>
 
 // FIXME: These should not be included outside of Serenity. This FIXME appears in several locations across the Ladybird
@@ -29,6 +30,8 @@ public:
         KeyUp,
     };
 
+    KeyEvent clone_without_chrome_data() const;
+
     Type type;
     KeyCode key { KeyCode::Key_Invalid };
     KeyModifier modifiers { KeyModifier::Mod_None };
@@ -47,6 +50,8 @@ public:
         DoubleClick,
     };
 
+    MouseEvent clone_without_chrome_data() const;
+
     Type type;
     Web::DevicePixelPoint position;
     Web::DevicePixelPoint screen_position;
@@ -62,3 +67,19 @@ public:
 using InputEvent = Variant<KeyEvent, MouseEvent>;
 
 }
+
+namespace IPC {
+
+template<>
+ErrorOr<void> encode(Encoder&, Web::KeyEvent const&);
+
+template<>
+ErrorOr<Web::KeyEvent> decode(Decoder&);
+
+template<>
+ErrorOr<void> encode(Encoder&, Web::MouseEvent const&);
+
+template<>
+ErrorOr<Web::MouseEvent> decode(Decoder&);
+
+}

Userland/Libraries/LibWebView/ViewImplementation.cpp

@@ -119,36 +119,12 @@ void ViewImplementation::enqueue_input_event(Web::InputEvent event)
     // process the next one.
     m_pending_input_events.enqueue(move(event));
 
-    // FIXME: Replace these IPCs with a singular "async_input_event".
     m_pending_input_events.tail().visit(
         [this](Web::KeyEvent const& event) {
-            switch (event.type) {
-            case Web::KeyEvent::Type::KeyDown:
-                client().async_key_down(m_client_state.page_index, event.key, event.modifiers, event.code_point);
-                break;
-            case Web::KeyEvent::Type::KeyUp:
-                client().async_key_up(m_client_state.page_index, event.key, event.modifiers, event.code_point);
-                break;
-            }
+            client().async_key_event(m_client_state.page_index, event.clone_without_chrome_data());
         },
         [this](Web::MouseEvent const& event) {
-            switch (event.type) {
-            case Web::MouseEvent::Type::MouseDown:
-                client().async_mouse_down(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
-                break;
-            case Web::MouseEvent::Type::MouseUp:
-                client().async_mouse_up(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
-                break;
-            case Web::MouseEvent::Type::MouseMove:
-                client().async_mouse_move(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
-                break;
-            case Web::MouseEvent::Type::MouseWheel:
-                client().async_mouse_wheel(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers, event.wheel_delta_x, event.wheel_delta_y);
-                break;
-            case Web::MouseEvent::Type::DoubleClick:
-                client().async_doubleclick(m_client_state.page_index, event.position, event.screen_position, event.button, event.buttons, event.modifiers);
-                break;
-            }
+            client().async_mouse_event(m_client_state.page_index, event.clone_without_chrome_data());
         });
 }
 

Userland/Services/WebContent/ConnectionFromClient.cpp

@@ -227,185 +227,84 @@ void ConnectionFromClient::process_next_input_event()
         return;
 
     auto event = m_input_event_queue.dequeue();
-    event.visit(
-        [&](QueuedMouseEvent const& event) {
-            auto maybe_page = page(event.page_id);
-            if (!maybe_page.has_value()) {
-                dbgln("ConnectionFromClient::process_next_input_event: No page with ID {}", event.page_id);
-                return;
-            }
-            auto& page = maybe_page.release_value();
 
+    auto maybe_page = page(event.page_id);
+    if (!maybe_page.has_value()) {
+        dbgln("ConnectionFromClient::process_next_input_event: No page with ID {}", event.page_id);
+        return;
+    }
+    auto& page = maybe_page->page();
+
+    auto handled = event.event.visit(
+        [&](Web::KeyEvent const& event) {
             switch (event.type) {
-            case QueuedMouseEvent::Type::MouseDown:
-                report_finished_handling_input_event(event.page_id, page.page().handle_mousedown(event.position, event.screen_position, event.button, event.buttons, event.modifiers));
-                break;
-            case QueuedMouseEvent::Type::MouseUp:
-                report_finished_handling_input_event(event.page_id, page.page().handle_mouseup(event.position, event.screen_position, event.button, event.buttons, event.modifiers));
-                break;
-            case QueuedMouseEvent::Type::MouseMove:
-                // NOTE: We have to notify the client about coalesced MouseMoves,
-                //       so we do that by saying none of them were handled by the web page.
-                for (size_t i = 0; i < event.coalesced_event_count; ++i) {
-                    report_finished_handling_input_event(event.page_id, false);
-                }
-                report_finished_handling_input_event(event.page_id, page.page().handle_mousemove(event.position, event.screen_position, event.buttons, event.modifiers));
-                break;
-            case QueuedMouseEvent::Type::DoubleClick:
-                report_finished_handling_input_event(event.page_id, page.page().handle_doubleclick(event.position, event.screen_position, event.button, event.buttons, event.modifiers));
-                break;
-            case QueuedMouseEvent::Type::MouseWheel:
-                for (size_t i = 0; i < event.coalesced_event_count; ++i) {
-                    report_finished_handling_input_event(event.page_id, false);
-                }
-                report_finished_handling_input_event(event.page_id, page.page().handle_mousewheel(event.position, event.screen_position, event.button, event.buttons, event.modifiers, event.wheel_delta_x, event.wheel_delta_y));
-                break;
+            case Web::KeyEvent::Type::KeyDown:
+                return page.handle_keydown(event.key, event.modifiers, event.code_point);
+            case Web::KeyEvent::Type::KeyUp:
+                return page.handle_keyup(event.key, event.modifiers, event.code_point);
             }
+            VERIFY_NOT_REACHED();
         },
-        [&](QueuedKeyboardEvent const& event) {
-            auto maybe_page = page(event.page_id);
-            if (!maybe_page.has_value()) {
-                dbgln("ConnectionFromClient::process_next_input_event: No page with ID {}", event.page_id);
-                return;
-            }
-            auto& page = maybe_page.release_value();
-
+        [&](Web::MouseEvent const& event) {
             switch (event.type) {
-            case QueuedKeyboardEvent::Type::KeyDown:
-                report_finished_handling_input_event(event.page_id, page.page().handle_keydown((KeyCode)event.key, event.modifiers, event.code_point));
-                break;
-            case QueuedKeyboardEvent::Type::KeyUp:
-                report_finished_handling_input_event(event.page_id, page.page().handle_keyup((KeyCode)event.key, event.modifiers, event.code_point));
-                break;
+            case Web::MouseEvent::Type::MouseDown:
+                return page.handle_mousedown(event.position, event.screen_position, event.button, event.buttons, event.modifiers);
+            case Web::MouseEvent::Type::MouseUp:
+                return page.handle_mouseup(event.position, event.screen_position, event.button, event.buttons, event.modifiers);
+            case Web::MouseEvent::Type::MouseMove:
+                return page.handle_mousemove(event.position, event.screen_position, event.buttons, event.modifiers);
+            case Web::MouseEvent::Type::MouseWheel:
+                return page.handle_mousewheel(event.position, event.screen_position, event.button, event.buttons, event.modifiers, event.wheel_delta_x, event.wheel_delta_y);
+            case Web::MouseEvent::Type::DoubleClick:
+                return page.handle_doubleclick(event.position, event.screen_position, event.button, event.buttons, event.modifiers);
             }
+            VERIFY_NOT_REACHED();
         });
 
+    // We have to notify the client about coalesced events, so we do that by saying none of them were handled by the web page.
+    for (size_t i = 0; i < event.coalesced_event_count; ++i)
+        report_finished_handling_input_event(event.page_id, false);
+    report_finished_handling_input_event(event.page_id, handled);
+
     if (!m_input_event_queue.is_empty())
         m_input_event_queue_timer->start();
 }
 
-void ConnectionFromClient::mouse_down(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers)
+void ConnectionFromClient::key_event(u64 page_id, Web::KeyEvent const& event)
 {
-    enqueue_input_event(
-        QueuedMouseEvent {
-            .type = QueuedMouseEvent::Type::MouseDown,
-            .position = position,
-            .screen_position = screen_position,
-            .button = button,
-            .buttons = buttons,
-            .modifiers = modifiers,
-            .page_id = page_id,
-        });
+    enqueue_input_event({ page_id, move(const_cast<Web::KeyEvent&>(event)), 0 });
 }
 
-void ConnectionFromClient::mouse_move(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, [[maybe_unused]] u32 button, u32 buttons, u32 modifiers)
+void ConnectionFromClient::mouse_event(u64 page_id, Web::MouseEvent const& event)
 {
-    auto event = QueuedMouseEvent {
-        .type = QueuedMouseEvent::Type::MouseMove,
-        .position = position,
-        .screen_position = screen_position,
-        .button = button,
-        .buttons = buttons,
-        .modifiers = modifiers,
-        .page_id = page_id,
-    };
+    // OPTIMIZATION: Coalesce consecutive unprocessed mouse move and wheel events.
+    auto should_coalesce = [&]() {
+        if (m_input_event_queue.is_empty())
+            return false;
 
-    // OPTIMIZATION: Coalesce with previous unprocessed event iff the previous event is also a MouseMove event.
-    if (!m_input_event_queue.is_empty()
-        && m_input_event_queue.tail().has<QueuedMouseEvent>()
-        && m_input_event_queue.tail().get<QueuedMouseEvent>().type == QueuedMouseEvent::Type::MouseMove
-        && m_input_event_queue.tail().get<QueuedMouseEvent>().page_id == page_id) {
-        event.coalesced_event_count = m_input_event_queue.tail().get<QueuedMouseEvent>().coalesced_event_count + 1;
-        m_input_event_queue.tail() = event;
-        return;
-    }
+        if (event.type != Web::MouseEvent::Type::MouseMove && event.type != Web::MouseEvent::Type::MouseWheel)
+            return false;
 
-    enqueue_input_event(move(event));
-}
+        if (m_input_event_queue.tail().page_id != page_id)
+            return false;
 
-void ConnectionFromClient::mouse_up(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers)
-{
-    enqueue_input_event(
-        QueuedMouseEvent {
-            .type = QueuedMouseEvent::Type::MouseUp,
-            .position = position,
-            .screen_position = screen_position,
-            .button = button,
-            .buttons = buttons,
-            .modifiers = modifiers,
-            .page_id = page_id,
-        });
-}
+        if (auto* mouse_event = m_input_event_queue.tail().event.get_pointer<Web::MouseEvent>())
+            return mouse_event->type == event.type;
 
-void ConnectionFromClient::mouse_wheel(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers, Web::DevicePixels wheel_delta_x, Web::DevicePixels wheel_delta_y)
-{
-    auto event = QueuedMouseEvent {
-        .type = QueuedMouseEvent::Type::MouseWheel,
-        .position = position,
-        .screen_position = screen_position,
-        .button = button,
-        .buttons = buttons,
-        .modifiers = modifiers,
-        .wheel_delta_x = wheel_delta_x,
-        .wheel_delta_y = wheel_delta_y,
-        .page_id = page_id,
+        return false;
     };
 
-    // OPTIMIZATION: Coalesce with previous unprocessed event if the previous event is also a MouseWheel event.
-    if (!m_input_event_queue.is_empty()
-        && m_input_event_queue.tail().has<QueuedMouseEvent>()
-        && m_input_event_queue.tail().get<QueuedMouseEvent>().type == QueuedMouseEvent::Type::MouseWheel
-        && m_input_event_queue.tail().get<QueuedMouseEvent>().page_id == page_id) {
-        auto const& last_event = m_input_event_queue.tail().get<QueuedMouseEvent>();
-        event.coalesced_event_count = last_event.coalesced_event_count + 1;
-        event.wheel_delta_x += last_event.wheel_delta_x;
-        event.wheel_delta_y += last_event.wheel_delta_y;
-        m_input_event_queue.tail() = event;
+    if (should_coalesce()) {
+        m_input_event_queue.tail().event = move(const_cast<Web::MouseEvent&>(event));
+        ++m_input_event_queue.tail().coalesced_event_count;
+
         return;
     }
 
-    enqueue_input_event(move(event));
-}
-
-void ConnectionFromClient::doubleclick(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers)
-{
-    enqueue_input_event(
-        QueuedMouseEvent {
-            .type = QueuedMouseEvent::Type::DoubleClick,
-            .position = position,
-            .screen_position = screen_position,
-            .button = button,
-            .buttons = buttons,
-            .modifiers = modifiers,
-            .page_id = page_id,
-        });
-}
-
-void ConnectionFromClient::key_down(u64 page_id, i32 key, u32 modifiers, u32 code_point)
-{
-    enqueue_input_event(
-        QueuedKeyboardEvent {
-            .type = QueuedKeyboardEvent::Type::KeyDown,
-            .key = key,
-            .modifiers = modifiers,
-            .code_point = code_point,
-            .page_id = page_id,
-        });
-}
-
-void ConnectionFromClient::key_up(u64 page_id, i32 key, u32 modifiers, u32 code_point)
-{
-    enqueue_input_event(
-        QueuedKeyboardEvent {
-            .type = QueuedKeyboardEvent::Type::KeyUp,
-            .key = key,
-            .modifiers = modifiers,
-            .code_point = code_point,
-            .page_id = page_id,
-        });
+    enqueue_input_event({ page_id, move(const_cast<Web::MouseEvent&>(event)), 0 });
 }
 
-void ConnectionFromClient::enqueue_input_event(Variant<QueuedMouseEvent, QueuedKeyboardEvent> event)
+void ConnectionFromClient::enqueue_input_event(QueuedInputEvent event)
 {
     m_input_event_queue.enqueue(move(event));
     m_input_event_queue_timer->start();

Userland/Services/WebContent/ConnectionFromClient.h

@@ -16,6 +16,7 @@
 #include <LibWeb/CSS/PreferredColorScheme.h>
 #include <LibWeb/Forward.h>
 #include <LibWeb/Loader/FileRequest.h>
+#include <LibWeb/Page/InputEvent.h>
 #include <LibWeb/Platform/Timer.h>
 #include <LibWebView/Forward.h>
 #include <WebContent/Forward.h>
@@ -56,13 +57,8 @@ private:
     virtual void load_url(u64 page_id, URL const&) override;
     virtual void load_html(u64 page_id, ByteString const&) override;
     virtual void set_viewport_rect(u64 page_id, Web::DevicePixelRect const&) override;
-    virtual void mouse_down(u64 page_id, Web::DevicePixelPoint, Web::DevicePixelPoint, u32, u32, u32) override;
-    virtual void mouse_move(u64 page_id, Web::DevicePixelPoint, Web::DevicePixelPoint, u32, u32, u32) override;
-    virtual void mouse_up(u64 page_id, Web::DevicePixelPoint, Web::DevicePixelPoint, u32, u32, u32) override;
-    virtual void mouse_wheel(u64 page_id, Web::DevicePixelPoint, Web::DevicePixelPoint, u32, u32, u32, Web::DevicePixels, Web::DevicePixels) override;
-    virtual void doubleclick(u64 page_id, Web::DevicePixelPoint, Web::DevicePixelPoint, u32, u32, u32) override;
-    virtual void key_down(u64 page_id, i32, u32, u32) override;
-    virtual void key_up(u64 page_id, i32, u32, u32) override;
+    virtual void key_event(u64 page_id, Web::KeyEvent const&) override;
+    virtual void mouse_event(u64 page_id, Web::MouseEvent const&) override;
     virtual void add_backing_store(u64 page_id, i32 front_bitmap_id, Gfx::ShareableBitmap const& front_bitmap, i32 back_bitmap_id, Gfx::ShareableBitmap const& back_bitmap) override;
     virtual void ready_to_paint(u64 page_id) override;
     virtual void debug_request(u64 page_id, ByteString const&, ByteString const&) override;
@@ -136,42 +132,16 @@ private:
     HashMap<int, Web::FileRequest> m_requested_files {};
     int last_id { 0 };
 
-    struct QueuedMouseEvent {
-        enum class Type {
-            MouseMove,
-            MouseDown,
-            MouseUp,
-            MouseWheel,
-            DoubleClick,
-        };
-        Type type {};
-        Web::DevicePixelPoint position {};
-        Web::DevicePixelPoint screen_position {};
-        u32 button {};
-        u32 buttons {};
-        u32 modifiers {};
-        Web::DevicePixels wheel_delta_x {};
-        Web::DevicePixels wheel_delta_y {};
-        size_t coalesced_event_count { 0 };
-        u64 page_id { 0 };
-    };
-
-    struct QueuedKeyboardEvent {
-        enum class Type {
-            KeyDown,
-            KeyUp,
-        };
-        Type type {};
-        i32 key {};
-        u32 modifiers {};
-        u32 code_point {};
+    struct QueuedInputEvent {
         u64 page_id { 0 };
+        Web::InputEvent event;
+        size_t coalesced_event_count { 0 };
     };
 
-    void enqueue_input_event(Variant<QueuedMouseEvent, QueuedKeyboardEvent>);
+    void enqueue_input_event(QueuedInputEvent);
     void process_next_input_event();
 
-    Queue<Variant<QueuedMouseEvent, QueuedKeyboardEvent>> m_input_event_queue;
+    Queue<QueuedInputEvent> m_input_event_queue;
 
     RefPtr<Web::Platform::Timer> m_input_event_queue_timer;
 };

Userland/Services/WebContent/WebContentServer.ipc

@@ -7,6 +7,7 @@
 #include <LibWeb/CSS/Selector.h>
 #include <LibWeb/HTML/ColorPickerUpdateState.h>
 #include <LibWeb/HTML/SelectedFile.h>
+#include <LibWeb/Page/InputEvent.h>
 #include <LibWeb/WebDriver/ExecuteScript.h>
 #include <LibWebView/Attribute.h>
 
@@ -29,14 +30,8 @@ endpoint WebContentServer
 
     set_viewport_rect(u64 page_id, Web::DevicePixelRect rect) =|
 
-    mouse_down(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers) =|
-    mouse_move(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers) =|
-    mouse_up(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers) =|
-    mouse_wheel(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers, Web::DevicePixels wheel_delta_x, Web::DevicePixels wheel_delta_y) =|
-    doubleclick(u64 page_id, Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, u32 button, u32 buttons, u32 modifiers) =|
-
-    key_down(u64 page_id, i32 key, u32 modifiers, u32 code_point) =|
-    key_up(u64 page_id, i32 key, u32 modifiers, u32 code_point) =|
+    key_event(u64 page_id, Web::KeyEvent event) =|
+    mouse_event(u64 page_id, Web::MouseEvent event) =|
 
     debug_request(u64 page_id, ByteString request, ByteString argument) =|
     get_source(u64 page_id) =|