LibWebView+WebContent: Drive repainting from WebContent process

With this change, chrome no longer has to ask the WebContent process
to paint the next frame into a specified bitmap. Instead, it allocates
bitmaps and sends them to WebContent, which then lets chrome know when
the painting is done.

This work is a preparation to move the execution of painting commands
into a separate thread. Now, it is much easier to start working on the
next frame while the current one is still rendering. This is because
WebContent does not have to inform chrome that the current frame is
ready before it can request the next frame.

Additionally, as a side bonus, we can now eliminate the
did_invalidate_content_rect and did_change_selection IPC calls. These
were used solely for the purpose of informing chrome that it needed to
request a repaint.

Ladybird/Android/src/main/cpp/WebViewImplementationNative.cpp

@@ -94,7 +94,6 @@ void WebViewImplementationNative::set_viewport_geometry(int w, int h)
 {
     m_viewport_rect = { { 0, 0 }, { w, h } };
     client().async_set_viewport_rect(m_viewport_rect);
-    request_repaint();
     handle_resize();
 }
 

Ladybird/AppKit/UI/LadybirdWebViewBridge.cpp

@@ -72,7 +72,6 @@ void WebViewBridge::set_viewport_rect(Gfx::IntRect viewport_rect, ForResize for_
     m_viewport_rect = viewport_rect;
 
     client().async_set_viewport_rect(m_viewport_rect.to_type<Web::DevicePixels>());
-    request_repaint();
 
     if (for_resize == ForResize::Yes) {
         handle_resize();

Ladybird/Qt/WebContentView.cpp

@@ -527,7 +527,6 @@ void WebContentView::set_device_pixel_ratio(double device_pixel_ratio)
     client().async_set_device_pixels_per_css_pixel(m_device_pixel_ratio * m_zoom_level);
     update_viewport_rect();
     handle_resize();
-    request_repaint();
 }
 
 void WebContentView::update_viewport_rect()
@@ -537,15 +536,12 @@ void WebContentView::update_viewport_rect()
     Gfx::IntRect rect(max(0, horizontalScrollBar()->value()), max(0, verticalScrollBar()->value()), scaled_width, scaled_height);
 
     set_viewport_rect(rect);
-
-    request_repaint();
 }
 
 void WebContentView::update_zoom()
 {
     client().async_set_device_pixels_per_css_pixel(m_device_pixel_ratio * m_zoom_level);
     update_viewport_rect();
-    request_repaint();
 }
 
 void WebContentView::showEvent(QShowEvent* event)
@@ -733,7 +729,6 @@ bool WebContentView::event(QEvent* event)
 
     if (event->type() == QEvent::PaletteChange) {
         update_palette();
-        request_repaint();
         return QAbstractScrollArea::event(event);
     }
 

Userland/Libraries/LibWeb/Page/EventHandler.cpp

@@ -522,7 +522,6 @@ bool EventHandler::handle_mousemove(CSSPixelPoint position, CSSPixelPoint screen
                 }
                 document.navigable()->set_needs_display();
             }
-            m_browsing_context->page().client().page_did_change_selection();
         }
     }
 

Userland/Libraries/LibWeb/Page/Page.cpp

@@ -106,6 +106,14 @@ DevicePixelPoint Page::css_to_device_point(CSSPixelPoint point) const
     };
 }
 
+DevicePixelRect Page::css_to_device_rect(CSSPixelRect rect) const
+{
+    return {
+        rect.location().to_type<double>() * client().device_pixels_per_css_pixel(),
+        rect.size().to_type<double>() * client().device_pixels_per_css_pixel(),
+    };
+}
+
 CSSPixelRect Page::device_to_css_rect(DevicePixelRect rect) const
 {
     auto scale = client().device_pixels_per_css_pixel();

Userland/Libraries/LibWeb/Page/Page.h

@@ -70,6 +70,7 @@ public:
 
     CSSPixelPoint device_to_css_point(DevicePixelPoint) const;
     DevicePixelPoint css_to_device_point(CSSPixelPoint) const;
+    DevicePixelRect css_to_device_rect(CSSPixelRect) const;
     CSSPixelRect device_to_css_rect(DevicePixelRect) const;
     DevicePixelRect enclosing_device_rect(CSSPixelRect) const;
     DevicePixelRect rounded_device_rect(CSSPixelRect) const;
@@ -238,7 +239,6 @@ public:
     virtual void page_did_create_new_document(Web::DOM::Document&) { }
     virtual void page_did_destroy_document(Web::DOM::Document&) { }
     virtual void page_did_finish_loading(const AK::URL&) { }
-    virtual void page_did_change_selection() { }
     virtual void page_did_request_cursor_change(Gfx::StandardCursor) { }
     virtual void page_did_request_context_menu(CSSPixelPoint) { }
     virtual void page_did_request_link_context_menu(CSSPixelPoint, AK::URL const&, [[maybe_unused]] ByteString const& target, [[maybe_unused]] unsigned modifiers) { }

Userland/Libraries/LibWebView/OutOfProcessWebView.cpp

@@ -206,7 +206,6 @@ void OutOfProcessWebView::theme_change_event(GUI::ThemeChangeEvent& event)
 {
     Super::theme_change_event(event);
     client().async_update_system_theme(Gfx::current_system_theme_buffer());
-    request_repaint();
 }
 
 void OutOfProcessWebView::screen_rects_change_event(GUI::ScreenRectsChangeEvent& event)
@@ -221,7 +220,6 @@ void OutOfProcessWebView::screen_rects_change_event(GUI::ScreenRectsChangeEvent&
 void OutOfProcessWebView::did_scroll()
 {
     client().async_set_viewport_rect(visible_content_rect().to_type<Web::DevicePixels>());
-    request_repaint();
 }
 
 ByteString OutOfProcessWebView::dump_layout_tree()

Userland/Libraries/LibWebView/ViewImplementation.cpp

@@ -50,36 +50,16 @@ WebContentClient const& ViewImplementation::client() const
 
 void ViewImplementation::server_did_paint(Badge<WebContentClient>, i32 bitmap_id, Gfx::IntSize size)
 {
-    if (m_client_state.back_bitmap.id != bitmap_id)
-        return;
-
-    m_client_state.has_usable_bitmap = true;
-    m_client_state.back_bitmap.pending_paints--;
-    m_client_state.back_bitmap.last_painted_size = size.to_type<Web::DevicePixels>();
-    swap(m_client_state.back_bitmap, m_client_state.front_bitmap);
-
-    // We don't need the backup bitmap anymore, so drop it.
-    m_backup_bitmap = nullptr;
-
-    if (on_ready_to_paint)
-        on_ready_to_paint();
-
-    if (m_client_state.got_repaint_requests_while_painting) {
-        m_client_state.got_repaint_requests_while_painting = false;
-        request_repaint();
+    if (m_client_state.back_bitmap.id == bitmap_id) {
+        m_client_state.has_usable_bitmap = true;
+        m_client_state.back_bitmap.last_painted_size = size.to_type<Web::DevicePixels>();
+        swap(m_client_state.back_bitmap, m_client_state.front_bitmap);
+        m_backup_bitmap = nullptr;
+        if (on_ready_to_paint)
+            on_ready_to_paint();
     }
 }
 
-void ViewImplementation::server_did_invalidate_content_rect(Badge<WebContentClient>, Gfx::IntRect)
-{
-    request_repaint();
-}
-
-void ViewImplementation::server_did_change_selection(Badge<WebContentClient>)
-{
-    request_repaint();
-}
-
 void ViewImplementation::load(AK::URL const& url)
 {
     m_url = url;
@@ -322,16 +302,14 @@ void ViewImplementation::resize_backing_stores_if_needed(WindowResizeInProgress
         m_client_state.back_bitmap = {};
     }
 
+    auto old_front_bitmap_id = m_client_state.front_bitmap.id;
+    auto old_back_bitmap_id = m_client_state.back_bitmap.id;
+
     auto reallocate_backing_store_if_needed = [&](SharedBitmap& backing_store) {
         if (!backing_store.bitmap || !backing_store.bitmap->size().contains(minimum_needed_size.to_type<int>())) {
             if (auto new_bitmap_or_error = Gfx::Bitmap::create_shareable(Gfx::BitmapFormat::BGRA8888, minimum_needed_size.to_type<int>()); !new_bitmap_or_error.is_error()) {
-                if (backing_store.bitmap)
-                    client().async_remove_backing_store(backing_store.id);
-
-                backing_store.pending_paints = 0;
                 backing_store.bitmap = new_bitmap_or_error.release_value();
                 backing_store.id = m_client_state.next_bitmap_id++;
-                client().async_add_backing_store(backing_store.id, backing_store.bitmap->to_shareable_bitmap());
             }
             backing_store.last_painted_size = viewport_rect.size();
         }
@@ -340,22 +318,11 @@ void ViewImplementation::resize_backing_stores_if_needed(WindowResizeInProgress
     reallocate_backing_store_if_needed(m_client_state.front_bitmap);
     reallocate_backing_store_if_needed(m_client_state.back_bitmap);
 
-    request_repaint();
-}
+    auto& front_bitmap = m_client_state.front_bitmap;
+    auto& back_bitmap = m_client_state.back_bitmap;
 
-void ViewImplementation::request_repaint()
-{
-    // If this widget was instantiated but not yet added to a window,
-    // it won't have a back bitmap yet, so we can just skip repaint requests.
-    if (!m_client_state.back_bitmap.bitmap)
-        return;
-    // Don't request a repaint until pending paint requests have finished.
-    if (m_client_state.back_bitmap.pending_paints) {
-        m_client_state.got_repaint_requests_while_painting = true;
-        return;
-    }
-    m_client_state.back_bitmap.pending_paints++;
-    client().async_paint(viewport_rect(), m_client_state.back_bitmap.id);
+    if (front_bitmap.id != old_front_bitmap_id || back_bitmap.id != old_back_bitmap_id)
+        client().async_add_backing_store(front_bitmap.id, front_bitmap.bitmap->to_shareable_bitmap(), back_bitmap.id, back_bitmap.bitmap->to_shareable_bitmap());
 }
 
 void ViewImplementation::handle_web_content_process_crash()

Userland/Libraries/LibWebView/ViewImplementation.h

@@ -39,8 +39,6 @@ public:
     String const& handle() const { return m_client_state.client_handle; }
 
     void server_did_paint(Badge<WebContentClient>, i32 bitmap_id, Gfx::IntSize size);
-    void server_did_invalidate_content_rect(Badge<WebContentClient>, Gfx::IntRect rect);
-    void server_did_change_selection(Badge<WebContentClient>);
 
     void load(AK::URL const&);
     void load_html(StringView);
@@ -194,7 +192,6 @@ protected:
     };
     void resize_backing_stores_if_needed(WindowResizeInProgress);
 
-    void request_repaint();
     void handle_resize();
 
     virtual void create_client() { }
@@ -203,7 +200,6 @@ protected:
 
     struct SharedBitmap {
         i32 id { -1 };
-        i32 pending_paints { 0 };
         Web::DevicePixelSize last_painted_size;
         RefPtr<Gfx::Bitmap> bitmap;
     };
@@ -215,7 +211,6 @@ protected:
         SharedBitmap back_bitmap;
         i32 next_bitmap_id { 0 };
         bool has_usable_bitmap { false };
-        bool got_repaint_requests_while_painting { false };
     } m_client_state;
 
     AK::URL m_url;

Userland/Libraries/LibWebView/WebContentClient.cpp

@@ -68,20 +68,6 @@ void WebContentClient::did_request_refresh()
         m_view.on_refresh();
 }
 
-void WebContentClient::did_invalidate_content_rect(Gfx::IntRect const& content_rect)
-{
-    dbgln_if(SPAM_DEBUG, "handle: WebContentClient::DidInvalidateContentRect! content_rect={}", content_rect);
-
-    // FIXME: Figure out a way to coalesce these messages to reduce unnecessary painting
-    m_view.server_did_invalidate_content_rect({}, content_rect);
-}
-
-void WebContentClient::did_change_selection()
-{
-    dbgln_if(SPAM_DEBUG, "handle: WebContentClient::DidChangeSelection!");
-    m_view.server_did_change_selection({});
-}
-
 void WebContentClient::did_request_cursor_change(i32 cursor_type)
 {
     if (cursor_type < 0 || cursor_type >= (i32)Gfx::StandardCursor::__Count) {

Userland/Libraries/LibWebView/WebContentClient.h

@@ -35,8 +35,6 @@ private:
     virtual void did_request_navigate_back() override;
     virtual void did_request_navigate_forward() override;
     virtual void did_request_refresh() override;
-    virtual void did_invalidate_content_rect(Gfx::IntRect const&) override;
-    virtual void did_change_selection() override;
     virtual void did_request_cursor_change(i32) override;
     virtual void did_layout(Gfx::IntSize) override;
     virtual void did_change_title(ByteString const&) override;

Userland/Services/WebContent/ConnectionFromClient.cpp

@@ -57,7 +57,6 @@ ConnectionFromClient::ConnectionFromClient(NonnullOwnPtr<Core::LocalSocket> sock
     : IPC::ConnectionFromClient<WebContentClientEndpoint, WebContentServerEndpoint>(*this, move(socket), 1)
     , m_page_host(PageHost::create(*this))
 {
-    m_paint_flush_timer = Web::Platform::Timer::create_single_shot(0, [this] { flush_pending_paint_requests(); });
     m_input_event_queue_timer = Web::Platform::Timer::create_single_shot(0, [this] { process_next_input_event(); });
 }
 
@@ -141,44 +140,12 @@ void ConnectionFromClient::set_viewport_rect(Web::DevicePixelRect const& rect)
     page().set_viewport_rect(rect);
 }
 
-void ConnectionFromClient::add_backing_store(i32 backing_store_id, Gfx::ShareableBitmap const& bitmap)
+void ConnectionFromClient::add_backing_store(i32 front_bitmap_id, Gfx::ShareableBitmap const& front_bitmap, i32 back_bitmap_id, Gfx::ShareableBitmap const& back_bitmap)
 {
-    m_backing_stores.set(backing_store_id, *const_cast<Gfx::ShareableBitmap&>(bitmap).bitmap());
-}
-
-void ConnectionFromClient::remove_backing_store(i32 backing_store_id)
-{
-    m_backing_stores.remove(backing_store_id);
-    m_pending_paint_requests.remove_all_matching([backing_store_id](auto& pending_repaint_request) { return pending_repaint_request.bitmap_id == backing_store_id; });
-}
-
-void ConnectionFromClient::paint(Web::DevicePixelRect const& content_rect, i32 backing_store_id)
-{
-    for (auto& pending_paint : m_pending_paint_requests) {
-        if (pending_paint.bitmap_id == backing_store_id) {
-            pending_paint.content_rect = content_rect;
-            return;
-        }
-    }
-
-    auto it = m_backing_stores.find(backing_store_id);
-    if (it == m_backing_stores.end()) {
-        did_misbehave("Client requested paint with backing store ID");
-        return;
-    }
-
-    auto& bitmap = *it->value;
-    m_pending_paint_requests.append({ content_rect, bitmap, backing_store_id });
-    m_paint_flush_timer->start();
-}
-
-void ConnectionFromClient::flush_pending_paint_requests()
-{
-    for (auto& pending_paint : m_pending_paint_requests) {
-        page().paint(pending_paint.content_rect, *pending_paint.bitmap);
-        async_did_paint(pending_paint.content_rect.to_type<int>(), pending_paint.bitmap_id);
-    }
-    m_pending_paint_requests.clear();
+    m_backing_stores.front_bitmap_id = front_bitmap_id;
+    m_backing_stores.back_bitmap_id = back_bitmap_id;
+    m_backing_stores.front_bitmap = *const_cast<Gfx::ShareableBitmap&>(front_bitmap).bitmap();
+    m_backing_stores.back_bitmap = *const_cast<Gfx::ShareableBitmap&>(back_bitmap).bitmap();
 }
 
 void ConnectionFromClient::process_next_input_event()
@@ -881,7 +848,6 @@ Messages::WebContentServer::GetSelectedTextResponse ConnectionFromClient::get_se
 void ConnectionFromClient::select_all()
 {
     page().page().focused_context().select_all();
-    page().page().client().page_did_change_selection();
 }
 
 Messages::WebContentServer::DumpLayoutTreeResponse ConnectionFromClient::dump_layout_tree()

Userland/Services/WebContent/ConnectionFromClient.h

@@ -44,6 +44,8 @@ public:
     PageHost& page_host() { return *m_page_host; }
     PageHost const& page_host() const { return *m_page_host; }
 
+    auto& backing_stores() { return m_backing_stores; }
+
 private:
     explicit ConnectionFromClient(NonnullOwnPtr<Core::LocalSocket>);
 
@@ -58,7 +60,6 @@ private:
     virtual void update_screen_rects(Vector<Web::DevicePixelRect> const&, u32) override;
     virtual void load_url(URL const&) override;
     virtual void load_html(ByteString const&) override;
-    virtual void paint(Web::DevicePixelRect const&, i32) override;
     virtual void set_viewport_rect(Web::DevicePixelRect const&) override;
     virtual void mouse_down(Web::DevicePixelPoint, Web::DevicePixelPoint, unsigned, unsigned, unsigned) override;
     virtual void mouse_move(Web::DevicePixelPoint, Web::DevicePixelPoint, unsigned, unsigned, unsigned) override;
@@ -67,8 +68,7 @@ private:
     virtual void doubleclick(Web::DevicePixelPoint, Web::DevicePixelPoint, unsigned, unsigned, unsigned) override;
     virtual void key_down(i32, unsigned, u32) override;
     virtual void key_up(i32, unsigned, u32) override;
-    virtual void add_backing_store(i32, Gfx::ShareableBitmap const&) override;
-    virtual void remove_backing_store(i32) override;
+    virtual void add_backing_store(i32 front_bitmap_id, Gfx::ShareableBitmap const& front_bitmap, i32 back_bitmap_id, Gfx::ShareableBitmap const& back_bitmap) override;
     virtual void debug_request(ByteString const&, ByteString const&) override;
     virtual void get_source() override;
     virtual void inspect_dom_tree() override;
@@ -132,20 +132,17 @@ private:
     virtual Messages::WebContentServer::GetSelectedTextResponse get_selected_text() override;
     virtual void select_all() override;
 
-    void flush_pending_paint_requests();
-
     void report_finished_handling_input_event(bool event_was_handled);
 
     NonnullOwnPtr<PageHost> m_page_host;
-    struct PaintRequest {
-        Web::DevicePixelRect content_rect;
-        NonnullRefPtr<Gfx::Bitmap> bitmap;
-        i32 bitmap_id { -1 };
-    };
-    Vector<PaintRequest> m_pending_paint_requests;
-    RefPtr<Web::Platform::Timer> m_paint_flush_timer;
 
-    HashMap<i32, NonnullRefPtr<Gfx::Bitmap>> m_backing_stores;
+    struct BackingStores {
+        i32 front_bitmap_id { -1 };
+        i32 back_bitmap_id { -1 };
+        RefPtr<Gfx::Bitmap> front_bitmap;
+        RefPtr<Gfx::Bitmap> back_bitmap;
+    };
+    BackingStores m_backing_stores;
 
     HashMap<Web::DOM::Document*, NonnullOwnPtr<WebContentConsoleClient>> m_console_clients;
     WeakPtr<WebContentConsoleClient> m_top_level_document_console_client;

Userland/Services/WebContent/PageClient.cpp

@@ -51,9 +51,21 @@ PageClient::PageClient(PageHost& owner, u64 id)
     , m_id(id)
 {
     setup_palette();
-    m_invalidation_coalescing_timer = Web::Platform::Timer::create_single_shot(0, [this] {
-        client().async_did_invalidate_content_rect({ m_invalidation_rect.x().value(), m_invalidation_rect.y().value(), m_invalidation_rect.width().value(), m_invalidation_rect.height().value() });
-        m_invalidation_rect = {};
+
+    m_repaint_timer = Web::Platform::Timer::create_single_shot(0, [this] {
+        if (!client().backing_stores().back_bitmap) {
+            warnln("Skip painting because back bitmap is not initialized");
+            return;
+        }
+
+        auto& back_bitmap = *client().backing_stores().back_bitmap;
+        auto viewport_rect = page().css_to_device_rect(page().top_level_traversable()->viewport_rect());
+        paint(viewport_rect, back_bitmap);
+
+        auto& backing_stores = client().backing_stores();
+        swap(backing_stores.front_bitmap, backing_stores.back_bitmap);
+        swap(backing_stores.front_bitmap_id, backing_stores.back_bitmap_id);
+        client().did_paint(viewport_rect.to_type<int>(), backing_stores.front_bitmap_id);
     });
 
 #ifdef HAS_ACCELERATED_GRAPHICS
@@ -63,6 +75,12 @@ PageClient::PageClient(PageHost& owner, u64 id)
 #endif
 }
 
+void PageClient::schedule_repaint()
+{
+    if (!m_repaint_timer->is_active())
+        m_repaint_timer->start();
+}
+
 void PageClient::visit_edges(JS::Cell::Visitor& visitor)
 {
     Base::visit_edges(visitor);
@@ -194,18 +212,12 @@ void PageClient::paint(Web::DevicePixelRect const& content_rect, Gfx::Bitmap& ta
 void PageClient::set_viewport_rect(Web::DevicePixelRect const& rect)
 {
     page().top_level_traversable()->set_viewport_rect(page().device_to_css_rect(rect));
+    schedule_repaint();
 }
 
-void PageClient::page_did_invalidate(Web::CSSPixelRect const& content_rect)
-{
-    m_invalidation_rect = m_invalidation_rect.united(page().enclosing_device_rect(content_rect));
-    if (!m_invalidation_coalescing_timer->is_active())
-        m_invalidation_coalescing_timer->start();
-}
-
-void PageClient::page_did_change_selection()
+void PageClient::page_did_invalidate(Web::CSSPixelRect const&)
 {
-    client().async_did_change_selection();
+    schedule_repaint();
 }
 
 void PageClient::page_did_request_cursor_change(Gfx::StandardCursor cursor)

Userland/Services/WebContent/PageClient.h

@@ -28,6 +28,8 @@ public:
 
     static void set_use_gpu_painter();
 
+    void schedule_repaint();
+
     virtual Web::Page& page() override { return *m_page; }
     virtual Web::Page const& page() const override { return *m_page; }
 
@@ -75,7 +77,6 @@ private:
     virtual double device_pixels_per_css_pixel() const override { return m_device_pixels_per_css_pixel; }
     virtual Web::CSS::PreferredColorScheme preferred_color_scheme() const override { return m_preferred_color_scheme; }
     virtual void page_did_invalidate(Web::CSSPixelRect const&) override;
-    virtual void page_did_change_selection() override;
     virtual void page_did_request_cursor_change(Gfx::StandardCursor) override;
     virtual void page_did_layout() override;
     virtual void page_did_change_title(ByteString const&) override;
@@ -148,8 +149,8 @@ private:
     bool m_should_show_line_box_borders { false };
     bool m_has_focus { false };
 
-    RefPtr<Web::Platform::Timer> m_invalidation_coalescing_timer;
-    Web::DevicePixelRect m_invalidation_rect;
+    RefPtr<Web::Platform::Timer> m_repaint_timer;
+
     Web::CSS::PreferredColorScheme m_preferred_color_scheme { Web::CSS::PreferredColorScheme::Auto };
 
     RefPtr<WebDriverConnection> m_webdriver;

Userland/Services/WebContent/WebContentClient.ipc

@@ -17,9 +17,7 @@ endpoint WebContentClient
     did_request_navigate_back() =|
     did_request_navigate_forward() =|
     did_request_refresh() =|
-    did_paint(Gfx::IntRect content_rect, i32 bitmap_id) =|
-    did_invalidate_content_rect(Gfx::IntRect content_rect) =|
-    did_change_selection() =|
+    did_paint(Gfx::IntRect content_rect, i32 bitmap_id) => ()
     did_request_cursor_change(i32 cursor_type) =|
     did_layout(Gfx::IntSize content_size) =|
     did_change_title(ByteString title) =|

Userland/Services/WebContent/WebContentServer.ipc

@@ -22,10 +22,8 @@ endpoint WebContentServer
     load_url(URL url) =|
     load_html(ByteString html) =|
 
-    add_backing_store(i32 backing_store_id, Gfx::ShareableBitmap bitmap) =|
-    remove_backing_store(i32 backing_store_id) =|
+    add_backing_store(i32 front_bitmap_id, Gfx::ShareableBitmap front_bitmap, i32 back_bitmap_id, Gfx::ShareableBitmap back_bitmap) =|
 
-    paint(Web::DevicePixelRect content_rect, i32 backing_store_id) =|
     set_viewport_rect(Web::DevicePixelRect rect) =|
 
     mouse_down(Web::DevicePixelPoint position, Web::DevicePixelPoint screen_position, unsigned button, unsigned buttons, unsigned modifiers) =|