Browser+WebContent+WebDriver: Move Take Element Screenshot to WebContent

Userland/Applications/Browser/BrowserWindow.cpp

@@ -607,10 +607,6 @@ void BrowserWindow::create_new_tab(URL url, bool activate)
         return active_tab().view().take_screenshot();
     };
 
-    new_tab.webdriver_endpoints().on_scroll_element_into_view = [this](i32 element_id) {
-        active_tab().view().scroll_element_into_view(element_id);
-    };
-
     new_tab.webdriver_endpoints().on_serialize_source = [this]() {
         return active_tab().view().serialize_source();
     };
@@ -619,10 +615,6 @@ void BrowserWindow::create_new_tab(URL url, bool activate)
         return active_tab().view().webdriver_execute_script(body, json_arguments, timeout, async);
     };
 
-    new_tab.webdriver_endpoints().on_take_element_screenshot = [this](i32 element_id) {
-        return active_tab().view().take_element_screenshot(element_id);
-    };
-
     new_tab.load(url);
 
     dbgln_if(SPAM_DEBUG, "Added new tab {:p}, loading {}", &new_tab, url);

Userland/Applications/Browser/WebDriverConnection.cpp

@@ -136,26 +136,4 @@ void WebDriverConnection::update_cookie(Web::Cookie::Cookie const& cookie)
     }
 }
 
-void WebDriverConnection::scroll_element_into_view(i32 element_id)
-{
-    dbgln("WebDriverConnection: scroll_element_into_view {}", element_id);
-    if (auto browser_window = m_browser_window.strong_ref()) {
-        auto& tab = browser_window->active_tab();
-        if (tab.webdriver_endpoints().on_scroll_element_into_view)
-            tab.webdriver_endpoints().on_scroll_element_into_view(element_id);
-    }
-}
-
-Messages::WebDriverSessionClient::TakeElementScreenshotResponse WebDriverConnection::take_element_screenshot(i32 element_id)
-{
-    dbgln_if(WEBDRIVER_DEBUG, "WebDriverConnection: take_element_screenshot {}", element_id);
-    if (auto browser_window = m_browser_window.strong_ref()) {
-        auto& tab = browser_window->active_tab();
-        if (tab.webdriver_endpoints().on_take_element_screenshot)
-            return { tab.webdriver_endpoints().on_take_element_screenshot(element_id) };
-    }
-
-    return { {} };
-}
-
 }

Userland/Applications/Browser/WebDriverConnection.h

@@ -48,8 +48,6 @@ public:
     virtual Messages::WebDriverSessionClient::GetNamedCookieResponse get_named_cookie(String const& name) override;
     virtual void add_cookie(Web::Cookie::ParsedCookie const&) override;
     virtual void update_cookie(Web::Cookie::Cookie const&) override;
-    virtual void scroll_element_into_view(i32 element_id) override;
-    virtual Messages::WebDriverSessionClient::TakeElementScreenshotResponse take_element_screenshot(i32 element_id) override;
 
 private:
     WebDriverConnection(NonnullOwnPtr<Core::Stream::LocalSocket> socket, NonnullRefPtr<BrowserWindow> browser_window);

Userland/Applications/Browser/WebDriverEndpoints.h

@@ -23,8 +23,6 @@ public:
     WebDriverEndpoints() = default;
     ~WebDriverEndpoints() = default;
 
-    Function<void(i32 element_id)> on_scroll_element_into_view;
-    Function<Gfx::ShareableBitmap(i32 element_id)> on_take_element_screenshot;
     Function<String()> on_serialize_source;
     Function<Messages::WebContentServer::WebdriverExecuteScriptResponse(String const& body, Vector<String> const& json_arguments, Optional<u64> const& timeout, bool async)> on_execute_script;
 };

Userland/Applications/Browser/WebDriverSessionClient.ipc

@@ -24,6 +24,4 @@ endpoint WebDriverSessionClient {
     get_named_cookie(String name) => (Optional<Web::Cookie::Cookie> cookie)
     add_cookie(Web::Cookie::ParsedCookie cookie) =|
     update_cookie(Web::Cookie::Cookie cookie) =|
-    scroll_element_into_view(i32 element_id) => ()
-    take_element_screenshot(i32 element_id) => (Gfx::ShareableBitmap data)
 }

Userland/Libraries/LibWebView/OutOfProcessWebView.cpp

@@ -554,11 +554,6 @@ OrderedHashMap<String, String> OutOfProcessWebView::get_session_storage_entries(
     return client().get_session_storage_entries();
 }
 
-void OutOfProcessWebView::scroll_element_into_view(i32 element_id)
-{
-    return client().scroll_element_into_view(element_id);
-}
-
 void OutOfProcessWebView::set_content_filters(Vector<String> filters)
 {
     client().async_set_content_filters(filters);
@@ -596,11 +591,6 @@ Gfx::ShareableBitmap OutOfProcessWebView::take_screenshot() const
     return {};
 }
 
-Gfx::ShareableBitmap OutOfProcessWebView::take_element_screenshot(i32 element_id)
-{
-    return client().take_element_screenshot(element_id);
-}
-
 Gfx::ShareableBitmap OutOfProcessWebView::take_document_screenshot()
 {
     return client().take_document_screenshot();

Userland/Libraries/LibWebView/OutOfProcessWebView.h

@@ -63,8 +63,6 @@ public:
     OrderedHashMap<String, String> get_local_storage_entries();
     OrderedHashMap<String, String> get_session_storage_entries();
 
-    void scroll_element_into_view(i32 element_id);
-
     void set_content_filters(Vector<String>);
     void set_proxy_mappings(Vector<String> proxies, HashMap<String, size_t> mappings);
     void set_preferred_color_scheme(Web::CSS::PreferredColorScheme);
@@ -76,7 +74,6 @@ public:
     void set_system_visibility_state(bool visible);
 
     Gfx::ShareableBitmap take_screenshot() const;
-    Gfx::ShareableBitmap take_element_screenshot(i32 element_id);
     Gfx::ShareableBitmap take_document_screenshot();
 
     Messages::WebContentServer::WebdriverExecuteScriptResponse webdriver_execute_script(String const& body, Vector<String> const& json_arguments, Optional<u64> const& timeout, bool async);

Userland/Services/WebContent/ConnectionFromClient.cpp

@@ -464,35 +464,6 @@ void ConnectionFromClient::js_console_request_messages(i32 start_index)
         m_console_client->send_messages(start_index);
 }
 
-static Optional<Web::DOM::Element&> find_element_by_id(i32 element_id)
-{
-    auto* node = Web::DOM::Node::from_id(element_id);
-    if (!node || !node->is_element())
-        return {};
-
-    return verify_cast<Web::DOM::Element>(*node);
-}
-
-// https://w3c.github.io/webdriver/#dfn-scrolls-into-view
-void ConnectionFromClient::scroll_element_into_view(i32 element_id)
-{
-    auto element = find_element_by_id(element_id);
-    if (!element.has_value())
-        return;
-
-    // 1. Let options be the following ScrollIntoViewOptions:
-    Web::DOM::ScrollIntoViewOptions options {};
-    // Logical scroll position "block"
-    //     "end"
-    options.block = Web::Bindings::ScrollLogicalPosition::End;
-    // Logical scroll position "inline"
-    //     "nearest"
-    options.inline_ = Web::Bindings::ScrollLogicalPosition::Nearest;
-
-    // 2. Run Function.[[Call]](scrollIntoView, options) with element as the this value.
-    element->scroll_into_view(options);
-}
-
 // https://w3c.github.io/webdriver/#dfn-calculate-the-absolute-position
 static Gfx::IntPoint calculate_absolute_position_of_element(Web::Page const& page, JS::NonnullGCPtr<Web::Geometry::DOMRect> rect)
 {
@@ -511,36 +482,6 @@ static Gfx::IntPoint calculate_absolute_position_of_element(Web::Page const& pag
     return { x, y };
 }
 
-static Gfx::IntRect calculate_absolute_rect_of_element(Web::Page const& page, Web::DOM::Element const& element)
-{
-    auto bounding_rect = element.get_bounding_client_rect();
-    auto coordinates = calculate_absolute_position_of_element(page, bounding_rect);
-
-    return Gfx::IntRect {
-        coordinates.x(),
-        coordinates.y(),
-        static_cast<int>(bounding_rect->width()),
-        static_cast<int>(bounding_rect->height())
-    };
-}
-
-Messages::WebContentServer::TakeElementScreenshotResponse ConnectionFromClient::take_element_screenshot(i32 element_id)
-{
-    auto element = find_element_by_id(element_id);
-    if (!element.has_value())
-        return { {} };
-
-    auto viewport_rect = page().top_level_browsing_context().viewport_rect();
-
-    auto rect = calculate_absolute_rect_of_element(page(), *element);
-    rect.intersect(viewport_rect);
-
-    auto bitmap = Gfx::Bitmap::try_create(Gfx::BitmapFormat::BGRA8888, rect.size()).release_value_but_fixme_should_propagate_errors();
-    m_page_host->paint(rect, *bitmap);
-
-    return { bitmap->to_shareable_bitmap() };
-}
-
 Messages::WebContentServer::TakeDocumentScreenshotResponse ConnectionFromClient::take_document_screenshot()
 {
     auto* document = page().top_level_browsing_context().active_document();

Userland/Services/WebContent/ConnectionFromClient.h

@@ -83,8 +83,6 @@ private:
     virtual void run_javascript(String const&) override;
     virtual void js_console_request_messages(i32) override;
 
-    virtual void scroll_element_into_view(i32 element_id) override;
-    virtual Messages::WebContentServer::TakeElementScreenshotResponse take_element_screenshot(i32 element_id) override;
     virtual Messages::WebContentServer::TakeDocumentScreenshotResponse take_document_screenshot() override;
 
     virtual Messages::WebContentServer::GetLocalStorageEntriesResponse get_local_storage_entries() override;

Userland/Services/WebContent/WebContentServer.ipc

@@ -42,8 +42,6 @@ endpoint WebContentServer
     js_console_input(String js_source) =|
     js_console_request_messages(i32 start_index) =|
 
-    scroll_element_into_view(i32 element_id) => ()
-    take_element_screenshot(i32 element_id) => (Gfx::ShareableBitmap data)
     take_document_screenshot() => (Gfx::ShareableBitmap data)
 
     webdriver_execute_script(String body, Vector<String> json_arguments, Optional<u64> timeout, bool async) => (Web::WebDriver::ExecuteScriptResultType result_type, String json_result)

Userland/Services/WebContent/WebDriverClient.ipc

@@ -22,4 +22,5 @@ endpoint WebDriverClient {
     get_element_rect(String element_id) => (Web::WebDriver::Response response)
     is_element_enabled(String element_id) => (Web::WebDriver::Response response)
     take_screenshot() => (Web::WebDriver::Response response)
+    take_element_screenshot(String element_id) => (Web::WebDriver::Response response)
 }

Userland/Services/WebContent/WebDriverConnection.cpp

@@ -139,6 +139,22 @@ static ErrorOr<Web::DOM::Element*, Web::WebDriver::Error> get_known_connected_el
     return static_cast<Web::DOM::Element*>(node);
 }
 
+// https://w3c.github.io/webdriver/#dfn-scrolls-into-view
+static void scroll_element_into_view(Web::DOM::Element& element)
+{
+    // 1. Let options be the following ScrollIntoViewOptions:
+    Web::DOM::ScrollIntoViewOptions options {};
+    // Logical scroll position "block"
+    //     "end"
+    options.block = Web::Bindings::ScrollLogicalPosition::End;
+    // Logical scroll position "inline"
+    //     "nearest"
+    options.inline_ = Web::Bindings::ScrollLogicalPosition::Nearest;
+
+    // 2. Run Function.[[Call]](scrollIntoView, options) with element as the this value.
+    element.scroll_into_view(options);
+}
+
 static ErrorOr<String, Web::WebDriver::Error> get_property(JsonValue const& payload, StringView key)
 {
     if (!payload.is_object())
@@ -740,6 +756,38 @@ Messages::WebDriverClient::TakeScreenshotResponse WebDriverConnection::take_scre
     return make_success_response(move(encoded_string));
 }
 
+// 17.2 Take Element Screenshot, https://w3c.github.io/webdriver/#dfn-take-element-screenshot
+Messages::WebDriverClient::TakeElementScreenshotResponse WebDriverConnection::take_element_screenshot(String const& element_id)
+{
+    // 1. If the current top-level browsing context is no longer open, return error with error code no such window.
+    TRY(ensure_open_top_level_browsing_context());
+
+    // FIXME: 2. Handle any user prompts and return its value if it is an error.
+
+    // 3. Let element be the result of trying to get a known connected element with url variable element id.
+    auto* element = TRY(get_known_connected_element(element_id));
+
+    // 4. Scroll into view the element.
+    scroll_element_into_view(*element);
+
+    // 5. When the user agent is next to run the animation frame callbacks:
+    //     a. Let element rect be element’s rectangle.
+    //     b. Let screenshot result be the result of trying to call draw a bounding box from the framebuffer, given element rect as an argument.
+    //     c. Let canvas be a canvas element of screenshot result’s data.
+    //     d. Let encoding result be the result of trying encoding a canvas as Base64 canvas.
+    //     e. Let encoded string be encoding result’s data.
+    auto element_rect = calculate_absolute_rect_of_element(m_page_host.page(), *element);
+
+    auto encoded_string = TRY(Web::WebDriver::capture_element_screenshot(
+        [&](auto const& rect, auto& bitmap) { m_page_host.paint(rect, bitmap); },
+        m_page_host.page(),
+        *element,
+        element_rect));
+
+    // 6. Return success with data encoded string.
+    return make_success_response(move(encoded_string));
+}
+
 // https://w3c.github.io/webdriver/#dfn-no-longer-open
 ErrorOr<void, Web::WebDriver::Error> WebDriverConnection::ensure_open_top_level_browsing_context()
 {

Userland/Services/WebContent/WebDriverConnection.h

@@ -52,6 +52,7 @@ private:
     virtual Messages::WebDriverClient::GetElementRectResponse get_element_rect(String const& element_id) override;
     virtual Messages::WebDriverClient::IsElementEnabledResponse is_element_enabled(String const& element_id) override;
     virtual Messages::WebDriverClient::TakeScreenshotResponse take_screenshot() override;
+    virtual Messages::WebDriverClient::TakeElementScreenshotResponse take_element_screenshot(String const& element_id) override;
 
     ErrorOr<void, Web::WebDriver::Error> ensure_open_top_level_browsing_context();
     void restore_the_window();

Userland/Services/WebDriver/Client.cpp

@@ -815,8 +815,7 @@ Web::WebDriver::Response Client::handle_take_element_screenshot(Vector<StringVie
 {
     dbgln_if(WEBDRIVER_DEBUG, "Handling GET /session/<session_id>/element/<element_id>/screenshot");
     auto* session = TRY(find_session_with_id(parameters[0]));
-    auto result = TRY(session->take_element_screenshot(parameters[1]));
-    return make_json_value(result);
+    return session->web_content_connection().take_element_screenshot(parameters[1]);
 }
 
 }

Userland/Services/WebDriver/Session.cpp

@@ -11,14 +11,11 @@
 #include "Session.h"
 #include "BrowserConnection.h"
 #include "Client.h"
-#include <AK/Base64.h>
 #include <AK/NumericLimits.h>
 #include <AK/Time.h>
-#include <AK/URL.h>
 #include <LibCore/LocalServer.h>
 #include <LibCore/Stream.h>
 #include <LibCore/System.h>
-#include <LibGfx/PNGWriter.h>
 #include <LibGfx/Point.h>
 #include <LibGfx/Rect.h>
 #include <LibGfx/Size.h>
@@ -300,18 +297,6 @@ Web::WebDriver::Response Session::get_window_handles() const
     return JsonValue { handles };
 }
 
-// https://w3c.github.io/webdriver/#dfn-get-a-known-connected-element
-static ErrorOr<i32, Web::WebDriver::Error> get_known_connected_element(StringView element_id)
-{
-    // NOTE: The whole concept of "connected elements" is not implemented yet. See get_or_create_a_web_element_reference().
-    //       For now the element is only represented by its ID.
-    auto maybe_element_id = element_id.to_int();
-    if (!maybe_element_id.has_value())
-        return Web::WebDriver::Error::from_code(Web::WebDriver::ErrorCode::InvalidArgument, "Element ID is not an integer");
-
-    return maybe_element_id.release_value();
-}
-
 // 13.1 Get Page Source, https://w3c.github.io/webdriver/#dfn-get-page-source
 Web::WebDriver::Response Session::get_source()
 {
@@ -669,60 +654,4 @@ Web::WebDriver::Response Session::delete_all_cookies()
     return JsonValue();
 }
 
-// https://w3c.github.io/webdriver/#dfn-encoding-a-canvas-as-base64
-static ErrorOr<String, Web::WebDriver::Error> encode_bitmap_as_canvas_element(Gfx::Bitmap const& bitmap)
-{
-    // FIXME: 1. If the canvas element’s bitmap’s origin-clean flag is set to false, return error with error code unable to capture screen.
-
-    // 2. If the canvas element’s bitmap has no pixels (i.e. either its horizontal dimension or vertical dimension is zero) then return error with error code unable to capture screen.
-    if (bitmap.width() == 0 || bitmap.height() == 0)
-        return Web::WebDriver::Error::from_code(Web::WebDriver::ErrorCode::UnableToCaptureScreen, "Captured screenshot is empty"sv);
-
-    // 3. Let file be a serialization of the canvas element’s bitmap as a file, using "image/png" as an argument.
-    auto file = Gfx::PNGWriter::encode(bitmap);
-
-    // 4. Let data url be a data: URL representing file. [RFC2397]
-    auto data_url = AK::URL::create_with_data("image/png"sv, encode_base64(file), true).to_string();
-
-    // 5. Let index be the index of "," in data url.
-    auto index = data_url.find(',');
-    VERIFY(index.has_value());
-
-    // 6. Let encoded string be a substring of data url using (index + 1) as the start argument.
-    auto encoded_string = data_url.substring(*index + 1);
-
-    // 7. Return success with data encoded string.
-    return encoded_string;
-}
-
-// 17.2 Take Element Screenshot, https://w3c.github.io/webdriver/#dfn-take-element-screenshot
-Web::WebDriver::Response Session::take_element_screenshot(StringView parameter_element_id)
-{
-    // 1. If the current top-level browsing context is no longer open, return error with error code no such window.
-    TRY(check_for_open_top_level_browsing_context_or_return_error());
-
-    // FIXME: 2. Handle any user prompts and return its value if it is an error.
-
-    // 3. Let element be the result of trying to get a known connected element with url variable element id.
-    auto element_id = TRY(get_known_connected_element(parameter_element_id));
-
-    // 4. Scroll into view the element.
-    m_browser_connection->scroll_element_into_view(element_id);
-
-    // 5. When the user agent is next to run the animation frame callbacks:
-    //     a. Let element rect be element’s rectangle.
-    //     b. Let screenshot result be the result of trying to call draw a bounding box from the framebuffer, given element rect as an argument.
-    auto screenshot = m_browser_connection->take_element_screenshot(element_id);
-    if (!screenshot.is_valid())
-        return Web::WebDriver::Error::from_code(Web::WebDriver::ErrorCode::UnableToCaptureScreen, "Unable to capture screenshot"sv);
-
-    //     c. Let canvas be a canvas element of screenshot result’s data.
-    //     d. Let encoding result be the result of trying encoding a canvas as Base64 canvas.
-    //     e. Let encoded string be encoding result’s data.
-    auto encoded_string = TRY(encode_bitmap_as_canvas_element(*screenshot.bitmap()));
-
-    // 6. Return success with data encoded string.
-    return JsonValue { encoded_string };
-}
-
 }