LibWeb: Re-implement WebDriver endpoint handling within Web::WebDriver

This moves communication and route matching for WebDriver endpoints into
LibWeb. This is to reduce the amount of duplication required to create a
WebDriver implementation for Ladybird.

In doing so, this introduces some cleanup of WebDriver handling. Routes
are now a compile-time array, and matching a route is nearly free of
allocations (we still allocate a Vector for parsed parameters). This
implementation also makes heavier use of TRY semantics to propagate
errors into one handler.

Userland/Libraries/LibWeb/CMakeLists.txt

@@ -438,6 +438,7 @@ set(SOURCES
     WebAssembly/WebAssemblyTableConstructor.cpp
     WebAssembly/WebAssemblyTableObject.cpp
     WebAssembly/WebAssemblyTablePrototype.cpp
+    WebDriver/Client.cpp
     WebDriver/ElementLocationStrategies.cpp
     WebDriver/Error.cpp
     WebDriver/ExecuteScript.cpp
@@ -476,7 +477,7 @@ set(GENERATED_SOURCES
 serenity_lib(LibWeb web)
 
 # NOTE: We link with LibSoftGPU here instead of lazy loading it via dlopen() so that we do not have to unveil the library and pledge prot_exec.
-target_link_libraries(LibWeb PRIVATE LibCore LibCrypto LibJS LibMarkdown LibGemini LibGL LibGUI LibGfx LibIPC LibLocale LibRegex LibSoftGPU LibSyntax LibTextCodec LibUnicode LibWasm LibXML LibIDL)
+target_link_libraries(LibWeb PRIVATE LibCore LibCrypto LibJS LibMarkdown LibHTTP LibGemini LibGL LibGUI LibGfx LibIPC LibLocale LibRegex LibSoftGPU LibSyntax LibTextCodec LibUnicode LibWasm LibXML LibIDL)
 link_with_locale_data(LibWeb)
 
 generate_js_bindings(LibWeb)

Userland/Libraries/LibWeb/WebDriver/Client.cpp

@@ -0,0 +1,312 @@
+/*
+ * Copyright (c) 2022, Florent Castelli <florent.castelli@gmail.com>
+ * Copyright (c) 2022, Sam Atkins <atkinssj@serenityos.org>
+ * Copyright (c) 2022, Tobias Christiansen <tobyase@serenityos.org>
+ * Copyright (c) 2022, Linus Groh <linusg@serenityos.org>
+ * Copyright (c) 2022, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/ByteBuffer.h>
+#include <AK/Debug.h>
+#include <AK/Format.h>
+#include <AK/JsonObject.h>
+#include <AK/JsonParser.h>
+#include <AK/JsonValue.h>
+#include <AK/Span.h>
+#include <AK/StringBuilder.h>
+#include <AK/StringView.h>
+#include <LibCore/DateTime.h>
+#include <LibHTTP/HttpResponse.h>
+#include <LibWeb/WebDriver/Client.h>
+
+namespace Web::WebDriver {
+
+using RouteHandler = Response (*)(Client&, Parameters, JsonValue);
+
+struct Route {
+    HTTP::HttpRequest::Method method {};
+    StringView path;
+    RouteHandler handler { nullptr };
+};
+
+struct MatchedRoute {
+    RouteHandler handler;
+    Vector<StringView> parameters;
+};
+
+// clang-format off
+// This would be formatted rather badly.
+#define ROUTE(method, path, handler)                          \
+    Route {                                                   \
+        HTTP::HttpRequest::method,                            \
+        path,                                                 \
+        [](auto& client, auto parameters, auto payload) {     \
+            return client.handler(parameters, move(payload)); \
+        }                                                     \
+    }
+// clang-format on
+
+// https://w3c.github.io/webdriver/#dfn-endpoints
+static constexpr auto s_webdriver_endpoints = Array {
+    ROUTE(POST, "/session"sv, new_session),
+    ROUTE(DELETE, "/session/:session_id"sv, delete_session),
+    ROUTE(GET, "/status"sv, get_status),
+    ROUTE(GET, "/session/:session_id/timeouts"sv, get_timeouts),
+    ROUTE(POST, "/session/:session_id/timeouts"sv, set_timeouts),
+    ROUTE(POST, "/session/:session_id/url"sv, navigate_to),
+    ROUTE(GET, "/session/:session_id/url"sv, get_current_url),
+    ROUTE(POST, "/session/:session_id/back"sv, back),
+    ROUTE(POST, "/session/:session_id/forward"sv, forward),
+    ROUTE(POST, "/session/:session_id/refresh"sv, refresh),
+    ROUTE(GET, "/session/:session_id/title"sv, get_title),
+    ROUTE(GET, "/session/:session_id/window"sv, get_window_handle),
+    ROUTE(DELETE, "/session/:session_id/window"sv, close_window),
+    ROUTE(GET, "/session/:session_id/window/handles"sv, get_window_handles),
+    ROUTE(GET, "/session/:session_id/window/rect"sv, get_window_rect),
+    ROUTE(POST, "/session/:session_id/window/rect"sv, set_window_rect),
+    ROUTE(POST, "/session/:session_id/window/maximize"sv, maximize_window),
+    ROUTE(POST, "/session/:session_id/window/minimize"sv, minimize_window),
+    ROUTE(POST, "/session/:session_id/element"sv, find_element),
+    ROUTE(POST, "/session/:session_id/elements"sv, find_elements),
+    ROUTE(POST, "/session/:session_id/element/:element_id/element"sv, find_element_from_element),
+    ROUTE(POST, "/session/:session_id/element/:element_id/elements"sv, find_elements_from_element),
+    ROUTE(GET, "/session/:session_id/element/:element_id/selected"sv, is_element_selected),
+    ROUTE(GET, "/session/:session_id/element/:element_id/attribute/:name"sv, get_element_attribute),
+    ROUTE(GET, "/session/:session_id/element/:element_id/property/:name"sv, get_element_property),
+    ROUTE(GET, "/session/:session_id/element/:element_id/css/:name"sv, get_element_css_value),
+    ROUTE(GET, "/session/:session_id/element/:element_id/text"sv, get_element_text),
+    ROUTE(GET, "/session/:session_id/element/:element_id/name"sv, get_element_tag_name),
+    ROUTE(GET, "/session/:session_id/element/:element_id/rect"sv, get_element_rect),
+    ROUTE(GET, "/session/:session_id/element/:element_id/enabled"sv, is_element_enabled),
+    ROUTE(GET, "/session/:session_id/source"sv, get_source),
+    ROUTE(POST, "/session/:session_id/execute/sync"sv, execute_script),
+    ROUTE(POST, "/session/:session_id/execute/async"sv, execute_async_script),
+    ROUTE(GET, "/session/:session_id/cookie"sv, get_all_cookies),
+    ROUTE(GET, "/session/:session_id/cookie/:name"sv, get_named_cookie),
+    ROUTE(POST, "/session/:session_id/cookie"sv, add_cookie),
+    ROUTE(DELETE, "/session/:session_id/cookie/:name"sv, delete_cookie),
+    ROUTE(DELETE, "/session/:session_id/cookie"sv, delete_all_cookies),
+    ROUTE(GET, "/session/:session_id/screenshot"sv, take_screenshot),
+    ROUTE(GET, "/session/:session_id/element/:element_id/screenshot"sv, take_element_screenshot),
+};
+
+// https://w3c.github.io/webdriver/#dfn-match-a-request
+static ErrorOr<MatchedRoute, Error> match_route(HTTP::HttpRequest const& request)
+{
+    dbgln_if(WEBDRIVER_DEBUG, "match_route({}, {})", HTTP::to_string(request.method()), request.resource());
+
+    auto request_path = request.resource().view();
+    Vector<StringView> parameters;
+
+    auto next_segment = [](auto& path) -> Optional<StringView> {
+        if (auto index = path.find('/'); index.has_value() && (*index + 1) < path.length()) {
+            path = path.substring_view(*index + 1);
+
+            if (index = path.find('/'); index.has_value())
+                return path.substring_view(0, *index);
+            return path;
+        }
+
+        path = {};
+        return {};
+    };
+
+    for (auto const& route : s_webdriver_endpoints) {
+        dbgln_if(WEBDRIVER_DEBUG, "- Checking {} {}", HTTP::to_string(route.method), route.path);
+        if (route.method != request.method())
+            continue;
+
+        auto route_path = route.path;
+        Optional<bool> match;
+
+        auto on_failed_match = [&]() {
+            request_path = request.resource();
+            parameters.clear();
+            match = false;
+        };
+
+        while (!match.has_value()) {
+            auto request_segment = next_segment(request_path);
+            auto route_segment = next_segment(route_path);
+
+            if (!request_segment.has_value() && !route_segment.has_value())
+                match = true;
+            else if (request_segment.has_value() != route_segment.has_value())
+                on_failed_match();
+            else if (route_segment->starts_with(':'))
+                parameters.append(*request_segment);
+            else if (request_segment != route_segment)
+                on_failed_match();
+        }
+
+        if (*match) {
+            dbgln_if(WEBDRIVER_DEBUG, "- Found match with parameters={}", parameters);
+            return MatchedRoute { route.handler, parameters };
+        }
+    }
+
+    return Error::from_code(ErrorCode::UnknownCommand, "The command was not recognized.");
+}
+
+Client::Client(NonnullOwnPtr<Core::Stream::BufferedTCPSocket> socket, Core::Object* parent)
+    : Core::Object(parent)
+    , m_socket(move(socket))
+{
+    m_socket->on_ready_to_read = [this] {
+        if (auto result = on_ready_to_read(); result.is_error()) {
+            result.error().visit(
+                [](AK::Error const& error) {
+                    warnln("Internal error: {}", error);
+                },
+                [this](WebDriver::Error const& error) {
+                    if (send_error_response(error).is_error())
+                        warnln("Could not send error response");
+                });
+
+            die();
+        }
+
+        m_request = {};
+    };
+}
+
+Client::~Client()
+{
+    m_socket->close();
+}
+
+void Client::die()
+{
+    deferred_invoke([this] { remove_from_parent(); });
+}
+
+ErrorOr<void, Client::WrappedError> Client::on_ready_to_read()
+{
+    // FIXME: All this should be moved to LibHTTP and be made spec compliant.
+    auto buffer = TRY(ByteBuffer::create_uninitialized(m_socket->buffer_size()));
+    StringBuilder builder;
+
+    for (;;) {
+        if (!TRY(m_socket->can_read_without_blocking()))
+            break;
+
+        auto data = TRY(m_socket->read(buffer));
+        TRY(builder.try_append(StringView { data }));
+
+        if (m_socket->is_eof())
+            break;
+    }
+
+    m_request = HTTP::HttpRequest::from_raw_request(builder.to_byte_buffer());
+    if (!m_request.has_value())
+        return {};
+
+    auto body = TRY(read_body_as_json());
+    TRY(handle_request(move(body)));
+
+    return {};
+}
+
+ErrorOr<JsonValue, Client::WrappedError> Client::read_body_as_json()
+{
+    // FIXME: If we received a multipart body here, this would fail badly.
+    // FIXME: Check the Content-Type is actually application/json.
+    size_t content_length = 0;
+
+    for (auto const& header : m_request->headers()) {
+        if (header.name.equals_ignoring_case("Content-Length"sv)) {
+            content_length = header.value.to_uint<size_t>(TrimWhitespace::Yes).value_or(0);
+            break;
+        }
+    }
+
+    if (content_length == 0)
+        return JsonValue {};
+
+    JsonParser json_parser(m_request->body());
+    return TRY(json_parser.parse());
+}
+
+ErrorOr<void, Client::WrappedError> Client::handle_request(JsonValue body)
+{
+    if constexpr (WEBDRIVER_DEBUG) {
+        dbgln("Got HTTP request: {} {}", m_request->method_name(), m_request->resource());
+        if (!body.is_null())
+            dbgln("Body: {}", body.to_string());
+    }
+
+    auto const& [handler, parameters] = TRY(match_route(*m_request));
+    auto result = TRY((*handler)(*this, parameters, move(body)));
+    return send_success_response(move(result));
+}
+
+ErrorOr<void, Client::WrappedError> Client::send_success_response(JsonValue result)
+{
+    auto content = result.serialized<StringBuilder>();
+
+    StringBuilder builder;
+    builder.append("HTTP/1.0 200 OK\r\n"sv);
+    builder.append("Server: WebDriver (SerenityOS)\r\n"sv);
+    builder.append("X-Frame-Options: SAMEORIGIN\r\n"sv);
+    builder.append("X-Content-Type-Options: nosniff\r\n"sv);
+    builder.append("Pragma: no-cache\r\n"sv);
+    builder.append("Content-Type: application/json; charset=utf-8\r\n"sv);
+    builder.appendff("Content-Length: {}\r\n", content.length());
+    builder.append("\r\n"sv);
+
+    auto builder_contents = builder.to_byte_buffer();
+    TRY(m_socket->write(builder_contents));
+
+    while (!content.is_empty()) {
+        auto bytes_sent = TRY(m_socket->write(content.bytes()));
+        content = content.substring_view(bytes_sent);
+    }
+
+    bool keep_alive = false;
+    if (auto it = m_request->headers().find_if([](auto& header) { return header.name.equals_ignoring_case("Connection"sv); }); !it.is_end())
+        keep_alive = it->value.trim_whitespace().equals_ignoring_case("keep-alive"sv);
+
+    if (!keep_alive)
+        die();
+
+    log_response(200);
+    return {};
+}
+
+ErrorOr<void, Client::WrappedError> Client::send_error_response(Error const& error)
+{
+    // FIXME: Implement to spec.
+    dbgln_if(WEBDRIVER_DEBUG, "Sending error response: {} {}: {}", error.http_status, error.error, error.message);
+    auto reason = HTTP::HttpResponse::reason_phrase_for_code(error.http_status);
+
+    JsonObject result;
+    result.set("error", error.error);
+    result.set("message", error.message);
+    result.set("stacktrace", "");
+    if (error.data.has_value())
+        result.set("data", *error.data);
+
+    StringBuilder content_builder;
+    result.serialize(content_builder);
+
+    StringBuilder header_builder;
+    header_builder.appendff("HTTP/1.0 {} {}\r\n", error.http_status, reason);
+    header_builder.append("Content-Type: application/json; charset=UTF-8\r\n"sv);
+    header_builder.appendff("Content-Length: {}\r\n", content_builder.length());
+    header_builder.append("\r\n"sv);
+
+    TRY(m_socket->write(header_builder.to_byte_buffer()));
+    TRY(m_socket->write(content_builder.to_byte_buffer()));
+
+    log_response(error.http_status);
+    return {};
+}
+
+void Client::log_response(unsigned code)
+{
+    outln("{} :: {:03d} :: {} {}", Core::DateTime::now().to_string(), code, m_request->method_name(), m_request->resource());
+}
+
+}

Userland/Libraries/LibWeb/WebDriver/Client.h

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2022, Florent Castelli <florent.castelli@gmail.com>
+ * Copyright (c) 2022, Linus Groh <linusg@serenityos.org>
+ * Copyright (c) 2022, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Error.h>
+#include <AK/NonnullOwnPtrVector.h>
+#include <AK/String.h>
+#include <AK/Variant.h>
+#include <LibCore/Object.h>
+#include <LibCore/Stream.h>
+#include <LibHTTP/Forward.h>
+#include <LibHTTP/HttpRequest.h>
+#include <LibWeb/WebDriver/Error.h>
+#include <LibWeb/WebDriver/Response.h>
+
+namespace Web::WebDriver {
+
+using Parameters = Span<StringView const>;
+
+class Client : public Core::Object {
+    C_OBJECT_ABSTRACT(Client);
+
+public:
+    virtual ~Client();
+
+    // 8. Sessions, https://w3c.github.io/webdriver/#sessions
+    virtual Response new_session(Parameters parameters, JsonValue payload) = 0;
+    virtual Response delete_session(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_status(Parameters parameters, JsonValue payload) = 0;
+
+    // 9. Timeouts, https://w3c.github.io/webdriver/#timeouts
+    virtual Response get_timeouts(Parameters parameters, JsonValue payload) = 0;
+    virtual Response set_timeouts(Parameters parameters, JsonValue payload) = 0;
+
+    // 10. Navigation, https://w3c.github.io/webdriver/#navigation
+    virtual Response navigate_to(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_current_url(Parameters parameters, JsonValue payload) = 0;
+    virtual Response back(Parameters parameters, JsonValue payload) = 0;
+    virtual Response forward(Parameters parameters, JsonValue payload) = 0;
+    virtual Response refresh(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_title(Parameters parameters, JsonValue payload) = 0;
+
+    // 11. Contexts, https://w3c.github.io/webdriver/#contexts
+    virtual Response get_window_handle(Parameters parameters, JsonValue payload) = 0;
+    virtual Response close_window(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_window_handles(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_window_rect(Parameters parameters, JsonValue payload) = 0;
+    virtual Response set_window_rect(Parameters parameters, JsonValue payload) = 0;
+    virtual Response maximize_window(Parameters parameters, JsonValue payload) = 0;
+    virtual Response minimize_window(Parameters parameters, JsonValue payload) = 0;
+    virtual Response fullscreen_window(Parameters parameters, JsonValue payload) = 0;
+
+    // 12. Elements, https://w3c.github.io/webdriver/#elements
+    virtual Response find_element(Parameters parameters, JsonValue payload) = 0;
+    virtual Response find_elements(Parameters parameters, JsonValue payload) = 0;
+    virtual Response find_element_from_element(Parameters parameters, JsonValue payload) = 0;
+    virtual Response find_elements_from_element(Parameters parameters, JsonValue payload) = 0;
+    virtual Response is_element_selected(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_element_attribute(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_element_property(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_element_css_value(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_element_text(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_element_tag_name(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_element_rect(Parameters parameters, JsonValue payload) = 0;
+    virtual Response is_element_enabled(Parameters parameters, JsonValue payload) = 0;
+
+    // 13. https://w3c.github.io/webdriver/#document, https://w3c.github.io/webdriver/#get-page-source
+    virtual Response get_source(Parameters parameters, JsonValue payload) = 0;
+    virtual Response execute_script(Parameters parameters, JsonValue payload) = 0;
+    virtual Response execute_async_script(Parameters parameters, JsonValue payload) = 0;
+
+    // 14. Cookies, https://w3c.github.io/webdriver/#cookies
+    virtual Response get_all_cookies(Parameters parameters, JsonValue payload) = 0;
+    virtual Response get_named_cookie(Parameters parameters, JsonValue payload) = 0;
+    virtual Response add_cookie(Parameters parameters, JsonValue payload) = 0;
+    virtual Response delete_cookie(Parameters parameters, JsonValue payload) = 0;
+    virtual Response delete_all_cookies(Parameters parameters, JsonValue payload) = 0;
+
+    // 17. Screen capture, https://w3c.github.io/webdriver/#screen-capture
+    virtual Response take_screenshot(Parameters parameters, JsonValue payload) = 0;
+    virtual Response take_element_screenshot(Parameters parameters, JsonValue payload) = 0;
+
+protected:
+    Client(NonnullOwnPtr<Core::Stream::BufferedTCPSocket>, Core::Object* parent);
+
+private:
+    using WrappedError = Variant<AK::Error, WebDriver::Error>;
+
+    void die();
+    ErrorOr<void, WrappedError> on_ready_to_read();
+    ErrorOr<JsonValue, WrappedError> read_body_as_json();
+    ErrorOr<void, WrappedError> handle_request(JsonValue body);
+    ErrorOr<void, WrappedError> send_success_response(JsonValue result);
+    ErrorOr<void, WrappedError> send_error_response(Error const& error);
+    void log_response(unsigned code);
+
+    NonnullOwnPtr<Core::Stream::BufferedTCPSocket> m_socket;
+    Optional<HTTP::HttpRequest> m_request;
+};
+
+}