Ladybird: Add RequestServer process to optionally replace Qt Networking

LibTLS still can't access many parts of the web, so let's hide this
behind a flag (with all the plumbing that entails).

Hopefully this can encourage folks to improve LibTLS's algorithm support
:^).

Ladybird/BrowserWindow.cpp

@@ -39,11 +39,12 @@ static QIcon const& app_icon()
     return icon;
 }
 
-BrowserWindow::BrowserWindow(Browser::CookieJar& cookie_jar, StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling enable_callgrind_profiling, WebView::UseJavaScriptBytecode use_javascript_bytecode)
+BrowserWindow::BrowserWindow(Browser::CookieJar& cookie_jar, StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling enable_callgrind_profiling, WebView::UseJavaScriptBytecode use_javascript_bytecode, UseLagomNetworking use_lagom_networking)
     : m_cookie_jar(cookie_jar)
     , m_webdriver_content_ipc_path(webdriver_content_ipc_path)
     , m_enable_callgrind_profiling(enable_callgrind_profiling)
     , m_use_javascript_bytecode(use_javascript_bytecode)
+    , m_use_lagom_networking(use_lagom_networking)
 {
     setWindowIcon(app_icon());
     m_tabs_container = new QTabWidget(this);
@@ -418,7 +419,7 @@ void BrowserWindow::debug_request(DeprecatedString const& request, DeprecatedStr
 
 Tab& BrowserWindow::new_tab(QString const& url, Web::HTML::ActivateTab activate_tab)
 {
-    auto tab = make<Tab>(this, m_webdriver_content_ipc_path, m_enable_callgrind_profiling, m_use_javascript_bytecode);
+    auto tab = make<Tab>(this, m_webdriver_content_ipc_path, m_enable_callgrind_profiling, m_use_javascript_bytecode, m_use_lagom_networking);
     auto tab_ptr = tab.ptr();
     m_tabs.append(std::move(tab));
 

Ladybird/BrowserWindow.h

@@ -26,7 +26,7 @@ class CookieJar;
 class BrowserWindow : public QMainWindow {
     Q_OBJECT
 public:
-    explicit BrowserWindow(Browser::CookieJar&, StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling, WebView::UseJavaScriptBytecode);
+    explicit BrowserWindow(Browser::CookieJar&, StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling, WebView::UseJavaScriptBytecode, UseLagomNetworking);
 
     WebContentView& view() const { return m_current_tab->view(); }
 
@@ -117,4 +117,5 @@ private:
     StringView m_webdriver_content_ipc_path;
     WebView::EnableCallgrindProfiling m_enable_callgrind_profiling;
     WebView::UseJavaScriptBytecode m_use_javascript_bytecode;
+    UseLagomNetworking m_use_lagom_networking;
 };

Ladybird/CMakeLists.txt

@@ -99,7 +99,7 @@ set(SOURCES
 )
 
 qt_add_executable(ladybird ${SOURCES})
-target_link_libraries(ladybird PRIVATE Qt::Core Qt::Gui Qt::Network Qt::Widgets LibCore LibFileSystem LibGfx LibGUI LibIPC LibJS LibMain LibWeb LibWebView LibSQL)
+target_link_libraries(ladybird PRIVATE Qt::Core Qt::Gui Qt::Network Qt::Widgets LibCore LibFileSystem LibGfx LibGUI LibIPC LibJS LibMain LibWeb LibWebView LibSQL LibProtocol)
 
 target_include_directories(ladybird PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 target_include_directories(ladybird PRIVATE ${SERENITY_SOURCE_DIR}/Userland/)
@@ -113,7 +113,7 @@ qt_add_executable(headless-browser
     Utilities.cpp)
 
 target_include_directories(headless-browser PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
-target_link_libraries(headless-browser PRIVATE Qt::Core LibWeb LibWebView LibWebSocket LibCrypto LibFileSystem LibGemini LibHTTP LibJS LibGfx LibMain LibTLS LibIPC LibJS LibDiff)
+target_link_libraries(headless-browser PRIVATE Qt::Core LibWeb LibWebView LibWebSocket LibCrypto LibFileSystem LibGemini LibHTTP LibJS LibGfx LibMain LibTLS LibIPC LibDiff LibProtocol)
 
 set_target_properties(ladybird PROPERTIES
     MACOSX_BUNDLE_GUI_IDENTIFIER org.SerenityOS.Ladybird
@@ -144,13 +144,15 @@ add_custom_target(debug${LADYBIRD_CUSTOM_TARGET_SUFFIX}
 add_subdirectory(SQLServer)
 add_subdirectory(WebContent)
 add_subdirectory(WebDriver)
-add_dependencies(ladybird SQLServer WebContent WebDriver headless-browser)
+add_subdirectory(RequestServer)
+add_dependencies(ladybird SQLServer WebContent WebDriver RequestServer headless-browser)
 
 if (APPLE)
     # FIXME: Create a proper app bundle for each helper process
     set(app_dir "$<TARGET_FILE_DIR:ladybird>")
     set(bundle_dir "$<TARGET_BUNDLE_DIR:ladybird>")
     add_custom_command(TARGET ladybird POST_BUILD
+        COMMAND "${CMAKE_COMMAND}" -E copy_if_different "$<TARGET_FILE:RequestServer>" "${app_dir}"
         COMMAND "${CMAKE_COMMAND}" -E copy_if_different "$<TARGET_FILE:SQLServer>" "${app_dir}"
         COMMAND "${CMAKE_COMMAND}" -E copy_if_different "$<TARGET_FILE:WebContent>" "${app_dir}"
         COMMAND "${CMAKE_COMMAND}" -E copy_if_different "$<TARGET_FILE:WebDriver>" "${app_dir}"

Ladybird/ConsoleWidget.cpp

@@ -28,7 +28,7 @@ ConsoleWidget::ConsoleWidget()
 {
     setLayout(new QVBoxLayout);
 
-    m_output_view = new WebContentView({}, WebView::EnableCallgrindProfiling::No, WebView::UseJavaScriptBytecode::No);
+    m_output_view = new WebContentView({}, WebView::EnableCallgrindProfiling::No, WebView::UseJavaScriptBytecode::No, UseLagomNetworking::No);
     if (is_using_dark_system_theme(*this))
         m_output_view->update_palette(WebContentView::PaletteMode::Dark);
 

Ladybird/HelperProcess.cpp

@@ -10,7 +10,8 @@ ErrorOr<NonnullRefPtr<WebView::WebContentClient>> launch_web_content_process(Web
     ReadonlySpan<String> candidate_web_content_paths,
     WebView::EnableCallgrindProfiling enable_callgrind_profiling,
     WebView::IsLayoutTestMode is_layout_test_mode,
-    WebView::UseJavaScriptBytecode use_javascript_bytecode)
+    WebView::UseJavaScriptBytecode use_javascript_bytecode,
+    UseLagomNetworking use_lagom_networking)
 {
     int socket_fds[2] {};
     TRY(Core::System::socketpair(AF_LOCAL, SOCK_STREAM, 0, socket_fds));
@@ -54,6 +55,8 @@ ErrorOr<NonnullRefPtr<WebView::WebContentClient>> launch_web_content_process(Web
                 arguments.append("--layout-test-mode"sv);
             if (use_javascript_bytecode == WebView::UseJavaScriptBytecode::Yes)
                 arguments.append("--use-bytecode"sv);
+            if (use_lagom_networking == UseLagomNetworking::Yes)
+                arguments.append("--use-lagom-networking"sv);
 
             result = Core::System::exec(arguments[0], arguments.span(), Core::System::SearchInPath::Yes);
             if (!result.is_error())
@@ -83,3 +86,60 @@ ErrorOr<NonnullRefPtr<WebView::WebContentClient>> launch_web_content_process(Web
 
     return new_client;
 }
+
+ErrorOr<NonnullRefPtr<Protocol::RequestClient>> launch_request_server_process(ReadonlySpan<String> candidate_request_server_paths)
+{
+    int socket_fds[2] {};
+    TRY(Core::System::socketpair(AF_LOCAL, SOCK_STREAM, 0, socket_fds));
+
+    int ui_fd = socket_fds[0];
+    int rc_fd = socket_fds[1];
+
+    int fd_passing_socket_fds[2] {};
+    TRY(Core::System::socketpair(AF_LOCAL, SOCK_STREAM, 0, fd_passing_socket_fds));
+
+    int ui_fd_passing_fd = fd_passing_socket_fds[0];
+    int rc_fd_passing_fd = fd_passing_socket_fds[1];
+
+    if (auto child_pid = TRY(Core::System::fork()); child_pid == 0) {
+        TRY(Core::System::close(ui_fd));
+        TRY(Core::System::close(ui_fd_passing_fd));
+
+        auto takeover_string = TRY(String::formatted("RequestServer:{}", rc_fd));
+        TRY(Core::System::setenv("SOCKET_TAKEOVER"sv, takeover_string, true));
+
+        auto fd_passing_socket_string = TRY(String::number(rc_fd_passing_fd));
+
+        ErrorOr<void> result;
+        for (auto const& path : candidate_request_server_paths) {
+
+            if (Core::System::access(path, X_OK).is_error())
+                continue;
+
+            auto arguments = Vector {
+                path.bytes_as_string_view(),
+                "--fd-passing-socket"sv,
+                fd_passing_socket_string,
+            };
+
+            result = Core::System::exec(arguments[0], arguments.span(), Core::System::SearchInPath::Yes);
+            if (!result.is_error())
+                break;
+        }
+
+        if (result.is_error())
+            warnln("Could not launch any of {}: {}", candidate_request_server_paths, result.error());
+        VERIFY_NOT_REACHED();
+    }
+
+    TRY(Core::System::close(rc_fd));
+    TRY(Core::System::close(rc_fd_passing_fd));
+
+    auto socket = TRY(Core::LocalSocket::adopt_fd(ui_fd));
+    TRY(socket->set_blocking(true));
+
+    auto new_client = TRY(try_make_ref_counted<Protocol::RequestClient>(move(socket)));
+    new_client->set_fd_passing_socket(TRY(Core::LocalSocket::adopt_fd(ui_fd_passing_fd)));
+
+    return new_client;
+}

Ladybird/HelperProcess.h

@@ -6,10 +6,11 @@
 
 #pragma once
 
+#include "Types.h"
 #include <AK/Error.h>
 #include <AK/Span.h>
 #include <AK/StringView.h>
-
+#include <LibProtocol/RequestClient.h>
 #include <LibWebView/ViewImplementation.h>
 #include <LibWebView/WebContentClient.h>
 
@@ -17,4 +18,7 @@ ErrorOr<NonnullRefPtr<WebView::WebContentClient>> launch_web_content_process(Web
     ReadonlySpan<String> candidate_web_content_paths,
     WebView::EnableCallgrindProfiling,
     WebView::IsLayoutTestMode,
-    WebView::UseJavaScriptBytecode);
+    WebView::UseJavaScriptBytecode,
+    UseLagomNetworking);
+
+ErrorOr<NonnullRefPtr<Protocol::RequestClient>> launch_request_server_process(ReadonlySpan<String> candidate_request_server_paths);

Ladybird/RequestServer/CMakeLists.txt

@@ -0,0 +1,25 @@
+set(REQUESTSERVER_SOURCE_DIR ${SERENITY_SOURCE_DIR}/Userland/Services/RequestServer)
+
+set(REQUESTSERVER_SOURCES
+    ${REQUESTSERVER_SOURCE_DIR}/ConnectionFromClient.cpp
+    ${REQUESTSERVER_SOURCE_DIR}/ConnectionCache.cpp
+    ${REQUESTSERVER_SOURCE_DIR}/Request.cpp
+    ${REQUESTSERVER_SOURCE_DIR}/GeminiRequest.cpp
+    ${REQUESTSERVER_SOURCE_DIR}/GeminiProtocol.cpp
+    ${REQUESTSERVER_SOURCE_DIR}/HttpRequest.cpp
+    ${REQUESTSERVER_SOURCE_DIR}/HttpProtocol.cpp
+    ${REQUESTSERVER_SOURCE_DIR}/HttpsRequest.cpp
+    ${REQUESTSERVER_SOURCE_DIR}/HttpsProtocol.cpp
+    ${REQUESTSERVER_SOURCE_DIR}/Protocol.cpp
+    ../Utilities.cpp
+    main.cpp
+)
+
+qt_add_executable(RequestServer ${REQUESTSERVER_SOURCES})
+
+target_include_directories(RequestServer PRIVATE ${SERENITY_SOURCE_DIR}/Userland/Services/)
+target_include_directories(RequestServer PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/..)
+target_link_libraries(RequestServer PRIVATE Qt::Core LibCore LibMain LibCrypto LibFileSystem LibGemini LibHTTP LibIPC LibMain LibTLS)
+if (ANDROID)
+    link_android_libs(RequestServer)
+endif()

Ladybird/RequestServer/main.cpp

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * Copyright (c) 2023, Andrew Kaster <akaster@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "../Utilities.h"
+#include <AK/LexicalPath.h>
+#include <AK/OwnPtr.h>
+#include <LibCore/ArgsParser.h>
+#include <LibCore/EventLoop.h>
+#include <LibCore/LocalServer.h>
+#include <LibCore/System.h>
+#include <LibFileSystem/FileSystem.h>
+#include <LibIPC/SingleServer.h>
+#include <LibMain/Main.h>
+#include <LibTLS/Certificate.h>
+#include <QCoreApplication>
+#include <RequestServer/ConnectionFromClient.h>
+#include <RequestServer/GeminiProtocol.h>
+#include <RequestServer/HttpProtocol.h>
+#include <RequestServer/HttpsProtocol.h>
+
+ErrorOr<String> find_certificates()
+{
+    auto cert_path = TRY(String::formatted("{}/res/ladybird/cacert.pem", s_serenity_resource_root));
+    if (!FileSystem::exists(cert_path)) {
+        auto app_dir = ak_deprecated_string_from_qstring(QCoreApplication::applicationDirPath());
+
+        cert_path = TRY(String::formatted("{}/cacert.pem", LexicalPath(app_dir).parent()));
+        if (!FileSystem::exists(cert_path))
+            return Error::from_string_view("Don't know how to load certs!"sv);
+    }
+    return cert_path;
+}
+
+ErrorOr<int> serenity_main(Main::Arguments arguments)
+{
+    QCoreApplication application(arguments.argc, arguments.argv);
+    platform_init();
+
+    // Ensure the certificates are read out here.
+    DefaultRootCACertificates::set_default_certificate_path(TRY(find_certificates()));
+    [[maybe_unused]] auto& certs = DefaultRootCACertificates::the();
+
+    int fd_passing_socket { -1 };
+
+    Core::ArgsParser args_parser;
+    args_parser.add_option(fd_passing_socket, "File descriptor of the fd passing socket", "fd-passing-socket", 'c', "fd-passing-socket");
+    args_parser.parse(arguments);
+
+    Core::EventLoop event_loop;
+
+    [[maybe_unused]] auto gemini = make<RequestServer::GeminiProtocol>();
+    [[maybe_unused]] auto http = make<RequestServer::HttpProtocol>();
+    [[maybe_unused]] auto https = make<RequestServer::HttpsProtocol>();
+
+    auto client = TRY(IPC::take_over_accepted_client_from_system_server<RequestServer::ConnectionFromClient>());
+    client->set_fd_passing_socket(TRY(Core::LocalSocket::adopt_fd(fd_passing_socket)));
+
+    auto result = event_loop.exec();
+
+    // FIXME: We exit instead of returning, so that protocol destructors don't get called.
+    //        The Protocol base class should probably do proper de-registration instead of
+    //        just VERIFY_NOT_REACHED().
+    exit(result);
+}

Ladybird/Tab.cpp

@@ -47,7 +47,7 @@ static QIcon create_tvg_icon_with_theme_colors(QString name, QPalette const& pal
     return QIcon(icon_engine);
 }
 
-Tab::Tab(BrowserWindow* window, StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling enable_callgrind_profiling, WebView::UseJavaScriptBytecode use_javascript_bytecode)
+Tab::Tab(BrowserWindow* window, StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling enable_callgrind_profiling, WebView::UseJavaScriptBytecode use_javascript_bytecode, UseLagomNetworking use_lagom_networking)
     : QWidget(window)
     , m_window(window)
 {
@@ -55,7 +55,7 @@ Tab::Tab(BrowserWindow* window, StringView webdriver_content_ipc_path, WebView::
     m_layout->setSpacing(0);
     m_layout->setContentsMargins(0, 0, 0, 0);
 
-    m_view = new WebContentView(webdriver_content_ipc_path, enable_callgrind_profiling, use_javascript_bytecode);
+    m_view = new WebContentView(webdriver_content_ipc_path, enable_callgrind_profiling, use_javascript_bytecode, use_lagom_networking);
     m_toolbar = new QToolBar(this);
     m_location_edit = new LocationEdit(this);
 

Ladybird/Tab.h

@@ -28,7 +28,7 @@ class InspectorWidget;
 class Tab final : public QWidget {
     Q_OBJECT
 public:
-    Tab(BrowserWindow* window, StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling, WebView::UseJavaScriptBytecode);
+    Tab(BrowserWindow* window, StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling, WebView::UseJavaScriptBytecode, UseLagomNetworking);
     virtual ~Tab() override;
 
     WebContentView& view() { return *m_view; }

Ladybird/Types.h

@@ -0,0 +1,12 @@
+/*
+ * Copyright (c) 2023, Andrew Kaster <akaster@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+enum UseLagomNetworking {
+    No,
+    Yes
+};

Ladybird/WebContent/CMakeLists.txt

@@ -10,6 +10,7 @@ set(WEBCONTENT_SOURCES
     ../AudioThread.cpp
     ../EventLoopImplementationQt.cpp
     ../FontPluginLadybird.cpp
+    ../HelperProcess.cpp
     ../ImageCodecPluginLadybird.cpp
     ../RequestManagerQt.cpp
     ../Utilities.cpp
@@ -23,7 +24,7 @@ qt_add_executable(WebContent ${WEBCONTENT_SOURCES})
 
 target_include_directories(WebContent PRIVATE ${SERENITY_SOURCE_DIR}/Userland/Services/)
 target_include_directories(WebContent PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/..)
-target_link_libraries(WebContent PRIVATE Qt::Core Qt::Network Qt::Multimedia LibAudio LibCore LibFileSystem LibGfx LibIPC LibJS LibMain LibWeb LibWebSocket)
+target_link_libraries(WebContent PRIVATE Qt::Core Qt::Network Qt::Multimedia LibAudio LibCore LibFileSystem LibGfx LibIPC LibJS LibMain LibWeb LibWebSocket LibProtocol LibSQL LibWebView)
 if (ANDROID)
     link_android_libs(WebContent)
 endif()

Ladybird/WebContent/main.cpp

@@ -7,6 +7,7 @@
 #include "../AudioCodecPluginLadybird.h"
 #include "../EventLoopImplementationQt.h"
 #include "../FontPluginLadybird.h"
+#include "../HelperProcess.h"
 #include "../ImageCodecPluginLadybird.h"
 #include "../RequestManagerQt.h"
 #include "../Utilities.h"
@@ -28,8 +29,8 @@
 #include <LibWeb/PermissionsPolicy/AutoplayAllowlist.h>
 #include <LibWeb/Platform/EventLoopPluginSerenity.h>
 #include <LibWeb/WebSockets/WebSocket.h>
+#include <LibWebView/RequestServerAdapter.h>
 #include <QCoreApplication>
-#include <QTimer>
 #include <WebContent/ConnectionFromClient.h>
 #include <WebContent/PageHost.h>
 #include <WebContent/WebDriverConnection.h>
@@ -55,7 +56,6 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
         return Ladybird::AudioCodecPluginLadybird::create(move(loader));
     });
 
-    Web::ResourceLoader::initialize(RequestManagerQt::create());
     Web::WebSockets::WebSocketClientManager::initialize(Ladybird::WebSocketClientManagerLadybird::create());
 
     Web::FrameLoader::set_default_favicon_path(DeprecatedString::formatted("{}/res/icons/16x16/app-browser.png", s_serenity_resource_root));
@@ -63,13 +63,23 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
     int webcontent_fd_passing_socket { -1 };
     bool is_layout_test_mode = false;
     bool use_javascript_bytecode = false;
+    bool use_lagom_networking = false;
 
     Core::ArgsParser args_parser;
     args_parser.add_option(webcontent_fd_passing_socket, "File descriptor of the passing socket for the WebContent connection", "webcontent-fd-passing-socket", 'c', "webcontent_fd_passing_socket");
     args_parser.add_option(is_layout_test_mode, "Is layout test mode", "layout-test-mode", 0);
     args_parser.add_option(use_javascript_bytecode, "Enable JavaScript bytecode VM", "use-bytecode", 0);
+    args_parser.add_option(use_lagom_networking, "Enable Lagom servers for networking", "use-lagom-networking", 0);
     args_parser.parse(arguments);
 
+    if (use_lagom_networking) {
+        auto candidate_request_server_paths = TRY(get_paths_for_helper_process("RequestServer"sv));
+        auto protocol_client = TRY(launch_request_server_process(candidate_request_server_paths));
+        Web::ResourceLoader::initialize(TRY(WebView::RequestServerAdapter::try_create(move(protocol_client))));
+    } else {
+        Web::ResourceLoader::initialize(RequestManagerQt::create());
+    }
+
     JS::Bytecode::Interpreter::set_enabled(use_javascript_bytecode);
 
     VERIFY(webcontent_fd_passing_socket >= 0);

Ladybird/WebContentView.cpp

@@ -51,8 +51,9 @@
 
 bool is_using_dark_system_theme(QWidget&);
 
-WebContentView::WebContentView(StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling enable_callgrind_profiling, WebView::UseJavaScriptBytecode use_javascript_bytecode)
+WebContentView::WebContentView(StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling enable_callgrind_profiling, WebView::UseJavaScriptBytecode use_javascript_bytecode, UseLagomNetworking use_lagom_networking)
     : WebView::ViewImplementation(use_javascript_bytecode)
+    , m_use_lagom_networking(use_lagom_networking)
     , m_webdriver_content_ipc_path(webdriver_content_ipc_path)
 {
     setMouseTracking(true);
@@ -531,7 +532,7 @@ void WebContentView::create_client(WebView::EnableCallgrindProfiling enable_call
     m_client_state = {};
 
     auto candidate_web_content_paths = get_paths_for_helper_process("WebContent"sv).release_value_but_fixme_should_propagate_errors();
-    auto new_client = launch_web_content_process(*this, candidate_web_content_paths, enable_callgrind_profiling, WebView::IsLayoutTestMode::No, use_javascript_bytecode()).release_value_but_fixme_should_propagate_errors();
+    auto new_client = launch_web_content_process(*this, candidate_web_content_paths, enable_callgrind_profiling, WebView::IsLayoutTestMode::No, use_javascript_bytecode(), m_use_lagom_networking).release_value_but_fixme_should_propagate_errors();
 
     m_client_state.client = new_client;
     m_client_state.client->on_web_content_process_crash = [this] {

Ladybird/WebContentView.h

@@ -7,6 +7,7 @@
 
 #pragma once
 
+#include "Types.h"
 #include <AK/DeprecatedString.h>
 #include <AK/Function.h>
 #include <AK/HashMap.h>
@@ -40,7 +41,7 @@ class WebContentView final
     , public WebView::ViewImplementation {
     Q_OBJECT
 public:
-    explicit WebContentView(StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling, WebView::UseJavaScriptBytecode);
+    explicit WebContentView(StringView webdriver_content_ipc_path, WebView::EnableCallgrindProfiling, WebView::UseJavaScriptBytecode, UseLagomNetworking);
     virtual ~WebContentView() override;
 
     Function<String(const AK::URL&, Web::HTML::ActivateTab)> on_tab_open_request;
@@ -108,6 +109,7 @@ private:
 
     qreal m_inverse_pixel_scaling_ratio { 1.0 };
     bool m_should_show_line_box_borders { false };
+    UseLagomNetworking m_use_lagom_networking {};
 
     QPointer<QDialog> m_dialog;
 

Ladybird/cmake/InstallRules.cmake

@@ -4,7 +4,7 @@ include(GNUInstallDirs)
 
 set(package ladybird)
 
-set(ladybird_applications ladybird SQLServer WebContent WebDriver headless-browser)
+set(ladybird_applications ladybird SQLServer WebContent WebDriver RequestServer headless-browser)
 
 set(app_install_targets ${ladybird_applications})
 if (ANDROID)
@@ -89,6 +89,7 @@ install(DIRECTORY
 install(FILES
     "${SERENITY_SOURCE_DIR}/Base/home/anon/.config/BrowserAutoplayAllowlist.txt"
     "${SERENITY_SOURCE_DIR}/Base/home/anon/.config/BrowserContentFilters.txt"
+    "${Lagom_BINARY_DIR}/cacert.pem"
   DESTINATION "${CMAKE_INSTALL_DATADIR}/res/ladybird"
   COMPONENT ladybird_Runtime
 )

Ladybird/main.cpp

@@ -71,6 +71,7 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
     bool enable_callgrind_profiling = false;
     bool enable_sql_database = false;
     bool use_ast_interpreter = false;
+    bool use_lagom_networking = false;
 
     Core::ArgsParser args_parser;
     args_parser.set_general_help("The Ladybird web browser :^)");
@@ -79,6 +80,7 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
     args_parser.add_option(enable_callgrind_profiling, "Enable Callgrind profiling", "enable-callgrind-profiling", 'P');
     args_parser.add_option(enable_sql_database, "Enable SQL database", "enable-sql-database", 0);
     args_parser.add_option(use_ast_interpreter, "Enable JavaScript AST interpreter (deprecated)", "ast", 0);
+    args_parser.add_option(use_lagom_networking, "Enable Lagom servers for networking", "enable-lagom-networking", 0);
     args_parser.parse(arguments);
 
     auto get_formatted_url = [&](StringView const& raw_url) -> ErrorOr<URL> {
@@ -101,7 +103,7 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
     auto cookie_jar = database ? TRY(Browser::CookieJar::create(*database)) : Browser::CookieJar::create();
 
     s_settings = adopt_own_if_nonnull(new Browser::Settings());
-    BrowserWindow window(cookie_jar, webdriver_content_ipc_path, enable_callgrind_profiling ? WebView::EnableCallgrindProfiling::Yes : WebView::EnableCallgrindProfiling::No, use_ast_interpreter ? WebView::UseJavaScriptBytecode::No : WebView::UseJavaScriptBytecode::Yes);
+    BrowserWindow window(cookie_jar, webdriver_content_ipc_path, enable_callgrind_profiling ? WebView::EnableCallgrindProfiling::Yes : WebView::EnableCallgrindProfiling::No, use_ast_interpreter ? WebView::UseJavaScriptBytecode::No : WebView::UseJavaScriptBytecode::Yes, use_lagom_networking ? UseLagomNetworking::Yes : UseLagomNetworking::No);
     window.setWindowTitle("Ladybird");
     window.resize(800, 600);
     window.show();

Meta/Lagom/CMakeLists.txt

@@ -407,6 +407,7 @@ if (BUILD_LAGOM)
         Manual
         Markdown
         PDF
+        Protocol
         Regex
         SoftGPU
         SQL
@@ -430,21 +431,26 @@ if (BUILD_LAGOM)
         list(APPEND LIBWEBVIEW_SOURCES "../../Userland/Libraries/LibWebView/StylePropertiesModel.cpp")
         list(APPEND LIBWEBVIEW_SOURCES "../../Userland/Libraries/LibWebView/ViewImplementation.cpp")
         list(APPEND LIBWEBVIEW_SOURCES "../../Userland/Libraries/LibWebView/WebContentClient.cpp")
+        list(APPEND LIBWEBVIEW_SOURCES "../../Userland/Libraries/LibWebView/RequestServerAdapter.cpp")
 
         compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/WebContent/WebContentServer.ipc WebContent/WebContentServerEndpoint.h)
         compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/WebContent/WebContentClient.ipc WebContent/WebContentClientEndpoint.h)
         compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/WebContent/WebDriverClient.ipc WebContent/WebDriverClientEndpoint.h)
         compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/WebContent/WebDriverServer.ipc WebContent/WebDriverServerEndpoint.h)
+        compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/RequestServer/RequestClient.ipc RequestServer/RequestClientEndpoint.h)
+        compile_ipc(${SERENITY_PROJECT_ROOT}/Userland/Services/RequestServer/RequestServer.ipc RequestServer/RequestServerEndpoint.h)
 
         list(APPEND LIBWEBVIEW_GENERATED_SOURCES WebContent/WebContentClientEndpoint.h)
         list(APPEND LIBWEBVIEW_GENERATED_SOURCES WebContent/WebContentServerEndpoint.h)
         list(APPEND LIBWEBVIEW_GENERATED_SOURCES WebContent/WebDriverClientEndpoint.h)
         list(APPEND LIBWEBVIEW_GENERATED_SOURCES WebContent/WebDriverServerEndpoint.h)
+        list(APPEND LIBWEBVIEW_GENERATED_SOURCES RequestServer/RequestClientEndpoint.h)
+        list(APPEND LIBWEBVIEW_GENERATED_SOURCES RequestServer/RequestServerEndpoint.h)
 
         set(GENERATED_SOURCES ${LIBWEBVIEW_GENERATED_SOURCES})
         lagom_lib(LibWebView webview
             SOURCES ${LIBWEBVIEW_SOURCES} ${LIBWEBVIEW_GENERATED_SOURCES}
-            LIBS LibGfx LibGUI LibIPC LibWeb)
+            LIBS LibGfx LibGUI LibIPC LibWeb LibProtocol)
         unset(GENERATED_SOURCES)
     endif()
 

Meta/gn/secondary/Ladybird/BUILD.gn

@@ -40,7 +40,7 @@ link_qt("ladybird_qt_components") {
 config("ladybird_config") {
   include_dirs = [
     "//Userland/Applications",
-    "//Userland",
+    "//Userland/Services",
   ]
   defines = [ "AK_DONT_REPLACE_STD" ]
 }
@@ -52,6 +52,7 @@ executable("ladybird_executable") {
   ]
   data_deps = [
     ":headless-browser",
+    "RequestServer",
     "SQLServer",
     "WebContent",
     "WebDriver",
@@ -67,6 +68,7 @@ executable("ladybird_executable") {
     "//Userland/Libraries/LibIPC",
     "//Userland/Libraries/LibJS",
     "//Userland/Libraries/LibMain",
+    "//Userland/Libraries/LibProtocol",
     "//Userland/Libraries/LibSQL",
     "//Userland/Libraries/LibWeb",
     "//Userland/Libraries/LibWebView",
@@ -119,6 +121,7 @@ executable("headless-browser") {
     "//Userland/Libraries/LibIPC",
     "//Userland/Libraries/LibJS",
     "//Userland/Libraries/LibMain",
+    "//Userland/Libraries/LibProtocol",
     "//Userland/Libraries/LibTLS",
     "//Userland/Libraries/LibWeb",
     "//Userland/Libraries/LibWebSocket",
@@ -142,11 +145,13 @@ if (current_os == "mac") {
     public_deps = [
       ":headless-browser",
       ":ladybird_executable",
+      "RequestServer",
       "SQLServer",
       "WebContent",
       "WebDriver",
     ]
     sources = [
+      "$root_out_dir/bin/RequestServer",
       "$root_out_dir/bin/SQLServer",
       "$root_out_dir/bin/WebContent",
       "$root_out_dir/bin/WebDriver",
@@ -176,6 +181,7 @@ if (current_os == "mac") {
       "//Userland/Libraries/LibJS",
       "//Userland/Libraries/LibLine",
       "//Userland/Libraries/LibMarkdown",
+      "//Userland/Libraries/LibProtocol",
       "//Userland/Libraries/LibRegex",
       "//Userland/Libraries/LibSQL",
       "//Userland/Libraries/LibSoftGPU",
@@ -209,6 +215,7 @@ if (current_os == "mac") {
       "$root_out_dir/lib/liblagom-js.dylib",
       "$root_out_dir/lib/liblagom-line.dylib",
       "$root_out_dir/lib/liblagom-markdown.dylib",
+      "$root_out_dir/lib/liblagom-protocol.dylib",
       "$root_out_dir/lib/liblagom-regex.dylib",
       "$root_out_dir/lib/liblagom-softgpu.dylib",
       "$root_out_dir/lib/liblagom-sql.dylib",
@@ -239,7 +246,9 @@ if (current_os == "mac") {
   }
 
   bundle_data("ladybird_config_resources") {
+    public_deps = [ "//Userland/Libraries/LibTLS:ca_certificates_download" ]
     sources = [
+      "$root_build_dir/cacert.pem",
       "//Base/home/anon/.config/BrowserAutoplayAllowlist.txt",
       "//Base/home/anon/.config/BrowserContentFilters.txt",
     ]

Meta/gn/secondary/Ladybird/RequestServer/BUILD.gn

@@ -0,0 +1,42 @@
+import("//Ladybird/link_qt.gni")
+
+link_qt("RequestServer_qt") {
+  qt_components = [ "Core" ]
+}
+
+executable("RequestServer") {
+  configs += [
+    "//Ladybird:ladybird_config",
+    ":RequestServer_qt",
+  ]
+  include_dirs = [
+    "//Userland/Libraries",
+    "//Userland/Services",
+  ]
+  deps = [
+    "//AK",
+    "//Userland/Libraries/LibCore",
+    "//Userland/Libraries/LibCrypto",
+    "//Userland/Libraries/LibFileSystem",
+    "//Userland/Libraries/LibGemini",
+    "//Userland/Libraries/LibHTTP",
+    "//Userland/Libraries/LibIPC",
+    "//Userland/Libraries/LibMain",
+    "//Userland/Libraries/LibProtocol",
+    "//Userland/Libraries/LibTLS",
+  ]
+  sources = [
+    "../Utilities.cpp",
+    "//Userland/Services/RequestServer/ConnectionCache.cpp",
+    "//Userland/Services/RequestServer/ConnectionFromClient.cpp",
+    "//Userland/Services/RequestServer/GeminiProtocol.cpp",
+    "//Userland/Services/RequestServer/GeminiRequest.cpp",
+    "//Userland/Services/RequestServer/HttpProtocol.cpp",
+    "//Userland/Services/RequestServer/HttpRequest.cpp",
+    "//Userland/Services/RequestServer/HttpsProtocol.cpp",
+    "//Userland/Services/RequestServer/HttpsRequest.cpp",
+    "//Userland/Services/RequestServer/Protocol.cpp",
+    "//Userland/Services/RequestServer/Request.cpp",
+    "main.cpp",
+  ]
+}

Meta/gn/secondary/Ladybird/WebContent/BUILD.gn

@@ -12,7 +12,6 @@ moc_qt_objects("generate_moc") {
 link_qt("WebContent_qt") {
   qt_components = [
     "Core",
-    "Gui",
     "Network",
     "Multimedia",
   ]
@@ -36,8 +35,10 @@ executable("WebContent") {
     "//Userland/Libraries/LibIPC",
     "//Userland/Libraries/LibJS",
     "//Userland/Libraries/LibMain",
+    "//Userland/Libraries/LibProtocol",
     "//Userland/Libraries/LibWeb",
     "//Userland/Libraries/LibWebSocket",
+    "//Userland/Libraries/LibWebView",
     "//Userland/Libraries/LibWebView:WebContentClientEndpoint",
     "//Userland/Libraries/LibWebView:WebContentServerEndpoint",
     "//Userland/Libraries/LibWebView:WebDriverClientEndpoint",
@@ -48,6 +49,7 @@ executable("WebContent") {
     "../AudioThread.cpp",
     "../EventLoopImplementationQt.cpp",
     "../FontPluginLadybird.cpp",
+    "../HelperProcess.cpp",
     "../ImageCodecPluginLadybird.cpp",
     "../RequestManagerQt.cpp",
     "../Utilities.cpp",

Meta/gn/secondary/Userland/Libraries/LibProtocol/BUILD.gn

@@ -0,0 +1,46 @@
+import("//Meta/gn/build/compiled_action.gni")
+
+compiled_action("RequestClientEndpoint") {
+  tool = "//Meta/Lagom/Tools/CodeGenerators/IPCCompiler"
+  inputs = [ "//Userland/Services/RequestServer/RequestClient.ipc" ]
+  outputs = [ "$root_gen_dir/RequestServer/RequestClientEndpoint.h" ]
+  args = [
+    rebase_path(inputs[0], root_build_dir),
+    "-o",
+    rebase_path(outputs[0], root_build_dir),
+  ]
+}
+
+compiled_action("RequestServerEndpoint") {
+  tool = "//Meta/Lagom/Tools/CodeGenerators/IPCCompiler"
+  inputs = [ "//Userland/Services/RequestServer/RequestServer.ipc" ]
+  outputs = [ "$root_gen_dir/RequestServer/RequestServerEndpoint.h" ]
+  args = [
+    rebase_path(inputs[0], root_build_dir),
+    "-o",
+    rebase_path(outputs[0], root_build_dir),
+  ]
+}
+
+shared_library("LibProtocol") {
+  output_name = "protocol"
+  include_dirs = [
+    "//Userland/Libraries",
+    "//Userland/Services",
+  ]
+  deps = [
+    ":RequestClientEndpoint",
+    ":RequestServerEndpoint",
+    "//AK",
+    "//Userland/Libraries/LibCore",
+    "//Userland/Libraries/LibIPC",
+  ]
+  sources = [
+    "Request.cpp",
+    "RequestClient.cpp",
+
+    # TODO: Add WebSocket sources + IPC
+  ]
+  sources += get_target_outputs(":RequestClientEndpoint") +
+             get_target_outputs(":RequestServerEndpoint")
+}

Meta/gn/secondary/Userland/Libraries/LibWebView/BUILD.gn

@@ -48,7 +48,8 @@ shared_library("LibWebView") {
   output_name = "webview"
   include_dirs = [
     "//Userland/Libraries",
-    "//Userland",
+    "//Userland/Services",
+    "//Userland/",
   ]
   deps = [
     ":WebContentClientEndpoint",
@@ -60,11 +61,13 @@ shared_library("LibWebView") {
     "//Userland/Libraries/LibGUI",
     "//Userland/Libraries/LibGfx",
     "//Userland/Libraries/LibIPC",
+    "//Userland/Libraries/LibProtocol",
     "//Userland/Libraries/LibWeb",
   ]
   sources = [
     "AccessibilityTreeModel.cpp",
     "DOMTreeModel.cpp",
+    "RequestServerAdapter.cpp",
     "StylePropertiesModel.cpp",
     "ViewImplementation.cpp",
     "WebContentClient.cpp",

Userland/Libraries/LibProtocol/CMakeLists.txt

@@ -1,16 +1,23 @@
 set(SOURCES
     Request.cpp
     RequestClient.cpp
-    WebSocket.cpp
-    WebSocketClient.cpp
 )
 
 set(GENERATED_SOURCES
     ../../Services/RequestServer/RequestClientEndpoint.h
     ../../Services/RequestServer/RequestServerEndpoint.h
-    ../../Services/WebSocket/WebSocketClientEndpoint.h
-    ../../Services/WebSocket/WebSocketServerEndpoint.h
 )
 
+if (SERENITYOS)
+    list(APPEND SOURCES
+        WebSocket.cpp
+        WebSocketClient.cpp
+    )
+    list(APPEND GENERATED_SOURCES 
+        ../../Services/WebSocket/WebSocketClientEndpoint.h
+        ../../Services/WebSocket/WebSocketServerEndpoint.h
+    )
+endif()
+
 serenity_lib(LibProtocol protocol)
 target_link_libraries(LibProtocol PRIVATE LibCore LibIPC)

Userland/Services/CMakeLists.txt

@@ -3,7 +3,6 @@ add_subdirectory(EchoServer)
 add_subdirectory(FileOperation)
 add_subdirectory(ImageDecoder)
 add_subdirectory(LookupServer)
-add_subdirectory(RequestServer)
 add_subdirectory(WebServer)
 add_subdirectory(WebSocket)
 
@@ -19,6 +18,7 @@ if (SERENITYOS)
     add_subdirectory(LoginServer)
     add_subdirectory(NetworkServer)
     add_subdirectory(NotificationServer)
+    add_subdirectory(RequestServer)
     add_subdirectory(SpiceAgent)
     add_subdirectory(SQLServer)
     add_subdirectory(SystemServer)

Userland/Utilities/headless-browser.cpp

@@ -58,7 +58,7 @@ public:
         (void)use_javascript_bytecode;
 #else
         auto candidate_web_content_paths = TRY(get_paths_for_helper_process("WebContent"sv));
-        view->m_client_state.client = TRY(launch_web_content_process(*view, candidate_web_content_paths, WebView::EnableCallgrindProfiling::No, is_layout_test_mode, use_javascript_bytecode));
+        view->m_client_state.client = TRY(launch_web_content_process(*view, candidate_web_content_paths, WebView::EnableCallgrindProfiling::No, is_layout_test_mode, use_javascript_bytecode, UseLagomNetworking::No));
 #endif
 
         view->client().async_update_system_theme(move(theme));