RequestServer: Remove now-unused HTTP related code

Userland/Libraries/LibWebView/CMakeLists.txt

@@ -67,6 +67,4 @@ if (ENABLE_INSTALL_HEADERS)
 
         install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${header}" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${subdirectory}")
     endforeach()
-
-    install(FILES "${SERENITY_PROJECT_ROOT}/Userland/Services/RequestServer/ConnectionCache.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/RequestServer")
 endif()

Userland/Services/RequestServer/CMakeLists.txt

@@ -3,14 +3,8 @@ compile_ipc(RequestClient.ipc RequestClientEndpoint.h)
 
 set(SOURCES
     ConnectionFromClient.cpp
-    ConnectionCache.cpp
     Request.cpp
-    HttpRequest.cpp
-    HttpProtocol.cpp
-    HttpsRequest.cpp
-    HttpsProtocol.cpp
     main.cpp
-    Protocol.cpp
 )
 
 set(GENERATED_SOURCES
@@ -19,4 +13,4 @@ set(GENERATED_SOURCES
 )
 
 serenity_bin(RequestServer)
-target_link_libraries(RequestServer PRIVATE LibCore LibCrypto LibIPC LibHTTP LibMain LibTLS LibWebSocket LibURL LibThreading)
+target_link_libraries(RequestServer PRIVATE LibCore LibCrypto LibIPC LibMain LibTLS LibWebSocket LibURL LibThreading)

Userland/Services/RequestServer/CacheLevel.h

@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2024, Andreas Kling <andreas@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+namespace RequestServer {
+
+enum class CacheLevel {
+    ResolveOnly,
+    CreateConnection,
+};
+
+}

Userland/Services/RequestServer/ConnectionCache.cpp

@@ -1,167 +0,0 @@
-/*
- * Copyright (c) 2021-2022, Ali Mohammad Pur <mpfard@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include "ConnectionCache.h"
-#include <AK/Debug.h>
-#include <AK/Find.h>
-#include <LibCore/EventLoop.h>
-
-namespace RequestServer::ConnectionCache {
-
-Threading::RWLockProtected<HashMap<ConnectionKey, NonnullOwnPtr<Vector<NonnullOwnPtr<Connection<Core::TCPSocket, Core::Socket>>>>>> g_tcp_connection_cache {};
-Threading::RWLockProtected<HashMap<ConnectionKey, NonnullOwnPtr<Vector<NonnullOwnPtr<Connection<TLS::TLSv12>>>>>> g_tls_connection_cache {};
-Threading::RWLockProtected<HashMap<ByteString, InferredServerProperties>> g_inferred_server_properties;
-
-void request_did_finish(URL::URL const& url, Core::Socket const* socket)
-{
-    if (!socket) {
-        dbgln("Request with a null socket finished for URL {}", url);
-        return;
-    }
-
-    dbgln_if(REQUESTSERVER_DEBUG, "Request for {} finished", url);
-
-    ConnectionKey partial_key { url.serialized_host().release_value_but_fixme_should_propagate_errors().to_byte_string(), url.port_or_default() };
-    auto fire_off_next_job = [&](auto& cache) {
-        using CacheType = typename RemoveCVReference<decltype(cache)>::ProtectedType;
-        auto [it, end] = cache.with_read_locked([&](auto const& cache) {
-            struct Result {
-                decltype(cache.begin()) it;
-                decltype(cache.end()) end;
-            };
-            return Result {
-                find_if(cache.begin(), cache.end(), [&](auto& connection) {
-                    return connection.key.hostname == partial_key.hostname && connection.key.port == partial_key.port;
-                }),
-                cache.end(),
-            };
-        });
-        if (it == end) {
-            dbgln("Request for URL {} finished, but we don't own that!", url);
-            return;
-        }
-        auto connection_it = it->value->find_if([&](auto& connection) { return connection->socket == socket; });
-        if (connection_it.is_end()) {
-            dbgln("Request for URL {} finished, but we don't have a socket for that!", url);
-            return;
-        }
-
-        auto& connection = *connection_it;
-        if constexpr (REQUESTSERVER_DEBUG) {
-            connection->job_data->timing_info.performing_request = AK::Duration::from_milliseconds(connection->job_data->timing_info.timer.elapsed_milliseconds());
-            connection->job_data->timing_info.timer.start();
-        }
-
-        auto& properties = g_inferred_server_properties.with_write_locked([&](auto& map) -> InferredServerProperties& { return map.ensure(partial_key.hostname); });
-        if (!connection->socket->is_open())
-            properties.requests_served_per_connection = min(properties.requests_served_per_connection, connection->max_queue_length + 1);
-
-        if (connection->request_queue.with_read_locked([](auto const& queue) { return queue.is_empty(); })) {
-            // Immediately mark the connection as finished, as new jobs will never be run if they are queued
-            // before the deferred_invoke() below runs otherwise.
-            connection->has_started = false;
-            connection->socket->set_notifications_enabled(false);
-
-            Core::deferred_invoke([&connection, &cache_entry = *it->value, key = it->key, &cache] {
-                if (connection->has_started)
-                    return;
-
-                connection->current_url = {};
-                connection->job_data = {};
-                connection->removal_timer->on_timeout = [ptr = connection.ptr(), &cache_entry, key = move(key), &cache]() mutable {
-                    Core::deferred_invoke([&, key = move(key), ptr] {
-                        if (ptr->has_started)
-                            return;
-
-                        dbgln_if(REQUESTSERVER_DEBUG, "Removing no-longer-used connection {} (socket {})", ptr, ptr->socket.ptr());
-                        cache.with_write_locked([&](CacheType& cache) {
-                            auto did_remove = cache_entry.remove_first_matching([&](auto& entry) { return entry == ptr; });
-                            VERIFY(did_remove);
-                            if (cache_entry.is_empty())
-                                cache.remove(key);
-                        });
-                    });
-                };
-                connection->removal_timer->start();
-            });
-        } else {
-            auto timer = Core::ElapsedTimer::start_new();
-            if (auto result = recreate_socket_if_needed(*connection, url); result.is_error()) {
-                if constexpr (REQUESTSERVER_DEBUG) {
-                    connection->job_data->timing_info.starting_connection += AK::Duration::from_milliseconds(timer.elapsed_milliseconds());
-                }
-                cache.with_read_locked([&](auto&) {
-                    dbgln("ConnectionCache request finish handler, reconnection failed with {}", result.error());
-                    connection->job_data->fail(Core::NetworkJob::Error::ConnectionFailed);
-                });
-                return;
-            }
-            if constexpr (REQUESTSERVER_DEBUG) {
-                connection->job_data->timing_info.starting_connection += AK::Duration::from_milliseconds(timer.elapsed_milliseconds());
-            }
-
-            connection->has_started = true;
-            Core::deferred_invoke([&connection = *connection, url, &cache] {
-                cache.with_read_locked([&](auto&) {
-                    dbgln_if(REQUESTSERVER_DEBUG, "Running next job in queue for connection {}", &connection);
-                    connection.timer.start();
-                    connection.current_url = url;
-                    connection.job_data = connection.request_queue.with_write_locked([](auto& queue) { return queue.take_first(); });
-                    if constexpr (REQUESTSERVER_DEBUG) {
-                        connection.job_data->timing_info.waiting_in_queue = AK::Duration::from_milliseconds(connection.job_data->timing_info.timer.elapsed_milliseconds() - connection.job_data->timing_info.performing_request.to_milliseconds());
-                        connection.job_data->timing_info.timer.start();
-                    }
-                    connection.socket->set_notifications_enabled(true);
-                    connection.job_data->start(*connection.socket);
-                });
-            });
-        }
-    };
-
-    if (is<Core::BufferedSocket<TLS::TLSv12>>(socket))
-        fire_off_next_job(g_tls_connection_cache);
-    else if (is<Core::BufferedSocket<Core::Socket>>(socket))
-        fire_off_next_job(g_tcp_connection_cache);
-    else
-        dbgln("Unknown socket {} finished for URL {}", socket, url);
-}
-
-void dump_jobs()
-{
-    g_tls_connection_cache.with_read_locked([](auto& cache) {
-        dbgln("=========== TLS Connection Cache ==========");
-        for (auto& connection : cache) {
-            dbgln(" - {}:{}", connection.key.hostname, connection.key.port);
-            for (auto& entry : *connection.value) {
-                dbgln("  - Connection {} (started={}) (socket={})", &entry, entry->has_started, entry->socket.ptr());
-                dbgln("    Currently loading {} ({} elapsed)", entry->current_url, entry->timer.is_valid() ? entry->timer.elapsed() : 0);
-                dbgln("    Request Queue:");
-                entry->request_queue.for_each_locked([](auto const& job) {
-                    dbgln("    - {}", &job);
-                });
-            }
-        }
-    });
-
-    g_tcp_connection_cache.with_read_locked([](auto& cache) {
-        dbgln("=========== TCP Connection Cache ==========");
-        for (auto& connection : cache) {
-            dbgln(" - {}:{}", connection.key.hostname, connection.key.port);
-            for (auto& entry : *connection.value) {
-                dbgln("  - Connection {} (started={}) (socket={})", &entry, entry->has_started, entry->socket.ptr());
-                dbgln("    Currently loading {} ({} elapsed)", entry->current_url, entry->timer.is_valid() ? entry->timer.elapsed() : 0);
-                dbgln("    Request Queue:");
-                entry->request_queue.for_each_locked([](auto const& job) {
-                    dbgln("    - {}", &job);
-                });
-            }
-        }
-    });
-}
-
-size_t hits;
-size_t misses;
-}

Userland/Services/RequestServer/ConnectionCache.h

@@ -1,386 +0,0 @@
-/*
- * Copyright (c) 2021-2022, Ali Mohammad Pur <mpfard@serenityos.org>
- * Copyright (c) 2022, the SerenityOS developers.
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#pragma once
-
-#include <AK/Debug.h>
-#include <AK/HashMap.h>
-#include <AK/Vector.h>
-#include <LibCore/ElapsedTimer.h>
-#include <LibCore/EventLoop.h>
-#include <LibCore/NetworkJob.h>
-#include <LibCore/SOCKSProxyClient.h>
-#include <LibCore/Timer.h>
-#include <LibTLS/TLSv12.h>
-#include <LibThreading/RWLockProtected.h>
-#include <LibURL/URL.h>
-
-namespace RequestServer {
-
-enum class CacheLevel {
-    ResolveOnly,
-    CreateConnection,
-};
-
-}
-
-namespace RequestServer::ConnectionCache {
-
-struct Proxy {
-    Core::ProxyData data;
-    OwnPtr<Core::SOCKSProxyClient> proxy_client_storage {};
-
-    template<typename SocketType, typename StorageType, typename... Args>
-    ErrorOr<NonnullOwnPtr<StorageType>> tunnel(URL::URL const& url, Args&&... args)
-    {
-        if (data.type == Core::ProxyData::Direct) {
-            return TRY(SocketType::connect(TRY(url.serialized_host()).to_byte_string(), url.port_or_default(), forward<Args>(args)...));
-        }
-        if (data.type == Core::ProxyData::SOCKS5) {
-            if constexpr (requires { SocketType::connect(declval<ByteString>(), *proxy_client_storage, forward<Args>(args)...); }) {
-                proxy_client_storage = TRY(Core::SOCKSProxyClient::connect(data.host_ipv4, data.port, Core::SOCKSProxyClient::Version::V5, TRY(url.serialized_host()).to_byte_string(), url.port_or_default()));
-                return TRY(SocketType::connect(TRY(url.serialized_host()).to_byte_string(), *proxy_client_storage, forward<Args>(args)...));
-            } else if constexpr (IsSame<SocketType, Core::TCPSocket>) {
-                return TRY(Core::SOCKSProxyClient::connect(data.host_ipv4, data.port, Core::SOCKSProxyClient::Version::V5, TRY(url.serialized_host()).to_byte_string(), url.port_or_default()));
-            } else {
-                return Error::from_string_literal("SOCKS5 not supported for this socket type");
-            }
-        }
-        VERIFY_NOT_REACHED();
-    }
-};
-
-struct JobData {
-    Function<void(Core::BufferedSocketBase&)> start {};
-    Function<void(Core::NetworkJob::Error)> fail {};
-    Function<Vector<TLS::Certificate>()> provide_client_certificates {};
-    struct TimingInfo {
-#if REQUESTSERVER_DEBUG
-        bool valid { true };
-        Core::ElapsedTimer timer {};
-        URL::URL url {};
-        AK::Duration waiting_in_queue {};
-        AK::Duration starting_connection {};
-        AK::Duration performing_request {};
-#endif
-    } timing_info {};
-
-    JobData(Function<void(Core::BufferedSocketBase&)> start, Function<void(Core::NetworkJob::Error)> fail, Function<Vector<TLS::Certificate>()> provide_client_certificates, TimingInfo timing_info)
-        : start(move(start))
-        , fail(move(fail))
-        , provide_client_certificates(move(provide_client_certificates))
-        , timing_info(move(timing_info))
-    {
-    }
-
-    JobData(JobData&& other)
-        : start(move(other.start))
-        , fail(move(other.fail))
-        , provide_client_certificates(move(other.provide_client_certificates))
-        , timing_info(move(other.timing_info))
-    {
-#if REQUESTSERVER_DEBUG
-        other.timing_info.valid = false;
-#endif
-    }
-
-#if REQUESTSERVER_DEBUG
-    ~JobData()
-    {
-        if (timing_info.valid) {
-            dbgln("JobData for {} timings:", timing_info.url);
-            dbgln("  - Waiting in queue: {}ms", timing_info.waiting_in_queue.to_milliseconds());
-            dbgln("  - Starting connection: {}ms", timing_info.starting_connection.to_milliseconds());
-            dbgln("  - Performing request: {}ms", timing_info.performing_request.to_milliseconds());
-        }
-    }
-#endif
-
-    static JobData create(auto job, [[maybe_unused]] URL::URL url)
-    {
-        return JobData {
-            [job](auto& socket) { job->start(socket); },
-            [job](auto error) { job->fail(error); },
-            [job] {
-                if constexpr (requires { job->on_certificate_requested; }) {
-                    if (job->on_certificate_requested)
-                        return job->on_certificate_requested();
-                } else {
-                    // "use" `job`, otherwise clang gets sad.
-                    (void)job;
-                }
-                return Vector<TLS::Certificate> {};
-            },
-            {
-#if REQUESTSERVER_DEBUG
-                .timer = Core::ElapsedTimer::start_new(Core::TimerType::Precise),
-                .url = move(url),
-                .waiting_in_queue = {},
-                .starting_connection = {},
-                .performing_request = {},
-#endif
-            },
-        };
-    }
-};
-
-template<typename Socket, typename SocketStorageType = Socket>
-struct Connection {
-    using QueueType = Vector<JobData>;
-    using SocketType = Socket;
-    using StorageType = SocketStorageType;
-
-    OwnPtr<Core::BufferedSocket<SocketStorageType>> socket;
-    Threading::RWLockProtected<QueueType> request_queue;
-    NonnullRefPtr<Core::Timer> removal_timer;
-    Atomic<bool> is_being_started { false };
-    bool has_started { false };
-    URL::URL current_url {};
-    Core::ElapsedTimer timer {};
-    Optional<JobData> job_data {};
-    Proxy proxy {};
-    size_t max_queue_length { 0 };
-};
-
-struct ConnectionKey {
-    ByteString hostname;
-    u16 port { 0 };
-    Core::ProxyData proxy_data {};
-
-    bool operator==(ConnectionKey const&) const = default;
-};
-
-};
-
-template<>
-struct AK::Traits<RequestServer::ConnectionCache::ConnectionKey> : public AK::DefaultTraits<RequestServer::ConnectionCache::ConnectionKey> {
-    static u32 hash(RequestServer::ConnectionCache::ConnectionKey const& key)
-    {
-        return pair_int_hash(pair_int_hash(key.proxy_data.host_ipv4, key.proxy_data.port), pair_int_hash(key.hostname.hash(), key.port));
-    }
-};
-
-namespace RequestServer::ConnectionCache {
-
-struct InferredServerProperties {
-    size_t requests_served_per_connection { NumericLimits<size_t>::max() };
-};
-
-extern Threading::RWLockProtected<HashMap<ConnectionKey, NonnullOwnPtr<Vector<NonnullOwnPtr<Connection<Core::TCPSocket, Core::Socket>>>>>> g_tcp_connection_cache;
-extern Threading::RWLockProtected<HashMap<ConnectionKey, NonnullOwnPtr<Vector<NonnullOwnPtr<Connection<TLS::TLSv12>>>>>> g_tls_connection_cache;
-extern Threading::RWLockProtected<HashMap<ByteString, InferredServerProperties>> g_inferred_server_properties;
-
-void request_did_finish(URL::URL const&, Core::Socket const*);
-void dump_jobs();
-
-constexpr static size_t MaxConcurrentConnectionsPerURL = 4;
-constexpr static size_t ConnectionKeepAliveTimeMilliseconds = 20'000;
-constexpr static size_t ConnectionCacheQueueHighWatermark = 4;
-
-template<typename T>
-ErrorOr<void> recreate_socket_if_needed(T& connection, URL::URL const& url)
-{
-    using SocketType = typename T::SocketType;
-    using SocketStorageType = typename T::StorageType;
-
-    if (!connection.socket || !connection.socket->is_open() || connection.socket->is_eof()) {
-        connection.socket = nullptr;
-        // Create another socket for the connection.
-        auto set_socket = [&](NonnullOwnPtr<SocketStorageType>&& socket) -> ErrorOr<void> {
-            connection.socket = TRY(Core::BufferedSocket<SocketStorageType>::create(move(socket)));
-            return {};
-        };
-
-        if constexpr (IsSame<TLS::TLSv12, SocketType>) {
-            TLS::Options options;
-            options.set_alert_handler([&connection](TLS::AlertDescription alert) {
-                Core::NetworkJob::Error reason;
-                if (alert == TLS::AlertDescription::HANDSHAKE_FAILURE)
-                    reason = Core::NetworkJob::Error::ProtocolFailed;
-                else if (alert == TLS::AlertDescription::DECRYPT_ERROR)
-                    reason = Core::NetworkJob::Error::ConnectionFailed;
-                else
-                    reason = Core::NetworkJob::Error::TransmissionFailed;
-
-                if (connection.job_data->fail)
-                    connection.job_data->fail(reason);
-            });
-            options.set_certificate_provider([&connection]() -> Vector<TLS::Certificate> {
-                if (connection.job_data->provide_client_certificates)
-                    return connection.job_data->provide_client_certificates();
-                return {};
-            });
-            TRY(set_socket(TRY((connection.proxy.template tunnel<SocketType, SocketStorageType>(url, move(options))))));
-        } else {
-            TRY(set_socket(TRY((connection.proxy.template tunnel<SocketType, SocketStorageType>(url)))));
-        }
-        dbgln_if(REQUESTSERVER_DEBUG, "Creating a new socket for {} -> {}", url, connection.socket.ptr());
-    }
-    return {};
-}
-
-extern size_t hits;
-extern size_t misses;
-
-template<typename Cache>
-void start_connection(const URL::URL& url, auto job, auto& sockets_for_url, size_t index, AK::Duration, Cache&);
-
-void ensure_connection(auto& cache, const URL::URL& url, auto job, Core::ProxyData proxy_data = {})
-{
-    using CacheEntryType = RemoveCVReference<decltype(*declval<typename RemoveCVReference<decltype(cache)>::ProtectedType>().begin()->value)>;
-
-    auto hostname = url.serialized_host().release_value_but_fixme_should_propagate_errors().to_byte_string();
-    auto& properties = g_inferred_server_properties.with_write_locked([&](auto& map) -> InferredServerProperties& { return map.ensure(hostname); });
-
-    auto& sockets_for_url = *cache.with_write_locked([&](auto& map) -> NonnullOwnPtr<CacheEntryType>& {
-        return map.ensure({ move(hostname), url.port_or_default(), proxy_data }, [] { return make<CacheEntryType>(); });
-    });
-
-    // Find the connection with an empty queue; if none exist, we'll find the least backed-up connection later.
-    // Note that servers that are known to serve a single request per connection (e.g. HTTP/1.0) usually have
-    // issues with concurrent connections, so we'll only allow one connection per URL in that case to avoid issues.
-    // This is a bit too aggressive, but there's no way to know if the server can handle concurrent connections
-    // without trying it out first, and that's not worth the effort as HTTP/1.0 is a legacy protocol anyway.
-    auto it = sockets_for_url.find_if([&](auto const& connection) {
-        return properties.requests_served_per_connection < 2
-            || connection->request_queue.with_read_locked([](auto const& queue) { return queue.size(); }) <= ConnectionCacheQueueHighWatermark;
-    });
-    auto did_add_new_connection = false;
-    auto failed_to_find_a_socket = it.is_end();
-
-    Proxy proxy { proxy_data };
-    size_t index;
-
-    auto timer = Core::ElapsedTimer::start_new();
-    if (failed_to_find_a_socket && sockets_for_url.size() < MaxConcurrentConnectionsPerURL) {
-        using ConnectionType = RemoveCVReference<decltype(*AK::Detail::declval<CacheEntryType>().at(0))>;
-        auto& connection = cache.with_write_locked([&](auto&) -> ConnectionType& {
-            index = sockets_for_url.size();
-            sockets_for_url.append(AK::make<ConnectionType>(
-                nullptr,
-                typename ConnectionType::QueueType {},
-                Core::Timer::create_single_shot(ConnectionKeepAliveTimeMilliseconds, nullptr),
-                true));
-            auto& connection = sockets_for_url.last();
-            connection->proxy = move(proxy);
-            return *connection;
-        });
-        ScopeGuard start_guard = [&] {
-            connection.is_being_started = false;
-        };
-        dbgln_if(REQUESTSERVER_DEBUG, "I will start a connection ({}) for URL {}", &connection, url);
-
-        auto connection_result = proxy.tunnel<typename ConnectionType::SocketType, typename ConnectionType::StorageType>(url);
-        misses++;
-        if (connection_result.is_error()) {
-            dbgln("ConnectionCache: Connection to {} failed: {}", url, connection_result.error());
-            Core::deferred_invoke([job] {
-                job->fail(Core::NetworkJob::Error::ConnectionFailed);
-            });
-            return;
-        }
-        auto socket_result = Core::BufferedSocket<typename ConnectionType::StorageType>::create(connection_result.release_value());
-        if (socket_result.is_error()) {
-            dbgln("ConnectionCache: Failed to make a buffered socket for {}: {}", url, socket_result.error());
-            Core::deferred_invoke([job] {
-                job->fail(Core::NetworkJob::Error::ConnectionFailed);
-            });
-            return;
-        }
-        did_add_new_connection = true;
-        connection.socket = socket_result.release_value();
-    }
-
-    auto elapsed = AK::Duration::from_milliseconds(timer.elapsed_milliseconds());
-
-    if (failed_to_find_a_socket) {
-        if (!did_add_new_connection) {
-            // Find the least backed-up connection (based on how many entries are in their request queue).
-            index = 0;
-            auto min_queue_size = (size_t)-1;
-            for (auto it = sockets_for_url.begin(); it != sockets_for_url.end(); ++it) {
-                if (auto queue_size = (*it)->request_queue.with_read_locked([](auto const& queue) { return queue.size(); }); min_queue_size > queue_size) {
-                    index = it.index();
-                    min_queue_size = queue_size;
-                }
-            }
-        }
-    } else {
-        index = it.index();
-        hits++;
-    }
-
-    dbgln_if(REQUESTSERVER_DEBUG, "ConnectionCache: Hits: {}, Misses: {}", RequestServer::ConnectionCache::hits, RequestServer::ConnectionCache::misses);
-    start_connection(url, job, sockets_for_url, index, elapsed, cache);
-}
-
-template<typename Cache>
-void start_connection(URL::URL const& url, auto job, auto& sockets_for_url, size_t index, AK::Duration setup_time, Cache& cache)
-{
-    if (sockets_for_url.is_empty()) {
-        Core::deferred_invoke([job] {
-            job->fail(Core::NetworkJob::Error::ConnectionFailed);
-        });
-        return;
-    }
-
-    auto& connection = *sockets_for_url[index];
-    if (connection.is_being_started) {
-        // Someone else is creating the connection, queue the job and let them handle it.
-        dbgln_if(REQUESTSERVER_DEBUG, "Enqueue request for URL {} in {} - {}", url, &connection, connection.socket.ptr());
-        auto size = connection.request_queue.with_write_locked([&](auto& queue) {
-            queue.append(JobData::create(job, url));
-            return queue.size();
-        });
-        connection.max_queue_length = max(connection.max_queue_length, size);
-        return;
-    }
-
-    if (!connection.has_started) {
-        connection.has_started = true;
-        Core::deferred_invoke([&connection, &cache, url, job, setup_time] {
-            (void)setup_time;
-            auto job_data = JobData::create(job, url);
-            if constexpr (REQUESTSERVER_DEBUG) {
-                job_data.timing_info.waiting_in_queue = AK::Duration::from_milliseconds(job_data.timing_info.timer.elapsed_milliseconds());
-                job_data.timing_info.timer.start();
-            }
-            if (auto result = recreate_socket_if_needed(connection, url); result.is_error()) {
-                dbgln_if(REQUESTSERVER_DEBUG, "ConnectionCache: request failed to start, failed to make a socket: {}", result.error());
-                if constexpr (REQUESTSERVER_DEBUG) {
-                    job_data.timing_info.starting_connection += AK::Duration::from_milliseconds(job_data.timing_info.timer.elapsed_milliseconds()) + setup_time;
-                    job_data.timing_info.timer.start();
-                }
-                Core::deferred_invoke([job] {
-                    job->fail(Core::NetworkJob::Error::ConnectionFailed);
-                });
-            } else {
-                cache.with_write_locked([&](auto&) {
-                    dbgln_if(REQUESTSERVER_DEBUG, "Immediately start request for url {} in {} - {}", url, &connection, connection.socket.ptr());
-                    connection.job_data = move(job_data);
-                    if constexpr (REQUESTSERVER_DEBUG) {
-                        connection.job_data->timing_info.starting_connection += AK::Duration::from_milliseconds(connection.job_data->timing_info.timer.elapsed_milliseconds()) + setup_time;
-                        connection.job_data->timing_info.timer.start();
-                    }
-                    connection.removal_timer->stop();
-                    connection.timer.start();
-                    connection.current_url = url;
-                    connection.socket->set_notifications_enabled(true);
-                    connection.job_data->start(*connection.socket);
-                });
-            }
-        });
-    } else {
-        dbgln_if(REQUESTSERVER_DEBUG, "Enqueue request for URL {} in {} - {}", url, &connection, connection.socket.ptr());
-        auto size = connection.request_queue.with_write_locked([&](auto& queue) {
-            queue.append(JobData::create(job, url));
-            return queue.size();
-        });
-        connection.max_queue_length = max(connection.max_queue_length, size);
-    }
-}
-}

Userland/Services/RequestServer/ConnectionFromClient.cpp

@@ -9,6 +9,7 @@
 #include <AK/NonnullOwnPtr.h>
 #include <AK/RefCounted.h>
 #include <AK/Weakable.h>
+#include <LibCore/EventLoop.h>
 #include <LibCore/Proxy.h>
 #include <LibCore/Socket.h>
 #include <LibWebSocket/ConnectionInfo.h>

Userland/Services/RequestServer/HttpCommon.h

@@ -1,115 +0,0 @@
-/*
- * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#pragma once
-
-#include <AK/ByteBuffer.h>
-#include <AK/ByteString.h>
-#include <AK/HashMap.h>
-#include <AK/NonnullOwnPtr.h>
-#include <AK/Optional.h>
-#include <AK/OwnPtr.h>
-#include <AK/Types.h>
-#include <LibHTTP/HttpRequest.h>
-#include <RequestServer/ConnectionCache.h>
-#include <RequestServer/ConnectionFromClient.h>
-#include <RequestServer/Request.h>
-
-namespace RequestServer::Detail {
-
-template<typename TSelf, typename TJob>
-void init(TSelf* self, TJob job)
-{
-    job->on_headers_received = [self](auto& headers, auto response_code) {
-        if (response_code.has_value())
-            self->set_status_code(response_code.value());
-        self->set_response_headers(headers);
-    };
-
-    job->on_finish = [self](bool success) {
-        Core::deferred_invoke([url = self->job().url(), socket = self->job().socket()] {
-            ConnectionCache::request_did_finish(url, socket);
-        });
-        if (auto* response = self->job().response()) {
-            self->set_status_code(response->code());
-            self->set_response_headers(response->headers());
-            self->set_downloaded_size(response->downloaded_size());
-        }
-
-        // if we didn't know the total size, pretend that the request finished successfully
-        // and set the total size to the downloaded size
-        if (!self->total_size().has_value())
-            self->did_progress(self->downloaded_size(), self->downloaded_size());
-
-        self->did_finish(success);
-    };
-    job->on_progress = [self](Optional<u64> total, u64 current) {
-        self->did_progress(total, current);
-    };
-    if constexpr (requires { job->on_certificate_requested; }) {
-        job->on_certificate_requested = [job, self] {
-            self->did_request_certificates();
-            Core::EventLoop::current().spin_until([&] {
-                return job->received_client_certificates();
-            });
-            return job->take_client_certificates();
-        };
-    }
-}
-
-template<typename TBadgedProtocol, typename TPipeResult>
-OwnPtr<Request> start_request(TBadgedProtocol&& protocol, i32 request_id, ConnectionFromClient& client, ByteString const& method, URL::URL const& url, HTTP::HeaderMap const& headers, ReadonlyBytes body, TPipeResult&& pipe_result, Core::ProxyData proxy_data = {})
-{
-    using TJob = typename TBadgedProtocol::Type::JobType;
-    using TRequest = typename TBadgedProtocol::Type::RequestType;
-
-    if (pipe_result.is_error()) {
-        return {};
-    }
-
-    HTTP::HttpRequest request;
-    if (method.equals_ignoring_ascii_case("post"sv))
-        request.set_method(HTTP::HttpRequest::Method::POST);
-    else if (method.equals_ignoring_ascii_case("head"sv))
-        request.set_method(HTTP::HttpRequest::Method::HEAD);
-    else if (method.equals_ignoring_ascii_case("delete"sv))
-        request.set_method(HTTP::HttpRequest::Method::DELETE);
-    else if (method.equals_ignoring_ascii_case("patch"sv))
-        request.set_method(HTTP::HttpRequest::Method::PATCH);
-    else if (method.equals_ignoring_ascii_case("options"sv))
-        request.set_method(HTTP::HttpRequest::Method::OPTIONS);
-    else if (method.equals_ignoring_ascii_case("trace"sv))
-        request.set_method(HTTP::HttpRequest::Method::TRACE);
-    else if (method.equals_ignoring_ascii_case("connect"sv))
-        request.set_method(HTTP::HttpRequest::Method::CONNECT);
-    else if (method.equals_ignoring_ascii_case("put"sv))
-        request.set_method(HTTP::HttpRequest::Method::PUT);
-    else
-        request.set_method(HTTP::HttpRequest::Method::GET);
-    request.set_url(url);
-    request.set_headers(headers);
-
-    auto allocated_body_result = ByteBuffer::copy(body);
-    if (allocated_body_result.is_error())
-        return {};
-    request.set_body(allocated_body_result.release_value());
-
-    auto output_stream = MUST(Core::File::adopt_fd(pipe_result.value().write_fd, Core::File::OpenMode::Write));
-    auto job = TJob::construct(move(request), *output_stream);
-    auto protocol_request = TRequest::create_with_job(forward<TBadgedProtocol>(protocol), client, static_cast<TJob&>(*job), move(output_stream), request_id);
-    protocol_request->set_request_fd(pipe_result.value().read_fd);
-
-    Core::deferred_invoke([=, job = job->template make_weak_ptr<TJob>()] {
-        if constexpr (IsSame<typename TBadgedProtocol::Type, HttpsProtocol>)
-            ConnectionCache::ensure_connection(ConnectionCache::g_tls_connection_cache, url, job, proxy_data);
-        else
-            ConnectionCache::ensure_connection(ConnectionCache::g_tcp_connection_cache, url, job, proxy_data);
-    });
-
-    return protocol_request;
-}
-
-}

Userland/Services/RequestServer/HttpProtocol.cpp

@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include <AK/Badge.h>
-#include <AK/ByteBuffer.h>
-#include <AK/ByteString.h>
-#include <AK/HashMap.h>
-#include <AK/OwnPtr.h>
-#include <LibURL/URL.h>
-#include <RequestServer/ConnectionFromClient.h>
-#include <RequestServer/HttpCommon.h>
-#include <RequestServer/HttpProtocol.h>
-#include <RequestServer/Request.h>
-
-namespace RequestServer {
-
-HttpProtocol::HttpProtocol()
-    : Protocol("http")
-{
-}
-
-OwnPtr<Request> HttpProtocol::start_request(i32 request_id, ConnectionFromClient& client, ByteString const& method, URL::URL const& url, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data)
-{
-    return Detail::start_request(Badge<HttpProtocol> {}, request_id, client, method, url, headers, body, get_pipe_for_request(), proxy_data);
-}
-
-void HttpProtocol::install()
-{
-    Protocol::install(adopt_own(*new HttpProtocol()));
-}
-
-}

Userland/Services/RequestServer/HttpProtocol.h

@@ -1,37 +0,0 @@
-/*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#pragma once
-
-#include <AK/ByteBuffer.h>
-#include <AK/ByteString.h>
-#include <AK/HashMap.h>
-#include <AK/OwnPtr.h>
-#include <LibHTTP/Job.h>
-#include <LibURL/URL.h>
-#include <RequestServer/ConnectionFromClient.h>
-#include <RequestServer/HttpRequest.h>
-#include <RequestServer/Protocol.h>
-#include <RequestServer/Request.h>
-
-namespace RequestServer {
-
-class HttpProtocol final : public Protocol {
-public:
-    using JobType = HTTP::Job;
-    using RequestType = HttpRequest;
-
-    ~HttpProtocol() override = default;
-
-    static void install();
-
-private:
-    HttpProtocol();
-
-    virtual OwnPtr<Request> start_request(i32, ConnectionFromClient&, ByteString const& method, URL::URL const&, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data = {}) override;
-};
-
-}

Userland/Services/RequestServer/HttpRequest.cpp

@@ -1,33 +0,0 @@
-/*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include <LibHTTP/Job.h>
-#include <RequestServer/HttpCommon.h>
-#include <RequestServer/HttpProtocol.h>
-#include <RequestServer/HttpRequest.h>
-
-namespace RequestServer {
-
-HttpRequest::HttpRequest(ConnectionFromClient& client, NonnullRefPtr<HTTP::Job> job, NonnullOwnPtr<Core::File>&& output_stream, i32 request_id)
-    : Request(client, move(output_stream), request_id)
-    , m_job(job)
-{
-    Detail::init(this, job);
-}
-
-HttpRequest::~HttpRequest()
-{
-    m_job->on_finish = nullptr;
-    m_job->on_progress = nullptr;
-    m_job->cancel();
-}
-
-NonnullOwnPtr<HttpRequest> HttpRequest::create_with_job(Badge<HttpProtocol>&&, ConnectionFromClient& client, NonnullRefPtr<HTTP::Job> job, NonnullOwnPtr<Core::File>&& output_stream, i32 request_id)
-{
-    return adopt_own(*new HttpRequest(client, move(job), move(output_stream), request_id));
-}
-
-}

Userland/Services/RequestServer/HttpRequest.h

@@ -1,34 +0,0 @@
-/*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#pragma once
-
-#include <AK/Badge.h>
-#include <AK/NonnullOwnPtr.h>
-#include <LibCore/Forward.h>
-#include <LibHTTP/Forward.h>
-#include <RequestServer/Request.h>
-
-namespace RequestServer {
-
-class HttpRequest final : public Request {
-public:
-    virtual ~HttpRequest() override;
-    static NonnullOwnPtr<HttpRequest> create_with_job(Badge<HttpProtocol>&&, ConnectionFromClient&, NonnullRefPtr<HTTP::Job>, NonnullOwnPtr<Core::File>&&, i32);
-
-    HTTP::Job& job() { return m_job; }
-    HTTP::Job const& job() const { return m_job; }
-
-    virtual URL::URL url() const override { return m_job->url(); }
-    virtual void cancel() override { m_job->cancel(); }
-
-private:
-    explicit HttpRequest(ConnectionFromClient&, NonnullRefPtr<HTTP::Job>, NonnullOwnPtr<Core::File>&&, i32);
-
-    NonnullRefPtr<HTTP::Job> m_job;
-};
-
-}

Userland/Services/RequestServer/HttpsProtocol.cpp

@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2018-2020, the SerenityOS developers.
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include <AK/Badge.h>
-#include <AK/ByteBuffer.h>
-#include <AK/ByteString.h>
-#include <AK/HashMap.h>
-#include <AK/OwnPtr.h>
-#include <LibURL/URL.h>
-#include <RequestServer/ConnectionFromClient.h>
-#include <RequestServer/HttpCommon.h>
-#include <RequestServer/HttpsProtocol.h>
-#include <RequestServer/Request.h>
-
-namespace RequestServer {
-
-HttpsProtocol::HttpsProtocol()
-    : Protocol("https")
-{
-}
-
-OwnPtr<Request> HttpsProtocol::start_request(i32 request_id, ConnectionFromClient& client, ByteString const& method, URL::URL const& url, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data)
-{
-    return Detail::start_request(Badge<HttpsProtocol> {}, request_id, client, method, url, headers, body, get_pipe_for_request(), proxy_data);
-}
-
-void HttpsProtocol::install()
-{
-    Protocol::install(adopt_own(*new HttpsProtocol()));
-}
-
-}

Userland/Services/RequestServer/HttpsProtocol.h

@@ -1,37 +0,0 @@
-/*
- * Copyright (c) 2018-2020, the SerenityOS developers.
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#pragma once
-
-#include <AK/ByteBuffer.h>
-#include <AK/ByteString.h>
-#include <AK/HashMap.h>
-#include <AK/OwnPtr.h>
-#include <LibHTTP/HttpsJob.h>
-#include <LibURL/URL.h>
-#include <RequestServer/ConnectionFromClient.h>
-#include <RequestServer/HttpsRequest.h>
-#include <RequestServer/Protocol.h>
-#include <RequestServer/Request.h>
-
-namespace RequestServer {
-
-class HttpsProtocol final : public Protocol {
-public:
-    using JobType = HTTP::HttpsJob;
-    using RequestType = HttpsRequest;
-
-    ~HttpsProtocol() override = default;
-
-    static void install();
-
-private:
-    HttpsProtocol();
-
-    virtual OwnPtr<Request> start_request(i32, ConnectionFromClient&, ByteString const& method, URL::URL const&, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data = {}) override;
-};
-
-}

Userland/Services/RequestServer/HttpsRequest.cpp

@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2020, the SerenityOS developers.
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include <LibHTTP/HttpsJob.h>
-#include <RequestServer/HttpCommon.h>
-#include <RequestServer/HttpsProtocol.h>
-#include <RequestServer/HttpsRequest.h>
-
-namespace RequestServer {
-
-HttpsRequest::HttpsRequest(ConnectionFromClient& client, NonnullRefPtr<HTTP::HttpsJob> job, NonnullOwnPtr<Core::File>&& output_stream, i32 request_id)
-    : Request(client, move(output_stream), request_id)
-    , m_job(job)
-{
-    Detail::init(this, job);
-}
-
-void HttpsRequest::set_certificate(ByteString certificate, ByteString key)
-{
-    m_job->set_certificate(move(certificate), move(key));
-}
-
-HttpsRequest::~HttpsRequest()
-{
-    m_job->on_finish = nullptr;
-    m_job->on_progress = nullptr;
-    m_job->cancel();
-}
-
-NonnullOwnPtr<HttpsRequest> HttpsRequest::create_with_job(Badge<HttpsProtocol>&&, ConnectionFromClient& client, NonnullRefPtr<HTTP::HttpsJob> job, NonnullOwnPtr<Core::File>&& output_stream, i32 request_id)
-{
-    return adopt_own(*new HttpsRequest(client, move(job), move(output_stream), request_id));
-}
-
-}

Userland/Services/RequestServer/HttpsRequest.h

@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2020, the SerenityOS developers.
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#pragma once
-
-#include <AK/Badge.h>
-#include <LibCore/Forward.h>
-#include <LibHTTP/HttpsJob.h>
-#include <RequestServer/Request.h>
-
-namespace RequestServer {
-
-class HttpsRequest final : public Request {
-public:
-    virtual ~HttpsRequest() override;
-    static NonnullOwnPtr<HttpsRequest> create_with_job(Badge<HttpsProtocol>&&, ConnectionFromClient&, NonnullRefPtr<HTTP::HttpsJob>, NonnullOwnPtr<Core::File>&&, i32);
-
-    HTTP::HttpsJob& job() { return m_job; }
-    HTTP::HttpsJob const& job() const { return m_job; }
-
-    virtual URL::URL url() const override { return m_job->url(); }
-    virtual void cancel() override { m_job->cancel(); }
-
-private:
-    explicit HttpsRequest(ConnectionFromClient&, NonnullRefPtr<HTTP::HttpsJob>, NonnullOwnPtr<Core::File>&&, i32);
-
-    virtual void set_certificate(ByteString certificate, ByteString key) override;
-
-    NonnullRefPtr<HTTP::HttpsJob> m_job;
-};
-
-}

Userland/Services/RequestServer/Protocol.cpp

@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include <AK/HashMap.h>
-#include <AK/NonnullOwnPtr.h>
-#include <LibCore/System.h>
-#include <RequestServer/Protocol.h>
-
-namespace RequestServer {
-
-static HashMap<ByteString, NonnullOwnPtr<Protocol>>& all_protocols()
-{
-    static HashMap<ByteString, NonnullOwnPtr<Protocol>> map;
-    return map;
-}
-
-Protocol* Protocol::find_by_name(ByteString const& name)
-{
-    return all_protocols().get(name).map([](auto& p) -> Protocol* { return p; }).value_or(nullptr);
-}
-
-Protocol::Protocol(ByteString const& name)
-    : m_name(name)
-{
-}
-
-ErrorOr<Protocol::Pipe> Protocol::get_pipe_for_request()
-{
-    auto fd_pair = TRY(Core::System::pipe2(O_NONBLOCK));
-    return Pipe { fd_pair[0], fd_pair[1] };
-}
-
-void Protocol::install(NonnullOwnPtr<Protocol> protocol)
-{
-    auto name = protocol->name();
-    all_protocols().set(move(name), move(protocol));
-}
-
-}

Userland/Services/RequestServer/Protocol.h

@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#pragma once
-
-#include <AK/RefPtr.h>
-#include <LibCore/Proxy.h>
-#include <LibHTTP/HeaderMap.h>
-#include <LibURL/URL.h>
-#include <RequestServer/Forward.h>
-
-namespace RequestServer {
-
-class Protocol {
-public:
-    virtual ~Protocol() = default;
-
-    ByteString const& name() const { return m_name; }
-    virtual OwnPtr<Request> start_request(i32, ConnectionFromClient&, ByteString const& method, URL::URL const&, HTTP::HeaderMap const& headers, ReadonlyBytes body, Core::ProxyData proxy_data = {}) = 0;
-
-    static Protocol* find_by_name(ByteString const&);
-
-protected:
-    explicit Protocol(ByteString const& name);
-    struct Pipe {
-        int read_fd { -1 };
-        int write_fd { -1 };
-    };
-    static ErrorOr<Pipe> get_pipe_for_request();
-
-    static void install(NonnullOwnPtr<Protocol>);
-
-private:
-    ByteString m_name;
-};
-
-}

Userland/Services/RequestServer/RequestServer.ipc

@@ -1,6 +1,7 @@
+#include <LibCore/Proxy.h>
 #include <LibHTTP/HeaderMap.h>
 #include <LibURL/URL.h>
-#include <RequestServer/ConnectionCache.h>
+#include <RequestServer/CacheLevel.h>
 
 endpoint RequestServer
 {