From e483bb70f0189d255eebc84a6d1dac44117c88dd Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Thu, 5 Sep 2024 13:20:09 +0200 Subject: [PATCH] RequestServer: Use libcurl for HTTP and HTTPS downloads This replaces the use of our home-grown implementations inherited from SerenityOS. --- Ladybird/RequestServer/CMakeLists.txt | 5 +- Ladybird/RequestServer/main.cpp | 7 + .../RequestServer/ConnectionFromClient.cpp | 454 +++++++++++------- .../RequestServer/ConnectionFromClient.h | 34 +- vcpkg.json | 6 + 5 files changed, 318 insertions(+), 188 deletions(-) diff --git a/Ladybird/RequestServer/CMakeLists.txt b/Ladybird/RequestServer/CMakeLists.txt index ca4e0d2ae83..da945a54d84 100644 --- a/Ladybird/RequestServer/CMakeLists.txt +++ b/Ladybird/RequestServer/CMakeLists.txt @@ -26,12 +26,15 @@ else() add_library(requestserver STATIC ${REQUESTSERVER_SOURCES}) endif() +find_package(PkgConfig) +find_package(CURL REQUIRED) + add_executable(RequestServer main.cpp) target_link_libraries(RequestServer PRIVATE requestserver) target_include_directories(requestserver PRIVATE ${LADYBIRD_SOURCE_DIR}/Userland/Services/) target_include_directories(requestserver PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/..) -target_link_libraries(requestserver PUBLIC LibCore LibMain LibCrypto LibFileSystem LibHTTP LibIPC LibMain LibTLS LibWebView LibWebSocket LibURL LibThreading) +target_link_libraries(requestserver PUBLIC LibCore LibMain LibCrypto LibFileSystem LibHTTP LibIPC LibMain LibTLS LibWebView LibWebSocket LibURL LibThreading CURL::libcurl) if (${CMAKE_SYSTEM_NAME} MATCHES "SunOS") # Solaris has socket and networking related functions in two extra libraries target_link_libraries(requestserver PUBLIC nsl socket) diff --git a/Ladybird/RequestServer/main.cpp b/Ladybird/RequestServer/main.cpp index d401a558cf2..0509619e65e 100644 --- a/Ladybird/RequestServer/main.cpp +++ b/Ladybird/RequestServer/main.cpp @@ -24,6 +24,10 @@ # include #endif +namespace RequestServer { +extern ByteString g_default_certificate_path; +} + static ErrorOr find_certificates(StringView serenity_resource_root) { auto cert_path = ByteString::formatted("{}/ladybird/cacert.pem", serenity_resource_root); @@ -54,6 +58,9 @@ ErrorOr serenity_main(Main::Arguments arguments) // Ensure the certificates are read out here. if (certificates.is_empty()) certificates.append(TRY(find_certificates(serenity_resource_root))); + else + RequestServer::g_default_certificate_path = certificates.first(); + DefaultRootCACertificates::set_default_certificate_paths(certificates.span()); [[maybe_unused]] auto& certs = DefaultRootCACertificates::the(); diff --git a/Userland/Services/RequestServer/ConnectionFromClient.cpp b/Userland/Services/RequestServer/ConnectionFromClient.cpp index d100e2a4ef2..26da6b5c2ca 100644 --- a/Userland/Services/RequestServer/ConnectionFromClient.cpp +++ b/Userland/Services/RequestServer/ConnectionFromClient.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020, Andreas Kling + * Copyright (c) 2018-2024, Andreas Kling * * SPDX-License-Identifier: BSD-2-Clause */ @@ -14,44 +14,194 @@ #include #include #include -#include -#include #include +#include #include namespace RequestServer { -template -struct Looper : public Threading::ThreadPoolLooper { - IterationDecision next(Pool& pool, bool wait); - Core::EventLoop event_loop; -}; +ByteString g_default_certificate_path; +static HashMap> s_connections; +static IDAllocator s_client_ids; -struct ThreadPoolEntry { - NonnullRefPtr client; - ConnectionFromClient::Work work; -}; -static Threading::ThreadPool s_thread_pool { - [](ThreadPoolEntry entry) { - entry.client->worker_do_work(move(entry.work)); +struct ConnectionFromClient::ActiveRequest { + CURLM* multi { nullptr }; + CURL* easy { nullptr }; + i32 request_id { 0 }; + RefPtr notifier; + WeakPtr client; + int writer_fd { 0 }; + HTTP::HeaderMap headers; + bool got_all_headers { false }; + size_t downloaded_so_far { 0 }; + String url; + ByteBuffer body; + + ActiveRequest(ConnectionFromClient& client, CURLM* multi, CURL* easy, i32 request_id, int writer_fd) + : multi(multi) + , easy(easy) + , request_id(request_id) + , client(client) + , writer_fd(writer_fd) + { + } + + ~ActiveRequest() + { + MUST(Core::System::close(writer_fd)); + auto result = curl_multi_remove_handle(multi, easy); + VERIFY(result == CURLM_OK); + curl_easy_cleanup(easy); + } + + void flush_headers_if_needed() + { + if (got_all_headers) + return; + got_all_headers = true; + long http_status_code = 0; + auto result = curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &http_status_code); + VERIFY(result == CURLE_OK); + client->async_headers_became_available(request_id, headers, http_status_code); } }; -static HashMap> s_connections; -static IDAllocator s_client_ids; +size_t ConnectionFromClient::on_header_received(void* buffer, size_t size, size_t nmemb, void* user_data) +{ + auto* request = static_cast(user_data); + size_t total_size = size * nmemb; + auto header_line = StringView { static_cast(buffer), total_size }; + if (auto colon_index = header_line.find(':'); colon_index.has_value()) { + auto name = header_line.substring_view(0, colon_index.value()).trim_whitespace(); + auto value = header_line.substring_view(colon_index.value() + 1, header_line.length() - colon_index.value() - 1).trim_whitespace(); + request->headers.set(name, value); + } + return total_size; +} + +size_t ConnectionFromClient::on_data_received(void* buffer, size_t size, size_t nmemb, void* user_data) +{ + auto* request = static_cast(user_data); + request->flush_headers_if_needed(); + + size_t total_size = size * nmemb; + + size_t remaining_length = total_size; + u8 const* remaining_data = static_cast(buffer); + while (remaining_length > 0) { + auto result = Core::System::write(request->writer_fd, { remaining_data, remaining_length }); + if (result.is_error()) { + if (result.error().code() != EAGAIN) { + dbgln("on_data_received: write failed: {}", result.error()); + VERIFY_NOT_REACHED(); + } + sched_yield(); + continue; + } + auto nwritten = result.value(); + if (nwritten == 0) { + dbgln("on_data_received: write returned 0"); + VERIFY_NOT_REACHED(); + } + remaining_data += nwritten; + remaining_length -= nwritten; + } + + Optional content_length_for_ipc; + curl_off_t content_length = -1; + auto res = curl_easy_getinfo(request->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &content_length); + if (res == CURLE_OK && content_length != -1) { + content_length_for_ipc = content_length; + } + + request->downloaded_so_far += total_size; + + request->client->async_request_progress( + request->request_id, + content_length_for_ipc, + request->downloaded_so_far); + + return total_size; +} + +int ConnectionFromClient::on_socket_callback(CURL*, int sockfd, int what, void* user_data, void*) +{ + auto* client = static_cast(user_data); + + if (what == CURL_POLL_REMOVE) { + client->m_read_notifiers.remove(sockfd); + client->m_write_notifiers.remove(sockfd); + return 0; + } + + if (what & CURL_POLL_IN) { + client->m_read_notifiers.ensure(sockfd, [client, sockfd, multi = client->m_curl_multi] { + auto notifier = Core::Notifier::construct(sockfd, Core::NotificationType::Read); + notifier->on_activation = [client, sockfd, multi] { + int still_running = 0; + auto result = curl_multi_socket_action(multi, sockfd, CURL_CSELECT_IN, &still_running); + VERIFY(result == CURLM_OK); + client->check_active_requests(); + }; + notifier->set_enabled(true); + return notifier; + }); + } + + if (what & CURL_POLL_OUT) { + client->m_write_notifiers.ensure(sockfd, [client, sockfd, multi = client->m_curl_multi] { + auto notifier = Core::Notifier::construct(sockfd, Core::NotificationType::Write); + notifier->on_activation = [client, sockfd, multi] { + int still_running = 0; + auto result = curl_multi_socket_action(multi, sockfd, CURL_CSELECT_OUT, &still_running); + VERIFY(result == CURLM_OK); + client->check_active_requests(); + }; + notifier->set_enabled(true); + return notifier; + }); + } + + return 0; +} + +int ConnectionFromClient::on_timeout_callback(void*, long timeout_ms, void* user_data) +{ + auto* client = static_cast(user_data); + if (timeout_ms < 0) { + client->m_timer->stop(); + } else { + client->m_timer->restart(timeout_ms); + } + return 0; +} ConnectionFromClient::ConnectionFromClient(NonnullOwnPtr socket) : IPC::ConnectionFromClient(*this, move(socket), s_client_ids.allocate()) { s_connections.set(client_id(), *this); + + m_curl_multi = curl_multi_init(); + + auto set_option = [this](auto option, auto value) { + auto result = curl_multi_setopt(m_curl_multi, option, value); + VERIFY(result == CURLM_OK); + }; + set_option(CURLMOPT_SOCKETFUNCTION, &on_socket_callback); + set_option(CURLMOPT_SOCKETDATA, this); + set_option(CURLMOPT_TIMERFUNCTION, &on_timeout_callback); + set_option(CURLMOPT_TIMERDATA, this); + + m_timer = Core::Timer::create_single_shot(0, [this] { + int still_running = 0; + auto result = curl_multi_socket_action(m_curl_multi, CURL_SOCKET_TIMEOUT, 0, &still_running); + VERIFY(result == CURLM_OK); + check_active_requests(); + }); } ConnectionFromClient::~ConnectionFromClient() { - m_requests.with_locked([](HashMap>& map) { - for (auto& entry : map) - entry.value->cancel(); - }); } class Job : public RefCounted @@ -96,85 +246,6 @@ private: inline static HashMap> s_jobs {}; }; -template -IterationDecision Looper::next(Pool& pool, bool wait) -{ - bool should_exit = false; - auto timer = Core::Timer::create_repeating(100, [&] { - if (Threading::ThreadPoolLooper::next(pool, false) == IterationDecision::Break) { - event_loop.quit(0); - should_exit = true; - } - }); - - timer->start(); - if (!wait) { - event_loop.deferred_invoke([&] { - event_loop.quit(0); - }); - } - - event_loop.exec(); - - if (should_exit) - return IterationDecision::Break; - return IterationDecision::Continue; -} - -void ConnectionFromClient::worker_do_work(Work work) -{ - work.visit( - [&](StartRequest& start_request) { - auto* protocol = Protocol::find_by_name(start_request.url.scheme().to_byte_string()); - if (!protocol) { - dbgln("StartRequest: No protocol handler for URL: '{}'", start_request.url); - auto lock = Threading::MutexLocker(m_ipc_mutex); - (void)post_message(Messages::RequestClient::RequestFinished(start_request.request_id, false, 0)); - return; - } - auto request = protocol->start_request(start_request.request_id, *this, start_request.method, start_request.url, start_request.request_headers, start_request.request_body, start_request.proxy_data); - if (!request) { - dbgln("StartRequest: Protocol handler failed to start request: '{}'", start_request.url); - auto lock = Threading::MutexLocker(m_ipc_mutex); - (void)post_message(Messages::RequestClient::RequestFinished(start_request.request_id, false, 0)); - return; - } - auto id = request->id(); - auto fd = request->request_fd(); - m_requests.with_locked([&](auto& map) { map.set(id, move(request)); }); - auto lock = Threading::MutexLocker(m_ipc_mutex); - (void)post_message(Messages::RequestClient::RequestStarted(start_request.request_id, IPC::File::adopt_fd(fd))); - }, - [&](EnsureConnection& ensure_connection) { - auto& url = ensure_connection.url; - auto& cache_level = ensure_connection.cache_level; - - if (cache_level == CacheLevel::ResolveOnly) { - Core::deferred_invoke([host = url.serialized_host().release_value_but_fixme_should_propagate_errors().to_byte_string()] { - dbgln("EnsureConnection: DNS-preload for {}", host); - auto resolved_host = Core::Socket::resolve_host(host, Core::Socket::SocketType::Stream); - if (resolved_host.is_error()) - dbgln("EnsureConnection: DNS-preload failed for {}", host); - }); - dbgln("EnsureConnection: DNS-preload for {} done", url); - return; - } - - dbgln("EnsureConnection: Pre-connect to {}", url); - auto do_preconnect = [=, job = Job::ensure(url)](auto& cache) { - ConnectionCache::ensure_connection(cache, url, move(job)); - }; - - if (url.scheme() == "http"sv) - do_preconnect(ConnectionCache::g_tcp_connection_cache); - else if (url.scheme() == "https"sv) - do_preconnect(ConnectionCache::g_tls_connection_cache); - else - dbgln("EnsureConnection: Invalid URL scheme: '{}'", url.scheme()); - }, - [&](Empty) {}); -} - void ConnectionFromClient::die() { auto client_id = this->client_id(); @@ -207,93 +278,148 @@ Messages::RequestServer::ConnectNewClientResponse ConnectionFromClient::connect_ return IPC::File::adopt_fd(socket_fds[1]); } -void ConnectionFromClient::enqueue(Work work) -{ - s_thread_pool.submit({ *this, move(work) }); -} - Messages::RequestServer::IsSupportedProtocolResponse ConnectionFromClient::is_supported_protocol(ByteString const& protocol) { - bool supported = Protocol::find_by_name(protocol.to_lowercase()); - return supported; + return protocol == "http"sv || protocol == "https"sv; } void ConnectionFromClient::start_request(i32 request_id, ByteString const& method, URL::URL const& url, HTTP::HeaderMap const& request_headers, ByteBuffer const& request_body, Core::ProxyData const& proxy_data) { if (!url.is_valid()) { dbgln("StartRequest: Invalid URL requested: '{}'", url); - auto lock = Threading::MutexLocker(m_ipc_mutex); - (void)post_message(Messages::RequestClient::RequestFinished(request_id, false, 0)); + async_request_finished(request_id, false, 0); return; } - auto headers = request_headers; - if (!headers.contains("Accept-Encoding")) - headers.set("Accept-Encoding", "gzip, deflate, br"); + auto* easy = curl_easy_init(); + if (!easy) { + dbgln("StartRequest: Failed to initialize curl easy handle"); + return; + } - enqueue(StartRequest { - .request_id = request_id, - .method = method, - .url = url, - .request_headers = move(headers), - .request_body = request_body, - .proxy_data = proxy_data, - }); + auto fds_or_error = Core::System::pipe2(O_NONBLOCK); + if (fds_or_error.is_error()) { + dbgln("StartRequest: Failed to create pipe: {}", fds_or_error.error()); + return; + } + + auto fds = fds_or_error.release_value(); + auto writer_fd = fds[1]; + auto reader_fd = fds[0]; + async_request_started(request_id, IPC::File::adopt_fd(reader_fd)); + + auto request = make(*this, m_curl_multi, easy, request_id, writer_fd); + request->url = url.to_string().value(); + + auto set_option = [easy](auto option, auto value) { + auto result = curl_easy_setopt(easy, option, value); + if (result != CURLE_OK) { + dbgln("StartRequest: Failed to set curl option: {}", curl_easy_strerror(result)); + return false; + } + return true; + }; + + if (!g_default_certificate_path.is_empty()) + set_option(CURLOPT_CAINFO, g_default_certificate_path.characters()); + + set_option(CURLOPT_ACCEPT_ENCODING, "gzip, deflate, br"); + set_option(CURLOPT_URL, url.to_string().value().to_byte_string().characters()); + set_option(CURLOPT_PORT, url.port_or_default()); + + if (method == "GET"sv) { + set_option(CURLOPT_HTTPGET, 1L); + } else if (method == "POST"sv) { + request->body = request_body; + set_option(CURLOPT_POSTFIELDSIZE, request->body.size()); + set_option(CURLOPT_POSTFIELDS, request->body.data()); + } else if (method == "HEAD") { + set_option(CURLOPT_NOBODY, 1L); + } + set_option(CURLOPT_CUSTOMREQUEST, method.characters()); + + set_option(CURLOPT_FOLLOWLOCATION, 0); + + struct curl_slist* curl_headers = nullptr; + for (auto const& header : request_headers.headers()) { + auto header_string = ByteString::formatted("{}: {}", header.name, header.value); + curl_headers = curl_slist_append(curl_headers, header_string.characters()); + } + set_option(CURLOPT_HTTPHEADER, curl_headers); + + // FIXME: Set up proxy if applicable + (void)proxy_data; + + set_option(CURLOPT_WRITEFUNCTION, &on_data_received); + set_option(CURLOPT_WRITEDATA, reinterpret_cast(request.ptr())); + + set_option(CURLOPT_HEADERFUNCTION, &on_header_received); + set_option(CURLOPT_HEADERDATA, reinterpret_cast(request.ptr())); + + auto result = curl_multi_add_handle(m_curl_multi, easy); + VERIFY(result == CURLM_OK); + + m_active_requests.set(request_id, move(request)); +} + +void ConnectionFromClient::check_active_requests() +{ + auto request_from_easy_handle = [this](CURL* easy) -> ActiveRequest* { + for (auto& it : m_active_requests) { + if (it.value->easy == easy) + return it.value.ptr(); + } + return nullptr; + }; + + int msgs_in_queue = 0; + while (auto* msg = curl_multi_info_read(m_curl_multi, &msgs_in_queue)) { + if (msg->msg != CURLMSG_DONE) + continue; + + auto* request = request_from_easy_handle(msg->easy_handle); + request->flush_headers_if_needed(); + + async_request_finished(request->request_id, msg->data.result == CURLE_OK, request->downloaded_so_far); + + m_active_requests.remove(request->request_id); + } } Messages::RequestServer::StopRequestResponse ConnectionFromClient::stop_request(i32 request_id) { - return m_requests.with_locked([&](auto& map) { - auto* request = const_cast(map.get(request_id).value_or(nullptr)); - bool success = false; - if (request) { - request->stop(); - map.remove(request_id); - success = true; - } - return success; - }); -} - -void ConnectionFromClient::did_receive_headers(Badge, Request& request) -{ - auto lock = Threading::MutexLocker(m_ipc_mutex); - async_headers_became_available(request.id(), request.response_headers(), request.status_code()); -} - -void ConnectionFromClient::did_finish_request(Badge, Request& request, bool success) -{ - if (request.total_size().has_value()) { - auto lock = Threading::MutexLocker(m_ipc_mutex); - async_request_finished(request.id(), success, request.total_size().value()); + auto request = m_active_requests.take(request_id); + if (!request.has_value()) { + dbgln("StopRequest: Request ID {} not found", request_id); + return false; } - m_requests.with_locked([&](auto& map) { map.remove(request.id()); }); + return true; } -void ConnectionFromClient::did_progress_request(Badge, Request& request) +void ConnectionFromClient::did_receive_headers(Badge, Request&) { - auto lock = Threading::MutexLocker(m_ipc_mutex); - async_request_progress(request.id(), request.total_size(), request.downloaded_size()); } -void ConnectionFromClient::did_request_certificates(Badge, Request& request) +void ConnectionFromClient::did_finish_request(Badge, Request&, bool) { - auto lock = Threading::MutexLocker(m_ipc_mutex); - async_certificate_requested(request.id()); +} + +void ConnectionFromClient::did_progress_request(Badge, Request&) +{ +} + +void ConnectionFromClient::did_request_certificates(Badge, Request&) +{ + TODO(); } Messages::RequestServer::SetCertificateResponse ConnectionFromClient::set_certificate(i32 request_id, ByteString const& certificate, ByteString const& key) { - return m_requests.with_locked([&](auto& map) { - auto* request = const_cast(map.get(request_id).value_or(nullptr)); - bool success = false; - if (request) { - request->set_certificate(certificate, key); - success = true; - } - return success; - }); + (void)request_id; + (void)certificate; + (void)key; + TODO(); } void ConnectionFromClient::ensure_connection(URL::URL const& url, ::RequestServer::CacheLevel const& cache_level) @@ -303,10 +429,8 @@ void ConnectionFromClient::ensure_connection(URL::URL const& url, ::RequestServe return; } - enqueue(EnsureConnection { - .url = url, - .cache_level = cache_level, - }); + (void)cache_level; + dbgln("FIXME: EnsureConnection: Pre-connect to {}", url); } static i32 s_next_websocket_id = 1; @@ -326,19 +450,15 @@ Messages::RequestServer::WebsocketConnectResponse ConnectionFromClient::websocke auto id = ++s_next_websocket_id; auto connection = WebSocket::WebSocket::create(move(connection_info)); connection->on_open = [this, id]() { - auto lock = Threading::MutexLocker(m_ipc_mutex); async_websocket_connected(id); }; connection->on_message = [this, id](auto message) { - auto lock = Threading::MutexLocker(m_ipc_mutex); async_websocket_received(id, message.is_text(), message.data()); }; connection->on_error = [this, id](auto message) { - auto lock = Threading::MutexLocker(m_ipc_mutex); async_websocket_errored(id, (i32)message); }; connection->on_close = [this, id](u16 code, ByteString reason, bool was_clean) { - auto lock = Threading::MutexLocker(m_ipc_mutex); async_websocket_closed(id, code, move(reason), was_clean); }; diff --git a/Userland/Services/RequestServer/ConnectionFromClient.h b/Userland/Services/RequestServer/ConnectionFromClient.h index 3aaae5d6262..ae2a0960fdc 100644 --- a/Userland/Services/RequestServer/ConnectionFromClient.h +++ b/Userland/Services/RequestServer/ConnectionFromClient.h @@ -32,23 +32,6 @@ public: void did_progress_request(Badge, Request&); void did_request_certificates(Badge, Request&); - struct StartRequest { - i32 request_id; - ByteString method; - URL::URL url; - HTTP::HeaderMap request_headers; - ByteBuffer request_body; - Core::ProxyData proxy_data; - }; - - struct EnsureConnection { - URL::URL url; - CacheLevel cache_level; - }; - - using Work = Variant; - void worker_do_work(Work); - private: explicit ConnectionFromClient(NonnullOwnPtr); @@ -68,12 +51,23 @@ private: virtual void dump_connection_info() override; - Threading::MutexProtected>> m_requests; HashMap> m_websockets; - void enqueue(Work); + struct ActiveRequest; + friend struct ActiveRequest; - Threading::Mutex m_ipc_mutex; + static int on_socket_callback(void*, int sockfd, int what, void* user_data, void*); + static int on_timeout_callback(void*, long timeout_ms, void* user_data); + static size_t on_header_received(void* buffer, size_t size, size_t nmemb, void* user_data); + static size_t on_data_received(void* buffer, size_t size, size_t nmemb, void* user_data); + + HashMap> m_active_requests; + + void check_active_requests(); + void* m_curl_multi { nullptr }; + RefPtr m_timer; + HashMap> m_read_notifiers; + HashMap> m_write_notifiers; }; } diff --git a/vcpkg.json b/vcpkg.json index f9976728b37..9e66254c654 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,6 +1,12 @@ { "builtin-baseline": "3508985146f1b1d248c67ead13f8f54be5b4f5da", "dependencies": [ + { + "name": "curl", + "features": [ + "brotli", "http2" + ] + }, { "name": "fontconfig", "platform": "linux | freebsd | openbsd | osx"