ResourceLoader.cpp 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390
  1. /*
  2. * Copyright (c) 2018-2022, Andreas Kling <kling@serenityos.org>
  3. * Copyright (c) 2022, Dex♪ <dexes.ttp@gmail.com>
  4. *
  5. * SPDX-License-Identifier: BSD-2-Clause
  6. */
  7. #include <AK/Base64.h>
  8. #include <AK/Debug.h>
  9. #include <AK/JsonObject.h>
  10. #include <LibCore/ElapsedTimer.h>
  11. #include <LibWeb/Cookie/Cookie.h>
  12. #include <LibWeb/Cookie/ParsedCookie.h>
  13. #include <LibWeb/Loader/ContentFilter.h>
  14. #include <LibWeb/Loader/LoadRequest.h>
  15. #include <LibWeb/Loader/ProxyMappings.h>
  16. #include <LibWeb/Loader/Resource.h>
  17. #include <LibWeb/Loader/ResourceLoader.h>
  18. #include <LibWeb/Platform/EventLoopPlugin.h>
  19. #include <LibWeb/Platform/Timer.h>
  20. #ifdef AK_OS_SERENITY
  21. # include <serenity.h>
  22. #endif
  23. namespace Web {
  24. ResourceLoaderConnectorRequest::ResourceLoaderConnectorRequest() = default;
  25. ResourceLoaderConnectorRequest::~ResourceLoaderConnectorRequest() = default;
  26. ResourceLoaderConnector::ResourceLoaderConnector() = default;
  27. ResourceLoaderConnector::~ResourceLoaderConnector() = default;
  28. static RefPtr<ResourceLoader> s_resource_loader;
  29. void ResourceLoader::initialize(RefPtr<ResourceLoaderConnector> connector)
  30. {
  31. if (connector)
  32. s_resource_loader = ResourceLoader::try_create(connector.release_nonnull()).release_value_but_fixme_should_propagate_errors();
  33. }
  34. ResourceLoader& ResourceLoader::the()
  35. {
  36. if (!s_resource_loader) {
  37. dbgln("Web::ResourceLoader was not initialized");
  38. VERIFY_NOT_REACHED();
  39. }
  40. return *s_resource_loader;
  41. }
  42. ErrorOr<NonnullRefPtr<ResourceLoader>> ResourceLoader::try_create(NonnullRefPtr<ResourceLoaderConnector> connector)
  43. {
  44. return adopt_nonnull_ref_or_enomem(new (nothrow) ResourceLoader(move(connector)));
  45. }
  46. ResourceLoader::ResourceLoader(NonnullRefPtr<ResourceLoaderConnector> connector)
  47. : m_connector(move(connector))
  48. , m_user_agent(default_user_agent)
  49. {
  50. }
  51. void ResourceLoader::prefetch_dns(AK::URL const& url)
  52. {
  53. if (ContentFilter::the().is_filtered(url)) {
  54. dbgln("ResourceLoader: Refusing to prefetch DNS for '{}': \033[31;1mURL was filtered\033[0m", url);
  55. return;
  56. }
  57. m_connector->prefetch_dns(url);
  58. }
  59. void ResourceLoader::preconnect(AK::URL const& url)
  60. {
  61. if (ContentFilter::the().is_filtered(url)) {
  62. dbgln("ResourceLoader: Refusing to pre-connect to '{}': \033[31;1mURL was filtered\033[0m", url);
  63. return;
  64. }
  65. m_connector->preconnect(url);
  66. }
  67. static HashMap<LoadRequest, NonnullRefPtr<Resource>> s_resource_cache;
  68. RefPtr<Resource> ResourceLoader::load_resource(Resource::Type type, LoadRequest& request)
  69. {
  70. if (!request.is_valid())
  71. return nullptr;
  72. bool use_cache = request.url().scheme() != "file";
  73. if (use_cache) {
  74. auto it = s_resource_cache.find(request);
  75. if (it != s_resource_cache.end()) {
  76. if (it->value->type() != type) {
  77. dbgln("FIXME: Not using cached resource for {} since there's a type mismatch.", request.url());
  78. } else {
  79. dbgln_if(CACHE_DEBUG, "Reusing cached resource for: {}", request.url());
  80. return it->value;
  81. }
  82. }
  83. }
  84. auto resource = Resource::create({}, type, request);
  85. if (use_cache)
  86. s_resource_cache.set(request, resource);
  87. load(
  88. request,
  89. [=](auto data, auto& headers, auto status_code) {
  90. const_cast<Resource&>(*resource).did_load({}, data, headers, status_code);
  91. },
  92. [=](auto& error, auto status_code) {
  93. const_cast<Resource&>(*resource).did_fail({}, error, status_code);
  94. });
  95. return resource;
  96. }
  97. static DeprecatedString sanitized_url_for_logging(AK::URL const& url)
  98. {
  99. if (url.scheme() == "data"sv)
  100. return DeprecatedString::formatted("[data URL, mime-type={}, size={}]", url.data_mime_type(), url.data_payload().length());
  101. return url.to_deprecated_string();
  102. }
  103. static void emit_signpost(DeprecatedString const& message, int id)
  104. {
  105. #ifdef AK_OS_SERENITY
  106. auto string_id = perf_register_string(message.characters(), message.length());
  107. perf_event(PERF_EVENT_SIGNPOST, string_id, id);
  108. #else
  109. (void)message;
  110. (void)id;
  111. #endif
  112. }
  113. static void store_response_cookies(Page& page, AK::URL const& url, DeprecatedString const& cookies)
  114. {
  115. auto set_cookie_json_value = MUST(JsonValue::from_string(cookies));
  116. VERIFY(set_cookie_json_value.type() == JsonValue::Type::Array);
  117. for (auto const& set_cookie_entry : set_cookie_json_value.as_array().values()) {
  118. VERIFY(set_cookie_entry.type() == JsonValue::Type::String);
  119. auto cookie = Cookie::parse_cookie(set_cookie_entry.as_string());
  120. if (!cookie.has_value())
  121. continue;
  122. page.client().page_did_set_cookie(url, cookie.value(), Cookie::Source::Http); // FIXME: Determine cookie source correctly
  123. }
  124. }
  125. static size_t resource_id = 0;
  126. void ResourceLoader::load(LoadRequest& request, Function<void(ReadonlyBytes, HashMap<DeprecatedString, DeprecatedString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> status_code)> success_callback, Function<void(DeprecatedString const&, Optional<u32> status_code)> error_callback, Optional<u32> timeout, Function<void()> timeout_callback)
  127. {
  128. auto& url = request.url();
  129. request.start_timer();
  130. auto id = resource_id++;
  131. auto url_for_logging = sanitized_url_for_logging(url);
  132. emit_signpost(DeprecatedString::formatted("Starting load: {}", url_for_logging), id);
  133. dbgln("ResourceLoader: Starting load of: \"{}\"", url_for_logging);
  134. auto const log_success = [url_for_logging, id](auto const& request) {
  135. auto load_time_ms = request.load_time().to_milliseconds();
  136. emit_signpost(DeprecatedString::formatted("Finished load: {}", url_for_logging), id);
  137. dbgln("ResourceLoader: Finished load of: \"{}\", Duration: {}ms", url_for_logging, load_time_ms);
  138. };
  139. auto const log_failure = [url_for_logging, id](auto const& request, auto const& error_message) {
  140. auto load_time_ms = request.load_time().to_milliseconds();
  141. emit_signpost(DeprecatedString::formatted("Failed load: {}", url_for_logging), id);
  142. dbgln("ResourceLoader: Failed load of: \"{}\", \033[31;1mError: {}\033[0m, Duration: {}ms", url_for_logging, error_message, load_time_ms);
  143. };
  144. if (is_port_blocked(url.port_or_default())) {
  145. log_failure(request, DeprecatedString::formatted("The port #{} is blocked", url.port_or_default()));
  146. return;
  147. }
  148. if (ContentFilter::the().is_filtered(url)) {
  149. auto filter_message = "URL was filtered"sv;
  150. log_failure(request, filter_message);
  151. error_callback(filter_message, {});
  152. return;
  153. }
  154. if (url.scheme() == "about") {
  155. dbgln_if(SPAM_DEBUG, "Loading about: URL {}", url);
  156. log_success(request);
  157. HashMap<DeprecatedString, DeprecatedString, CaseInsensitiveStringTraits> response_headers;
  158. response_headers.set("Content-Type", "text/html; charset=UTF-8");
  159. Platform::EventLoopPlugin::the().deferred_invoke([success_callback = move(success_callback), response_headers = move(response_headers)] {
  160. success_callback(DeprecatedString::empty().to_byte_buffer(), response_headers, {});
  161. });
  162. return;
  163. }
  164. if (url.scheme() == "data") {
  165. dbgln_if(SPAM_DEBUG, "ResourceLoader loading a data URL with mime-type: '{}', base64={}, payload='{}'",
  166. url.data_mime_type(),
  167. url.data_payload_is_base64(),
  168. url.data_payload());
  169. ByteBuffer data;
  170. if (url.data_payload_is_base64()) {
  171. auto data_maybe = decode_base64(url.data_payload());
  172. if (data_maybe.is_error()) {
  173. auto error_message = data_maybe.error().string_literal();
  174. log_failure(request, error_message);
  175. error_callback(error_message, {});
  176. return;
  177. }
  178. data = data_maybe.value();
  179. } else {
  180. data = url.data_payload().to_byte_buffer();
  181. }
  182. log_success(request);
  183. Platform::EventLoopPlugin::the().deferred_invoke([data = move(data), success_callback = move(success_callback)] {
  184. success_callback(data, {}, {});
  185. });
  186. return;
  187. }
  188. if (url.scheme() == "file") {
  189. if (request.page().has_value())
  190. m_page = request.page().value();
  191. if (!m_page.has_value())
  192. return;
  193. FileRequest file_request(url.path(), [this, success_callback = move(success_callback), error_callback = move(error_callback), log_success, log_failure, request](ErrorOr<i32> file_or_error) {
  194. --m_pending_loads;
  195. if (on_load_counter_change)
  196. on_load_counter_change();
  197. if (file_or_error.is_error()) {
  198. log_failure(request, file_or_error.error());
  199. if (error_callback)
  200. error_callback(DeprecatedString::formatted("{}", file_or_error.error()), file_or_error.error().code());
  201. return;
  202. }
  203. auto const fd = file_or_error.value();
  204. auto maybe_file = Core::File::adopt_fd(fd, Core::File::OpenMode::Read);
  205. if (maybe_file.is_error()) {
  206. log_failure(request, maybe_file.error());
  207. if (error_callback)
  208. error_callback(DeprecatedString::formatted("{}", maybe_file.error()), maybe_file.error().code());
  209. return;
  210. }
  211. auto file = maybe_file.release_value();
  212. auto maybe_data = file->read_until_eof();
  213. if (maybe_data.is_error()) {
  214. log_failure(request, maybe_data.error());
  215. if (error_callback)
  216. error_callback(DeprecatedString::formatted("{}", maybe_data.error()), maybe_data.error().code());
  217. return;
  218. }
  219. auto data = maybe_data.release_value();
  220. log_success(request);
  221. success_callback(data, {}, {});
  222. });
  223. m_page->client().request_file(move(file_request));
  224. ++m_pending_loads;
  225. if (on_load_counter_change)
  226. on_load_counter_change();
  227. return;
  228. }
  229. if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "gemini") {
  230. auto proxy = ProxyMappings::the().proxy_for_url(url);
  231. HashMap<DeprecatedString, DeprecatedString> headers;
  232. headers.set("User-Agent", m_user_agent);
  233. headers.set("Accept-Encoding", "gzip, deflate, br");
  234. for (auto& it : request.headers()) {
  235. headers.set(it.key, it.value);
  236. }
  237. auto protocol_request = m_connector->start_request(request.method(), url, headers, request.body(), proxy);
  238. if (!protocol_request) {
  239. auto start_request_failure_msg = "Failed to initiate load"sv;
  240. log_failure(request, start_request_failure_msg);
  241. if (error_callback)
  242. error_callback(start_request_failure_msg, {});
  243. return;
  244. }
  245. if (timeout.has_value() && timeout.value() > 0) {
  246. auto timer = Platform::Timer::create_single_shot(timeout.value(), nullptr);
  247. timer->on_timeout = [timer, protocol_request, timeout_callback = move(timeout_callback)] {
  248. protocol_request->stop();
  249. if (timeout_callback)
  250. timeout_callback();
  251. };
  252. timer->start();
  253. }
  254. m_active_requests.set(*protocol_request);
  255. protocol_request->on_buffered_request_finish = [this, success_callback = move(success_callback), error_callback = move(error_callback), log_success, log_failure, request, &protocol_request = *protocol_request](bool success, auto, auto& response_headers, auto status_code, ReadonlyBytes payload) mutable {
  256. --m_pending_loads;
  257. if (on_load_counter_change)
  258. on_load_counter_change();
  259. if (request.page().has_value()) {
  260. if (auto set_cookie = response_headers.get("Set-Cookie"); set_cookie.has_value())
  261. store_response_cookies(request.page().value(), request.url(), *set_cookie);
  262. }
  263. if (!success || (status_code.has_value() && *status_code >= 400 && *status_code <= 599 && (payload.is_empty() || !request.is_main_resource()))) {
  264. StringBuilder error_builder;
  265. if (status_code.has_value())
  266. error_builder.appendff("Load failed: {}", *status_code);
  267. else
  268. error_builder.append("Load failed"sv);
  269. log_failure(request, error_builder.string_view());
  270. if (error_callback)
  271. error_callback(error_builder.to_deprecated_string(), status_code);
  272. return;
  273. }
  274. log_success(request);
  275. success_callback(payload, response_headers, status_code);
  276. Platform::EventLoopPlugin::the().deferred_invoke([this, &protocol_request] {
  277. m_active_requests.remove(protocol_request);
  278. });
  279. };
  280. protocol_request->set_should_buffer_all_input(true);
  281. protocol_request->on_certificate_requested = []() -> ResourceLoaderConnectorRequest::CertificateAndKey {
  282. return {};
  283. };
  284. ++m_pending_loads;
  285. if (on_load_counter_change)
  286. on_load_counter_change();
  287. return;
  288. }
  289. auto not_implemented_error = DeprecatedString::formatted("Protocol not implemented: {}", url.scheme());
  290. log_failure(request, not_implemented_error);
  291. if (error_callback)
  292. error_callback(not_implemented_error, {});
  293. }
  294. void ResourceLoader::load(const AK::URL& url, Function<void(ReadonlyBytes, HashMap<DeprecatedString, DeprecatedString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> status_code)> success_callback, Function<void(DeprecatedString const&, Optional<u32> status_code)> error_callback, Optional<u32> timeout, Function<void()> timeout_callback)
  295. {
  296. LoadRequest request;
  297. request.set_url(url);
  298. load(request, move(success_callback), move(error_callback), timeout, move(timeout_callback));
  299. }
  300. bool ResourceLoader::is_port_blocked(int port)
  301. {
  302. int ports[] { 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42,
  303. 43, 53, 77, 79, 87, 95, 101, 102, 103, 104, 109, 110, 111, 113,
  304. 115, 117, 119, 123, 135, 139, 143, 179, 389, 465, 512, 513, 514,
  305. 515, 526, 530, 531, 532, 540, 556, 563, 587, 601, 636, 993, 995,
  306. 2049, 3659, 4045, 6000, 6379, 6665, 6666, 6667, 6668, 6669, 9000 };
  307. for (auto blocked_port : ports)
  308. if (port == blocked_port)
  309. return true;
  310. return false;
  311. }
  312. void ResourceLoader::clear_cache()
  313. {
  314. dbgln_if(CACHE_DEBUG, "Clearing {} items from ResourceLoader cache", s_resource_cache.size());
  315. s_resource_cache.clear();
  316. }
  317. void ResourceLoader::evict_from_cache(LoadRequest const& request)
  318. {
  319. dbgln_if(CACHE_DEBUG, "Removing resource {} from cache", request.url());
  320. s_resource_cache.remove(request);
  321. }
  322. }