LibCore+LibHTTP: Move out the HTTP handler and add HTTPS

Author: https://github.com/alimpfard
Commit: https://github.com/SerenityOS/serenity/commit/7670e5ccf0d
Pull-request: https://github.com/SerenityOS/serenity/pull/1661
Reviewed-by: https://github.com/Dexesttp
Reviewed-by: https://github.com/awesomekling
Reviewed-by: https://github.com/itamar8910

Applications/Browser/Tab.h

@@ -30,6 +30,7 @@
 #include <AK/URL.h>
 #include <LibGUI/Widget.h>
 #include <LibWeb/Forward.h>
+#include <LibHTTP/HttpJob.h>
 
 namespace Browser {
 

Libraries/LibCore/Forward.h

@@ -36,9 +36,6 @@ class ElapsedTime;
 class Event;
 class EventLoop;
 class File;
-class HttpJob;
-class HttpRequest;
-class HttpResponse;
 class IODevice;
 class LocalServer;
 class LocalSocket;

Libraries/LibCore/Makefile

@@ -9,9 +9,6 @@ OBJS = \
     EventLoop.o \
     File.o \
     Gzip.o \
-    HttpJob.o \
-    HttpRequest.o \
-    HttpResponse.o \
     IODevice.o \
     LocalServer.o \
     LocalSocket.o \

Libraries/LibGUI/ToolBarContainer.h

@@ -28,6 +28,7 @@
 
 #include <LibGUI/Frame.h>
 #include <LibGUI/ToolBar.h>
+#include <LibHTTP/HttpJob.h>
 
 namespace GUI {
 

Libraries/LibHTTP/Forward.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+namespace HTTP {
+
+class HttpRequest;
+class HttpResponse;
+class HttpJob;
+class HttpsJob;
+
+}

Libraries/LibHTTP/HttpJob.cpp

@@ -25,15 +25,15 @@
  */
 
 #include <LibCore/Gzip.h>
-#include <LibCore/HttpJob.h>
-#include <LibCore/HttpResponse.h>
 #include <LibCore/TCPSocket.h>
+#include <LibHTTP/HttpJob.h>
+#include <LibHTTP/HttpResponse.h>
 #include <stdio.h>
 #include <unistd.h>
 
-//#define CHTTPJOB_DEBUG
+//#define HTTPJOB_DEBUG
 
-namespace Core {
+namespace HTTP {
 
 static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& content_encoding)
 {
@@ -42,7 +42,7 @@ static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& c
 #endif
 
     if (content_encoding == "gzip") {
-        if (!Gzip::is_compressed(buf)) {
+        if (!Core::Gzip::is_compressed(buf)) {
             dbg() << "HttpJob::handle_content_encoding: buf is not gzip compressed!";
         }
 
@@ -50,7 +50,7 @@ static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& c
         dbg() << "HttpJob::handle_content_encoding: buf is gzip compressed!";
 #endif
 
-        auto uncompressed = Gzip::decompress(buf);
+        auto uncompressed = Core::Gzip::decompress(buf);
         if (!uncompressed.has_value()) {
             dbg() << "HttpJob::handle_content_encoding: Gzip::decompress() failed. Returning original buffer.";
             return buf;
@@ -87,7 +87,7 @@ void HttpJob::on_socket_connected()
 
     bool success = m_socket->send(raw_request);
     if (!success)
-        return deferred_invoke([this](auto&) { did_fail(NetworkJob::Error::TransmissionFailed); });
+        return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
 
     m_socket->on_ready_to_read = [&] {
         if (is_cancelled())
@@ -98,18 +98,18 @@ void HttpJob::on_socket_connected()
             auto line = m_socket->read_line(PAGE_SIZE);
             if (line.is_null()) {
                 fprintf(stderr, "HttpJob: Expected HTTP status\n");
-                return deferred_invoke([this](auto&) { did_fail(NetworkJob::Error::TransmissionFailed); });
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
             }
             auto parts = String::copy(line, Chomp).split(' ');
             if (parts.size() < 3) {
                 fprintf(stderr, "HttpJob: Expected 3-part HTTP status, got '%s'\n", line.data());
-                return deferred_invoke([this](auto&) { did_fail(NetworkJob::Error::ProtocolFailed); });
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
             }
             bool ok;
             m_code = parts[1].to_uint(ok);
             if (!ok) {
                 fprintf(stderr, "HttpJob: Expected numeric HTTP status\n");
-                return deferred_invoke([this](auto&) { did_fail(NetworkJob::Error::ProtocolFailed); });
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
             }
             m_state = State::InHeaders;
             return;
@@ -120,7 +120,7 @@ void HttpJob::on_socket_connected()
             auto line = m_socket->read_line(PAGE_SIZE);
             if (line.is_null()) {
                 fprintf(stderr, "HttpJob: Expected HTTP header\n");
-                return did_fail(NetworkJob::Error::ProtocolFailed);
+                return did_fail(Core::NetworkJob::Error::ProtocolFailed);
             }
             auto chomped_line = String::copy(line, Chomp);
             if (chomped_line.is_empty()) {
@@ -130,12 +130,12 @@ void HttpJob::on_socket_connected()
             auto parts = chomped_line.split(':');
             if (parts.is_empty()) {
                 fprintf(stderr, "HttpJob: Expected HTTP header with key/value\n");
-                return deferred_invoke([this](auto&) { did_fail(NetworkJob::Error::ProtocolFailed); });
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
             }
             auto name = parts[0];
             if (chomped_line.length() < name.length() + 2) {
                 fprintf(stderr, "HttpJob: Malformed HTTP header: '%s' (%zu)\n", chomped_line.characters(), chomped_line.length());
-                return deferred_invoke([this](auto&) { did_fail(NetworkJob::Error::ProtocolFailed); });
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
             }
             auto value = chomped_line.substring(name.length() + 2, chomped_line.length() - name.length() - 2);
             m_headers.set(name, value);
@@ -150,7 +150,7 @@ void HttpJob::on_socket_connected()
         if (!payload) {
             if (m_socket->eof())
                 return finish_up();
-            return deferred_invoke([this](auto&) { did_fail(NetworkJob::Error::ProtocolFailed); });
+            return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
         }
         m_received_buffers.append(payload);
         m_received_size += payload.size();
@@ -189,7 +189,7 @@ void HttpJob::finish_up()
 void HttpJob::start()
 {
     ASSERT(!m_socket);
-    m_socket = TCPSocket::construct(this);
+    m_socket = Core::TCPSocket::construct(this);
     m_socket->on_connected = [this] {
 #ifdef CHTTPJOB_DEBUG
         dbg() << "HttpJob: on_connected callback";
@@ -199,7 +199,7 @@ void HttpJob::start()
     bool success = m_socket->connect(m_request.url().host(), m_request.url().port());
     if (!success) {
         deferred_invoke([this](auto&) {
-            return did_fail(NetworkJob::Error::ConnectionFailed);
+            return did_fail(Core::NetworkJob::Error::ConnectionFailed);
         });
     }
 }

Libraries/LibHTTP/HttpJob.h

@@ -27,13 +27,14 @@
 #pragma once
 
 #include <AK/HashMap.h>
-#include <LibCore/HttpRequest.h>
-#include <LibCore/HttpResponse.h>
 #include <LibCore/NetworkJob.h>
+#include <LibCore/TCPSocket.h>
+#include <LibHTTP/HttpRequest.h>
+#include <LibHTTP/HttpResponse.h>
 
-namespace Core {
+namespace HTTP {
 
-class HttpJob final : public NetworkJob {
+class HttpJob final : public Core::NetworkJob {
     C_OBJECT(HttpJob)
 public:
     explicit HttpJob(const HttpRequest&);
@@ -42,8 +43,8 @@ public:
     virtual void start() override;
     virtual void shutdown() override;
 
-    HttpResponse* response() { return static_cast<HttpResponse*>(NetworkJob::response()); }
-    const HttpResponse* response() const { return static_cast<const HttpResponse*>(NetworkJob::response()); }
+    HttpResponse* response() { return static_cast<HttpResponse*>(Core::NetworkJob::response()); }
+    const HttpResponse* response() const { return static_cast<const HttpResponse*>(Core::NetworkJob::response()); }
 
 private:
     void on_socket_connected();
@@ -57,7 +58,7 @@ private:
     };
 
     HttpRequest m_request;
-    RefPtr<TCPSocket> m_socket;
+    RefPtr<Core::Socket> m_socket;
     State m_state { State::InStatus };
     int m_code { -1 };
     HashMap<String, String> m_headers;

Libraries/LibHTTP/HttpRequest.cpp

@@ -25,10 +25,10 @@
  */
 
 #include <AK/StringBuilder.h>
-#include <LibCore/HttpJob.h>
-#include <LibCore/HttpRequest.h>
+#include <LibHTTP/HttpJob.h>
+#include <LibHTTP/HttpRequest.h>
 
-namespace Core {
+namespace HTTP {
 
 HttpRequest::HttpRequest()
 {
@@ -38,7 +38,7 @@ HttpRequest::~HttpRequest()
 {
 }
 
-RefPtr<NetworkJob> HttpRequest::schedule()
+RefPtr<Core::NetworkJob> HttpRequest::schedule()
 {
     auto job = HttpJob::construct(*this);
     job->start();
@@ -65,9 +65,9 @@ ByteBuffer HttpRequest::to_raw_request() const
     builder.append(method_name());
     builder.append(' ');
     builder.append(m_url.path());
-    builder.append(" HTTP/1.0\r\nHost: ");
+    builder.append(" HTTP/1.1\r\nHost: ");
     builder.append(m_url.host());
-    builder.append("\r\n\r\n");
+    builder.append("\r\nConnection: close\r\n\r\n");
     return builder.to_byte_buffer();
 }
 

Libraries/LibHTTP/HttpRequest.h

@@ -32,7 +32,7 @@
 #include <AK/Vector.h>
 #include <LibCore/Forward.h>
 
-namespace Core {
+namespace HTTP {
 
 class HttpRequest {
 public:
@@ -63,7 +63,7 @@ public:
     String method_name() const;
     ByteBuffer to_raw_request() const;
 
-    RefPtr<NetworkJob> schedule();
+    RefPtr<Core::NetworkJob> schedule();
 
     static Optional<HttpRequest> from_raw_request(const ByteBuffer&);
 

Libraries/LibHTTP/HttpResponse.cpp

@@ -24,12 +24,12 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <LibCore/HttpResponse.h>
+#include <LibHTTP/HttpResponse.h>
 
-namespace Core {
+namespace HTTP {
 
 HttpResponse::HttpResponse(int code, HashMap<String, String>&& headers, ByteBuffer&& payload)
-    : NetworkResponse(move(payload))
+    : Core::NetworkResponse(move(payload))
     , m_code(code)
     , m_headers(move(headers))
 {

Libraries/LibHTTP/HttpResponse.h

@@ -30,9 +30,9 @@
 #include <AK/String.h>
 #include <LibCore/NetworkResponse.h>
 
-namespace Core {
+namespace HTTP {
 
-class HttpResponse : public NetworkResponse {
+class HttpResponse : public Core::NetworkResponse {
 public:
     virtual ~HttpResponse() override;
     static NonnullRefPtr<HttpResponse> create(int code, HashMap<String, String>&& headers, ByteBuffer&& payload)

Libraries/LibHTTP/HttpsJob.cpp

@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <LibCore/EventLoop.h>
+#include <LibCore/Gzip.h>
+#include <LibHTTP/HttpResponse.h>
+#include <LibHTTP/HttpsJob.h>
+#include <LibTLS/TLSv12.h>
+#include <stdio.h>
+#include <unistd.h>
+
+//#define HTTPJOB_DEBUG
+
+namespace HTTP {
+
+static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& content_encoding)
+{
+#ifdef CHTTPJOB_DEBUG
+    dbg() << "HttpsJob::handle_content_encoding: buf has content_encoding = " << content_encoding;
+#endif
+
+    if (content_encoding == "gzip") {
+        if (!Core::Gzip::is_compressed(buf)) {
+            dbg() << "HttpsJob::handle_content_encoding: buf is not gzip compressed!";
+        }
+
+#ifdef CHTTPJOB_DEBUG
+        dbg() << "HttpsJob::handle_content_encoding: buf is gzip compressed!";
+#endif
+
+        auto uncompressed = Core::Gzip::decompress(buf);
+        if (!uncompressed.has_value()) {
+            dbg() << "HttpsJob::handle_content_encoding: Gzip::decompress() failed. Returning original buffer.";
+            return buf;
+        }
+
+#ifdef CHTTPJOB_DEBUG
+        dbg() << "HttpsJob::handle_content_encoding: Gzip::decompress() successful.\n"
+              << "  Input size = " << buf.size() << "\n"
+              << "  Output size = " << uncompressed.value().size();
+#endif
+
+        return uncompressed.value();
+    }
+
+    return buf;
+}
+
+HttpsJob::HttpsJob(const HttpRequest& request)
+    : m_request(request)
+{
+}
+
+HttpsJob::~HttpsJob()
+{
+    m_socket = nullptr;
+}
+
+void HttpsJob::on_socket_connected()
+{
+
+    m_socket->on_tls_ready_to_write = [&](TLS::TLSv12& tls) {
+        if (m_sent_data)
+            return;
+        m_sent_data = true;
+        auto raw_request = m_request.to_raw_request();
+#if 0
+        dbg() << "HttpsJob: raw_request:";
+        dbg() << String::copy(raw_request).characters();
+#endif
+        bool success = tls.write(raw_request);
+        if (!success)
+            deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
+    };
+
+    m_socket->on_tls_ready_to_read = [&](TLS::TLSv12& tls) {
+        dbg() << " ON TLS READY TO READ: " << (u16)m_state;
+        if (is_cancelled())
+            return;
+        if (m_state == State::InStatus) {
+            if (!tls.can_read_line()) {
+                dbg() << " cannot read line";
+                return;
+            }
+            auto line = tls.read_line(PAGE_SIZE);
+            if (line.is_null()) {
+                fprintf(stderr, "HttpsJob: Expected HTTP status\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
+            }
+            auto parts = String::copy(line, Chomp).split(' ');
+            if (parts.size() < 3) {
+                fprintf(stderr, "HttpsJob: Expected 3-part HTTP status, got '%s'\n", line.data());
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            bool ok;
+            m_code = parts[1].to_uint(ok);
+            if (!ok) {
+                fprintf(stderr, "HttpsJob: Expected numeric HTTP status\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            m_state = State::InHeaders;
+            return;
+        }
+        if (m_state == State::InHeaders) {
+            if (!tls.can_read_line())
+                return;
+            auto line = tls.read_line(PAGE_SIZE);
+            if (line.is_null()) {
+                fprintf(stderr, "HttpsJob: Expected HTTP header\n");
+                return did_fail(Core::NetworkJob::Error::ProtocolFailed);
+            }
+            auto chomped_line = String::copy(line, Chomp);
+            if (chomped_line.is_empty()) {
+                m_state = State::InBody;
+                return;
+            }
+            auto parts = chomped_line.split(':');
+            if (parts.is_empty()) {
+                fprintf(stderr, "HttpsJob: Expected HTTP header with key/value\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            auto name = parts[0];
+            if (chomped_line.length() < name.length() + 2) {
+                fprintf(stderr, "HttpsJob: Malformed HTTP header: '%s' (%zu)\n", chomped_line.characters(), chomped_line.length());
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            auto value = chomped_line.substring(name.length() + 2, chomped_line.length() - name.length() - 2);
+            m_headers.set(name, value);
+#ifdef CHTTPJOB_DEBUG
+            dbg() << "HttpsJob: [" << name << "] = '" << value << "'";
+#endif
+            return;
+        }
+        ASSERT(m_state == State::InBody);
+        ASSERT(tls.can_read());
+        auto payload = tls.read(64 * KB);
+        if (!payload) {
+            if (tls.eof())
+                return finish_up();
+            return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+        }
+        dbg() << "Read payload, " << payload.size() << " bytes";
+        m_received_buffers.append(payload);
+        m_received_size += payload.size();
+
+        auto content_length_header = m_headers.get("Content-Length");
+        if (content_length_header.has_value()) {
+            dbg() << "content length is " << content_length_header.value() << ", we have " << m_received_size;
+            bool ok;
+            if (m_received_size >= content_length_header.value().to_uint(ok) && ok)
+                finish_up();
+        } else {
+            // no content-length, assume closed connection
+            finish_up();
+        }
+    };
+}
+
+void HttpsJob::finish_up()
+{
+    m_state = State::Finished;
+    auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
+    u8* flat_ptr = flattened_buffer.data();
+    for (auto& received_buffer : m_received_buffers) {
+        memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
+        flat_ptr += received_buffer.size();
+    }
+    m_received_buffers.clear();
+
+    auto content_encoding = m_headers.get("Content-Encoding");
+    if (content_encoding.has_value()) {
+        flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
+    }
+
+    auto response = HttpResponse::create(m_code, move(m_headers), move(flattened_buffer));
+    deferred_invoke([this, response](auto&) {
+        did_finish(move(response));
+    });
+}
+
+void HttpsJob::start()
+{
+    ASSERT(!m_socket);
+    m_socket = TLS::TLSv12::construct(this);
+    m_socket->on_tls_connected = [this] {
+#ifdef CHTTPJOB_DEBUG
+        dbg() << "HttpsJob: on_connected callback";
+#endif
+        on_socket_connected();
+    };
+    m_socket->on_tls_error = [&](auto) {
+        finish_up();
+    };
+    m_socket->on_tls_finished = [&] {
+        finish_up();
+    };
+    bool success = ((TLS::TLSv12&)*m_socket).connect(m_request.url().host(), m_request.url().port());
+    if (!success) {
+        deferred_invoke([this](auto&) {
+            return did_fail(Core::NetworkJob::Error::ConnectionFailed);
+        });
+    }
+}
+
+void HttpsJob::shutdown()
+{
+    if (!m_socket)
+        return;
+    m_socket->on_tls_ready_to_read = nullptr;
+    m_socket->on_tls_connected = nullptr;
+    remove_child(*m_socket);
+    m_socket = nullptr;
+}
+}

Libraries/LibHTTP/HttpsJob.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+#include <LibCore/NetworkJob.h>
+#include <LibHTTP/HttpRequest.h>
+#include <LibHTTP/HttpResponse.h>
+#include <LibTLS/TLSv12.h>
+
+namespace HTTP {
+
+class HttpsJob final : public Core::NetworkJob {
+    C_OBJECT(HttpsJob)
+public:
+    explicit HttpsJob(const HttpRequest&);
+    virtual ~HttpsJob() override;
+
+    virtual void start() override;
+    virtual void shutdown() override;
+
+    HttpResponse* response() { return static_cast<HttpResponse*>(Core::NetworkJob::response()); }
+    const HttpResponse* response() const { return static_cast<const HttpResponse*>(Core::NetworkJob::response()); }
+
+private:
+    RefPtr<TLS::TLSv12> construct_socket() { return TLS::TLSv12::construct(this); }
+    void on_socket_connected();
+    void finish_up();
+
+    enum class State {
+        InStatus,
+        InHeaders,
+        InBody,
+        Finished,
+    };
+
+    HttpRequest m_request;
+    RefPtr<TLS::TLSv12> m_socket;
+    State m_state { State::InStatus };
+    int m_code { -1 };
+    HashMap<String, String> m_headers;
+    Vector<ByteBuffer> m_received_buffers;
+    size_t m_received_size { 0 };
+    bool m_sent_data { false };
+};
+
+}

Libraries/LibHTTP/Makefile

@@ -0,0 +1,18 @@
+OBJS = HttpResponse.o \
+	   HttpRequest.o \
+	   HttpJob.o \
+	   HttpsJob.o
+
+LIBRARY = libhttp.a
+
+LIB_DEPS = Core
+
+POST_LIBRARY_BUILD = $(QUIET) $(MAKE) install
+
+install:
+	mkdir -p $(SERENITY_BASE_DIR)/Root/usr/include/sys/
+	mkdir -p $(SERENITY_BASE_DIR)/Root/usr/lib/
+	cp *.h $(SERENITY_BASE_DIR)/Root/usr/include/
+	cp $(LIBRARY) $(SERENITY_BASE_DIR)/Root/usr/lib/
+
+include ../../Makefile.common

Libraries/LibWeb/ResourceLoader.cpp

@@ -105,7 +105,7 @@ void ResourceLoader::load(const URL& url, Function<void(const ByteBuffer&)> succ
         return;
     }
 
-    if (url.protocol() == "http") {
+    if (url.protocol() == "http" || url.protocol() == "https") {
         auto download = protocol_client().start_download(url.to_string());
         if (!download) {
             if (error_callback)

Servers/ProtocolServer/HttpDownload.cpp

@@ -24,11 +24,11 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <LibCore/HttpJob.h>
-#include <LibCore/HttpResponse.h>
+#include <LibHTTP/HttpJob.h>
+#include <LibHTTP/HttpResponse.h>
 #include <ProtocolServer/HttpDownload.h>
 
-HttpDownload::HttpDownload(PSClientConnection& client, NonnullRefPtr<Core::HttpJob>&& job)
+HttpDownload::HttpDownload(PSClientConnection& client, NonnullRefPtr<HTTP::HttpJob>&& job)
     : Download(client)
     , m_job(job)
 {
@@ -43,7 +43,7 @@ HttpDownload::~HttpDownload()
 {
 }
 
-NonnullRefPtr<HttpDownload> HttpDownload::create_with_job(Badge<HttpProtocol>, PSClientConnection& client, NonnullRefPtr<Core::HttpJob>&& job)
+NonnullRefPtr<HttpDownload> HttpDownload::create_with_job(Badge<HttpProtocol>, PSClientConnection& client, NonnullRefPtr<HTTP::HttpJob>&& job)
 {
     return adopt(*new HttpDownload(client, move(job)));
 }

Servers/ProtocolServer/HttpDownload.h

@@ -28,6 +28,7 @@
 
 #include <AK/Badge.h>
 #include <LibCore/Forward.h>
+#include <LibHTTP/HttpJob.h>
 #include <ProtocolServer/Download.h>
 
 class HttpProtocol;
@@ -35,10 +36,10 @@ class HttpProtocol;
 class HttpDownload final : public Download {
 public:
     virtual ~HttpDownload() override;
-    static NonnullRefPtr<HttpDownload> create_with_job(Badge<HttpProtocol>, PSClientConnection&, NonnullRefPtr<Core::HttpJob>&&);
+    static NonnullRefPtr<HttpDownload> create_with_job(Badge<HttpProtocol>, PSClientConnection&, NonnullRefPtr<HTTP::HttpJob>&&);
 
 private:
-    explicit HttpDownload(PSClientConnection&, NonnullRefPtr<Core::HttpJob>&&);
+    explicit HttpDownload(PSClientConnection&, NonnullRefPtr<HTTP::HttpJob>&&);
 
-    NonnullRefPtr<Core::HttpJob> m_job;
+    NonnullRefPtr<HTTP::HttpJob> m_job;
 };

Servers/ProtocolServer/HttpProtocol.cpp

@@ -24,8 +24,8 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <LibCore/HttpJob.h>
-#include <LibCore/HttpRequest.h>
+#include <LibHTTP/HttpJob.h>
+#include <LibHTTP/HttpRequest.h>
 #include <ProtocolServer/HttpDownload.h>
 #include <ProtocolServer/HttpProtocol.h>
 
@@ -40,11 +40,11 @@ HttpProtocol::~HttpProtocol()
 
 RefPtr<Download> HttpProtocol::start_download(PSClientConnection& client, const URL& url)
 {
-    Core::HttpRequest request;
-    request.set_method(Core::HttpRequest::Method::GET);
+    HTTP::HttpRequest request;
+    request.set_method(HTTP::HttpRequest::Method::GET);
     request.set_url(url);
     auto job = request.schedule();
     if (!job)
         return nullptr;
-    return HttpDownload::create_with_job({}, client, (Core::HttpJob&)*job);
+    return HttpDownload::create_with_job({}, client, (HTTP::HttpJob&)*job);
 }

Servers/ProtocolServer/HttpsDownload.cpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <LibHTTP/HttpResponse.h>
+#include <LibHTTP/HttpsJob.h>
+#include <ProtocolServer/HttpsDownload.h>
+
+HttpsDownload::HttpsDownload(PSClientConnection& client, NonnullRefPtr<HTTP::HttpsJob>&& job)
+    : Download(client)
+    , m_job(job)
+{
+    m_job->on_finish = [this](bool success) {
+        if (m_job->response())
+            set_payload(m_job->response()->payload());
+        did_finish(success);
+    };
+}
+
+HttpsDownload::~HttpsDownload()
+{
+}
+
+NonnullRefPtr<HttpsDownload> HttpsDownload::create_with_job(Badge<HttpsProtocol>, PSClientConnection& client, NonnullRefPtr<HTTP::HttpsJob>&& job)
+{
+    return adopt(*new HttpsDownload(client, move(job)));
+}

Servers/ProtocolServer/HttpsDownload.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <AK/Badge.h>
+#include <LibCore/Forward.h>
+#include <LibHTTP/HttpsJob.h>
+#include <ProtocolServer/Download.h>
+
+class HttpsProtocol;
+
+class HttpsDownload final : public Download {
+public:
+    virtual ~HttpsDownload() override;
+    static NonnullRefPtr<HttpsDownload> create_with_job(Badge<HttpsProtocol>, PSClientConnection&, NonnullRefPtr<HTTP::HttpsJob>&&);
+
+private:
+    explicit HttpsDownload(PSClientConnection&, NonnullRefPtr<HTTP::HttpsJob>&&);
+
+    NonnullRefPtr<HTTP::HttpsJob> m_job;
+};

Servers/ProtocolServer/HttpsProtocol.cpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2018-2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <LibHTTP/HttpRequest.h>
+#include <LibHTTP/HttpsJob.h>
+#include <ProtocolServer/HttpsDownload.h>
+#include <ProtocolServer/HttpsProtocol.h>
+
+HttpsProtocol::HttpsProtocol()
+    : Protocol("https")
+{
+}
+
+HttpsProtocol::~HttpsProtocol()
+{
+}
+
+RefPtr<Download> HttpsProtocol::start_download(PSClientConnection& client, const URL& url)
+{
+    HTTP::HttpRequest request;
+    request.set_method(HTTP::HttpRequest::Method::GET);
+    request.set_url(url);
+    auto job = HTTP::HttpsJob::construct(request);
+    job->start();
+    return HttpsDownload::create_with_job({}, client, (HTTP::HttpsJob&)*job);
+}

Servers/ProtocolServer/HttpsProtocol.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2018-2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <ProtocolServer/Protocol.h>
+
+class HttpsProtocol final : public Protocol {
+public:
+    HttpsProtocol();
+    virtual ~HttpsProtocol() override;
+
+    virtual RefPtr<Download> start_download(PSClientConnection&, const URL&) override;
+};

Servers/ProtocolServer/Makefile

@@ -4,11 +4,13 @@ OBJS = \
     Download.o \
     HttpProtocol.o \
     HttpDownload.o \
+    HttpsProtocol.o \
+    HttpsDownload.o \
     main.o
 
 PROGRAM = ProtocolServer
 
-LIB_DEPS = Core IPC
+LIB_DEPS = HTTP TLS Crypto Core IPC
 
 EXTRA_CLEAN = ProtocolServerEndpoint.h ProtocolClientEndpoint.h
 

Servers/ProtocolServer/main.cpp

@@ -28,6 +28,7 @@
 #include <LibCore/LocalServer.h>
 #include <LibIPC/ClientConnection.h>
 #include <ProtocolServer/HttpProtocol.h>
+#include <ProtocolServer/HttpsProtocol.h>
 #include <ProtocolServer/PSClientConnection.h>
 
 int main(int, char**)
@@ -43,6 +44,7 @@ int main(int, char**)
         return 1;
     }
     (void)*new HttpProtocol;
+    (void)*new HttpsProtocol;
     auto server = Core::LocalServer::construct();
     bool ok = server->take_over_from_system_server();
     ASSERT(ok);

Servers/WebServer/Client.cpp

@@ -30,7 +30,7 @@
 #include <LibCore/DateTime.h>
 #include <LibCore/DirIterator.h>
 #include <LibCore/File.h>
-#include <LibCore/HttpRequest.h>
+#include <LibHTTP/HttpRequest.h>
 #include <stdio.h>
 #include <sys/stat.h>
 #include <time.h>
@@ -67,7 +67,7 @@ void Client::start()
 
 void Client::handle_request(ByteBuffer raw_request)
 {
-    auto request_or_error = Core::HttpRequest::from_raw_request(raw_request);
+    auto request_or_error = HTTP::HttpRequest::from_raw_request(raw_request);
     if (!request_or_error.has_value())
         return;
     auto& request = request_or_error.value();
@@ -77,7 +77,7 @@ void Client::handle_request(ByteBuffer raw_request)
         dbg() << "    " << header.name << " => " << header.value;
     }
 
-    if (request.method() != Core::HttpRequest::Method::GET) {
+    if (request.method() != HTTP::HttpRequest::Method::GET) {
         send_error_response(403, "Forbidden, bro!", request);
         return;
     }
@@ -122,7 +122,7 @@ void Client::handle_request(ByteBuffer raw_request)
     send_response(file->read_all(), request);
 }
 
-void Client::send_response(StringView response, const Core::HttpRequest& request)
+void Client::send_response(StringView response, const HTTP::HttpRequest& request)
 {
     StringBuilder builder;
     builder.append("HTTP/1.0 200 OK\r\n");
@@ -136,7 +136,7 @@ void Client::send_response(StringView response, const Core::HttpRequest& request
     log_response(200, request);
 }
 
-void Client::send_redirect(StringView redirect_path, const Core::HttpRequest& request)
+void Client::send_redirect(StringView redirect_path, const HTTP::HttpRequest& request)
 {
     StringBuilder builder;
     builder.append("HTTP/1.0 301 Moved Permanently\r\n");
@@ -150,7 +150,7 @@ void Client::send_redirect(StringView redirect_path, const Core::HttpRequest& re
     log_response(301, request);
 }
 
-void Client::handle_directory_listing(const String& requested_path, const String& real_path, const Core::HttpRequest& request)
+void Client::handle_directory_listing(const String& requested_path, const String& real_path, const HTTP::HttpRequest& request)
 {
     StringBuilder builder;
 
@@ -203,7 +203,7 @@ void Client::handle_directory_listing(const String& requested_path, const String
     send_response(builder.to_string(), request);
 }
 
-void Client::send_error_response(unsigned code, const StringView& message, const Core::HttpRequest& request)
+void Client::send_error_response(unsigned code, const StringView& message, const HTTP::HttpRequest& request)
 {
     StringBuilder builder;
     builder.appendf("HTTP/1.0 %u ", code);
@@ -218,7 +218,7 @@ void Client::send_error_response(unsigned code, const StringView& message, const
     log_response(code, request);
 }
 
-void Client::log_response(unsigned code, const Core::HttpRequest& request)
+void Client::log_response(unsigned code, const HTTP::HttpRequest& request)
 {
     printf("%s :: %03u :: %s %s\n",
         Core::DateTime::now().to_string().characters(),

Servers/WebServer/Client.h

@@ -28,6 +28,7 @@
 
 #include <LibCore/Object.h>
 #include <LibCore/TCPSocket.h>
+#include <LibHTTP/Forward.h>
 
 namespace WebServer {
 
@@ -40,12 +41,12 @@ private:
     Client(NonnullRefPtr<Core::TCPSocket>, Core::Object* parent);
 
     void handle_request(ByteBuffer);
-    void send_response(StringView, const Core::HttpRequest&);
-    void send_redirect(StringView redirect, const Core::HttpRequest& request);
-    void send_error_response(unsigned code, const StringView& message, const Core::HttpRequest&);
+    void send_response(StringView, const HTTP::HttpRequest&);
+    void send_redirect(StringView redirect, const HTTP::HttpRequest& request);
+    void send_error_response(unsigned code, const StringView& message, const HTTP::HttpRequest&);
     void die();
-    void log_response(unsigned code, const Core::HttpRequest&);
-    void handle_directory_listing(const String& requested_path, const String& real_path, const Core::HttpRequest&);
+    void log_response(unsigned code, const HTTP::HttpRequest&);
+    void handle_directory_listing(const String& requested_path, const String& real_path, const HTTP::HttpRequest&);
 
     NonnullRefPtr<Core::TCPSocket> m_socket;
 };

Servers/WebServer/Makefile

@@ -4,6 +4,6 @@ OBJS = \
 
 PROGRAM = WebServer
 
-LIB_DEPS = Core
+LIB_DEPS = HTTP Core
 
 include ../../Makefile.common