LibIMAP: Support for the FETCH command (*mostly)

This commit doesn't include support for FETCH BODY, because it's a bit
big already. Rest assured, FETCH is the most complicated IMAP command,
and we'll go back to simple boring ones shortly.

Userland/Libraries/LibIMAP/Client.cpp

@@ -124,6 +124,10 @@ static ReadonlyBytes command_byte_buffer(CommandType command)
         return "LIST"sv.bytes();
     case CommandType::Select:
         return "SELECT"sv.bytes();
+    case CommandType::Fetch:
+        return "FETCH"sv.bytes();
+    case CommandType::UIDFetch:
+        return "UID FETCH"sv.bytes();
     }
     VERIFY_NOT_REACHED();
 }
@@ -177,6 +181,12 @@ RefPtr<Promise<Optional<SolidResponse>>> Client::list(StringView reference_name,
     return cast_promise<SolidResponse>(send_command(move(command)));
 }
 
+RefPtr<Promise<Optional<SolidResponse>>> Client::fetch(FetchCommand request, bool uid)
+{
+    auto command = Command { uid ? CommandType::UIDFetch : CommandType::Fetch, m_current_command, { request.serialize() } };
+    return cast_promise<SolidResponse>(send_command(move(command)));
+}
+
 RefPtr<Promise<Optional<Response>>> Client::send_simple_command(CommandType type)
 {
     auto command = Command { type, m_current_command, {} };

Userland/Libraries/LibIMAP/Client.h

@@ -24,6 +24,7 @@ public:
     RefPtr<Promise<Optional<SolidResponse>>> login(StringView username, StringView password);
     RefPtr<Promise<Optional<SolidResponse>>> list(StringView reference_name, StringView mailbox_name);
     RefPtr<Promise<Optional<SolidResponse>>> select(StringView string);
+    RefPtr<Promise<Optional<SolidResponse>>> fetch(FetchCommand request, bool uid);
     RefPtr<Promise<Optional<ContinueRequest>>> idle();
     RefPtr<Promise<Optional<SolidResponse>>> finish_idle();
 

Userland/Libraries/LibIMAP/Objects.cpp

@@ -8,4 +8,105 @@
 
 namespace IMAP {
 
+String Sequence::serialize() const
+{
+    if (start == end) {
+        return AK::String::formatted("{}", start);
+    } else {
+        auto start_char = start != -1 ? String::formatted("{}", start) : "*";
+        auto end_char = end != -1 ? String::formatted("{}", end) : "*";
+        return String::formatted("{}:{}", start_char, end_char);
+    }
+}
+
+String FetchCommand::DataItem::Section::serialize() const
+{
+    StringBuilder headers_builder;
+    switch (type) {
+    case SectionType::Header:
+        return "HEADER";
+    case SectionType::HeaderFields:
+    case SectionType::HeaderFieldsNot: {
+        if (type == SectionType::HeaderFields)
+            headers_builder.append("HEADER.FIELDS (");
+        else
+            headers_builder.append("HEADERS.FIELDS.NOT (");
+
+        bool first = true;
+        for (auto& field : headers.value()) {
+            if (!first)
+                headers_builder.append(" ");
+            headers_builder.append(field);
+            first = false;
+        }
+        headers_builder.append(")");
+        return headers_builder.build();
+    }
+    case SectionType::Text:
+        return "TEXT";
+    case SectionType::Parts: {
+        StringBuilder sb;
+        bool first = true;
+        for (int part : parts.value()) {
+            if (!first)
+                sb.append(".");
+            sb.appendff("{}", part);
+            first = false;
+        }
+        if (ends_with_mime) {
+            sb.append(".MIME");
+        }
+        return sb.build();
+    }
+    }
+    VERIFY_NOT_REACHED();
+}
+String FetchCommand::DataItem::serialize() const
+{
+    switch (type) {
+    case DataItemType::Envelope:
+        return "ENVELOPE";
+    case DataItemType::Flags:
+        return "FLAGS";
+    case DataItemType::InternalDate:
+        return "INTERNALDATE";
+    case DataItemType::UID:
+        return "UID";
+    case DataItemType::PeekBody:
+        TODO();
+    case DataItemType::BodySection:
+        StringBuilder sb;
+        sb.appendff("BODY[{}]", section.value().serialize());
+        if (partial_fetch) {
+            sb.appendff("<{}.{}>", start, octets);
+        }
+
+        return sb.build();
+    }
+    VERIFY_NOT_REACHED();
+}
+String FetchCommand::serialize()
+{
+    StringBuilder sequence_builder;
+    bool first = true;
+    for (auto& sequence : sequence_set) {
+        if (!first) {
+            sequence_builder.append(",");
+        }
+        sequence_builder.append(sequence.serialize());
+        first = false;
+    }
+
+    StringBuilder data_items_builder;
+    first = true;
+    for (auto& data_item : data_items) {
+        if (!first) {
+            data_items_builder.append(" ");
+        }
+        data_items_builder.append(data_item.serialize());
+        first = false;
+    }
+
+    return AK::String::formatted("{} ({})", sequence_builder.build(), data_items_builder.build());
+}
 }

Userland/Libraries/LibIMAP/Objects.h

@@ -18,12 +18,14 @@
 namespace IMAP {
 enum class CommandType {
     Capability,
+    Fetch,
     Idle,
     List,
     Login,
     Logout,
     Noop,
     Select,
+    UIDFetch,
 };
 
 enum class MailboxFlag : unsigned {
@@ -53,11 +55,72 @@ enum class ResponseType : unsigned {
     UIDValidity = 1u << 6,
     Unseen = 1u << 7,
     PermanentFlags = 1u << 8,
+    Fetch = 1u << 9,
     Bye = 1u << 13,
 };
 
+enum class FetchResponseType : unsigned {
+    Body = 1u << 1,
+    UID = 1u << 2,
+    InternalDate = 1u << 3,
+    Envelope = 1u << 4,
+    Flags = 1u << 5,
+};
+
 class Parser;
 
+// Set -1 for '*' i.e highest possible value.
+struct Sequence {
+    int start;
+    int end;
+
+    [[nodiscard]] String serialize() const;
+};
+
+struct FetchCommand {
+    enum class DataItemType {
+        Envelope,
+        Flags,
+        InternalDate,
+        UID,
+        PeekBody,
+        BodySection
+    };
+
+    struct DataItem {
+        enum class SectionType {
+            Header,
+            HeaderFields,
+            HeaderFieldsNot,
+            Text,
+            Parts
+        };
+        struct Section {
+            SectionType type;
+
+            Optional<Vector<int>> parts {};
+            bool ends_with_mime {};
+
+            Optional<Vector<String>> headers {};
+
+            [[nodiscard]] String serialize() const;
+        };
+
+        DataItemType type;
+
+        Optional<Section> section {};
+        bool partial_fetch { false };
+        int start { 0 };
+        int octets { 0 };
+
+        [[nodiscard]] String serialize() const;
+    };
+
+    Vector<Sequence> sequence_set;
+    Vector<DataItem> data_items;
+
+    String serialize();
+};
 struct Command {
 public:
     CommandType type;
@@ -77,6 +140,115 @@ struct ListItem {
     String name;
 };
 
+struct Address {
+    Optional<String> name;
+    Optional<String> source_route;
+    Optional<String> mailbox;
+    Optional<String> host;
+};
+struct Envelope {
+    Optional<String> date; // Format of date not specified.
+    Optional<String> subject;
+    Optional<Vector<Address>> from;
+    Optional<Vector<Address>> sender;
+    Optional<Vector<Address>> reply_to;
+    Optional<Vector<Address>> to;
+    Optional<Vector<Address>> cc;
+    Optional<Vector<Address>> bcc;
+    Optional<String> in_reply_to;
+    Optional<String> message_id;
+};
+
+class FetchResponseData {
+public:
+    [[nodiscard]] unsigned response_type() const
+    {
+        return m_response_type;
+    }
+
+    [[nodiscard]] bool contains_response_type(FetchResponseType response_type) const
+    {
+        return (static_cast<unsigned>(response_type) & m_response_type) != 0;
+    }
+
+    void add_response_type(FetchResponseType type)
+    {
+        m_response_type |= static_cast<unsigned>(type);
+    }
+
+    void add_body_data(FetchCommand::DataItem&& data_item, Optional<String>&& body)
+    {
+        add_response_type(FetchResponseType::Body);
+        m_bodies.append({ move(data_item), move(body) });
+    }
+
+    Vector<Tuple<FetchCommand::DataItem, Optional<String>>>& body_data()
+    {
+        VERIFY(contains_response_type(FetchResponseType::Body));
+        return m_bodies;
+    }
+
+    void set_uid(unsigned uid)
+    {
+        add_response_type(FetchResponseType::UID);
+        m_uid = uid;
+    }
+
+    [[nodiscard]] unsigned uid() const
+    {
+        VERIFY(contains_response_type(FetchResponseType::UID));
+        return m_uid;
+    }
+
+    void set_internal_date(Core::DateTime time)
+    {
+        add_response_type(FetchResponseType::InternalDate);
+        m_internal_date = time;
+    }
+
+    Core::DateTime& internal_date()
+    {
+        VERIFY(contains_response_type(FetchResponseType::InternalDate));
+        return m_internal_date;
+    }
+
+    void set_envelope(Envelope&& envelope)
+    {
+        add_response_type(FetchResponseType::Envelope);
+        m_envelope = move(envelope);
+    }
+
+    Envelope& envelope()
+    {
+        VERIFY(contains_response_type(FetchResponseType::Envelope));
+        return m_envelope;
+    }
+
+    void set_flags(Vector<String>&& flags)
+    {
+        add_response_type(FetchResponseType::Flags);
+        m_flags = move(flags);
+    }
+
+    Vector<String>& flags()
+    {
+        VERIFY(contains_response_type(FetchResponseType::Flags));
+        return m_flags;
+    }
+
+    FetchResponseData()
+    {
+    }
+
+private:
+    Vector<String> m_flags;
+    Vector<Tuple<FetchCommand::DataItem, Optional<String>>> m_bodies;
+    Core::DateTime m_internal_date;
+    Envelope m_envelope;
+    unsigned m_uid { 0 };
+    unsigned m_response_type { 0 };
+};
+
 class ResponseData {
 public:
     [[nodiscard]] unsigned response_type() const
@@ -212,6 +384,18 @@ public:
         return m_permanent_flags;
     }
 
+    void add_fetch_response(unsigned message, FetchResponseData&& data)
+    {
+        add_response_type(ResponseType::Fetch);
+        m_fetch_responses.append(Tuple<unsigned, FetchResponseData> { move(message), move(data) });
+    }
+
+    Vector<Tuple<unsigned, FetchResponseData>>& fetch_data()
+    {
+        VERIFY(contains_response_type(ResponseType::Fetch));
+        return m_fetch_responses;
+    }
+
     void set_bye(Optional<String> message)
     {
         add_response_type(ResponseType::Bye);
@@ -238,6 +422,7 @@ private:
     unsigned m_unseen {};
     Vector<String> m_permanent_flags;
     Vector<String> m_flags;
+    Vector<Tuple<unsigned, FetchResponseData>> m_fetch_responses;
     Optional<String> m_bye_message;
 };
 

Userland/Libraries/LibIMAP/Parser.cpp

@@ -137,6 +137,9 @@ void Parser::parse_untagged()
         } else if (data_type.matches("RECENT")) {
             m_response.data().set_recent(number.value());
             consume("\r\n");
+        } else if (data_type.matches("FETCH")) {
+            auto fetch_response = parse_fetch_response();
+            m_response.data().add_fetch_response(number.value(), move(fetch_response));
         }
         return;
     }
@@ -214,6 +217,87 @@ Optional<StringView> Parser::parse_nstring()
         return { parse_string() };
 }
 
+FetchResponseData Parser::parse_fetch_response()
+{
+    consume(" (");
+    auto fetch_response = FetchResponseData();
+
+    while (!try_consume(")")) {
+        auto data_item = parse_fetch_data_item();
+        switch (data_item.type) {
+        case FetchCommand::DataItemType::Envelope: {
+            consume(" (");
+            auto date = parse_nstring();
+            consume(" ");
+            auto subject = parse_nstring();
+            consume(" ");
+            auto from = parse_address_list();
+            consume(" ");
+            auto sender = parse_address_list();
+            consume(" ");
+            auto reply_to = parse_address_list();
+            consume(" ");
+            auto to = parse_address_list();
+            consume(" ");
+            auto cc = parse_address_list();
+            consume(" ");
+            auto bcc = parse_address_list();
+            consume(" ");
+            auto in_reply_to = parse_nstring();
+            consume(" ");
+            auto message_id = parse_nstring();
+            consume(")");
+            Envelope envelope = {
+                date.has_value() ? Optional<String>(date.value()) : Optional<String>(),
+                subject.has_value() ? Optional<String>(subject.value()) : Optional<String>(),
+                from,
+                sender,
+                reply_to,
+                to,
+                cc,
+                bcc,
+                in_reply_to.has_value() ? Optional<String>(in_reply_to.value()) : Optional<String>(),
+                message_id.has_value() ? Optional<String>(message_id.value()) : Optional<String>(),
+            };
+            fetch_response.set_envelope(move(envelope));
+            break;
+        }
+        case FetchCommand::DataItemType::Flags: {
+            consume(" ");
+            auto flags = parse_list(+[](StringView x) { return String(x); });
+            fetch_response.set_flags(move(flags));
+            break;
+        }
+        case FetchCommand::DataItemType::InternalDate: {
+            consume(" \"");
+            auto date_view = parse_while([](u8 x) { return x != '"'; });
+            consume("\"");
+            auto date = Core::DateTime::parse("%d-%b-%Y %H:%M:%S %z", date_view).value();
+            fetch_response.set_internal_date(date);
+            break;
+        }
+        case FetchCommand::DataItemType::UID: {
+            consume(" ");
+            fetch_response.set_uid(parse_number());
+            break;
+        }
+        case FetchCommand::DataItemType::PeekBody:
+            // Spec doesn't allow for this in a response.
+            m_parsing_failed = true;
+            break;
+        case FetchCommand::DataItemType::BodySection: {
+            auto body = parse_nstring();
+            fetch_response.add_body_data(move(data_item), body.has_value() ? body.release_value() : Optional<String>());
+            break;
+        }
+        }
+        if (!at_end() && m_buffer[position] != ')')
+            consume(" ");
+    }
+    consume("\r\n");
+    return fetch_response;
+}
+
 StringView Parser::parse_literal_string()
 {
     consume("{");
@@ -351,4 +435,126 @@ StringView Parser::parse_while(Function<bool(u8)> should_consume)
     return StringView(m_buffer.data() + position - chars, chars);
 }
 
+FetchCommand::DataItem Parser::parse_fetch_data_item()
+{
+    auto msg_attr = parse_while([](u8 x) { return is_ascii_alpha(x) != 0; });
+    if (msg_attr.equals_ignoring_case("BODY") && try_consume("[")) {
+        auto data_item = FetchCommand::DataItem {
+            .type = FetchCommand::DataItemType::BodySection,
+            .section = { {} }
+        };
+        auto section_type = parse_while([](u8 x) { return x != ']' && x != ' '; });
+        if (section_type.equals_ignoring_case("HEADER.FIELDS")) {
+            data_item.section->type = FetchCommand::DataItem::SectionType::HeaderFields;
+            data_item.section->headers = Vector<String>();
+            consume(" ");
+            auto headers = parse_list(+[](StringView x) { return x; });
+            for (auto& header : headers) {
+                data_item.section->headers->append(header);
+            }
+            consume("]");
+        } else if (section_type.equals_ignoring_case("HEADER.FIELDS.NOT")) {
+            data_item.section->type = FetchCommand::DataItem::SectionType::HeaderFieldsNot;
+            data_item.section->headers = Vector<String>();
+            consume(" (");
+            auto headers = parse_list(+[](StringView x) { return x; });
+            for (auto& header : headers) {
+                data_item.section->headers->append(header);
+            }
+            consume("]");
+        } else if (is_ascii_digit(section_type[0])) {
+            data_item.section->type = FetchCommand::DataItem::SectionType::Parts;
+            data_item.section->parts = Vector<int>();
+
+            while (!try_consume("]")) {
+                auto num = parse_number();
+                if (num != (unsigned)-1) {
+                    data_item.section->parts->append((int)num);
+                    continue;
+                }
+                auto atom = parse_atom();
+                if (atom.equals_ignoring_case("MIME")) {
+                    data_item.section->ends_with_mime = true;
+                    continue;
+                }
+            }
+        } else if (section_type.equals_ignoring_case("TEXT")) {
+            data_item.section->type = FetchCommand::DataItem::SectionType::Text;
+        } else if (section_type.equals_ignoring_case("HEADER")) {
+            data_item.section->type = FetchCommand::DataItem::SectionType::Header;
+        } else {
+            dbgln("Unmatched section type {}", section_type);
+            m_parsing_failed = true;
+        }
+        if (try_consume("<")) {
+            auto start = parse_number();
+            data_item.partial_fetch = true;
+            data_item.start = (int)start;
+            consume(">");
+        }
+        try_consume(" ");
+        return data_item;
+    } else if (msg_attr.equals_ignoring_case("FLAGS")) {
+        return FetchCommand::DataItem {
+            .type = FetchCommand::DataItemType::Flags
+        };
+    } else if (msg_attr.equals_ignoring_case("UID")) {
+        return FetchCommand::DataItem {
+            .type = FetchCommand::DataItemType::UID
+        };
+    } else if (msg_attr.equals_ignoring_case("INTERNALDATE")) {
+        return FetchCommand::DataItem {
+            .type = FetchCommand::DataItemType::InternalDate
+        };
+    } else if (msg_attr.equals_ignoring_case("ENVELOPE")) {
+        return FetchCommand::DataItem {
+            .type = FetchCommand::DataItemType::Envelope
+        };
+    } else {
+        dbgln("msg_attr not matched: {}", msg_attr);
+        m_parsing_failed = true;
+        return FetchCommand::DataItem {};
+    }
 }
+Optional<Vector<Address>> Parser::parse_address_list()
+{
+    if (try_consume("NIL"))
+        return {};
+
+    auto addresses = Vector<Address>();
+    consume("(");
+    while (!try_consume(")")) {
+        addresses.append(parse_address());
+        if (!at_end() && m_buffer[position] != ')')
+            consume(" ");
+    }
+    return { addresses };
+}
+
+Address Parser::parse_address()
+{
+    consume("(");
+    auto address = Address();
+    // I hate this so much. Why is there no Optional.map??
+    auto name = parse_nstring();
+    address.name = name.has_value() ? Optional<String>(name.value()) : Optional<String>();
+    consume(" ");
+    auto source_route = parse_nstring();
+    address.source_route = source_route.has_value() ? Optional<String>(source_route.value()) : Optional<String>();
+    consume(" ");
+    auto mailbox = parse_nstring();
+    address.mailbox = mailbox.has_value() ? Optional<String>(mailbox.value()) : Optional<String>();
+    consume(" ");
+    auto host = parse_nstring();
+    address.host = host.has_value() ? Optional<String>(host.value()) : Optional<String>();
+    consume(")");
+    return address;
+}
+StringView Parser::parse_astring()
+{
+    if (!at_end() && (m_buffer[position] == '{' || m_buffer[position] == '"'))
+        return parse_string();
+    else
+        return parse_atom();
+}
+}

Userland/Libraries/LibIMAP/Parser.h

@@ -60,7 +60,13 @@ private:
 
     ListItem parse_list_item();
 
+    FetchCommand::DataItem parse_fetch_data_item();
+
+    FetchResponseData parse_fetch_response();
+
     StringView parse_literal_string();
+    Optional<Vector<Address>> parse_address_list();
+    Address parse_address();
     StringView parse_astring();
 };
 }