ImageDecoder+LibImageDecoder+LibWeb: Support animations in ImageDecoder

The ImageDecoder service now returns a list of image frames, each with
a duration value.

The code for in-process image decoding is removed from LibWeb, an all
image decode requests are sent out-of-process to ImageDecoder. :^)

This won't scale super well to very long and/or large animations, but
we can work on improving that separately. The main goal here is simply
to stop doing any image decoding inside LibWeb.

Fixes #5165.

Userland/Libraries/LibImageDecoderClient/Client.cpp

@@ -46,20 +46,30 @@ void Client::handle(const Messages::ImageDecoderClient::Dummy&)
 {
 }
 
-RefPtr<Gfx::Bitmap> Client::decode_image(const ByteBuffer& encoded_data)
+Optional<DecodedImage> Client::decode_image(const ByteBuffer& encoded_data)
 {
     if (encoded_data.is_empty())
-        return nullptr;
+        return {};
 
     auto encoded_buffer = Core::AnonymousBuffer::create_with_size(encoded_data.size());
     if (!encoded_buffer.is_valid()) {
         dbgln("Could not allocate encoded buffer");
-        return nullptr;
+        return {};
     }
 
     memcpy(encoded_buffer.data<void>(), encoded_data.data(), encoded_data.size());
     auto response = send_sync<Messages::ImageDecoderServer::DecodeImage>(move(encoded_buffer));
-    return response->bitmap().bitmap();
+
+    DecodedImage image;
+    image.is_animated = response->is_animated();
+    image.loop_count = response->loop_count();
+    image.frames.resize(response->bitmaps().size());
+    for (size_t i = 0; i < image.frames.size(); ++i) {
+        auto& frame = image.frames[i];
+        frame.bitmap = response->bitmaps()[i].bitmap();
+        frame.duration = response->durations()[i];
+    }
+    return move(image);
 }
 
 }

Userland/Libraries/LibImageDecoderClient/Client.h

@@ -33,6 +33,17 @@
 
 namespace ImageDecoderClient {
 
+struct Frame {
+    RefPtr<Gfx::Bitmap> bitmap;
+    u32 duration { 0 };
+};
+
+struct DecodedImage {
+    bool is_animated { false };
+    u32 loop_count { 0 };
+    Vector<Frame> frames;
+};
+
 class Client
     : public IPC::ServerConnection<ImageDecoderClientEndpoint, ImageDecoderServerEndpoint>
     , public ImageDecoderClientEndpoint {
@@ -41,7 +52,7 @@ class Client
 public:
     virtual void handshake() override;
 
-    RefPtr<Gfx::Bitmap> decode_image(const ByteBuffer&);
+    Optional<DecodedImage> decode_image(const ByteBuffer&);
 
 private:
     Client();

Userland/Libraries/LibWeb/HTML/HTMLImageElement.cpp

@@ -94,7 +94,7 @@ RefPtr<Layout::Node> HTMLImageElement::create_layout_node()
 
 const Gfx::Bitmap* HTMLImageElement::bitmap() const
 {
-    return m_image_loader.bitmap();
+    return m_image_loader.bitmap(m_image_loader.current_frame_index());
 }
 
 }

Userland/Libraries/LibWeb/Layout/ImageBox.cpp

@@ -114,7 +114,7 @@ void ImageBox::paint(PaintContext& context, PaintPhase phase)
             if (alt.is_empty())
                 alt = image_element.src();
             context.painter().draw_text(enclosing_int_rect(absolute_rect()), alt, Gfx::TextAlignment::Center, computed_values().color(), Gfx::TextElision::Right);
-        } else if (auto bitmap = m_image_loader.bitmap()) {
+        } else if (auto bitmap = m_image_loader.bitmap(m_image_loader.current_frame_index())) {
             context.painter().draw_scaled_bitmap(enclosing_int_rect(absolute_rect()), *bitmap, bitmap->rect());
         }
     }

Userland/Libraries/LibWeb/Loader/ImageLoader.cpp

@@ -80,15 +80,10 @@ void ImageLoader::resource_did_load()
         }
     }
 
-    if (resource()->should_decode_in_process()) {
-        auto& decoder = resource()->ensure_decoder();
-
-        if (decoder.is_animated() && decoder.frame_count() > 1) {
-            const auto& first_frame = decoder.frame(0);
-            m_timer->set_interval(first_frame.duration);
-            m_timer->on_timeout = [this] { animate(); };
-            m_timer->start();
-        }
+    if (resource()->is_animated() && resource()->frame_count() > 1) {
+        m_timer->set_interval(resource()->frame_duration(0));
+        m_timer->on_timeout = [this] { animate(); };
+        m_timer->start();
     }
 
     if (on_load)
@@ -100,18 +95,16 @@ void ImageLoader::animate()
     if (!m_visible_in_viewport)
         return;
 
-    auto& decoder = resource()->ensure_decoder();
-
-    m_current_frame_index = (m_current_frame_index + 1) % decoder.frame_count();
-    const auto& current_frame = decoder.frame(m_current_frame_index);
+    m_current_frame_index = (m_current_frame_index + 1) % resource()->frame_count();
+    auto current_frame_duration = resource()->frame_duration(m_current_frame_index);
 
-    if (current_frame.duration != m_timer->interval()) {
-        m_timer->restart(current_frame.duration);
+    if (current_frame_duration != m_timer->interval()) {
+        m_timer->restart(current_frame_duration);
     }
 
-    if (m_current_frame_index == decoder.frame_count() - 1) {
+    if (m_current_frame_index == resource()->frame_count() - 1) {
         ++m_loops_completed;
-        if (m_loops_completed > 0 && m_loops_completed == decoder.loop_count()) {
+        if (m_loops_completed > 0 && m_loops_completed == resource()->loop_count()) {
             m_timer->stop();
         }
     }
@@ -132,34 +125,28 @@ bool ImageLoader::has_image() const
 {
     if (!resource())
         return false;
-    if (resource()->should_decode_in_process())
-        return const_cast<ImageResource*>(resource())->ensure_decoder().bitmap();
-    return true;
+    return bitmap(0);
 }
 
 unsigned ImageLoader::width() const
 {
     if (!resource())
         return 0;
-    if (resource()->should_decode_in_process())
-        return const_cast<ImageResource*>(resource())->ensure_decoder().width();
-    return bitmap() ? bitmap()->width() : 0;
+    return bitmap(0) ? bitmap(0)->width() : 0;
 }
 
 unsigned ImageLoader::height() const
 {
     if (!resource())
         return 0;
-    if (resource()->should_decode_in_process())
-        return const_cast<ImageResource*>(resource())->ensure_decoder().height();
-    return bitmap() ? bitmap()->height() : 0;
+    return bitmap(0) ? bitmap(0)->height() : 0;
 }
 
-const Gfx::Bitmap* ImageLoader::bitmap() const
+const Gfx::Bitmap* ImageLoader::bitmap(size_t frame_index) const
 {
     if (!resource())
         return nullptr;
-    return resource()->bitmap(m_current_frame_index);
+    return resource()->bitmap(frame_index);
 }
 
 }

Userland/Libraries/LibWeb/Loader/ImageLoader.h

@@ -38,7 +38,8 @@ public:
 
     void load(const URL&);
 
-    const Gfx::Bitmap* bitmap() const;
+    const Gfx::Bitmap* bitmap(size_t index) const;
+    size_t current_frame_index() const { return m_current_frame_index; }
 
     bool has_image() const;
 

Userland/Libraries/LibWeb/Loader/ImageResource.cpp

@@ -41,43 +41,51 @@ ImageResource::~ImageResource()
 {
 }
 
-bool ImageResource::should_decode_in_process() const
+int ImageResource::frame_duration(size_t frame_index) const
 {
-    return mime_type() == "image/gif";
+    decode_if_needed();
+    if (frame_index >= m_decoded_frames.size())
+        return 0;
+    return m_decoded_frames[frame_index].duration;
 }
 
-Gfx::ImageDecoder& ImageResource::ensure_decoder()
+void ImageResource::decode_if_needed() const
 {
-    if (!m_decoder)
-        m_decoder = Gfx::ImageDecoder::create(encoded_data());
-    return *m_decoder;
+    if (!has_encoded_data())
+        return;
+
+    if (m_has_attempted_decode)
+        return;
+
+    if (!m_decoded_frames.is_empty())
+        return;
+
+    auto image_decoder_client = ImageDecoderClient::Client::construct();
+    auto image = image_decoder_client->decode_image(encoded_data());
+    if (image.has_value()) {
+        m_loop_count = image.value().loop_count;
+        m_animated = image.value().is_animated;
+        m_decoded_frames.resize(image.value().frames.size());
+        for (size_t i = 0; i < m_decoded_frames.size(); ++i) {
+            auto& frame = m_decoded_frames[i];
+            frame.bitmap = image.value().frames[i].bitmap;
+            frame.duration = image.value().frames[i].duration;
+        }
+    }
+
+    m_has_attempted_decode = true;
 }
 
 const Gfx::Bitmap* ImageResource::bitmap(size_t frame_index) const
 {
-    if (!has_encoded_data())
+    decode_if_needed();
+    if (frame_index >= m_decoded_frames.size())
         return nullptr;
-
-    if (should_decode_in_process()) {
-        if (!m_decoder)
-            return nullptr;
-        if (m_decoder->is_animated())
-            m_decoded_image = m_decoder->frame(frame_index).image;
-        else
-            m_decoded_image = m_decoder->bitmap();
-    } else if (!m_decoded_image && !m_has_attempted_decode) {
-        auto image_decoder_client = ImageDecoderClient::Client::construct();
-        m_decoded_image = image_decoder_client->decode_image(encoded_data());
-        m_has_attempted_decode = true;
-    }
-    return m_decoded_image;
+    return m_decoded_frames[frame_index].bitmap;
 }
 
 void ImageResource::update_volatility()
 {
-    if (!m_decoder)
-        return;
-
     bool visible_in_viewport = false;
     for_each_client([&](auto& client) {
         if (static_cast<const ImageResourceClient&>(client).is_visible_in_viewport())
@@ -85,15 +93,28 @@ void ImageResource::update_volatility()
     });
 
     if (!visible_in_viewport) {
-        m_decoder->set_volatile();
+        for (auto& frame : m_decoded_frames) {
+            if (frame.bitmap)
+                frame.bitmap->set_volatile();
+        }
         return;
     }
 
-    bool still_has_decoded_image = m_decoder->set_nonvolatile();
+    bool still_has_decoded_image = true;
+    for (auto& frame : m_decoded_frames) {
+        if (!frame.bitmap) {
+            still_has_decoded_image = false;
+        } else {
+            bool still_has_frame = frame.bitmap->set_nonvolatile();
+            if (!still_has_frame)
+                still_has_decoded_image = false;
+        }
+    }
     if (still_has_decoded_image)
         return;
 
-    m_decoder = nullptr;
+    m_decoded_frames.clear();
+    m_has_attempted_decode = false;
 }
 
 ImageResourceClient::~ImageResourceClient()

Userland/Libraries/LibWeb/Loader/ImageResource.h

@@ -35,17 +35,40 @@ class ImageResource final : public Resource {
 
 public:
     virtual ~ImageResource() override;
-    Gfx::ImageDecoder& ensure_decoder();
-    const Gfx::Bitmap* bitmap(size_t frame_index = 0) const;
 
-    bool should_decode_in_process() const;
+    struct Frame {
+        RefPtr<Gfx::Bitmap> bitmap;
+        size_t duration { 0 };
+    };
+
+    const Gfx::Bitmap* bitmap(size_t frame_index = 0) const;
+    int frame_duration(size_t frame_index) const;
+    size_t frame_count() const
+    {
+        decode_if_needed();
+        return m_decoded_frames.size();
+    }
+    bool is_animated() const
+    {
+        decode_if_needed();
+        return m_animated;
+    }
+    size_t loop_count() const
+    {
+        decode_if_needed();
+        return m_loop_count;
+    }
 
     void update_volatility();
 
 private:
     explicit ImageResource(const LoadRequest&);
-    RefPtr<Gfx::ImageDecoder> m_decoder;
-    mutable RefPtr<Gfx::Bitmap> m_decoded_image;
+
+    void decode_if_needed() const;
+
+    mutable bool m_animated { false };
+    mutable int m_loop_count { 0 };
+    mutable Vector<Frame> m_decoded_frames;
     mutable bool m_has_attempted_decode { false };
 };
 

Userland/Services/ImageDecoder/ClientConnection.cpp

@@ -68,16 +68,33 @@ OwnPtr<Messages::ImageDecoderServer::DecodeImageResponse> ClientConnection::hand
     }
 
     auto decoder = Gfx::ImageDecoder::create(encoded_buffer.data<u8>(), encoded_buffer.size());
-    auto bitmap = decoder->bitmap();
 
-    if (!bitmap) {
+    if (!decoder->frame_count()) {
 #if IMAGE_DECODER_DEBUG
         dbgln("Could not decode image from encoded data");
 #endif
-        return make<Messages::ImageDecoderServer::DecodeImageResponse>(Gfx::ShareableBitmap());
+        return make<Messages::ImageDecoderServer::DecodeImageResponse>(false, 0, Vector<Gfx::ShareableBitmap> {}, Vector<u32> {});
     }
 
-    return make<Messages::ImageDecoderServer::DecodeImageResponse>(bitmap->to_shareable_bitmap());
+    Vector<Gfx::ShareableBitmap> bitmaps;
+    Vector<u32> durations;
+    for (size_t i = 0; i < decoder->frame_count(); ++i) {
+        // FIXME: All image decoder plugins should be rewritten to return frame() instead of bitmap().
+        //        Non-animated images can simply return 1 frame.
+        Gfx::ImageFrameDescriptor frame;
+        if (decoder->is_animated()) {
+            frame = decoder->frame(i);
+        } else {
+            frame.image = decoder->bitmap();
+        }
+        if (frame.image)
+            bitmaps.append(frame.image->to_shareable_bitmap());
+        else
+            bitmaps.append(Gfx::ShareableBitmap {});
+        durations.append(frame.duration);
+    }
+
+    return make<Messages::ImageDecoderServer::DecodeImageResponse>(decoder->is_animated(), decoder->loop_count(), bitmaps, durations);
 }
 
 }

Userland/Services/ImageDecoder/ImageDecoderServer.ipc

@@ -2,5 +2,5 @@ endpoint ImageDecoderServer = 7001
 {
     Greet(i32 client_pid) => (i32 client_id, i32 server_pid)
 
-    DecodeImage(Core::AnonymousBuffer data) => (Gfx::ShareableBitmap bitmap)
+    DecodeImage(Core::AnonymousBuffer data) => (bool is_animated, u32 loop_count, Vector<Gfx::ShareableBitmap> bitmaps, Vector<u32> durations)
 }