LibGfx: Don't allow creating bitmaps whose sizes would overflow

If the area or size_in_bytes calculation for a Gfx::Bitmap would
overflow, we now refuse to create such a bitmap and return nullptr.

Thanks to @itamar8910 for finding this! :^)

Games/Solitaire/Card.cpp

@@ -80,7 +80,7 @@ static RefPtr<Gfx::Bitmap> s_background;
 
 Card::Card(Type type, uint8_t value)
     : m_rect(Gfx::Rect({}, { width, height }))
-    , m_front(Gfx::Bitmap::create(Gfx::BitmapFormat::RGB32, { width, height }))
+    , m_front(*Gfx::Bitmap::create(Gfx::BitmapFormat::RGB32, { width, height }))
     , m_type(type)
     , m_value(value)
 {

Libraries/LibGUI/Window.cpp

@@ -251,10 +251,12 @@ void Window::event(Core::Event& event)
         bool created_new_backing_store = !m_back_bitmap;
         if (!m_back_bitmap) {
             m_back_bitmap = create_backing_bitmap(paint_event.window_size());
+            ASSERT(m_back_bitmap);
         } else if (m_double_buffering_enabled) {
             bool still_has_pixels = m_back_bitmap->shared_buffer()->set_nonvolatile();
             if (!still_has_pixels) {
                 m_back_bitmap = create_backing_bitmap(paint_event.window_size());
+                ASSERT(m_back_bitmap);
                 created_new_backing_store = true;
             }
         }
@@ -514,6 +516,7 @@ void Window::flip(const Vector<Gfx::Rect, 32>& dirty_rects)
 
     if (!m_back_bitmap || m_back_bitmap->size() != m_front_bitmap->size()) {
         m_back_bitmap = create_backing_bitmap(m_front_bitmap->size());
+        ASSERT(m_back_bitmap);
         memcpy(m_back_bitmap->scanline(0), m_front_bitmap->scanline(0), m_front_bitmap->size_in_bytes());
         m_back_bitmap->shared_buffer()->set_volatile();
         return;
@@ -527,7 +530,7 @@ void Window::flip(const Vector<Gfx::Rect, 32>& dirty_rects)
     m_back_bitmap->shared_buffer()->set_volatile();
 }
 
-NonnullRefPtr<Gfx::Bitmap> Window::create_shared_bitmap(Gfx::BitmapFormat format, const Gfx::Size& size)
+RefPtr<Gfx::Bitmap> Window::create_shared_bitmap(Gfx::BitmapFormat format, const Gfx::Size& size)
 {
     ASSERT(WindowServerConnection::the().server_pid());
     ASSERT(!size.is_empty());
@@ -539,7 +542,7 @@ NonnullRefPtr<Gfx::Bitmap> Window::create_shared_bitmap(Gfx::BitmapFormat format
     return Gfx::Bitmap::create_with_shared_buffer(format, *shared_buffer, size);
 }
 
-NonnullRefPtr<Gfx::Bitmap> Window::create_backing_bitmap(const Gfx::Size& size)
+RefPtr<Gfx::Bitmap> Window::create_backing_bitmap(const Gfx::Size& size)
 {
     auto format = m_has_alpha_channel ? Gfx::BitmapFormat::RGBA32 : Gfx::BitmapFormat::RGB32;
     return create_shared_bitmap(format, size);
@@ -561,6 +564,7 @@ void Window::set_icon(const Gfx::Bitmap* icon)
         return;
 
     m_icon = create_shared_bitmap(Gfx::BitmapFormat::RGBA32, icon->size());
+    ASSERT(m_icon);
     {
         Painter painter(*m_icon);
         painter.blit({ 0, 0 }, *icon, icon->rect());

Libraries/LibGUI/Window.h

@@ -189,8 +189,8 @@ protected:
 private:
     virtual bool is_window() const override final { return true; }
 
-    NonnullRefPtr<Gfx::Bitmap> create_backing_bitmap(const Gfx::Size&);
-    NonnullRefPtr<Gfx::Bitmap> create_shared_bitmap(Gfx::BitmapFormat, const Gfx::Size&);
+    RefPtr<Gfx::Bitmap> create_backing_bitmap(const Gfx::Size&);
+    RefPtr<Gfx::Bitmap> create_shared_bitmap(Gfx::BitmapFormat, const Gfx::Size&);
     void set_current_backing_bitmap(Gfx::Bitmap&, bool flush_immediately = false);
     void flip(const Vector<Gfx::Rect, 32>& dirty_rects);
     void force_update();

Libraries/LibGfx/Bitmap.cpp

@@ -38,13 +38,30 @@
 
 namespace Gfx {
 
-NonnullRefPtr<Bitmap> Bitmap::create(BitmapFormat format, const Size& size)
+static bool size_would_overflow(BitmapFormat format, const Size& size)
 {
+    if (size.width() < 0 || size.height() < 0)
+        return true;
+    size_t bpp = Bitmap::bpp_for_format(format);
+    size_t result = 0;
+    if (__builtin_mul_overflow((size_t)size.width(), (size_t)size.height(), &result))
+        return true;
+    if (__builtin_mul_overflow(result, bpp, &result))
+        return true;
+    return false;
+}
+
+RefPtr<Bitmap> Bitmap::create(BitmapFormat format, const Size& size)
+{
+    if (size_would_overflow(format, size))
+        return nullptr;
     return adopt(*new Bitmap(format, size, Purgeable::No));
 }
 
-NonnullRefPtr<Bitmap> Bitmap::create_purgeable(BitmapFormat format, const Size& size)
+RefPtr<Bitmap> Bitmap::create_purgeable(BitmapFormat format, const Size& size)
 {
+    if (size_would_overflow(format, size))
+        return nullptr;
     return adopt(*new Bitmap(format, size, Purgeable::Yes));
 }
 
@@ -55,6 +72,7 @@ Bitmap::Bitmap(BitmapFormat format, const Size& size, Purgeable purgeable)
     , m_purgeable(purgeable == Purgeable::Yes)
 {
     ASSERT(!m_size.is_empty());
+    ASSERT(!size_would_overflow(format, size));
     if (format == BitmapFormat::Indexed8)
         m_palette = new RGBA32[256];
     int map_flags = purgeable == Purgeable::Yes ? (MAP_PURGEABLE | MAP_PRIVATE) : (MAP_ANONYMOUS | MAP_PRIVATE);
@@ -63,8 +81,10 @@ Bitmap::Bitmap(BitmapFormat format, const Size& size, Purgeable purgeable)
     m_needs_munmap = true;
 }
 
-NonnullRefPtr<Bitmap> Bitmap::create_wrapper(BitmapFormat format, const Size& size, size_t pitch, RGBA32* data)
+RefPtr<Bitmap> Bitmap::create_wrapper(BitmapFormat format, const Size& size, size_t pitch, RGBA32* data)
 {
+    if (size_would_overflow(format, size))
+        return nullptr;
     return adopt(*new Bitmap(format, size, pitch, data));
 }
 
@@ -79,12 +99,15 @@ Bitmap::Bitmap(BitmapFormat format, const Size& size, size_t pitch, RGBA32* data
     , m_pitch(pitch)
     , m_format(format)
 {
+    ASSERT(!size_would_overflow(format, size));
     if (format == BitmapFormat::Indexed8)
         m_palette = new RGBA32[256];
 }
 
-NonnullRefPtr<Bitmap> Bitmap::create_with_shared_buffer(BitmapFormat format, NonnullRefPtr<SharedBuffer>&& shared_buffer, const Size& size)
+RefPtr<Bitmap> Bitmap::create_with_shared_buffer(BitmapFormat format, NonnullRefPtr<SharedBuffer>&& shared_buffer, const Size& size)
 {
+    if (size_would_overflow(format, size))
+        return nullptr;
     return adopt(*new Bitmap(format, move(shared_buffer), size));
 }
 
@@ -96,14 +119,17 @@ Bitmap::Bitmap(BitmapFormat format, NonnullRefPtr<SharedBuffer>&& shared_buffer,
     , m_shared_buffer(move(shared_buffer))
 {
     ASSERT(format != BitmapFormat::Indexed8);
+    ASSERT(!size_would_overflow(format, size));
 }
 
-NonnullRefPtr<Gfx::Bitmap> Bitmap::rotated(Gfx::RotationDirection rotation_direction) const
+RefPtr<Gfx::Bitmap> Bitmap::rotated(Gfx::RotationDirection rotation_direction) const
 {
     auto w = this->width();
     auto h = this->height();
 
     auto new_bitmap = Gfx::Bitmap::create(this->format(), { h, w });
+    if (!new_bitmap)
+        return nullptr;
 
     for (int i = 0; i < w; i++) {
         for (int j = 0; j < h; j++) {
@@ -120,12 +146,14 @@ NonnullRefPtr<Gfx::Bitmap> Bitmap::rotated(Gfx::RotationDirection rotation_direc
     return new_bitmap;
 }
 
-NonnullRefPtr<Gfx::Bitmap> Bitmap::flipped(Gfx::Orientation orientation) const
+RefPtr<Gfx::Bitmap> Bitmap::flipped(Gfx::Orientation orientation) const
 {
     auto w = this->width();
     auto h = this->height();
 
     auto new_bitmap = Gfx::Bitmap::create(this->format(), { w, h });
+    if (!new_bitmap)
+        return nullptr;
 
     for (int i = 0; i < w; i++) {
         for (int j = 0; j < h; j++) {
@@ -140,12 +168,14 @@ NonnullRefPtr<Gfx::Bitmap> Bitmap::flipped(Gfx::Orientation orientation) const
     return new_bitmap;
 }
 
-NonnullRefPtr<Bitmap> Bitmap::to_bitmap_backed_by_shared_buffer() const
+RefPtr<Bitmap> Bitmap::to_bitmap_backed_by_shared_buffer() const
 {
     if (m_shared_buffer)
         return *this;
     auto buffer = SharedBuffer::create_with_size(size_in_bytes());
     auto bitmap = Bitmap::create_with_shared_buffer(m_format, *buffer, m_size);
+    if (!bitmap)
+        return nullptr;
     memcpy(buffer->data(), scanline(0), size_in_bytes());
     return bitmap;
 }
@@ -210,6 +240,8 @@ int Bitmap::shbuf_id() const
 ShareableBitmap Bitmap::to_shareable_bitmap(pid_t peer_pid) const
 {
     auto bitmap = to_bitmap_backed_by_shared_buffer();
+    if (!bitmap)
+        return {};
     if (peer_pid > 0)
         bitmap->shared_buffer()->share_with(peer_pid);
     return ShareableBitmap(*bitmap);

Libraries/LibGfx/Bitmap.h

@@ -49,15 +49,15 @@ enum RotationDirection {
 
 class Bitmap : public RefCounted<Bitmap> {
 public:
-    static NonnullRefPtr<Bitmap> create(BitmapFormat, const Size&);
-    static NonnullRefPtr<Bitmap> create_purgeable(BitmapFormat, const Size&);
-    static NonnullRefPtr<Bitmap> create_wrapper(BitmapFormat, const Size&, size_t pitch, RGBA32*);
+    static RefPtr<Bitmap> create(BitmapFormat, const Size&);
+    static RefPtr<Bitmap> create_purgeable(BitmapFormat, const Size&);
+    static RefPtr<Bitmap> create_wrapper(BitmapFormat, const Size&, size_t pitch, RGBA32*);
     static RefPtr<Bitmap> load_from_file(const StringView& path);
-    static NonnullRefPtr<Bitmap> create_with_shared_buffer(BitmapFormat, NonnullRefPtr<SharedBuffer>&&, const Size&);
+    static RefPtr<Bitmap> create_with_shared_buffer(BitmapFormat, NonnullRefPtr<SharedBuffer>&&, const Size&);
 
-    NonnullRefPtr<Gfx::Bitmap> rotated(Gfx::RotationDirection) const;
-    NonnullRefPtr<Gfx::Bitmap> flipped(Gfx::Orientation) const;
-    NonnullRefPtr<Bitmap> to_bitmap_backed_by_shared_buffer() const;
+    RefPtr<Gfx::Bitmap> rotated(Gfx::RotationDirection) const;
+    RefPtr<Gfx::Bitmap> flipped(Gfx::Orientation) const;
+    RefPtr<Bitmap> to_bitmap_backed_by_shared_buffer() const;
 
     ShareableBitmap to_shareable_bitmap(pid_t peer_pid = -1) const;
 
@@ -79,9 +79,9 @@ public:
     SharedBuffer* shared_buffer() { return m_shared_buffer.ptr(); }
     const SharedBuffer* shared_buffer() const { return m_shared_buffer.ptr(); }
 
-    unsigned bpp() const
+    static unsigned bpp_for_format(BitmapFormat format)
     {
-        switch (m_format) {
+        switch (format) {
         case BitmapFormat::Indexed8:
             return 8;
         case BitmapFormat::RGB32:
@@ -94,6 +94,11 @@ public:
         }
     }
 
+    unsigned bpp() const
+    {
+        return bpp_for_format(m_format);
+    }
+
     void fill(Color);
 
     bool has_alpha_channel() const { return m_format == BitmapFormat::RGBA32; }