LibC: Make `*alloc` return `NULL` in case of failure (POSIX)

Tests/LibC/CMakeLists.txt

@@ -9,6 +9,7 @@ set(TEST_SOURCES
     TestLibCSetjmp.cpp
     TestLibCString.cpp
     TestLibCTime.cpp
+    TestMalloc.cpp
     TestMemmem.cpp
     TestQsort.cpp
     TestRaise.cpp

Tests/LibC/TestMalloc.cpp

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2022, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibTest/TestCase.h>
+
+#include <LibC/mallocdefs.h>
+#include <errno.h>
+#include <stdlib.h>
+
+TEST_CASE(malloc_limits)
+{
+    EXPECT_NO_CRASH("Allocation of 0 size should succed at allocation and release", [] {
+        errno = 0;
+        void* ptr = malloc(0);
+        EXPECT_EQ(errno, 0);
+        free(ptr);
+        return Test::Crash::Failure::DidNotCrash;
+    });
+
+    EXPECT_NO_CRASH("Allocation of the maximum `size_t` value should fails with `ENOMEM`", [] {
+        errno = 0;
+        void* ptr = malloc(NumericLimits<size_t>::max());
+        EXPECT_EQ(errno, ENOMEM);
+        EXPECT_EQ(ptr, nullptr);
+        free(ptr);
+        return Test::Crash::Failure::DidNotCrash;
+    });
+
+    EXPECT_NO_CRASH("Allocation of the maximum `size_t` value that does not overflow should fails with `ENOMEM`", [] {
+        errno = 0;
+        void* ptr = malloc(NumericLimits<size_t>::max() - ChunkedBlock::block_size - sizeof(BigAllocationBlock));
+        EXPECT_EQ(errno, ENOMEM);
+        EXPECT_EQ(ptr, nullptr);
+        free(ptr);
+        return Test::Crash::Failure::DidNotCrash;
+    });
+}

Userland/Libraries/LibC/malloc.cpp

@@ -183,7 +183,11 @@ static void* os_alloc(size_t size, const char* name)
     flags |= MAP_RANDOMIZED;
 #endif
     auto* ptr = serenity_mmap(nullptr, size, PROT_READ | PROT_WRITE, flags, 0, 0, ChunkedBlock::block_size, name);
-    VERIFY(ptr != MAP_FAILED);
+    VERIFY(ptr != nullptr);
+    if (ptr == MAP_FAILED) {
+        errno = ENOMEM;
+        return nullptr;
+    }
     return ptr;
 }
 
@@ -228,6 +232,11 @@ static void* malloc_impl(size_t size, CallerWillInitializeMemory caller_will_ini
 
     if (!allocator) {
         size_t real_size = round_up_to_power_of_two(sizeof(BigAllocationBlock) + size, ChunkedBlock::block_size);
+        if (real_size < size) {
+            dbgln_if(MALLOC_DEBUG, "LibC: Detected overflow trying to do big allocation of size {} for {}", real_size, size);
+            errno = ENOMEM;
+            return nullptr;
+        }
 #ifdef RECYCLE_BIG_ALLOCATIONS
         if (auto* allocator = big_allocator_for_size(real_size)) {
             if (!allocator->blocks.is_empty()) {
@@ -253,8 +262,12 @@ static void* malloc_impl(size_t size, CallerWillInitializeMemory caller_will_ini
             }
         }
 #endif
-        g_malloc_stats.number_of_big_allocs++;
         auto* block = (BigAllocationBlock*)os_alloc(real_size, "malloc: BigAllocationBlock");
+        if (block == nullptr) {
+            dbgln_if(MALLOC_DEBUG, "LibC: Failed to do big allocation of size {} for {}", real_size, size);
+            return nullptr;
+        }
+        g_malloc_stats.number_of_big_allocs++;
         new (block) BigAllocationBlock(real_size);
         ue_notify_malloc(&block->m_slot[0], size);
         return &block->m_slot[0];
@@ -309,6 +322,9 @@ static void* malloc_impl(size_t size, CallerWillInitializeMemory caller_will_ini
         char buffer[64];
         snprintf(buffer, sizeof(buffer), "malloc: ChunkedBlock(%zu)", good_size);
         block = (ChunkedBlock*)os_alloc(ChunkedBlock::block_size, buffer);
+        if (block == nullptr) {
+            return nullptr;
+        }
         new (block) ChunkedBlock(good_size);
         allocator->usable_blocks.append(*block);
         ++allocator->block_count;