Kernel: Crash the current process on OOM (instead of panicking kernel)

This patch adds PageFaultResponse::OutOfMemory which informs the fault
handler that we were unable to allocate a necessary physical page and
cannot continue.

In response to this, the kernel will crash the current process. Because
we are OOM, we can't symbolicate the crash like we normally would
(since the ELF symbolication code needs to allocate), so we also
communicate to Process::crash() that we're out of memory.

Now we can survive "allocate 300 MB" (only the allocate process dies.)
This is definitely not perfect and can easily end up killing a random
innocent other process who happened to allocate one page at the wrong
time, but it's a *lot* better than panicking on OOM. :^)

Kernel/Arch/i386/CPU.cpp

@@ -167,7 +167,7 @@ static void dump(const RegisterState& regs)
     }
 }
 
-void handle_crash(RegisterState& regs, const char* description, int signal)
+void handle_crash(RegisterState& regs, const char* description, int signal, bool out_of_memory)
 {
     if (!Process::current) {
         klog() << description << " with !current";
@@ -188,7 +188,7 @@ void handle_crash(RegisterState& regs, const char* description, int signal)
     }
 
     cli();
-    Process::current->crash(signal, regs.eip);
+    Process::current->crash(signal, regs.eip, out_of_memory);
 }
 
 EH_ENTRY_NO_CODE(6, illegal_instruction);
@@ -254,10 +254,12 @@ void page_fault_handler(RegisterState regs)
 
     auto response = MM.handle_page_fault(PageFault(regs.exception_code, VirtualAddress(fault_address)));
 
-    if (response == PageFaultResponse::ShouldCrash) {
-        if (Thread::current->has_signal_handler(SIGSEGV)) {
-            Thread::current->send_urgent_signal_to_self(SIGSEGV);
-            return;
+    if (response == PageFaultResponse::ShouldCrash || response == PageFaultResponse::OutOfMemory) {
+        if (response != PageFaultResponse::OutOfMemory) {
+            if (Thread::current->has_signal_handler(SIGSEGV)) {
+                Thread::current->send_urgent_signal_to_self(SIGSEGV);
+                return;
+            }
         }
 
         dbg() << "Unrecoverable page fault, "
@@ -287,7 +289,7 @@ void page_fault_handler(RegisterState regs)
             dbg() << "Note: Address " << VirtualAddress(fault_address) << " looks like a possible nullptr dereference";
         }
 
-        handle_crash(regs, "Page Fault", SIGSEGV);
+        handle_crash(regs, "Page Fault", SIGSEGV, response == PageFaultResponse::OutOfMemory);
     } else if (response == PageFaultResponse::Continue) {
 #ifdef PAGE_FAULT_DEBUG
         dbg() << "Continuing after resolved page fault";

Kernel/Arch/i386/CPU.h

@@ -265,7 +265,7 @@ u16 gdt_alloc_entry();
 void gdt_free_entry(u16);
 Descriptor& get_gdt_entry(u16 selector);
 void write_gdt_entry(u16 selector, Descriptor&);
-void handle_crash(RegisterState&, const char* description, int signal);
+void handle_crash(RegisterState&, const char* description, int signal, bool out_of_memory = false);
 
 [[noreturn]] static inline void hang()
 {

Kernel/Process.cpp

@@ -1540,21 +1540,25 @@ int Process::sys$sigreturn(RegisterState& registers)
     return smuggled_eax;
 }
 
-void Process::crash(int signal, u32 eip)
+void Process::crash(int signal, u32 eip, bool out_of_memory)
 {
     ASSERT_INTERRUPTS_DISABLED();
     ASSERT(!is_dead());
     ASSERT(Process::current == this);
 
-    if (eip >= 0xc0000000 && g_kernel_symbols_available) {
-        auto* symbol = symbolicate_kernel_address(eip);
-        dbg() << "\033[31;1m" << String::format("%p", eip) << "  " << (symbol ? demangle(symbol->name) : "(k?)") << " +" << (symbol ? eip - symbol->address : 0) << "\033[0m\n";
-    } else if (auto elf_bundle = this->elf_bundle()) {
-        dbg() << "\033[31;1m" << String::format("%p", eip) << "  " << elf_bundle->elf_loader->symbolicate(eip) << "\033[0m\n";
+    if (out_of_memory) {
+        dbg() << "\033[31;1mOut of memory\033[m, killing: " << *this;
     } else {
-        dbg() << "\033[31;1m" << String::format("%p", eip) << "  (?)\033[0m\n";
+        if (eip >= 0xc0000000 && g_kernel_symbols_available) {
+            auto* symbol = symbolicate_kernel_address(eip);
+            dbg() << "\033[31;1m" << String::format("%p", eip) << "  " << (symbol ? demangle(symbol->name) : "(k?)") << " +" << (symbol ? eip - symbol->address : 0) << "\033[0m\n";
+        } else if (auto elf_bundle = this->elf_bundle()) {
+            dbg() << "\033[31;1m" << String::format("%p", eip) << "  " << elf_bundle->elf_loader->symbolicate(eip) << "\033[0m\n";
+        } else {
+            dbg() << "\033[31;1m" << String::format("%p", eip) << "  (?)\033[0m\n";
+        }
+        dump_backtrace();
     }
-    dump_backtrace();
     m_termination_signal = signal;
     dump_regions();
     ASSERT(is_ring3());

Kernel/Process.h

@@ -314,7 +314,7 @@ public:
 
     static void initialize();
 
-    [[noreturn]] void crash(int signal, u32 eip);
+    [[noreturn]] void crash(int signal, u32 eip, bool out_of_memory = false);
     [[nodiscard]] static siginfo_t reap(Process&);
 
     const TTY* tty() const { return m_tty; }

Kernel/VM/Region.cpp

@@ -368,7 +368,7 @@ PageFaultResponse Region::handle_zero_fault(size_t page_index_in_region)
     auto page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
     if (page.is_null()) {
         klog() << "MM: handle_zero_fault was unable to allocate a physical page";
-        return PageFaultResponse::ShouldCrash;
+        return PageFaultResponse::OutOfMemory;
     }
 
 #ifdef PAGE_FAULT_DEBUG
@@ -401,7 +401,7 @@ PageFaultResponse Region::handle_cow_fault(size_t page_index_in_region)
     auto page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::No);
     if (page.is_null()) {
         klog() << "MM: handle_cow_fault was unable to allocate a physical page";
-        return PageFaultResponse::ShouldCrash;
+        return PageFaultResponse::OutOfMemory;
     }
     auto physical_page_to_copy = move(page_slot);
     u8* dest_ptr = MM.quickmap_page(*page);
@@ -463,7 +463,7 @@ PageFaultResponse Region::handle_inode_fault(size_t page_index_in_region)
     vmobject_physical_page_entry = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::No);
     if (vmobject_physical_page_entry.is_null()) {
         klog() << "MM: handle_inode_fault was unable to allocate a physical page";
-        return PageFaultResponse::ShouldCrash;
+        return PageFaultResponse::OutOfMemory;
     }
 
     u8* dest_ptr = MM.quickmap_page(*vmobject_physical_page_entry);

Kernel/VM/Region.h

@@ -40,6 +40,7 @@ class VMObject;
 
 enum class PageFaultResponse {
     ShouldCrash,
+    OutOfMemory,
     Continue,
 };