Kernel: Move Memory/PageDirectory.{cpp,h} to arch-specific directory

The handling of page tables is very architecture specific, so belongs
in the Arch directory. Some parts were already architecture-specific,
however this commit moves the rest of the PageDirectory class into the
Arch directory.

While we're here the aarch64/PageDirectory.{h,cpp} files are updated to
be aarch64 specific, by renaming some members and removing x86_64
specific code.

Kernel/Arch/aarch64/CPU.h

@@ -12,9 +12,14 @@
 namespace Kernel {
 
 void initialize_exceptions();
-void init_page_tables();
-void unmap_identity_map();
 void panic_without_mmu(StringView);
 void dbgln_without_mmu(StringView);
 
+namespace Memory {
+
+void init_page_tables();
+void unmap_identity_map();
+
+}
+
 }

Kernel/Arch/aarch64/Dummy.cpp

@@ -57,13 +57,11 @@ READONLY_AFTER_INIT PhysicalAddress end_of_prekernel_image;
 READONLY_AFTER_INIT size_t physical_to_virtual_offset;
 // READONLY_AFTER_INIT FlatPtr kernel_mapping_base;
 READONLY_AFTER_INIT FlatPtr kernel_load_base;
-#if ARCH(X86_64)
 READONLY_AFTER_INIT PhysicalAddress boot_pml4t;
-#endif
 READONLY_AFTER_INIT PhysicalAddress boot_pdpt;
 READONLY_AFTER_INIT PhysicalAddress boot_pd0;
 READONLY_AFTER_INIT PhysicalAddress boot_pd_kernel;
-READONLY_AFTER_INIT Kernel::PageTableEntry* boot_pd_kernel_pt1023;
+READONLY_AFTER_INIT Kernel::Memory::PageTableEntry* boot_pd_kernel_pt1023;
 READONLY_AFTER_INIT char const* kernel_cmdline;
 READONLY_AFTER_INIT u32 multiboot_flags;
 READONLY_AFTER_INIT multiboot_memory_map_t* multiboot_memory_map;

Kernel/Arch/aarch64/MMU.cpp

@@ -26,7 +26,7 @@ extern u8 page_tables_phys_end[];
 extern u8 start_of_kernel_image[];
 extern u8 end_of_kernel_image[];
 
-namespace Kernel {
+namespace Kernel::Memory {
 
 // physical memory
 constexpr u32 START_OF_NORMAL_MEMORY = 0x00000000;

Kernel/Arch/aarch64/PageDirectory.cpp

@@ -5,8 +5,17 @@
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
+#include <AK/Singleton.h>
+#include <Kernel/Arch/CPU.h>
+#include <Kernel/Arch/PageDirectory.h>
 #include <Kernel/Arch/aarch64/ASM_wrapper.h>
-#include <Kernel/Memory/PageDirectory.h>
+#include <Kernel/InterruptDisabler.h>
+#include <Kernel/Memory/MemoryManager.h>
+#include <Kernel/Prekernel/Prekernel.h>
+#include <Kernel/Process.h>
+#include <Kernel/Random.h>
+#include <Kernel/Sections.h>
+#include <Kernel/Thread.h>
 
 namespace Kernel::Memory {
 
@@ -36,4 +45,68 @@ void activate_page_directory(PageDirectory const&, Thread*)
     TODO_AARCH64();
 }
 
+UNMAP_AFTER_INIT NonnullLockRefPtr<PageDirectory> PageDirectory::must_create_kernel_page_directory()
+{
+    return adopt_lock_ref_if_nonnull(new (nothrow) PageDirectory).release_nonnull();
+}
+
+ErrorOr<NonnullLockRefPtr<PageDirectory>> PageDirectory::try_create_for_userspace()
+{
+    auto directory = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) PageDirectory));
+
+    directory->m_root_table = TRY(MM.allocate_physical_page());
+
+    directory->m_directory_table = TRY(MM.allocate_physical_page());
+    auto kernel_pd_index = (kernel_mapping_base >> 30) & 0x1ffu;
+    for (size_t i = 0; i < kernel_pd_index; i++) {
+        directory->m_directory_pages[i] = TRY(MM.allocate_physical_page());
+    }
+
+    // Share the top 1 GiB of kernel-only mappings (>=kernel_mapping_base)
+    directory->m_directory_pages[kernel_pd_index] = MM.kernel_page_directory().m_directory_pages[kernel_pd_index];
+
+    {
+        InterruptDisabler disabler;
+        auto& table = *(PageDirectoryPointerTable*)MM.quickmap_page(*directory->m_root_table);
+        table.raw[0] = (FlatPtr)directory->m_directory_table->paddr().as_ptr() | TABLE_DESCRIPTOR;
+        MM.unquickmap_page();
+    }
+
+    {
+        InterruptDisabler disabler;
+        auto& table = *(PageDirectoryPointerTable*)MM.quickmap_page(*directory->m_directory_table);
+        for (size_t i = 0; i < sizeof(m_directory_pages) / sizeof(m_directory_pages[0]); i++) {
+            if (directory->m_directory_pages[i]) {
+                table.raw[i] = (FlatPtr)directory->m_directory_pages[i]->paddr().as_ptr() | PAGE_DESCRIPTOR;
+            }
+        }
+        MM.unquickmap_page();
+    }
+
+    register_page_directory(directory);
+    return directory;
+}
+
+PageDirectory::PageDirectory() = default;
+
+UNMAP_AFTER_INIT void PageDirectory::allocate_kernel_directory()
+{
+    // Adopt the page tables already set up by boot.S
+    dmesgln("MM: boot_pml4t @ {}", boot_pml4t);
+    m_root_table = PhysicalPage::create(boot_pml4t, MayReturnToFreeList::No);
+    dmesgln("MM: boot_pdpt @ {}", boot_pdpt);
+    dmesgln("MM: boot_pd0 @ {}", boot_pd0);
+    dmesgln("MM: boot_pd_kernel @ {}", boot_pd_kernel);
+    m_directory_table = PhysicalPage::create(boot_pdpt, MayReturnToFreeList::No);
+    m_directory_pages[0] = PhysicalPage::create(boot_pd0, MayReturnToFreeList::No);
+    m_directory_pages[(kernel_mapping_base >> 30) & 0x1ff] = PhysicalPage::create(boot_pd_kernel, MayReturnToFreeList::No);
+}
+
+PageDirectory::~PageDirectory()
+{
+    if (is_root_table_initialized()) {
+        deregister_page_directory(this);
+    }
+}
+
 }

Kernel/Arch/aarch64/PageDirectory.h

@@ -7,12 +7,18 @@
 
 #pragma once
 
+#include <AK/AtomicRefCounted.h>
 #include <AK/Badge.h>
+#include <AK/HashMap.h>
+#include <AK/IntrusiveRedBlackTree.h>
+#include <AK/RefPtr.h>
 #include <AK/Types.h>
 #include <Kernel/Forward.h>
+#include <Kernel/Locking/Spinlock.h>
+#include <Kernel/Memory/PhysicalPage.h>
 #include <Kernel/PhysicalAddress.h>
 
-namespace Kernel {
+namespace Kernel::Memory {
 
 // 4KiB page size was chosen to make this code slightly simpler
 constexpr u32 GRANULE_SIZE = 0x1000;
@@ -171,4 +177,51 @@ public:
     u64 raw[512];
 };
 
+class PageDirectory final : public AtomicRefCounted<PageDirectory> {
+    friend class MemoryManager;
+
+public:
+    static ErrorOr<NonnullLockRefPtr<PageDirectory>> try_create_for_userspace();
+    static NonnullLockRefPtr<PageDirectory> must_create_kernel_page_directory();
+    static LockRefPtr<PageDirectory> find_current();
+
+    ~PageDirectory();
+
+    void allocate_kernel_directory();
+
+    FlatPtr ttbr0() const
+    {
+        return m_root_table->paddr().get();
+    }
+
+    bool is_root_table_initialized() const
+    {
+        return m_root_table;
+    }
+
+    AddressSpace* address_space() { return m_space; }
+    AddressSpace const* address_space() const { return m_space; }
+
+    void set_space(Badge<AddressSpace>, AddressSpace& space) { m_space = &space; }
+
+    RecursiveSpinlock<LockRank::None>& get_lock() { return m_lock; }
+
+    // This has to be public to let the global singleton access the member pointer
+    IntrusiveRedBlackTreeNode<FlatPtr, PageDirectory, RawPtr<PageDirectory>> m_tree_node;
+
+private:
+    PageDirectory();
+    static void register_page_directory(PageDirectory* directory);
+    static void deregister_page_directory(PageDirectory* directory);
+
+    AddressSpace* m_space { nullptr };
+    RefPtr<PhysicalPage> m_root_table;
+    RefPtr<PhysicalPage> m_directory_table;
+    RefPtr<PhysicalPage> m_directory_pages[512];
+    RecursiveSpinlock<LockRank::None> m_lock {};
+};
+
+void activate_kernel_page_directory(PageDirectory const& pgd);
+void activate_page_directory(PageDirectory const& pgd, Thread* current_thread);
+
 }

Kernel/Arch/aarch64/pre_init.cpp

@@ -27,7 +27,7 @@ extern "C" [[noreturn]] void pre_init()
     initialize_exceptions();
 
     // Next step is to set up page tables and enable the MMU.
-    init_page_tables();
+    Memory::init_page_tables();
 
     // At this point the MMU is enabled, physical memory is identity mapped,
     // and the kernel is also mapped into higher virtual memory. However we are still executing
@@ -49,7 +49,7 @@ extern "C" [[noreturn]] void pre_init()
         : "x0");
 
     // We can now unmap the identity map as everything is running in high virtual memory at this point.
-    unmap_identity_map();
+    Memory::unmap_identity_map();
 
     // Clear the frame pointer (x29) and link register (x30) to make sure the kernel cannot backtrace
     // into this code, and jump to actual init function in the kernel.

Kernel/Arch/x86_64/Interrupts/APIC.cpp

@@ -8,6 +8,7 @@
 #include <AK/Singleton.h>
 #include <AK/Types.h>
 #include <Kernel/Arch/Delay.h>
+#include <Kernel/Arch/PageDirectory.h>
 #include <Kernel/Arch/x86_64/Interrupts/APIC.h>
 #include <Kernel/Arch/x86_64/MSR.h>
 #include <Kernel/Arch/x86_64/ProcessorInfo.h>
@@ -17,7 +18,6 @@
 #include <Kernel/Interrupts/SpuriousInterruptHandler.h>
 #include <Kernel/Memory/AnonymousVMObject.h>
 #include <Kernel/Memory/MemoryManager.h>
-#include <Kernel/Memory/PageDirectory.h>
 #include <Kernel/Memory/TypedMapping.h>
 #include <Kernel/Panic.h>
 #include <Kernel/Scheduler.h>

Kernel/Arch/x86_64/PageDirectory.cpp

@@ -6,7 +6,14 @@
  */
 
 #include <AK/Singleton.h>
-#include <Kernel/Memory/PageDirectory.h>
+#include <Kernel/Arch/CPU.h>
+#include <Kernel/Arch/PageDirectory.h>
+#include <Kernel/InterruptDisabler.h>
+#include <Kernel/Memory/MemoryManager.h>
+#include <Kernel/Prekernel/Prekernel.h>
+#include <Kernel/Process.h>
+#include <Kernel/Random.h>
+#include <Kernel/Sections.h>
 #include <Kernel/Thread.h>
 
 namespace Kernel::Memory {
@@ -49,4 +56,92 @@ void activate_page_directory(PageDirectory const& pgd, Thread* current_thread)
     write_cr3(pgd.cr3());
 }
 
+UNMAP_AFTER_INIT NonnullLockRefPtr<PageDirectory> PageDirectory::must_create_kernel_page_directory()
+{
+    return adopt_lock_ref_if_nonnull(new (nothrow) PageDirectory).release_nonnull();
+}
+
+ErrorOr<NonnullLockRefPtr<PageDirectory>> PageDirectory::try_create_for_userspace()
+{
+    auto directory = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) PageDirectory));
+
+    directory->m_pml4t = TRY(MM.allocate_physical_page());
+
+    directory->m_directory_table = TRY(MM.allocate_physical_page());
+    auto kernel_pd_index = (kernel_mapping_base >> 30) & 0x1ffu;
+    for (size_t i = 0; i < kernel_pd_index; i++) {
+        directory->m_directory_pages[i] = TRY(MM.allocate_physical_page());
+    }
+
+    // Share the top 1 GiB of kernel-only mappings (>=kernel_mapping_base)
+    directory->m_directory_pages[kernel_pd_index] = MM.kernel_page_directory().m_directory_pages[kernel_pd_index];
+
+    {
+        InterruptDisabler disabler;
+        auto& table = *(PageDirectoryPointerTable*)MM.quickmap_page(*directory->m_pml4t);
+        table.raw[0] = (FlatPtr)directory->m_directory_table->paddr().as_ptr() | 7;
+        MM.unquickmap_page();
+    }
+
+    {
+        InterruptDisabler disabler;
+        auto& table = *(PageDirectoryPointerTable*)MM.quickmap_page(*directory->m_directory_table);
+        for (size_t i = 0; i < sizeof(m_directory_pages) / sizeof(m_directory_pages[0]); i++) {
+            if (directory->m_directory_pages[i]) {
+                table.raw[i] = (FlatPtr)directory->m_directory_pages[i]->paddr().as_ptr() | 7;
+            }
+        }
+
+        // 2 ** MAXPHYADDR - 1
+        // Where MAXPHYADDR = physical_address_bit_width
+        u64 max_physical_address = (1ULL << Processor::current().physical_address_bit_width()) - 1;
+
+        // bit 63 = no execute
+        // bit 7 = page size
+        // bit 5 = accessed
+        // bit 4 = cache disable
+        // bit 3 = write through
+        // bit 2 = user/supervisor
+        // bit 1 = read/write
+        // bit 0 = present
+        constexpr u64 pdpte_bit_flags = 0x80000000000000BF;
+
+        // This is to notify us of bugs where we're:
+        // 1. Going over what the processor is capable of.
+        // 2. Writing into the reserved bits (51:MAXPHYADDR), where doing so throws a GPF
+        //    when writing out the PDPT pointer to CR3.
+        // The reason we're not checking the page directory's physical address directly is because
+        // we're checking for sign extension when putting it into a PDPTE. See issue #4584.
+        for (auto table_entry : table.raw)
+            VERIFY((table_entry & ~pdpte_bit_flags) <= max_physical_address);
+
+        MM.unquickmap_page();
+    }
+
+    register_page_directory(directory);
+    return directory;
+}
+
+PageDirectory::PageDirectory() = default;
+
+UNMAP_AFTER_INIT void PageDirectory::allocate_kernel_directory()
+{
+    // Adopt the page tables already set up by boot.S
+    dmesgln("MM: boot_pml4t @ {}", boot_pml4t);
+    m_pml4t = PhysicalPage::create(boot_pml4t, MayReturnToFreeList::No);
+    dmesgln("MM: boot_pdpt @ {}", boot_pdpt);
+    dmesgln("MM: boot_pd0 @ {}", boot_pd0);
+    dmesgln("MM: boot_pd_kernel @ {}", boot_pd_kernel);
+    m_directory_table = PhysicalPage::create(boot_pdpt, MayReturnToFreeList::No);
+    m_directory_pages[0] = PhysicalPage::create(boot_pd0, MayReturnToFreeList::No);
+    m_directory_pages[(kernel_mapping_base >> 30) & 0x1ff] = PhysicalPage::create(boot_pd_kernel, MayReturnToFreeList::No);
+}
+
+PageDirectory::~PageDirectory()
+{
+    if (is_cr3_initialized()) {
+        deregister_page_directory(this);
+    }
+}
+
 }

Kernel/Arch/x86_64/PageDirectory.h

@@ -6,12 +6,19 @@
 
 #pragma once
 
+#include <AK/AtomicRefCounted.h>
 #include <AK/Badge.h>
+#include <AK/HashMap.h>
+#include <AK/IntrusiveRedBlackTree.h>
+#include <AK/RefPtr.h>
 #include <AK/Types.h>
 #include <Kernel/Forward.h>
+#include <Kernel/Locking/LockRank.h>
+#include <Kernel/Locking/Spinlock.h>
+#include <Kernel/Memory/PhysicalPage.h>
 #include <Kernel/PhysicalAddress.h>
 
-namespace Kernel {
+namespace Kernel::Memory {
 
 class PageDirectoryEntry {
 public:
@@ -151,4 +158,51 @@ public:
     u64 raw[512];
 };
 
+class PageDirectory final : public AtomicRefCounted<PageDirectory> {
+    friend class MemoryManager;
+
+public:
+    static ErrorOr<NonnullLockRefPtr<PageDirectory>> try_create_for_userspace();
+    static NonnullLockRefPtr<PageDirectory> must_create_kernel_page_directory();
+    static LockRefPtr<PageDirectory> find_current();
+
+    ~PageDirectory();
+
+    void allocate_kernel_directory();
+
+    FlatPtr cr3() const
+    {
+        return m_pml4t->paddr().get();
+    }
+
+    bool is_cr3_initialized() const
+    {
+        return m_pml4t;
+    }
+
+    AddressSpace* address_space() { return m_space; }
+    AddressSpace const* address_space() const { return m_space; }
+
+    void set_space(Badge<AddressSpace>, AddressSpace& space) { m_space = &space; }
+
+    RecursiveSpinlock<LockRank::None>& get_lock() { return m_lock; }
+
+    // This has to be public to let the global singleton access the member pointer
+    IntrusiveRedBlackTreeNode<FlatPtr, PageDirectory, RawPtr<PageDirectory>> m_tree_node;
+
+private:
+    PageDirectory();
+    static void register_page_directory(PageDirectory* directory);
+    static void deregister_page_directory(PageDirectory* directory);
+
+    AddressSpace* m_space { nullptr };
+    RefPtr<PhysicalPage> m_pml4t;
+    RefPtr<PhysicalPage> m_directory_table;
+    RefPtr<PhysicalPage> m_directory_pages[512];
+    RecursiveSpinlock<LockRank::None> m_lock {};
+};
+
+void activate_kernel_page_directory(PageDirectory const& pgd);
+void activate_page_directory(PageDirectory const& pgd, Thread* current_thread);
+
 }

Kernel/Arch/x86_64/Processor.cpp

@@ -30,7 +30,7 @@
 #include <Kernel/Arch/x86_64/ProcessorInfo.h>
 #include <Kernel/ScopedCritical.h>
 
-#include <Kernel/Memory/PageDirectory.h>
+#include <Kernel/Arch/PageDirectory.h>
 #include <Kernel/Memory/ScopedAddressSpaceSwitcher.h>
 
 namespace Kernel {

Kernel/Arch/x86_64/Processor.h

@@ -12,7 +12,6 @@
 #include <AK/Types.h>
 
 #include <Kernel/Arch/DeferredCallEntry.h>
-#include <Kernel/Arch/PageDirectory.h>
 #include <Kernel/Arch/ProcessorSpecificDataID.h>
 #include <Kernel/Arch/x86_64/ASM_wrapper.h>
 #include <Kernel/Arch/x86_64/CPUID.h>

Kernel/Arch/x86_64/init.cpp

@@ -126,7 +126,7 @@ READONLY_AFTER_INIT PhysicalAddress boot_pml4t;
 READONLY_AFTER_INIT PhysicalAddress boot_pdpt;
 READONLY_AFTER_INIT PhysicalAddress boot_pd0;
 READONLY_AFTER_INIT PhysicalAddress boot_pd_kernel;
-READONLY_AFTER_INIT PageTableEntry* boot_pd_kernel_pt1023;
+READONLY_AFTER_INIT Memory::PageTableEntry* boot_pd_kernel_pt1023;
 READONLY_AFTER_INIT char const* kernel_cmdline;
 READONLY_AFTER_INIT u32 multiboot_flags;
 READONLY_AFTER_INIT multiboot_memory_map_t* multiboot_memory_map;
@@ -158,7 +158,7 @@ extern "C" [[noreturn]] UNMAP_AFTER_INIT void init(BootInfo const& boot_info)
     boot_pdpt = PhysicalAddress { boot_info.boot_pdpt };
     boot_pd0 = PhysicalAddress { boot_info.boot_pd0 };
     boot_pd_kernel = PhysicalAddress { boot_info.boot_pd_kernel };
-    boot_pd_kernel_pt1023 = (PageTableEntry*)boot_info.boot_pd_kernel_pt1023;
+    boot_pd_kernel_pt1023 = (Memory::PageTableEntry*)boot_info.boot_pd_kernel_pt1023;
     kernel_cmdline = (char const*)boot_info.kernel_cmdline;
     multiboot_flags = boot_info.multiboot_flags;
     multiboot_memory_map = (multiboot_memory_map_t*)boot_info.multiboot_memory_map;

Kernel/BootInfo.h

@@ -10,7 +10,7 @@
 #include <Kernel/PhysicalAddress.h>
 #include <Kernel/VirtualAddress.h>
 
-namespace Kernel {
+namespace Kernel::Memory {
 class PageTableEntry;
 }
 
@@ -22,12 +22,12 @@ extern "C" FlatPtr kernel_load_base;
 #if ARCH(X86_64)
 extern "C" u32 gdt64ptr;
 extern "C" u16 code64_sel;
-extern "C" PhysicalAddress boot_pml4t;
 #endif
+extern "C" PhysicalAddress boot_pml4t;
 extern "C" PhysicalAddress boot_pdpt;
 extern "C" PhysicalAddress boot_pd0;
 extern "C" PhysicalAddress boot_pd_kernel;
-extern "C" Kernel::PageTableEntry* boot_pd_kernel_pt1023;
+extern "C" Kernel::Memory::PageTableEntry* boot_pd_kernel_pt1023;
 extern "C" char const* kernel_cmdline;
 extern "C" u32 multiboot_flags;
 extern "C" multiboot_memory_map_t* multiboot_memory_map;

Kernel/CMakeLists.txt

@@ -222,7 +222,6 @@ set(KERNEL_SOURCES
     Memory/AnonymousVMObject.cpp
     Memory/InodeVMObject.cpp
     Memory/MemoryManager.cpp
-    Memory/PageDirectory.cpp
     Memory/PhysicalPage.cpp
     Memory/PhysicalRegion.cpp
     Memory/PhysicalZone.cpp

Kernel/Memory/AddressSpace.h

@@ -9,10 +9,10 @@
 
 #include <AK/RedBlackTree.h>
 #include <AK/Vector.h>
+#include <Kernel/Arch/PageDirectory.h>
 #include <Kernel/Library/LockWeakPtr.h>
 #include <Kernel/Locking/SpinlockProtected.h>
 #include <Kernel/Memory/AllocationStrategy.h>
-#include <Kernel/Memory/PageDirectory.h>
 #include <Kernel/Memory/Region.h>
 #include <Kernel/Memory/RegionTree.h>
 #include <Kernel/UnixTypes.h>

Kernel/Memory/MemoryManager.cpp

@@ -18,7 +18,6 @@
 #include <Kernel/KSyms.h>
 #include <Kernel/Memory/AnonymousVMObject.h>
 #include <Kernel/Memory/MemoryManager.h>
-#include <Kernel/Memory/PageDirectory.h>
 #include <Kernel/Memory/PhysicalRegion.h>
 #include <Kernel/Memory/SharedInodeVMObject.h>
 #include <Kernel/Multiboot.h>

Kernel/Memory/MemoryManager.h

@@ -21,15 +21,13 @@
 #include <Kernel/Memory/RegionTree.h>
 #include <Kernel/Memory/VMObject.h>
 
-namespace Kernel {
-class PageDirectoryEntry;
-class PageTableEntry;
-}
-
 struct KmallocGlobalData;
 
 namespace Kernel::Memory {
 
+class PageDirectoryEntry;
+class PageTableEntry;
+
 ErrorOr<FlatPtr> page_round_up(FlatPtr x);
 
 constexpr FlatPtr page_round_down(FlatPtr x)

Kernel/Memory/PageDirectory.cpp

@@ -1,119 +0,0 @@
-/*
- * Copyright (c) 2018-2022, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include <AK/Singleton.h>
-#include <Kernel/Arch/CPU.h>
-#include <Kernel/Arch/PageDirectory.h>
-#include <Kernel/InterruptDisabler.h>
-#include <Kernel/Memory/MemoryManager.h>
-#include <Kernel/Memory/PageDirectory.h>
-#include <Kernel/Prekernel/Prekernel.h>
-#include <Kernel/Process.h>
-#include <Kernel/Random.h>
-#include <Kernel/Sections.h>
-
-extern u8 start_of_kernel_image[];
-extern u8 end_of_kernel_image[];
-
-namespace Kernel::Memory {
-
-UNMAP_AFTER_INIT NonnullLockRefPtr<PageDirectory> PageDirectory::must_create_kernel_page_directory()
-{
-    return adopt_lock_ref_if_nonnull(new (nothrow) PageDirectory).release_nonnull();
-}
-
-ErrorOr<NonnullLockRefPtr<PageDirectory>> PageDirectory::try_create_for_userspace()
-{
-    auto directory = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) PageDirectory));
-
-#if ARCH(X86_64)
-    directory->m_pml4t = TRY(MM.allocate_physical_page());
-#endif
-
-    directory->m_directory_table = TRY(MM.allocate_physical_page());
-    auto kernel_pd_index = (kernel_mapping_base >> 30) & 0x1ffu;
-    for (size_t i = 0; i < kernel_pd_index; i++) {
-        directory->m_directory_pages[i] = TRY(MM.allocate_physical_page());
-    }
-
-    // Share the top 1 GiB of kernel-only mappings (>=kernel_mapping_base)
-    directory->m_directory_pages[kernel_pd_index] = MM.kernel_page_directory().m_directory_pages[kernel_pd_index];
-
-#if ARCH(X86_64)
-    {
-        InterruptDisabler disabler;
-        auto& table = *(PageDirectoryPointerTable*)MM.quickmap_page(*directory->m_pml4t);
-        table.raw[0] = (FlatPtr)directory->m_directory_table->paddr().as_ptr() | 7;
-        MM.unquickmap_page();
-    }
-#endif
-
-    {
-        InterruptDisabler disabler;
-        auto& table = *(PageDirectoryPointerTable*)MM.quickmap_page(*directory->m_directory_table);
-        for (size_t i = 0; i < sizeof(m_directory_pages) / sizeof(m_directory_pages[0]); i++) {
-            if (directory->m_directory_pages[i]) {
-                table.raw[i] = (FlatPtr)directory->m_directory_pages[i]->paddr().as_ptr() | 7;
-            }
-        }
-
-#if ARCH(X86_64)
-        // 2 ** MAXPHYADDR - 1
-        // Where MAXPHYADDR = physical_address_bit_width
-        u64 max_physical_address = (1ULL << Processor::current().physical_address_bit_width()) - 1;
-
-        // bit 63 = no execute
-        // bit 7 = page size
-        // bit 5 = accessed
-        // bit 4 = cache disable
-        // bit 3 = write through
-        // bit 2 = user/supervisor
-        // bit 1 = read/write
-        // bit 0 = present
-        constexpr u64 pdpte_bit_flags = 0x80000000000000BF;
-
-        // This is to notify us of bugs where we're:
-        // 1. Going over what the processor is capable of.
-        // 2. Writing into the reserved bits (51:MAXPHYADDR), where doing so throws a GPF
-        //    when writing out the PDPT pointer to CR3.
-        // The reason we're not checking the page directory's physical address directly is because
-        // we're checking for sign extension when putting it into a PDPTE. See issue #4584.
-        for (auto table_entry : table.raw)
-            VERIFY((table_entry & ~pdpte_bit_flags) <= max_physical_address);
-#endif
-
-        MM.unquickmap_page();
-    }
-
-    register_page_directory(directory);
-    return directory;
-}
-
-PageDirectory::PageDirectory() = default;
-
-UNMAP_AFTER_INIT void PageDirectory::allocate_kernel_directory()
-{
-    // Adopt the page tables already set up by boot.S
-#if ARCH(X86_64)
-    dmesgln("MM: boot_pml4t @ {}", boot_pml4t);
-    m_pml4t = PhysicalPage::create(boot_pml4t, MayReturnToFreeList::No);
-#endif
-    dmesgln("MM: boot_pdpt @ {}", boot_pdpt);
-    dmesgln("MM: boot_pd0 @ {}", boot_pd0);
-    dmesgln("MM: boot_pd_kernel @ {}", boot_pd_kernel);
-    m_directory_table = PhysicalPage::create(boot_pdpt, MayReturnToFreeList::No);
-    m_directory_pages[0] = PhysicalPage::create(boot_pd0, MayReturnToFreeList::No);
-    m_directory_pages[(kernel_mapping_base >> 30) & 0x1ff] = PhysicalPage::create(boot_pd_kernel, MayReturnToFreeList::No);
-}
-
-PageDirectory::~PageDirectory()
-{
-    if (is_cr3_initialized()) {
-        deregister_page_directory(this);
-    }
-}
-
-}

Kernel/Memory/PageDirectory.h

@@ -1,77 +0,0 @@
-/*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#pragma once
-
-#include <AK/AtomicRefCounted.h>
-#include <AK/Badge.h>
-#include <AK/HashMap.h>
-#include <AK/IntrusiveRedBlackTree.h>
-#include <AK/RefPtr.h>
-#include <Kernel/Forward.h>
-#include <Kernel/Locking/Spinlock.h>
-#include <Kernel/Memory/PhysicalPage.h>
-
-namespace Kernel::Memory {
-
-class PageDirectory final : public AtomicRefCounted<PageDirectory> {
-    friend class MemoryManager;
-
-public:
-    static ErrorOr<NonnullLockRefPtr<PageDirectory>> try_create_for_userspace();
-    static NonnullLockRefPtr<PageDirectory> must_create_kernel_page_directory();
-    static LockRefPtr<PageDirectory> find_current();
-
-    ~PageDirectory();
-
-    void allocate_kernel_directory();
-
-    FlatPtr cr3() const
-    {
-#if ARCH(X86_64)
-        return m_pml4t->paddr().get();
-#else
-        return m_directory_table->paddr().get();
-#endif
-    }
-
-    bool is_cr3_initialized() const
-    {
-#if ARCH(X86_64)
-        return m_pml4t;
-#else
-        return m_directory_table;
-#endif
-    }
-
-    AddressSpace* address_space() { return m_space; }
-    AddressSpace const* address_space() const { return m_space; }
-
-    void set_space(Badge<AddressSpace>, AddressSpace& space) { m_space = &space; }
-
-    RecursiveSpinlock<LockRank::None>& get_lock() { return m_lock; }
-
-    // This has to be public to let the global singleton access the member pointer
-    IntrusiveRedBlackTreeNode<FlatPtr, PageDirectory, RawPtr<PageDirectory>> m_tree_node;
-
-private:
-    PageDirectory();
-    static void register_page_directory(PageDirectory* directory);
-    static void deregister_page_directory(PageDirectory* directory);
-
-    AddressSpace* m_space { nullptr };
-#if ARCH(X86_64)
-    RefPtr<PhysicalPage> m_pml4t;
-#endif
-    RefPtr<PhysicalPage> m_directory_table;
-    RefPtr<PhysicalPage> m_directory_pages[512];
-    RecursiveSpinlock<LockRank::None> m_lock {};
-};
-
-void activate_kernel_page_directory(PageDirectory const& pgd);
-void activate_page_directory(PageDirectory const& pgd, Thread* current_thread);
-
-}

Kernel/Memory/Region.cpp

@@ -12,7 +12,6 @@
 #include <Kernel/InterruptDisabler.h>
 #include <Kernel/Memory/AnonymousVMObject.h>
 #include <Kernel/Memory/MemoryManager.h>
-#include <Kernel/Memory/PageDirectory.h>
 #include <Kernel/Memory/Region.h>
 #include <Kernel/Memory/SharedInodeVMObject.h>
 #include <Kernel/Panic.h>

Kernel/Process.cpp

@@ -19,6 +19,7 @@
 #endif
 #include <Kernel/API/POSIX/errno.h>
 #include <Kernel/API/POSIX/sys/limits.h>
+#include <Kernel/Arch/PageDirectory.h>
 #include <Kernel/Devices/NullDevice.h>
 #include <Kernel/FileSystem/Custody.h>
 #include <Kernel/FileSystem/OpenFileDescription.h>
@@ -26,7 +27,6 @@
 #include <Kernel/KBufferBuilder.h>
 #include <Kernel/KSyms.h>
 #include <Kernel/Memory/AnonymousVMObject.h>
-#include <Kernel/Memory/PageDirectory.h>
 #include <Kernel/Memory/SharedInodeVMObject.h>
 #include <Kernel/Panic.h>
 #include <Kernel/PerformanceEventBuffer.h>

Kernel/Syscalls/mmap.cpp

@@ -6,6 +6,7 @@
  */
 
 #include <Kernel/API/VirtualMemoryAnnotations.h>
+#include <Kernel/Arch/PageDirectory.h>
 #include <Kernel/Arch/SafeMem.h>
 #include <Kernel/Arch/SmapDisabler.h>
 #include <Kernel/Arch/x86_64/MSR.h>
@@ -13,7 +14,6 @@
 #include <Kernel/FileSystem/OpenFileDescription.h>
 #include <Kernel/Memory/AnonymousVMObject.h>
 #include <Kernel/Memory/MemoryManager.h>
-#include <Kernel/Memory/PageDirectory.h>
 #include <Kernel/Memory/PrivateInodeVMObject.h>
 #include <Kernel/Memory/Region.h>
 #include <Kernel/Memory/SharedInodeVMObject.h>

Kernel/Thread.cpp

@@ -10,6 +10,7 @@
 #include <AK/TemporaryChange.h>
 #include <AK/Time.h>
 #include <Kernel/API/POSIX/signal_numbers.h>
+#include <Kernel/Arch/PageDirectory.h>
 #include <Kernel/Arch/SmapDisabler.h>
 #include <Kernel/Arch/TrapFrame.h>
 #include <Kernel/Debug.h>
@@ -18,7 +19,6 @@
 #include <Kernel/InterruptDisabler.h>
 #include <Kernel/KSyms.h>
 #include <Kernel/Memory/MemoryManager.h>
-#include <Kernel/Memory/PageDirectory.h>
 #include <Kernel/Memory/ScopedAddressSpaceSwitcher.h>
 #include <Kernel/Panic.h>
 #include <Kernel/PerformanceEventBuffer.h>