From e8f543c3908bad1c79e77d2eff1aebef461cd6b8 Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Sun, 3 Apr 2022 13:28:16 +0200 Subject: [PATCH] Kernel: Use intrusive RegionTree solution for kernel regions as well This patch ports MemoryManager to RegionTree as well. The biggest difference between this and the userspace code is that kernel regions are owned by extant OwnPtr objects spread around the kernel, while userspace regions are owned by the AddressSpace itself. For kernelspace, there are a couple of situations where we need to make large VM reservations that never get backed by regular VMObjects (for example the kernel image reservation, or the big kmalloc range.) Since we can't make a VM reservation without a Region object anymore, this patch adds a way to create unbacked Region objects that can be used for this exact purpose. They have no internal VMObject.) --- Kernel/CMakeLists.txt | 1 - Kernel/Forward.h | 1 - Kernel/Heap/kmalloc.cpp | 10 +++++--- Kernel/Memory/MemoryManager.cpp | 43 ++++++++++++++++++++++----------- Kernel/Memory/MemoryManager.h | 5 +++- Kernel/Memory/PageDirectory.cpp | 11 +-------- Kernel/Memory/PageDirectory.h | 5 ---- Kernel/Memory/Region.cpp | 25 +++++++++++-------- Kernel/Memory/Region.h | 6 +++-- Kernel/Memory/RegionTree.cpp | 8 ++++++ Kernel/Memory/RegionTree.h | 5 ++++ Kernel/Memory/VirtualRange.h | 2 -- 12 files changed, 72 insertions(+), 50 deletions(-) diff --git a/Kernel/CMakeLists.txt b/Kernel/CMakeLists.txt index bc71f366b5f..94871b8ddce 100644 --- a/Kernel/CMakeLists.txt +++ b/Kernel/CMakeLists.txt @@ -177,7 +177,6 @@ set(KERNEL_SOURCES Memory/SharedInodeVMObject.cpp Memory/VMObject.cpp Memory/VirtualRange.cpp - Memory/VirtualRangeAllocator.cpp MiniStdLib.cpp Locking/LockRank.cpp Locking/Mutex.cpp diff --git a/Kernel/Forward.h b/Kernel/Forward.h index f80ca4d5ec9..7fcedb1d2ac 100644 --- a/Kernel/Forward.h +++ b/Kernel/Forward.h @@ -80,7 +80,6 @@ class Region; class SharedInodeVMObject; class VMObject; class VirtualRange; -class VirtualRangeAllocator; } class Spinlock; diff --git a/Kernel/Heap/kmalloc.cpp b/Kernel/Heap/kmalloc.cpp index d0f5fdf63b1..a2b12b6e7a0 100644 --- a/Kernel/Heap/kmalloc.cpp +++ b/Kernel/Heap/kmalloc.cpp @@ -353,20 +353,22 @@ struct KmallocGlobalData { void enable_expansion() { // FIXME: This range can be much bigger on 64-bit, but we need to figure something out for 32-bit. - auto virtual_range = MM.kernel_page_directory().range_allocator().try_allocate_anywhere(64 * MiB, 1 * MiB); + auto reserved_region = MUST(MM.region_tree().allocate_unbacked_anywhere(64 * MiB, 1 * MiB)); expansion_data = KmallocGlobalData::ExpansionData { - .virtual_range = virtual_range.value(), - .next_virtual_address = virtual_range.value().base(), + .virtual_range = reserved_region->range(), + .next_virtual_address = reserved_region->range().base(), }; // Make sure the entire kmalloc VM range is backed by page tables. // This avoids having to deal with lazy page table allocation during heap expansion. SpinlockLocker mm_locker(Memory::s_mm_lock); SpinlockLocker pd_locker(MM.kernel_page_directory().get_lock()); - for (auto vaddr = virtual_range.value().base(); vaddr < virtual_range.value().end(); vaddr = vaddr.offset(PAGE_SIZE)) { + for (auto vaddr = reserved_region->range().base(); vaddr < reserved_region->range().end(); vaddr = vaddr.offset(PAGE_SIZE)) { MM.ensure_pte(MM.kernel_page_directory(), vaddr); } + + (void)reserved_region.leak_ptr(); } struct ExpansionData { diff --git a/Kernel/Memory/MemoryManager.cpp b/Kernel/Memory/MemoryManager.cpp index 833ae228121..874bc868a2c 100644 --- a/Kernel/Memory/MemoryManager.cpp +++ b/Kernel/Memory/MemoryManager.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -74,7 +75,14 @@ bool MemoryManager::is_initialized() return s_the != nullptr; } +static UNMAP_AFTER_INIT VirtualRange kernel_virtual_range() +{ + auto kernel_range_start = kernel_mapping_base + 2 * MiB; // The first 2 MiB are used for mapping the pre-kernel + return VirtualRange { VirtualAddress(kernel_range_start), KERNEL_PD_END - kernel_range_start }; +} + UNMAP_AFTER_INIT MemoryManager::MemoryManager() + : m_region_tree(kernel_virtual_range()) { s_the = this; @@ -439,13 +447,20 @@ UNMAP_AFTER_INIT void MemoryManager::initialize_physical_pages() // Create the bare page directory. This is not a fully constructed page directory and merely contains the allocators! m_kernel_page_directory = PageDirectory::must_create_kernel_page_directory(); - // Allocate a virtual address range for our array - auto range_or_error = m_kernel_page_directory->range_allocator().try_allocate_anywhere(physical_page_array_pages * PAGE_SIZE); - if (range_or_error.is_error()) { - dmesgln("MM: Could not allocate {} bytes to map physical page array!", physical_page_array_pages * PAGE_SIZE); - VERIFY_NOT_REACHED(); + { + // Carve out the whole page directory covering the kernel image to make MemoryManager::initialize_physical_pages() happy + FlatPtr start_of_range = ((FlatPtr)start_of_kernel_image & ~(FlatPtr)0x1fffff); + FlatPtr end_of_range = ((FlatPtr)end_of_kernel_image & ~(FlatPtr)0x1fffff) + 0x200000; + auto reserved_range = MUST(m_region_tree.try_allocate_specific(VirtualAddress(start_of_range), end_of_range - start_of_range)); + (void)MUST(Region::create_unbacked(reserved_range)).leak_ptr(); + } + + // Allocate a virtual address range for our array + auto range = MUST(m_region_tree.try_allocate_anywhere(physical_page_array_pages * PAGE_SIZE)); + + { + (void)MUST(Region::create_unbacked(range)).leak_ptr(); } - auto range = range_or_error.release_value(); // Now that we have our special m_physical_pages_region region with enough pages to hold the entire array // try to map the entire region into kernel space so we always have it @@ -651,7 +666,7 @@ Region* MemoryManager::kernel_region_from_vaddr(VirtualAddress vaddr) return nullptr; SpinlockLocker lock(s_mm_lock); - auto* region = MM.m_kernel_regions.find_largest_not_above(vaddr.get()); + auto* region = MM.m_region_tree.regions().find_largest_not_above(vaddr.get()); if (!region || !region->contains(vaddr)) return nullptr; return region; @@ -757,7 +772,7 @@ ErrorOr> MemoryManager::allocate_contiguous_kernel_region( VERIFY(!(size % PAGE_SIZE)); SpinlockLocker lock(kernel_page_directory().get_lock()); auto vmobject = TRY(AnonymousVMObject::try_create_physically_contiguous_with_size(size)); - auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size)); + auto range = TRY(m_region_tree.try_allocate_anywhere(size)); return allocate_kernel_region_with_vmobject(range, move(vmobject), name, access, cacheable); } @@ -796,7 +811,7 @@ ErrorOr> MemoryManager::allocate_kernel_region(size_t size VERIFY(!(size % PAGE_SIZE)); auto vmobject = TRY(AnonymousVMObject::try_create_with_size(size, strategy)); SpinlockLocker lock(kernel_page_directory().get_lock()); - auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size)); + auto range = TRY(m_region_tree.try_allocate_anywhere(size)); return allocate_kernel_region_with_vmobject(range, move(vmobject), name, access, cacheable); } @@ -805,7 +820,7 @@ ErrorOr> MemoryManager::allocate_kernel_region(PhysicalAdd VERIFY(!(size % PAGE_SIZE)); auto vmobject = TRY(AnonymousVMObject::try_create_for_physical_range(paddr, size)); SpinlockLocker lock(kernel_page_directory().get_lock()); - auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size)); + auto range = TRY(m_region_tree.try_allocate_anywhere(size)); return allocate_kernel_region_with_vmobject(range, move(vmobject), name, access, cacheable); } @@ -823,7 +838,7 @@ ErrorOr> MemoryManager::allocate_kernel_region_with_vmobje { VERIFY(!(size % PAGE_SIZE)); SpinlockLocker lock(kernel_page_directory().get_lock()); - auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size)); + auto range = TRY(m_region_tree.try_allocate_anywhere(size)); return allocate_kernel_region_with_vmobject(range, vmobject, name, access, cacheable); } @@ -1146,14 +1161,14 @@ void MemoryManager::register_kernel_region(Region& region) { VERIFY(region.is_kernel()); SpinlockLocker lock(s_mm_lock); - m_kernel_regions.insert(region.vaddr().get(), region); + m_region_tree.regions().insert(region.vaddr().get(), region); } void MemoryManager::unregister_kernel_region(Region& region) { VERIFY(region.is_kernel()); SpinlockLocker lock(s_mm_lock); - m_kernel_regions.remove(region.vaddr().get()); + m_region_tree.regions().remove(region.vaddr().get()); } void MemoryManager::dump_kernel_regions() @@ -1167,7 +1182,7 @@ void MemoryManager::dump_kernel_regions() dbgln("BEGIN{} END{} SIZE{} ACCESS NAME", addr_padding, addr_padding, addr_padding); SpinlockLocker lock(s_mm_lock); - for (auto const& region : m_kernel_regions) { + for (auto const& region : m_region_tree.regions()) { dbgln("{:p} -- {:p} {:p} {:c}{:c}{:c}{:c}{:c}{:c} {}", region.vaddr().get(), region.vaddr().offset(region.size() - 1).get(), diff --git a/Kernel/Memory/MemoryManager.h b/Kernel/Memory/MemoryManager.h index 73a230d40b5..cac1fdce238 100644 --- a/Kernel/Memory/MemoryManager.h +++ b/Kernel/Memory/MemoryManager.h @@ -18,6 +18,7 @@ #include #include #include +#include #include namespace Kernel { @@ -245,6 +246,8 @@ public: IterationDecision for_each_physical_memory_range(Function); + auto& region_tree() { return m_region_tree; } + private: MemoryManager(); ~MemoryManager(); @@ -297,7 +300,7 @@ private: PhysicalPageEntry* m_physical_page_entries { nullptr }; size_t m_physical_page_entries_count { 0 }; - IntrusiveRedBlackTree<&Region::m_tree_node> m_kernel_regions; + RegionTree m_region_tree; Vector m_used_memory_ranges; Vector m_physical_memory_ranges; diff --git a/Kernel/Memory/PageDirectory.cpp b/Kernel/Memory/PageDirectory.cpp index a1056074aa9..5ce21fd65f8 100644 --- a/Kernel/Memory/PageDirectory.cpp +++ b/Kernel/Memory/PageDirectory.cpp @@ -22,16 +22,7 @@ namespace Kernel::Memory { UNMAP_AFTER_INIT NonnullRefPtr PageDirectory::must_create_kernel_page_directory() { - auto directory = adopt_ref_if_nonnull(new (nothrow) PageDirectory).release_nonnull(); - - auto kernel_range_start = kernel_mapping_base + 2 * MiB; // The first 2 MiB are used for mapping the pre-kernel - MUST(directory->m_range_allocator.initialize_with_range(VirtualAddress(kernel_range_start), KERNEL_PD_END - kernel_range_start)); - // Carve out the whole page directory covering the kernel image to make MemoryManager::initialize_physical_pages() happy - FlatPtr start_of_range = ((FlatPtr)start_of_kernel_image & ~(FlatPtr)0x1fffff); - FlatPtr end_of_range = ((FlatPtr)end_of_kernel_image & ~(FlatPtr)0x1fffff) + 0x200000; - MUST(directory->m_range_allocator.try_allocate_specific(VirtualAddress(start_of_range), end_of_range - start_of_range)); - - return directory; + return adopt_ref_if_nonnull(new (nothrow) PageDirectory).release_nonnull(); } ErrorOr> PageDirectory::try_create_for_userspace() diff --git a/Kernel/Memory/PageDirectory.h b/Kernel/Memory/PageDirectory.h index 5bad876153b..2e09106e0b2 100644 --- a/Kernel/Memory/PageDirectory.h +++ b/Kernel/Memory/PageDirectory.h @@ -13,7 +13,6 @@ #include #include #include -#include namespace Kernel::Memory { @@ -47,9 +46,6 @@ public: #endif } - VirtualRangeAllocator& range_allocator() { return m_range_allocator; } - VirtualRangeAllocator const& range_allocator() const { return m_range_allocator; } - AddressSpace* address_space() { return m_space; } AddressSpace const* address_space() const { return m_space; } @@ -66,7 +62,6 @@ private: static void deregister_page_directory(PageDirectory* directory); AddressSpace* m_space { nullptr }; - VirtualRangeAllocator m_range_allocator; #if ARCH(X86_64) RefPtr m_pml4t; #endif diff --git a/Kernel/Memory/Region.cpp b/Kernel/Memory/Region.cpp index 962e8087e03..69d48bbea1e 100644 --- a/Kernel/Memory/Region.cpp +++ b/Kernel/Memory/Region.cpp @@ -22,6 +22,13 @@ namespace Kernel::Memory { +Region::Region(VirtualRange const& range) + : m_range(range) +{ + if (is_kernel()) + MM.register_kernel_region(*this); +} + Region::Region(VirtualRange const& range, NonnullRefPtr vmobject, size_t offset_in_vmobject, OwnPtr name, Region::Access access, Cacheable cacheable, bool shared) : m_range(range) , m_offset_in_vmobject(offset_in_vmobject) @@ -56,10 +63,7 @@ Region::~Region() if (m_page_directory) { SpinlockLocker pd_locker(m_page_directory->get_lock()); if (!is_readable() && !is_writable() && !is_executable()) { - // If the region is "PROT_NONE", we didn't map it in the first place, - // so all we need to do here is deallocate the VM. - if (is_kernel()) - m_page_directory->range_allocator().deallocate(range()); + // If the region is "PROT_NONE", we didn't map it in the first place. } else { SpinlockLocker mm_locker(s_mm_lock); unmap_with_locks_held(ShouldDeallocateVirtualRange::Yes, ShouldFlushTLB::Yes, pd_locker, mm_locker); @@ -68,6 +72,11 @@ Region::~Region() } } +ErrorOr> Region::create_unbacked(VirtualRange const& range) +{ + return adopt_nonnull_own_or_enomem(new (nothrow) Region(range)); +} + ErrorOr> Region::try_clone() { VERIFY(Process::has_current()); @@ -84,7 +93,7 @@ ErrorOr> Region::try_clone() region_name = TRY(m_name->try_clone()); auto region = TRY(Region::try_create_user_accessible( - m_range, m_vmobject, m_offset_in_vmobject, move(region_name), access(), m_cacheable ? Cacheable::Yes : Cacheable::No, m_shared)); + m_range, vmobject(), m_offset_in_vmobject, move(region_name), access(), m_cacheable ? Cacheable::Yes : Cacheable::No, m_shared)); region->set_mmap(m_mmap); region->set_shared(m_shared); region->set_syscall_region(is_syscall_region()); @@ -259,7 +268,7 @@ void Region::unmap(ShouldDeallocateVirtualRange should_deallocate_range, ShouldF unmap_with_locks_held(should_deallocate_range, should_flush_tlb, pd_locker, mm_locker); } -void Region::unmap_with_locks_held(ShouldDeallocateVirtualRange deallocate_range, ShouldFlushTLB should_flush_tlb, SpinlockLocker&, SpinlockLocker&) +void Region::unmap_with_locks_held(ShouldDeallocateVirtualRange, ShouldFlushTLB should_flush_tlb, SpinlockLocker&, SpinlockLocker&) { if (!m_page_directory) return; @@ -270,10 +279,6 @@ void Region::unmap_with_locks_held(ShouldDeallocateVirtualRange deallocate_range } if (should_flush_tlb == ShouldFlushTLB::Yes) MemoryManager::flush_tlb(m_page_directory, vaddr(), page_count()); - if (deallocate_range == ShouldDeallocateVirtualRange::Yes) { - if (is_kernel()) - m_page_directory->range_allocator().deallocate(range()); - } m_page_directory = nullptr; } diff --git a/Kernel/Memory/Region.h b/Kernel/Memory/Region.h index 9b07a53c72c..bc73075c851 100644 --- a/Kernel/Memory/Region.h +++ b/Kernel/Memory/Region.h @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include @@ -56,6 +56,7 @@ public: static ErrorOr> try_create_user_accessible(VirtualRange const&, NonnullRefPtr, size_t offset_in_vmobject, OwnPtr name, Region::Access access, Cacheable, bool shared); static ErrorOr> try_create_kernel_only(VirtualRange const&, NonnullRefPtr, size_t offset_in_vmobject, OwnPtr name, Region::Access access, Cacheable = Cacheable::Yes); + static ErrorOr> create_unbacked(VirtualRange const&); ~Region(); @@ -198,6 +199,7 @@ public: void set_syscall_region(bool b) { m_syscall_region = b; } private: + explicit Region(VirtualRange const&); Region(VirtualRange const&, NonnullRefPtr, size_t offset_in_vmobject, OwnPtr, Region::Access access, Cacheable, bool shared); [[nodiscard]] bool remap_vmobject_page(size_t page_index, bool with_flush = true); @@ -220,7 +222,7 @@ private: RefPtr m_page_directory; VirtualRange m_range; size_t m_offset_in_vmobject { 0 }; - NonnullRefPtr m_vmobject; + RefPtr m_vmobject; OwnPtr m_name; u8 m_access { Region::None }; bool m_shared : 1 { false }; diff --git a/Kernel/Memory/RegionTree.cpp b/Kernel/Memory/RegionTree.cpp index b2cb2ce7085..67d3ef15806 100644 --- a/Kernel/Memory/RegionTree.cpp +++ b/Kernel/Memory/RegionTree.cpp @@ -140,4 +140,12 @@ ErrorOr RegionTree::try_allocate_randomized(size_t size, size_t al return try_allocate_anywhere(size, alignment); } + +ErrorOr> RegionTree::allocate_unbacked_anywhere(size_t size, size_t alignment) +{ + SpinlockLocker locker(m_lock); + auto range = TRY(try_allocate_anywhere(size, alignment)); + return Region::create_unbacked(range); +} + } diff --git a/Kernel/Memory/RegionTree.h b/Kernel/Memory/RegionTree.h index 386ba475b16..9a00db9f43a 100644 --- a/Kernel/Memory/RegionTree.h +++ b/Kernel/Memory/RegionTree.h @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -31,6 +32,8 @@ public: VirtualRange total_range() const { return m_total_range; } + ErrorOr> allocate_unbacked_anywhere(size_t size, size_t alignment = PAGE_SIZE); + ErrorOr try_allocate_anywhere(size_t size, size_t alignment = PAGE_SIZE); ErrorOr try_allocate_specific(VirtualAddress base, size_t size); ErrorOr try_allocate_randomized(size_t size, size_t alignment = PAGE_SIZE); @@ -38,6 +41,8 @@ public: void delete_all_regions_assuming_they_are_unmapped(); private: + Spinlock m_lock; + IntrusiveRedBlackTree<&Region::m_tree_node> m_regions; VirtualRange const m_total_range; }; diff --git a/Kernel/Memory/VirtualRange.h b/Kernel/Memory/VirtualRange.h index 3d69fa6b2bb..071f78bb371 100644 --- a/Kernel/Memory/VirtualRange.h +++ b/Kernel/Memory/VirtualRange.h @@ -13,8 +13,6 @@ namespace Kernel::Memory { class VirtualRange { - friend class VirtualRangeAllocator; - public: VirtualRange() = delete; VirtualRange(VirtualAddress base, size_t size)