MemoryManager.cpp 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130
  1. /*
  2. * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <AK/Assertions.h>
  7. #include <AK/Memory.h>
  8. #include <AK/StringView.h>
  9. #include <Kernel/Arch/x86/PageFault.h>
  10. #include <Kernel/BootInfo.h>
  11. #include <Kernel/CMOS.h>
  12. #include <Kernel/FileSystem/Inode.h>
  13. #include <Kernel/Heap/kmalloc.h>
  14. #include <Kernel/Memory/AnonymousVMObject.h>
  15. #include <Kernel/Memory/MemoryManager.h>
  16. #include <Kernel/Memory/PageDirectory.h>
  17. #include <Kernel/Memory/PhysicalRegion.h>
  18. #include <Kernel/Memory/SharedInodeVMObject.h>
  19. #include <Kernel/Multiboot.h>
  20. #include <Kernel/Panic.h>
  21. #include <Kernel/Process.h>
  22. #include <Kernel/Sections.h>
  23. #include <Kernel/StdLib.h>
  24. extern u8 start_of_kernel_image[];
  25. extern u8 end_of_kernel_image[];
  26. extern u8 start_of_kernel_text[];
  27. extern u8 start_of_kernel_data[];
  28. extern u8 end_of_kernel_bss[];
  29. extern u8 start_of_ro_after_init[];
  30. extern u8 end_of_ro_after_init[];
  31. extern u8 start_of_unmap_after_init[];
  32. extern u8 end_of_unmap_after_init[];
  33. extern u8 start_of_kernel_ksyms[];
  34. extern u8 end_of_kernel_ksyms[];
  35. extern multiboot_module_entry_t multiboot_copy_boot_modules_array[16];
  36. extern size_t multiboot_copy_boot_modules_count;
  37. // Treat the super pages as logically separate from .bss
  38. // FIXME: Find a solution so we don't need to expand this range each time
  39. // we are in a situation too many drivers try to allocate super pages.
  40. __attribute__((section(".super_pages"))) static u8 super_pages[4 * MiB];
  41. namespace Kernel::Memory {
  42. // NOTE: We can NOT use Singleton for this class, because
  43. // MemoryManager::initialize is called *before* global constructors are
  44. // run. If we do, then Singleton would get re-initialized, causing
  45. // the memory manager to be initialized twice!
  46. static MemoryManager* s_the;
  47. RecursiveSpinlock s_mm_lock { LockRank::MemoryManager };
  48. MemoryManager& MemoryManager::the()
  49. {
  50. return *s_the;
  51. }
  52. bool MemoryManager::is_initialized()
  53. {
  54. return s_the != nullptr;
  55. }
  56. UNMAP_AFTER_INIT MemoryManager::MemoryManager()
  57. {
  58. s_the = this;
  59. SpinlockLocker lock(s_mm_lock);
  60. parse_memory_map();
  61. write_cr3(kernel_page_directory().cr3());
  62. protect_kernel_image();
  63. // We're temporarily "committing" to two pages that we need to allocate below
  64. auto committed_pages = commit_user_physical_pages(2).release_value();
  65. m_shared_zero_page = committed_pages.take_one();
  66. // We're wasting a page here, we just need a special tag (physical
  67. // address) so that we know when we need to lazily allocate a page
  68. // that we should be drawing this page from the committed pool rather
  69. // than potentially failing if no pages are available anymore.
  70. // By using a tag we don't have to query the VMObject for every page
  71. // whether it was committed or not
  72. m_lazy_committed_page = committed_pages.take_one();
  73. }
  74. UNMAP_AFTER_INIT MemoryManager::~MemoryManager()
  75. {
  76. }
  77. UNMAP_AFTER_INIT void MemoryManager::protect_kernel_image()
  78. {
  79. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  80. // Disable writing to the kernel text and rodata segments.
  81. for (auto i = start_of_kernel_text; i < start_of_kernel_data; i += PAGE_SIZE) {
  82. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  83. pte.set_writable(false);
  84. }
  85. if (Processor::current().has_feature(CPUFeature::NX)) {
  86. // Disable execution of the kernel data, bss and heap segments.
  87. for (auto i = start_of_kernel_data; i < end_of_kernel_image; i += PAGE_SIZE) {
  88. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  89. pte.set_execute_disabled(true);
  90. }
  91. }
  92. }
  93. UNMAP_AFTER_INIT void MemoryManager::protect_readonly_after_init_memory()
  94. {
  95. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  96. SpinlockLocker mm_lock(s_mm_lock);
  97. // Disable writing to the .ro_after_init section
  98. for (auto i = (FlatPtr)&start_of_ro_after_init; i < (FlatPtr)&end_of_ro_after_init; i += PAGE_SIZE) {
  99. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  100. pte.set_writable(false);
  101. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  102. }
  103. }
  104. void MemoryManager::unmap_text_after_init()
  105. {
  106. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  107. SpinlockLocker mm_lock(s_mm_lock);
  108. auto start = page_round_down((FlatPtr)&start_of_unmap_after_init);
  109. auto end = page_round_up((FlatPtr)&end_of_unmap_after_init);
  110. // Unmap the entire .unmap_after_init section
  111. for (auto i = start; i < end; i += PAGE_SIZE) {
  112. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  113. pte.clear();
  114. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  115. }
  116. dmesgln("Unmapped {} KiB of kernel text after init! :^)", (end - start) / KiB);
  117. }
  118. void MemoryManager::unmap_ksyms_after_init()
  119. {
  120. SpinlockLocker mm_lock(s_mm_lock);
  121. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  122. auto start = page_round_down((FlatPtr)start_of_kernel_ksyms);
  123. auto end = page_round_up((FlatPtr)end_of_kernel_ksyms);
  124. // Unmap the entire .ksyms section
  125. for (auto i = start; i < end; i += PAGE_SIZE) {
  126. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  127. pte.clear();
  128. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  129. }
  130. dmesgln("Unmapped {} KiB of kernel symbols after init! :^)", (end - start) / KiB);
  131. }
  132. UNMAP_AFTER_INIT void MemoryManager::register_reserved_ranges()
  133. {
  134. VERIFY(!m_physical_memory_ranges.is_empty());
  135. ContiguousReservedMemoryRange range;
  136. for (auto& current_range : m_physical_memory_ranges) {
  137. if (current_range.type != PhysicalMemoryRangeType::Reserved) {
  138. if (range.start.is_null())
  139. continue;
  140. m_reserved_memory_ranges.append(ContiguousReservedMemoryRange { range.start, current_range.start.get() - range.start.get() });
  141. range.start.set((FlatPtr) nullptr);
  142. continue;
  143. }
  144. if (!range.start.is_null()) {
  145. continue;
  146. }
  147. range.start = current_range.start;
  148. }
  149. if (m_physical_memory_ranges.last().type != PhysicalMemoryRangeType::Reserved)
  150. return;
  151. if (range.start.is_null())
  152. return;
  153. m_reserved_memory_ranges.append(ContiguousReservedMemoryRange { range.start, m_physical_memory_ranges.last().start.get() + m_physical_memory_ranges.last().length - range.start.get() });
  154. }
  155. bool MemoryManager::is_allowed_to_mmap_to_userspace(PhysicalAddress start_address, VirtualRange const& range) const
  156. {
  157. VERIFY(!m_reserved_memory_ranges.is_empty());
  158. // Note: Guard against overflow in case someone tries to mmap on the edge of
  159. // the RAM
  160. if (start_address.offset_addition_would_overflow(range.size()))
  161. return false;
  162. auto end_address = start_address.offset(range.size());
  163. for (auto& current_range : m_reserved_memory_ranges) {
  164. if (current_range.start > start_address)
  165. continue;
  166. if (current_range.start.offset(current_range.length) < end_address)
  167. continue;
  168. return true;
  169. }
  170. return false;
  171. }
  172. UNMAP_AFTER_INIT void MemoryManager::parse_memory_map()
  173. {
  174. // Register used memory regions that we know of.
  175. m_used_memory_ranges.ensure_capacity(4);
  176. m_used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::LowMemory, PhysicalAddress(0x00000000), PhysicalAddress(1 * MiB) });
  177. m_used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::Prekernel, start_of_prekernel_image, end_of_prekernel_image });
  178. m_used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::Kernel, PhysicalAddress(virtual_to_low_physical((FlatPtr)start_of_kernel_image)), PhysicalAddress(page_round_up(virtual_to_low_physical((FlatPtr)end_of_kernel_image))) });
  179. if (multiboot_flags & 0x4) {
  180. auto* bootmods_start = multiboot_copy_boot_modules_array;
  181. auto* bootmods_end = bootmods_start + multiboot_copy_boot_modules_count;
  182. for (auto* bootmod = bootmods_start; bootmod < bootmods_end; bootmod++) {
  183. m_used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::BootModule, PhysicalAddress(bootmod->start), PhysicalAddress(bootmod->end) });
  184. }
  185. }
  186. auto* mmap_begin = multiboot_memory_map;
  187. auto* mmap_end = multiboot_memory_map + multiboot_memory_map_count;
  188. struct ContiguousPhysicalVirtualRange {
  189. PhysicalAddress lower;
  190. PhysicalAddress upper;
  191. };
  192. Vector<ContiguousPhysicalVirtualRange> contiguous_physical_ranges;
  193. for (auto* mmap = mmap_begin; mmap < mmap_end; mmap++) {
  194. // We have to copy these onto the stack, because we take a reference to these when printing them out,
  195. // and doing so on a packed struct field is UB.
  196. auto address = mmap->addr;
  197. auto length = mmap->len;
  198. ArmedScopeGuard write_back_guard = [&]() {
  199. mmap->addr = address;
  200. mmap->len = length;
  201. };
  202. dmesgln("MM: Multiboot mmap: address={:p}, length={}, type={}", address, length, mmap->type);
  203. auto start_address = PhysicalAddress(address);
  204. switch (mmap->type) {
  205. case (MULTIBOOT_MEMORY_AVAILABLE):
  206. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Usable, start_address, length });
  207. break;
  208. case (MULTIBOOT_MEMORY_RESERVED):
  209. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Reserved, start_address, length });
  210. break;
  211. case (MULTIBOOT_MEMORY_ACPI_RECLAIMABLE):
  212. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_Reclaimable, start_address, length });
  213. break;
  214. case (MULTIBOOT_MEMORY_NVS):
  215. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_NVS, start_address, length });
  216. break;
  217. case (MULTIBOOT_MEMORY_BADRAM):
  218. dmesgln("MM: Warning, detected bad memory range!");
  219. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::BadMemory, start_address, length });
  220. break;
  221. default:
  222. dbgln("MM: Unknown range!");
  223. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Unknown, start_address, length });
  224. break;
  225. }
  226. if (mmap->type != MULTIBOOT_MEMORY_AVAILABLE)
  227. continue;
  228. // Fix up unaligned memory regions.
  229. auto diff = (FlatPtr)address % PAGE_SIZE;
  230. if (diff != 0) {
  231. dmesgln("MM: Got an unaligned physical_region from the bootloader; correcting {:p} by {} bytes", address, diff);
  232. diff = PAGE_SIZE - diff;
  233. address += diff;
  234. length -= diff;
  235. }
  236. if ((length % PAGE_SIZE) != 0) {
  237. dmesgln("MM: Got an unaligned physical_region from the bootloader; correcting length {} by {} bytes", length, length % PAGE_SIZE);
  238. length -= length % PAGE_SIZE;
  239. }
  240. if (length < PAGE_SIZE) {
  241. dmesgln("MM: Memory physical_region from bootloader is too small; we want >= {} bytes, but got {} bytes", PAGE_SIZE, length);
  242. continue;
  243. }
  244. for (PhysicalSize page_base = address; page_base <= (address + length); page_base += PAGE_SIZE) {
  245. auto addr = PhysicalAddress(page_base);
  246. // Skip used memory ranges.
  247. bool should_skip = false;
  248. for (auto& used_range : m_used_memory_ranges) {
  249. if (addr.get() >= used_range.start.get() && addr.get() <= used_range.end.get()) {
  250. should_skip = true;
  251. break;
  252. }
  253. }
  254. if (should_skip)
  255. continue;
  256. if (contiguous_physical_ranges.is_empty() || contiguous_physical_ranges.last().upper.offset(PAGE_SIZE) != addr) {
  257. contiguous_physical_ranges.append(ContiguousPhysicalVirtualRange {
  258. .lower = addr,
  259. .upper = addr,
  260. });
  261. } else {
  262. contiguous_physical_ranges.last().upper = addr;
  263. }
  264. }
  265. }
  266. for (auto& range : contiguous_physical_ranges) {
  267. m_user_physical_regions.append(PhysicalRegion::try_create(range.lower, range.upper).release_nonnull());
  268. }
  269. // Super pages are guaranteed to be in the first 16MB of physical memory
  270. VERIFY(virtual_to_low_physical((FlatPtr)super_pages) + sizeof(super_pages) < 0x1000000);
  271. // Append statically-allocated super physical physical_region.
  272. m_super_physical_region = PhysicalRegion::try_create(
  273. PhysicalAddress(virtual_to_low_physical(FlatPtr(super_pages))),
  274. PhysicalAddress(virtual_to_low_physical(FlatPtr(super_pages + sizeof(super_pages)))));
  275. VERIFY(m_super_physical_region);
  276. m_system_memory_info.super_physical_pages += m_super_physical_region->size();
  277. for (auto& region : m_user_physical_regions)
  278. m_system_memory_info.user_physical_pages += region.size();
  279. register_reserved_ranges();
  280. for (auto& range : m_reserved_memory_ranges) {
  281. dmesgln("MM: Contiguous reserved range from {}, length is {}", range.start, range.length);
  282. }
  283. initialize_physical_pages();
  284. VERIFY(m_system_memory_info.super_physical_pages > 0);
  285. VERIFY(m_system_memory_info.user_physical_pages > 0);
  286. // We start out with no committed pages
  287. m_system_memory_info.user_physical_pages_uncommitted = m_system_memory_info.user_physical_pages;
  288. for (auto& used_range : m_used_memory_ranges) {
  289. dmesgln("MM: {} range @ {} - {} (size {:#x})", UserMemoryRangeTypeNames[to_underlying(used_range.type)], used_range.start, used_range.end.offset(-1), used_range.end.as_ptr() - used_range.start.as_ptr());
  290. }
  291. dmesgln("MM: Super physical region: {} - {} (size {:#x})", m_super_physical_region->lower(), m_super_physical_region->upper().offset(-1), PAGE_SIZE * m_super_physical_region->size());
  292. m_super_physical_region->initialize_zones();
  293. for (auto& region : m_user_physical_regions) {
  294. dmesgln("MM: User physical region: {} - {} (size {:#x})", region.lower(), region.upper().offset(-1), PAGE_SIZE * region.size());
  295. region.initialize_zones();
  296. }
  297. }
  298. UNMAP_AFTER_INIT void MemoryManager::initialize_physical_pages()
  299. {
  300. // We assume that the physical page range is contiguous and doesn't contain huge gaps!
  301. PhysicalAddress highest_physical_address;
  302. for (auto& range : m_used_memory_ranges) {
  303. if (range.end.get() > highest_physical_address.get())
  304. highest_physical_address = range.end;
  305. }
  306. for (auto& region : m_physical_memory_ranges) {
  307. auto range_end = PhysicalAddress(region.start).offset(region.length);
  308. if (range_end.get() > highest_physical_address.get())
  309. highest_physical_address = range_end;
  310. }
  311. // Calculate how many total physical pages the array will have
  312. m_physical_page_entries_count = PhysicalAddress::physical_page_index(highest_physical_address.get()) + 1;
  313. VERIFY(m_physical_page_entries_count != 0);
  314. VERIFY(!Checked<decltype(m_physical_page_entries_count)>::multiplication_would_overflow(m_physical_page_entries_count, sizeof(PhysicalPageEntry)));
  315. // Calculate how many bytes the array will consume
  316. auto physical_page_array_size = m_physical_page_entries_count * sizeof(PhysicalPageEntry);
  317. auto physical_page_array_pages = page_round_up(physical_page_array_size) / PAGE_SIZE;
  318. VERIFY(physical_page_array_pages * PAGE_SIZE >= physical_page_array_size);
  319. // Calculate how many page tables we will need to be able to map them all
  320. auto needed_page_table_count = (physical_page_array_pages + 512 - 1) / 512;
  321. auto physical_page_array_pages_and_page_tables_count = physical_page_array_pages + needed_page_table_count;
  322. // Now that we know how much memory we need for a contiguous array of PhysicalPage instances, find a memory region that can fit it
  323. PhysicalRegion* found_region { nullptr };
  324. Optional<size_t> found_region_index;
  325. for (size_t i = 0; i < m_user_physical_regions.size(); ++i) {
  326. auto& region = m_user_physical_regions[i];
  327. if (region.size() >= physical_page_array_pages_and_page_tables_count) {
  328. found_region = &region;
  329. found_region_index = i;
  330. break;
  331. }
  332. }
  333. if (!found_region) {
  334. dmesgln("MM: Need {} bytes for physical page management, but no memory region is large enough!", physical_page_array_pages_and_page_tables_count);
  335. VERIFY_NOT_REACHED();
  336. }
  337. VERIFY(m_system_memory_info.user_physical_pages >= physical_page_array_pages_and_page_tables_count);
  338. m_system_memory_info.user_physical_pages -= physical_page_array_pages_and_page_tables_count;
  339. if (found_region->size() == physical_page_array_pages_and_page_tables_count) {
  340. // We're stealing the entire region
  341. m_physical_pages_region = m_user_physical_regions.take(*found_region_index);
  342. } else {
  343. m_physical_pages_region = found_region->try_take_pages_from_beginning(physical_page_array_pages_and_page_tables_count);
  344. }
  345. m_used_memory_ranges.append({ UsedMemoryRangeType::PhysicalPages, m_physical_pages_region->lower(), m_physical_pages_region->upper() });
  346. // Create the bare page directory. This is not a fully constructed page directory and merely contains the allocators!
  347. m_kernel_page_directory = PageDirectory::must_create_kernel_page_directory();
  348. // Allocate a virtual address range for our array
  349. auto range_or_error = m_kernel_page_directory->range_allocator().try_allocate_anywhere(physical_page_array_pages * PAGE_SIZE);
  350. if (range_or_error.is_error()) {
  351. dmesgln("MM: Could not allocate {} bytes to map physical page array!", physical_page_array_pages * PAGE_SIZE);
  352. VERIFY_NOT_REACHED();
  353. }
  354. auto range = range_or_error.release_value();
  355. // Now that we have our special m_physical_pages_region region with enough pages to hold the entire array
  356. // try to map the entire region into kernel space so we always have it
  357. // We can't use ensure_pte here because it would try to allocate a PhysicalPage and we don't have the array
  358. // mapped yet so we can't create them
  359. SpinlockLocker lock(s_mm_lock);
  360. // Create page tables at the beginning of m_physical_pages_region, followed by the PhysicalPageEntry array
  361. auto page_tables_base = m_physical_pages_region->lower();
  362. auto physical_page_array_base = page_tables_base.offset(needed_page_table_count * PAGE_SIZE);
  363. auto physical_page_array_current_page = physical_page_array_base.get();
  364. auto virtual_page_array_base = range.base().get();
  365. auto virtual_page_array_current_page = virtual_page_array_base;
  366. for (size_t pt_index = 0; pt_index < needed_page_table_count; pt_index++) {
  367. auto virtual_page_base_for_this_pt = virtual_page_array_current_page;
  368. auto pt_paddr = page_tables_base.offset(pt_index * PAGE_SIZE);
  369. auto* pt = reinterpret_cast<PageTableEntry*>(quickmap_page(pt_paddr));
  370. __builtin_memset(pt, 0, PAGE_SIZE);
  371. for (size_t pte_index = 0; pte_index < PAGE_SIZE / sizeof(PageTableEntry); pte_index++) {
  372. auto& pte = pt[pte_index];
  373. pte.set_physical_page_base(physical_page_array_current_page);
  374. pte.set_user_allowed(false);
  375. pte.set_writable(true);
  376. if (Processor::current().has_feature(CPUFeature::NX))
  377. pte.set_execute_disabled(false);
  378. pte.set_global(true);
  379. pte.set_present(true);
  380. physical_page_array_current_page += PAGE_SIZE;
  381. virtual_page_array_current_page += PAGE_SIZE;
  382. }
  383. unquickmap_page();
  384. // Hook the page table into the kernel page directory
  385. u32 page_directory_index = (virtual_page_base_for_this_pt >> 21) & 0x1ff;
  386. auto* pd = reinterpret_cast<PageDirectoryEntry*>(quickmap_page(boot_pd_kernel));
  387. PageDirectoryEntry& pde = pd[page_directory_index];
  388. VERIFY(!pde.is_present()); // Nothing should be using this PD yet
  389. // We can't use ensure_pte quite yet!
  390. pde.set_page_table_base(pt_paddr.get());
  391. pde.set_user_allowed(false);
  392. pde.set_present(true);
  393. pde.set_writable(true);
  394. pde.set_global(true);
  395. unquickmap_page();
  396. flush_tlb_local(VirtualAddress(virtual_page_base_for_this_pt));
  397. }
  398. // We now have the entire PhysicalPageEntry array mapped!
  399. m_physical_page_entries = (PhysicalPageEntry*)range.base().get();
  400. for (size_t i = 0; i < m_physical_page_entries_count; i++)
  401. new (&m_physical_page_entries[i]) PageTableEntry();
  402. // Now we should be able to allocate PhysicalPage instances,
  403. // so finish setting up the kernel page directory
  404. m_kernel_page_directory->allocate_kernel_directory();
  405. // Now create legit PhysicalPage objects for the page tables we created, so that
  406. // we can put them into kernel_page_directory().m_page_tables
  407. auto& kernel_page_tables = kernel_page_directory().m_page_tables;
  408. virtual_page_array_current_page = virtual_page_array_base;
  409. for (size_t pt_index = 0; pt_index < needed_page_table_count; pt_index++) {
  410. VERIFY(virtual_page_array_current_page <= range.end().get());
  411. auto pt_paddr = page_tables_base.offset(pt_index * PAGE_SIZE);
  412. auto physical_page_index = PhysicalAddress::physical_page_index(pt_paddr.get());
  413. auto& physical_page_entry = m_physical_page_entries[physical_page_index];
  414. auto physical_page = adopt_ref(*new (&physical_page_entry.allocated.physical_page) PhysicalPage(MayReturnToFreeList::No));
  415. auto result = kernel_page_tables.set(virtual_page_array_current_page & ~0x1fffff, move(physical_page));
  416. VERIFY(result == AK::HashSetResult::InsertedNewEntry);
  417. virtual_page_array_current_page += (PAGE_SIZE / sizeof(PageTableEntry)) * PAGE_SIZE;
  418. }
  419. dmesgln("MM: Physical page entries: {}", range);
  420. }
  421. PhysicalPageEntry& MemoryManager::get_physical_page_entry(PhysicalAddress physical_address)
  422. {
  423. VERIFY(m_physical_page_entries);
  424. auto physical_page_entry_index = PhysicalAddress::physical_page_index(physical_address.get());
  425. VERIFY(physical_page_entry_index < m_physical_page_entries_count);
  426. return m_physical_page_entries[physical_page_entry_index];
  427. }
  428. PhysicalAddress MemoryManager::get_physical_address(PhysicalPage const& physical_page)
  429. {
  430. PhysicalPageEntry const& physical_page_entry = *reinterpret_cast<PhysicalPageEntry const*>((u8 const*)&physical_page - __builtin_offsetof(PhysicalPageEntry, allocated.physical_page));
  431. VERIFY(m_physical_page_entries);
  432. size_t physical_page_entry_index = &physical_page_entry - m_physical_page_entries;
  433. VERIFY(physical_page_entry_index < m_physical_page_entries_count);
  434. return PhysicalAddress((PhysicalPtr)physical_page_entry_index * PAGE_SIZE);
  435. }
  436. PageTableEntry* MemoryManager::pte(PageDirectory& page_directory, VirtualAddress vaddr)
  437. {
  438. VERIFY_INTERRUPTS_DISABLED();
  439. VERIFY(s_mm_lock.is_locked_by_current_processor());
  440. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  441. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  442. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  443. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  444. auto* pd = quickmap_pd(const_cast<PageDirectory&>(page_directory), page_directory_table_index);
  445. PageDirectoryEntry const& pde = pd[page_directory_index];
  446. if (!pde.is_present())
  447. return nullptr;
  448. return &quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()))[page_table_index];
  449. }
  450. PageTableEntry* MemoryManager::ensure_pte(PageDirectory& page_directory, VirtualAddress vaddr)
  451. {
  452. VERIFY_INTERRUPTS_DISABLED();
  453. VERIFY(s_mm_lock.is_locked_by_current_processor());
  454. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  455. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  456. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  457. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  458. auto* pd = quickmap_pd(page_directory, page_directory_table_index);
  459. PageDirectoryEntry& pde = pd[page_directory_index];
  460. if (!pde.is_present()) {
  461. bool did_purge = false;
  462. auto page_table = allocate_user_physical_page(ShouldZeroFill::Yes, &did_purge);
  463. if (!page_table) {
  464. dbgln("MM: Unable to allocate page table to map {}", vaddr);
  465. return nullptr;
  466. }
  467. if (did_purge) {
  468. // If any memory had to be purged, ensure_pte may have been called as part
  469. // of the purging process. So we need to re-map the pd in this case to ensure
  470. // we're writing to the correct underlying physical page
  471. pd = quickmap_pd(page_directory, page_directory_table_index);
  472. VERIFY(&pde == &pd[page_directory_index]); // Sanity check
  473. VERIFY(!pde.is_present()); // Should have not changed
  474. }
  475. pde.set_page_table_base(page_table->paddr().get());
  476. pde.set_user_allowed(true);
  477. pde.set_present(true);
  478. pde.set_writable(true);
  479. pde.set_global(&page_directory == m_kernel_page_directory.ptr());
  480. // Use page_directory_table_index and page_directory_index as key
  481. // This allows us to release the page table entry when no longer needed
  482. auto result = page_directory.m_page_tables.set(vaddr.get() & ~(FlatPtr)0x1fffff, page_table.release_nonnull());
  483. // If you're hitting this VERIFY on x86_64 chances are a 64-bit pointer was truncated somewhere
  484. VERIFY(result == AK::HashSetResult::InsertedNewEntry);
  485. }
  486. return &quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()))[page_table_index];
  487. }
  488. void MemoryManager::release_pte(PageDirectory& page_directory, VirtualAddress vaddr, bool is_last_release)
  489. {
  490. VERIFY_INTERRUPTS_DISABLED();
  491. VERIFY(s_mm_lock.is_locked_by_current_processor());
  492. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  493. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  494. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  495. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  496. auto* pd = quickmap_pd(page_directory, page_directory_table_index);
  497. PageDirectoryEntry& pde = pd[page_directory_index];
  498. if (pde.is_present()) {
  499. auto* page_table = quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()));
  500. auto& pte = page_table[page_table_index];
  501. pte.clear();
  502. if (is_last_release || page_table_index == 0x1ff) {
  503. // If this is the last PTE in a region or the last PTE in a page table then
  504. // check if we can also release the page table
  505. bool all_clear = true;
  506. for (u32 i = 0; i <= 0x1ff; i++) {
  507. if (!page_table[i].is_null()) {
  508. all_clear = false;
  509. break;
  510. }
  511. }
  512. if (all_clear) {
  513. pde.clear();
  514. auto result = page_directory.m_page_tables.remove(vaddr.get() & ~0x1fffff);
  515. VERIFY(result);
  516. }
  517. }
  518. }
  519. }
  520. UNMAP_AFTER_INIT void MemoryManager::initialize(u32 cpu)
  521. {
  522. ProcessorSpecific<MemoryManagerData>::initialize();
  523. if (cpu == 0) {
  524. new MemoryManager;
  525. kmalloc_enable_expand();
  526. }
  527. }
  528. Region* MemoryManager::kernel_region_from_vaddr(VirtualAddress vaddr)
  529. {
  530. SpinlockLocker lock(s_mm_lock);
  531. auto* region_ptr = MM.m_kernel_regions.find_largest_not_above(vaddr.get());
  532. if (!region_ptr)
  533. return nullptr;
  534. return *region_ptr;
  535. }
  536. Region* MemoryManager::find_user_region_from_vaddr_no_lock(AddressSpace& space, VirtualAddress vaddr)
  537. {
  538. VERIFY(space.get_lock().is_locked_by_current_processor());
  539. return space.find_region_containing({ vaddr, 1 });
  540. }
  541. Region* MemoryManager::find_user_region_from_vaddr(AddressSpace& space, VirtualAddress vaddr)
  542. {
  543. SpinlockLocker lock(space.get_lock());
  544. return find_user_region_from_vaddr_no_lock(space, vaddr);
  545. }
  546. void MemoryManager::validate_syscall_preconditions(AddressSpace& space, RegisterState const& regs)
  547. {
  548. // We take the space lock once here and then use the no_lock variants
  549. // to avoid excessive spinlock recursion in this extremely common path.
  550. SpinlockLocker lock(space.get_lock());
  551. auto unlock_and_handle_crash = [&lock, &regs](const char* description, int signal) {
  552. lock.unlock();
  553. handle_crash(regs, description, signal);
  554. };
  555. {
  556. VirtualAddress userspace_sp = VirtualAddress { regs.userspace_sp() };
  557. if (!MM.validate_user_stack_no_lock(space, userspace_sp)) {
  558. dbgln("Invalid stack pointer: {}", userspace_sp);
  559. return unlock_and_handle_crash("Bad stack on syscall entry", SIGSEGV);
  560. }
  561. }
  562. {
  563. VirtualAddress ip = VirtualAddress { regs.ip() };
  564. auto* calling_region = MM.find_user_region_from_vaddr_no_lock(space, ip);
  565. if (!calling_region) {
  566. dbgln("Syscall from {:p} which has no associated region", ip);
  567. return unlock_and_handle_crash("Syscall from unknown region", SIGSEGV);
  568. }
  569. if (calling_region->is_writable()) {
  570. dbgln("Syscall from writable memory at {:p}", ip);
  571. return unlock_and_handle_crash("Syscall from writable memory", SIGSEGV);
  572. }
  573. if (space.enforces_syscall_regions() && !calling_region->is_syscall_region()) {
  574. dbgln("Syscall from non-syscall region");
  575. return unlock_and_handle_crash("Syscall from non-syscall region", SIGSEGV);
  576. }
  577. }
  578. }
  579. Region* MemoryManager::find_region_from_vaddr(VirtualAddress vaddr)
  580. {
  581. if (auto* region = kernel_region_from_vaddr(vaddr))
  582. return region;
  583. auto page_directory = PageDirectory::find_by_cr3(read_cr3());
  584. if (!page_directory)
  585. return nullptr;
  586. VERIFY(page_directory->address_space());
  587. return find_user_region_from_vaddr(*page_directory->address_space(), vaddr);
  588. }
  589. PageFaultResponse MemoryManager::handle_page_fault(PageFault const& fault)
  590. {
  591. VERIFY_INTERRUPTS_DISABLED();
  592. if (Processor::current_in_irq()) {
  593. dbgln("CPU[{}] BUG! Page fault while handling IRQ! code={}, vaddr={}, irq level: {}",
  594. Processor::current_id(), fault.code(), fault.vaddr(), Processor::current_in_irq());
  595. dump_kernel_regions();
  596. return PageFaultResponse::ShouldCrash;
  597. }
  598. dbgln_if(PAGE_FAULT_DEBUG, "MM: CPU[{}] handle_page_fault({:#04x}) at {}", Processor::current_id(), fault.code(), fault.vaddr());
  599. auto* region = find_region_from_vaddr(fault.vaddr());
  600. if (!region) {
  601. return PageFaultResponse::ShouldCrash;
  602. }
  603. return region->handle_fault(fault);
  604. }
  605. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_contiguous_kernel_region(size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  606. {
  607. VERIFY(!(size % PAGE_SIZE));
  608. SpinlockLocker lock(kernel_page_directory().get_lock());
  609. auto vmobject = TRY(AnonymousVMObject::try_create_physically_contiguous_with_size(size));
  610. auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size));
  611. return allocate_kernel_region_with_vmobject(range, move(vmobject), name, access, cacheable);
  612. }
  613. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region(size_t size, StringView name, Region::Access access, AllocationStrategy strategy, Region::Cacheable cacheable)
  614. {
  615. VERIFY(!(size % PAGE_SIZE));
  616. auto vmobject = TRY(AnonymousVMObject::try_create_with_size(size, strategy));
  617. SpinlockLocker lock(kernel_page_directory().get_lock());
  618. auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size));
  619. return allocate_kernel_region_with_vmobject(range, move(vmobject), name, access, cacheable);
  620. }
  621. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region(PhysicalAddress paddr, size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  622. {
  623. VERIFY(!(size % PAGE_SIZE));
  624. auto vmobject = TRY(AnonymousVMObject::try_create_for_physical_range(paddr, size));
  625. SpinlockLocker lock(kernel_page_directory().get_lock());
  626. auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size));
  627. return allocate_kernel_region_with_vmobject(range, move(vmobject), name, access, cacheable);
  628. }
  629. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region_with_vmobject(VirtualRange const& range, VMObject& vmobject, StringView name, Region::Access access, Region::Cacheable cacheable)
  630. {
  631. OwnPtr<KString> name_kstring;
  632. if (!name.is_null())
  633. name_kstring = TRY(KString::try_create(name));
  634. auto region = TRY(Region::try_create_kernel_only(range, vmobject, 0, move(name_kstring), access, cacheable));
  635. TRY(region->map(kernel_page_directory()));
  636. return region;
  637. }
  638. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region_with_vmobject(VMObject& vmobject, size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  639. {
  640. VERIFY(!(size % PAGE_SIZE));
  641. SpinlockLocker lock(kernel_page_directory().get_lock());
  642. auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size));
  643. return allocate_kernel_region_with_vmobject(range, vmobject, name, access, cacheable);
  644. }
  645. ErrorOr<CommittedPhysicalPageSet> MemoryManager::commit_user_physical_pages(size_t page_count)
  646. {
  647. VERIFY(page_count > 0);
  648. SpinlockLocker lock(s_mm_lock);
  649. if (m_system_memory_info.user_physical_pages_uncommitted < page_count)
  650. return ENOMEM;
  651. m_system_memory_info.user_physical_pages_uncommitted -= page_count;
  652. m_system_memory_info.user_physical_pages_committed += page_count;
  653. return CommittedPhysicalPageSet { {}, page_count };
  654. }
  655. void MemoryManager::uncommit_user_physical_pages(Badge<CommittedPhysicalPageSet>, size_t page_count)
  656. {
  657. VERIFY(page_count > 0);
  658. SpinlockLocker lock(s_mm_lock);
  659. VERIFY(m_system_memory_info.user_physical_pages_committed >= page_count);
  660. m_system_memory_info.user_physical_pages_uncommitted += page_count;
  661. m_system_memory_info.user_physical_pages_committed -= page_count;
  662. }
  663. void MemoryManager::deallocate_physical_page(PhysicalAddress paddr)
  664. {
  665. SpinlockLocker lock(s_mm_lock);
  666. // Are we returning a user page?
  667. for (auto& region : m_user_physical_regions) {
  668. if (!region.contains(paddr))
  669. continue;
  670. region.return_page(paddr);
  671. --m_system_memory_info.user_physical_pages_used;
  672. // Always return pages to the uncommitted pool. Pages that were
  673. // committed and allocated are only freed upon request. Once
  674. // returned there is no guarantee being able to get them back.
  675. ++m_system_memory_info.user_physical_pages_uncommitted;
  676. return;
  677. }
  678. // If it's not a user page, it should be a supervisor page.
  679. if (!m_super_physical_region->contains(paddr))
  680. PANIC("MM: deallocate_user_physical_page couldn't figure out region for page @ {}", paddr);
  681. m_super_physical_region->return_page(paddr);
  682. --m_system_memory_info.super_physical_pages_used;
  683. }
  684. RefPtr<PhysicalPage> MemoryManager::find_free_user_physical_page(bool committed)
  685. {
  686. VERIFY(s_mm_lock.is_locked());
  687. RefPtr<PhysicalPage> page;
  688. if (committed) {
  689. // Draw from the committed pages pool. We should always have these pages available
  690. VERIFY(m_system_memory_info.user_physical_pages_committed > 0);
  691. m_system_memory_info.user_physical_pages_committed--;
  692. } else {
  693. // We need to make sure we don't touch pages that we have committed to
  694. if (m_system_memory_info.user_physical_pages_uncommitted == 0)
  695. return {};
  696. m_system_memory_info.user_physical_pages_uncommitted--;
  697. }
  698. for (auto& region : m_user_physical_regions) {
  699. page = region.take_free_page();
  700. if (!page.is_null()) {
  701. ++m_system_memory_info.user_physical_pages_used;
  702. break;
  703. }
  704. }
  705. VERIFY(!committed || !page.is_null());
  706. return page;
  707. }
  708. NonnullRefPtr<PhysicalPage> MemoryManager::allocate_committed_user_physical_page(Badge<CommittedPhysicalPageSet>, ShouldZeroFill should_zero_fill)
  709. {
  710. SpinlockLocker lock(s_mm_lock);
  711. auto page = find_free_user_physical_page(true);
  712. if (should_zero_fill == ShouldZeroFill::Yes) {
  713. auto* ptr = quickmap_page(*page);
  714. memset(ptr, 0, PAGE_SIZE);
  715. unquickmap_page();
  716. }
  717. return page.release_nonnull();
  718. }
  719. RefPtr<PhysicalPage> MemoryManager::allocate_user_physical_page(ShouldZeroFill should_zero_fill, bool* did_purge)
  720. {
  721. SpinlockLocker lock(s_mm_lock);
  722. auto page = find_free_user_physical_page(false);
  723. bool purged_pages = false;
  724. if (!page) {
  725. // We didn't have a single free physical page. Let's try to free something up!
  726. // First, we look for a purgeable VMObject in the volatile state.
  727. for_each_vmobject([&](auto& vmobject) {
  728. if (!vmobject.is_anonymous())
  729. return IterationDecision::Continue;
  730. auto& anonymous_vmobject = static_cast<AnonymousVMObject&>(vmobject);
  731. if (!anonymous_vmobject.is_purgeable() || !anonymous_vmobject.is_volatile())
  732. return IterationDecision::Continue;
  733. if (auto purged_page_count = anonymous_vmobject.purge()) {
  734. dbgln("MM: Purge saved the day! Purged {} pages from AnonymousVMObject", purged_page_count);
  735. page = find_free_user_physical_page(false);
  736. purged_pages = true;
  737. VERIFY(page);
  738. return IterationDecision::Break;
  739. }
  740. return IterationDecision::Continue;
  741. });
  742. if (!page) {
  743. dmesgln("MM: no user physical pages available");
  744. return {};
  745. }
  746. }
  747. if (should_zero_fill == ShouldZeroFill::Yes) {
  748. auto* ptr = quickmap_page(*page);
  749. memset(ptr, 0, PAGE_SIZE);
  750. unquickmap_page();
  751. }
  752. if (did_purge)
  753. *did_purge = purged_pages;
  754. return page;
  755. }
  756. NonnullRefPtrVector<PhysicalPage> MemoryManager::allocate_contiguous_supervisor_physical_pages(size_t size)
  757. {
  758. VERIFY(!(size % PAGE_SIZE));
  759. SpinlockLocker lock(s_mm_lock);
  760. size_t count = ceil_div(size, static_cast<size_t>(PAGE_SIZE));
  761. auto physical_pages = m_super_physical_region->take_contiguous_free_pages(count);
  762. if (physical_pages.is_empty()) {
  763. dmesgln("MM: no super physical pages available");
  764. VERIFY_NOT_REACHED();
  765. return {};
  766. }
  767. {
  768. auto region_or_error = MM.allocate_kernel_region(physical_pages[0].paddr(), PAGE_SIZE * count, "MemoryManager Allocation Sanitization", Region::Access::Read | Region::Access::Write);
  769. if (region_or_error.is_error())
  770. TODO();
  771. auto cleanup_region = region_or_error.release_value();
  772. fast_u32_fill((u32*)cleanup_region->vaddr().as_ptr(), 0, (PAGE_SIZE * count) / sizeof(u32));
  773. }
  774. m_system_memory_info.super_physical_pages_used += count;
  775. return physical_pages;
  776. }
  777. RefPtr<PhysicalPage> MemoryManager::allocate_supervisor_physical_page()
  778. {
  779. SpinlockLocker lock(s_mm_lock);
  780. auto page = m_super_physical_region->take_free_page();
  781. if (!page) {
  782. dmesgln("MM: no super physical pages available");
  783. VERIFY_NOT_REACHED();
  784. return {};
  785. }
  786. fast_u32_fill((u32*)page->paddr().offset(physical_to_virtual_offset).as_ptr(), 0, PAGE_SIZE / sizeof(u32));
  787. ++m_system_memory_info.super_physical_pages_used;
  788. return page;
  789. }
  790. void MemoryManager::enter_process_address_space(Process& process)
  791. {
  792. enter_address_space(process.address_space());
  793. }
  794. void MemoryManager::enter_address_space(AddressSpace& space)
  795. {
  796. auto current_thread = Thread::current();
  797. VERIFY(current_thread != nullptr);
  798. SpinlockLocker lock(s_mm_lock);
  799. current_thread->regs().cr3 = space.page_directory().cr3();
  800. write_cr3(space.page_directory().cr3());
  801. }
  802. void MemoryManager::flush_tlb_local(VirtualAddress vaddr, size_t page_count)
  803. {
  804. Processor::flush_tlb_local(vaddr, page_count);
  805. }
  806. void MemoryManager::flush_tlb(PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count)
  807. {
  808. Processor::flush_tlb(page_directory, vaddr, page_count);
  809. }
  810. PageDirectoryEntry* MemoryManager::quickmap_pd(PageDirectory& directory, size_t pdpt_index)
  811. {
  812. VERIFY(s_mm_lock.is_locked_by_current_processor());
  813. auto& mm_data = get_data();
  814. auto& pte = boot_pd_kernel_pt1023[(KERNEL_QUICKMAP_PD - KERNEL_PT1024_BASE) / PAGE_SIZE];
  815. auto pd_paddr = directory.m_directory_pages[pdpt_index]->paddr();
  816. if (pte.physical_page_base() != pd_paddr.get()) {
  817. pte.set_physical_page_base(pd_paddr.get());
  818. pte.set_present(true);
  819. pte.set_writable(true);
  820. pte.set_user_allowed(false);
  821. // Because we must continue to hold the MM lock while we use this
  822. // mapping, it is sufficient to only flush on the current CPU. Other
  823. // CPUs trying to use this API must wait on the MM lock anyway
  824. flush_tlb_local(VirtualAddress(KERNEL_QUICKMAP_PD));
  825. } else {
  826. // Even though we don't allow this to be called concurrently, it's
  827. // possible that this PD was mapped on a different CPU and we don't
  828. // broadcast the flush. If so, we still need to flush the TLB.
  829. if (mm_data.m_last_quickmap_pd != pd_paddr)
  830. flush_tlb_local(VirtualAddress(KERNEL_QUICKMAP_PD));
  831. }
  832. mm_data.m_last_quickmap_pd = pd_paddr;
  833. return (PageDirectoryEntry*)KERNEL_QUICKMAP_PD;
  834. }
  835. PageTableEntry* MemoryManager::quickmap_pt(PhysicalAddress pt_paddr)
  836. {
  837. VERIFY(s_mm_lock.is_locked_by_current_processor());
  838. auto& mm_data = get_data();
  839. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[(KERNEL_QUICKMAP_PT - KERNEL_PT1024_BASE) / PAGE_SIZE];
  840. if (pte.physical_page_base() != pt_paddr.get()) {
  841. pte.set_physical_page_base(pt_paddr.get());
  842. pte.set_present(true);
  843. pte.set_writable(true);
  844. pte.set_user_allowed(false);
  845. // Because we must continue to hold the MM lock while we use this
  846. // mapping, it is sufficient to only flush on the current CPU. Other
  847. // CPUs trying to use this API must wait on the MM lock anyway
  848. flush_tlb_local(VirtualAddress(KERNEL_QUICKMAP_PT));
  849. } else {
  850. // Even though we don't allow this to be called concurrently, it's
  851. // possible that this PT was mapped on a different CPU and we don't
  852. // broadcast the flush. If so, we still need to flush the TLB.
  853. if (mm_data.m_last_quickmap_pt != pt_paddr)
  854. flush_tlb_local(VirtualAddress(KERNEL_QUICKMAP_PT));
  855. }
  856. mm_data.m_last_quickmap_pt = pt_paddr;
  857. return (PageTableEntry*)KERNEL_QUICKMAP_PT;
  858. }
  859. u8* MemoryManager::quickmap_page(PhysicalAddress const& physical_address)
  860. {
  861. VERIFY_INTERRUPTS_DISABLED();
  862. VERIFY(s_mm_lock.is_locked_by_current_processor());
  863. auto& mm_data = get_data();
  864. mm_data.m_quickmap_prev_flags = mm_data.m_quickmap_in_use.lock();
  865. VirtualAddress vaddr(KERNEL_QUICKMAP_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  866. u32 pte_idx = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  867. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[pte_idx];
  868. if (pte.physical_page_base() != physical_address.get()) {
  869. pte.set_physical_page_base(physical_address.get());
  870. pte.set_present(true);
  871. pte.set_writable(true);
  872. pte.set_user_allowed(false);
  873. flush_tlb_local(vaddr);
  874. }
  875. return vaddr.as_ptr();
  876. }
  877. void MemoryManager::unquickmap_page()
  878. {
  879. VERIFY_INTERRUPTS_DISABLED();
  880. VERIFY(s_mm_lock.is_locked_by_current_processor());
  881. auto& mm_data = get_data();
  882. VERIFY(mm_data.m_quickmap_in_use.is_locked());
  883. VirtualAddress vaddr(KERNEL_QUICKMAP_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  884. u32 pte_idx = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  885. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[pte_idx];
  886. pte.clear();
  887. flush_tlb_local(vaddr);
  888. mm_data.m_quickmap_in_use.unlock(mm_data.m_quickmap_prev_flags);
  889. }
  890. bool MemoryManager::validate_user_stack_no_lock(AddressSpace& space, VirtualAddress vaddr) const
  891. {
  892. VERIFY(space.get_lock().is_locked_by_current_processor());
  893. if (!is_user_address(vaddr))
  894. return false;
  895. auto* region = find_user_region_from_vaddr_no_lock(space, vaddr);
  896. return region && region->is_user() && region->is_stack();
  897. }
  898. bool MemoryManager::validate_user_stack(AddressSpace& space, VirtualAddress vaddr) const
  899. {
  900. SpinlockLocker lock(space.get_lock());
  901. return validate_user_stack_no_lock(space, vaddr);
  902. }
  903. void MemoryManager::register_region(Region& region)
  904. {
  905. SpinlockLocker lock(s_mm_lock);
  906. if (region.is_kernel())
  907. m_kernel_regions.insert(region.vaddr().get(), &region);
  908. }
  909. void MemoryManager::unregister_region(Region& region)
  910. {
  911. SpinlockLocker lock(s_mm_lock);
  912. if (region.is_kernel())
  913. m_kernel_regions.remove(region.vaddr().get());
  914. }
  915. void MemoryManager::dump_kernel_regions()
  916. {
  917. dbgln("Kernel regions:");
  918. #if ARCH(I386)
  919. auto addr_padding = "";
  920. #else
  921. auto addr_padding = " ";
  922. #endif
  923. dbgln("BEGIN{} END{} SIZE{} ACCESS NAME",
  924. addr_padding, addr_padding, addr_padding);
  925. SpinlockLocker lock(s_mm_lock);
  926. for (auto* region_ptr : m_kernel_regions) {
  927. auto& region = *region_ptr;
  928. dbgln("{:p} -- {:p} {:p} {:c}{:c}{:c}{:c}{:c}{:c} {}",
  929. region.vaddr().get(),
  930. region.vaddr().offset(region.size() - 1).get(),
  931. region.size(),
  932. region.is_readable() ? 'R' : ' ',
  933. region.is_writable() ? 'W' : ' ',
  934. region.is_executable() ? 'X' : ' ',
  935. region.is_shared() ? 'S' : ' ',
  936. region.is_stack() ? 'T' : ' ',
  937. region.is_syscall_region() ? 'C' : ' ',
  938. region.name());
  939. }
  940. }
  941. void MemoryManager::set_page_writable_direct(VirtualAddress vaddr, bool writable)
  942. {
  943. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  944. SpinlockLocker lock(s_mm_lock);
  945. auto* pte = ensure_pte(kernel_page_directory(), vaddr);
  946. VERIFY(pte);
  947. if (pte->is_writable() == writable)
  948. return;
  949. pte->set_writable(writable);
  950. flush_tlb(&kernel_page_directory(), vaddr);
  951. }
  952. CommittedPhysicalPageSet::~CommittedPhysicalPageSet()
  953. {
  954. if (m_page_count)
  955. MM.uncommit_user_physical_pages({}, m_page_count);
  956. }
  957. NonnullRefPtr<PhysicalPage> CommittedPhysicalPageSet::take_one()
  958. {
  959. VERIFY(m_page_count > 0);
  960. --m_page_count;
  961. return MM.allocate_committed_user_physical_page({}, MemoryManager::ShouldZeroFill::Yes);
  962. }
  963. void CommittedPhysicalPageSet::uncommit_one()
  964. {
  965. VERIFY(m_page_count > 0);
  966. --m_page_count;
  967. MM.uncommit_user_physical_pages({}, 1);
  968. }
  969. void MemoryManager::copy_physical_page(PhysicalPage& physical_page, u8 page_buffer[PAGE_SIZE])
  970. {
  971. SpinlockLocker locker(s_mm_lock);
  972. auto* quickmapped_page = quickmap_page(physical_page);
  973. memcpy(page_buffer, quickmapped_page, PAGE_SIZE);
  974. unquickmap_page();
  975. }
  976. }