MemoryManager.cpp 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179
  1. /*
  2. * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <AK/Assertions.h>
  7. #include <AK/Memory.h>
  8. #include <AK/StringView.h>
  9. #include <Kernel/Arch/x86/PageFault.h>
  10. #include <Kernel/BootInfo.h>
  11. #include <Kernel/CMOS.h>
  12. #include <Kernel/FileSystem/Inode.h>
  13. #include <Kernel/Heap/kmalloc.h>
  14. #include <Kernel/Memory/AnonymousVMObject.h>
  15. #include <Kernel/Memory/MemoryManager.h>
  16. #include <Kernel/Memory/PageDirectory.h>
  17. #include <Kernel/Memory/PhysicalRegion.h>
  18. #include <Kernel/Memory/SharedInodeVMObject.h>
  19. #include <Kernel/Multiboot.h>
  20. #include <Kernel/Panic.h>
  21. #include <Kernel/Process.h>
  22. #include <Kernel/Sections.h>
  23. #include <Kernel/StdLib.h>
  24. extern u8 start_of_kernel_image[];
  25. extern u8 end_of_kernel_image[];
  26. extern u8 start_of_kernel_text[];
  27. extern u8 start_of_kernel_data[];
  28. extern u8 end_of_kernel_bss[];
  29. extern u8 start_of_ro_after_init[];
  30. extern u8 end_of_ro_after_init[];
  31. extern u8 start_of_unmap_after_init[];
  32. extern u8 end_of_unmap_after_init[];
  33. extern u8 start_of_kernel_ksyms[];
  34. extern u8 end_of_kernel_ksyms[];
  35. extern multiboot_module_entry_t multiboot_copy_boot_modules_array[16];
  36. extern size_t multiboot_copy_boot_modules_count;
  37. // Treat the super pages as logically separate from .bss
  38. // FIXME: Find a solution so we don't need to expand this range each time
  39. // we are in a situation too many drivers try to allocate super pages.
  40. __attribute__((section(".super_pages"))) static u8 super_pages[4 * MiB];
  41. namespace Kernel::Memory {
  42. ErrorOr<FlatPtr> page_round_up(FlatPtr x)
  43. {
  44. if (x > (explode_byte(0xFF) & ~0xFFF)) {
  45. return Error::from_errno(EINVAL);
  46. }
  47. return (((FlatPtr)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1));
  48. }
  49. // NOTE: We can NOT use Singleton for this class, because
  50. // MemoryManager::initialize is called *before* global constructors are
  51. // run. If we do, then Singleton would get re-initialized, causing
  52. // the memory manager to be initialized twice!
  53. static MemoryManager* s_the;
  54. RecursiveSpinlock s_mm_lock { LockRank::MemoryManager };
  55. MemoryManager& MemoryManager::the()
  56. {
  57. return *s_the;
  58. }
  59. bool MemoryManager::is_initialized()
  60. {
  61. return s_the != nullptr;
  62. }
  63. UNMAP_AFTER_INIT MemoryManager::MemoryManager()
  64. {
  65. s_the = this;
  66. SpinlockLocker lock(s_mm_lock);
  67. parse_memory_map();
  68. write_cr3(kernel_page_directory().cr3());
  69. protect_kernel_image();
  70. // We're temporarily "committing" to two pages that we need to allocate below
  71. auto committed_pages = commit_user_physical_pages(2).release_value();
  72. m_shared_zero_page = committed_pages.take_one();
  73. // We're wasting a page here, we just need a special tag (physical
  74. // address) so that we know when we need to lazily allocate a page
  75. // that we should be drawing this page from the committed pool rather
  76. // than potentially failing if no pages are available anymore.
  77. // By using a tag we don't have to query the VMObject for every page
  78. // whether it was committed or not
  79. m_lazy_committed_page = committed_pages.take_one();
  80. }
  81. UNMAP_AFTER_INIT MemoryManager::~MemoryManager()
  82. {
  83. }
  84. UNMAP_AFTER_INIT void MemoryManager::protect_kernel_image()
  85. {
  86. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  87. // Disable writing to the kernel text and rodata segments.
  88. for (auto i = start_of_kernel_text; i < start_of_kernel_data; i += PAGE_SIZE) {
  89. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  90. pte.set_writable(false);
  91. }
  92. if (Processor::current().has_feature(CPUFeature::NX)) {
  93. // Disable execution of the kernel data, bss and heap segments.
  94. for (auto i = start_of_kernel_data; i < end_of_kernel_image; i += PAGE_SIZE) {
  95. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  96. pte.set_execute_disabled(true);
  97. }
  98. }
  99. }
  100. UNMAP_AFTER_INIT void MemoryManager::unmap_prekernel()
  101. {
  102. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  103. SpinlockLocker mm_lock(s_mm_lock);
  104. auto start = start_of_prekernel_image.page_base().get();
  105. auto end = end_of_prekernel_image.page_base().get();
  106. for (auto i = start; i <= end; i += PAGE_SIZE)
  107. release_pte(kernel_page_directory(), VirtualAddress(i), i == end ? IsLastPTERelease::Yes : IsLastPTERelease::No, UnsafeIgnoreMissingPageTable::Yes);
  108. flush_tlb(&kernel_page_directory(), VirtualAddress(start), (end - start) / PAGE_SIZE);
  109. }
  110. UNMAP_AFTER_INIT void MemoryManager::protect_readonly_after_init_memory()
  111. {
  112. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  113. SpinlockLocker mm_lock(s_mm_lock);
  114. // Disable writing to the .ro_after_init section
  115. for (auto i = (FlatPtr)&start_of_ro_after_init; i < (FlatPtr)&end_of_ro_after_init; i += PAGE_SIZE) {
  116. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  117. pte.set_writable(false);
  118. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  119. }
  120. }
  121. void MemoryManager::unmap_text_after_init()
  122. {
  123. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  124. SpinlockLocker mm_lock(s_mm_lock);
  125. auto start = page_round_down((FlatPtr)&start_of_unmap_after_init);
  126. auto end = page_round_up((FlatPtr)&end_of_unmap_after_init).release_value_but_fixme_should_propagate_errors();
  127. // Unmap the entire .unmap_after_init section
  128. for (auto i = start; i < end; i += PAGE_SIZE) {
  129. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  130. pte.clear();
  131. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  132. }
  133. dmesgln("Unmapped {} KiB of kernel text after init! :^)", (end - start) / KiB);
  134. }
  135. UNMAP_AFTER_INIT void MemoryManager::protect_ksyms_after_init()
  136. {
  137. SpinlockLocker mm_lock(s_mm_lock);
  138. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  139. auto start = page_round_down((FlatPtr)start_of_kernel_ksyms);
  140. auto end = page_round_up((FlatPtr)end_of_kernel_ksyms).release_value_but_fixme_should_propagate_errors();
  141. for (auto i = start; i < end; i += PAGE_SIZE) {
  142. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  143. pte.set_writable(false);
  144. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  145. }
  146. dmesgln("Write-protected kernel symbols after init.");
  147. }
  148. IterationDecision MemoryManager::for_each_physical_memory_range(Function<IterationDecision(PhysicalMemoryRange const&)> callback)
  149. {
  150. VERIFY(!m_physical_memory_ranges.is_empty());
  151. for (auto& current_range : m_physical_memory_ranges) {
  152. IterationDecision decision = callback(current_range);
  153. if (decision != IterationDecision::Continue)
  154. return decision;
  155. }
  156. return IterationDecision::Continue;
  157. }
  158. UNMAP_AFTER_INIT void MemoryManager::register_reserved_ranges()
  159. {
  160. VERIFY(!m_physical_memory_ranges.is_empty());
  161. ContiguousReservedMemoryRange range;
  162. for (auto& current_range : m_physical_memory_ranges) {
  163. if (current_range.type != PhysicalMemoryRangeType::Reserved) {
  164. if (range.start.is_null())
  165. continue;
  166. m_reserved_memory_ranges.append(ContiguousReservedMemoryRange { range.start, current_range.start.get() - range.start.get() });
  167. range.start.set((FlatPtr) nullptr);
  168. continue;
  169. }
  170. if (!range.start.is_null()) {
  171. continue;
  172. }
  173. range.start = current_range.start;
  174. }
  175. if (m_physical_memory_ranges.last().type != PhysicalMemoryRangeType::Reserved)
  176. return;
  177. if (range.start.is_null())
  178. return;
  179. m_reserved_memory_ranges.append(ContiguousReservedMemoryRange { range.start, m_physical_memory_ranges.last().start.get() + m_physical_memory_ranges.last().length - range.start.get() });
  180. }
  181. bool MemoryManager::is_allowed_to_mmap_to_userspace(PhysicalAddress start_address, VirtualRange const& range) const
  182. {
  183. VERIFY(!m_reserved_memory_ranges.is_empty());
  184. // Note: Guard against overflow in case someone tries to mmap on the edge of
  185. // the RAM
  186. if (start_address.offset_addition_would_overflow(range.size()))
  187. return false;
  188. auto end_address = start_address.offset(range.size());
  189. for (auto& current_range : m_reserved_memory_ranges) {
  190. if (current_range.start > start_address)
  191. continue;
  192. if (current_range.start.offset(current_range.length) < end_address)
  193. continue;
  194. return true;
  195. }
  196. return false;
  197. }
  198. UNMAP_AFTER_INIT void MemoryManager::parse_memory_map()
  199. {
  200. // Register used memory regions that we know of.
  201. m_used_memory_ranges.ensure_capacity(4);
  202. m_used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::LowMemory, PhysicalAddress(0x00000000), PhysicalAddress(1 * MiB) });
  203. m_used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::Kernel, PhysicalAddress(virtual_to_low_physical((FlatPtr)start_of_kernel_image)), PhysicalAddress(page_round_up(virtual_to_low_physical((FlatPtr)end_of_kernel_image)).release_value_but_fixme_should_propagate_errors()) });
  204. if (multiboot_flags & 0x4) {
  205. auto* bootmods_start = multiboot_copy_boot_modules_array;
  206. auto* bootmods_end = bootmods_start + multiboot_copy_boot_modules_count;
  207. for (auto* bootmod = bootmods_start; bootmod < bootmods_end; bootmod++) {
  208. m_used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::BootModule, PhysicalAddress(bootmod->start), PhysicalAddress(bootmod->end) });
  209. }
  210. }
  211. auto* mmap_begin = multiboot_memory_map;
  212. auto* mmap_end = multiboot_memory_map + multiboot_memory_map_count;
  213. struct ContiguousPhysicalVirtualRange {
  214. PhysicalAddress lower;
  215. PhysicalAddress upper;
  216. };
  217. Vector<ContiguousPhysicalVirtualRange> contiguous_physical_ranges;
  218. for (auto* mmap = mmap_begin; mmap < mmap_end; mmap++) {
  219. // We have to copy these onto the stack, because we take a reference to these when printing them out,
  220. // and doing so on a packed struct field is UB.
  221. auto address = mmap->addr;
  222. auto length = mmap->len;
  223. ArmedScopeGuard write_back_guard = [&]() {
  224. mmap->addr = address;
  225. mmap->len = length;
  226. };
  227. dmesgln("MM: Multiboot mmap: address={:p}, length={}, type={}", address, length, mmap->type);
  228. auto start_address = PhysicalAddress(address);
  229. switch (mmap->type) {
  230. case (MULTIBOOT_MEMORY_AVAILABLE):
  231. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Usable, start_address, length });
  232. break;
  233. case (MULTIBOOT_MEMORY_RESERVED):
  234. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Reserved, start_address, length });
  235. break;
  236. case (MULTIBOOT_MEMORY_ACPI_RECLAIMABLE):
  237. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_Reclaimable, start_address, length });
  238. break;
  239. case (MULTIBOOT_MEMORY_NVS):
  240. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_NVS, start_address, length });
  241. break;
  242. case (MULTIBOOT_MEMORY_BADRAM):
  243. dmesgln("MM: Warning, detected bad memory range!");
  244. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::BadMemory, start_address, length });
  245. break;
  246. default:
  247. dbgln("MM: Unknown range!");
  248. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Unknown, start_address, length });
  249. break;
  250. }
  251. if (mmap->type != MULTIBOOT_MEMORY_AVAILABLE)
  252. continue;
  253. // Fix up unaligned memory regions.
  254. auto diff = (FlatPtr)address % PAGE_SIZE;
  255. if (diff != 0) {
  256. dmesgln("MM: Got an unaligned physical_region from the bootloader; correcting {:p} by {} bytes", address, diff);
  257. diff = PAGE_SIZE - diff;
  258. address += diff;
  259. length -= diff;
  260. }
  261. if ((length % PAGE_SIZE) != 0) {
  262. dmesgln("MM: Got an unaligned physical_region from the bootloader; correcting length {} by {} bytes", length, length % PAGE_SIZE);
  263. length -= length % PAGE_SIZE;
  264. }
  265. if (length < PAGE_SIZE) {
  266. dmesgln("MM: Memory physical_region from bootloader is too small; we want >= {} bytes, but got {} bytes", PAGE_SIZE, length);
  267. continue;
  268. }
  269. for (PhysicalSize page_base = address; page_base <= (address + length); page_base += PAGE_SIZE) {
  270. auto addr = PhysicalAddress(page_base);
  271. // Skip used memory ranges.
  272. bool should_skip = false;
  273. for (auto& used_range : m_used_memory_ranges) {
  274. if (addr.get() >= used_range.start.get() && addr.get() <= used_range.end.get()) {
  275. should_skip = true;
  276. break;
  277. }
  278. }
  279. if (should_skip)
  280. continue;
  281. if (contiguous_physical_ranges.is_empty() || contiguous_physical_ranges.last().upper.offset(PAGE_SIZE) != addr) {
  282. contiguous_physical_ranges.append(ContiguousPhysicalVirtualRange {
  283. .lower = addr,
  284. .upper = addr,
  285. });
  286. } else {
  287. contiguous_physical_ranges.last().upper = addr;
  288. }
  289. }
  290. }
  291. for (auto& range : contiguous_physical_ranges) {
  292. m_user_physical_regions.append(PhysicalRegion::try_create(range.lower, range.upper).release_nonnull());
  293. }
  294. // Super pages are guaranteed to be in the first 16MB of physical memory
  295. VERIFY(virtual_to_low_physical((FlatPtr)super_pages) + sizeof(super_pages) < 0x1000000);
  296. // Append statically-allocated super physical physical_region.
  297. m_super_physical_region = PhysicalRegion::try_create(
  298. PhysicalAddress(virtual_to_low_physical(FlatPtr(super_pages))),
  299. PhysicalAddress(virtual_to_low_physical(FlatPtr(super_pages + sizeof(super_pages)))));
  300. VERIFY(m_super_physical_region);
  301. m_system_memory_info.super_physical_pages += m_super_physical_region->size();
  302. for (auto& region : m_user_physical_regions)
  303. m_system_memory_info.user_physical_pages += region.size();
  304. register_reserved_ranges();
  305. for (auto& range : m_reserved_memory_ranges) {
  306. dmesgln("MM: Contiguous reserved range from {}, length is {}", range.start, range.length);
  307. }
  308. initialize_physical_pages();
  309. VERIFY(m_system_memory_info.super_physical_pages > 0);
  310. VERIFY(m_system_memory_info.user_physical_pages > 0);
  311. // We start out with no committed pages
  312. m_system_memory_info.user_physical_pages_uncommitted = m_system_memory_info.user_physical_pages;
  313. for (auto& used_range : m_used_memory_ranges) {
  314. dmesgln("MM: {} range @ {} - {} (size {:#x})", UserMemoryRangeTypeNames[to_underlying(used_range.type)], used_range.start, used_range.end.offset(-1), used_range.end.as_ptr() - used_range.start.as_ptr());
  315. }
  316. dmesgln("MM: Super physical region: {} - {} (size {:#x})", m_super_physical_region->lower(), m_super_physical_region->upper().offset(-1), PAGE_SIZE * m_super_physical_region->size());
  317. m_super_physical_region->initialize_zones();
  318. for (auto& region : m_user_physical_regions) {
  319. dmesgln("MM: User physical region: {} - {} (size {:#x})", region.lower(), region.upper().offset(-1), PAGE_SIZE * region.size());
  320. region.initialize_zones();
  321. }
  322. }
  323. UNMAP_AFTER_INIT void MemoryManager::initialize_physical_pages()
  324. {
  325. // We assume that the physical page range is contiguous and doesn't contain huge gaps!
  326. PhysicalAddress highest_physical_address;
  327. for (auto& range : m_used_memory_ranges) {
  328. if (range.end.get() > highest_physical_address.get())
  329. highest_physical_address = range.end;
  330. }
  331. for (auto& region : m_physical_memory_ranges) {
  332. auto range_end = PhysicalAddress(region.start).offset(region.length);
  333. if (range_end.get() > highest_physical_address.get())
  334. highest_physical_address = range_end;
  335. }
  336. // Calculate how many total physical pages the array will have
  337. m_physical_page_entries_count = PhysicalAddress::physical_page_index(highest_physical_address.get()) + 1;
  338. VERIFY(m_physical_page_entries_count != 0);
  339. VERIFY(!Checked<decltype(m_physical_page_entries_count)>::multiplication_would_overflow(m_physical_page_entries_count, sizeof(PhysicalPageEntry)));
  340. // Calculate how many bytes the array will consume
  341. auto physical_page_array_size = m_physical_page_entries_count * sizeof(PhysicalPageEntry);
  342. auto physical_page_array_pages = page_round_up(physical_page_array_size).release_value_but_fixme_should_propagate_errors() / PAGE_SIZE;
  343. VERIFY(physical_page_array_pages * PAGE_SIZE >= physical_page_array_size);
  344. // Calculate how many page tables we will need to be able to map them all
  345. auto needed_page_table_count = (physical_page_array_pages + 512 - 1) / 512;
  346. auto physical_page_array_pages_and_page_tables_count = physical_page_array_pages + needed_page_table_count;
  347. // Now that we know how much memory we need for a contiguous array of PhysicalPage instances, find a memory region that can fit it
  348. PhysicalRegion* found_region { nullptr };
  349. Optional<size_t> found_region_index;
  350. for (size_t i = 0; i < m_user_physical_regions.size(); ++i) {
  351. auto& region = m_user_physical_regions[i];
  352. if (region.size() >= physical_page_array_pages_and_page_tables_count) {
  353. found_region = &region;
  354. found_region_index = i;
  355. break;
  356. }
  357. }
  358. if (!found_region) {
  359. dmesgln("MM: Need {} bytes for physical page management, but no memory region is large enough!", physical_page_array_pages_and_page_tables_count);
  360. VERIFY_NOT_REACHED();
  361. }
  362. VERIFY(m_system_memory_info.user_physical_pages >= physical_page_array_pages_and_page_tables_count);
  363. m_system_memory_info.user_physical_pages -= physical_page_array_pages_and_page_tables_count;
  364. if (found_region->size() == physical_page_array_pages_and_page_tables_count) {
  365. // We're stealing the entire region
  366. m_physical_pages_region = m_user_physical_regions.take(*found_region_index);
  367. } else {
  368. m_physical_pages_region = found_region->try_take_pages_from_beginning(physical_page_array_pages_and_page_tables_count);
  369. }
  370. m_used_memory_ranges.append({ UsedMemoryRangeType::PhysicalPages, m_physical_pages_region->lower(), m_physical_pages_region->upper() });
  371. // Create the bare page directory. This is not a fully constructed page directory and merely contains the allocators!
  372. m_kernel_page_directory = PageDirectory::must_create_kernel_page_directory();
  373. // Allocate a virtual address range for our array
  374. auto range_or_error = m_kernel_page_directory->range_allocator().try_allocate_anywhere(physical_page_array_pages * PAGE_SIZE);
  375. if (range_or_error.is_error()) {
  376. dmesgln("MM: Could not allocate {} bytes to map physical page array!", physical_page_array_pages * PAGE_SIZE);
  377. VERIFY_NOT_REACHED();
  378. }
  379. auto range = range_or_error.release_value();
  380. // Now that we have our special m_physical_pages_region region with enough pages to hold the entire array
  381. // try to map the entire region into kernel space so we always have it
  382. // We can't use ensure_pte here because it would try to allocate a PhysicalPage and we don't have the array
  383. // mapped yet so we can't create them
  384. SpinlockLocker lock(s_mm_lock);
  385. // Create page tables at the beginning of m_physical_pages_region, followed by the PhysicalPageEntry array
  386. auto page_tables_base = m_physical_pages_region->lower();
  387. auto physical_page_array_base = page_tables_base.offset(needed_page_table_count * PAGE_SIZE);
  388. auto physical_page_array_current_page = physical_page_array_base.get();
  389. auto virtual_page_array_base = range.base().get();
  390. auto virtual_page_array_current_page = virtual_page_array_base;
  391. for (size_t pt_index = 0; pt_index < needed_page_table_count; pt_index++) {
  392. auto virtual_page_base_for_this_pt = virtual_page_array_current_page;
  393. auto pt_paddr = page_tables_base.offset(pt_index * PAGE_SIZE);
  394. auto* pt = reinterpret_cast<PageTableEntry*>(quickmap_page(pt_paddr));
  395. __builtin_memset(pt, 0, PAGE_SIZE);
  396. for (size_t pte_index = 0; pte_index < PAGE_SIZE / sizeof(PageTableEntry); pte_index++) {
  397. auto& pte = pt[pte_index];
  398. pte.set_physical_page_base(physical_page_array_current_page);
  399. pte.set_user_allowed(false);
  400. pte.set_writable(true);
  401. if (Processor::current().has_feature(CPUFeature::NX))
  402. pte.set_execute_disabled(false);
  403. pte.set_global(true);
  404. pte.set_present(true);
  405. physical_page_array_current_page += PAGE_SIZE;
  406. virtual_page_array_current_page += PAGE_SIZE;
  407. }
  408. unquickmap_page();
  409. // Hook the page table into the kernel page directory
  410. u32 page_directory_index = (virtual_page_base_for_this_pt >> 21) & 0x1ff;
  411. auto* pd = reinterpret_cast<PageDirectoryEntry*>(quickmap_page(boot_pd_kernel));
  412. PageDirectoryEntry& pde = pd[page_directory_index];
  413. VERIFY(!pde.is_present()); // Nothing should be using this PD yet
  414. // We can't use ensure_pte quite yet!
  415. pde.set_page_table_base(pt_paddr.get());
  416. pde.set_user_allowed(false);
  417. pde.set_present(true);
  418. pde.set_writable(true);
  419. pde.set_global(true);
  420. unquickmap_page();
  421. flush_tlb_local(VirtualAddress(virtual_page_base_for_this_pt));
  422. }
  423. // We now have the entire PhysicalPageEntry array mapped!
  424. m_physical_page_entries = (PhysicalPageEntry*)range.base().get();
  425. for (size_t i = 0; i < m_physical_page_entries_count; i++)
  426. new (&m_physical_page_entries[i]) PageTableEntry();
  427. // Now we should be able to allocate PhysicalPage instances,
  428. // so finish setting up the kernel page directory
  429. m_kernel_page_directory->allocate_kernel_directory();
  430. // Now create legit PhysicalPage objects for the page tables we created, so that
  431. // we can put them into kernel_page_directory().m_page_tables
  432. auto& kernel_page_tables = kernel_page_directory().m_page_tables;
  433. virtual_page_array_current_page = virtual_page_array_base;
  434. for (size_t pt_index = 0; pt_index < needed_page_table_count; pt_index++) {
  435. VERIFY(virtual_page_array_current_page <= range.end().get());
  436. auto pt_paddr = page_tables_base.offset(pt_index * PAGE_SIZE);
  437. auto physical_page_index = PhysicalAddress::physical_page_index(pt_paddr.get());
  438. auto& physical_page_entry = m_physical_page_entries[physical_page_index];
  439. auto physical_page = adopt_ref(*new (&physical_page_entry.allocated.physical_page) PhysicalPage(MayReturnToFreeList::No));
  440. auto result = kernel_page_tables.set(virtual_page_array_current_page & ~0x1fffff, move(physical_page));
  441. VERIFY(result == AK::HashSetResult::InsertedNewEntry);
  442. virtual_page_array_current_page += (PAGE_SIZE / sizeof(PageTableEntry)) * PAGE_SIZE;
  443. }
  444. dmesgln("MM: Physical page entries: {}", range);
  445. }
  446. PhysicalPageEntry& MemoryManager::get_physical_page_entry(PhysicalAddress physical_address)
  447. {
  448. VERIFY(m_physical_page_entries);
  449. auto physical_page_entry_index = PhysicalAddress::physical_page_index(physical_address.get());
  450. VERIFY(physical_page_entry_index < m_physical_page_entries_count);
  451. return m_physical_page_entries[physical_page_entry_index];
  452. }
  453. PhysicalAddress MemoryManager::get_physical_address(PhysicalPage const& physical_page)
  454. {
  455. PhysicalPageEntry const& physical_page_entry = *reinterpret_cast<PhysicalPageEntry const*>((u8 const*)&physical_page - __builtin_offsetof(PhysicalPageEntry, allocated.physical_page));
  456. VERIFY(m_physical_page_entries);
  457. size_t physical_page_entry_index = &physical_page_entry - m_physical_page_entries;
  458. VERIFY(physical_page_entry_index < m_physical_page_entries_count);
  459. return PhysicalAddress((PhysicalPtr)physical_page_entry_index * PAGE_SIZE);
  460. }
  461. PageTableEntry* MemoryManager::pte(PageDirectory& page_directory, VirtualAddress vaddr)
  462. {
  463. VERIFY_INTERRUPTS_DISABLED();
  464. VERIFY(s_mm_lock.is_locked_by_current_processor());
  465. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  466. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  467. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  468. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  469. auto* pd = quickmap_pd(const_cast<PageDirectory&>(page_directory), page_directory_table_index);
  470. PageDirectoryEntry const& pde = pd[page_directory_index];
  471. if (!pde.is_present())
  472. return nullptr;
  473. return &quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()))[page_table_index];
  474. }
  475. PageTableEntry* MemoryManager::ensure_pte(PageDirectory& page_directory, VirtualAddress vaddr)
  476. {
  477. VERIFY_INTERRUPTS_DISABLED();
  478. VERIFY(s_mm_lock.is_locked_by_current_processor());
  479. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  480. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  481. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  482. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  483. auto* pd = quickmap_pd(page_directory, page_directory_table_index);
  484. PageDirectoryEntry& pde = pd[page_directory_index];
  485. if (!pde.is_present()) {
  486. bool did_purge = false;
  487. auto page_table = allocate_user_physical_page(ShouldZeroFill::Yes, &did_purge);
  488. if (!page_table) {
  489. dbgln("MM: Unable to allocate page table to map {}", vaddr);
  490. return nullptr;
  491. }
  492. if (did_purge) {
  493. // If any memory had to be purged, ensure_pte may have been called as part
  494. // of the purging process. So we need to re-map the pd in this case to ensure
  495. // we're writing to the correct underlying physical page
  496. pd = quickmap_pd(page_directory, page_directory_table_index);
  497. VERIFY(&pde == &pd[page_directory_index]); // Sanity check
  498. VERIFY(!pde.is_present()); // Should have not changed
  499. }
  500. pde.set_page_table_base(page_table->paddr().get());
  501. pde.set_user_allowed(true);
  502. pde.set_present(true);
  503. pde.set_writable(true);
  504. pde.set_global(&page_directory == m_kernel_page_directory.ptr());
  505. // Use page_directory_table_index and page_directory_index as key
  506. // This allows us to release the page table entry when no longer needed
  507. auto result = page_directory.m_page_tables.set(vaddr.get() & ~(FlatPtr)0x1fffff, page_table.release_nonnull());
  508. // If you're hitting this VERIFY on x86_64 chances are a 64-bit pointer was truncated somewhere
  509. VERIFY(result == AK::HashSetResult::InsertedNewEntry);
  510. }
  511. return &quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()))[page_table_index];
  512. }
  513. void MemoryManager::release_pte(PageDirectory& page_directory, VirtualAddress vaddr, IsLastPTERelease is_last_pte_release, UnsafeIgnoreMissingPageTable unsafe_ignore_missing_page_table)
  514. {
  515. VERIFY_INTERRUPTS_DISABLED();
  516. VERIFY(s_mm_lock.is_locked_by_current_processor());
  517. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  518. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  519. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  520. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  521. auto* pd = quickmap_pd(page_directory, page_directory_table_index);
  522. PageDirectoryEntry& pde = pd[page_directory_index];
  523. if (pde.is_present()) {
  524. auto* page_table = quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()));
  525. auto& pte = page_table[page_table_index];
  526. pte.clear();
  527. if (is_last_pte_release == IsLastPTERelease::Yes || page_table_index == 0x1ff) {
  528. // If this is the last PTE in a region or the last PTE in a page table then
  529. // check if we can also release the page table
  530. bool all_clear = true;
  531. for (u32 i = 0; i <= 0x1ff; i++) {
  532. if (!page_table[i].is_null()) {
  533. all_clear = false;
  534. break;
  535. }
  536. }
  537. if (all_clear) {
  538. pde.clear();
  539. auto result = page_directory.m_page_tables.remove(vaddr.get() & ~0x1fffff);
  540. VERIFY(unsafe_ignore_missing_page_table == UnsafeIgnoreMissingPageTable::Yes || result);
  541. }
  542. }
  543. }
  544. }
  545. UNMAP_AFTER_INIT void MemoryManager::initialize(u32 cpu)
  546. {
  547. ProcessorSpecific<MemoryManagerData>::initialize();
  548. if (cpu == 0) {
  549. new MemoryManager;
  550. kmalloc_enable_expand();
  551. }
  552. }
  553. Region* MemoryManager::kernel_region_from_vaddr(VirtualAddress vaddr)
  554. {
  555. SpinlockLocker lock(s_mm_lock);
  556. auto* region_ptr = MM.m_kernel_regions.find_largest_not_above(vaddr.get());
  557. if (!region_ptr)
  558. return nullptr;
  559. return (*region_ptr)->contains(vaddr) ? *region_ptr : nullptr;
  560. }
  561. Region* MemoryManager::find_user_region_from_vaddr_no_lock(AddressSpace& space, VirtualAddress vaddr)
  562. {
  563. VERIFY(space.get_lock().is_locked_by_current_processor());
  564. return space.find_region_containing({ vaddr, 1 });
  565. }
  566. Region* MemoryManager::find_user_region_from_vaddr(AddressSpace& space, VirtualAddress vaddr)
  567. {
  568. SpinlockLocker lock(space.get_lock());
  569. return find_user_region_from_vaddr_no_lock(space, vaddr);
  570. }
  571. void MemoryManager::validate_syscall_preconditions(AddressSpace& space, RegisterState const& regs)
  572. {
  573. // We take the space lock once here and then use the no_lock variants
  574. // to avoid excessive spinlock recursion in this extremely common path.
  575. SpinlockLocker lock(space.get_lock());
  576. auto unlock_and_handle_crash = [&lock, &regs](const char* description, int signal) {
  577. lock.unlock();
  578. handle_crash(regs, description, signal);
  579. };
  580. {
  581. VirtualAddress userspace_sp = VirtualAddress { regs.userspace_sp() };
  582. if (!MM.validate_user_stack_no_lock(space, userspace_sp)) {
  583. dbgln("Invalid stack pointer: {}", userspace_sp);
  584. return unlock_and_handle_crash("Bad stack on syscall entry", SIGSEGV);
  585. }
  586. }
  587. {
  588. VirtualAddress ip = VirtualAddress { regs.ip() };
  589. auto* calling_region = MM.find_user_region_from_vaddr_no_lock(space, ip);
  590. if (!calling_region) {
  591. dbgln("Syscall from {:p} which has no associated region", ip);
  592. return unlock_and_handle_crash("Syscall from unknown region", SIGSEGV);
  593. }
  594. if (calling_region->is_writable()) {
  595. dbgln("Syscall from writable memory at {:p}", ip);
  596. return unlock_and_handle_crash("Syscall from writable memory", SIGSEGV);
  597. }
  598. if (space.enforces_syscall_regions() && !calling_region->is_syscall_region()) {
  599. dbgln("Syscall from non-syscall region");
  600. return unlock_and_handle_crash("Syscall from non-syscall region", SIGSEGV);
  601. }
  602. }
  603. }
  604. Region* MemoryManager::find_region_from_vaddr(VirtualAddress vaddr)
  605. {
  606. if (auto* region = kernel_region_from_vaddr(vaddr))
  607. return region;
  608. auto page_directory = PageDirectory::find_by_cr3(read_cr3());
  609. if (!page_directory)
  610. return nullptr;
  611. VERIFY(page_directory->address_space());
  612. return find_user_region_from_vaddr(*page_directory->address_space(), vaddr);
  613. }
  614. PageFaultResponse MemoryManager::handle_page_fault(PageFault const& fault)
  615. {
  616. VERIFY_INTERRUPTS_DISABLED();
  617. if (Processor::current_in_irq()) {
  618. dbgln("CPU[{}] BUG! Page fault while handling IRQ! code={}, vaddr={}, irq level: {}",
  619. Processor::current_id(), fault.code(), fault.vaddr(), Processor::current_in_irq());
  620. dump_kernel_regions();
  621. return PageFaultResponse::ShouldCrash;
  622. }
  623. dbgln_if(PAGE_FAULT_DEBUG, "MM: CPU[{}] handle_page_fault({:#04x}) at {}", Processor::current_id(), fault.code(), fault.vaddr());
  624. auto* region = find_region_from_vaddr(fault.vaddr());
  625. if (!region) {
  626. return PageFaultResponse::ShouldCrash;
  627. }
  628. return region->handle_fault(fault);
  629. }
  630. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_contiguous_kernel_region(size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  631. {
  632. VERIFY(!(size % PAGE_SIZE));
  633. SpinlockLocker lock(kernel_page_directory().get_lock());
  634. auto vmobject = TRY(AnonymousVMObject::try_create_physically_contiguous_with_size(size));
  635. auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size));
  636. return allocate_kernel_region_with_vmobject(range, move(vmobject), name, access, cacheable);
  637. }
  638. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::allocate_dma_buffer_page(StringView name, Memory::Region::Access access, RefPtr<Memory::PhysicalPage>& dma_buffer_page)
  639. {
  640. dma_buffer_page = allocate_supervisor_physical_page();
  641. if (dma_buffer_page.is_null())
  642. return ENOMEM;
  643. auto region_or_error = allocate_kernel_region(dma_buffer_page->paddr(), PAGE_SIZE, name, access);
  644. return region_or_error;
  645. }
  646. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::allocate_dma_buffer_pages(size_t size, StringView name, Memory::Region::Access access, NonnullRefPtrVector<Memory::PhysicalPage>& dma_buffer_pages)
  647. {
  648. VERIFY(!(size % PAGE_SIZE));
  649. dma_buffer_pages = allocate_contiguous_supervisor_physical_pages(size);
  650. if (dma_buffer_pages.is_empty())
  651. return ENOMEM;
  652. auto region_or_error = allocate_kernel_region(dma_buffer_pages.first().paddr(), size, name, access);
  653. return region_or_error;
  654. }
  655. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region(size_t size, StringView name, Region::Access access, AllocationStrategy strategy, Region::Cacheable cacheable)
  656. {
  657. VERIFY(!(size % PAGE_SIZE));
  658. auto vmobject = TRY(AnonymousVMObject::try_create_with_size(size, strategy));
  659. SpinlockLocker lock(kernel_page_directory().get_lock());
  660. auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size));
  661. return allocate_kernel_region_with_vmobject(range, move(vmobject), name, access, cacheable);
  662. }
  663. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region(PhysicalAddress paddr, size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  664. {
  665. VERIFY(!(size % PAGE_SIZE));
  666. auto vmobject = TRY(AnonymousVMObject::try_create_for_physical_range(paddr, size));
  667. SpinlockLocker lock(kernel_page_directory().get_lock());
  668. auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size));
  669. return allocate_kernel_region_with_vmobject(range, move(vmobject), name, access, cacheable);
  670. }
  671. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region_with_vmobject(VirtualRange const& range, VMObject& vmobject, StringView name, Region::Access access, Region::Cacheable cacheable)
  672. {
  673. OwnPtr<KString> name_kstring;
  674. if (!name.is_null())
  675. name_kstring = TRY(KString::try_create(name));
  676. auto region = TRY(Region::try_create_kernel_only(range, vmobject, 0, move(name_kstring), access, cacheable));
  677. TRY(region->map(kernel_page_directory()));
  678. return region;
  679. }
  680. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region_with_vmobject(VMObject& vmobject, size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  681. {
  682. VERIFY(!(size % PAGE_SIZE));
  683. SpinlockLocker lock(kernel_page_directory().get_lock());
  684. auto range = TRY(kernel_page_directory().range_allocator().try_allocate_anywhere(size));
  685. return allocate_kernel_region_with_vmobject(range, vmobject, name, access, cacheable);
  686. }
  687. ErrorOr<CommittedPhysicalPageSet> MemoryManager::commit_user_physical_pages(size_t page_count)
  688. {
  689. VERIFY(page_count > 0);
  690. SpinlockLocker lock(s_mm_lock);
  691. if (m_system_memory_info.user_physical_pages_uncommitted < page_count)
  692. return ENOMEM;
  693. m_system_memory_info.user_physical_pages_uncommitted -= page_count;
  694. m_system_memory_info.user_physical_pages_committed += page_count;
  695. return CommittedPhysicalPageSet { {}, page_count };
  696. }
  697. void MemoryManager::uncommit_user_physical_pages(Badge<CommittedPhysicalPageSet>, size_t page_count)
  698. {
  699. VERIFY(page_count > 0);
  700. SpinlockLocker lock(s_mm_lock);
  701. VERIFY(m_system_memory_info.user_physical_pages_committed >= page_count);
  702. m_system_memory_info.user_physical_pages_uncommitted += page_count;
  703. m_system_memory_info.user_physical_pages_committed -= page_count;
  704. }
  705. void MemoryManager::deallocate_physical_page(PhysicalAddress paddr)
  706. {
  707. SpinlockLocker lock(s_mm_lock);
  708. // Are we returning a user page?
  709. for (auto& region : m_user_physical_regions) {
  710. if (!region.contains(paddr))
  711. continue;
  712. region.return_page(paddr);
  713. --m_system_memory_info.user_physical_pages_used;
  714. // Always return pages to the uncommitted pool. Pages that were
  715. // committed and allocated are only freed upon request. Once
  716. // returned there is no guarantee being able to get them back.
  717. ++m_system_memory_info.user_physical_pages_uncommitted;
  718. return;
  719. }
  720. // If it's not a user page, it should be a supervisor page.
  721. if (!m_super_physical_region->contains(paddr))
  722. PANIC("MM: deallocate_user_physical_page couldn't figure out region for page @ {}", paddr);
  723. m_super_physical_region->return_page(paddr);
  724. --m_system_memory_info.super_physical_pages_used;
  725. }
  726. RefPtr<PhysicalPage> MemoryManager::find_free_user_physical_page(bool committed)
  727. {
  728. VERIFY(s_mm_lock.is_locked());
  729. RefPtr<PhysicalPage> page;
  730. if (committed) {
  731. // Draw from the committed pages pool. We should always have these pages available
  732. VERIFY(m_system_memory_info.user_physical_pages_committed > 0);
  733. m_system_memory_info.user_physical_pages_committed--;
  734. } else {
  735. // We need to make sure we don't touch pages that we have committed to
  736. if (m_system_memory_info.user_physical_pages_uncommitted == 0)
  737. return {};
  738. m_system_memory_info.user_physical_pages_uncommitted--;
  739. }
  740. for (auto& region : m_user_physical_regions) {
  741. page = region.take_free_page();
  742. if (!page.is_null()) {
  743. ++m_system_memory_info.user_physical_pages_used;
  744. break;
  745. }
  746. }
  747. VERIFY(!committed || !page.is_null());
  748. return page;
  749. }
  750. NonnullRefPtr<PhysicalPage> MemoryManager::allocate_committed_user_physical_page(Badge<CommittedPhysicalPageSet>, ShouldZeroFill should_zero_fill)
  751. {
  752. SpinlockLocker lock(s_mm_lock);
  753. auto page = find_free_user_physical_page(true);
  754. if (should_zero_fill == ShouldZeroFill::Yes) {
  755. auto* ptr = quickmap_page(*page);
  756. memset(ptr, 0, PAGE_SIZE);
  757. unquickmap_page();
  758. }
  759. return page.release_nonnull();
  760. }
  761. RefPtr<PhysicalPage> MemoryManager::allocate_user_physical_page(ShouldZeroFill should_zero_fill, bool* did_purge)
  762. {
  763. SpinlockLocker lock(s_mm_lock);
  764. auto page = find_free_user_physical_page(false);
  765. bool purged_pages = false;
  766. if (!page) {
  767. // We didn't have a single free physical page. Let's try to free something up!
  768. // First, we look for a purgeable VMObject in the volatile state.
  769. for_each_vmobject([&](auto& vmobject) {
  770. if (!vmobject.is_anonymous())
  771. return IterationDecision::Continue;
  772. auto& anonymous_vmobject = static_cast<AnonymousVMObject&>(vmobject);
  773. if (!anonymous_vmobject.is_purgeable() || !anonymous_vmobject.is_volatile())
  774. return IterationDecision::Continue;
  775. if (auto purged_page_count = anonymous_vmobject.purge()) {
  776. dbgln("MM: Purge saved the day! Purged {} pages from AnonymousVMObject", purged_page_count);
  777. page = find_free_user_physical_page(false);
  778. purged_pages = true;
  779. VERIFY(page);
  780. return IterationDecision::Break;
  781. }
  782. return IterationDecision::Continue;
  783. });
  784. if (!page) {
  785. dmesgln("MM: no user physical pages available");
  786. return {};
  787. }
  788. }
  789. if (should_zero_fill == ShouldZeroFill::Yes) {
  790. auto* ptr = quickmap_page(*page);
  791. memset(ptr, 0, PAGE_SIZE);
  792. unquickmap_page();
  793. }
  794. if (did_purge)
  795. *did_purge = purged_pages;
  796. return page;
  797. }
  798. NonnullRefPtrVector<PhysicalPage> MemoryManager::allocate_contiguous_supervisor_physical_pages(size_t size)
  799. {
  800. VERIFY(!(size % PAGE_SIZE));
  801. SpinlockLocker lock(s_mm_lock);
  802. size_t count = ceil_div(size, static_cast<size_t>(PAGE_SIZE));
  803. auto physical_pages = m_super_physical_region->take_contiguous_free_pages(count);
  804. if (physical_pages.is_empty()) {
  805. dmesgln("MM: no super physical pages available");
  806. VERIFY_NOT_REACHED();
  807. return {};
  808. }
  809. {
  810. auto region_or_error = MM.allocate_kernel_region(physical_pages[0].paddr(), PAGE_SIZE * count, "MemoryManager Allocation Sanitization", Region::Access::Read | Region::Access::Write);
  811. if (region_or_error.is_error())
  812. TODO();
  813. auto cleanup_region = region_or_error.release_value();
  814. fast_u32_fill((u32*)cleanup_region->vaddr().as_ptr(), 0, (PAGE_SIZE * count) / sizeof(u32));
  815. }
  816. m_system_memory_info.super_physical_pages_used += count;
  817. return physical_pages;
  818. }
  819. RefPtr<PhysicalPage> MemoryManager::allocate_supervisor_physical_page()
  820. {
  821. SpinlockLocker lock(s_mm_lock);
  822. auto page = m_super_physical_region->take_free_page();
  823. if (!page) {
  824. dmesgln("MM: no super physical pages available");
  825. VERIFY_NOT_REACHED();
  826. return {};
  827. }
  828. fast_u32_fill((u32*)page->paddr().offset(physical_to_virtual_offset).as_ptr(), 0, PAGE_SIZE / sizeof(u32));
  829. ++m_system_memory_info.super_physical_pages_used;
  830. return page;
  831. }
  832. void MemoryManager::enter_process_address_space(Process& process)
  833. {
  834. enter_address_space(process.address_space());
  835. }
  836. void MemoryManager::enter_address_space(AddressSpace& space)
  837. {
  838. auto current_thread = Thread::current();
  839. VERIFY(current_thread != nullptr);
  840. SpinlockLocker lock(s_mm_lock);
  841. current_thread->regs().cr3 = space.page_directory().cr3();
  842. write_cr3(space.page_directory().cr3());
  843. }
  844. void MemoryManager::flush_tlb_local(VirtualAddress vaddr, size_t page_count)
  845. {
  846. Processor::flush_tlb_local(vaddr, page_count);
  847. }
  848. void MemoryManager::flush_tlb(PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count)
  849. {
  850. Processor::flush_tlb(page_directory, vaddr, page_count);
  851. }
  852. PageDirectoryEntry* MemoryManager::quickmap_pd(PageDirectory& directory, size_t pdpt_index)
  853. {
  854. VERIFY(s_mm_lock.is_locked_by_current_processor());
  855. auto& mm_data = get_data();
  856. auto& pte = boot_pd_kernel_pt1023[(KERNEL_QUICKMAP_PD - KERNEL_PT1024_BASE) / PAGE_SIZE];
  857. auto pd_paddr = directory.m_directory_pages[pdpt_index]->paddr();
  858. if (pte.physical_page_base() != pd_paddr.get()) {
  859. pte.set_physical_page_base(pd_paddr.get());
  860. pte.set_present(true);
  861. pte.set_writable(true);
  862. pte.set_user_allowed(false);
  863. // Because we must continue to hold the MM lock while we use this
  864. // mapping, it is sufficient to only flush on the current CPU. Other
  865. // CPUs trying to use this API must wait on the MM lock anyway
  866. flush_tlb_local(VirtualAddress(KERNEL_QUICKMAP_PD));
  867. } else {
  868. // Even though we don't allow this to be called concurrently, it's
  869. // possible that this PD was mapped on a different CPU and we don't
  870. // broadcast the flush. If so, we still need to flush the TLB.
  871. if (mm_data.m_last_quickmap_pd != pd_paddr)
  872. flush_tlb_local(VirtualAddress(KERNEL_QUICKMAP_PD));
  873. }
  874. mm_data.m_last_quickmap_pd = pd_paddr;
  875. return (PageDirectoryEntry*)KERNEL_QUICKMAP_PD;
  876. }
  877. PageTableEntry* MemoryManager::quickmap_pt(PhysicalAddress pt_paddr)
  878. {
  879. VERIFY(s_mm_lock.is_locked_by_current_processor());
  880. auto& mm_data = get_data();
  881. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[(KERNEL_QUICKMAP_PT - KERNEL_PT1024_BASE) / PAGE_SIZE];
  882. if (pte.physical_page_base() != pt_paddr.get()) {
  883. pte.set_physical_page_base(pt_paddr.get());
  884. pte.set_present(true);
  885. pte.set_writable(true);
  886. pte.set_user_allowed(false);
  887. // Because we must continue to hold the MM lock while we use this
  888. // mapping, it is sufficient to only flush on the current CPU. Other
  889. // CPUs trying to use this API must wait on the MM lock anyway
  890. flush_tlb_local(VirtualAddress(KERNEL_QUICKMAP_PT));
  891. } else {
  892. // Even though we don't allow this to be called concurrently, it's
  893. // possible that this PT was mapped on a different CPU and we don't
  894. // broadcast the flush. If so, we still need to flush the TLB.
  895. if (mm_data.m_last_quickmap_pt != pt_paddr)
  896. flush_tlb_local(VirtualAddress(KERNEL_QUICKMAP_PT));
  897. }
  898. mm_data.m_last_quickmap_pt = pt_paddr;
  899. return (PageTableEntry*)KERNEL_QUICKMAP_PT;
  900. }
  901. u8* MemoryManager::quickmap_page(PhysicalAddress const& physical_address)
  902. {
  903. VERIFY_INTERRUPTS_DISABLED();
  904. VERIFY(s_mm_lock.is_locked_by_current_processor());
  905. auto& mm_data = get_data();
  906. mm_data.m_quickmap_prev_flags = mm_data.m_quickmap_in_use.lock();
  907. VirtualAddress vaddr(KERNEL_QUICKMAP_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  908. u32 pte_idx = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  909. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[pte_idx];
  910. if (pte.physical_page_base() != physical_address.get()) {
  911. pte.set_physical_page_base(physical_address.get());
  912. pte.set_present(true);
  913. pte.set_writable(true);
  914. pte.set_user_allowed(false);
  915. flush_tlb_local(vaddr);
  916. }
  917. return vaddr.as_ptr();
  918. }
  919. void MemoryManager::unquickmap_page()
  920. {
  921. VERIFY_INTERRUPTS_DISABLED();
  922. VERIFY(s_mm_lock.is_locked_by_current_processor());
  923. auto& mm_data = get_data();
  924. VERIFY(mm_data.m_quickmap_in_use.is_locked());
  925. VirtualAddress vaddr(KERNEL_QUICKMAP_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  926. u32 pte_idx = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  927. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[pte_idx];
  928. pte.clear();
  929. flush_tlb_local(vaddr);
  930. mm_data.m_quickmap_in_use.unlock(mm_data.m_quickmap_prev_flags);
  931. }
  932. bool MemoryManager::validate_user_stack_no_lock(AddressSpace& space, VirtualAddress vaddr) const
  933. {
  934. VERIFY(space.get_lock().is_locked_by_current_processor());
  935. if (!is_user_address(vaddr))
  936. return false;
  937. auto* region = find_user_region_from_vaddr_no_lock(space, vaddr);
  938. return region && region->is_user() && region->is_stack();
  939. }
  940. bool MemoryManager::validate_user_stack(AddressSpace& space, VirtualAddress vaddr) const
  941. {
  942. SpinlockLocker lock(space.get_lock());
  943. return validate_user_stack_no_lock(space, vaddr);
  944. }
  945. void MemoryManager::register_region(Region& region)
  946. {
  947. SpinlockLocker lock(s_mm_lock);
  948. if (region.is_kernel())
  949. m_kernel_regions.insert(region.vaddr().get(), &region);
  950. }
  951. void MemoryManager::unregister_region(Region& region)
  952. {
  953. SpinlockLocker lock(s_mm_lock);
  954. if (region.is_kernel())
  955. m_kernel_regions.remove(region.vaddr().get());
  956. }
  957. void MemoryManager::dump_kernel_regions()
  958. {
  959. dbgln("Kernel regions:");
  960. #if ARCH(I386)
  961. auto addr_padding = "";
  962. #else
  963. auto addr_padding = " ";
  964. #endif
  965. dbgln("BEGIN{} END{} SIZE{} ACCESS NAME",
  966. addr_padding, addr_padding, addr_padding);
  967. SpinlockLocker lock(s_mm_lock);
  968. for (auto* region_ptr : m_kernel_regions) {
  969. auto& region = *region_ptr;
  970. dbgln("{:p} -- {:p} {:p} {:c}{:c}{:c}{:c}{:c}{:c} {}",
  971. region.vaddr().get(),
  972. region.vaddr().offset(region.size() - 1).get(),
  973. region.size(),
  974. region.is_readable() ? 'R' : ' ',
  975. region.is_writable() ? 'W' : ' ',
  976. region.is_executable() ? 'X' : ' ',
  977. region.is_shared() ? 'S' : ' ',
  978. region.is_stack() ? 'T' : ' ',
  979. region.is_syscall_region() ? 'C' : ' ',
  980. region.name());
  981. }
  982. }
  983. void MemoryManager::set_page_writable_direct(VirtualAddress vaddr, bool writable)
  984. {
  985. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  986. SpinlockLocker lock(s_mm_lock);
  987. auto* pte = ensure_pte(kernel_page_directory(), vaddr);
  988. VERIFY(pte);
  989. if (pte->is_writable() == writable)
  990. return;
  991. pte->set_writable(writable);
  992. flush_tlb(&kernel_page_directory(), vaddr);
  993. }
  994. CommittedPhysicalPageSet::~CommittedPhysicalPageSet()
  995. {
  996. if (m_page_count)
  997. MM.uncommit_user_physical_pages({}, m_page_count);
  998. }
  999. NonnullRefPtr<PhysicalPage> CommittedPhysicalPageSet::take_one()
  1000. {
  1001. VERIFY(m_page_count > 0);
  1002. --m_page_count;
  1003. return MM.allocate_committed_user_physical_page({}, MemoryManager::ShouldZeroFill::Yes);
  1004. }
  1005. void CommittedPhysicalPageSet::uncommit_one()
  1006. {
  1007. VERIFY(m_page_count > 0);
  1008. --m_page_count;
  1009. MM.uncommit_user_physical_pages({}, 1);
  1010. }
  1011. void MemoryManager::copy_physical_page(PhysicalPage& physical_page, u8 page_buffer[PAGE_SIZE])
  1012. {
  1013. SpinlockLocker locker(s_mm_lock);
  1014. auto* quickmapped_page = quickmap_page(physical_page);
  1015. memcpy(page_buffer, quickmapped_page, PAGE_SIZE);
  1016. unquickmap_page();
  1017. }
  1018. }