MemoryManager.cpp 50 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219
  1. /*
  2. * Copyright (c) 2018-2022, Andreas Kling <kling@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <AK/Assertions.h>
  7. #include <AK/Memory.h>
  8. #include <AK/StringView.h>
  9. #include <Kernel/Arch/CPU.h>
  10. #include <Kernel/Arch/PageDirectory.h>
  11. #include <Kernel/Arch/PageFault.h>
  12. #include <Kernel/Arch/RegisterState.h>
  13. #include <Kernel/BootInfo.h>
  14. #include <Kernel/CMOS.h>
  15. #include <Kernel/FileSystem/Inode.h>
  16. #include <Kernel/Heap/kmalloc.h>
  17. #include <Kernel/KSyms.h>
  18. #include <Kernel/Memory/AnonymousVMObject.h>
  19. #include <Kernel/Memory/MemoryManager.h>
  20. #include <Kernel/Memory/PageDirectory.h>
  21. #include <Kernel/Memory/PhysicalRegion.h>
  22. #include <Kernel/Memory/SharedInodeVMObject.h>
  23. #include <Kernel/Multiboot.h>
  24. #include <Kernel/Panic.h>
  25. #include <Kernel/Prekernel/Prekernel.h>
  26. #include <Kernel/Process.h>
  27. #include <Kernel/Sections.h>
  28. #include <Kernel/StdLib.h>
  29. extern u8 start_of_kernel_image[];
  30. extern u8 end_of_kernel_image[];
  31. extern u8 start_of_kernel_text[];
  32. extern u8 start_of_kernel_data[];
  33. extern u8 end_of_kernel_bss[];
  34. extern u8 start_of_ro_after_init[];
  35. extern u8 end_of_ro_after_init[];
  36. extern u8 start_of_unmap_after_init[];
  37. extern u8 end_of_unmap_after_init[];
  38. extern u8 start_of_kernel_ksyms[];
  39. extern u8 end_of_kernel_ksyms[];
  40. extern multiboot_module_entry_t multiboot_copy_boot_modules_array[16];
  41. extern size_t multiboot_copy_boot_modules_count;
  42. namespace Kernel::Memory {
  43. ErrorOr<FlatPtr> page_round_up(FlatPtr x)
  44. {
  45. if (x > (explode_byte(0xFF) & ~0xFFF)) {
  46. return Error::from_errno(EINVAL);
  47. }
  48. return (((FlatPtr)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1));
  49. }
  50. // NOTE: We can NOT use Singleton for this class, because
  51. // MemoryManager::initialize is called *before* global constructors are
  52. // run. If we do, then Singleton would get re-initialized, causing
  53. // the memory manager to be initialized twice!
  54. static MemoryManager* s_the;
  55. // The MM lock protects:
  56. // - all data members of MemoryManager
  57. // - the quickmap mechanism
  58. // - the PTE/PDE mapping mechanism
  59. RecursiveSpinlock s_mm_lock { LockRank::MemoryManager };
  60. MemoryManager& MemoryManager::the()
  61. {
  62. return *s_the;
  63. }
  64. bool MemoryManager::is_initialized()
  65. {
  66. return s_the != nullptr;
  67. }
  68. static UNMAP_AFTER_INIT VirtualRange kernel_virtual_range()
  69. {
  70. size_t kernel_range_start = kernel_mapping_base + 2 * MiB; // The first 2 MiB are used for mapping the pre-kernel
  71. return VirtualRange { VirtualAddress(kernel_range_start), KERNEL_PD_END - kernel_range_start };
  72. }
  73. UNMAP_AFTER_INIT MemoryManager::MemoryManager()
  74. : m_region_tree(kernel_virtual_range())
  75. {
  76. s_the = this;
  77. SpinlockLocker lock(s_mm_lock);
  78. parse_memory_map();
  79. activate_kernel_page_directory(kernel_page_directory());
  80. protect_kernel_image();
  81. // We're temporarily "committing" to two pages that we need to allocate below
  82. auto committed_pages = commit_physical_pages(2).release_value();
  83. m_shared_zero_page = committed_pages.take_one();
  84. // We're wasting a page here, we just need a special tag (physical
  85. // address) so that we know when we need to lazily allocate a page
  86. // that we should be drawing this page from the committed pool rather
  87. // than potentially failing if no pages are available anymore.
  88. // By using a tag we don't have to query the VMObject for every page
  89. // whether it was committed or not
  90. m_lazy_committed_page = committed_pages.take_one();
  91. }
  92. UNMAP_AFTER_INIT MemoryManager::~MemoryManager() = default;
  93. UNMAP_AFTER_INIT void MemoryManager::protect_kernel_image()
  94. {
  95. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  96. // Disable writing to the kernel text and rodata segments.
  97. for (auto const* i = start_of_kernel_text; i < start_of_kernel_data; i += PAGE_SIZE) {
  98. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  99. pte.set_writable(false);
  100. }
  101. if (Processor::current().has_nx()) {
  102. // Disable execution of the kernel data, bss and heap segments.
  103. for (auto const* i = start_of_kernel_data; i < end_of_kernel_image; i += PAGE_SIZE) {
  104. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  105. pte.set_execute_disabled(true);
  106. }
  107. }
  108. }
  109. UNMAP_AFTER_INIT void MemoryManager::unmap_prekernel()
  110. {
  111. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  112. SpinlockLocker mm_lock(s_mm_lock);
  113. auto start = start_of_prekernel_image.page_base().get();
  114. auto end = end_of_prekernel_image.page_base().get();
  115. for (auto i = start; i <= end; i += PAGE_SIZE)
  116. release_pte(kernel_page_directory(), VirtualAddress(i), i == end ? IsLastPTERelease::Yes : IsLastPTERelease::No);
  117. flush_tlb(&kernel_page_directory(), VirtualAddress(start), (end - start) / PAGE_SIZE);
  118. }
  119. UNMAP_AFTER_INIT void MemoryManager::protect_readonly_after_init_memory()
  120. {
  121. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  122. SpinlockLocker mm_lock(s_mm_lock);
  123. // Disable writing to the .ro_after_init section
  124. for (auto i = (FlatPtr)&start_of_ro_after_init; i < (FlatPtr)&end_of_ro_after_init; i += PAGE_SIZE) {
  125. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  126. pte.set_writable(false);
  127. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  128. }
  129. }
  130. void MemoryManager::unmap_text_after_init()
  131. {
  132. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  133. SpinlockLocker mm_lock(s_mm_lock);
  134. auto start = page_round_down((FlatPtr)&start_of_unmap_after_init);
  135. auto end = page_round_up((FlatPtr)&end_of_unmap_after_init).release_value_but_fixme_should_propagate_errors();
  136. // Unmap the entire .unmap_after_init section
  137. for (auto i = start; i < end; i += PAGE_SIZE) {
  138. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  139. pte.clear();
  140. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  141. }
  142. dmesgln("Unmapped {} KiB of kernel text after init! :^)", (end - start) / KiB);
  143. }
  144. UNMAP_AFTER_INIT void MemoryManager::protect_ksyms_after_init()
  145. {
  146. SpinlockLocker mm_lock(s_mm_lock);
  147. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  148. auto start = page_round_down((FlatPtr)start_of_kernel_ksyms);
  149. auto end = page_round_up((FlatPtr)end_of_kernel_ksyms).release_value_but_fixme_should_propagate_errors();
  150. for (auto i = start; i < end; i += PAGE_SIZE) {
  151. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  152. pte.set_writable(false);
  153. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  154. }
  155. dmesgln("Write-protected kernel symbols after init.");
  156. }
  157. IterationDecision MemoryManager::for_each_physical_memory_range(Function<IterationDecision(PhysicalMemoryRange const&)> callback)
  158. {
  159. VERIFY(!m_physical_memory_ranges.is_empty());
  160. for (auto& current_range : m_physical_memory_ranges) {
  161. IterationDecision decision = callback(current_range);
  162. if (decision != IterationDecision::Continue)
  163. return decision;
  164. }
  165. return IterationDecision::Continue;
  166. }
  167. UNMAP_AFTER_INIT void MemoryManager::register_reserved_ranges()
  168. {
  169. VERIFY(!m_physical_memory_ranges.is_empty());
  170. ContiguousReservedMemoryRange range;
  171. for (auto& current_range : m_physical_memory_ranges) {
  172. if (current_range.type != PhysicalMemoryRangeType::Reserved) {
  173. if (range.start.is_null())
  174. continue;
  175. m_reserved_memory_ranges.append(ContiguousReservedMemoryRange { range.start, current_range.start.get() - range.start.get() });
  176. range.start.set((FlatPtr) nullptr);
  177. continue;
  178. }
  179. if (!range.start.is_null()) {
  180. continue;
  181. }
  182. range.start = current_range.start;
  183. }
  184. if (m_physical_memory_ranges.last().type != PhysicalMemoryRangeType::Reserved)
  185. return;
  186. if (range.start.is_null())
  187. return;
  188. m_reserved_memory_ranges.append(ContiguousReservedMemoryRange { range.start, m_physical_memory_ranges.last().start.get() + m_physical_memory_ranges.last().length - range.start.get() });
  189. }
  190. bool MemoryManager::is_allowed_to_read_physical_memory_for_userspace(PhysicalAddress start_address, size_t read_length) const
  191. {
  192. // Note: Guard against overflow in case someone tries to mmap on the edge of
  193. // the RAM
  194. if (start_address.offset_addition_would_overflow(read_length))
  195. return false;
  196. auto end_address = start_address.offset(read_length);
  197. for (auto const& current_range : m_reserved_memory_ranges) {
  198. if (current_range.start > start_address)
  199. continue;
  200. if (current_range.start.offset(current_range.length) < end_address)
  201. continue;
  202. return true;
  203. }
  204. return false;
  205. }
  206. UNMAP_AFTER_INIT void MemoryManager::parse_memory_map()
  207. {
  208. // Register used memory regions that we know of.
  209. m_used_memory_ranges.ensure_capacity(4);
  210. m_used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::LowMemory, PhysicalAddress(0x00000000), PhysicalAddress(1 * MiB) });
  211. m_used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::Kernel, PhysicalAddress(virtual_to_low_physical((FlatPtr)start_of_kernel_image)), PhysicalAddress(page_round_up(virtual_to_low_physical((FlatPtr)end_of_kernel_image)).release_value_but_fixme_should_propagate_errors()) });
  212. if (multiboot_flags & 0x4) {
  213. auto* bootmods_start = multiboot_copy_boot_modules_array;
  214. auto* bootmods_end = bootmods_start + multiboot_copy_boot_modules_count;
  215. for (auto* bootmod = bootmods_start; bootmod < bootmods_end; bootmod++) {
  216. m_used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::BootModule, PhysicalAddress(bootmod->start), PhysicalAddress(bootmod->end) });
  217. }
  218. }
  219. auto* mmap_begin = multiboot_memory_map;
  220. auto* mmap_end = multiboot_memory_map + multiboot_memory_map_count;
  221. struct ContiguousPhysicalVirtualRange {
  222. PhysicalAddress lower;
  223. PhysicalAddress upper;
  224. };
  225. Vector<ContiguousPhysicalVirtualRange> contiguous_physical_ranges;
  226. for (auto* mmap = mmap_begin; mmap < mmap_end; mmap++) {
  227. // We have to copy these onto the stack, because we take a reference to these when printing them out,
  228. // and doing so on a packed struct field is UB.
  229. auto address = mmap->addr;
  230. auto length = mmap->len;
  231. ArmedScopeGuard write_back_guard = [&]() {
  232. mmap->addr = address;
  233. mmap->len = length;
  234. };
  235. dmesgln("MM: Multiboot mmap: address={:p}, length={}, type={}", address, length, mmap->type);
  236. auto start_address = PhysicalAddress(address);
  237. switch (mmap->type) {
  238. case (MULTIBOOT_MEMORY_AVAILABLE):
  239. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Usable, start_address, length });
  240. break;
  241. case (MULTIBOOT_MEMORY_RESERVED):
  242. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Reserved, start_address, length });
  243. break;
  244. case (MULTIBOOT_MEMORY_ACPI_RECLAIMABLE):
  245. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_Reclaimable, start_address, length });
  246. break;
  247. case (MULTIBOOT_MEMORY_NVS):
  248. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_NVS, start_address, length });
  249. break;
  250. case (MULTIBOOT_MEMORY_BADRAM):
  251. dmesgln("MM: Warning, detected bad memory range!");
  252. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::BadMemory, start_address, length });
  253. break;
  254. default:
  255. dbgln("MM: Unknown range!");
  256. m_physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Unknown, start_address, length });
  257. break;
  258. }
  259. if (mmap->type != MULTIBOOT_MEMORY_AVAILABLE)
  260. continue;
  261. // Fix up unaligned memory regions.
  262. auto diff = (FlatPtr)address % PAGE_SIZE;
  263. if (diff != 0) {
  264. dmesgln("MM: Got an unaligned physical_region from the bootloader; correcting {:p} by {} bytes", address, diff);
  265. diff = PAGE_SIZE - diff;
  266. address += diff;
  267. length -= diff;
  268. }
  269. if ((length % PAGE_SIZE) != 0) {
  270. dmesgln("MM: Got an unaligned physical_region from the bootloader; correcting length {} by {} bytes", length, length % PAGE_SIZE);
  271. length -= length % PAGE_SIZE;
  272. }
  273. if (length < PAGE_SIZE) {
  274. dmesgln("MM: Memory physical_region from bootloader is too small; we want >= {} bytes, but got {} bytes", PAGE_SIZE, length);
  275. continue;
  276. }
  277. for (PhysicalSize page_base = address; page_base <= (address + length); page_base += PAGE_SIZE) {
  278. auto addr = PhysicalAddress(page_base);
  279. // Skip used memory ranges.
  280. bool should_skip = false;
  281. for (auto& used_range : m_used_memory_ranges) {
  282. if (addr.get() >= used_range.start.get() && addr.get() <= used_range.end.get()) {
  283. should_skip = true;
  284. break;
  285. }
  286. }
  287. if (should_skip)
  288. continue;
  289. if (contiguous_physical_ranges.is_empty() || contiguous_physical_ranges.last().upper.offset(PAGE_SIZE) != addr) {
  290. contiguous_physical_ranges.append(ContiguousPhysicalVirtualRange {
  291. .lower = addr,
  292. .upper = addr,
  293. });
  294. } else {
  295. contiguous_physical_ranges.last().upper = addr;
  296. }
  297. }
  298. }
  299. for (auto& range : contiguous_physical_ranges) {
  300. m_physical_regions.append(PhysicalRegion::try_create(range.lower, range.upper).release_nonnull());
  301. }
  302. for (auto& region : m_physical_regions)
  303. m_system_memory_info.physical_pages += region.size();
  304. register_reserved_ranges();
  305. for (auto& range : m_reserved_memory_ranges) {
  306. dmesgln("MM: Contiguous reserved range from {}, length is {}", range.start, range.length);
  307. }
  308. initialize_physical_pages();
  309. VERIFY(m_system_memory_info.physical_pages > 0);
  310. // We start out with no committed pages
  311. m_system_memory_info.physical_pages_uncommitted = m_system_memory_info.physical_pages;
  312. for (auto& used_range : m_used_memory_ranges) {
  313. dmesgln("MM: {} range @ {} - {} (size {:#x})", UserMemoryRangeTypeNames[to_underlying(used_range.type)], used_range.start, used_range.end.offset(-1), used_range.end.as_ptr() - used_range.start.as_ptr());
  314. }
  315. for (auto& region : m_physical_regions) {
  316. dmesgln("MM: User physical region: {} - {} (size {:#x})", region.lower(), region.upper().offset(-1), PAGE_SIZE * region.size());
  317. region.initialize_zones();
  318. }
  319. }
  320. UNMAP_AFTER_INIT void MemoryManager::initialize_physical_pages()
  321. {
  322. // We assume that the physical page range is contiguous and doesn't contain huge gaps!
  323. PhysicalAddress highest_physical_address;
  324. for (auto& range : m_used_memory_ranges) {
  325. if (range.end.get() > highest_physical_address.get())
  326. highest_physical_address = range.end;
  327. }
  328. for (auto& region : m_physical_memory_ranges) {
  329. auto range_end = PhysicalAddress(region.start).offset(region.length);
  330. if (range_end.get() > highest_physical_address.get())
  331. highest_physical_address = range_end;
  332. }
  333. // Calculate how many total physical pages the array will have
  334. m_physical_page_entries_count = PhysicalAddress::physical_page_index(highest_physical_address.get()) + 1;
  335. VERIFY(m_physical_page_entries_count != 0);
  336. VERIFY(!Checked<decltype(m_physical_page_entries_count)>::multiplication_would_overflow(m_physical_page_entries_count, sizeof(PhysicalPageEntry)));
  337. // Calculate how many bytes the array will consume
  338. auto physical_page_array_size = m_physical_page_entries_count * sizeof(PhysicalPageEntry);
  339. auto physical_page_array_pages = page_round_up(physical_page_array_size).release_value_but_fixme_should_propagate_errors() / PAGE_SIZE;
  340. VERIFY(physical_page_array_pages * PAGE_SIZE >= physical_page_array_size);
  341. // Calculate how many page tables we will need to be able to map them all
  342. auto needed_page_table_count = (physical_page_array_pages + 512 - 1) / 512;
  343. auto physical_page_array_pages_and_page_tables_count = physical_page_array_pages + needed_page_table_count;
  344. // Now that we know how much memory we need for a contiguous array of PhysicalPage instances, find a memory region that can fit it
  345. PhysicalRegion* found_region { nullptr };
  346. Optional<size_t> found_region_index;
  347. for (size_t i = 0; i < m_physical_regions.size(); ++i) {
  348. auto& region = m_physical_regions[i];
  349. if (region.size() >= physical_page_array_pages_and_page_tables_count) {
  350. found_region = &region;
  351. found_region_index = i;
  352. break;
  353. }
  354. }
  355. if (!found_region) {
  356. dmesgln("MM: Need {} bytes for physical page management, but no memory region is large enough!", physical_page_array_pages_and_page_tables_count);
  357. VERIFY_NOT_REACHED();
  358. }
  359. VERIFY(m_system_memory_info.physical_pages >= physical_page_array_pages_and_page_tables_count);
  360. m_system_memory_info.physical_pages -= physical_page_array_pages_and_page_tables_count;
  361. if (found_region->size() == physical_page_array_pages_and_page_tables_count) {
  362. // We're stealing the entire region
  363. m_physical_pages_region = m_physical_regions.take(*found_region_index);
  364. } else {
  365. m_physical_pages_region = found_region->try_take_pages_from_beginning(physical_page_array_pages_and_page_tables_count);
  366. }
  367. m_used_memory_ranges.append({ UsedMemoryRangeType::PhysicalPages, m_physical_pages_region->lower(), m_physical_pages_region->upper() });
  368. // Create the bare page directory. This is not a fully constructed page directory and merely contains the allocators!
  369. m_kernel_page_directory = PageDirectory::must_create_kernel_page_directory();
  370. {
  371. // Carve out the whole page directory covering the kernel image to make MemoryManager::initialize_physical_pages() happy
  372. FlatPtr start_of_range = ((FlatPtr)start_of_kernel_image & ~(FlatPtr)0x1fffff);
  373. FlatPtr end_of_range = ((FlatPtr)end_of_kernel_image & ~(FlatPtr)0x1fffff) + 0x200000;
  374. MUST(m_region_tree.place_specifically(*MUST(Region::create_unbacked()).leak_ptr(), VirtualRange { VirtualAddress(start_of_range), end_of_range - start_of_range }));
  375. }
  376. // Allocate a virtual address range for our array
  377. // This looks awkward, but it basically creates a dummy region to occupy the address range permanently.
  378. auto& region = *MUST(Region::create_unbacked()).leak_ptr();
  379. MUST(m_region_tree.place_anywhere(region, RandomizeVirtualAddress::No, physical_page_array_pages * PAGE_SIZE));
  380. auto range = region.range();
  381. // Now that we have our special m_physical_pages_region region with enough pages to hold the entire array
  382. // try to map the entire region into kernel space so we always have it
  383. // We can't use ensure_pte here because it would try to allocate a PhysicalPage and we don't have the array
  384. // mapped yet so we can't create them
  385. SpinlockLocker lock(s_mm_lock);
  386. // Create page tables at the beginning of m_physical_pages_region, followed by the PhysicalPageEntry array
  387. auto page_tables_base = m_physical_pages_region->lower();
  388. auto physical_page_array_base = page_tables_base.offset(needed_page_table_count * PAGE_SIZE);
  389. auto physical_page_array_current_page = physical_page_array_base.get();
  390. auto virtual_page_array_base = range.base().get();
  391. auto virtual_page_array_current_page = virtual_page_array_base;
  392. for (size_t pt_index = 0; pt_index < needed_page_table_count; pt_index++) {
  393. auto virtual_page_base_for_this_pt = virtual_page_array_current_page;
  394. auto pt_paddr = page_tables_base.offset(pt_index * PAGE_SIZE);
  395. auto* pt = reinterpret_cast<PageTableEntry*>(quickmap_page(pt_paddr));
  396. __builtin_memset(pt, 0, PAGE_SIZE);
  397. for (size_t pte_index = 0; pte_index < PAGE_SIZE / sizeof(PageTableEntry); pte_index++) {
  398. auto& pte = pt[pte_index];
  399. pte.set_physical_page_base(physical_page_array_current_page);
  400. pte.set_user_allowed(false);
  401. pte.set_writable(true);
  402. if (Processor::current().has_nx())
  403. pte.set_execute_disabled(false);
  404. pte.set_global(true);
  405. pte.set_present(true);
  406. physical_page_array_current_page += PAGE_SIZE;
  407. virtual_page_array_current_page += PAGE_SIZE;
  408. }
  409. unquickmap_page();
  410. // Hook the page table into the kernel page directory
  411. u32 page_directory_index = (virtual_page_base_for_this_pt >> 21) & 0x1ff;
  412. auto* pd = reinterpret_cast<PageDirectoryEntry*>(quickmap_page(boot_pd_kernel));
  413. PageDirectoryEntry& pde = pd[page_directory_index];
  414. VERIFY(!pde.is_present()); // Nothing should be using this PD yet
  415. // We can't use ensure_pte quite yet!
  416. pde.set_page_table_base(pt_paddr.get());
  417. pde.set_user_allowed(false);
  418. pde.set_present(true);
  419. pde.set_writable(true);
  420. pde.set_global(true);
  421. unquickmap_page();
  422. flush_tlb_local(VirtualAddress(virtual_page_base_for_this_pt));
  423. }
  424. // We now have the entire PhysicalPageEntry array mapped!
  425. m_physical_page_entries = (PhysicalPageEntry*)range.base().get();
  426. for (size_t i = 0; i < m_physical_page_entries_count; i++)
  427. new (&m_physical_page_entries[i]) PageTableEntry();
  428. // Now we should be able to allocate PhysicalPage instances,
  429. // so finish setting up the kernel page directory
  430. m_kernel_page_directory->allocate_kernel_directory();
  431. // Now create legit PhysicalPage objects for the page tables we created.
  432. virtual_page_array_current_page = virtual_page_array_base;
  433. for (size_t pt_index = 0; pt_index < needed_page_table_count; pt_index++) {
  434. VERIFY(virtual_page_array_current_page <= range.end().get());
  435. auto pt_paddr = page_tables_base.offset(pt_index * PAGE_SIZE);
  436. auto physical_page_index = PhysicalAddress::physical_page_index(pt_paddr.get());
  437. auto& physical_page_entry = m_physical_page_entries[physical_page_index];
  438. auto physical_page = adopt_lock_ref(*new (&physical_page_entry.allocated.physical_page) PhysicalPage(MayReturnToFreeList::No));
  439. // NOTE: This leaked ref is matched by the unref in MemoryManager::release_pte()
  440. (void)physical_page.leak_ref();
  441. virtual_page_array_current_page += (PAGE_SIZE / sizeof(PageTableEntry)) * PAGE_SIZE;
  442. }
  443. dmesgln("MM: Physical page entries: {}", range);
  444. }
  445. PhysicalPageEntry& MemoryManager::get_physical_page_entry(PhysicalAddress physical_address)
  446. {
  447. VERIFY(m_physical_page_entries);
  448. auto physical_page_entry_index = PhysicalAddress::physical_page_index(physical_address.get());
  449. VERIFY(physical_page_entry_index < m_physical_page_entries_count);
  450. return m_physical_page_entries[physical_page_entry_index];
  451. }
  452. PhysicalAddress MemoryManager::get_physical_address(PhysicalPage const& physical_page)
  453. {
  454. PhysicalPageEntry const& physical_page_entry = *reinterpret_cast<PhysicalPageEntry const*>((u8 const*)&physical_page - __builtin_offsetof(PhysicalPageEntry, allocated.physical_page));
  455. VERIFY(m_physical_page_entries);
  456. size_t physical_page_entry_index = &physical_page_entry - m_physical_page_entries;
  457. VERIFY(physical_page_entry_index < m_physical_page_entries_count);
  458. return PhysicalAddress((PhysicalPtr)physical_page_entry_index * PAGE_SIZE);
  459. }
  460. PageTableEntry* MemoryManager::pte(PageDirectory& page_directory, VirtualAddress vaddr)
  461. {
  462. VERIFY_INTERRUPTS_DISABLED();
  463. VERIFY(s_mm_lock.is_locked_by_current_processor());
  464. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  465. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  466. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  467. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  468. auto* pd = quickmap_pd(const_cast<PageDirectory&>(page_directory), page_directory_table_index);
  469. PageDirectoryEntry const& pde = pd[page_directory_index];
  470. if (!pde.is_present())
  471. return nullptr;
  472. return &quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()))[page_table_index];
  473. }
  474. PageTableEntry* MemoryManager::ensure_pte(PageDirectory& page_directory, VirtualAddress vaddr)
  475. {
  476. VERIFY_INTERRUPTS_DISABLED();
  477. VERIFY(s_mm_lock.is_locked_by_current_processor());
  478. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  479. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  480. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  481. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  482. auto* pd = quickmap_pd(page_directory, page_directory_table_index);
  483. auto& pde = pd[page_directory_index];
  484. if (pde.is_present())
  485. return &quickmap_pt(PhysicalAddress(pde.page_table_base()))[page_table_index];
  486. bool did_purge = false;
  487. auto page_table_or_error = allocate_physical_page(ShouldZeroFill::Yes, &did_purge);
  488. if (page_table_or_error.is_error()) {
  489. dbgln("MM: Unable to allocate page table to map {}", vaddr);
  490. return nullptr;
  491. }
  492. auto page_table = page_table_or_error.release_value();
  493. if (did_purge) {
  494. // If any memory had to be purged, ensure_pte may have been called as part
  495. // of the purging process. So we need to re-map the pd in this case to ensure
  496. // we're writing to the correct underlying physical page
  497. pd = quickmap_pd(page_directory, page_directory_table_index);
  498. VERIFY(&pde == &pd[page_directory_index]); // Sanity check
  499. VERIFY(!pde.is_present()); // Should have not changed
  500. }
  501. pde.set_page_table_base(page_table->paddr().get());
  502. pde.set_user_allowed(true);
  503. pde.set_present(true);
  504. pde.set_writable(true);
  505. pde.set_global(&page_directory == m_kernel_page_directory.ptr());
  506. // NOTE: This leaked ref is matched by the unref in MemoryManager::release_pte()
  507. (void)page_table.leak_ref();
  508. return &quickmap_pt(PhysicalAddress(pde.page_table_base()))[page_table_index];
  509. }
  510. void MemoryManager::release_pte(PageDirectory& page_directory, VirtualAddress vaddr, IsLastPTERelease is_last_pte_release)
  511. {
  512. VERIFY_INTERRUPTS_DISABLED();
  513. VERIFY(s_mm_lock.is_locked_by_current_processor());
  514. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  515. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  516. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  517. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  518. auto* pd = quickmap_pd(page_directory, page_directory_table_index);
  519. PageDirectoryEntry& pde = pd[page_directory_index];
  520. if (pde.is_present()) {
  521. auto* page_table = quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()));
  522. auto& pte = page_table[page_table_index];
  523. pte.clear();
  524. if (is_last_pte_release == IsLastPTERelease::Yes || page_table_index == 0x1ff) {
  525. // If this is the last PTE in a region or the last PTE in a page table then
  526. // check if we can also release the page table
  527. bool all_clear = true;
  528. for (u32 i = 0; i <= 0x1ff; i++) {
  529. if (!page_table[i].is_null()) {
  530. all_clear = false;
  531. break;
  532. }
  533. }
  534. if (all_clear) {
  535. get_physical_page_entry(PhysicalAddress { pde.page_table_base() }).allocated.physical_page.unref();
  536. pde.clear();
  537. }
  538. }
  539. }
  540. }
  541. UNMAP_AFTER_INIT void MemoryManager::initialize(u32 cpu)
  542. {
  543. ProcessorSpecific<MemoryManagerData>::initialize();
  544. if (cpu == 0) {
  545. new MemoryManager;
  546. kmalloc_enable_expand();
  547. }
  548. }
  549. Region* MemoryManager::kernel_region_from_vaddr(VirtualAddress address)
  550. {
  551. if (is_user_address(address))
  552. return nullptr;
  553. return MM.m_region_tree.find_region_containing(address);
  554. }
  555. Region* MemoryManager::find_user_region_from_vaddr_no_lock(AddressSpace& space, VirtualAddress vaddr)
  556. {
  557. VERIFY(space.get_lock().is_locked_by_current_processor());
  558. return space.find_region_containing({ vaddr, 1 });
  559. }
  560. Region* MemoryManager::find_user_region_from_vaddr(AddressSpace& space, VirtualAddress vaddr)
  561. {
  562. SpinlockLocker lock(space.get_lock());
  563. return find_user_region_from_vaddr_no_lock(space, vaddr);
  564. }
  565. void MemoryManager::validate_syscall_preconditions(AddressSpace& space, RegisterState const& regs)
  566. {
  567. // We take the space lock once here and then use the no_lock variants
  568. // to avoid excessive spinlock recursion in this extremely common path.
  569. SpinlockLocker lock(space.get_lock());
  570. auto unlock_and_handle_crash = [&lock, &regs](char const* description, int signal) {
  571. lock.unlock();
  572. handle_crash(regs, description, signal);
  573. };
  574. {
  575. VirtualAddress userspace_sp = VirtualAddress { regs.userspace_sp() };
  576. if (!MM.validate_user_stack_no_lock(space, userspace_sp)) {
  577. dbgln("Invalid stack pointer: {}", userspace_sp);
  578. return unlock_and_handle_crash("Bad stack on syscall entry", SIGSEGV);
  579. }
  580. }
  581. {
  582. VirtualAddress ip = VirtualAddress { regs.ip() };
  583. auto* calling_region = MM.find_user_region_from_vaddr_no_lock(space, ip);
  584. if (!calling_region) {
  585. dbgln("Syscall from {:p} which has no associated region", ip);
  586. return unlock_and_handle_crash("Syscall from unknown region", SIGSEGV);
  587. }
  588. if (calling_region->is_writable()) {
  589. dbgln("Syscall from writable memory at {:p}", ip);
  590. return unlock_and_handle_crash("Syscall from writable memory", SIGSEGV);
  591. }
  592. if (space.enforces_syscall_regions() && !calling_region->is_syscall_region()) {
  593. dbgln("Syscall from non-syscall region");
  594. return unlock_and_handle_crash("Syscall from non-syscall region", SIGSEGV);
  595. }
  596. }
  597. }
  598. Region* MemoryManager::find_region_from_vaddr(VirtualAddress vaddr)
  599. {
  600. if (auto* region = kernel_region_from_vaddr(vaddr))
  601. return region;
  602. auto page_directory = PageDirectory::find_current();
  603. if (!page_directory)
  604. return nullptr;
  605. VERIFY(page_directory->address_space());
  606. return find_user_region_from_vaddr(*page_directory->address_space(), vaddr);
  607. }
  608. PageFaultResponse MemoryManager::handle_page_fault(PageFault const& fault)
  609. {
  610. auto faulted_in_range = [&fault](auto const* start, auto const* end) {
  611. return fault.vaddr() >= VirtualAddress { start } && fault.vaddr() < VirtualAddress { end };
  612. };
  613. if (faulted_in_range(&start_of_ro_after_init, &end_of_ro_after_init))
  614. PANIC("Attempt to write into READONLY_AFTER_INIT section");
  615. if (faulted_in_range(&start_of_unmap_after_init, &end_of_unmap_after_init)) {
  616. auto const* kernel_symbol = symbolicate_kernel_address(fault.vaddr().get());
  617. PANIC("Attempt to access UNMAP_AFTER_INIT section ({:p}: {})", fault.vaddr(), kernel_symbol ? kernel_symbol->name : "(Unknown)");
  618. }
  619. if (faulted_in_range(&start_of_kernel_ksyms, &end_of_kernel_ksyms))
  620. PANIC("Attempt to access KSYMS section");
  621. if (Processor::current_in_irq()) {
  622. dbgln("CPU[{}] BUG! Page fault while handling IRQ! code={}, vaddr={}, irq level: {}",
  623. Processor::current_id(), fault.code(), fault.vaddr(), Processor::current_in_irq());
  624. dump_kernel_regions();
  625. return PageFaultResponse::ShouldCrash;
  626. }
  627. dbgln_if(PAGE_FAULT_DEBUG, "MM: CPU[{}] handle_page_fault({:#04x}) at {}", Processor::current_id(), fault.code(), fault.vaddr());
  628. auto* region = find_region_from_vaddr(fault.vaddr());
  629. if (!region) {
  630. return PageFaultResponse::ShouldCrash;
  631. }
  632. return region->handle_fault(fault);
  633. }
  634. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_contiguous_kernel_region(size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  635. {
  636. VERIFY(!(size % PAGE_SIZE));
  637. OwnPtr<KString> name_kstring;
  638. if (!name.is_null())
  639. name_kstring = TRY(KString::try_create(name));
  640. auto vmobject = TRY(AnonymousVMObject::try_create_physically_contiguous_with_size(size));
  641. auto region = TRY(Region::create_unplaced(move(vmobject), 0, move(name_kstring), access, cacheable));
  642. TRY(m_region_tree.place_anywhere(*region, RandomizeVirtualAddress::No, size));
  643. TRY(region->map(kernel_page_directory()));
  644. return region;
  645. }
  646. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::allocate_dma_buffer_page(StringView name, Memory::Region::Access access, LockRefPtr<Memory::PhysicalPage>& dma_buffer_page)
  647. {
  648. dma_buffer_page = TRY(allocate_physical_page());
  649. // Do not enable Cache for this region as physical memory transfers are performed (Most architectures have this behaviour by default)
  650. return allocate_kernel_region(dma_buffer_page->paddr(), PAGE_SIZE, name, access, Region::Cacheable::No);
  651. }
  652. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::allocate_dma_buffer_page(StringView name, Memory::Region::Access access)
  653. {
  654. LockRefPtr<Memory::PhysicalPage> dma_buffer_page;
  655. return allocate_dma_buffer_page(name, access, dma_buffer_page);
  656. }
  657. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::allocate_dma_buffer_pages(size_t size, StringView name, Memory::Region::Access access, NonnullLockRefPtrVector<Memory::PhysicalPage>& dma_buffer_pages)
  658. {
  659. VERIFY(!(size % PAGE_SIZE));
  660. dma_buffer_pages = TRY(allocate_contiguous_physical_pages(size));
  661. // Do not enable Cache for this region as physical memory transfers are performed (Most architectures have this behaviour by default)
  662. return allocate_kernel_region(dma_buffer_pages.first().paddr(), size, name, access, Region::Cacheable::No);
  663. }
  664. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::allocate_dma_buffer_pages(size_t size, StringView name, Memory::Region::Access access)
  665. {
  666. VERIFY(!(size % PAGE_SIZE));
  667. NonnullLockRefPtrVector<Memory::PhysicalPage> dma_buffer_pages;
  668. return allocate_dma_buffer_pages(size, name, access, dma_buffer_pages);
  669. }
  670. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region(size_t size, StringView name, Region::Access access, AllocationStrategy strategy, Region::Cacheable cacheable)
  671. {
  672. VERIFY(!(size % PAGE_SIZE));
  673. OwnPtr<KString> name_kstring;
  674. if (!name.is_null())
  675. name_kstring = TRY(KString::try_create(name));
  676. auto vmobject = TRY(AnonymousVMObject::try_create_with_size(size, strategy));
  677. auto region = TRY(Region::create_unplaced(move(vmobject), 0, move(name_kstring), access, cacheable));
  678. TRY(m_region_tree.place_anywhere(*region, RandomizeVirtualAddress::No, size));
  679. TRY(region->map(kernel_page_directory()));
  680. return region;
  681. }
  682. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region(PhysicalAddress paddr, size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  683. {
  684. VERIFY(!(size % PAGE_SIZE));
  685. auto vmobject = TRY(AnonymousVMObject::try_create_for_physical_range(paddr, size));
  686. OwnPtr<KString> name_kstring;
  687. if (!name.is_null())
  688. name_kstring = TRY(KString::try_create(name));
  689. auto region = TRY(Region::create_unplaced(move(vmobject), 0, move(name_kstring), access, cacheable));
  690. TRY(m_region_tree.place_anywhere(*region, RandomizeVirtualAddress::No, size, PAGE_SIZE));
  691. TRY(region->map(kernel_page_directory()));
  692. return region;
  693. }
  694. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region_with_vmobject(VMObject& vmobject, size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  695. {
  696. VERIFY(!(size % PAGE_SIZE));
  697. OwnPtr<KString> name_kstring;
  698. if (!name.is_null())
  699. name_kstring = TRY(KString::try_create(name));
  700. auto region = TRY(Region::create_unplaced(vmobject, 0, move(name_kstring), access, cacheable));
  701. TRY(m_region_tree.place_anywhere(*region, RandomizeVirtualAddress::No, size));
  702. TRY(region->map(kernel_page_directory()));
  703. return region;
  704. }
  705. ErrorOr<CommittedPhysicalPageSet> MemoryManager::commit_physical_pages(size_t page_count)
  706. {
  707. VERIFY(page_count > 0);
  708. SpinlockLocker lock(s_mm_lock);
  709. if (m_system_memory_info.physical_pages_uncommitted < page_count) {
  710. dbgln("MM: Unable to commit {} pages, have only {}", page_count, m_system_memory_info.physical_pages_uncommitted);
  711. Process::for_each([&](Process const& process) {
  712. dbgln("{}({}) resident:{}, shared:{}, virtual:{}",
  713. process.name(),
  714. process.pid(),
  715. process.address_space().amount_resident() / PAGE_SIZE,
  716. process.address_space().amount_shared() / PAGE_SIZE,
  717. process.address_space().amount_virtual() / PAGE_SIZE);
  718. return IterationDecision::Continue;
  719. });
  720. return ENOMEM;
  721. }
  722. m_system_memory_info.physical_pages_uncommitted -= page_count;
  723. m_system_memory_info.physical_pages_committed += page_count;
  724. return CommittedPhysicalPageSet { {}, page_count };
  725. }
  726. void MemoryManager::uncommit_physical_pages(Badge<CommittedPhysicalPageSet>, size_t page_count)
  727. {
  728. VERIFY(page_count > 0);
  729. SpinlockLocker lock(s_mm_lock);
  730. VERIFY(m_system_memory_info.physical_pages_committed >= page_count);
  731. m_system_memory_info.physical_pages_uncommitted += page_count;
  732. m_system_memory_info.physical_pages_committed -= page_count;
  733. }
  734. void MemoryManager::deallocate_physical_page(PhysicalAddress paddr)
  735. {
  736. SpinlockLocker lock(s_mm_lock);
  737. // Are we returning a user page?
  738. for (auto& region : m_physical_regions) {
  739. if (!region.contains(paddr))
  740. continue;
  741. region.return_page(paddr);
  742. --m_system_memory_info.physical_pages_used;
  743. // Always return pages to the uncommitted pool. Pages that were
  744. // committed and allocated are only freed upon request. Once
  745. // returned there is no guarantee being able to get them back.
  746. ++m_system_memory_info.physical_pages_uncommitted;
  747. return;
  748. }
  749. PANIC("MM: deallocate_physical_page couldn't figure out region for page @ {}", paddr);
  750. }
  751. LockRefPtr<PhysicalPage> MemoryManager::find_free_physical_page(bool committed)
  752. {
  753. VERIFY(s_mm_lock.is_locked());
  754. LockRefPtr<PhysicalPage> page;
  755. if (committed) {
  756. // Draw from the committed pages pool. We should always have these pages available
  757. VERIFY(m_system_memory_info.physical_pages_committed > 0);
  758. m_system_memory_info.physical_pages_committed--;
  759. } else {
  760. // We need to make sure we don't touch pages that we have committed to
  761. if (m_system_memory_info.physical_pages_uncommitted == 0)
  762. return {};
  763. m_system_memory_info.physical_pages_uncommitted--;
  764. }
  765. for (auto& region : m_physical_regions) {
  766. page = region.take_free_page();
  767. if (!page.is_null()) {
  768. ++m_system_memory_info.physical_pages_used;
  769. break;
  770. }
  771. }
  772. VERIFY(!page.is_null());
  773. return page;
  774. }
  775. NonnullLockRefPtr<PhysicalPage> MemoryManager::allocate_committed_physical_page(Badge<CommittedPhysicalPageSet>, ShouldZeroFill should_zero_fill)
  776. {
  777. SpinlockLocker lock(s_mm_lock);
  778. auto page = find_free_physical_page(true);
  779. if (should_zero_fill == ShouldZeroFill::Yes) {
  780. auto* ptr = quickmap_page(*page);
  781. memset(ptr, 0, PAGE_SIZE);
  782. unquickmap_page();
  783. }
  784. return page.release_nonnull();
  785. }
  786. ErrorOr<NonnullLockRefPtr<PhysicalPage>> MemoryManager::allocate_physical_page(ShouldZeroFill should_zero_fill, bool* did_purge)
  787. {
  788. SpinlockLocker lock(s_mm_lock);
  789. auto page = find_free_physical_page(false);
  790. bool purged_pages = false;
  791. if (!page) {
  792. // We didn't have a single free physical page. Let's try to free something up!
  793. // First, we look for a purgeable VMObject in the volatile state.
  794. for_each_vmobject([&](auto& vmobject) {
  795. if (!vmobject.is_anonymous())
  796. return IterationDecision::Continue;
  797. auto& anonymous_vmobject = static_cast<AnonymousVMObject&>(vmobject);
  798. if (!anonymous_vmobject.is_purgeable() || !anonymous_vmobject.is_volatile())
  799. return IterationDecision::Continue;
  800. if (auto purged_page_count = anonymous_vmobject.purge()) {
  801. dbgln("MM: Purge saved the day! Purged {} pages from AnonymousVMObject", purged_page_count);
  802. page = find_free_physical_page(false);
  803. purged_pages = true;
  804. VERIFY(page);
  805. return IterationDecision::Break;
  806. }
  807. return IterationDecision::Continue;
  808. });
  809. }
  810. if (!page) {
  811. // Second, we look for a file-backed VMObject with clean pages.
  812. for_each_vmobject([&](auto& vmobject) {
  813. if (!vmobject.is_inode())
  814. return IterationDecision::Continue;
  815. auto& inode_vmobject = static_cast<InodeVMObject&>(vmobject);
  816. if (auto released_page_count = inode_vmobject.try_release_clean_pages(1)) {
  817. dbgln("MM: Clean inode release saved the day! Released {} pages from InodeVMObject", released_page_count);
  818. page = find_free_physical_page(false);
  819. VERIFY(page);
  820. return IterationDecision::Break;
  821. }
  822. return IterationDecision::Continue;
  823. });
  824. }
  825. if (!page) {
  826. dmesgln("MM: no physical pages available");
  827. return ENOMEM;
  828. }
  829. if (should_zero_fill == ShouldZeroFill::Yes) {
  830. auto* ptr = quickmap_page(*page);
  831. memset(ptr, 0, PAGE_SIZE);
  832. unquickmap_page();
  833. }
  834. if (did_purge)
  835. *did_purge = purged_pages;
  836. return page.release_nonnull();
  837. }
  838. ErrorOr<NonnullLockRefPtrVector<PhysicalPage>> MemoryManager::allocate_contiguous_physical_pages(size_t size)
  839. {
  840. VERIFY(!(size % PAGE_SIZE));
  841. SpinlockLocker mm_lock(s_mm_lock);
  842. size_t page_count = ceil_div(size, static_cast<size_t>(PAGE_SIZE));
  843. // We need to make sure we don't touch pages that we have committed to
  844. if (m_system_memory_info.physical_pages_uncommitted < page_count)
  845. return ENOMEM;
  846. for (auto& physical_region : m_physical_regions) {
  847. auto physical_pages = physical_region.take_contiguous_free_pages(page_count);
  848. if (!physical_pages.is_empty()) {
  849. // Note: The locking semantics are a bit awkward here, the region destructor can
  850. // deadlock due to invalid lock ordering if we still hold the s_mm_lock when the
  851. // region is constructed or destructed. Since we are now done enumerating anyway,
  852. // it's safe to just unlock here and have MM.allocate_kernel_region do the right thing.
  853. mm_lock.unlock();
  854. {
  855. auto cleanup_region = TRY(MM.allocate_kernel_region(physical_pages[0].paddr(), PAGE_SIZE * page_count, "MemoryManager Allocation Sanitization"sv, Region::Access::Read | Region::Access::Write));
  856. memset(cleanup_region->vaddr().as_ptr(), 0, PAGE_SIZE * page_count);
  857. }
  858. m_system_memory_info.physical_pages_uncommitted -= page_count;
  859. m_system_memory_info.physical_pages_used += page_count;
  860. return physical_pages;
  861. }
  862. }
  863. dmesgln("MM: no contiguous physical pages available");
  864. return ENOMEM;
  865. }
  866. void MemoryManager::enter_process_address_space(Process& process)
  867. {
  868. enter_address_space(process.address_space());
  869. }
  870. void MemoryManager::enter_address_space(AddressSpace& space)
  871. {
  872. auto* current_thread = Thread::current();
  873. VERIFY(current_thread != nullptr);
  874. SpinlockLocker lock(s_mm_lock);
  875. current_thread->regs().cr3 = space.page_directory().cr3();
  876. activate_page_directory(space.page_directory(), current_thread);
  877. }
  878. void MemoryManager::flush_tlb_local(VirtualAddress vaddr, size_t page_count)
  879. {
  880. Processor::flush_tlb_local(vaddr, page_count);
  881. }
  882. void MemoryManager::flush_tlb(PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count)
  883. {
  884. Processor::flush_tlb(page_directory, vaddr, page_count);
  885. }
  886. PageDirectoryEntry* MemoryManager::quickmap_pd(PageDirectory& directory, size_t pdpt_index)
  887. {
  888. VERIFY_INTERRUPTS_DISABLED();
  889. VERIFY(s_mm_lock.is_locked_by_current_processor());
  890. VirtualAddress vaddr(KERNEL_QUICKMAP_PD_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  891. size_t pte_index = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  892. auto& pte = boot_pd_kernel_pt1023[pte_index];
  893. auto pd_paddr = directory.m_directory_pages[pdpt_index]->paddr();
  894. if (pte.physical_page_base() != pd_paddr.get()) {
  895. pte.set_physical_page_base(pd_paddr.get());
  896. pte.set_present(true);
  897. pte.set_writable(true);
  898. pte.set_user_allowed(false);
  899. flush_tlb_local(vaddr);
  900. }
  901. return (PageDirectoryEntry*)vaddr.get();
  902. }
  903. PageTableEntry* MemoryManager::quickmap_pt(PhysicalAddress pt_paddr)
  904. {
  905. VERIFY_INTERRUPTS_DISABLED();
  906. VERIFY(s_mm_lock.is_locked_by_current_processor());
  907. VirtualAddress vaddr(KERNEL_QUICKMAP_PT_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  908. size_t pte_index = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  909. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[pte_index];
  910. if (pte.physical_page_base() != pt_paddr.get()) {
  911. pte.set_physical_page_base(pt_paddr.get());
  912. pte.set_present(true);
  913. pte.set_writable(true);
  914. pte.set_user_allowed(false);
  915. flush_tlb_local(vaddr);
  916. }
  917. return (PageTableEntry*)vaddr.get();
  918. }
  919. u8* MemoryManager::quickmap_page(PhysicalAddress const& physical_address)
  920. {
  921. VERIFY_INTERRUPTS_DISABLED();
  922. VERIFY(s_mm_lock.is_locked_by_current_processor());
  923. auto& mm_data = get_data();
  924. mm_data.m_quickmap_prev_flags = mm_data.m_quickmap_in_use.lock();
  925. VirtualAddress vaddr(KERNEL_QUICKMAP_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  926. u32 pte_idx = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  927. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[pte_idx];
  928. if (pte.physical_page_base() != physical_address.get()) {
  929. pte.set_physical_page_base(physical_address.get());
  930. pte.set_present(true);
  931. pte.set_writable(true);
  932. pte.set_user_allowed(false);
  933. flush_tlb_local(vaddr);
  934. }
  935. return vaddr.as_ptr();
  936. }
  937. void MemoryManager::unquickmap_page()
  938. {
  939. VERIFY_INTERRUPTS_DISABLED();
  940. VERIFY(s_mm_lock.is_locked_by_current_processor());
  941. auto& mm_data = get_data();
  942. VERIFY(mm_data.m_quickmap_in_use.is_locked());
  943. VirtualAddress vaddr(KERNEL_QUICKMAP_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  944. u32 pte_idx = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  945. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[pte_idx];
  946. pte.clear();
  947. flush_tlb_local(vaddr);
  948. mm_data.m_quickmap_in_use.unlock(mm_data.m_quickmap_prev_flags);
  949. }
  950. bool MemoryManager::validate_user_stack_no_lock(AddressSpace& space, VirtualAddress vaddr) const
  951. {
  952. VERIFY(space.get_lock().is_locked_by_current_processor());
  953. if (!is_user_address(vaddr))
  954. return false;
  955. auto* region = find_user_region_from_vaddr_no_lock(space, vaddr);
  956. return region && region->is_user() && region->is_stack();
  957. }
  958. bool MemoryManager::validate_user_stack(AddressSpace& space, VirtualAddress vaddr) const
  959. {
  960. SpinlockLocker lock(space.get_lock());
  961. return validate_user_stack_no_lock(space, vaddr);
  962. }
  963. void MemoryManager::unregister_kernel_region(Region& region)
  964. {
  965. VERIFY(region.is_kernel());
  966. m_region_tree.remove(region);
  967. }
  968. void MemoryManager::dump_kernel_regions()
  969. {
  970. dbgln("Kernel regions:");
  971. #if ARCH(I386)
  972. char const* addr_padding = "";
  973. #else
  974. char const* addr_padding = " ";
  975. #endif
  976. dbgln("BEGIN{} END{} SIZE{} ACCESS NAME",
  977. addr_padding, addr_padding, addr_padding);
  978. SpinlockLocker lock(s_mm_lock);
  979. SpinlockLocker tree_locker(m_region_tree.get_lock());
  980. for (auto const& region : m_region_tree.regions()) {
  981. dbgln("{:p} -- {:p} {:p} {:c}{:c}{:c}{:c}{:c}{:c} {}",
  982. region.vaddr().get(),
  983. region.vaddr().offset(region.size() - 1).get(),
  984. region.size(),
  985. region.is_readable() ? 'R' : ' ',
  986. region.is_writable() ? 'W' : ' ',
  987. region.is_executable() ? 'X' : ' ',
  988. region.is_shared() ? 'S' : ' ',
  989. region.is_stack() ? 'T' : ' ',
  990. region.is_syscall_region() ? 'C' : ' ',
  991. region.name());
  992. }
  993. }
  994. void MemoryManager::set_page_writable_direct(VirtualAddress vaddr, bool writable)
  995. {
  996. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  997. SpinlockLocker lock(s_mm_lock);
  998. auto* pte = ensure_pte(kernel_page_directory(), vaddr);
  999. VERIFY(pte);
  1000. if (pte->is_writable() == writable)
  1001. return;
  1002. pte->set_writable(writable);
  1003. flush_tlb(&kernel_page_directory(), vaddr);
  1004. }
  1005. CommittedPhysicalPageSet::~CommittedPhysicalPageSet()
  1006. {
  1007. if (m_page_count)
  1008. MM.uncommit_physical_pages({}, m_page_count);
  1009. }
  1010. NonnullLockRefPtr<PhysicalPage> CommittedPhysicalPageSet::take_one()
  1011. {
  1012. VERIFY(m_page_count > 0);
  1013. --m_page_count;
  1014. return MM.allocate_committed_physical_page({}, MemoryManager::ShouldZeroFill::Yes);
  1015. }
  1016. void CommittedPhysicalPageSet::uncommit_one()
  1017. {
  1018. VERIFY(m_page_count > 0);
  1019. --m_page_count;
  1020. MM.uncommit_physical_pages({}, 1);
  1021. }
  1022. void MemoryManager::copy_physical_page(PhysicalPage& physical_page, u8 page_buffer[PAGE_SIZE])
  1023. {
  1024. SpinlockLocker locker(s_mm_lock);
  1025. auto* quickmapped_page = quickmap_page(physical_page);
  1026. memcpy(page_buffer, quickmapped_page, PAGE_SIZE);
  1027. unquickmap_page();
  1028. }
  1029. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::create_identity_mapped_region(PhysicalAddress address, size_t size)
  1030. {
  1031. auto vmobject = TRY(Memory::AnonymousVMObject::try_create_for_physical_range(address, size));
  1032. auto region = TRY(Memory::Region::create_unplaced(move(vmobject), 0, {}, Memory::Region::Access::ReadWriteExecute));
  1033. Memory::VirtualRange range { VirtualAddress { (FlatPtr)address.get() }, size };
  1034. region->m_range = range;
  1035. TRY(region->map(MM.kernel_page_directory()));
  1036. return region;
  1037. }
  1038. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_unbacked_region_anywhere(size_t size, size_t alignment)
  1039. {
  1040. auto region = TRY(Region::create_unbacked());
  1041. TRY(m_region_tree.place_anywhere(*region, RandomizeVirtualAddress::No, size, alignment));
  1042. return region;
  1043. }
  1044. }