MemoryManager.cpp 53 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238
  1. /*
  2. * Copyright (c) 2018-2022, Andreas Kling <kling@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <AK/Assertions.h>
  7. #include <AK/Memory.h>
  8. #include <AK/NonnullRefPtrVector.h>
  9. #include <AK/StringView.h>
  10. #include <Kernel/Arch/CPU.h>
  11. #include <Kernel/Arch/PageDirectory.h>
  12. #include <Kernel/Arch/PageFault.h>
  13. #include <Kernel/Arch/RegisterState.h>
  14. #include <Kernel/BootInfo.h>
  15. #include <Kernel/FileSystem/Inode.h>
  16. #include <Kernel/Heap/kmalloc.h>
  17. #include <Kernel/InterruptDisabler.h>
  18. #include <Kernel/KSyms.h>
  19. #include <Kernel/Memory/AnonymousVMObject.h>
  20. #include <Kernel/Memory/MemoryManager.h>
  21. #include <Kernel/Memory/PageDirectory.h>
  22. #include <Kernel/Memory/PhysicalRegion.h>
  23. #include <Kernel/Memory/SharedInodeVMObject.h>
  24. #include <Kernel/Multiboot.h>
  25. #include <Kernel/Panic.h>
  26. #include <Kernel/Prekernel/Prekernel.h>
  27. #include <Kernel/Process.h>
  28. #include <Kernel/Sections.h>
  29. #include <Kernel/StdLib.h>
  30. extern u8 start_of_kernel_image[];
  31. extern u8 end_of_kernel_image[];
  32. extern u8 start_of_kernel_text[];
  33. extern u8 start_of_kernel_data[];
  34. extern u8 end_of_kernel_bss[];
  35. extern u8 start_of_ro_after_init[];
  36. extern u8 end_of_ro_after_init[];
  37. extern u8 start_of_unmap_after_init[];
  38. extern u8 end_of_unmap_after_init[];
  39. extern u8 start_of_kernel_ksyms[];
  40. extern u8 end_of_kernel_ksyms[];
  41. extern multiboot_module_entry_t multiboot_copy_boot_modules_array[16];
  42. extern size_t multiboot_copy_boot_modules_count;
  43. namespace Kernel::Memory {
  44. ErrorOr<FlatPtr> page_round_up(FlatPtr x)
  45. {
  46. if (x > (explode_byte(0xFF) & ~0xFFF)) {
  47. return Error::from_errno(EINVAL);
  48. }
  49. return (((FlatPtr)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1));
  50. }
  51. // NOTE: We can NOT use Singleton for this class, because
  52. // MemoryManager::initialize is called *before* global constructors are
  53. // run. If we do, then Singleton would get re-initialized, causing
  54. // the memory manager to be initialized twice!
  55. static MemoryManager* s_the;
  56. MemoryManager& MemoryManager::the()
  57. {
  58. return *s_the;
  59. }
  60. bool MemoryManager::is_initialized()
  61. {
  62. return s_the != nullptr;
  63. }
  64. static UNMAP_AFTER_INIT VirtualRange kernel_virtual_range()
  65. {
  66. #if ARCH(AARCH64)
  67. // NOTE: We currently identity map the kernel image for aarch64, so the kernel virtual range
  68. // is the complete memory range.
  69. return VirtualRange { VirtualAddress((FlatPtr)0), 0x3F000000 };
  70. #else
  71. size_t kernel_range_start = kernel_mapping_base + 2 * MiB; // The first 2 MiB are used for mapping the pre-kernel
  72. return VirtualRange { VirtualAddress(kernel_range_start), KERNEL_PD_END - kernel_range_start };
  73. #endif
  74. }
  75. MemoryManager::GlobalData::GlobalData()
  76. : region_tree(kernel_virtual_range())
  77. {
  78. }
  79. UNMAP_AFTER_INIT MemoryManager::MemoryManager()
  80. : m_global_data(LockRank::None)
  81. {
  82. s_the = this;
  83. parse_memory_map();
  84. activate_kernel_page_directory(kernel_page_directory());
  85. protect_kernel_image();
  86. // We're temporarily "committing" to two pages that we need to allocate below
  87. auto committed_pages = commit_physical_pages(2).release_value();
  88. m_shared_zero_page = committed_pages.take_one();
  89. // We're wasting a page here, we just need a special tag (physical
  90. // address) so that we know when we need to lazily allocate a page
  91. // that we should be drawing this page from the committed pool rather
  92. // than potentially failing if no pages are available anymore.
  93. // By using a tag we don't have to query the VMObject for every page
  94. // whether it was committed or not
  95. m_lazy_committed_page = committed_pages.take_one();
  96. }
  97. UNMAP_AFTER_INIT MemoryManager::~MemoryManager() = default;
  98. UNMAP_AFTER_INIT void MemoryManager::protect_kernel_image()
  99. {
  100. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  101. // Disable writing to the kernel text and rodata segments.
  102. for (auto const* i = start_of_kernel_text; i < start_of_kernel_data; i += PAGE_SIZE) {
  103. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  104. pte.set_writable(false);
  105. }
  106. if (Processor::current().has_nx()) {
  107. // Disable execution of the kernel data, bss and heap segments.
  108. for (auto const* i = start_of_kernel_data; i < end_of_kernel_image; i += PAGE_SIZE) {
  109. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  110. pte.set_execute_disabled(true);
  111. }
  112. }
  113. }
  114. UNMAP_AFTER_INIT void MemoryManager::unmap_prekernel()
  115. {
  116. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  117. auto start = start_of_prekernel_image.page_base().get();
  118. auto end = end_of_prekernel_image.page_base().get();
  119. for (auto i = start; i <= end; i += PAGE_SIZE)
  120. release_pte(kernel_page_directory(), VirtualAddress(i), i == end ? IsLastPTERelease::Yes : IsLastPTERelease::No);
  121. flush_tlb(&kernel_page_directory(), VirtualAddress(start), (end - start) / PAGE_SIZE);
  122. }
  123. UNMAP_AFTER_INIT void MemoryManager::protect_readonly_after_init_memory()
  124. {
  125. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  126. // Disable writing to the .ro_after_init section
  127. for (auto i = (FlatPtr)&start_of_ro_after_init; i < (FlatPtr)&end_of_ro_after_init; i += PAGE_SIZE) {
  128. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  129. pte.set_writable(false);
  130. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  131. }
  132. }
  133. void MemoryManager::unmap_text_after_init()
  134. {
  135. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  136. auto start = page_round_down((FlatPtr)&start_of_unmap_after_init);
  137. auto end = page_round_up((FlatPtr)&end_of_unmap_after_init).release_value_but_fixme_should_propagate_errors();
  138. // Unmap the entire .unmap_after_init section
  139. for (auto i = start; i < end; i += PAGE_SIZE) {
  140. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  141. pte.clear();
  142. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  143. }
  144. dmesgln("Unmapped {} KiB of kernel text after init! :^)", (end - start) / KiB);
  145. }
  146. UNMAP_AFTER_INIT void MemoryManager::protect_ksyms_after_init()
  147. {
  148. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  149. auto start = page_round_down((FlatPtr)start_of_kernel_ksyms);
  150. auto end = page_round_up((FlatPtr)end_of_kernel_ksyms).release_value_but_fixme_should_propagate_errors();
  151. for (auto i = start; i < end; i += PAGE_SIZE) {
  152. auto& pte = *ensure_pte(kernel_page_directory(), VirtualAddress(i));
  153. pte.set_writable(false);
  154. flush_tlb(&kernel_page_directory(), VirtualAddress(i));
  155. }
  156. dmesgln("Write-protected kernel symbols after init.");
  157. }
  158. IterationDecision MemoryManager::for_each_physical_memory_range(Function<IterationDecision(PhysicalMemoryRange const&)> callback)
  159. {
  160. return m_global_data.with([&](auto& global_data) {
  161. VERIFY(!global_data.physical_memory_ranges.is_empty());
  162. for (auto& current_range : global_data.physical_memory_ranges) {
  163. IterationDecision decision = callback(current_range);
  164. if (decision != IterationDecision::Continue)
  165. return decision;
  166. }
  167. return IterationDecision::Continue;
  168. });
  169. }
  170. UNMAP_AFTER_INIT void MemoryManager::register_reserved_ranges()
  171. {
  172. m_global_data.with([&](auto& global_data) {
  173. VERIFY(!global_data.physical_memory_ranges.is_empty());
  174. ContiguousReservedMemoryRange range;
  175. for (auto& current_range : global_data.physical_memory_ranges) {
  176. if (current_range.type != PhysicalMemoryRangeType::Reserved) {
  177. if (range.start.is_null())
  178. continue;
  179. global_data.reserved_memory_ranges.append(ContiguousReservedMemoryRange { range.start, current_range.start.get() - range.start.get() });
  180. range.start.set((FlatPtr) nullptr);
  181. continue;
  182. }
  183. if (!range.start.is_null()) {
  184. continue;
  185. }
  186. range.start = current_range.start;
  187. }
  188. if (global_data.physical_memory_ranges.last().type != PhysicalMemoryRangeType::Reserved)
  189. return;
  190. if (range.start.is_null())
  191. return;
  192. global_data.reserved_memory_ranges.append(ContiguousReservedMemoryRange { range.start, global_data.physical_memory_ranges.last().start.get() + global_data.physical_memory_ranges.last().length - range.start.get() });
  193. });
  194. }
  195. bool MemoryManager::is_allowed_to_read_physical_memory_for_userspace(PhysicalAddress start_address, size_t read_length) const
  196. {
  197. // Note: Guard against overflow in case someone tries to mmap on the edge of
  198. // the RAM
  199. if (start_address.offset_addition_would_overflow(read_length))
  200. return false;
  201. auto end_address = start_address.offset(read_length);
  202. return m_global_data.with([&](auto& global_data) {
  203. for (auto const& current_range : global_data.reserved_memory_ranges) {
  204. if (current_range.start > start_address)
  205. continue;
  206. if (current_range.start.offset(current_range.length) < end_address)
  207. continue;
  208. return true;
  209. }
  210. return false;
  211. });
  212. }
  213. UNMAP_AFTER_INIT void MemoryManager::parse_memory_map()
  214. {
  215. // Register used memory regions that we know of.
  216. m_global_data.with([&](auto& global_data) {
  217. global_data.used_memory_ranges.ensure_capacity(4);
  218. #if ARCH(X86_64)
  219. global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::LowMemory, PhysicalAddress(0x00000000), PhysicalAddress(1 * MiB) });
  220. #endif
  221. global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::Kernel, PhysicalAddress(virtual_to_low_physical((FlatPtr)start_of_kernel_image)), PhysicalAddress(page_round_up(virtual_to_low_physical((FlatPtr)end_of_kernel_image)).release_value_but_fixme_should_propagate_errors()) });
  222. if (multiboot_flags & 0x4) {
  223. auto* bootmods_start = multiboot_copy_boot_modules_array;
  224. auto* bootmods_end = bootmods_start + multiboot_copy_boot_modules_count;
  225. for (auto* bootmod = bootmods_start; bootmod < bootmods_end; bootmod++) {
  226. global_data.used_memory_ranges.append(UsedMemoryRange { UsedMemoryRangeType::BootModule, PhysicalAddress(bootmod->start), PhysicalAddress(bootmod->end) });
  227. }
  228. }
  229. auto* mmap_begin = multiboot_memory_map;
  230. auto* mmap_end = multiboot_memory_map + multiboot_memory_map_count;
  231. struct ContiguousPhysicalVirtualRange {
  232. PhysicalAddress lower;
  233. PhysicalAddress upper;
  234. };
  235. Vector<ContiguousPhysicalVirtualRange> contiguous_physical_ranges;
  236. for (auto* mmap = mmap_begin; mmap < mmap_end; mmap++) {
  237. // We have to copy these onto the stack, because we take a reference to these when printing them out,
  238. // and doing so on a packed struct field is UB.
  239. auto address = mmap->addr;
  240. auto length = mmap->len;
  241. ArmedScopeGuard write_back_guard = [&]() {
  242. mmap->addr = address;
  243. mmap->len = length;
  244. };
  245. dmesgln("MM: Multiboot mmap: address={:p}, length={}, type={}", address, length, mmap->type);
  246. auto start_address = PhysicalAddress(address);
  247. switch (mmap->type) {
  248. case (MULTIBOOT_MEMORY_AVAILABLE):
  249. global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Usable, start_address, length });
  250. break;
  251. case (MULTIBOOT_MEMORY_RESERVED):
  252. #if ARCH(X86_64)
  253. // Workaround for https://gitlab.com/qemu-project/qemu/-/commit/8504f129450b909c88e199ca44facd35d38ba4de
  254. // That commit added a reserved 12GiB entry for the benefit of virtual firmware.
  255. // We can safely ignore this block as it isn't actually reserved on any real hardware.
  256. // From: https://lore.kernel.org/all/20220701161014.3850-1-joao.m.martins@oracle.com/
  257. // "Always add the HyperTransport range into e820 even when the relocation isn't
  258. // done *and* there's >= 40 phys bit that would put max phyusical boundary to 1T
  259. // This should allow virtual firmware to avoid the reserved range at the
  260. // 1T boundary on VFs with big bars."
  261. if (address != 0x000000fd00000000 || length != (0x000000ffffffffff - 0x000000fd00000000) + 1)
  262. #endif
  263. global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Reserved, start_address, length });
  264. break;
  265. case (MULTIBOOT_MEMORY_ACPI_RECLAIMABLE):
  266. global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_Reclaimable, start_address, length });
  267. break;
  268. case (MULTIBOOT_MEMORY_NVS):
  269. global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::ACPI_NVS, start_address, length });
  270. break;
  271. case (MULTIBOOT_MEMORY_BADRAM):
  272. dmesgln("MM: Warning, detected bad memory range!");
  273. global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::BadMemory, start_address, length });
  274. break;
  275. default:
  276. dbgln("MM: Unknown range!");
  277. global_data.physical_memory_ranges.append(PhysicalMemoryRange { PhysicalMemoryRangeType::Unknown, start_address, length });
  278. break;
  279. }
  280. if (mmap->type != MULTIBOOT_MEMORY_AVAILABLE)
  281. continue;
  282. // Fix up unaligned memory regions.
  283. auto diff = (FlatPtr)address % PAGE_SIZE;
  284. if (diff != 0) {
  285. dmesgln("MM: Got an unaligned physical_region from the bootloader; correcting {:p} by {} bytes", address, diff);
  286. diff = PAGE_SIZE - diff;
  287. address += diff;
  288. length -= diff;
  289. }
  290. if ((length % PAGE_SIZE) != 0) {
  291. dmesgln("MM: Got an unaligned physical_region from the bootloader; correcting length {} by {} bytes", length, length % PAGE_SIZE);
  292. length -= length % PAGE_SIZE;
  293. }
  294. if (length < PAGE_SIZE) {
  295. dmesgln("MM: Memory physical_region from bootloader is too small; we want >= {} bytes, but got {} bytes", PAGE_SIZE, length);
  296. continue;
  297. }
  298. for (PhysicalSize page_base = address; page_base <= (address + length); page_base += PAGE_SIZE) {
  299. auto addr = PhysicalAddress(page_base);
  300. // Skip used memory ranges.
  301. bool should_skip = false;
  302. for (auto& used_range : global_data.used_memory_ranges) {
  303. if (addr.get() >= used_range.start.get() && addr.get() <= used_range.end.get()) {
  304. should_skip = true;
  305. break;
  306. }
  307. }
  308. if (should_skip)
  309. continue;
  310. if (contiguous_physical_ranges.is_empty() || contiguous_physical_ranges.last().upper.offset(PAGE_SIZE) != addr) {
  311. contiguous_physical_ranges.append(ContiguousPhysicalVirtualRange {
  312. .lower = addr,
  313. .upper = addr,
  314. });
  315. } else {
  316. contiguous_physical_ranges.last().upper = addr;
  317. }
  318. }
  319. }
  320. for (auto& range : contiguous_physical_ranges) {
  321. global_data.physical_regions.append(PhysicalRegion::try_create(range.lower, range.upper).release_nonnull());
  322. }
  323. for (auto& region : global_data.physical_regions)
  324. global_data.system_memory_info.physical_pages += region.size();
  325. register_reserved_ranges();
  326. for (auto& range : global_data.reserved_memory_ranges) {
  327. dmesgln("MM: Contiguous reserved range from {}, length is {}", range.start, range.length);
  328. }
  329. initialize_physical_pages();
  330. VERIFY(global_data.system_memory_info.physical_pages > 0);
  331. // We start out with no committed pages
  332. global_data.system_memory_info.physical_pages_uncommitted = global_data.system_memory_info.physical_pages;
  333. for (auto& used_range : global_data.used_memory_ranges) {
  334. dmesgln("MM: {} range @ {} - {} (size {:#x})", UserMemoryRangeTypeNames[to_underlying(used_range.type)], used_range.start, used_range.end.offset(-1), used_range.end.as_ptr() - used_range.start.as_ptr());
  335. }
  336. for (auto& region : global_data.physical_regions) {
  337. dmesgln("MM: User physical region: {} - {} (size {:#x})", region.lower(), region.upper().offset(-1), PAGE_SIZE * region.size());
  338. region.initialize_zones();
  339. }
  340. });
  341. }
  342. UNMAP_AFTER_INIT void MemoryManager::initialize_physical_pages()
  343. {
  344. m_global_data.with([&](auto& global_data) {
  345. // We assume that the physical page range is contiguous and doesn't contain huge gaps!
  346. PhysicalAddress highest_physical_address;
  347. for (auto& range : global_data.used_memory_ranges) {
  348. if (range.end.get() > highest_physical_address.get())
  349. highest_physical_address = range.end;
  350. }
  351. for (auto& region : global_data.physical_memory_ranges) {
  352. auto range_end = PhysicalAddress(region.start).offset(region.length);
  353. if (range_end.get() > highest_physical_address.get())
  354. highest_physical_address = range_end;
  355. }
  356. // Calculate how many total physical pages the array will have
  357. m_physical_page_entries_count = PhysicalAddress::physical_page_index(highest_physical_address.get()) + 1;
  358. VERIFY(m_physical_page_entries_count != 0);
  359. VERIFY(!Checked<decltype(m_physical_page_entries_count)>::multiplication_would_overflow(m_physical_page_entries_count, sizeof(PhysicalPageEntry)));
  360. // Calculate how many bytes the array will consume
  361. auto physical_page_array_size = m_physical_page_entries_count * sizeof(PhysicalPageEntry);
  362. auto physical_page_array_pages = page_round_up(physical_page_array_size).release_value_but_fixme_should_propagate_errors() / PAGE_SIZE;
  363. VERIFY(physical_page_array_pages * PAGE_SIZE >= physical_page_array_size);
  364. // Calculate how many page tables we will need to be able to map them all
  365. auto needed_page_table_count = (physical_page_array_pages + 512 - 1) / 512;
  366. auto physical_page_array_pages_and_page_tables_count = physical_page_array_pages + needed_page_table_count;
  367. // Now that we know how much memory we need for a contiguous array of PhysicalPage instances, find a memory region that can fit it
  368. PhysicalRegion* found_region { nullptr };
  369. Optional<size_t> found_region_index;
  370. for (size_t i = 0; i < global_data.physical_regions.size(); ++i) {
  371. auto& region = global_data.physical_regions[i];
  372. if (region.size() >= physical_page_array_pages_and_page_tables_count) {
  373. found_region = &region;
  374. found_region_index = i;
  375. break;
  376. }
  377. }
  378. if (!found_region) {
  379. dmesgln("MM: Need {} bytes for physical page management, but no memory region is large enough!", physical_page_array_pages_and_page_tables_count);
  380. VERIFY_NOT_REACHED();
  381. }
  382. VERIFY(global_data.system_memory_info.physical_pages >= physical_page_array_pages_and_page_tables_count);
  383. global_data.system_memory_info.physical_pages -= physical_page_array_pages_and_page_tables_count;
  384. if (found_region->size() == physical_page_array_pages_and_page_tables_count) {
  385. // We're stealing the entire region
  386. global_data.physical_pages_region = global_data.physical_regions.take(*found_region_index);
  387. } else {
  388. global_data.physical_pages_region = found_region->try_take_pages_from_beginning(physical_page_array_pages_and_page_tables_count);
  389. }
  390. global_data.used_memory_ranges.append({ UsedMemoryRangeType::PhysicalPages, global_data.physical_pages_region->lower(), global_data.physical_pages_region->upper() });
  391. // Create the bare page directory. This is not a fully constructed page directory and merely contains the allocators!
  392. m_kernel_page_directory = PageDirectory::must_create_kernel_page_directory();
  393. {
  394. // Carve out the whole page directory covering the kernel image to make MemoryManager::initialize_physical_pages() happy
  395. FlatPtr start_of_range = ((FlatPtr)start_of_kernel_image & ~(FlatPtr)0x1fffff);
  396. FlatPtr end_of_range = ((FlatPtr)end_of_kernel_image & ~(FlatPtr)0x1fffff) + 0x200000;
  397. MUST(global_data.region_tree.place_specifically(*MUST(Region::create_unbacked()).leak_ptr(), VirtualRange { VirtualAddress(start_of_range), end_of_range - start_of_range }));
  398. }
  399. // Allocate a virtual address range for our array
  400. // This looks awkward, but it basically creates a dummy region to occupy the address range permanently.
  401. auto& region = *MUST(Region::create_unbacked()).leak_ptr();
  402. MUST(global_data.region_tree.place_anywhere(region, RandomizeVirtualAddress::No, physical_page_array_pages * PAGE_SIZE));
  403. auto range = region.range();
  404. // Now that we have our special m_physical_pages_region region with enough pages to hold the entire array
  405. // try to map the entire region into kernel space so we always have it
  406. // We can't use ensure_pte here because it would try to allocate a PhysicalPage and we don't have the array
  407. // mapped yet so we can't create them
  408. // Create page tables at the beginning of m_physical_pages_region, followed by the PhysicalPageEntry array
  409. auto page_tables_base = global_data.physical_pages_region->lower();
  410. auto physical_page_array_base = page_tables_base.offset(needed_page_table_count * PAGE_SIZE);
  411. auto physical_page_array_current_page = physical_page_array_base.get();
  412. auto virtual_page_array_base = range.base().get();
  413. auto virtual_page_array_current_page = virtual_page_array_base;
  414. for (size_t pt_index = 0; pt_index < needed_page_table_count; pt_index++) {
  415. auto virtual_page_base_for_this_pt = virtual_page_array_current_page;
  416. auto pt_paddr = page_tables_base.offset(pt_index * PAGE_SIZE);
  417. auto* pt = reinterpret_cast<PageTableEntry*>(quickmap_page(pt_paddr));
  418. __builtin_memset(pt, 0, PAGE_SIZE);
  419. for (size_t pte_index = 0; pte_index < PAGE_SIZE / sizeof(PageTableEntry); pte_index++) {
  420. auto& pte = pt[pte_index];
  421. pte.set_physical_page_base(physical_page_array_current_page);
  422. pte.set_user_allowed(false);
  423. pte.set_writable(true);
  424. if (Processor::current().has_nx())
  425. pte.set_execute_disabled(false);
  426. pte.set_global(true);
  427. pte.set_present(true);
  428. physical_page_array_current_page += PAGE_SIZE;
  429. virtual_page_array_current_page += PAGE_SIZE;
  430. }
  431. unquickmap_page();
  432. // Hook the page table into the kernel page directory
  433. u32 page_directory_index = (virtual_page_base_for_this_pt >> 21) & 0x1ff;
  434. auto* pd = reinterpret_cast<PageDirectoryEntry*>(quickmap_page(boot_pd_kernel));
  435. PageDirectoryEntry& pde = pd[page_directory_index];
  436. VERIFY(!pde.is_present()); // Nothing should be using this PD yet
  437. // We can't use ensure_pte quite yet!
  438. pde.set_page_table_base(pt_paddr.get());
  439. pde.set_user_allowed(false);
  440. pde.set_present(true);
  441. pde.set_writable(true);
  442. pde.set_global(true);
  443. unquickmap_page();
  444. flush_tlb_local(VirtualAddress(virtual_page_base_for_this_pt));
  445. }
  446. // We now have the entire PhysicalPageEntry array mapped!
  447. m_physical_page_entries = (PhysicalPageEntry*)range.base().get();
  448. for (size_t i = 0; i < m_physical_page_entries_count; i++)
  449. new (&m_physical_page_entries[i]) PageTableEntry();
  450. // Now we should be able to allocate PhysicalPage instances,
  451. // so finish setting up the kernel page directory
  452. m_kernel_page_directory->allocate_kernel_directory();
  453. // Now create legit PhysicalPage objects for the page tables we created.
  454. virtual_page_array_current_page = virtual_page_array_base;
  455. for (size_t pt_index = 0; pt_index < needed_page_table_count; pt_index++) {
  456. VERIFY(virtual_page_array_current_page <= range.end().get());
  457. auto pt_paddr = page_tables_base.offset(pt_index * PAGE_SIZE);
  458. auto physical_page_index = PhysicalAddress::physical_page_index(pt_paddr.get());
  459. auto& physical_page_entry = m_physical_page_entries[physical_page_index];
  460. auto physical_page = adopt_lock_ref(*new (&physical_page_entry.allocated.physical_page) PhysicalPage(MayReturnToFreeList::No));
  461. // NOTE: This leaked ref is matched by the unref in MemoryManager::release_pte()
  462. (void)physical_page.leak_ref();
  463. virtual_page_array_current_page += (PAGE_SIZE / sizeof(PageTableEntry)) * PAGE_SIZE;
  464. }
  465. dmesgln("MM: Physical page entries: {}", range);
  466. });
  467. }
  468. PhysicalPageEntry& MemoryManager::get_physical_page_entry(PhysicalAddress physical_address)
  469. {
  470. auto physical_page_entry_index = PhysicalAddress::physical_page_index(physical_address.get());
  471. VERIFY(physical_page_entry_index < m_physical_page_entries_count);
  472. return m_physical_page_entries[physical_page_entry_index];
  473. }
  474. PhysicalAddress MemoryManager::get_physical_address(PhysicalPage const& physical_page)
  475. {
  476. PhysicalPageEntry const& physical_page_entry = *reinterpret_cast<PhysicalPageEntry const*>((u8 const*)&physical_page - __builtin_offsetof(PhysicalPageEntry, allocated.physical_page));
  477. size_t physical_page_entry_index = &physical_page_entry - m_physical_page_entries;
  478. VERIFY(physical_page_entry_index < m_physical_page_entries_count);
  479. return PhysicalAddress((PhysicalPtr)physical_page_entry_index * PAGE_SIZE);
  480. }
  481. PageTableEntry* MemoryManager::pte(PageDirectory& page_directory, VirtualAddress vaddr)
  482. {
  483. VERIFY_INTERRUPTS_DISABLED();
  484. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  485. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  486. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  487. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  488. auto* pd = quickmap_pd(const_cast<PageDirectory&>(page_directory), page_directory_table_index);
  489. PageDirectoryEntry const& pde = pd[page_directory_index];
  490. if (!pde.is_present())
  491. return nullptr;
  492. return &quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()))[page_table_index];
  493. }
  494. PageTableEntry* MemoryManager::ensure_pte(PageDirectory& page_directory, VirtualAddress vaddr)
  495. {
  496. VERIFY_INTERRUPTS_DISABLED();
  497. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  498. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  499. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  500. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  501. auto* pd = quickmap_pd(page_directory, page_directory_table_index);
  502. auto& pde = pd[page_directory_index];
  503. if (pde.is_present())
  504. return &quickmap_pt(PhysicalAddress(pde.page_table_base()))[page_table_index];
  505. bool did_purge = false;
  506. auto page_table_or_error = allocate_physical_page(ShouldZeroFill::Yes, &did_purge);
  507. if (page_table_or_error.is_error()) {
  508. dbgln("MM: Unable to allocate page table to map {}", vaddr);
  509. return nullptr;
  510. }
  511. auto page_table = page_table_or_error.release_value();
  512. if (did_purge) {
  513. // If any memory had to be purged, ensure_pte may have been called as part
  514. // of the purging process. So we need to re-map the pd in this case to ensure
  515. // we're writing to the correct underlying physical page
  516. pd = quickmap_pd(page_directory, page_directory_table_index);
  517. VERIFY(&pde == &pd[page_directory_index]); // Sanity check
  518. VERIFY(!pde.is_present()); // Should have not changed
  519. }
  520. pde.set_page_table_base(page_table->paddr().get());
  521. pde.set_user_allowed(true);
  522. pde.set_present(true);
  523. pde.set_writable(true);
  524. pde.set_global(&page_directory == m_kernel_page_directory.ptr());
  525. // NOTE: This leaked ref is matched by the unref in MemoryManager::release_pte()
  526. (void)page_table.leak_ref();
  527. return &quickmap_pt(PhysicalAddress(pde.page_table_base()))[page_table_index];
  528. }
  529. void MemoryManager::release_pte(PageDirectory& page_directory, VirtualAddress vaddr, IsLastPTERelease is_last_pte_release)
  530. {
  531. VERIFY_INTERRUPTS_DISABLED();
  532. VERIFY(page_directory.get_lock().is_locked_by_current_processor());
  533. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x1ff;
  534. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  535. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  536. auto* pd = quickmap_pd(page_directory, page_directory_table_index);
  537. PageDirectoryEntry& pde = pd[page_directory_index];
  538. if (pde.is_present()) {
  539. auto* page_table = quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()));
  540. auto& pte = page_table[page_table_index];
  541. pte.clear();
  542. if (is_last_pte_release == IsLastPTERelease::Yes || page_table_index == 0x1ff) {
  543. // If this is the last PTE in a region or the last PTE in a page table then
  544. // check if we can also release the page table
  545. bool all_clear = true;
  546. for (u32 i = 0; i <= 0x1ff; i++) {
  547. if (!page_table[i].is_null()) {
  548. all_clear = false;
  549. break;
  550. }
  551. }
  552. if (all_clear) {
  553. get_physical_page_entry(PhysicalAddress { pde.page_table_base() }).allocated.physical_page.unref();
  554. pde.clear();
  555. }
  556. }
  557. }
  558. }
  559. UNMAP_AFTER_INIT void MemoryManager::initialize(u32 cpu)
  560. {
  561. ProcessorSpecific<MemoryManagerData>::initialize();
  562. if (cpu == 0) {
  563. new MemoryManager;
  564. kmalloc_enable_expand();
  565. }
  566. }
  567. Region* MemoryManager::kernel_region_from_vaddr(VirtualAddress address)
  568. {
  569. if (is_user_address(address))
  570. return nullptr;
  571. return MM.m_global_data.with([&](auto& global_data) {
  572. return global_data.region_tree.find_region_containing(address);
  573. });
  574. }
  575. Region* MemoryManager::find_user_region_from_vaddr(AddressSpace& space, VirtualAddress vaddr)
  576. {
  577. return space.find_region_containing({ vaddr, 1 });
  578. }
  579. void MemoryManager::validate_syscall_preconditions(Process& process, RegisterState const& regs)
  580. {
  581. bool should_crash = false;
  582. char const* crash_description = nullptr;
  583. int crash_signal = 0;
  584. auto unlock_and_handle_crash = [&](char const* description, int signal) {
  585. should_crash = true;
  586. crash_description = description;
  587. crash_signal = signal;
  588. };
  589. process.address_space().with([&](auto& space) -> void {
  590. VirtualAddress userspace_sp = VirtualAddress { regs.userspace_sp() };
  591. if (!MM.validate_user_stack(*space, userspace_sp)) {
  592. dbgln("Invalid stack pointer: {}", userspace_sp);
  593. return unlock_and_handle_crash("Bad stack on syscall entry", SIGSEGV);
  594. }
  595. VirtualAddress ip = VirtualAddress { regs.ip() };
  596. auto* calling_region = MM.find_user_region_from_vaddr(*space, ip);
  597. if (!calling_region) {
  598. dbgln("Syscall from {:p} which has no associated region", ip);
  599. return unlock_and_handle_crash("Syscall from unknown region", SIGSEGV);
  600. }
  601. if (calling_region->is_writable()) {
  602. dbgln("Syscall from writable memory at {:p}", ip);
  603. return unlock_and_handle_crash("Syscall from writable memory", SIGSEGV);
  604. }
  605. if (space->enforces_syscall_regions() && !calling_region->is_syscall_region()) {
  606. dbgln("Syscall from non-syscall region");
  607. return unlock_and_handle_crash("Syscall from non-syscall region", SIGSEGV);
  608. }
  609. });
  610. if (should_crash) {
  611. handle_crash(regs, crash_description, crash_signal);
  612. }
  613. }
  614. Region* MemoryManager::find_region_from_vaddr(VirtualAddress vaddr)
  615. {
  616. if (auto* region = kernel_region_from_vaddr(vaddr))
  617. return region;
  618. auto page_directory = PageDirectory::find_current();
  619. if (!page_directory)
  620. return nullptr;
  621. VERIFY(page_directory->address_space());
  622. return find_user_region_from_vaddr(*page_directory->address_space(), vaddr);
  623. }
  624. PageFaultResponse MemoryManager::handle_page_fault(PageFault const& fault)
  625. {
  626. auto faulted_in_range = [&fault](auto const* start, auto const* end) {
  627. return fault.vaddr() >= VirtualAddress { start } && fault.vaddr() < VirtualAddress { end };
  628. };
  629. if (faulted_in_range(&start_of_ro_after_init, &end_of_ro_after_init))
  630. PANIC("Attempt to write into READONLY_AFTER_INIT section");
  631. if (faulted_in_range(&start_of_unmap_after_init, &end_of_unmap_after_init)) {
  632. auto const* kernel_symbol = symbolicate_kernel_address(fault.vaddr().get());
  633. PANIC("Attempt to access UNMAP_AFTER_INIT section ({:p}: {})", fault.vaddr(), kernel_symbol ? kernel_symbol->name : "(Unknown)");
  634. }
  635. if (faulted_in_range(&start_of_kernel_ksyms, &end_of_kernel_ksyms))
  636. PANIC("Attempt to access KSYMS section");
  637. if (Processor::current_in_irq()) {
  638. dbgln("CPU[{}] BUG! Page fault while handling IRQ! code={}, vaddr={}, irq level: {}",
  639. Processor::current_id(), fault.code(), fault.vaddr(), Processor::current_in_irq());
  640. dump_kernel_regions();
  641. return PageFaultResponse::ShouldCrash;
  642. }
  643. dbgln_if(PAGE_FAULT_DEBUG, "MM: CPU[{}] handle_page_fault({:#04x}) at {}", Processor::current_id(), fault.code(), fault.vaddr());
  644. auto* region = find_region_from_vaddr(fault.vaddr());
  645. if (!region) {
  646. return PageFaultResponse::ShouldCrash;
  647. }
  648. return region->handle_fault(fault);
  649. }
  650. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_contiguous_kernel_region(size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  651. {
  652. VERIFY(!(size % PAGE_SIZE));
  653. OwnPtr<KString> name_kstring;
  654. if (!name.is_null())
  655. name_kstring = TRY(KString::try_create(name));
  656. auto vmobject = TRY(AnonymousVMObject::try_create_physically_contiguous_with_size(size));
  657. auto region = TRY(Region::create_unplaced(move(vmobject), 0, move(name_kstring), access, cacheable));
  658. TRY(m_global_data.with([&](auto& global_data) { return global_data.region_tree.place_anywhere(*region, RandomizeVirtualAddress::No, size); }));
  659. TRY(region->map(kernel_page_directory()));
  660. return region;
  661. }
  662. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::allocate_dma_buffer_page(StringView name, Memory::Region::Access access, RefPtr<Memory::PhysicalPage>& dma_buffer_page)
  663. {
  664. dma_buffer_page = TRY(allocate_physical_page());
  665. // Do not enable Cache for this region as physical memory transfers are performed (Most architectures have this behaviour by default)
  666. return allocate_kernel_region(dma_buffer_page->paddr(), PAGE_SIZE, name, access, Region::Cacheable::No);
  667. }
  668. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::allocate_dma_buffer_page(StringView name, Memory::Region::Access access)
  669. {
  670. RefPtr<Memory::PhysicalPage> dma_buffer_page;
  671. return allocate_dma_buffer_page(name, access, dma_buffer_page);
  672. }
  673. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::allocate_dma_buffer_pages(size_t size, StringView name, Memory::Region::Access access, NonnullRefPtrVector<Memory::PhysicalPage>& dma_buffer_pages)
  674. {
  675. VERIFY(!(size % PAGE_SIZE));
  676. dma_buffer_pages = TRY(allocate_contiguous_physical_pages(size));
  677. // Do not enable Cache for this region as physical memory transfers are performed (Most architectures have this behaviour by default)
  678. return allocate_kernel_region(dma_buffer_pages.first().paddr(), size, name, access, Region::Cacheable::No);
  679. }
  680. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::allocate_dma_buffer_pages(size_t size, StringView name, Memory::Region::Access access)
  681. {
  682. VERIFY(!(size % PAGE_SIZE));
  683. NonnullRefPtrVector<Memory::PhysicalPage> dma_buffer_pages;
  684. return allocate_dma_buffer_pages(size, name, access, dma_buffer_pages);
  685. }
  686. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region(size_t size, StringView name, Region::Access access, AllocationStrategy strategy, Region::Cacheable cacheable)
  687. {
  688. VERIFY(!(size % PAGE_SIZE));
  689. OwnPtr<KString> name_kstring;
  690. if (!name.is_null())
  691. name_kstring = TRY(KString::try_create(name));
  692. auto vmobject = TRY(AnonymousVMObject::try_create_with_size(size, strategy));
  693. auto region = TRY(Region::create_unplaced(move(vmobject), 0, move(name_kstring), access, cacheable));
  694. TRY(m_global_data.with([&](auto& global_data) { return global_data.region_tree.place_anywhere(*region, RandomizeVirtualAddress::No, size); }));
  695. TRY(region->map(kernel_page_directory()));
  696. return region;
  697. }
  698. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region(PhysicalAddress paddr, size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  699. {
  700. VERIFY(!(size % PAGE_SIZE));
  701. auto vmobject = TRY(AnonymousVMObject::try_create_for_physical_range(paddr, size));
  702. OwnPtr<KString> name_kstring;
  703. if (!name.is_null())
  704. name_kstring = TRY(KString::try_create(name));
  705. auto region = TRY(Region::create_unplaced(move(vmobject), 0, move(name_kstring), access, cacheable));
  706. TRY(m_global_data.with([&](auto& global_data) { return global_data.region_tree.place_anywhere(*region, RandomizeVirtualAddress::No, size, PAGE_SIZE); }));
  707. TRY(region->map(kernel_page_directory()));
  708. return region;
  709. }
  710. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_kernel_region_with_vmobject(VMObject& vmobject, size_t size, StringView name, Region::Access access, Region::Cacheable cacheable)
  711. {
  712. VERIFY(!(size % PAGE_SIZE));
  713. OwnPtr<KString> name_kstring;
  714. if (!name.is_null())
  715. name_kstring = TRY(KString::try_create(name));
  716. auto region = TRY(Region::create_unplaced(vmobject, 0, move(name_kstring), access, cacheable));
  717. TRY(m_global_data.with([&](auto& global_data) { return global_data.region_tree.place_anywhere(*region, RandomizeVirtualAddress::No, size); }));
  718. TRY(region->map(kernel_page_directory()));
  719. return region;
  720. }
  721. ErrorOr<CommittedPhysicalPageSet> MemoryManager::commit_physical_pages(size_t page_count)
  722. {
  723. VERIFY(page_count > 0);
  724. auto result = m_global_data.with([&](auto& global_data) -> ErrorOr<CommittedPhysicalPageSet> {
  725. if (global_data.system_memory_info.physical_pages_uncommitted < page_count) {
  726. dbgln("MM: Unable to commit {} pages, have only {}", page_count, global_data.system_memory_info.physical_pages_uncommitted);
  727. return ENOMEM;
  728. }
  729. global_data.system_memory_info.physical_pages_uncommitted -= page_count;
  730. global_data.system_memory_info.physical_pages_committed += page_count;
  731. return CommittedPhysicalPageSet { {}, page_count };
  732. });
  733. if (result.is_error()) {
  734. Process::for_each_ignoring_jails([&](Process const& process) {
  735. size_t amount_resident = 0;
  736. size_t amount_shared = 0;
  737. size_t amount_virtual = 0;
  738. process.address_space().with([&](auto& space) {
  739. amount_resident = space->amount_resident();
  740. amount_shared = space->amount_shared();
  741. amount_virtual = space->amount_virtual();
  742. });
  743. dbgln("{}({}) resident:{}, shared:{}, virtual:{}",
  744. process.name(),
  745. process.pid(),
  746. amount_resident / PAGE_SIZE,
  747. amount_shared / PAGE_SIZE,
  748. amount_virtual / PAGE_SIZE);
  749. return IterationDecision::Continue;
  750. });
  751. }
  752. return result;
  753. }
  754. void MemoryManager::uncommit_physical_pages(Badge<CommittedPhysicalPageSet>, size_t page_count)
  755. {
  756. VERIFY(page_count > 0);
  757. m_global_data.with([&](auto& global_data) {
  758. VERIFY(global_data.system_memory_info.physical_pages_committed >= page_count);
  759. global_data.system_memory_info.physical_pages_uncommitted += page_count;
  760. global_data.system_memory_info.physical_pages_committed -= page_count;
  761. });
  762. }
  763. void MemoryManager::deallocate_physical_page(PhysicalAddress paddr)
  764. {
  765. return m_global_data.with([&](auto& global_data) {
  766. // Are we returning a user page?
  767. for (auto& region : global_data.physical_regions) {
  768. if (!region.contains(paddr))
  769. continue;
  770. region.return_page(paddr);
  771. --global_data.system_memory_info.physical_pages_used;
  772. // Always return pages to the uncommitted pool. Pages that were
  773. // committed and allocated are only freed upon request. Once
  774. // returned there is no guarantee being able to get them back.
  775. ++global_data.system_memory_info.physical_pages_uncommitted;
  776. return;
  777. }
  778. PANIC("MM: deallocate_physical_page couldn't figure out region for page @ {}", paddr);
  779. });
  780. }
  781. RefPtr<PhysicalPage> MemoryManager::find_free_physical_page(bool committed)
  782. {
  783. RefPtr<PhysicalPage> page;
  784. m_global_data.with([&](auto& global_data) {
  785. if (committed) {
  786. // Draw from the committed pages pool. We should always have these pages available
  787. VERIFY(global_data.system_memory_info.physical_pages_committed > 0);
  788. global_data.system_memory_info.physical_pages_committed--;
  789. } else {
  790. // We need to make sure we don't touch pages that we have committed to
  791. if (global_data.system_memory_info.physical_pages_uncommitted == 0)
  792. return;
  793. global_data.system_memory_info.physical_pages_uncommitted--;
  794. }
  795. for (auto& region : global_data.physical_regions) {
  796. page = region.take_free_page();
  797. if (!page.is_null()) {
  798. ++global_data.system_memory_info.physical_pages_used;
  799. break;
  800. }
  801. }
  802. });
  803. if (page.is_null())
  804. dbgln("MM: couldn't find free physical page. Continuing...");
  805. return page;
  806. }
  807. NonnullRefPtr<PhysicalPage> MemoryManager::allocate_committed_physical_page(Badge<CommittedPhysicalPageSet>, ShouldZeroFill should_zero_fill)
  808. {
  809. auto page = find_free_physical_page(true);
  810. VERIFY(page);
  811. if (should_zero_fill == ShouldZeroFill::Yes) {
  812. InterruptDisabler disabler;
  813. auto* ptr = quickmap_page(*page);
  814. memset(ptr, 0, PAGE_SIZE);
  815. unquickmap_page();
  816. }
  817. return page.release_nonnull();
  818. }
  819. ErrorOr<NonnullRefPtr<PhysicalPage>> MemoryManager::allocate_physical_page(ShouldZeroFill should_zero_fill, bool* did_purge)
  820. {
  821. return m_global_data.with([&](auto&) -> ErrorOr<NonnullRefPtr<PhysicalPage>> {
  822. auto page = find_free_physical_page(false);
  823. bool purged_pages = false;
  824. if (!page) {
  825. // We didn't have a single free physical page. Let's try to free something up!
  826. // First, we look for a purgeable VMObject in the volatile state.
  827. for_each_vmobject([&](auto& vmobject) {
  828. if (!vmobject.is_anonymous())
  829. return IterationDecision::Continue;
  830. auto& anonymous_vmobject = static_cast<AnonymousVMObject&>(vmobject);
  831. if (!anonymous_vmobject.is_purgeable() || !anonymous_vmobject.is_volatile())
  832. return IterationDecision::Continue;
  833. if (auto purged_page_count = anonymous_vmobject.purge()) {
  834. dbgln("MM: Purge saved the day! Purged {} pages from AnonymousVMObject", purged_page_count);
  835. page = find_free_physical_page(false);
  836. purged_pages = true;
  837. VERIFY(page);
  838. return IterationDecision::Break;
  839. }
  840. return IterationDecision::Continue;
  841. });
  842. }
  843. if (!page) {
  844. // Second, we look for a file-backed VMObject with clean pages.
  845. for_each_vmobject([&](auto& vmobject) {
  846. if (!vmobject.is_inode())
  847. return IterationDecision::Continue;
  848. auto& inode_vmobject = static_cast<InodeVMObject&>(vmobject);
  849. if (auto released_page_count = inode_vmobject.try_release_clean_pages(1)) {
  850. dbgln("MM: Clean inode release saved the day! Released {} pages from InodeVMObject", released_page_count);
  851. page = find_free_physical_page(false);
  852. VERIFY(page);
  853. return IterationDecision::Break;
  854. }
  855. return IterationDecision::Continue;
  856. });
  857. }
  858. if (!page) {
  859. dmesgln("MM: no physical pages available");
  860. return ENOMEM;
  861. }
  862. if (should_zero_fill == ShouldZeroFill::Yes) {
  863. auto* ptr = quickmap_page(*page);
  864. memset(ptr, 0, PAGE_SIZE);
  865. unquickmap_page();
  866. }
  867. if (did_purge)
  868. *did_purge = purged_pages;
  869. return page.release_nonnull();
  870. });
  871. }
  872. ErrorOr<NonnullRefPtrVector<PhysicalPage>> MemoryManager::allocate_contiguous_physical_pages(size_t size)
  873. {
  874. VERIFY(!(size % PAGE_SIZE));
  875. size_t page_count = ceil_div(size, static_cast<size_t>(PAGE_SIZE));
  876. auto physical_pages = TRY(m_global_data.with([&](auto& global_data) -> ErrorOr<NonnullRefPtrVector<PhysicalPage>> {
  877. // We need to make sure we don't touch pages that we have committed to
  878. if (global_data.system_memory_info.physical_pages_uncommitted < page_count)
  879. return ENOMEM;
  880. for (auto& physical_region : global_data.physical_regions) {
  881. auto physical_pages = physical_region.take_contiguous_free_pages(page_count);
  882. if (!physical_pages.is_empty()) {
  883. global_data.system_memory_info.physical_pages_uncommitted -= page_count;
  884. global_data.system_memory_info.physical_pages_used += page_count;
  885. return physical_pages;
  886. }
  887. }
  888. dmesgln("MM: no contiguous physical pages available");
  889. return ENOMEM;
  890. }));
  891. {
  892. auto cleanup_region = TRY(MM.allocate_kernel_region(physical_pages[0].paddr(), PAGE_SIZE * page_count, {}, Region::Access::Read | Region::Access::Write));
  893. memset(cleanup_region->vaddr().as_ptr(), 0, PAGE_SIZE * page_count);
  894. }
  895. return physical_pages;
  896. }
  897. void MemoryManager::enter_process_address_space(Process& process)
  898. {
  899. process.address_space().with([](auto& space) {
  900. enter_address_space(*space);
  901. });
  902. }
  903. void MemoryManager::enter_address_space(AddressSpace& space)
  904. {
  905. auto* current_thread = Thread::current();
  906. VERIFY(current_thread != nullptr);
  907. activate_page_directory(space.page_directory(), current_thread);
  908. }
  909. void MemoryManager::flush_tlb_local(VirtualAddress vaddr, size_t page_count)
  910. {
  911. Processor::flush_tlb_local(vaddr, page_count);
  912. }
  913. void MemoryManager::flush_tlb(PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count)
  914. {
  915. Processor::flush_tlb(page_directory, vaddr, page_count);
  916. }
  917. PageDirectoryEntry* MemoryManager::quickmap_pd(PageDirectory& directory, size_t pdpt_index)
  918. {
  919. VERIFY_INTERRUPTS_DISABLED();
  920. VirtualAddress vaddr(KERNEL_QUICKMAP_PD_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  921. size_t pte_index = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  922. auto& pte = boot_pd_kernel_pt1023[pte_index];
  923. auto pd_paddr = directory.m_directory_pages[pdpt_index]->paddr();
  924. if (pte.physical_page_base() != pd_paddr.get()) {
  925. pte.set_physical_page_base(pd_paddr.get());
  926. pte.set_present(true);
  927. pte.set_writable(true);
  928. pte.set_user_allowed(false);
  929. flush_tlb_local(vaddr);
  930. }
  931. return (PageDirectoryEntry*)vaddr.get();
  932. }
  933. PageTableEntry* MemoryManager::quickmap_pt(PhysicalAddress pt_paddr)
  934. {
  935. VERIFY_INTERRUPTS_DISABLED();
  936. VirtualAddress vaddr(KERNEL_QUICKMAP_PT_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  937. size_t pte_index = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  938. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[pte_index];
  939. if (pte.physical_page_base() != pt_paddr.get()) {
  940. pte.set_physical_page_base(pt_paddr.get());
  941. pte.set_present(true);
  942. pte.set_writable(true);
  943. pte.set_user_allowed(false);
  944. flush_tlb_local(vaddr);
  945. }
  946. return (PageTableEntry*)vaddr.get();
  947. }
  948. u8* MemoryManager::quickmap_page(PhysicalAddress const& physical_address)
  949. {
  950. VERIFY_INTERRUPTS_DISABLED();
  951. auto& mm_data = get_data();
  952. mm_data.m_quickmap_previous_interrupts_state = mm_data.m_quickmap_in_use.lock();
  953. VirtualAddress vaddr(KERNEL_QUICKMAP_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  954. u32 pte_idx = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  955. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[pte_idx];
  956. if (pte.physical_page_base() != physical_address.get()) {
  957. pte.set_physical_page_base(physical_address.get());
  958. pte.set_present(true);
  959. pte.set_writable(true);
  960. pte.set_user_allowed(false);
  961. flush_tlb_local(vaddr);
  962. }
  963. return vaddr.as_ptr();
  964. }
  965. void MemoryManager::unquickmap_page()
  966. {
  967. VERIFY_INTERRUPTS_DISABLED();
  968. auto& mm_data = get_data();
  969. VERIFY(mm_data.m_quickmap_in_use.is_locked());
  970. VirtualAddress vaddr(KERNEL_QUICKMAP_PER_CPU_BASE + Processor::current_id() * PAGE_SIZE);
  971. u32 pte_idx = (vaddr.get() - KERNEL_PT1024_BASE) / PAGE_SIZE;
  972. auto& pte = ((PageTableEntry*)boot_pd_kernel_pt1023)[pte_idx];
  973. pte.clear();
  974. flush_tlb_local(vaddr);
  975. mm_data.m_quickmap_in_use.unlock(mm_data.m_quickmap_previous_interrupts_state);
  976. }
  977. bool MemoryManager::validate_user_stack(AddressSpace& space, VirtualAddress vaddr) const
  978. {
  979. if (!is_user_address(vaddr))
  980. return false;
  981. auto* region = find_user_region_from_vaddr(space, vaddr);
  982. return region && region->is_user() && region->is_stack();
  983. }
  984. void MemoryManager::unregister_kernel_region(Region& region)
  985. {
  986. VERIFY(region.is_kernel());
  987. m_global_data.with([&](auto& global_data) { global_data.region_tree.remove(region); });
  988. }
  989. void MemoryManager::dump_kernel_regions()
  990. {
  991. dbgln("Kernel regions:");
  992. char const* addr_padding = " ";
  993. dbgln("BEGIN{} END{} SIZE{} ACCESS NAME",
  994. addr_padding, addr_padding, addr_padding);
  995. m_global_data.with([&](auto& global_data) {
  996. for (auto& region : global_data.region_tree.regions()) {
  997. dbgln("{:p} -- {:p} {:p} {:c}{:c}{:c}{:c}{:c}{:c} {}",
  998. region.vaddr().get(),
  999. region.vaddr().offset(region.size() - 1).get(),
  1000. region.size(),
  1001. region.is_readable() ? 'R' : ' ',
  1002. region.is_writable() ? 'W' : ' ',
  1003. region.is_executable() ? 'X' : ' ',
  1004. region.is_shared() ? 'S' : ' ',
  1005. region.is_stack() ? 'T' : ' ',
  1006. region.is_syscall_region() ? 'C' : ' ',
  1007. region.name());
  1008. }
  1009. });
  1010. }
  1011. void MemoryManager::set_page_writable_direct(VirtualAddress vaddr, bool writable)
  1012. {
  1013. SpinlockLocker page_lock(kernel_page_directory().get_lock());
  1014. auto* pte = ensure_pte(kernel_page_directory(), vaddr);
  1015. VERIFY(pte);
  1016. if (pte->is_writable() == writable)
  1017. return;
  1018. pte->set_writable(writable);
  1019. flush_tlb(&kernel_page_directory(), vaddr);
  1020. }
  1021. CommittedPhysicalPageSet::~CommittedPhysicalPageSet()
  1022. {
  1023. if (m_page_count)
  1024. MM.uncommit_physical_pages({}, m_page_count);
  1025. }
  1026. NonnullRefPtr<PhysicalPage> CommittedPhysicalPageSet::take_one()
  1027. {
  1028. VERIFY(m_page_count > 0);
  1029. --m_page_count;
  1030. return MM.allocate_committed_physical_page({}, MemoryManager::ShouldZeroFill::Yes);
  1031. }
  1032. void CommittedPhysicalPageSet::uncommit_one()
  1033. {
  1034. VERIFY(m_page_count > 0);
  1035. --m_page_count;
  1036. MM.uncommit_physical_pages({}, 1);
  1037. }
  1038. void MemoryManager::copy_physical_page(PhysicalPage& physical_page, u8 page_buffer[PAGE_SIZE])
  1039. {
  1040. auto* quickmapped_page = quickmap_page(physical_page);
  1041. memcpy(page_buffer, quickmapped_page, PAGE_SIZE);
  1042. unquickmap_page();
  1043. }
  1044. ErrorOr<NonnullOwnPtr<Memory::Region>> MemoryManager::create_identity_mapped_region(PhysicalAddress address, size_t size)
  1045. {
  1046. auto vmobject = TRY(Memory::AnonymousVMObject::try_create_for_physical_range(address, size));
  1047. auto region = TRY(Memory::Region::create_unplaced(move(vmobject), 0, {}, Memory::Region::Access::ReadWriteExecute));
  1048. Memory::VirtualRange range { VirtualAddress { (FlatPtr)address.get() }, size };
  1049. region->m_range = range;
  1050. TRY(region->map(MM.kernel_page_directory()));
  1051. return region;
  1052. }
  1053. ErrorOr<NonnullOwnPtr<Region>> MemoryManager::allocate_unbacked_region_anywhere(size_t size, size_t alignment)
  1054. {
  1055. auto region = TRY(Region::create_unbacked());
  1056. TRY(m_global_data.with([&](auto& global_data) { return global_data.region_tree.place_anywhere(*region, RandomizeVirtualAddress::No, size, alignment); }));
  1057. return region;
  1058. }
  1059. MemoryManager::SystemMemoryInfo MemoryManager::get_system_memory_info()
  1060. {
  1061. return m_global_data.with([&](auto& global_data) {
  1062. auto physical_pages_unused = global_data.system_memory_info.physical_pages_committed + global_data.system_memory_info.physical_pages_uncommitted;
  1063. VERIFY(global_data.system_memory_info.physical_pages == (global_data.system_memory_info.physical_pages_used + physical_pages_unused));
  1064. return global_data.system_memory_info;
  1065. });
  1066. }
  1067. }