MemoryManager.cpp 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781
  1. #include "CMOS.h"
  2. #include "Process.h"
  3. #include "StdLib.h"
  4. #include <AK/Assertions.h>
  5. #include <AK/kstdio.h>
  6. #include <Kernel/Arch/i386/CPU.h>
  7. #include <Kernel/FileSystem/Inode.h>
  8. #include <Kernel/Multiboot.h>
  9. #include <Kernel/VM/AnonymousVMObject.h>
  10. #include <Kernel/VM/InodeVMObject.h>
  11. #include <Kernel/VM/MemoryManager.h>
  12. #include <Kernel/VM/PurgeableVMObject.h>
  13. //#define MM_DEBUG
  14. //#define PAGE_FAULT_DEBUG
  15. static MemoryManager* s_the;
  16. MemoryManager& MM
  17. {
  18. return *s_the;
  19. }
  20. MemoryManager::MemoryManager()
  21. {
  22. m_kernel_page_directory = PageDirectory::create_kernel_page_directory();
  23. initialize_paging();
  24. kprintf("MM initialized.\n");
  25. }
  26. MemoryManager::~MemoryManager()
  27. {
  28. }
  29. void MemoryManager::initialize_paging()
  30. {
  31. if (!g_cpu_supports_pae) {
  32. kprintf("x86: Cannot boot on machines without PAE support.\n");
  33. hang();
  34. }
  35. #ifdef MM_DEBUG
  36. dbgprintf("MM: Kernel page directory @ %p\n", kernel_page_directory().cr3());
  37. #endif
  38. parse_memory_map();
  39. #ifdef MM_DEBUG
  40. dbgprintf("MM: Installing page directory\n");
  41. #endif
  42. // Turn on CR4.PAE
  43. asm volatile(
  44. "mov %cr4, %eax\n"
  45. "orl $0x20, %eax\n"
  46. "mov %eax, %cr4\n");
  47. if (g_cpu_supports_pge) {
  48. // Turn on CR4.PGE so the CPU will respect the G bit in page tables.
  49. asm volatile(
  50. "mov %cr4, %eax\n"
  51. "orl $0x80, %eax\n"
  52. "mov %eax, %cr4\n");
  53. kprintf("x86: PGE support enabled\n");
  54. } else {
  55. kprintf("x86: PGE support not detected\n");
  56. }
  57. if (g_cpu_supports_smep) {
  58. // Turn on CR4.SMEP
  59. asm volatile(
  60. "mov %cr4, %eax\n"
  61. "orl $0x100000, %eax\n"
  62. "mov %eax, %cr4\n");
  63. kprintf("x86: SMEP support enabled\n");
  64. } else {
  65. kprintf("x86: SMEP support not detected\n");
  66. }
  67. if (g_cpu_supports_smap) {
  68. // Turn on CR4.SMAP
  69. kprintf("x86: Enabling SMAP\n");
  70. asm volatile(
  71. "mov %cr4, %eax\n"
  72. "orl $0x200000, %eax\n"
  73. "mov %eax, %cr4\n");
  74. kprintf("x86: SMAP support enabled\n");
  75. } else {
  76. kprintf("x86: SMAP support not detected\n");
  77. }
  78. if (g_cpu_supports_nx) {
  79. // Turn on IA32_EFER.NXE
  80. asm volatile(
  81. "movl $0xc0000080, %ecx\n"
  82. "rdmsr\n"
  83. "orl $0x800, %eax\n"
  84. "wrmsr\n");
  85. kprintf("x86: NX support enabled\n");
  86. } else {
  87. kprintf("x86: NX support not detected\n");
  88. }
  89. asm volatile("movl %%eax, %%cr3" ::"a"(kernel_page_directory().cr3()));
  90. asm volatile(
  91. "movl %%cr0, %%eax\n"
  92. "orl $0x80010001, %%eax\n"
  93. "movl %%eax, %%cr0\n" ::
  94. : "%eax", "memory");
  95. setup_low_1mb();
  96. protect_kernel_image();
  97. #ifdef MM_DEBUG
  98. dbgprintf("MM: Paging initialized.\n");
  99. #endif
  100. }
  101. void MemoryManager::protect_kernel_image()
  102. {
  103. // Disable writing to the kernel text and rodata segments.
  104. extern u32 start_of_kernel_text;
  105. extern u32 start_of_kernel_data;
  106. for (size_t i = (u32)&start_of_kernel_text; i < (u32)&start_of_kernel_data; i += PAGE_SIZE) {
  107. auto& pte = ensure_pte(kernel_page_directory(), VirtualAddress(i));
  108. pte.set_writable(false);
  109. }
  110. if (g_cpu_supports_nx) {
  111. // Disable execution of the kernel data and bss segments.
  112. extern u32 end_of_kernel_bss;
  113. for (size_t i = (u32)&start_of_kernel_data; i < (u32)&end_of_kernel_bss; i += PAGE_SIZE) {
  114. auto& pte = ensure_pte(kernel_page_directory(), VirtualAddress(i));
  115. pte.set_execute_disabled(true);
  116. }
  117. }
  118. }
  119. void MemoryManager::setup_low_1mb()
  120. {
  121. m_low_page_table = allocate_user_physical_page(ShouldZeroFill::Yes);
  122. auto* pd_zero = quickmap_pd(kernel_page_directory(), 0);
  123. pd_zero[1].set_present(false);
  124. pd_zero[2].set_present(false);
  125. pd_zero[3].set_present(false);
  126. auto& pde_zero = pd_zero[0];
  127. pde_zero.set_page_table_base(m_low_page_table->paddr().get());
  128. pde_zero.set_present(true);
  129. pde_zero.set_huge(false);
  130. pde_zero.set_writable(true);
  131. pde_zero.set_user_allowed(false);
  132. if (g_cpu_supports_nx)
  133. pde_zero.set_execute_disabled(true);
  134. for (u32 offset = 0; offset < (2 * MB); offset += PAGE_SIZE) {
  135. auto& page_table_page = m_low_page_table;
  136. auto& pte = quickmap_pt(page_table_page->paddr())[offset / PAGE_SIZE];
  137. pte.set_physical_page_base(offset);
  138. pte.set_user_allowed(false);
  139. pte.set_present(offset != 0);
  140. pte.set_writable(offset < (1 * MB));
  141. }
  142. }
  143. void MemoryManager::parse_memory_map()
  144. {
  145. RefPtr<PhysicalRegion> region;
  146. bool region_is_super = false;
  147. auto* mmap = (multiboot_memory_map_t*)(low_physical_to_virtual(multiboot_info_ptr->mmap_addr));
  148. for (; (unsigned long)mmap < (low_physical_to_virtual(multiboot_info_ptr->mmap_addr)) + (multiboot_info_ptr->mmap_length); mmap = (multiboot_memory_map_t*)((unsigned long)mmap + mmap->size + sizeof(mmap->size))) {
  149. kprintf("MM: Multiboot mmap: base_addr = 0x%x%08x, length = 0x%x%08x, type = 0x%x\n",
  150. (u32)(mmap->addr >> 32),
  151. (u32)(mmap->addr & 0xffffffff),
  152. (u32)(mmap->len >> 32),
  153. (u32)(mmap->len & 0xffffffff),
  154. (u32)mmap->type);
  155. if (mmap->type != MULTIBOOT_MEMORY_AVAILABLE)
  156. continue;
  157. // FIXME: Maybe make use of stuff below the 1MB mark?
  158. if (mmap->addr < (1 * MB))
  159. continue;
  160. if ((mmap->addr + mmap->len) > 0xffffffff)
  161. continue;
  162. auto diff = (u32)mmap->addr % PAGE_SIZE;
  163. if (diff != 0) {
  164. kprintf("MM: got an unaligned region base from the bootloader; correcting %p by %d bytes\n", mmap->addr, diff);
  165. diff = PAGE_SIZE - diff;
  166. mmap->addr += diff;
  167. mmap->len -= diff;
  168. }
  169. if ((mmap->len % PAGE_SIZE) != 0) {
  170. kprintf("MM: got an unaligned region length from the bootloader; correcting %d by %d bytes\n", mmap->len, mmap->len % PAGE_SIZE);
  171. mmap->len -= mmap->len % PAGE_SIZE;
  172. }
  173. if (mmap->len < PAGE_SIZE) {
  174. kprintf("MM: memory region from bootloader is too small; we want >= %d bytes, but got %d bytes\n", PAGE_SIZE, mmap->len);
  175. continue;
  176. }
  177. #ifdef MM_DEBUG
  178. kprintf("MM: considering memory at %p - %p\n",
  179. (u32)mmap->addr, (u32)(mmap->addr + mmap->len));
  180. #endif
  181. for (size_t page_base = mmap->addr; page_base < (mmap->addr + mmap->len); page_base += PAGE_SIZE) {
  182. auto addr = PhysicalAddress(page_base);
  183. if (page_base < 7 * MB) {
  184. // nothing
  185. } else if (page_base >= 7 * MB && page_base < 8 * MB) {
  186. if (region.is_null() || !region_is_super || region->upper().offset(PAGE_SIZE) != addr) {
  187. m_super_physical_regions.append(PhysicalRegion::create(addr, addr));
  188. region = m_super_physical_regions.last();
  189. region_is_super = true;
  190. } else {
  191. region->expand(region->lower(), addr);
  192. }
  193. } else {
  194. if (region.is_null() || region_is_super || region->upper().offset(PAGE_SIZE) != addr) {
  195. m_user_physical_regions.append(PhysicalRegion::create(addr, addr));
  196. region = m_user_physical_regions.last();
  197. region_is_super = false;
  198. } else {
  199. region->expand(region->lower(), addr);
  200. }
  201. }
  202. }
  203. }
  204. for (auto& region : m_super_physical_regions)
  205. m_super_physical_pages += region.finalize_capacity();
  206. for (auto& region : m_user_physical_regions)
  207. m_user_physical_pages += region.finalize_capacity();
  208. }
  209. PageTableEntry& MemoryManager::ensure_pte(PageDirectory& page_directory, VirtualAddress vaddr)
  210. {
  211. ASSERT_INTERRUPTS_DISABLED();
  212. u32 page_directory_table_index = (vaddr.get() >> 30) & 0x3;
  213. u32 page_directory_index = (vaddr.get() >> 21) & 0x1ff;
  214. u32 page_table_index = (vaddr.get() >> 12) & 0x1ff;
  215. auto* pd = quickmap_pd(page_directory, page_directory_table_index);
  216. PageDirectoryEntry& pde = pd[page_directory_index];
  217. if (!pde.is_present()) {
  218. #ifdef MM_DEBUG
  219. dbgprintf("MM: PDE %u not present (requested for V%p), allocating\n", page_directory_index, vaddr.get());
  220. #endif
  221. auto page_table = allocate_user_physical_page(ShouldZeroFill::Yes);
  222. #ifdef MM_DEBUG
  223. dbgprintf("MM: PD K%p (%s) at P%p allocated page table #%u (for V%p) at P%p\n",
  224. &page_directory,
  225. &page_directory == m_kernel_page_directory ? "Kernel" : "User",
  226. page_directory.cr3(),
  227. page_directory_index,
  228. vaddr.get(),
  229. page_table->paddr().get());
  230. #endif
  231. pde.set_page_table_base(page_table->paddr().get());
  232. pde.set_user_allowed(true);
  233. pde.set_present(true);
  234. pde.set_writable(true);
  235. pde.set_global(&page_directory == m_kernel_page_directory.ptr());
  236. page_directory.m_physical_pages.set(page_directory_index, move(page_table));
  237. }
  238. return quickmap_pt(PhysicalAddress((u32)pde.page_table_base()))[page_table_index];
  239. }
  240. void MemoryManager::map_protected(VirtualAddress vaddr, size_t length)
  241. {
  242. InterruptDisabler disabler;
  243. ASSERT(vaddr.is_page_aligned());
  244. for (u32 offset = 0; offset < length; offset += PAGE_SIZE) {
  245. auto pte_address = vaddr.offset(offset);
  246. auto& pte = ensure_pte(kernel_page_directory(), pte_address);
  247. pte.set_physical_page_base(pte_address.get());
  248. pte.set_user_allowed(false);
  249. pte.set_present(false);
  250. pte.set_writable(false);
  251. flush_tlb(pte_address);
  252. }
  253. }
  254. void MemoryManager::create_identity_mapping(PageDirectory& page_directory, VirtualAddress vaddr, size_t size)
  255. {
  256. InterruptDisabler disabler;
  257. ASSERT((vaddr.get() & ~PAGE_MASK) == 0);
  258. for (u32 offset = 0; offset < size; offset += PAGE_SIZE) {
  259. auto pte_address = vaddr.offset(offset);
  260. auto& pte = ensure_pte(page_directory, pte_address);
  261. pte.set_physical_page_base(pte_address.get());
  262. pte.set_user_allowed(false);
  263. pte.set_present(true);
  264. pte.set_writable(true);
  265. flush_tlb(pte_address);
  266. }
  267. }
  268. void MemoryManager::initialize()
  269. {
  270. s_the = new MemoryManager;
  271. }
  272. Region* MemoryManager::kernel_region_from_vaddr(VirtualAddress vaddr)
  273. {
  274. if (vaddr.get() < 0xc0000000)
  275. return nullptr;
  276. for (auto& region : MM.m_kernel_regions) {
  277. if (region.contains(vaddr))
  278. return &region;
  279. }
  280. return nullptr;
  281. }
  282. Region* MemoryManager::user_region_from_vaddr(Process& process, VirtualAddress vaddr)
  283. {
  284. // FIXME: Use a binary search tree (maybe red/black?) or some other more appropriate data structure!
  285. for (auto& region : process.m_regions) {
  286. if (region.contains(vaddr))
  287. return &region;
  288. }
  289. dbg() << process << " Couldn't find user region for " << vaddr;
  290. if (auto* kreg = kernel_region_from_vaddr(vaddr)) {
  291. dbg() << process << " OTOH, there is a kernel region: " << kreg->range() << ": " << kreg->name();
  292. } else {
  293. dbg() << process << " AND no kernel region either";
  294. }
  295. process.dump_regions();
  296. kprintf("Kernel regions:\n");
  297. kprintf("BEGIN END SIZE ACCESS NAME\n");
  298. for (auto& region : MM.m_kernel_regions) {
  299. kprintf("%08x -- %08x %08x %c%c%c%c%c%c %s\n",
  300. region.vaddr().get(),
  301. region.vaddr().offset(region.size() - 1).get(),
  302. region.size(),
  303. region.is_readable() ? 'R' : ' ',
  304. region.is_writable() ? 'W' : ' ',
  305. region.is_executable() ? 'X' : ' ',
  306. region.is_shared() ? 'S' : ' ',
  307. region.is_stack() ? 'T' : ' ',
  308. region.vmobject().is_purgeable() ? 'P' : ' ',
  309. region.name().characters());
  310. }
  311. return nullptr;
  312. }
  313. Region* MemoryManager::region_from_vaddr(Process& process, VirtualAddress vaddr)
  314. {
  315. if (auto* region = kernel_region_from_vaddr(vaddr))
  316. return region;
  317. return user_region_from_vaddr(process, vaddr);
  318. }
  319. const Region* MemoryManager::region_from_vaddr(const Process& process, VirtualAddress vaddr)
  320. {
  321. if (auto* region = kernel_region_from_vaddr(vaddr))
  322. return region;
  323. return user_region_from_vaddr(const_cast<Process&>(process), vaddr);
  324. }
  325. Region* MemoryManager::region_from_vaddr(VirtualAddress vaddr)
  326. {
  327. if (auto* region = kernel_region_from_vaddr(vaddr))
  328. return region;
  329. auto page_directory = PageDirectory::find_by_cr3(cpu_cr3());
  330. if (!page_directory)
  331. return nullptr;
  332. ASSERT(page_directory->process());
  333. return user_region_from_vaddr(*page_directory->process(), vaddr);
  334. }
  335. PageFaultResponse MemoryManager::handle_page_fault(const PageFault& fault)
  336. {
  337. ASSERT_INTERRUPTS_DISABLED();
  338. ASSERT(current);
  339. #ifdef PAGE_FAULT_DEBUG
  340. dbgprintf("MM: handle_page_fault(%w) at V%p\n", fault.code(), fault.vaddr().get());
  341. #endif
  342. auto* region = region_from_vaddr(fault.vaddr());
  343. if (!region) {
  344. kprintf("NP(error) fault at invalid address V%p\n", fault.vaddr().get());
  345. return PageFaultResponse::ShouldCrash;
  346. }
  347. return region->handle_fault(fault);
  348. }
  349. OwnPtr<Region> MemoryManager::allocate_kernel_region(size_t size, const StringView& name, u8 access, bool user_accessible, bool should_commit, bool cacheable)
  350. {
  351. InterruptDisabler disabler;
  352. ASSERT(!(size % PAGE_SIZE));
  353. auto range = kernel_page_directory().range_allocator().allocate_anywhere(size);
  354. ASSERT(range.is_valid());
  355. OwnPtr<Region> region;
  356. if (user_accessible)
  357. region = Region::create_user_accessible(range, name, access, cacheable);
  358. else
  359. region = Region::create_kernel_only(range, name, access, cacheable);
  360. region->set_page_directory(kernel_page_directory());
  361. // FIXME: It would be cool if these could zero-fill on demand instead.
  362. if (should_commit)
  363. region->commit();
  364. return region;
  365. }
  366. OwnPtr<Region> MemoryManager::allocate_kernel_region(PhysicalAddress paddr, size_t size, const StringView& name, u8 access, bool user_accessible, bool cacheable)
  367. {
  368. InterruptDisabler disabler;
  369. ASSERT(!(size % PAGE_SIZE));
  370. auto range = kernel_page_directory().range_allocator().allocate_anywhere(size);
  371. ASSERT(range.is_valid());
  372. OwnPtr<Region> region;
  373. if (user_accessible)
  374. region = Region::create_user_accessible(range, AnonymousVMObject::create_for_physical_range(paddr, size), 0, name, access, cacheable);
  375. else
  376. region = Region::create_kernel_only(range, AnonymousVMObject::create_for_physical_range(paddr, size), 0, name, access, cacheable);
  377. region->map(kernel_page_directory());
  378. return region;
  379. }
  380. OwnPtr<Region> MemoryManager::allocate_user_accessible_kernel_region(size_t size, const StringView& name, u8 access, bool cacheable)
  381. {
  382. return allocate_kernel_region(size, name, access, true, true, cacheable);
  383. }
  384. OwnPtr<Region> MemoryManager::allocate_kernel_region_with_vmobject(VMObject& vmobject, size_t size, const StringView& name, u8 access, bool user_accessible, bool cacheable)
  385. {
  386. InterruptDisabler disabler;
  387. ASSERT(!(size % PAGE_SIZE));
  388. auto range = kernel_page_directory().range_allocator().allocate_anywhere(size);
  389. ASSERT(range.is_valid());
  390. OwnPtr<Region> region;
  391. if (user_accessible)
  392. region = Region::create_user_accessible(range, vmobject, 0, name, access, cacheable);
  393. else
  394. region = Region::create_kernel_only(range, vmobject, 0, name, access, cacheable);
  395. region->map(kernel_page_directory());
  396. return region;
  397. }
  398. void MemoryManager::deallocate_user_physical_page(PhysicalPage&& page)
  399. {
  400. for (auto& region : m_user_physical_regions) {
  401. if (!region.contains(page)) {
  402. kprintf(
  403. "MM: deallocate_user_physical_page: %p not in %p -> %p\n",
  404. page.paddr().get(), region.lower().get(), region.upper().get());
  405. continue;
  406. }
  407. region.return_page(move(page));
  408. --m_user_physical_pages_used;
  409. return;
  410. }
  411. kprintf("MM: deallocate_user_physical_page couldn't figure out region for user page @ %p\n", page.paddr().get());
  412. ASSERT_NOT_REACHED();
  413. }
  414. RefPtr<PhysicalPage> MemoryManager::find_free_user_physical_page()
  415. {
  416. RefPtr<PhysicalPage> page;
  417. for (auto& region : m_user_physical_regions) {
  418. page = region.take_free_page(false);
  419. if (!page.is_null())
  420. break;
  421. }
  422. return page;
  423. }
  424. RefPtr<PhysicalPage> MemoryManager::allocate_user_physical_page(ShouldZeroFill should_zero_fill)
  425. {
  426. InterruptDisabler disabler;
  427. RefPtr<PhysicalPage> page = find_free_user_physical_page();
  428. if (!page) {
  429. if (m_user_physical_regions.is_empty()) {
  430. kprintf("MM: no user physical regions available (?)\n");
  431. }
  432. for_each_vmobject([&](auto& vmobject) {
  433. if (vmobject.is_purgeable()) {
  434. auto& purgeable_vmobject = static_cast<PurgeableVMObject&>(vmobject);
  435. int purged_page_count = purgeable_vmobject.purge_with_interrupts_disabled({});
  436. if (purged_page_count) {
  437. kprintf("MM: Purge saved the day! Purged %d pages from PurgeableVMObject{%p}\n", purged_page_count, &purgeable_vmobject);
  438. page = find_free_user_physical_page();
  439. ASSERT(page);
  440. return IterationDecision::Break;
  441. }
  442. }
  443. return IterationDecision::Continue;
  444. });
  445. if (!page) {
  446. kprintf("MM: no user physical pages available\n");
  447. ASSERT_NOT_REACHED();
  448. return {};
  449. }
  450. }
  451. #ifdef MM_DEBUG
  452. dbgprintf("MM: allocate_user_physical_page vending P%p\n", page->paddr().get());
  453. #endif
  454. if (should_zero_fill == ShouldZeroFill::Yes) {
  455. auto* ptr = (u32*)quickmap_page(*page);
  456. memset(ptr, 0, PAGE_SIZE);
  457. unquickmap_page();
  458. }
  459. ++m_user_physical_pages_used;
  460. return page;
  461. }
  462. void MemoryManager::deallocate_supervisor_physical_page(PhysicalPage&& page)
  463. {
  464. for (auto& region : m_super_physical_regions) {
  465. if (!region.contains(page)) {
  466. kprintf(
  467. "MM: deallocate_supervisor_physical_page: %p not in %p -> %p\n",
  468. page.paddr().get(), region.lower().get(), region.upper().get());
  469. continue;
  470. }
  471. region.return_page(move(page));
  472. --m_super_physical_pages_used;
  473. return;
  474. }
  475. kprintf("MM: deallocate_supervisor_physical_page couldn't figure out region for super page @ %p\n", page.paddr().get());
  476. ASSERT_NOT_REACHED();
  477. }
  478. RefPtr<PhysicalPage> MemoryManager::allocate_supervisor_physical_page()
  479. {
  480. InterruptDisabler disabler;
  481. RefPtr<PhysicalPage> page;
  482. for (auto& region : m_super_physical_regions) {
  483. page = region.take_free_page(true);
  484. if (page.is_null())
  485. continue;
  486. }
  487. if (!page) {
  488. if (m_super_physical_regions.is_empty()) {
  489. kprintf("MM: no super physical regions available (?)\n");
  490. }
  491. kprintf("MM: no super physical pages available\n");
  492. ASSERT_NOT_REACHED();
  493. return {};
  494. }
  495. #ifdef MM_DEBUG
  496. dbgprintf("MM: allocate_supervisor_physical_page vending P%p\n", page->paddr().get());
  497. #endif
  498. fast_u32_fill((u32*)page->paddr().offset(0xc0000000).as_ptr(), 0, PAGE_SIZE / sizeof(u32));
  499. ++m_super_physical_pages_used;
  500. return page;
  501. }
  502. void MemoryManager::enter_process_paging_scope(Process& process)
  503. {
  504. ASSERT(current);
  505. InterruptDisabler disabler;
  506. current->tss().cr3 = process.page_directory().cr3();
  507. asm volatile("movl %%eax, %%cr3" ::"a"(process.page_directory().cr3())
  508. : "memory");
  509. }
  510. void MemoryManager::flush_entire_tlb()
  511. {
  512. asm volatile(
  513. "mov %%cr3, %%eax\n"
  514. "mov %%eax, %%cr3\n" ::
  515. : "%eax", "memory");
  516. }
  517. void MemoryManager::flush_tlb(VirtualAddress vaddr)
  518. {
  519. #ifdef MM_DEBUG
  520. dbgprintf("MM: Flush page V%p\n", vaddr.get());
  521. #endif
  522. asm volatile("invlpg %0"
  523. :
  524. : "m"(*(char*)vaddr.get())
  525. : "memory");
  526. }
  527. extern "C" PageTableEntry boot_pd3_pde1023_pt[1024];
  528. PageDirectoryEntry* MemoryManager::quickmap_pd(PageDirectory& directory, size_t pdpt_index)
  529. {
  530. auto& pte = boot_pd3_pde1023_pt[4];
  531. auto pd_paddr = directory.m_directory_pages[pdpt_index]->paddr();
  532. if (pte.physical_page_base() != pd_paddr.as_ptr()) {
  533. #ifdef MM_DEBUG
  534. dbgprintf("quickmap_pd: Mapping P%p at 0xffe04000 in pte @ %p\n", directory.m_directory_pages[pdpt_index]->paddr().as_ptr(), &pte);
  535. #endif
  536. pte.set_physical_page_base(pd_paddr.get());
  537. pte.set_present(true);
  538. pte.set_writable(true);
  539. pte.set_user_allowed(false);
  540. flush_tlb(VirtualAddress(0xffe04000));
  541. }
  542. return (PageDirectoryEntry*)0xffe04000;
  543. }
  544. PageTableEntry* MemoryManager::quickmap_pt(PhysicalAddress pt_paddr)
  545. {
  546. auto& pte = boot_pd3_pde1023_pt[8];
  547. if (pte.physical_page_base() != pt_paddr.as_ptr()) {
  548. #ifdef MM_DEBUG
  549. dbgprintf("quickmap_pt: Mapping P%p at 0xffe08000 in pte @ %p\n", pt_paddr.as_ptr(), &pte);
  550. #endif
  551. pte.set_physical_page_base(pt_paddr.get());
  552. pte.set_present(true);
  553. pte.set_writable(true);
  554. pte.set_user_allowed(false);
  555. flush_tlb(VirtualAddress(0xffe08000));
  556. }
  557. return (PageTableEntry*)0xffe08000;
  558. }
  559. void MemoryManager::map_for_kernel(VirtualAddress vaddr, PhysicalAddress paddr, bool cache_disabled)
  560. {
  561. auto& pte = ensure_pte(kernel_page_directory(), vaddr);
  562. pte.set_physical_page_base(paddr.get());
  563. pte.set_present(true);
  564. pte.set_writable(true);
  565. pte.set_user_allowed(false);
  566. pte.set_cache_disabled(cache_disabled);
  567. flush_tlb(vaddr);
  568. }
  569. u8* MemoryManager::quickmap_page(PhysicalPage& physical_page)
  570. {
  571. ASSERT_INTERRUPTS_DISABLED();
  572. ASSERT(!m_quickmap_in_use);
  573. m_quickmap_in_use = true;
  574. auto& pte = boot_pd3_pde1023_pt[0];
  575. if (pte.physical_page_base() != physical_page.paddr().as_ptr()) {
  576. #ifdef MM_DEBUG
  577. dbgprintf("quickmap_page: Mapping P%p at 0xffe00000 in pte @ %p\n", physical_page.paddr().as_ptr(), &pte);
  578. #endif
  579. pte.set_physical_page_base(physical_page.paddr().get());
  580. pte.set_present(true);
  581. pte.set_writable(true);
  582. pte.set_user_allowed(false);
  583. flush_tlb(VirtualAddress(0xffe00000));
  584. }
  585. return (u8*)0xffe00000;
  586. }
  587. void MemoryManager::unquickmap_page()
  588. {
  589. ASSERT_INTERRUPTS_DISABLED();
  590. ASSERT(m_quickmap_in_use);
  591. auto& pte = boot_pd3_pde1023_pt[0];
  592. pte.set_physical_page_base(0);
  593. pte.set_present(false);
  594. flush_tlb(VirtualAddress(0xffe00000));
  595. m_quickmap_in_use = false;
  596. }
  597. template<MemoryManager::AccessSpace space, MemoryManager::AccessType access_type>
  598. bool MemoryManager::validate_range(const Process& process, VirtualAddress base_vaddr, size_t size) const
  599. {
  600. ASSERT(size);
  601. VirtualAddress vaddr = base_vaddr.page_base();
  602. VirtualAddress end_vaddr = base_vaddr.offset(size - 1).page_base();
  603. if (end_vaddr < vaddr) {
  604. dbg() << *current << " Shenanigans! Asked to validate " << base_vaddr << " size=" << size;
  605. return false;
  606. }
  607. const Region* region = nullptr;
  608. while (vaddr <= end_vaddr) {
  609. if (!region || !region->contains(vaddr)) {
  610. if (space == AccessSpace::Kernel)
  611. region = kernel_region_from_vaddr(vaddr);
  612. if (!region || !region->contains(vaddr))
  613. region = user_region_from_vaddr(const_cast<Process&>(process), vaddr);
  614. if (!region
  615. || (space == AccessSpace::User && !region->is_user_accessible())
  616. || (access_type == AccessType::Read && !region->is_readable())
  617. || (access_type == AccessType::Write && !region->is_writable())) {
  618. return false;
  619. }
  620. }
  621. vaddr = vaddr.offset(PAGE_SIZE);
  622. }
  623. return true;
  624. }
  625. bool MemoryManager::validate_user_stack(const Process& process, VirtualAddress vaddr) const
  626. {
  627. if (!is_user_address(vaddr))
  628. return false;
  629. auto* region = user_region_from_vaddr(const_cast<Process&>(process), vaddr);
  630. return region && region->is_user_accessible() && region->is_stack();
  631. }
  632. bool MemoryManager::validate_kernel_read(const Process& process, VirtualAddress vaddr, size_t size) const
  633. {
  634. return validate_range<AccessSpace::Kernel, AccessType::Read>(process, vaddr, size);
  635. }
  636. bool MemoryManager::validate_user_read(const Process& process, VirtualAddress vaddr, size_t size) const
  637. {
  638. if (!is_user_address(vaddr))
  639. return false;
  640. return validate_range<AccessSpace::User, AccessType::Read>(process, vaddr, size);
  641. }
  642. bool MemoryManager::validate_user_write(const Process& process, VirtualAddress vaddr, size_t size) const
  643. {
  644. if (!is_user_address(vaddr))
  645. return false;
  646. return validate_range<AccessSpace::User, AccessType::Write>(process, vaddr, size);
  647. }
  648. void MemoryManager::register_vmobject(VMObject& vmobject)
  649. {
  650. InterruptDisabler disabler;
  651. m_vmobjects.append(&vmobject);
  652. }
  653. void MemoryManager::unregister_vmobject(VMObject& vmobject)
  654. {
  655. InterruptDisabler disabler;
  656. m_vmobjects.remove(&vmobject);
  657. }
  658. void MemoryManager::register_region(Region& region)
  659. {
  660. InterruptDisabler disabler;
  661. if (region.vaddr().get() >= 0xc0000000)
  662. m_kernel_regions.append(&region);
  663. else
  664. m_user_regions.append(&region);
  665. }
  666. void MemoryManager::unregister_region(Region& region)
  667. {
  668. InterruptDisabler disabler;
  669. if (region.vaddr().get() >= 0xc0000000)
  670. m_kernel_regions.remove(&region);
  671. else
  672. m_user_regions.remove(&region);
  673. }
  674. ProcessPagingScope::ProcessPagingScope(Process& process)
  675. {
  676. ASSERT(current);
  677. MM.enter_process_paging_scope(process);
  678. }
  679. ProcessPagingScope::~ProcessPagingScope()
  680. {
  681. MM.enter_process_paging_scope(current->process());
  682. }