MemoryManager.cpp 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806
  1. #include "CMOS.h"
  2. #include "Process.h"
  3. #include "StdLib.h"
  4. #include <AK/Assertions.h>
  5. #include <AK/kstdio.h>
  6. #include <Kernel/Arch/i386/CPU.h>
  7. #include <Kernel/FileSystem/Inode.h>
  8. #include <Kernel/Multiboot.h>
  9. #include <Kernel/VM/MemoryManager.h>
  10. //#define MM_DEBUG
  11. //#define PAGE_FAULT_DEBUG
  12. static MemoryManager* s_the;
  13. MemoryManager& MM
  14. {
  15. return *s_the;
  16. }
  17. MemoryManager::MemoryManager()
  18. {
  19. m_kernel_page_directory = PageDirectory::create_at_fixed_address(PhysicalAddress(0x4000));
  20. m_page_table_zero = (PageTableEntry*)0x6000;
  21. m_page_table_one = (PageTableEntry*)0x7000;
  22. initialize_paging();
  23. kprintf("MM initialized.\n");
  24. }
  25. MemoryManager::~MemoryManager()
  26. {
  27. }
  28. void MemoryManager::populate_page_directory(PageDirectory& page_directory)
  29. {
  30. page_directory.m_directory_page = allocate_supervisor_physical_page();
  31. page_directory.entries()[0].copy_from({}, kernel_page_directory().entries()[0]);
  32. page_directory.entries()[1].copy_from({}, kernel_page_directory().entries()[1]);
  33. // Defer to the kernel page tables for 0xC0000000-0xFFFFFFFF
  34. for (int i = 768; i < 1024; ++i)
  35. page_directory.entries()[i].copy_from({}, kernel_page_directory().entries()[i]);
  36. }
  37. void MemoryManager::initialize_paging()
  38. {
  39. memset(m_page_table_zero, 0, PAGE_SIZE);
  40. memset(m_page_table_one, 0, PAGE_SIZE);
  41. #ifdef MM_DEBUG
  42. dbgprintf("MM: Kernel page directory @ %p\n", kernel_page_directory().cr3());
  43. #endif
  44. #ifdef MM_DEBUG
  45. dbgprintf("MM: Protect against null dereferences\n");
  46. #endif
  47. // Make null dereferences crash.
  48. map_protected(VirtualAddress(0), PAGE_SIZE);
  49. #ifdef MM_DEBUG
  50. dbgprintf("MM: Identity map bottom 5MB\n");
  51. #endif
  52. // The bottom 5 MB (except for the null page) are identity mapped & supervisor only.
  53. // Every process shares these mappings.
  54. create_identity_mapping(kernel_page_directory(), VirtualAddress(PAGE_SIZE), (5 * MB) - PAGE_SIZE);
  55. // Basic memory map:
  56. // 0 -> 512 kB Kernel code. Root page directory & PDE 0.
  57. // (last page before 1MB) Used by quickmap_page().
  58. // 1 MB -> 3 MB kmalloc_eternal() space.
  59. // 3 MB -> 4 MB kmalloc() space.
  60. // 4 MB -> 5 MB Supervisor physical pages (available for allocation!)
  61. // 5 MB -> 0xc0000000 Userspace physical pages (available for allocation!)
  62. // 0xc0000000-0xffffffff Kernel-only virtual address space
  63. #ifdef MM_DEBUG
  64. dbgprintf("MM: Quickmap will use %p\n", m_quickmap_addr.get());
  65. #endif
  66. m_quickmap_addr = VirtualAddress((1 * MB) - PAGE_SIZE);
  67. RefPtr<PhysicalRegion> region;
  68. bool region_is_super = false;
  69. for (auto* mmap = (multiboot_memory_map_t*)multiboot_info_ptr->mmap_addr; (unsigned long)mmap < multiboot_info_ptr->mmap_addr + multiboot_info_ptr->mmap_length; mmap = (multiboot_memory_map_t*)((unsigned long)mmap + mmap->size + sizeof(mmap->size))) {
  70. kprintf("MM: Multiboot mmap: base_addr = 0x%x%08x, length = 0x%x%08x, type = 0x%x\n",
  71. (u32)(mmap->addr >> 32),
  72. (u32)(mmap->addr & 0xffffffff),
  73. (u32)(mmap->len >> 32),
  74. (u32)(mmap->len & 0xffffffff),
  75. (u32)mmap->type);
  76. if (mmap->type != MULTIBOOT_MEMORY_AVAILABLE)
  77. continue;
  78. // FIXME: Maybe make use of stuff below the 1MB mark?
  79. if (mmap->addr < (1 * MB))
  80. continue;
  81. #ifdef MM_DEBUG
  82. kprintf("MM: considering memory at %p - %p\n",
  83. (u32)mmap->addr, (u32)(mmap->addr + mmap->len));
  84. #endif
  85. for (size_t page_base = mmap->addr; page_base < (mmap->addr + mmap->len); page_base += PAGE_SIZE) {
  86. auto addr = PhysicalAddress(page_base);
  87. if (page_base < 4 * MB) {
  88. // nothing
  89. } else if (page_base >= 4 * MB && page_base < 5 * MB) {
  90. if (region.is_null() || !region_is_super || region->upper().offset(PAGE_SIZE) != addr) {
  91. m_super_physical_regions.append(PhysicalRegion::create(addr, addr));
  92. region = m_super_physical_regions.last();
  93. region_is_super = true;
  94. } else {
  95. region->expand(region->lower(), addr);
  96. }
  97. } else {
  98. if (region.is_null() || region_is_super || region->upper().offset(PAGE_SIZE) != addr) {
  99. m_user_physical_regions.append(PhysicalRegion::create(addr, addr));
  100. region = &m_user_physical_regions.last();
  101. region_is_super = false;
  102. } else {
  103. region->expand(region->lower(), addr);
  104. }
  105. }
  106. }
  107. }
  108. for (auto& region : m_super_physical_regions)
  109. m_super_physical_pages += region.finalize_capacity();
  110. for (auto& region : m_user_physical_regions)
  111. m_user_physical_pages += region.finalize_capacity();
  112. #ifdef MM_DEBUG
  113. dbgprintf("MM: Installing page directory\n");
  114. #endif
  115. asm volatile("movl %%eax, %%cr3" ::"a"(kernel_page_directory().cr3()));
  116. asm volatile(
  117. "movl %%cr0, %%eax\n"
  118. "orl $0x80000001, %%eax\n"
  119. "movl %%eax, %%cr0\n" ::
  120. : "%eax", "memory");
  121. #ifdef MM_DEBUG
  122. dbgprintf("MM: Paging initialized.\n");
  123. #endif
  124. }
  125. RefPtr<PhysicalPage> MemoryManager::allocate_page_table(PageDirectory& page_directory, unsigned index)
  126. {
  127. ASSERT(!page_directory.m_physical_pages.contains(index));
  128. auto physical_page = allocate_supervisor_physical_page();
  129. if (!physical_page)
  130. return nullptr;
  131. page_directory.m_physical_pages.set(index, physical_page);
  132. return physical_page;
  133. }
  134. void MemoryManager::remove_identity_mapping(PageDirectory& page_directory, VirtualAddress vaddr, size_t size)
  135. {
  136. InterruptDisabler disabler;
  137. // FIXME: ASSERT(vaddr is 4KB aligned);
  138. for (u32 offset = 0; offset < size; offset += PAGE_SIZE) {
  139. auto pte_address = vaddr.offset(offset);
  140. auto& pte = ensure_pte(page_directory, pte_address);
  141. pte.set_physical_page_base(0);
  142. pte.set_user_allowed(false);
  143. pte.set_present(true);
  144. pte.set_writable(true);
  145. flush_tlb(pte_address);
  146. }
  147. }
  148. PageTableEntry& MemoryManager::ensure_pte(PageDirectory& page_directory, VirtualAddress vaddr)
  149. {
  150. ASSERT_INTERRUPTS_DISABLED();
  151. u32 page_directory_index = (vaddr.get() >> 22) & 0x3ff;
  152. u32 page_table_index = (vaddr.get() >> 12) & 0x3ff;
  153. PageDirectoryEntry& pde = page_directory.entries()[page_directory_index];
  154. if (!pde.is_present()) {
  155. #ifdef MM_DEBUG
  156. dbgprintf("MM: PDE %u not present (requested for L%x), allocating\n", page_directory_index, vaddr.get());
  157. #endif
  158. if (page_directory_index == 0) {
  159. ASSERT(&page_directory == m_kernel_page_directory);
  160. pde.set_page_table_base((u32)m_page_table_zero);
  161. pde.set_user_allowed(false);
  162. pde.set_present(true);
  163. pde.set_writable(true);
  164. } else if (page_directory_index == 1) {
  165. ASSERT(&page_directory == m_kernel_page_directory);
  166. pde.set_page_table_base((u32)m_page_table_one);
  167. pde.set_user_allowed(false);
  168. pde.set_present(true);
  169. pde.set_writable(true);
  170. } else {
  171. //ASSERT(&page_directory != m_kernel_page_directory.ptr());
  172. auto page_table = allocate_page_table(page_directory, page_directory_index);
  173. #ifdef MM_DEBUG
  174. dbgprintf("MM: PD K%x (%s) at P%x allocated page table #%u (for L%x) at P%x\n",
  175. &page_directory,
  176. &page_directory == m_kernel_page_directory ? "Kernel" : "User",
  177. page_directory.cr3(),
  178. page_directory_index,
  179. vaddr.get(),
  180. page_table->paddr().get());
  181. #endif
  182. pde.set_page_table_base(page_table->paddr().get());
  183. pde.set_user_allowed(true);
  184. pde.set_present(true);
  185. pde.set_writable(true);
  186. page_directory.m_physical_pages.set(page_directory_index, move(page_table));
  187. }
  188. }
  189. return pde.page_table_base()[page_table_index];
  190. }
  191. void MemoryManager::map_protected(VirtualAddress vaddr, size_t length)
  192. {
  193. InterruptDisabler disabler;
  194. ASSERT(vaddr.is_page_aligned());
  195. for (u32 offset = 0; offset < length; offset += PAGE_SIZE) {
  196. auto pte_address = vaddr.offset(offset);
  197. auto& pte = ensure_pte(kernel_page_directory(), pte_address);
  198. pte.set_physical_page_base(pte_address.get());
  199. pte.set_user_allowed(false);
  200. pte.set_present(false);
  201. pte.set_writable(false);
  202. flush_tlb(pte_address);
  203. }
  204. }
  205. void MemoryManager::create_identity_mapping(PageDirectory& page_directory, VirtualAddress vaddr, size_t size)
  206. {
  207. InterruptDisabler disabler;
  208. ASSERT((vaddr.get() & ~PAGE_MASK) == 0);
  209. for (u32 offset = 0; offset < size; offset += PAGE_SIZE) {
  210. auto pte_address = vaddr.offset(offset);
  211. auto& pte = ensure_pte(page_directory, pte_address);
  212. pte.set_physical_page_base(pte_address.get());
  213. pte.set_user_allowed(false);
  214. pte.set_present(true);
  215. pte.set_writable(true);
  216. page_directory.flush(pte_address);
  217. }
  218. }
  219. void MemoryManager::initialize()
  220. {
  221. s_the = new MemoryManager;
  222. }
  223. Region* MemoryManager::kernel_region_from_vaddr(VirtualAddress vaddr)
  224. {
  225. if (vaddr.get() < 0xc0000000)
  226. return nullptr;
  227. for (auto& region : MM.m_kernel_regions) {
  228. if (region->contains(vaddr))
  229. return region;
  230. }
  231. return nullptr;
  232. }
  233. Region* MemoryManager::user_region_from_vaddr(Process& process, VirtualAddress vaddr)
  234. {
  235. // FIXME: Use a binary search tree (maybe red/black?) or some other more appropriate data structure!
  236. for (auto& region : process.m_regions) {
  237. if (region.contains(vaddr))
  238. return &region;
  239. }
  240. dbg() << process << " Couldn't find user region for " << vaddr;
  241. return nullptr;
  242. }
  243. Region* MemoryManager::region_from_vaddr(Process& process, VirtualAddress vaddr)
  244. {
  245. ASSERT_INTERRUPTS_DISABLED();
  246. if (auto* region = kernel_region_from_vaddr(vaddr))
  247. return region;
  248. return user_region_from_vaddr(process, vaddr);
  249. }
  250. const Region* MemoryManager::region_from_vaddr(const Process& process, VirtualAddress vaddr)
  251. {
  252. if (auto* region = kernel_region_from_vaddr(vaddr))
  253. return region;
  254. return user_region_from_vaddr(const_cast<Process&>(process), vaddr);
  255. }
  256. bool MemoryManager::zero_page(Region& region, unsigned page_index_in_region)
  257. {
  258. ASSERT_INTERRUPTS_DISABLED();
  259. auto& vmo = region.vmo();
  260. auto& vmo_page = vmo.physical_pages()[region.first_page_index() + page_index_in_region];
  261. sti();
  262. LOCKER(vmo.m_paging_lock);
  263. cli();
  264. if (!vmo_page.is_null()) {
  265. #ifdef PAGE_FAULT_DEBUG
  266. dbgprintf("MM: zero_page() but page already present. Fine with me!\n");
  267. #endif
  268. remap_region_page(region, page_index_in_region);
  269. return true;
  270. }
  271. auto physical_page = allocate_user_physical_page(ShouldZeroFill::Yes);
  272. #ifdef PAGE_FAULT_DEBUG
  273. dbgprintf(" >> ZERO P%x\n", physical_page->paddr().get());
  274. #endif
  275. region.set_should_cow(page_index_in_region, false);
  276. vmo.physical_pages()[page_index_in_region] = move(physical_page);
  277. remap_region_page(region, page_index_in_region);
  278. return true;
  279. }
  280. bool MemoryManager::copy_on_write(Region& region, unsigned page_index_in_region)
  281. {
  282. ASSERT_INTERRUPTS_DISABLED();
  283. auto& vmo = region.vmo();
  284. if (vmo.physical_pages()[page_index_in_region]->ref_count() == 1) {
  285. #ifdef PAGE_FAULT_DEBUG
  286. dbgprintf(" >> It's a COW page but nobody is sharing it anymore. Remap r/w\n");
  287. #endif
  288. region.set_should_cow(page_index_in_region, false);
  289. remap_region_page(region, page_index_in_region);
  290. return true;
  291. }
  292. #ifdef PAGE_FAULT_DEBUG
  293. dbgprintf(" >> It's a COW page and it's time to COW!\n");
  294. #endif
  295. auto physical_page_to_copy = move(vmo.physical_pages()[page_index_in_region]);
  296. auto physical_page = allocate_user_physical_page(ShouldZeroFill::No);
  297. u8* dest_ptr = quickmap_page(*physical_page);
  298. const u8* src_ptr = region.vaddr().offset(page_index_in_region * PAGE_SIZE).as_ptr();
  299. #ifdef PAGE_FAULT_DEBUG
  300. dbgprintf(" >> COW P%x <- P%x\n", physical_page->paddr().get(), physical_page_to_copy->paddr().get());
  301. #endif
  302. memcpy(dest_ptr, src_ptr, PAGE_SIZE);
  303. vmo.physical_pages()[page_index_in_region] = move(physical_page);
  304. unquickmap_page();
  305. region.set_should_cow(page_index_in_region, false);
  306. remap_region_page(region, page_index_in_region);
  307. return true;
  308. }
  309. bool MemoryManager::page_in_from_inode(Region& region, unsigned page_index_in_region)
  310. {
  311. ASSERT(region.page_directory());
  312. auto& vmo = region.vmo();
  313. ASSERT(!vmo.is_anonymous());
  314. ASSERT(vmo.inode());
  315. auto& vmo_page = vmo.physical_pages()[region.first_page_index() + page_index_in_region];
  316. InterruptFlagSaver saver;
  317. sti();
  318. LOCKER(vmo.m_paging_lock);
  319. cli();
  320. if (!vmo_page.is_null()) {
  321. dbgprintf("MM: page_in_from_inode() but page already present. Fine with me!\n");
  322. remap_region_page(region, page_index_in_region);
  323. return true;
  324. }
  325. #ifdef MM_DEBUG
  326. dbgprintf("MM: page_in_from_inode ready to read from inode\n");
  327. #endif
  328. sti();
  329. u8 page_buffer[PAGE_SIZE];
  330. auto& inode = *vmo.inode();
  331. auto nread = inode.read_bytes(vmo.inode_offset() + ((region.first_page_index() + page_index_in_region) * PAGE_SIZE), PAGE_SIZE, page_buffer, nullptr);
  332. if (nread < 0) {
  333. kprintf("MM: page_in_from_inode had error (%d) while reading!\n", nread);
  334. return false;
  335. }
  336. if (nread < PAGE_SIZE) {
  337. // If we read less than a page, zero out the rest to avoid leaking uninitialized data.
  338. memset(page_buffer + nread, 0, PAGE_SIZE - nread);
  339. }
  340. cli();
  341. vmo_page = allocate_user_physical_page(ShouldZeroFill::No);
  342. if (vmo_page.is_null()) {
  343. kprintf("MM: page_in_from_inode was unable to allocate a physical page\n");
  344. return false;
  345. }
  346. remap_region_page(region, page_index_in_region);
  347. u8* dest_ptr = region.vaddr().offset(page_index_in_region * PAGE_SIZE).as_ptr();
  348. memcpy(dest_ptr, page_buffer, PAGE_SIZE);
  349. return true;
  350. }
  351. Region* MemoryManager::region_from_vaddr(VirtualAddress vaddr)
  352. {
  353. if (auto* region = kernel_region_from_vaddr(vaddr))
  354. return region;
  355. auto page_directory = PageDirectory::find_by_pdb(cpu_cr3());
  356. if (!page_directory)
  357. return nullptr;
  358. ASSERT(page_directory->process());
  359. return user_region_from_vaddr(*page_directory->process(), vaddr);
  360. }
  361. PageFaultResponse MemoryManager::handle_page_fault(const PageFault& fault)
  362. {
  363. ASSERT_INTERRUPTS_DISABLED();
  364. ASSERT(current);
  365. #ifdef PAGE_FAULT_DEBUG
  366. dbgprintf("MM: handle_page_fault(%w) at L%x\n", fault.code(), fault.vaddr().get());
  367. #endif
  368. ASSERT(fault.vaddr() != m_quickmap_addr);
  369. if (fault.type() == PageFault::Type::PageNotPresent && fault.vaddr().get() >= 0xc0000000) {
  370. auto* current_page_directory = reinterpret_cast<PageDirectoryEntry*>(cpu_cr3());
  371. u32 page_directory_index = (fault.vaddr().get() >> 22) & 0x3ff;
  372. auto& kernel_pde = kernel_page_directory().entries()[page_directory_index];
  373. auto& current_pde = current_page_directory[page_directory_index];
  374. if (kernel_pde.is_present() && !current_pde.is_present()) {
  375. dbg() << "NP(kernel): Copying new kernel mapping for " << fault.vaddr() << " into current page directory";
  376. current_pde.copy_from({}, kernel_pde);
  377. flush_tlb(fault.vaddr().page_base());
  378. return PageFaultResponse::Continue;
  379. }
  380. }
  381. auto* region = region_from_vaddr(fault.vaddr());
  382. if (!region) {
  383. kprintf("NP(error) fault at invalid address L%x\n", fault.vaddr().get());
  384. return PageFaultResponse::ShouldCrash;
  385. }
  386. auto page_index_in_region = region->page_index_from_address(fault.vaddr());
  387. if (fault.type() == PageFault::Type::PageNotPresent) {
  388. if (region->vmo().inode()) {
  389. #ifdef PAGE_FAULT_DEBUG
  390. dbgprintf("NP(inode) fault in Region{%p}[%u]\n", region, page_index_in_region);
  391. #endif
  392. page_in_from_inode(*region, page_index_in_region);
  393. return PageFaultResponse::Continue;
  394. }
  395. #ifdef PAGE_FAULT_DEBUG
  396. dbgprintf("NP(zero) fault in Region{%p}[%u]\n", region, page_index_in_region);
  397. #endif
  398. zero_page(*region, page_index_in_region);
  399. return PageFaultResponse::Continue;
  400. }
  401. ASSERT(fault.type() == PageFault::Type::ProtectionViolation);
  402. if (fault.access() == PageFault::Access::Write && region->should_cow(page_index_in_region)) {
  403. #ifdef PAGE_FAULT_DEBUG
  404. dbgprintf("PV(cow) fault in Region{%p}[%u]\n", region, page_index_in_region);
  405. #endif
  406. bool success = copy_on_write(*region, page_index_in_region);
  407. ASSERT(success);
  408. return PageFaultResponse::Continue;
  409. }
  410. kprintf("PV(error) fault in Region{%p}[%u] at L%x\n", region, page_index_in_region, fault.vaddr().get());
  411. return PageFaultResponse::ShouldCrash;
  412. }
  413. RefPtr<Region> MemoryManager::allocate_kernel_region(size_t size, const StringView& name, bool user_accessible)
  414. {
  415. InterruptDisabler disabler;
  416. ASSERT(!(size % PAGE_SIZE));
  417. auto range = kernel_page_directory().range_allocator().allocate_anywhere(size);
  418. ASSERT(range.is_valid());
  419. RefPtr<Region> region;
  420. if (user_accessible)
  421. region = Region::create_user_accessible(range, name, PROT_READ | PROT_WRITE | PROT_EXEC, false);
  422. else
  423. region = Region::create_kernel_only(range, name, PROT_READ | PROT_WRITE | PROT_EXEC, false);
  424. MM.map_region_at_address(*m_kernel_page_directory, *region, range.base());
  425. // FIXME: It would be cool if these could zero-fill on demand instead.
  426. region->commit();
  427. return region;
  428. }
  429. RefPtr<Region> MemoryManager::allocate_user_accessible_kernel_region(size_t size, const StringView& name)
  430. {
  431. return allocate_kernel_region(size, name, true);
  432. }
  433. void MemoryManager::deallocate_user_physical_page(PhysicalPage&& page)
  434. {
  435. for (auto& region : m_user_physical_regions) {
  436. if (!region.contains(page)) {
  437. kprintf(
  438. "MM: deallocate_user_physical_page: %p not in %p -> %p\n",
  439. page.paddr(), region.lower().get(), region.upper().get());
  440. continue;
  441. }
  442. region.return_page(move(page));
  443. --m_user_physical_pages_used;
  444. return;
  445. }
  446. kprintf("MM: deallocate_user_physical_page couldn't figure out region for user page @ %p\n", page.paddr());
  447. ASSERT_NOT_REACHED();
  448. }
  449. RefPtr<PhysicalPage> MemoryManager::allocate_user_physical_page(ShouldZeroFill should_zero_fill)
  450. {
  451. InterruptDisabler disabler;
  452. RefPtr<PhysicalPage> page;
  453. for (auto& region : m_user_physical_regions) {
  454. page = region.take_free_page(false);
  455. if (page.is_null())
  456. continue;
  457. }
  458. if (!page) {
  459. if (m_user_physical_regions.is_empty()) {
  460. kprintf("MM: no user physical regions available (?)\n");
  461. }
  462. kprintf("MM: no user physical pages available\n");
  463. ASSERT_NOT_REACHED();
  464. return {};
  465. }
  466. #ifdef MM_DEBUG
  467. dbgprintf("MM: allocate_user_physical_page vending P%p\n", page->paddr().get());
  468. #endif
  469. if (should_zero_fill == ShouldZeroFill::Yes) {
  470. auto* ptr = (u32*)quickmap_page(*page);
  471. fast_u32_fill(ptr, 0, PAGE_SIZE / sizeof(u32));
  472. unquickmap_page();
  473. }
  474. ++m_user_physical_pages_used;
  475. return page;
  476. }
  477. void MemoryManager::deallocate_supervisor_physical_page(PhysicalPage&& page)
  478. {
  479. for (auto& region : m_super_physical_regions) {
  480. if (!region.contains(page)) {
  481. kprintf(
  482. "MM: deallocate_supervisor_physical_page: %p not in %p -> %p\n",
  483. page.paddr(), region.lower().get(), region.upper().get());
  484. continue;
  485. }
  486. region.return_page(move(page));
  487. --m_super_physical_pages_used;
  488. return;
  489. }
  490. kprintf("MM: deallocate_supervisor_physical_page couldn't figure out region for super page @ %p\n", page.paddr());
  491. ASSERT_NOT_REACHED();
  492. }
  493. RefPtr<PhysicalPage> MemoryManager::allocate_supervisor_physical_page()
  494. {
  495. InterruptDisabler disabler;
  496. RefPtr<PhysicalPage> page;
  497. for (auto& region : m_super_physical_regions) {
  498. page = region.take_free_page(true);
  499. if (page.is_null())
  500. continue;
  501. }
  502. if (!page) {
  503. if (m_super_physical_regions.is_empty()) {
  504. kprintf("MM: no super physical regions available (?)\n");
  505. }
  506. kprintf("MM: no super physical pages available\n");
  507. ASSERT_NOT_REACHED();
  508. return {};
  509. }
  510. #ifdef MM_DEBUG
  511. dbgprintf("MM: allocate_supervisor_physical_page vending P%p\n", page->paddr().get());
  512. #endif
  513. fast_u32_fill((u32*)page->paddr().as_ptr(), 0, PAGE_SIZE / sizeof(u32));
  514. ++m_super_physical_pages_used;
  515. return page;
  516. }
  517. void MemoryManager::enter_process_paging_scope(Process& process)
  518. {
  519. ASSERT(current);
  520. InterruptDisabler disabler;
  521. current->tss().cr3 = process.page_directory().cr3();
  522. asm volatile("movl %%eax, %%cr3" ::"a"(process.page_directory().cr3())
  523. : "memory");
  524. }
  525. void MemoryManager::flush_entire_tlb()
  526. {
  527. asm volatile(
  528. "mov %%cr3, %%eax\n"
  529. "mov %%eax, %%cr3\n" ::
  530. : "%eax", "memory");
  531. }
  532. void MemoryManager::flush_tlb(VirtualAddress vaddr)
  533. {
  534. asm volatile("invlpg %0"
  535. :
  536. : "m"(*(char*)vaddr.get())
  537. : "memory");
  538. }
  539. void MemoryManager::map_for_kernel(VirtualAddress vaddr, PhysicalAddress paddr)
  540. {
  541. auto& pte = ensure_pte(kernel_page_directory(), vaddr);
  542. pte.set_physical_page_base(paddr.get());
  543. pte.set_present(true);
  544. pte.set_writable(true);
  545. pte.set_user_allowed(false);
  546. flush_tlb(vaddr);
  547. }
  548. u8* MemoryManager::quickmap_page(PhysicalPage& physical_page)
  549. {
  550. ASSERT_INTERRUPTS_DISABLED();
  551. ASSERT(!m_quickmap_in_use);
  552. m_quickmap_in_use = true;
  553. auto page_vaddr = m_quickmap_addr;
  554. auto& pte = ensure_pte(kernel_page_directory(), page_vaddr);
  555. pte.set_physical_page_base(physical_page.paddr().get());
  556. pte.set_present(true);
  557. pte.set_writable(true);
  558. pte.set_user_allowed(false);
  559. flush_tlb(page_vaddr);
  560. ASSERT((u32)pte.physical_page_base() == physical_page.paddr().get());
  561. #ifdef MM_DEBUG
  562. dbgprintf("MM: >> quickmap_page L%x => P%x @ PTE=%p\n", page_vaddr, physical_page.paddr().get(), pte.ptr());
  563. #endif
  564. return page_vaddr.as_ptr();
  565. }
  566. void MemoryManager::unquickmap_page()
  567. {
  568. ASSERT_INTERRUPTS_DISABLED();
  569. ASSERT(m_quickmap_in_use);
  570. auto page_vaddr = m_quickmap_addr;
  571. auto& pte = ensure_pte(kernel_page_directory(), page_vaddr);
  572. #ifdef MM_DEBUG
  573. auto old_physical_address = pte.physical_page_base();
  574. #endif
  575. pte.set_physical_page_base(0);
  576. pte.set_present(false);
  577. pte.set_writable(false);
  578. flush_tlb(page_vaddr);
  579. #ifdef MM_DEBUG
  580. dbgprintf("MM: >> unquickmap_page L%x =/> P%x\n", page_vaddr, old_physical_address);
  581. #endif
  582. m_quickmap_in_use = false;
  583. }
  584. void MemoryManager::remap_region_page(Region& region, unsigned page_index_in_region)
  585. {
  586. ASSERT(region.page_directory());
  587. InterruptDisabler disabler;
  588. auto page_vaddr = region.vaddr().offset(page_index_in_region * PAGE_SIZE);
  589. auto& pte = ensure_pte(*region.page_directory(), page_vaddr);
  590. auto& physical_page = region.vmo().physical_pages()[page_index_in_region];
  591. ASSERT(physical_page);
  592. pte.set_physical_page_base(physical_page->paddr().get());
  593. pte.set_present(true); // FIXME: Maybe we should use the is_readable flag here?
  594. if (region.should_cow(page_index_in_region))
  595. pte.set_writable(false);
  596. else
  597. pte.set_writable(region.is_writable());
  598. pte.set_cache_disabled(!region.vmo().m_allow_cpu_caching);
  599. pte.set_write_through(!region.vmo().m_allow_cpu_caching);
  600. pte.set_user_allowed(region.is_user_accessible());
  601. region.page_directory()->flush(page_vaddr);
  602. #ifdef MM_DEBUG
  603. dbgprintf("MM: >> remap_region_page (PD=%x, PTE=P%x) '%s' L%x => P%x (@%p)\n", region.page_directory()->cr3(), pte.ptr(), region.name().characters(), page_vaddr.get(), physical_page->paddr().get(), physical_page.ptr());
  604. #endif
  605. }
  606. void MemoryManager::remap_region(PageDirectory& page_directory, Region& region)
  607. {
  608. InterruptDisabler disabler;
  609. ASSERT(region.page_directory() == &page_directory);
  610. map_region_at_address(page_directory, region, region.vaddr());
  611. }
  612. void MemoryManager::map_region_at_address(PageDirectory& page_directory, Region& region, VirtualAddress vaddr)
  613. {
  614. InterruptDisabler disabler;
  615. region.set_page_directory(page_directory);
  616. auto& vmo = region.vmo();
  617. #ifdef MM_DEBUG
  618. dbgprintf("MM: map_region_at_address will map VMO pages %u - %u (VMO page count: %u)\n", region.first_page_index(), region.last_page_index(), vmo.page_count());
  619. #endif
  620. for (size_t i = 0; i < region.page_count(); ++i) {
  621. auto page_vaddr = vaddr.offset(i * PAGE_SIZE);
  622. auto& pte = ensure_pte(page_directory, page_vaddr);
  623. auto& physical_page = vmo.physical_pages()[region.first_page_index() + i];
  624. if (physical_page) {
  625. pte.set_physical_page_base(physical_page->paddr().get());
  626. pte.set_present(true); // FIXME: Maybe we should use the is_readable flag here?
  627. // FIXME: It seems wrong that the *region* cow map is essentially using *VMO* relative indices.
  628. if (region.should_cow(region.first_page_index() + i))
  629. pte.set_writable(false);
  630. else
  631. pte.set_writable(region.is_writable());
  632. pte.set_cache_disabled(!region.vmo().m_allow_cpu_caching);
  633. pte.set_write_through(!region.vmo().m_allow_cpu_caching);
  634. } else {
  635. pte.set_physical_page_base(0);
  636. pte.set_present(false);
  637. pte.set_writable(region.is_writable());
  638. }
  639. pte.set_user_allowed(region.is_user_accessible());
  640. page_directory.flush(page_vaddr);
  641. #ifdef MM_DEBUG
  642. dbgprintf("MM: >> map_region_at_address (PD=%x) '%s' L%x => P%x (@%p)\n", &page_directory, region.name().characters(), page_vaddr, physical_page ? physical_page->paddr().get() : 0, physical_page.ptr());
  643. #endif
  644. }
  645. }
  646. bool MemoryManager::unmap_region(Region& region)
  647. {
  648. ASSERT(region.page_directory());
  649. InterruptDisabler disabler;
  650. for (size_t i = 0; i < region.page_count(); ++i) {
  651. auto vaddr = region.vaddr().offset(i * PAGE_SIZE);
  652. auto& pte = ensure_pte(*region.page_directory(), vaddr);
  653. pte.set_physical_page_base(0);
  654. pte.set_present(false);
  655. pte.set_writable(false);
  656. pte.set_user_allowed(false);
  657. region.page_directory()->flush(vaddr);
  658. #ifdef MM_DEBUG
  659. auto& physical_page = region.vmo().physical_pages()[region.first_page_index() + i];
  660. dbgprintf("MM: >> Unmapped L%x => P%x <<\n", vaddr, physical_page ? physical_page->paddr().get() : 0);
  661. #endif
  662. }
  663. region.release_page_directory();
  664. return true;
  665. }
  666. bool MemoryManager::map_region(Process& process, Region& region)
  667. {
  668. map_region_at_address(process.page_directory(), region, region.vaddr());
  669. return true;
  670. }
  671. bool MemoryManager::validate_user_read(const Process& process, VirtualAddress vaddr) const
  672. {
  673. auto* region = region_from_vaddr(process, vaddr);
  674. return region && region->is_readable();
  675. }
  676. bool MemoryManager::validate_user_write(const Process& process, VirtualAddress vaddr) const
  677. {
  678. auto* region = region_from_vaddr(process, vaddr);
  679. return region && region->is_writable();
  680. }
  681. void MemoryManager::register_vmo(VMObject& vmo)
  682. {
  683. InterruptDisabler disabler;
  684. m_vmos.set(&vmo);
  685. }
  686. void MemoryManager::unregister_vmo(VMObject& vmo)
  687. {
  688. InterruptDisabler disabler;
  689. m_vmos.remove(&vmo);
  690. }
  691. void MemoryManager::register_region(Region& region)
  692. {
  693. InterruptDisabler disabler;
  694. if (region.vaddr().get() >= 0xc0000000)
  695. m_kernel_regions.set(&region);
  696. else
  697. m_user_regions.set(&region);
  698. }
  699. void MemoryManager::unregister_region(Region& region)
  700. {
  701. InterruptDisabler disabler;
  702. if (region.vaddr().get() >= 0xc0000000)
  703. m_kernel_regions.remove(&region);
  704. else
  705. m_user_regions.remove(&region);
  706. }
  707. ProcessPagingScope::ProcessPagingScope(Process& process)
  708. {
  709. ASSERT(current);
  710. MM.enter_process_paging_scope(process);
  711. }
  712. ProcessPagingScope::~ProcessPagingScope()
  713. {
  714. MM.enter_process_paging_scope(current->process());
  715. }