MemoryManager.cpp 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767
  1. #include "CMOS.h"
  2. #include "Process.h"
  3. #include "StdLib.h"
  4. #include <AK/Assertions.h>
  5. #include <AK/kstdio.h>
  6. #include <Kernel/Arch/i386/CPU.h>
  7. #include <Kernel/FileSystem/Inode.h>
  8. #include <Kernel/Multiboot.h>
  9. #include <Kernel/VM/AnonymousVMObject.h>
  10. #include <Kernel/VM/InodeVMObject.h>
  11. #include <Kernel/VM/MemoryManager.h>
  12. //#define MM_DEBUG
  13. //#define PAGE_FAULT_DEBUG
  14. static MemoryManager* s_the;
  15. MemoryManager& MM
  16. {
  17. return *s_the;
  18. }
  19. MemoryManager::MemoryManager()
  20. {
  21. m_kernel_page_directory = PageDirectory::create_at_fixed_address(PhysicalAddress(0x4000));
  22. m_page_table_zero = (PageTableEntry*)0x6000;
  23. m_page_table_one = (PageTableEntry*)0x7000;
  24. initialize_paging();
  25. kprintf("MM initialized.\n");
  26. }
  27. MemoryManager::~MemoryManager()
  28. {
  29. }
  30. void MemoryManager::populate_page_directory(PageDirectory& page_directory)
  31. {
  32. page_directory.m_directory_page = allocate_supervisor_physical_page();
  33. page_directory.entries()[0].copy_from({}, kernel_page_directory().entries()[0]);
  34. page_directory.entries()[1].copy_from({}, kernel_page_directory().entries()[1]);
  35. // Defer to the kernel page tables for 0xC0000000-0xFFFFFFFF
  36. for (int i = 768; i < 1024; ++i)
  37. page_directory.entries()[i].copy_from({}, kernel_page_directory().entries()[i]);
  38. }
  39. void MemoryManager::initialize_paging()
  40. {
  41. memset(m_page_table_zero, 0, PAGE_SIZE);
  42. memset(m_page_table_one, 0, PAGE_SIZE);
  43. #ifdef MM_DEBUG
  44. dbgprintf("MM: Kernel page directory @ %p\n", kernel_page_directory().cr3());
  45. #endif
  46. #ifdef MM_DEBUG
  47. dbgprintf("MM: Protect against null dereferences\n");
  48. #endif
  49. // Make null dereferences crash.
  50. map_protected(VirtualAddress(0), PAGE_SIZE);
  51. #ifdef MM_DEBUG
  52. dbgprintf("MM: Identity map bottom 5MB\n");
  53. #endif
  54. // The bottom 5 MB (except for the null page) are identity mapped & supervisor only.
  55. // Every process shares these mappings.
  56. create_identity_mapping(kernel_page_directory(), VirtualAddress(PAGE_SIZE), (5 * MB) - PAGE_SIZE);
  57. // Basic memory map:
  58. // 0 -> 512 kB Kernel code. Root page directory & PDE 0.
  59. // (last page before 1MB) Used by quickmap_page().
  60. // 1 MB -> 3 MB kmalloc_eternal() space.
  61. // 3 MB -> 4 MB kmalloc() space.
  62. // 4 MB -> 5 MB Supervisor physical pages (available for allocation!)
  63. // 5 MB -> 0xc0000000 Userspace physical pages (available for allocation!)
  64. // 0xc0000000-0xffffffff Kernel-only virtual address space
  65. #ifdef MM_DEBUG
  66. dbgprintf("MM: Quickmap will use %p\n", m_quickmap_addr.get());
  67. #endif
  68. m_quickmap_addr = VirtualAddress((1 * MB) - PAGE_SIZE);
  69. RefPtr<PhysicalRegion> region;
  70. bool region_is_super = false;
  71. for (auto* mmap = (multiboot_memory_map_t*)multiboot_info_ptr->mmap_addr; (unsigned long)mmap < multiboot_info_ptr->mmap_addr + multiboot_info_ptr->mmap_length; mmap = (multiboot_memory_map_t*)((unsigned long)mmap + mmap->size + sizeof(mmap->size))) {
  72. kprintf("MM: Multiboot mmap: base_addr = 0x%x%08x, length = 0x%x%08x, type = 0x%x\n",
  73. (u32)(mmap->addr >> 32),
  74. (u32)(mmap->addr & 0xffffffff),
  75. (u32)(mmap->len >> 32),
  76. (u32)(mmap->len & 0xffffffff),
  77. (u32)mmap->type);
  78. if (mmap->type != MULTIBOOT_MEMORY_AVAILABLE)
  79. continue;
  80. // FIXME: Maybe make use of stuff below the 1MB mark?
  81. if (mmap->addr < (1 * MB))
  82. continue;
  83. if ((mmap->addr + mmap->len) > 0xffffffff)
  84. continue;
  85. auto diff = (u32)mmap->addr % PAGE_SIZE;
  86. if (diff != 0) {
  87. kprintf("MM: got an unaligned region base from the bootloader; correcting %p by %d bytes\n", mmap->addr, diff);
  88. diff = PAGE_SIZE - diff;
  89. mmap->addr += diff;
  90. mmap->len -= diff;
  91. }
  92. if ((mmap->len % PAGE_SIZE) != 0) {
  93. kprintf("MM: got an unaligned region length from the bootloader; correcting %d by %d bytes\n", mmap->len, mmap->len % PAGE_SIZE);
  94. mmap->len -= mmap->len % PAGE_SIZE;
  95. }
  96. if (mmap->len < PAGE_SIZE) {
  97. kprintf("MM: memory region from bootloader is too small; we want >= %d bytes, but got %d bytes\n", PAGE_SIZE, mmap->len);
  98. continue;
  99. }
  100. #ifdef MM_DEBUG
  101. kprintf("MM: considering memory at %p - %p\n",
  102. (u32)mmap->addr, (u32)(mmap->addr + mmap->len));
  103. #endif
  104. for (size_t page_base = mmap->addr; page_base < (mmap->addr + mmap->len); page_base += PAGE_SIZE) {
  105. auto addr = PhysicalAddress(page_base);
  106. if (page_base < 4 * MB) {
  107. // nothing
  108. } else if (page_base >= 4 * MB && page_base < 5 * MB) {
  109. if (region.is_null() || !region_is_super || region->upper().offset(PAGE_SIZE) != addr) {
  110. m_super_physical_regions.append(PhysicalRegion::create(addr, addr));
  111. region = m_super_physical_regions.last();
  112. region_is_super = true;
  113. } else {
  114. region->expand(region->lower(), addr);
  115. }
  116. } else {
  117. if (region.is_null() || region_is_super || region->upper().offset(PAGE_SIZE) != addr) {
  118. m_user_physical_regions.append(PhysicalRegion::create(addr, addr));
  119. region = &m_user_physical_regions.last();
  120. region_is_super = false;
  121. } else {
  122. region->expand(region->lower(), addr);
  123. }
  124. }
  125. }
  126. }
  127. for (auto& region : m_super_physical_regions)
  128. m_super_physical_pages += region.finalize_capacity();
  129. for (auto& region : m_user_physical_regions)
  130. m_user_physical_pages += region.finalize_capacity();
  131. #ifdef MM_DEBUG
  132. dbgprintf("MM: Installing page directory\n");
  133. #endif
  134. asm volatile("movl %%eax, %%cr3" ::"a"(kernel_page_directory().cr3()));
  135. asm volatile(
  136. "movl %%cr0, %%eax\n"
  137. "orl $0x80000001, %%eax\n"
  138. "movl %%eax, %%cr0\n" ::
  139. : "%eax", "memory");
  140. #ifdef MM_DEBUG
  141. dbgprintf("MM: Paging initialized.\n");
  142. #endif
  143. }
  144. PageTableEntry& MemoryManager::ensure_pte(PageDirectory& page_directory, VirtualAddress vaddr)
  145. {
  146. ASSERT_INTERRUPTS_DISABLED();
  147. u32 page_directory_index = (vaddr.get() >> 22) & 0x3ff;
  148. u32 page_table_index = (vaddr.get() >> 12) & 0x3ff;
  149. PageDirectoryEntry& pde = page_directory.entries()[page_directory_index];
  150. if (!pde.is_present()) {
  151. #ifdef MM_DEBUG
  152. dbgprintf("MM: PDE %u not present (requested for V%p), allocating\n", page_directory_index, vaddr.get());
  153. #endif
  154. if (page_directory_index == 0) {
  155. ASSERT(&page_directory == m_kernel_page_directory);
  156. pde.set_page_table_base((u32)m_page_table_zero);
  157. pde.set_user_allowed(false);
  158. pde.set_present(true);
  159. pde.set_writable(true);
  160. } else if (page_directory_index == 1) {
  161. ASSERT(&page_directory == m_kernel_page_directory);
  162. pde.set_page_table_base((u32)m_page_table_one);
  163. pde.set_user_allowed(false);
  164. pde.set_present(true);
  165. pde.set_writable(true);
  166. } else {
  167. //ASSERT(&page_directory != m_kernel_page_directory.ptr());
  168. auto page_table = allocate_supervisor_physical_page();
  169. #ifdef MM_DEBUG
  170. dbgprintf("MM: PD K%p (%s) at P%p allocated page table #%u (for V%p) at P%p\n",
  171. &page_directory,
  172. &page_directory == m_kernel_page_directory ? "Kernel" : "User",
  173. page_directory.cr3(),
  174. page_directory_index,
  175. vaddr.get(),
  176. page_table->paddr().get());
  177. #endif
  178. pde.set_page_table_base(page_table->paddr().get());
  179. pde.set_user_allowed(true);
  180. pde.set_present(true);
  181. pde.set_writable(true);
  182. page_directory.m_physical_pages.set(page_directory_index, move(page_table));
  183. }
  184. }
  185. return pde.page_table_base()[page_table_index];
  186. }
  187. void MemoryManager::map_protected(VirtualAddress vaddr, size_t length)
  188. {
  189. InterruptDisabler disabler;
  190. ASSERT(vaddr.is_page_aligned());
  191. for (u32 offset = 0; offset < length; offset += PAGE_SIZE) {
  192. auto pte_address = vaddr.offset(offset);
  193. auto& pte = ensure_pte(kernel_page_directory(), pte_address);
  194. pte.set_physical_page_base(pte_address.get());
  195. pte.set_user_allowed(false);
  196. pte.set_present(false);
  197. pte.set_writable(false);
  198. flush_tlb(pte_address);
  199. }
  200. }
  201. void MemoryManager::create_identity_mapping(PageDirectory& page_directory, VirtualAddress vaddr, size_t size)
  202. {
  203. InterruptDisabler disabler;
  204. ASSERT((vaddr.get() & ~PAGE_MASK) == 0);
  205. for (u32 offset = 0; offset < size; offset += PAGE_SIZE) {
  206. auto pte_address = vaddr.offset(offset);
  207. auto& pte = ensure_pte(page_directory, pte_address);
  208. pte.set_physical_page_base(pte_address.get());
  209. pte.set_user_allowed(false);
  210. pte.set_present(true);
  211. pte.set_writable(true);
  212. page_directory.flush(pte_address);
  213. }
  214. }
  215. void MemoryManager::initialize()
  216. {
  217. s_the = new MemoryManager;
  218. }
  219. Region* MemoryManager::kernel_region_from_vaddr(VirtualAddress vaddr)
  220. {
  221. if (vaddr.get() < 0xc0000000)
  222. return nullptr;
  223. for (auto& region : MM.m_kernel_regions) {
  224. if (region.contains(vaddr))
  225. return &region;
  226. }
  227. return nullptr;
  228. }
  229. Region* MemoryManager::user_region_from_vaddr(Process& process, VirtualAddress vaddr)
  230. {
  231. // FIXME: Use a binary search tree (maybe red/black?) or some other more appropriate data structure!
  232. for (auto& region : process.m_regions) {
  233. if (region.contains(vaddr))
  234. return &region;
  235. }
  236. dbg() << process << " Couldn't find user region for " << vaddr;
  237. return nullptr;
  238. }
  239. Region* MemoryManager::region_from_vaddr(Process& process, VirtualAddress vaddr)
  240. {
  241. ASSERT_INTERRUPTS_DISABLED();
  242. if (auto* region = kernel_region_from_vaddr(vaddr))
  243. return region;
  244. return user_region_from_vaddr(process, vaddr);
  245. }
  246. const Region* MemoryManager::region_from_vaddr(const Process& process, VirtualAddress vaddr)
  247. {
  248. if (auto* region = kernel_region_from_vaddr(vaddr))
  249. return region;
  250. return user_region_from_vaddr(const_cast<Process&>(process), vaddr);
  251. }
  252. bool MemoryManager::zero_page(Region& region, unsigned page_index_in_region)
  253. {
  254. ASSERT_INTERRUPTS_DISABLED();
  255. ASSERT(region.vmobject().is_anonymous());
  256. auto& vmobject = region.vmobject();
  257. auto& vmobject_physical_page_entry = vmobject.physical_pages()[region.first_page_index() + page_index_in_region];
  258. // NOTE: We don't need to acquire the VMObject's lock.
  259. // This function is already exclusive due to interrupts being blocked.
  260. if (!vmobject_physical_page_entry.is_null()) {
  261. #ifdef PAGE_FAULT_DEBUG
  262. dbgprintf("MM: zero_page() but page already present. Fine with me!\n");
  263. #endif
  264. region.remap_page(page_index_in_region);
  265. return true;
  266. }
  267. if (current)
  268. current->process().did_zero_fault();
  269. auto physical_page = allocate_user_physical_page(ShouldZeroFill::Yes);
  270. #ifdef PAGE_FAULT_DEBUG
  271. dbgprintf(" >> ZERO P%p\n", physical_page->paddr().get());
  272. #endif
  273. vmobject_physical_page_entry = move(physical_page);
  274. region.remap_page(page_index_in_region);
  275. return true;
  276. }
  277. bool MemoryManager::copy_on_write(Region& region, unsigned page_index_in_region)
  278. {
  279. ASSERT_INTERRUPTS_DISABLED();
  280. auto& vmobject = region.vmobject();
  281. auto& vmobject_physical_page_entry = vmobject.physical_pages()[region.first_page_index() + page_index_in_region];
  282. if (vmobject_physical_page_entry->ref_count() == 1) {
  283. #ifdef PAGE_FAULT_DEBUG
  284. dbgprintf(" >> It's a COW page but nobody is sharing it anymore. Remap r/w\n");
  285. #endif
  286. region.set_should_cow(page_index_in_region, false);
  287. region.remap_page(page_index_in_region);
  288. return true;
  289. }
  290. if (current)
  291. current->process().did_cow_fault();
  292. #ifdef PAGE_FAULT_DEBUG
  293. dbgprintf(" >> It's a COW page and it's time to COW!\n");
  294. #endif
  295. auto physical_page_to_copy = move(vmobject_physical_page_entry);
  296. auto physical_page = allocate_user_physical_page(ShouldZeroFill::No);
  297. u8* dest_ptr = quickmap_page(*physical_page);
  298. const u8* src_ptr = region.vaddr().offset(page_index_in_region * PAGE_SIZE).as_ptr();
  299. #ifdef PAGE_FAULT_DEBUG
  300. dbgprintf(" >> COW P%p <- P%p\n", physical_page->paddr().get(), physical_page_to_copy->paddr().get());
  301. #endif
  302. memcpy(dest_ptr, src_ptr, PAGE_SIZE);
  303. vmobject_physical_page_entry = move(physical_page);
  304. unquickmap_page();
  305. region.set_should_cow(page_index_in_region, false);
  306. region.remap_page(page_index_in_region);
  307. return true;
  308. }
  309. bool MemoryManager::page_in_from_inode(Region& region, unsigned page_index_in_region)
  310. {
  311. ASSERT(region.page_directory());
  312. ASSERT(region.vmobject().is_inode());
  313. auto& vmobject = region.vmobject();
  314. auto& inode_vmobject = static_cast<InodeVMObject&>(vmobject);
  315. auto& vmobject_physical_page_entry = inode_vmobject.physical_pages()[region.first_page_index() + page_index_in_region];
  316. InterruptFlagSaver saver;
  317. sti();
  318. LOCKER(vmobject.m_paging_lock);
  319. cli();
  320. if (!vmobject_physical_page_entry.is_null()) {
  321. #ifdef PAGE_FAULT_DEBUG
  322. dbgprintf("MM: page_in_from_inode() but page already present. Fine with me!\n");
  323. #endif
  324. region.remap_page(page_index_in_region);
  325. return true;
  326. }
  327. if (current)
  328. current->process().did_inode_fault();
  329. #ifdef MM_DEBUG
  330. dbgprintf("MM: page_in_from_inode ready to read from inode\n");
  331. #endif
  332. sti();
  333. u8 page_buffer[PAGE_SIZE];
  334. auto& inode = inode_vmobject.inode();
  335. auto nread = inode.read_bytes((region.first_page_index() + page_index_in_region) * PAGE_SIZE, PAGE_SIZE, page_buffer, nullptr);
  336. if (nread < 0) {
  337. kprintf("MM: page_in_from_inode had error (%d) while reading!\n", nread);
  338. return false;
  339. }
  340. if (nread < PAGE_SIZE) {
  341. // If we read less than a page, zero out the rest to avoid leaking uninitialized data.
  342. memset(page_buffer + nread, 0, PAGE_SIZE - nread);
  343. }
  344. cli();
  345. vmobject_physical_page_entry = allocate_user_physical_page(ShouldZeroFill::No);
  346. if (vmobject_physical_page_entry.is_null()) {
  347. kprintf("MM: page_in_from_inode was unable to allocate a physical page\n");
  348. return false;
  349. }
  350. region.remap_page(page_index_in_region);
  351. u8* dest_ptr = region.vaddr().offset(page_index_in_region * PAGE_SIZE).as_ptr();
  352. memcpy(dest_ptr, page_buffer, PAGE_SIZE);
  353. return true;
  354. }
  355. Region* MemoryManager::region_from_vaddr(VirtualAddress vaddr)
  356. {
  357. if (auto* region = kernel_region_from_vaddr(vaddr))
  358. return region;
  359. auto page_directory = PageDirectory::find_by_pdb(cpu_cr3());
  360. if (!page_directory)
  361. return nullptr;
  362. ASSERT(page_directory->process());
  363. return user_region_from_vaddr(*page_directory->process(), vaddr);
  364. }
  365. PageFaultResponse MemoryManager::handle_page_fault(const PageFault& fault)
  366. {
  367. ASSERT_INTERRUPTS_DISABLED();
  368. ASSERT(current);
  369. #ifdef PAGE_FAULT_DEBUG
  370. dbgprintf("MM: handle_page_fault(%w) at V%p\n", fault.code(), fault.vaddr().get());
  371. #endif
  372. ASSERT(fault.vaddr() != m_quickmap_addr);
  373. if (fault.type() == PageFault::Type::PageNotPresent && fault.vaddr().get() >= 0xc0000000) {
  374. auto* current_page_directory = reinterpret_cast<PageDirectoryEntry*>(cpu_cr3());
  375. u32 page_directory_index = (fault.vaddr().get() >> 22) & 0x3ff;
  376. auto& kernel_pde = kernel_page_directory().entries()[page_directory_index];
  377. auto& current_pde = current_page_directory[page_directory_index];
  378. if (kernel_pde.is_present() && !current_pde.is_present()) {
  379. dbg() << "NP(kernel): Copying new kernel mapping for " << fault.vaddr() << " into current page directory";
  380. current_pde.copy_from({}, kernel_pde);
  381. flush_tlb(fault.vaddr().page_base());
  382. return PageFaultResponse::Continue;
  383. }
  384. }
  385. auto* region = region_from_vaddr(fault.vaddr());
  386. if (!region) {
  387. kprintf("NP(error) fault at invalid address V%p\n", fault.vaddr().get());
  388. return PageFaultResponse::ShouldCrash;
  389. }
  390. auto page_index_in_region = region->page_index_from_address(fault.vaddr());
  391. if (fault.type() == PageFault::Type::PageNotPresent) {
  392. if (region->vmobject().is_inode()) {
  393. #ifdef PAGE_FAULT_DEBUG
  394. dbgprintf("NP(inode) fault in Region{%p}[%u]\n", region, page_index_in_region);
  395. #endif
  396. page_in_from_inode(*region, page_index_in_region);
  397. return PageFaultResponse::Continue;
  398. }
  399. #ifdef PAGE_FAULT_DEBUG
  400. dbgprintf("NP(zero) fault in Region{%p}[%u]\n", region, page_index_in_region);
  401. #endif
  402. zero_page(*region, page_index_in_region);
  403. return PageFaultResponse::Continue;
  404. }
  405. ASSERT(fault.type() == PageFault::Type::ProtectionViolation);
  406. if (fault.access() == PageFault::Access::Write && region->should_cow(page_index_in_region)) {
  407. #ifdef PAGE_FAULT_DEBUG
  408. dbgprintf("PV(cow) fault in Region{%p}[%u]\n", region, page_index_in_region);
  409. #endif
  410. bool success = copy_on_write(*region, page_index_in_region);
  411. ASSERT(success);
  412. return PageFaultResponse::Continue;
  413. }
  414. kprintf("PV(error) fault in Region{%p}[%u] at V%p\n", region, page_index_in_region, fault.vaddr().get());
  415. return PageFaultResponse::ShouldCrash;
  416. }
  417. OwnPtr<Region> MemoryManager::allocate_kernel_region(size_t size, const StringView& name, bool user_accessible, bool should_commit)
  418. {
  419. InterruptDisabler disabler;
  420. ASSERT(!(size % PAGE_SIZE));
  421. auto range = kernel_page_directory().range_allocator().allocate_anywhere(size);
  422. ASSERT(range.is_valid());
  423. OwnPtr<Region> region;
  424. if (user_accessible)
  425. region = Region::create_user_accessible(range, name, PROT_READ | PROT_WRITE | PROT_EXEC);
  426. else
  427. region = Region::create_kernel_only(range, name, PROT_READ | PROT_WRITE | PROT_EXEC);
  428. region->map(kernel_page_directory());
  429. // FIXME: It would be cool if these could zero-fill on demand instead.
  430. if (should_commit)
  431. region->commit();
  432. return region;
  433. }
  434. OwnPtr<Region> MemoryManager::allocate_user_accessible_kernel_region(size_t size, const StringView& name)
  435. {
  436. return allocate_kernel_region(size, name, true);
  437. }
  438. void MemoryManager::deallocate_user_physical_page(PhysicalPage&& page)
  439. {
  440. for (auto& region : m_user_physical_regions) {
  441. if (!region.contains(page)) {
  442. kprintf(
  443. "MM: deallocate_user_physical_page: %p not in %p -> %p\n",
  444. page.paddr(), region.lower().get(), region.upper().get());
  445. continue;
  446. }
  447. region.return_page(move(page));
  448. --m_user_physical_pages_used;
  449. return;
  450. }
  451. kprintf("MM: deallocate_user_physical_page couldn't figure out region for user page @ %p\n", page.paddr());
  452. ASSERT_NOT_REACHED();
  453. }
  454. RefPtr<PhysicalPage> MemoryManager::allocate_user_physical_page(ShouldZeroFill should_zero_fill)
  455. {
  456. InterruptDisabler disabler;
  457. RefPtr<PhysicalPage> page;
  458. for (auto& region : m_user_physical_regions) {
  459. page = region.take_free_page(false);
  460. if (page.is_null())
  461. continue;
  462. }
  463. if (!page) {
  464. if (m_user_physical_regions.is_empty()) {
  465. kprintf("MM: no user physical regions available (?)\n");
  466. }
  467. kprintf("MM: no user physical pages available\n");
  468. ASSERT_NOT_REACHED();
  469. return {};
  470. }
  471. #ifdef MM_DEBUG
  472. dbgprintf("MM: allocate_user_physical_page vending P%p\n", page->paddr().get());
  473. #endif
  474. if (should_zero_fill == ShouldZeroFill::Yes) {
  475. auto* ptr = (u32*)quickmap_page(*page);
  476. fast_u32_fill(ptr, 0, PAGE_SIZE / sizeof(u32));
  477. unquickmap_page();
  478. }
  479. ++m_user_physical_pages_used;
  480. return page;
  481. }
  482. void MemoryManager::deallocate_supervisor_physical_page(PhysicalPage&& page)
  483. {
  484. for (auto& region : m_super_physical_regions) {
  485. if (!region.contains(page)) {
  486. kprintf(
  487. "MM: deallocate_supervisor_physical_page: %p not in %p -> %p\n",
  488. page.paddr(), region.lower().get(), region.upper().get());
  489. continue;
  490. }
  491. region.return_page(move(page));
  492. --m_super_physical_pages_used;
  493. return;
  494. }
  495. kprintf("MM: deallocate_supervisor_physical_page couldn't figure out region for super page @ %p\n", page.paddr());
  496. ASSERT_NOT_REACHED();
  497. }
  498. RefPtr<PhysicalPage> MemoryManager::allocate_supervisor_physical_page()
  499. {
  500. InterruptDisabler disabler;
  501. RefPtr<PhysicalPage> page;
  502. for (auto& region : m_super_physical_regions) {
  503. page = region.take_free_page(true);
  504. if (page.is_null())
  505. continue;
  506. }
  507. if (!page) {
  508. if (m_super_physical_regions.is_empty()) {
  509. kprintf("MM: no super physical regions available (?)\n");
  510. }
  511. kprintf("MM: no super physical pages available\n");
  512. ASSERT_NOT_REACHED();
  513. return {};
  514. }
  515. #ifdef MM_DEBUG
  516. dbgprintf("MM: allocate_supervisor_physical_page vending P%p\n", page->paddr().get());
  517. #endif
  518. fast_u32_fill((u32*)page->paddr().as_ptr(), 0, PAGE_SIZE / sizeof(u32));
  519. ++m_super_physical_pages_used;
  520. return page;
  521. }
  522. void MemoryManager::enter_process_paging_scope(Process& process)
  523. {
  524. ASSERT(current);
  525. InterruptDisabler disabler;
  526. current->tss().cr3 = process.page_directory().cr3();
  527. asm volatile("movl %%eax, %%cr3" ::"a"(process.page_directory().cr3())
  528. : "memory");
  529. }
  530. void MemoryManager::flush_entire_tlb()
  531. {
  532. asm volatile(
  533. "mov %%cr3, %%eax\n"
  534. "mov %%eax, %%cr3\n" ::
  535. : "%eax", "memory");
  536. }
  537. void MemoryManager::flush_tlb(VirtualAddress vaddr)
  538. {
  539. asm volatile("invlpg %0"
  540. :
  541. : "m"(*(char*)vaddr.get())
  542. : "memory");
  543. }
  544. void MemoryManager::map_for_kernel(VirtualAddress vaddr, PhysicalAddress paddr, bool cache_disabled)
  545. {
  546. auto& pte = ensure_pte(kernel_page_directory(), vaddr);
  547. pte.set_physical_page_base(paddr.get());
  548. pte.set_present(true);
  549. pte.set_writable(true);
  550. pte.set_user_allowed(false);
  551. pte.set_cache_disabled(cache_disabled);
  552. flush_tlb(vaddr);
  553. }
  554. u8* MemoryManager::quickmap_page(PhysicalPage& physical_page)
  555. {
  556. ASSERT_INTERRUPTS_DISABLED();
  557. ASSERT(!m_quickmap_in_use);
  558. m_quickmap_in_use = true;
  559. auto page_vaddr = m_quickmap_addr;
  560. auto& pte = ensure_pte(kernel_page_directory(), page_vaddr);
  561. pte.set_physical_page_base(physical_page.paddr().get());
  562. pte.set_present(true);
  563. pte.set_writable(true);
  564. pte.set_user_allowed(false);
  565. flush_tlb(page_vaddr);
  566. ASSERT((u32)pte.physical_page_base() == physical_page.paddr().get());
  567. #ifdef MM_DEBUG
  568. dbg() << "MM: >> quickmap_page " << page_vaddr << " => " << physical_page.paddr() << " @ PTE=" << (void*)pte.raw() << " {" << &pte << "}";
  569. #endif
  570. return page_vaddr.as_ptr();
  571. }
  572. void MemoryManager::unquickmap_page()
  573. {
  574. ASSERT_INTERRUPTS_DISABLED();
  575. ASSERT(m_quickmap_in_use);
  576. auto page_vaddr = m_quickmap_addr;
  577. auto& pte = ensure_pte(kernel_page_directory(), page_vaddr);
  578. #ifdef MM_DEBUG
  579. auto old_physical_address = pte.physical_page_base();
  580. #endif
  581. pte.set_physical_page_base(0);
  582. pte.set_present(false);
  583. pte.set_writable(false);
  584. flush_tlb(page_vaddr);
  585. #ifdef MM_DEBUG
  586. dbg() << "MM: >> unquickmap_page " << page_vaddr << " =/> " << old_physical_address;
  587. #endif
  588. m_quickmap_in_use = false;
  589. }
  590. void MemoryManager::remap_region(PageDirectory& page_directory, Region& region)
  591. {
  592. InterruptDisabler disabler;
  593. ASSERT(region.page_directory() == &page_directory);
  594. map_region_at_address(page_directory, region, region.vaddr());
  595. }
  596. void MemoryManager::map_region_at_address(PageDirectory& page_directory, Region& region, VirtualAddress vaddr)
  597. {
  598. InterruptDisabler disabler;
  599. region.set_page_directory(page_directory);
  600. auto& vmo = region.vmobject();
  601. #ifdef MM_DEBUG
  602. dbgprintf("MM: map_region_at_address will map VMO pages %u - %u (VMO page count: %u)\n", region.first_page_index(), region.last_page_index(), vmo.page_count());
  603. #endif
  604. for (size_t i = 0; i < region.page_count(); ++i) {
  605. auto page_vaddr = vaddr.offset(i * PAGE_SIZE);
  606. auto& pte = ensure_pte(page_directory, page_vaddr);
  607. auto& physical_page = vmo.physical_pages()[region.first_page_index() + i];
  608. if (physical_page) {
  609. pte.set_physical_page_base(physical_page->paddr().get());
  610. pte.set_present(true); // FIXME: Maybe we should use the is_readable flag here?
  611. // FIXME: It seems wrong that the *region* cow map is essentially using *VMO* relative indices.
  612. if (region.should_cow(region.first_page_index() + i))
  613. pte.set_writable(false);
  614. else
  615. pte.set_writable(region.is_writable());
  616. } else {
  617. pte.set_physical_page_base(0);
  618. pte.set_present(false);
  619. pte.set_writable(region.is_writable());
  620. }
  621. pte.set_user_allowed(region.is_user_accessible());
  622. page_directory.flush(page_vaddr);
  623. #ifdef MM_DEBUG
  624. dbgprintf("MM: >> map_region_at_address (PD=%p) '%s' V%p => P%p (@%p)\n", &page_directory, region.name().characters(), page_vaddr, physical_page ? physical_page->paddr().get() : 0, physical_page.ptr());
  625. #endif
  626. }
  627. }
  628. bool MemoryManager::validate_user_read(const Process& process, VirtualAddress vaddr) const
  629. {
  630. auto* region = region_from_vaddr(process, vaddr);
  631. return region && region->is_readable();
  632. }
  633. bool MemoryManager::validate_user_write(const Process& process, VirtualAddress vaddr) const
  634. {
  635. auto* region = region_from_vaddr(process, vaddr);
  636. return region && region->is_writable();
  637. }
  638. void MemoryManager::register_vmo(VMObject& vmo)
  639. {
  640. InterruptDisabler disabler;
  641. m_vmobjects.append(&vmo);
  642. }
  643. void MemoryManager::unregister_vmo(VMObject& vmo)
  644. {
  645. InterruptDisabler disabler;
  646. m_vmobjects.remove(&vmo);
  647. }
  648. void MemoryManager::register_region(Region& region)
  649. {
  650. InterruptDisabler disabler;
  651. if (region.vaddr().get() >= 0xc0000000)
  652. m_kernel_regions.append(&region);
  653. else
  654. m_user_regions.append(&region);
  655. }
  656. void MemoryManager::unregister_region(Region& region)
  657. {
  658. InterruptDisabler disabler;
  659. if (region.vaddr().get() >= 0xc0000000)
  660. m_kernel_regions.remove(&region);
  661. else
  662. m_user_regions.remove(&region);
  663. }
  664. ProcessPagingScope::ProcessPagingScope(Process& process)
  665. {
  666. ASSERT(current);
  667. MM.enter_process_paging_scope(process);
  668. }
  669. ProcessPagingScope::~ProcessPagingScope()
  670. {
  671. MM.enter_process_paging_scope(current->process());
  672. }