AnonymousVMObject.cpp 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481
  1. /*
  2. * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <Kernel/Arch/x86/SmapDisabler.h>
  7. #include <Kernel/Debug.h>
  8. #include <Kernel/Process.h>
  9. #include <Kernel/VM/AnonymousVMObject.h>
  10. #include <Kernel/VM/MemoryManager.h>
  11. #include <Kernel/VM/PhysicalPage.h>
  12. namespace Kernel {
  13. RefPtr<VMObject> AnonymousVMObject::try_clone()
  14. {
  15. // We need to acquire our lock so we copy a sane state
  16. ScopedSpinLock lock(m_lock);
  17. // We're the parent. Since we're about to become COW we need to
  18. // commit the number of pages that we need to potentially allocate
  19. // so that the parent is still guaranteed to be able to have all
  20. // non-volatile memory available.
  21. size_t need_cow_pages = 0;
  22. // We definitely need to commit non-volatile areas
  23. for_each_nonvolatile_range([&](VolatilePageRange const& nonvolatile_range) {
  24. need_cow_pages += nonvolatile_range.count;
  25. });
  26. dbgln_if(COMMIT_DEBUG, "Cloning {:p}, need {} committed cow pages", this, need_cow_pages);
  27. if (!MM.commit_user_physical_pages(need_cow_pages))
  28. return {};
  29. // Create or replace the committed cow pages. When cloning a previously
  30. // cloned vmobject, we want to essentially "fork", leaving us and the
  31. // new clone with one set of shared committed cow pages, and the original
  32. // one would keep the one it still has. This ensures that the original
  33. // one and this one, as well as the clone have sufficient resources
  34. // to cow all pages as needed
  35. m_shared_committed_cow_pages = try_create<CommittedCowPages>(need_cow_pages);
  36. if (!m_shared_committed_cow_pages) {
  37. MM.uncommit_user_physical_pages(need_cow_pages);
  38. return {};
  39. }
  40. // Both original and clone become COW. So create a COW map for ourselves
  41. // or reset all pages to be copied again if we were previously cloned
  42. ensure_or_reset_cow_map();
  43. // FIXME: If this allocation fails, we need to rollback all changes.
  44. return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(*this));
  45. }
  46. RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_with_size(size_t size, AllocationStrategy commit)
  47. {
  48. if (commit == AllocationStrategy::Reserve || commit == AllocationStrategy::AllocateNow) {
  49. // We need to attempt to commit before actually creating the object
  50. if (!MM.commit_user_physical_pages(ceil_div(size, static_cast<size_t>(PAGE_SIZE))))
  51. return {};
  52. }
  53. return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(size, commit));
  54. }
  55. RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_with_physical_pages(Span<NonnullRefPtr<PhysicalPage>> physical_pages)
  56. {
  57. return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(physical_pages));
  58. }
  59. RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_for_physical_range(PhysicalAddress paddr, size_t size)
  60. {
  61. if (paddr.offset(size) < paddr) {
  62. dbgln("Shenanigans! try_create_for_physical_range({}, {}) would wrap around", paddr, size);
  63. return nullptr;
  64. }
  65. return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(paddr, size));
  66. }
  67. AnonymousVMObject::AnonymousVMObject(size_t size, AllocationStrategy strategy)
  68. : VMObject(size)
  69. , m_volatile_ranges_cache({ 0, page_count() })
  70. , m_unused_committed_pages(strategy == AllocationStrategy::Reserve ? page_count() : 0)
  71. {
  72. if (strategy == AllocationStrategy::AllocateNow) {
  73. // Allocate all pages right now. We know we can get all because we committed the amount needed
  74. for (size_t i = 0; i < page_count(); ++i)
  75. physical_pages()[i] = MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
  76. } else {
  77. auto& initial_page = (strategy == AllocationStrategy::Reserve) ? MM.lazy_committed_page() : MM.shared_zero_page();
  78. for (size_t i = 0; i < page_count(); ++i)
  79. physical_pages()[i] = initial_page;
  80. }
  81. }
  82. AnonymousVMObject::AnonymousVMObject(PhysicalAddress paddr, size_t size)
  83. : VMObject(size)
  84. , m_volatile_ranges_cache({ 0, page_count() })
  85. {
  86. VERIFY(paddr.page_base() == paddr);
  87. for (size_t i = 0; i < page_count(); ++i)
  88. physical_pages()[i] = PhysicalPage::create(paddr.offset(i * PAGE_SIZE), MayReturnToFreeList::No);
  89. }
  90. AnonymousVMObject::AnonymousVMObject(Span<NonnullRefPtr<PhysicalPage>> physical_pages)
  91. : VMObject(physical_pages.size() * PAGE_SIZE)
  92. , m_volatile_ranges_cache({ 0, page_count() })
  93. {
  94. for (size_t i = 0; i < physical_pages.size(); ++i) {
  95. m_physical_pages[i] = physical_pages[i];
  96. }
  97. }
  98. AnonymousVMObject::AnonymousVMObject(AnonymousVMObject const& other)
  99. : VMObject(other)
  100. , m_volatile_ranges_cache({ 0, page_count() }) // do *not* clone this
  101. , m_volatile_ranges_cache_dirty(true) // do *not* clone this
  102. , m_purgeable_ranges() // do *not* clone this
  103. , m_unused_committed_pages(other.m_unused_committed_pages)
  104. , m_cow_map() // do *not* clone this
  105. , m_shared_committed_cow_pages(other.m_shared_committed_cow_pages) // share the pool
  106. {
  107. // We can't really "copy" a spinlock. But we're holding it. Clear in the clone
  108. VERIFY(other.m_lock.is_locked());
  109. m_lock.initialize();
  110. // The clone also becomes COW
  111. ensure_or_reset_cow_map();
  112. if (m_unused_committed_pages > 0) {
  113. // The original vmobject didn't use up all committed pages. When
  114. // cloning (fork) we will overcommit. For this purpose we drop all
  115. // lazy-commit references and replace them with shared zero pages.
  116. for (size_t i = 0; i < page_count(); i++) {
  117. auto& phys_page = m_physical_pages[i];
  118. if (phys_page && phys_page->is_lazy_committed_page()) {
  119. phys_page = MM.shared_zero_page();
  120. if (--m_unused_committed_pages == 0)
  121. break;
  122. }
  123. }
  124. VERIFY(m_unused_committed_pages == 0);
  125. }
  126. }
  127. AnonymousVMObject::~AnonymousVMObject()
  128. {
  129. // Return any unused committed pages
  130. if (m_unused_committed_pages > 0)
  131. MM.uncommit_user_physical_pages(m_unused_committed_pages);
  132. }
  133. int AnonymousVMObject::purge()
  134. {
  135. MutexLocker locker(m_paging_lock);
  136. return purge_impl();
  137. }
  138. int AnonymousVMObject::purge_with_interrupts_disabled(Badge<MemoryManager>)
  139. {
  140. VERIFY_INTERRUPTS_DISABLED();
  141. if (m_paging_lock.is_locked())
  142. return 0;
  143. return purge_impl();
  144. }
  145. void AnonymousVMObject::set_was_purged(VolatilePageRange const& range)
  146. {
  147. VERIFY(m_lock.is_locked());
  148. for (auto* purgeable_ranges : m_purgeable_ranges)
  149. purgeable_ranges->set_was_purged(range);
  150. }
  151. int AnonymousVMObject::purge_impl()
  152. {
  153. int purged_page_count = 0;
  154. ScopedSpinLock lock(m_lock);
  155. for_each_volatile_range([&](auto const& range) {
  156. int purged_in_range = 0;
  157. auto range_end = range.base + range.count;
  158. for (size_t i = range.base; i < range_end; i++) {
  159. auto& phys_page = m_physical_pages[i];
  160. if (phys_page && !phys_page->is_shared_zero_page()) {
  161. VERIFY(!phys_page->is_lazy_committed_page());
  162. ++purged_in_range;
  163. }
  164. phys_page = MM.shared_zero_page();
  165. }
  166. if (purged_in_range > 0) {
  167. purged_page_count += purged_in_range;
  168. set_was_purged(range);
  169. for_each_region([&](auto& region) {
  170. if (&region.vmobject() == this) {
  171. if (auto owner = region.get_owner()) {
  172. // we need to hold a reference the process here (if there is one) as we may not own this region
  173. dmesgln("Purged {} pages from region {} owned by {} at {} - {}",
  174. purged_in_range,
  175. region.name(),
  176. *owner,
  177. region.vaddr_from_page_index(range.base),
  178. region.vaddr_from_page_index(range.base + range.count));
  179. } else {
  180. dmesgln("Purged {} pages from region {} (no ownership) at {} - {}",
  181. purged_in_range,
  182. region.name(),
  183. region.vaddr_from_page_index(range.base),
  184. region.vaddr_from_page_index(range.base + range.count));
  185. }
  186. region.remap_vmobject_page_range(range.base, range.count);
  187. }
  188. });
  189. }
  190. });
  191. return purged_page_count;
  192. }
  193. void AnonymousVMObject::register_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
  194. {
  195. ScopedSpinLock lock(m_lock);
  196. purgeable_page_ranges.set_vmobject(this);
  197. VERIFY(!m_purgeable_ranges.contains_slow(&purgeable_page_ranges));
  198. m_purgeable_ranges.append(&purgeable_page_ranges);
  199. }
  200. void AnonymousVMObject::unregister_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
  201. {
  202. ScopedSpinLock lock(m_lock);
  203. for (size_t i = 0; i < m_purgeable_ranges.size(); i++) {
  204. if (m_purgeable_ranges[i] != &purgeable_page_ranges)
  205. continue;
  206. purgeable_page_ranges.set_vmobject(nullptr);
  207. m_purgeable_ranges.remove(i);
  208. return;
  209. }
  210. VERIFY_NOT_REACHED();
  211. }
  212. bool AnonymousVMObject::is_any_volatile() const
  213. {
  214. ScopedSpinLock lock(m_lock);
  215. for (auto& volatile_ranges : m_purgeable_ranges) {
  216. ScopedSpinLock lock(volatile_ranges->m_volatile_ranges_lock);
  217. if (!volatile_ranges->is_empty())
  218. return true;
  219. }
  220. return false;
  221. }
  222. size_t AnonymousVMObject::remove_lazy_commit_pages(VolatilePageRange const& range)
  223. {
  224. VERIFY(m_lock.is_locked());
  225. size_t removed_count = 0;
  226. auto range_end = range.base + range.count;
  227. for (size_t i = range.base; i < range_end; i++) {
  228. auto& phys_page = m_physical_pages[i];
  229. if (phys_page && phys_page->is_lazy_committed_page()) {
  230. phys_page = MM.shared_zero_page();
  231. removed_count++;
  232. VERIFY(m_unused_committed_pages > 0);
  233. if (--m_unused_committed_pages == 0)
  234. break;
  235. }
  236. }
  237. return removed_count;
  238. }
  239. void AnonymousVMObject::update_volatile_cache()
  240. {
  241. VERIFY(m_lock.is_locked());
  242. VERIFY(m_volatile_ranges_cache_dirty);
  243. m_volatile_ranges_cache.clear();
  244. for_each_nonvolatile_range([&](VolatilePageRange const& range) {
  245. m_volatile_ranges_cache.add_unchecked(range);
  246. });
  247. m_volatile_ranges_cache_dirty = false;
  248. }
  249. void AnonymousVMObject::range_made_volatile(VolatilePageRange const& range)
  250. {
  251. VERIFY(m_lock.is_locked());
  252. if (m_unused_committed_pages == 0)
  253. return;
  254. // We need to check this range for any pages that are marked for
  255. // lazy committed allocation and turn them into shared zero pages
  256. // and also adjust the m_unused_committed_pages for each such page.
  257. // Take into account all the other views as well.
  258. size_t uncommit_page_count = 0;
  259. for_each_volatile_range([&](auto const& r) {
  260. auto intersected = range.intersected(r);
  261. if (!intersected.is_empty()) {
  262. uncommit_page_count += remove_lazy_commit_pages(intersected);
  263. if (m_unused_committed_pages == 0)
  264. return IterationDecision::Break;
  265. }
  266. return IterationDecision::Continue;
  267. });
  268. // Return those committed pages back to the system
  269. if (uncommit_page_count > 0) {
  270. dbgln_if(COMMIT_DEBUG, "Uncommit {} lazy-commit pages from {:p}", uncommit_page_count, this);
  271. MM.uncommit_user_physical_pages(uncommit_page_count);
  272. }
  273. m_volatile_ranges_cache_dirty = true;
  274. }
  275. void AnonymousVMObject::range_made_nonvolatile(VolatilePageRange const&)
  276. {
  277. VERIFY(m_lock.is_locked());
  278. m_volatile_ranges_cache_dirty = true;
  279. }
  280. size_t AnonymousVMObject::count_needed_commit_pages_for_nonvolatile_range(VolatilePageRange const& range)
  281. {
  282. VERIFY(m_lock.is_locked());
  283. VERIFY(!range.is_empty());
  284. size_t need_commit_pages = 0;
  285. auto range_end = range.base + range.count;
  286. for (size_t page_index = range.base; page_index < range_end; page_index++) {
  287. // COW pages are accounted for in m_shared_committed_cow_pages
  288. if (!m_cow_map.is_null() && m_cow_map.get(page_index))
  289. continue;
  290. auto& phys_page = m_physical_pages[page_index];
  291. if (phys_page && phys_page->is_shared_zero_page())
  292. need_commit_pages++;
  293. }
  294. return need_commit_pages;
  295. }
  296. size_t AnonymousVMObject::mark_committed_pages_for_nonvolatile_range(VolatilePageRange const& range, size_t mark_total)
  297. {
  298. VERIFY(m_lock.is_locked());
  299. VERIFY(!range.is_empty());
  300. VERIFY(mark_total > 0);
  301. size_t pages_updated = 0;
  302. auto range_end = range.base + range.count;
  303. for (size_t page_index = range.base; page_index < range_end; page_index++) {
  304. // COW pages are accounted for in m_shared_committed_cow_pages
  305. if (!m_cow_map.is_null() && m_cow_map.get(page_index))
  306. continue;
  307. auto& phys_page = m_physical_pages[page_index];
  308. if (phys_page && phys_page->is_shared_zero_page()) {
  309. phys_page = MM.lazy_committed_page();
  310. if (++pages_updated == mark_total)
  311. break;
  312. }
  313. }
  314. dbgln_if(COMMIT_DEBUG, "Added {} lazy-commit pages to {:p}", pages_updated, this);
  315. m_unused_committed_pages += pages_updated;
  316. return pages_updated;
  317. }
  318. NonnullRefPtr<PhysicalPage> AnonymousVMObject::allocate_committed_page(Badge<Region>, size_t page_index)
  319. {
  320. {
  321. ScopedSpinLock lock(m_lock);
  322. VERIFY(m_unused_committed_pages > 0);
  323. // We shouldn't have any committed page tags in volatile regions
  324. VERIFY([&]() {
  325. for (auto* purgeable_ranges : m_purgeable_ranges) {
  326. if (purgeable_ranges->is_volatile(page_index))
  327. return false;
  328. }
  329. return true;
  330. }());
  331. m_unused_committed_pages--;
  332. }
  333. return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
  334. }
  335. Bitmap& AnonymousVMObject::ensure_cow_map()
  336. {
  337. if (m_cow_map.is_null())
  338. m_cow_map = Bitmap { page_count(), true };
  339. return m_cow_map;
  340. }
  341. void AnonymousVMObject::ensure_or_reset_cow_map()
  342. {
  343. if (m_cow_map.is_null())
  344. ensure_cow_map();
  345. else
  346. m_cow_map.fill(true);
  347. }
  348. bool AnonymousVMObject::should_cow(size_t page_index, bool is_shared) const
  349. {
  350. auto& page = physical_pages()[page_index];
  351. if (page && (page->is_shared_zero_page() || page->is_lazy_committed_page()))
  352. return true;
  353. if (is_shared)
  354. return false;
  355. return !m_cow_map.is_null() && m_cow_map.get(page_index);
  356. }
  357. void AnonymousVMObject::set_should_cow(size_t page_index, bool cow)
  358. {
  359. ensure_cow_map().set(page_index, cow);
  360. }
  361. size_t AnonymousVMObject::cow_pages() const
  362. {
  363. if (m_cow_map.is_null())
  364. return 0;
  365. return m_cow_map.count_slow(true);
  366. }
  367. bool AnonymousVMObject::is_nonvolatile(size_t page_index)
  368. {
  369. if (m_volatile_ranges_cache_dirty)
  370. update_volatile_cache();
  371. return !m_volatile_ranges_cache.contains(page_index);
  372. }
  373. PageFaultResponse AnonymousVMObject::handle_cow_fault(size_t page_index, VirtualAddress vaddr)
  374. {
  375. VERIFY_INTERRUPTS_DISABLED();
  376. ScopedSpinLock lock(m_lock);
  377. auto& page_slot = physical_pages()[page_index];
  378. bool have_committed = m_shared_committed_cow_pages && is_nonvolatile(page_index);
  379. if (page_slot->ref_count() == 1) {
  380. dbgln_if(PAGE_FAULT_DEBUG, " >> It's a COW page but nobody is sharing it anymore. Remap r/w");
  381. set_should_cow(page_index, false);
  382. if (have_committed) {
  383. if (m_shared_committed_cow_pages->return_one())
  384. m_shared_committed_cow_pages = nullptr;
  385. }
  386. return PageFaultResponse::Continue;
  387. }
  388. RefPtr<PhysicalPage> page;
  389. if (have_committed) {
  390. dbgln_if(PAGE_FAULT_DEBUG, " >> It's a committed COW page and it's time to COW!");
  391. page = m_shared_committed_cow_pages->allocate_one();
  392. } else {
  393. dbgln_if(PAGE_FAULT_DEBUG, " >> It's a COW page and it's time to COW!");
  394. page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::No);
  395. if (page.is_null()) {
  396. dmesgln("MM: handle_cow_fault was unable to allocate a physical page");
  397. return PageFaultResponse::OutOfMemory;
  398. }
  399. }
  400. u8* dest_ptr = MM.quickmap_page(*page);
  401. dbgln_if(PAGE_FAULT_DEBUG, " >> COW {} <- {}", page->paddr(), page_slot->paddr());
  402. {
  403. SmapDisabler disabler;
  404. void* fault_at;
  405. if (!safe_memcpy(dest_ptr, vaddr.as_ptr(), PAGE_SIZE, fault_at)) {
  406. if ((u8*)fault_at >= dest_ptr && (u8*)fault_at <= dest_ptr + PAGE_SIZE)
  407. dbgln(" >> COW: error copying page {}/{} to {}/{}: failed to write to page at {}",
  408. page_slot->paddr(), vaddr, page->paddr(), VirtualAddress(dest_ptr), VirtualAddress(fault_at));
  409. else if ((u8*)fault_at >= vaddr.as_ptr() && (u8*)fault_at <= vaddr.as_ptr() + PAGE_SIZE)
  410. dbgln(" >> COW: error copying page {}/{} to {}/{}: failed to read from page at {}",
  411. page_slot->paddr(), vaddr, page->paddr(), VirtualAddress(dest_ptr), VirtualAddress(fault_at));
  412. else
  413. VERIFY_NOT_REACHED();
  414. }
  415. }
  416. page_slot = move(page);
  417. MM.unquickmap_page();
  418. set_should_cow(page_index, false);
  419. return PageFaultResponse::Continue;
  420. }
  421. }