AnonymousVMObject.cpp 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381
  1. /*
  2. * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <Kernel/Arch/x86/SmapDisabler.h>
  7. #include <Kernel/Debug.h>
  8. #include <Kernel/Process.h>
  9. #include <Kernel/VM/AnonymousVMObject.h>
  10. #include <Kernel/VM/MemoryManager.h>
  11. #include <Kernel/VM/PhysicalPage.h>
  12. namespace Kernel {
  13. RefPtr<VMObject> AnonymousVMObject::try_clone()
  14. {
  15. // We need to acquire our lock so we copy a sane state
  16. ScopedSpinLock lock(m_lock);
  17. // We're the parent. Since we're about to become COW we need to
  18. // commit the number of pages that we need to potentially allocate
  19. // so that the parent is still guaranteed to be able to have all
  20. // non-volatile memory available.
  21. size_t new_cow_pages_needed = 0;
  22. if (is_volatile()) {
  23. // NOTE: If this object is currently volatile, we don't own any committed pages.
  24. } else {
  25. new_cow_pages_needed = page_count();
  26. }
  27. dbgln_if(COMMIT_DEBUG, "Cloning {:p}, need {} committed cow pages", this, new_cow_pages_needed);
  28. if (!MM.commit_user_physical_pages(new_cow_pages_needed))
  29. return {};
  30. // Create or replace the committed cow pages. When cloning a previously
  31. // cloned vmobject, we want to essentially "fork", leaving us and the
  32. // new clone with one set of shared committed cow pages, and the original
  33. // one would keep the one it still has. This ensures that the original
  34. // one and this one, as well as the clone have sufficient resources
  35. // to cow all pages as needed
  36. m_shared_committed_cow_pages = try_create<CommittedCowPages>(new_cow_pages_needed);
  37. if (!m_shared_committed_cow_pages) {
  38. MM.uncommit_user_physical_pages(new_cow_pages_needed);
  39. return {};
  40. }
  41. // Both original and clone become COW. So create a COW map for ourselves
  42. // or reset all pages to be copied again if we were previously cloned
  43. ensure_or_reset_cow_map();
  44. // FIXME: If this allocation fails, we need to rollback all changes.
  45. return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(*this));
  46. }
  47. RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_with_size(size_t size, AllocationStrategy commit)
  48. {
  49. if (commit == AllocationStrategy::Reserve || commit == AllocationStrategy::AllocateNow) {
  50. // We need to attempt to commit before actually creating the object
  51. if (!MM.commit_user_physical_pages(ceil_div(size, static_cast<size_t>(PAGE_SIZE))))
  52. return {};
  53. }
  54. return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(size, commit));
  55. }
  56. RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_purgeable_with_size(size_t size, AllocationStrategy commit)
  57. {
  58. if (commit == AllocationStrategy::Reserve || commit == AllocationStrategy::AllocateNow) {
  59. // We need to attempt to commit before actually creating the object
  60. if (!MM.commit_user_physical_pages(ceil_div(size, static_cast<size_t>(PAGE_SIZE))))
  61. return {};
  62. }
  63. auto vmobject = adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(size, commit));
  64. if (!vmobject)
  65. return {};
  66. vmobject->m_purgeable = true;
  67. return vmobject;
  68. }
  69. RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_with_physical_pages(Span<NonnullRefPtr<PhysicalPage>> physical_pages)
  70. {
  71. return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(physical_pages));
  72. }
  73. RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_for_physical_range(PhysicalAddress paddr, size_t size)
  74. {
  75. if (paddr.offset(size) < paddr) {
  76. dbgln("Shenanigans! try_create_for_physical_range({}, {}) would wrap around", paddr, size);
  77. return nullptr;
  78. }
  79. return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(paddr, size));
  80. }
  81. AnonymousVMObject::AnonymousVMObject(size_t size, AllocationStrategy strategy)
  82. : VMObject(size)
  83. , m_unused_committed_pages(strategy == AllocationStrategy::Reserve ? page_count() : 0)
  84. {
  85. if (strategy == AllocationStrategy::AllocateNow) {
  86. // Allocate all pages right now. We know we can get all because we committed the amount needed
  87. for (size_t i = 0; i < page_count(); ++i)
  88. physical_pages()[i] = MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
  89. } else {
  90. auto& initial_page = (strategy == AllocationStrategy::Reserve) ? MM.lazy_committed_page() : MM.shared_zero_page();
  91. for (size_t i = 0; i < page_count(); ++i)
  92. physical_pages()[i] = initial_page;
  93. }
  94. }
  95. AnonymousVMObject::AnonymousVMObject(PhysicalAddress paddr, size_t size)
  96. : VMObject(size)
  97. {
  98. VERIFY(paddr.page_base() == paddr);
  99. for (size_t i = 0; i < page_count(); ++i)
  100. physical_pages()[i] = PhysicalPage::create(paddr.offset(i * PAGE_SIZE), MayReturnToFreeList::No);
  101. }
  102. AnonymousVMObject::AnonymousVMObject(Span<NonnullRefPtr<PhysicalPage>> physical_pages)
  103. : VMObject(physical_pages.size() * PAGE_SIZE)
  104. {
  105. for (size_t i = 0; i < physical_pages.size(); ++i) {
  106. m_physical_pages[i] = physical_pages[i];
  107. }
  108. }
  109. AnonymousVMObject::AnonymousVMObject(AnonymousVMObject const& other)
  110. : VMObject(other)
  111. , m_unused_committed_pages(other.m_unused_committed_pages)
  112. , m_cow_map() // do *not* clone this
  113. , m_shared_committed_cow_pages(other.m_shared_committed_cow_pages) // share the pool
  114. {
  115. // We can't really "copy" a spinlock. But we're holding it. Clear in the clone
  116. VERIFY(other.m_lock.is_locked());
  117. m_lock.initialize();
  118. // The clone also becomes COW
  119. ensure_or_reset_cow_map();
  120. if (m_unused_committed_pages > 0) {
  121. // The original vmobject didn't use up all committed pages. When
  122. // cloning (fork) we will overcommit. For this purpose we drop all
  123. // lazy-commit references and replace them with shared zero pages.
  124. for (size_t i = 0; i < page_count(); i++) {
  125. auto& phys_page = m_physical_pages[i];
  126. if (phys_page && phys_page->is_lazy_committed_page()) {
  127. phys_page = MM.shared_zero_page();
  128. if (--m_unused_committed_pages == 0)
  129. break;
  130. }
  131. }
  132. VERIFY(m_unused_committed_pages == 0);
  133. }
  134. }
  135. AnonymousVMObject::~AnonymousVMObject()
  136. {
  137. // Return any unused committed pages
  138. if (m_unused_committed_pages > 0)
  139. MM.uncommit_user_physical_pages(m_unused_committed_pages);
  140. }
  141. size_t AnonymousVMObject::purge()
  142. {
  143. ScopedSpinLock lock(m_lock);
  144. if (!is_purgeable() || !is_volatile())
  145. return 0;
  146. size_t total_pages_purged = 0;
  147. for (auto& page : m_physical_pages) {
  148. VERIFY(page);
  149. if (page->is_shared_zero_page())
  150. continue;
  151. page = MM.shared_zero_page();
  152. ++total_pages_purged;
  153. }
  154. m_was_purged = true;
  155. for_each_region([](Region& region) {
  156. region.remap();
  157. });
  158. return total_pages_purged;
  159. }
  160. KResult AnonymousVMObject::set_volatile(bool is_volatile, bool& was_purged)
  161. {
  162. VERIFY(is_purgeable());
  163. ScopedSpinLock locker(m_lock);
  164. was_purged = m_was_purged;
  165. if (m_volatile == is_volatile)
  166. return KSuccess;
  167. if (is_volatile) {
  168. // When a VMObject is made volatile, it gives up all of its committed memory.
  169. // Any physical pages already allocated remain in the VMObject for now, but the kernel is free to take them at any moment.
  170. for (auto& page : m_physical_pages) {
  171. if (page && page->is_lazy_committed_page())
  172. page = MM.shared_zero_page();
  173. }
  174. if (m_unused_committed_pages) {
  175. MM.uncommit_user_physical_pages(m_unused_committed_pages);
  176. m_unused_committed_pages = 0;
  177. }
  178. m_volatile = true;
  179. m_was_purged = false;
  180. return KSuccess;
  181. }
  182. // When a VMObject is made non-volatile, we try to commit however many pages are not currently available.
  183. // If that fails, we return false to indicate that memory allocation failed.
  184. size_t committed_pages_needed = 0;
  185. for (auto& page : m_physical_pages) {
  186. VERIFY(page);
  187. if (page->is_shared_zero_page())
  188. ++committed_pages_needed;
  189. }
  190. if (!committed_pages_needed) {
  191. m_volatile = false;
  192. return KSuccess;
  193. }
  194. if (!MM.commit_user_physical_pages(committed_pages_needed))
  195. return ENOMEM;
  196. m_unused_committed_pages = committed_pages_needed;
  197. for (auto& page : m_physical_pages) {
  198. if (page->is_shared_zero_page())
  199. page = MM.lazy_committed_page();
  200. }
  201. m_volatile = false;
  202. m_was_purged = false;
  203. return KSuccess;
  204. }
  205. NonnullRefPtr<PhysicalPage> AnonymousVMObject::allocate_committed_page(Badge<Region>)
  206. {
  207. {
  208. ScopedSpinLock lock(m_lock);
  209. VERIFY(m_unused_committed_pages > 0);
  210. --m_unused_committed_pages;
  211. }
  212. return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
  213. }
  214. Bitmap& AnonymousVMObject::ensure_cow_map()
  215. {
  216. if (m_cow_map.is_null())
  217. m_cow_map = Bitmap { page_count(), true };
  218. return m_cow_map;
  219. }
  220. void AnonymousVMObject::ensure_or_reset_cow_map()
  221. {
  222. if (m_cow_map.is_null())
  223. ensure_cow_map();
  224. else
  225. m_cow_map.fill(true);
  226. }
  227. bool AnonymousVMObject::should_cow(size_t page_index, bool is_shared) const
  228. {
  229. auto& page = physical_pages()[page_index];
  230. if (page && (page->is_shared_zero_page() || page->is_lazy_committed_page()))
  231. return true;
  232. if (is_shared)
  233. return false;
  234. return !m_cow_map.is_null() && m_cow_map.get(page_index);
  235. }
  236. void AnonymousVMObject::set_should_cow(size_t page_index, bool cow)
  237. {
  238. ensure_cow_map().set(page_index, cow);
  239. }
  240. size_t AnonymousVMObject::cow_pages() const
  241. {
  242. if (m_cow_map.is_null())
  243. return 0;
  244. return m_cow_map.count_slow(true);
  245. }
  246. PageFaultResponse AnonymousVMObject::handle_cow_fault(size_t page_index, VirtualAddress vaddr)
  247. {
  248. VERIFY_INTERRUPTS_DISABLED();
  249. ScopedSpinLock lock(m_lock);
  250. if (is_volatile()) {
  251. // A COW fault in a volatile region? Userspace is writing to volatile memory, this is a bug. Crash.
  252. dbgln("COW fault in volatile region, will crash.");
  253. return PageFaultResponse::ShouldCrash;
  254. }
  255. auto& page_slot = physical_pages()[page_index];
  256. bool have_committed = m_shared_committed_cow_pages;
  257. if (page_slot->ref_count() == 1) {
  258. dbgln_if(PAGE_FAULT_DEBUG, " >> It's a COW page but nobody is sharing it anymore. Remap r/w");
  259. set_should_cow(page_index, false);
  260. if (have_committed) {
  261. if (m_shared_committed_cow_pages->return_one())
  262. m_shared_committed_cow_pages = nullptr;
  263. }
  264. return PageFaultResponse::Continue;
  265. }
  266. RefPtr<PhysicalPage> page;
  267. if (have_committed) {
  268. dbgln_if(PAGE_FAULT_DEBUG, " >> It's a committed COW page and it's time to COW!");
  269. page = m_shared_committed_cow_pages->allocate_one();
  270. } else {
  271. dbgln_if(PAGE_FAULT_DEBUG, " >> It's a COW page and it's time to COW!");
  272. page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::No);
  273. if (page.is_null()) {
  274. dmesgln("MM: handle_cow_fault was unable to allocate a physical page");
  275. return PageFaultResponse::OutOfMemory;
  276. }
  277. }
  278. u8* dest_ptr = MM.quickmap_page(*page);
  279. dbgln_if(PAGE_FAULT_DEBUG, " >> COW {} <- {}", page->paddr(), page_slot->paddr());
  280. {
  281. SmapDisabler disabler;
  282. void* fault_at;
  283. if (!safe_memcpy(dest_ptr, vaddr.as_ptr(), PAGE_SIZE, fault_at)) {
  284. if ((u8*)fault_at >= dest_ptr && (u8*)fault_at <= dest_ptr + PAGE_SIZE)
  285. dbgln(" >> COW: error copying page {}/{} to {}/{}: failed to write to page at {}",
  286. page_slot->paddr(), vaddr, page->paddr(), VirtualAddress(dest_ptr), VirtualAddress(fault_at));
  287. else if ((u8*)fault_at >= vaddr.as_ptr() && (u8*)fault_at <= vaddr.as_ptr() + PAGE_SIZE)
  288. dbgln(" >> COW: error copying page {}/{} to {}/{}: failed to read from page at {}",
  289. page_slot->paddr(), vaddr, page->paddr(), VirtualAddress(dest_ptr), VirtualAddress(fault_at));
  290. else
  291. VERIFY_NOT_REACHED();
  292. }
  293. }
  294. page_slot = move(page);
  295. MM.unquickmap_page();
  296. set_should_cow(page_index, false);
  297. return PageFaultResponse::Continue;
  298. }
  299. CommittedCowPages::CommittedCowPages(size_t committed_pages)
  300. : m_committed_pages(committed_pages)
  301. {
  302. }
  303. CommittedCowPages::~CommittedCowPages()
  304. {
  305. // Return unused committed pages
  306. if (m_committed_pages > 0)
  307. MM.uncommit_user_physical_pages(m_committed_pages);
  308. }
  309. NonnullRefPtr<PhysicalPage> CommittedCowPages::allocate_one()
  310. {
  311. VERIFY(m_committed_pages > 0);
  312. m_committed_pages--;
  313. return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
  314. }
  315. bool CommittedCowPages::return_one()
  316. {
  317. VERIFY(m_committed_pages > 0);
  318. m_committed_pages--;
  319. MM.uncommit_user_physical_pages(1);
  320. return m_committed_pages == 0;
  321. }
  322. }