mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2024-11-22 23:50:19 +00:00
Kernel: Implement lazy committed page allocation
By designating a committed page pool we can guarantee to have physical pages available for lazy allocation in mappings. However, when forking we will overcommit. The assumption is that worst-case it's better for the fork to die due to insufficient physical memory on COW access than the parent that created the region. If a fork wants to ensure that all memory is available (trigger a commit) then it can use madvise. This also means that fork now can gracefully fail if we don't have enough physical pages available.
This commit is contained in:
Tom
Andreas Kling
parent
e21cc4cff6
commit
b2a52f6208
Notes:
sideshowbarker
2024-07-19 00:14:34 +09:00
Author: https://github.com/tomuta Commit: https://github.com/SerenityOS/serenity/commit/b2a52f62089 Pull-request: https://github.com/SerenityOS/serenity/pull/4707
20 changed files with 329 additions and 67 deletions
|
|
@ -145,7 +145,9 @@ Region* Process::allocate_region(const Range& range, const String& name, int pro
|
|||
{
|
||||
ASSERT(range.is_valid());
|
||||
auto vmobject = PurgeableVMObject::create_with_size(range.size());
|
||||
auto region = Region::create_user_accessible(this, range, vmobject, 0, name, prot_to_region_access_flags(prot));
|
||||
if (!vmobject)
|
||||
return nullptr;
|
||||
auto region = Region::create_user_accessible(this, range, vmobject.release_nonnull(), 0, name, prot_to_region_access_flags(prot));
|
||||
if (!region->map(page_directory()))
|
||||
return nullptr;
|
||||
if (should_commit && region->can_commit() && !region->commit())
|
||||
|
|
|
|||
|
|
@ -48,12 +48,12 @@ private:
|
|||
};
|
||||
|
||||
public:
|
||||
SharedBuffer(int id, int size)
|
||||
SharedBuffer(int id, NonnullRefPtr<PurgeableVMObject>&& vmobject)
|
||||
: m_shbuf_id(id)
|
||||
, m_vmobject(PurgeableVMObject::create_with_size(size))
|
||||
, m_vmobject(move(vmobject))
|
||||
{
|
||||
#ifdef SHARED_BUFFER_DEBUG
|
||||
dbg() << "Created shared buffer " << m_shbuf_id << " of size " << size;
|
||||
dbg() << "Created shared buffer " << m_shbuf_id << " of size " << m_vmobject->size();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -88,7 +88,14 @@ pid_t Process::sys$fork(RegisterState& regs)
|
|||
#ifdef FORK_DEBUG
|
||||
dbg() << "fork: cloning Region{" << ®ion << "} '" << region.name() << "' @ " << region.vaddr();
|
||||
#endif
|
||||
auto& child_region = child->add_region(region.clone());
|
||||
auto region_clone = region.clone();
|
||||
if (!region_clone) {
|
||||
dbg() << "fork: Cannot clone region, insufficient memory";
|
||||
// TODO: tear down new process?
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
auto& child_region = child->add_region(region_clone.release_nonnull());
|
||||
child_region.map(child->page_directory());
|
||||
|
||||
if (®ion == m_master_tls_region.unsafe_ptr())
|
||||
|
|
|
|||
|
|
@ -52,10 +52,14 @@ int Process::sys$shbuf_create(int size, void** buffer)
|
|||
return -EINVAL;
|
||||
size = PAGE_ROUND_UP(size);
|
||||
|
||||
auto vmobject = PurgeableVMObject::create_with_size(size);
|
||||
if (!vmobject)
|
||||
return -ENOMEM;
|
||||
|
||||
LOCKER(shared_buffers().lock());
|
||||
static int s_next_shbuf_id;
|
||||
int shbuf_id = ++s_next_shbuf_id;
|
||||
auto shared_buffer = make<SharedBuffer>(shbuf_id, size);
|
||||
auto shared_buffer = make<SharedBuffer>(shbuf_id, vmobject.release_nonnull());
|
||||
shared_buffer->share_with(m_pid);
|
||||
|
||||
void* address = shared_buffer->ref_for_process_and_get_address(*this);
|
||||
|
|
|
|||
|
|
@ -51,13 +51,15 @@ NonnullRefPtr<AnonymousVMObject> AnonymousVMObject::create_with_physical_page(Ph
|
|||
return vmobject;
|
||||
}
|
||||
|
||||
AnonymousVMObject::AnonymousVMObject(size_t size)
|
||||
AnonymousVMObject::AnonymousVMObject(size_t size, bool initialize_pages)
|
||||
: VMObject(size)
|
||||
{
|
||||
if (initialize_pages) {
|
||||
#ifndef MAP_SHARED_ZERO_PAGE_LAZILY
|
||||
for (size_t i = 0; i < page_count(); ++i)
|
||||
physical_pages()[i] = MM.shared_zero_page();
|
||||
for (size_t i = 0; i < page_count(); ++i)
|
||||
physical_pages()[i] = MM.shared_zero_page();
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
AnonymousVMObject::AnonymousVMObject(PhysicalAddress paddr, size_t size)
|
||||
|
|
@ -77,9 +79,14 @@ AnonymousVMObject::~AnonymousVMObject()
|
|||
{
|
||||
}
|
||||
|
||||
NonnullRefPtr<VMObject> AnonymousVMObject::clone()
|
||||
RefPtr<VMObject> AnonymousVMObject::clone()
|
||||
{
|
||||
return adopt(*new AnonymousVMObject(*this));
|
||||
}
|
||||
|
||||
RefPtr<PhysicalPage> AnonymousVMObject::allocate_committed_page(size_t)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,10 +38,12 @@ public:
|
|||
static NonnullRefPtr<AnonymousVMObject> create_with_size(size_t);
|
||||
static RefPtr<AnonymousVMObject> create_for_physical_range(PhysicalAddress, size_t);
|
||||
static NonnullRefPtr<AnonymousVMObject> create_with_physical_page(PhysicalPage&);
|
||||
virtual NonnullRefPtr<VMObject> clone() override;
|
||||
virtual RefPtr<VMObject> clone() override;
|
||||
|
||||
virtual RefPtr<PhysicalPage> allocate_committed_page(size_t);
|
||||
|
||||
protected:
|
||||
explicit AnonymousVMObject(size_t);
|
||||
explicit AnonymousVMObject(size_t, bool initialize_pages = true);
|
||||
explicit AnonymousVMObject(const AnonymousVMObject&);
|
||||
|
||||
virtual const char* class_name() const override { return "AnonymousVMObject"; }
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ ContiguousVMObject::~ContiguousVMObject()
|
|||
{
|
||||
}
|
||||
|
||||
NonnullRefPtr<VMObject> ContiguousVMObject::clone()
|
||||
RefPtr<VMObject> ContiguousVMObject::clone()
|
||||
{
|
||||
ASSERT_NOT_REACHED();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ private:
|
|||
explicit ContiguousVMObject(const ContiguousVMObject&);
|
||||
|
||||
virtual const char* class_name() const override { return "ContiguousVMObject"; }
|
||||
virtual NonnullRefPtr<VMObject> clone() override;
|
||||
virtual RefPtr<VMObject> clone() override;
|
||||
|
||||
ContiguousVMObject& operator=(const ContiguousVMObject&) = delete;
|
||||
ContiguousVMObject& operator=(ContiguousVMObject&&) = delete;
|
||||
|
|
|
|||
|
|
@ -78,7 +78,19 @@ MemoryManager::MemoryManager()
|
|||
write_cr3(kernel_page_directory().cr3());
|
||||
protect_kernel_image();
|
||||
|
||||
m_shared_zero_page = allocate_user_physical_page();
|
||||
// We're temporarily "committing" to two pages that we need to allocate below
|
||||
if (!commit_user_physical_pages(2))
|
||||
ASSERT_NOT_REACHED();
|
||||
|
||||
m_shared_zero_page = allocate_committed_user_physical_page();
|
||||
|
||||
// We're wasting a page here, we just need a special tag (physical
|
||||
// address) so that we know when we need to lazily allocate a page
|
||||
// that we should be drawing this page from the committed pool rather
|
||||
// than potentially failing if no pages are available anymore.
|
||||
// By using a tag we don't have to query the VMObject for every page
|
||||
// whether it was committed or not
|
||||
m_lazy_committed_page = allocate_committed_user_physical_page();
|
||||
}
|
||||
|
||||
MemoryManager::~MemoryManager()
|
||||
|
|
@ -192,6 +204,9 @@ void MemoryManager::parse_memory_map()
|
|||
|
||||
ASSERT(m_super_physical_pages > 0);
|
||||
ASSERT(m_user_physical_pages > 0);
|
||||
|
||||
// We start out with no committed pages
|
||||
m_user_physical_pages_uncommitted = m_user_physical_pages;
|
||||
}
|
||||
|
||||
PageTableEntry* MemoryManager::pte(PageDirectory& page_directory, VirtualAddress vaddr)
|
||||
|
|
@ -469,6 +484,28 @@ OwnPtr<Region> MemoryManager::allocate_kernel_region_with_vmobject(VMObject& vmo
|
|||
return allocate_kernel_region_with_vmobject(range, vmobject, name, access, user_accessible, cacheable);
|
||||
}
|
||||
|
||||
bool MemoryManager::commit_user_physical_pages(size_t page_count)
|
||||
{
|
||||
ASSERT(page_count > 0);
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
if (m_user_physical_pages_uncommitted < page_count)
|
||||
return false;
|
||||
|
||||
m_user_physical_pages_uncommitted -= page_count;
|
||||
m_user_physical_pages_committed += page_count;
|
||||
return true;
|
||||
}
|
||||
|
||||
void MemoryManager::uncommit_user_physical_pages(size_t page_count)
|
||||
{
|
||||
ASSERT(page_count > 0);
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
ASSERT(m_user_physical_pages_committed >= page_count);
|
||||
|
||||
m_user_physical_pages_uncommitted += page_count;
|
||||
m_user_physical_pages_committed -= page_count;
|
||||
}
|
||||
|
||||
void MemoryManager::deallocate_user_physical_page(const PhysicalPage& page)
|
||||
{
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
|
|
@ -481,6 +518,10 @@ void MemoryManager::deallocate_user_physical_page(const PhysicalPage& page)
|
|||
region.return_page(page);
|
||||
--m_user_physical_pages_used;
|
||||
|
||||
// Always return pages to the uncommitted pool. Pages that were
|
||||
// committed and allocated are only freed upon request. Once
|
||||
// returned there is no guarantee being able to get them back.
|
||||
++m_user_physical_pages_uncommitted;
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -488,22 +529,47 @@ void MemoryManager::deallocate_user_physical_page(const PhysicalPage& page)
|
|||
ASSERT_NOT_REACHED();
|
||||
}
|
||||
|
||||
RefPtr<PhysicalPage> MemoryManager::find_free_user_physical_page()
|
||||
RefPtr<PhysicalPage> MemoryManager::find_free_user_physical_page(bool committed)
|
||||
{
|
||||
ASSERT(s_mm_lock.is_locked());
|
||||
RefPtr<PhysicalPage> page;
|
||||
if (committed) {
|
||||
// Draw from the committed pages pool. We should always have these pages available
|
||||
ASSERT(m_user_physical_pages_committed > 0);
|
||||
m_user_physical_pages_committed--;
|
||||
} else {
|
||||
// We need to make sure we don't touch pages that we have committed to
|
||||
if (m_user_physical_pages_uncommitted == 0)
|
||||
return {};
|
||||
m_user_physical_pages_uncommitted--;
|
||||
}
|
||||
for (auto& region : m_user_physical_regions) {
|
||||
page = region.take_free_page(false);
|
||||
if (!page.is_null())
|
||||
if (!page.is_null()) {
|
||||
++m_user_physical_pages_used;
|
||||
break;
|
||||
}
|
||||
}
|
||||
ASSERT(!committed || !page.is_null());
|
||||
return page;
|
||||
}
|
||||
|
||||
NonnullRefPtr<PhysicalPage> MemoryManager::allocate_committed_user_physical_page(ShouldZeroFill should_zero_fill)
|
||||
{
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
auto page = find_free_user_physical_page(true);
|
||||
if (should_zero_fill == ShouldZeroFill::Yes) {
|
||||
auto* ptr = quickmap_page(*page);
|
||||
memset(ptr, 0, PAGE_SIZE);
|
||||
unquickmap_page();
|
||||
}
|
||||
return page.release_nonnull();
|
||||
}
|
||||
|
||||
RefPtr<PhysicalPage> MemoryManager::allocate_user_physical_page(ShouldZeroFill should_zero_fill, bool* did_purge)
|
||||
{
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
auto page = find_free_user_physical_page();
|
||||
auto page = find_free_user_physical_page(false);
|
||||
bool purged_pages = false;
|
||||
|
||||
if (!page) {
|
||||
|
|
@ -515,7 +581,7 @@ RefPtr<PhysicalPage> MemoryManager::allocate_user_physical_page(ShouldZeroFill s
|
|||
int purged_page_count = static_cast<PurgeableVMObject&>(vmobject).purge_with_interrupts_disabled({});
|
||||
if (purged_page_count) {
|
||||
klog() << "MM: Purge saved the day! Purged " << purged_page_count << " pages from PurgeableVMObject{" << &vmobject << "}";
|
||||
page = find_free_user_physical_page();
|
||||
page = find_free_user_physical_page(false);
|
||||
purged_pages = true;
|
||||
ASSERT(page);
|
||||
return IterationDecision::Break;
|
||||
|
|
@ -541,8 +607,6 @@ RefPtr<PhysicalPage> MemoryManager::allocate_user_physical_page(ShouldZeroFill s
|
|||
|
||||
if (did_purge)
|
||||
*did_purge = purged_pages;
|
||||
|
||||
++m_user_physical_pages_used;
|
||||
return page;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -110,6 +110,9 @@ public:
|
|||
Yes
|
||||
};
|
||||
|
||||
bool commit_user_physical_pages(size_t);
|
||||
void uncommit_user_physical_pages(size_t);
|
||||
NonnullRefPtr<PhysicalPage> allocate_committed_user_physical_page(ShouldZeroFill = ShouldZeroFill::Yes);
|
||||
RefPtr<PhysicalPage> allocate_user_physical_page(ShouldZeroFill = ShouldZeroFill::Yes, bool* did_purge = nullptr);
|
||||
RefPtr<PhysicalPage> allocate_supervisor_physical_page();
|
||||
NonnullRefPtrVector<PhysicalPage> allocate_contiguous_supervisor_physical_pages(size_t size);
|
||||
|
|
@ -155,6 +158,7 @@ public:
|
|||
void dump_kernel_regions();
|
||||
|
||||
PhysicalPage& shared_zero_page() { return *m_shared_zero_page; }
|
||||
PhysicalPage& lazy_committed_page() { return *m_lazy_committed_page; }
|
||||
|
||||
PageDirectory& kernel_page_directory() { return *m_kernel_page_directory; }
|
||||
|
||||
|
|
@ -185,7 +189,7 @@ private:
|
|||
|
||||
static Region* find_region_from_vaddr(VirtualAddress);
|
||||
|
||||
RefPtr<PhysicalPage> find_free_user_physical_page();
|
||||
RefPtr<PhysicalPage> find_free_user_physical_page(bool);
|
||||
u8* quickmap_page(PhysicalPage&);
|
||||
void unquickmap_page();
|
||||
|
||||
|
|
@ -200,9 +204,12 @@ private:
|
|||
RefPtr<PhysicalPage> m_low_page_table;
|
||||
|
||||
RefPtr<PhysicalPage> m_shared_zero_page;
|
||||
RefPtr<PhysicalPage> m_lazy_committed_page;
|
||||
|
||||
unsigned m_user_physical_pages { 0 };
|
||||
unsigned m_user_physical_pages_used { 0 };
|
||||
unsigned m_user_physical_pages_committed { 0 };
|
||||
unsigned m_user_physical_pages_uncommitted { 0 };
|
||||
unsigned m_super_physical_pages { 0 };
|
||||
unsigned m_super_physical_pages_used { 0 };
|
||||
|
||||
|
|
@ -250,4 +257,9 @@ inline bool PhysicalPage::is_shared_zero_page() const
|
|||
return this == &MM.shared_zero_page();
|
||||
}
|
||||
|
||||
inline bool PhysicalPage::is_lazy_committed_page() const
|
||||
{
|
||||
return this == &MM.lazy_committed_page();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ public:
|
|||
u32 ref_count() const { return m_ref_count.load(AK::memory_order_consume); }
|
||||
|
||||
bool is_shared_zero_page() const;
|
||||
bool is_lazy_committed_page() const;
|
||||
|
||||
private:
|
||||
PhysicalPage(PhysicalAddress paddr, bool supervisor, bool may_return_to_freelist = true);
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ NonnullRefPtr<PrivateInodeVMObject> PrivateInodeVMObject::create_with_inode(Inod
|
|||
return adopt(*new PrivateInodeVMObject(inode, inode.size()));
|
||||
}
|
||||
|
||||
NonnullRefPtr<VMObject> PrivateInodeVMObject::clone()
|
||||
RefPtr<VMObject> PrivateInodeVMObject::clone()
|
||||
{
|
||||
return adopt(*new PrivateInodeVMObject(*this));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ public:
|
|||
virtual ~PrivateInodeVMObject() override;
|
||||
|
||||
static NonnullRefPtr<PrivateInodeVMObject> create_with_inode(Inode&);
|
||||
virtual NonnullRefPtr<VMObject> clone() override;
|
||||
virtual RefPtr<VMObject> clone() override;
|
||||
|
||||
private:
|
||||
virtual bool is_private_inode() const override { return true; }
|
||||
|
|
|
|||
|
|
@ -44,10 +44,10 @@ inline LogStream& operator<<(const LogStream& stream, const VolatilePageRange& r
|
|||
|
||||
static void dump_volatile_page_ranges(const Vector<VolatilePageRange>& ranges)
|
||||
{
|
||||
for (size_t i = 0; i < ranges.size(); i++) {
|
||||
const auto& range = ranges[i];
|
||||
klog() << " [" << i << "] " << range;
|
||||
}
|
||||
for (size_t i = 0; i < ranges.size(); i++) {
|
||||
const auto& range = ranges[i];
|
||||
klog() << " [" << i << "] " << range;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -185,7 +185,7 @@ bool VolatilePageRanges::intersects(const VolatilePageRange& range) const
|
|||
}
|
||||
|
||||
PurgeablePageRanges::PurgeablePageRanges(const VMObject& vmobject)
|
||||
: m_volatile_ranges({0, vmobject.is_purgeable() ? static_cast<const PurgeableVMObject&>(vmobject).page_count() : 0})
|
||||
: m_volatile_ranges({ 0, vmobject.is_purgeable() ? static_cast<const PurgeableVMObject&>(vmobject).page_count() : 0 })
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -193,8 +193,23 @@ bool PurgeablePageRanges::add_volatile_range(const VolatilePageRange& range)
|
|||
{
|
||||
if (range.is_empty())
|
||||
return false;
|
||||
|
||||
// Since we may need to call into PurgeableVMObject we need to acquire
|
||||
// its lock as well, and acquire it first. This is important so that
|
||||
// we don't deadlock when a page fault (e.g. on another processor)
|
||||
// happens that is meant to lazy-allocate a committed page. It would
|
||||
// call into PurgeableVMObject::range_made_volatile, which then would
|
||||
// also call into this object and need to acquire m_lock. By acquiring
|
||||
// the vmobject lock first in both cases, we avoid deadlocking.
|
||||
// We can access m_vmobject without any locks for that purpose because
|
||||
// add_volatile_range and remove_volatile_range can only be called
|
||||
// by same object that calls set_vmobject.
|
||||
ScopedSpinLock vmobject_lock(m_vmobject->m_lock);
|
||||
ScopedSpinLock lock(m_volatile_ranges_lock);
|
||||
return m_volatile_ranges.add(range);
|
||||
bool added = m_volatile_ranges.add(range);
|
||||
if (added)
|
||||
m_vmobject->range_made_volatile(range);
|
||||
return added;
|
||||
}
|
||||
|
||||
bool PurgeablePageRanges::remove_volatile_range(const VolatilePageRange& range, bool& was_purged)
|
||||
|
|
@ -202,6 +217,7 @@ bool PurgeablePageRanges::remove_volatile_range(const VolatilePageRange& range,
|
|||
if (range.is_empty())
|
||||
return false;
|
||||
ScopedSpinLock lock(m_volatile_ranges_lock);
|
||||
ASSERT(m_vmobject);
|
||||
return m_volatile_ranges.remove(range, was_purged);
|
||||
}
|
||||
|
||||
|
|
@ -213,35 +229,73 @@ bool PurgeablePageRanges::is_volatile_range(const VolatilePageRange& range) cons
|
|||
return m_volatile_ranges.intersects(range);
|
||||
}
|
||||
|
||||
bool PurgeablePageRanges::is_volatile(size_t index) const
|
||||
{
|
||||
ScopedSpinLock lock(m_volatile_ranges_lock);
|
||||
return m_volatile_ranges.contains(index);
|
||||
}
|
||||
|
||||
void PurgeablePageRanges::set_was_purged(const VolatilePageRange& range)
|
||||
{
|
||||
ScopedSpinLock lock(m_volatile_ranges_lock);
|
||||
m_volatile_ranges.add({range.base, range.count, true});
|
||||
m_volatile_ranges.add({ range.base, range.count, true });
|
||||
}
|
||||
|
||||
NonnullRefPtr<PurgeableVMObject> PurgeableVMObject::create_with_size(size_t size)
|
||||
void PurgeablePageRanges::set_vmobject(PurgeableVMObject* vmobject)
|
||||
{
|
||||
// No lock needed here
|
||||
if (vmobject) {
|
||||
ASSERT(!m_vmobject);
|
||||
m_vmobject = vmobject;
|
||||
} else {
|
||||
ASSERT(m_vmobject);
|
||||
m_vmobject = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
RefPtr<PurgeableVMObject> PurgeableVMObject::create_with_size(size_t size)
|
||||
{
|
||||
// We need to attempt to commit before actually creating the object
|
||||
if (!MM.commit_user_physical_pages(ceil_div(size, PAGE_SIZE)))
|
||||
return {};
|
||||
return adopt(*new PurgeableVMObject(size));
|
||||
}
|
||||
|
||||
PurgeableVMObject::PurgeableVMObject(size_t size)
|
||||
: AnonymousVMObject(size)
|
||||
: AnonymousVMObject(size, false)
|
||||
, m_unused_committed_pages(page_count())
|
||||
{
|
||||
for (size_t i = 0; i < page_count(); ++i)
|
||||
physical_pages()[i] = MM.lazy_committed_page();
|
||||
}
|
||||
|
||||
PurgeableVMObject::PurgeableVMObject(const PurgeableVMObject& other)
|
||||
: AnonymousVMObject(other)
|
||||
, m_purgeable_ranges() // do *not* clone this
|
||||
, m_unused_committed_pages(other.m_unused_committed_pages)
|
||||
{
|
||||
// TODO: what about m_lock?
|
||||
// We can't really "copy" a spinlock. But we're holding it. Clear in the clone
|
||||
ASSERT(other.m_lock.is_locked());
|
||||
m_lock.initialize();
|
||||
}
|
||||
|
||||
PurgeableVMObject::~PurgeableVMObject()
|
||||
{
|
||||
if (m_unused_committed_pages > 0)
|
||||
MM.uncommit_user_physical_pages(m_unused_committed_pages);
|
||||
}
|
||||
|
||||
NonnullRefPtr<VMObject> PurgeableVMObject::clone()
|
||||
RefPtr<VMObject> PurgeableVMObject::clone()
|
||||
{
|
||||
// We need to acquire our lock so we copy a sane state
|
||||
ScopedSpinLock lock(m_lock);
|
||||
if (m_unused_committed_pages > 0) {
|
||||
// We haven't used up all committed pages. In order to be able
|
||||
// to clone ourselves, we need to be able to commit the same number
|
||||
// of pages first
|
||||
if (!MM.commit_user_physical_pages(m_unused_committed_pages))
|
||||
return {};
|
||||
}
|
||||
return adopt(*new PurgeableVMObject(*this));
|
||||
}
|
||||
|
||||
|
|
@ -275,8 +329,10 @@ int PurgeableVMObject::purge_impl()
|
|||
auto range_end = range.base + range.count;
|
||||
for (size_t i = range.base; i < range_end; i++) {
|
||||
auto& phys_page = m_physical_pages[i];
|
||||
if (phys_page && !phys_page->is_shared_zero_page())
|
||||
if (phys_page && !phys_page->is_shared_zero_page()) {
|
||||
ASSERT(!phys_page->is_lazy_committed_page());
|
||||
++purged_in_range;
|
||||
}
|
||||
phys_page = MM.shared_zero_page();
|
||||
}
|
||||
|
||||
|
|
@ -291,7 +347,7 @@ int PurgeableVMObject::purge_impl()
|
|||
} else {
|
||||
klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " (no ownership) at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count);
|
||||
}
|
||||
region.remap_page_range(range.base, range.count, false);
|
||||
region.remap_page_range(range.base, range.count);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -303,6 +359,7 @@ int PurgeableVMObject::purge_impl()
|
|||
void PurgeableVMObject::register_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
|
||||
{
|
||||
ScopedSpinLock lock(m_lock);
|
||||
purgeable_page_ranges.set_vmobject(this);
|
||||
ASSERT(!m_purgeable_ranges.contains_slow(&purgeable_page_ranges));
|
||||
m_purgeable_ranges.append(&purgeable_page_ranges);
|
||||
}
|
||||
|
|
@ -313,6 +370,7 @@ void PurgeableVMObject::unregister_purgeable_page_ranges(PurgeablePageRanges& pu
|
|||
for (size_t i = 0; i < m_purgeable_ranges.size(); i++) {
|
||||
if (m_purgeable_ranges[i] != &purgeable_page_ranges)
|
||||
continue;
|
||||
purgeable_page_ranges.set_vmobject(nullptr);
|
||||
m_purgeable_ranges.remove(i);
|
||||
return;
|
||||
}
|
||||
|
|
@ -330,4 +388,72 @@ bool PurgeableVMObject::is_any_volatile() const
|
|||
return false;
|
||||
}
|
||||
|
||||
size_t PurgeableVMObject::remove_lazy_commit_pages(const VolatilePageRange& range)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
|
||||
size_t removed_count = 0;
|
||||
auto range_end = range.base + range.count;
|
||||
for (size_t i = range.base; i < range_end; i++) {
|
||||
auto& phys_page = m_physical_pages[i];
|
||||
if (phys_page && phys_page->is_lazy_committed_page()) {
|
||||
phys_page = MM.shared_zero_page();
|
||||
removed_count++;
|
||||
ASSERT(m_unused_committed_pages > 0);
|
||||
m_unused_committed_pages--;
|
||||
// if (--m_unused_committed_pages == 0)
|
||||
// break;
|
||||
}
|
||||
}
|
||||
return removed_count;
|
||||
}
|
||||
|
||||
void PurgeableVMObject::range_made_volatile(const VolatilePageRange& range)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
|
||||
if (m_unused_committed_pages == 0)
|
||||
return;
|
||||
|
||||
// We need to check this range for any pages that are marked for
|
||||
// lazy committed allocation and turn them into shared zero pages
|
||||
// and also adjust the m_unused_committed_pages for each such page.
|
||||
// Take into account all the other views as well.
|
||||
size_t uncommit_page_count = 0;
|
||||
for_each_volatile_range([&](const auto& r) {
|
||||
auto intersected = range.intersected(r);
|
||||
if (!intersected.is_empty()) {
|
||||
uncommit_page_count += remove_lazy_commit_pages(intersected);
|
||||
// if (m_unused_committed_pages == 0)
|
||||
// return IterationDecision::Break;
|
||||
}
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
|
||||
// Return those committed pages back to the system
|
||||
if (uncommit_page_count > 0)
|
||||
MM.uncommit_user_physical_pages(uncommit_page_count);
|
||||
}
|
||||
|
||||
RefPtr<PhysicalPage> PurgeableVMObject::allocate_committed_page(size_t page_index)
|
||||
{
|
||||
{
|
||||
ScopedSpinLock lock(m_lock);
|
||||
|
||||
ASSERT(m_unused_committed_pages > 0);
|
||||
|
||||
// We should't have any committed page tags in volatile regions
|
||||
ASSERT([&]() {
|
||||
for (auto* purgeable_ranges : m_purgeable_ranges) {
|
||||
if (purgeable_ranges->is_volatile(page_index))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}());
|
||||
|
||||
m_unused_committed_pages--;
|
||||
}
|
||||
return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -136,6 +136,10 @@ public:
|
|||
}
|
||||
|
||||
bool intersects(const VolatilePageRange&) const;
|
||||
bool contains(size_t index) const
|
||||
{
|
||||
return intersects({ index, 1 });
|
||||
}
|
||||
|
||||
bool add(const VolatilePageRange&);
|
||||
bool remove(const VolatilePageRange&, bool&);
|
||||
|
|
@ -152,6 +156,7 @@ class PurgeableVMObject;
|
|||
|
||||
class PurgeablePageRanges {
|
||||
friend class PurgeableVMObject;
|
||||
|
||||
public:
|
||||
PurgeablePageRanges(const VMObject&);
|
||||
|
||||
|
|
@ -168,23 +173,32 @@ public:
|
|||
bool add_volatile_range(const VolatilePageRange& range);
|
||||
bool remove_volatile_range(const VolatilePageRange& range, bool& was_purged);
|
||||
bool is_volatile_range(const VolatilePageRange& range) const;
|
||||
bool is_volatile(size_t) const;
|
||||
|
||||
bool is_empty() const { return m_volatile_ranges.is_empty(); }
|
||||
|
||||
void set_was_purged(const VolatilePageRange&);
|
||||
|
||||
const VolatilePageRanges& volatile_ranges() const { return m_volatile_ranges; }
|
||||
|
||||
protected:
|
||||
void set_vmobject(PurgeableVMObject*);
|
||||
|
||||
VolatilePageRanges m_volatile_ranges;
|
||||
mutable SpinLock<u8> m_volatile_ranges_lock;
|
||||
mutable RecursiveSpinLock m_volatile_ranges_lock;
|
||||
PurgeableVMObject* m_vmobject { nullptr };
|
||||
};
|
||||
|
||||
class PurgeableVMObject final : public AnonymousVMObject {
|
||||
friend class PurgeablePageRanges;
|
||||
|
||||
public:
|
||||
virtual ~PurgeableVMObject() override;
|
||||
|
||||
static NonnullRefPtr<PurgeableVMObject> create_with_size(size_t);
|
||||
virtual NonnullRefPtr<VMObject> clone() override;
|
||||
static RefPtr<PurgeableVMObject> create_with_size(size_t);
|
||||
virtual RefPtr<VMObject> clone() override;
|
||||
|
||||
virtual RefPtr<PhysicalPage> allocate_committed_page(size_t) override;
|
||||
|
||||
void register_purgeable_page_ranges(PurgeablePageRanges&);
|
||||
void unregister_purgeable_page_ranges(PurgeablePageRanges&);
|
||||
|
|
@ -202,11 +216,13 @@ public:
|
|||
// volatile ranges that all share, because those are the only
|
||||
// pages we can actually purge
|
||||
for (auto* purgeable_range : m_purgeable_ranges) {
|
||||
ScopedSpinLock purgeable_lock(purgeable_range->m_volatile_ranges_lock);
|
||||
for (auto& r1 : purgeable_range->volatile_ranges().ranges()) {
|
||||
VolatilePageRange range(r1);
|
||||
for (auto* purgeable_range2 : m_purgeable_ranges) {
|
||||
if (purgeable_range2 == purgeable_range)
|
||||
continue;
|
||||
ScopedSpinLock purgeable2_lock(purgeable_range2->m_volatile_ranges_lock);
|
||||
if (purgeable_range2->is_empty()) {
|
||||
// If just one doesn't allow any purging, we can
|
||||
// immediately bail
|
||||
|
|
@ -230,6 +246,8 @@ public:
|
|||
return IterationDecision::Continue;
|
||||
}
|
||||
|
||||
size_t get_lazy_committed_page_count() const;
|
||||
|
||||
private:
|
||||
explicit PurgeableVMObject(size_t);
|
||||
explicit PurgeableVMObject(const PurgeableVMObject&);
|
||||
|
|
@ -238,6 +256,8 @@ private:
|
|||
|
||||
int purge_impl();
|
||||
void set_was_purged(const VolatilePageRange&);
|
||||
size_t remove_lazy_commit_pages(const VolatilePageRange&);
|
||||
void range_made_volatile(const VolatilePageRange&);
|
||||
|
||||
PurgeableVMObject& operator=(const PurgeableVMObject&) = delete;
|
||||
PurgeableVMObject& operator=(PurgeableVMObject&&) = delete;
|
||||
|
|
@ -247,6 +267,7 @@ private:
|
|||
|
||||
Vector<PurgeablePageRanges*> m_purgeable_ranges;
|
||||
mutable SpinLock<u8> m_lock;
|
||||
size_t m_unused_committed_pages { 0 };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ void Region::unregister_purgeable_page_ranges()
|
|||
}
|
||||
}
|
||||
|
||||
NonnullOwnPtr<Region> Region::clone()
|
||||
OwnPtr<Region> Region::clone()
|
||||
{
|
||||
ASSERT(Process::current());
|
||||
|
||||
|
|
@ -122,13 +122,17 @@ NonnullOwnPtr<Region> Region::clone()
|
|||
if (vmobject().is_inode())
|
||||
ASSERT(vmobject().is_private_inode());
|
||||
|
||||
auto vmobject_clone = m_vmobject->clone();
|
||||
if (!vmobject_clone)
|
||||
return {};
|
||||
|
||||
#ifdef MM_DEBUG
|
||||
dbg() << "Region::clone(): CoWing " << name() << " (" << vaddr() << ")";
|
||||
#endif
|
||||
// Set up a COW region. The parent (this) region becomes COW as well!
|
||||
ensure_cow_map().fill(true);
|
||||
remap();
|
||||
auto clone_region = Region::create_user_accessible(get_owner().ptr(), m_range, m_vmobject->clone(), m_offset_in_vmobject, m_name, m_access);
|
||||
auto clone_region = Region::create_user_accessible(get_owner().ptr(), m_range, vmobject_clone.release_nonnull(), m_offset_in_vmobject, m_name, m_access);
|
||||
clone_region->set_purgeable_page_ranges(*this);
|
||||
clone_region->ensure_cow_map();
|
||||
if (m_stack) {
|
||||
|
|
@ -187,7 +191,7 @@ auto Region::set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, b
|
|||
// Attempt to remap the page range. We want to make sure we have
|
||||
// enough memory, if not we need to inform the caller of that
|
||||
// fact
|
||||
if (!remap_page_range(first_page_index, last_page_index - first_page_index, true))
|
||||
if (!remap_page_range(first_page_index, last_page_index - first_page_index))
|
||||
return SetVolatileError::OutOfMemory;
|
||||
}
|
||||
}
|
||||
|
|
@ -224,10 +228,15 @@ bool Region::commit(size_t page_index)
|
|||
auto& vmobject_physical_page_entry = physical_page_slot(page_index);
|
||||
if (!vmobject_physical_page_entry.is_null() && !vmobject_physical_page_entry->is_shared_zero_page())
|
||||
return true;
|
||||
auto physical_page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
||||
if (!physical_page) {
|
||||
klog() << "MM: commit was unable to allocate a physical page";
|
||||
return false;
|
||||
RefPtr<PhysicalPage> physical_page;
|
||||
if (vmobject_physical_page_entry->is_lazy_committed_page()) {
|
||||
physical_page = static_cast<AnonymousVMObject&>(*m_vmobject).allocate_committed_page(page_index);
|
||||
} else {
|
||||
physical_page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
||||
if (!physical_page) {
|
||||
klog() << "MM: commit was unable to allocate a physical page";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
vmobject_physical_page_entry = move(physical_page);
|
||||
remap_page(page_index, false); // caller is in charge of flushing tlb
|
||||
|
|
@ -292,7 +301,7 @@ NonnullOwnPtr<Region> Region::create_kernel_only(const Range& range, NonnullRefP
|
|||
bool Region::should_cow(size_t page_index) const
|
||||
{
|
||||
auto* page = physical_page(page_index);
|
||||
if (page && page->is_shared_zero_page())
|
||||
if (page && (page->is_shared_zero_page() || page->is_lazy_committed_page()))
|
||||
return true;
|
||||
if (m_shared)
|
||||
return false;
|
||||
|
|
@ -344,7 +353,7 @@ bool Region::map_individual_page_impl(size_t page_index)
|
|||
return true;
|
||||
}
|
||||
|
||||
bool Region::remap_page_range(size_t page_index, size_t page_count, bool do_commit)
|
||||
bool Region::remap_page_range(size_t page_index, size_t page_count)
|
||||
{
|
||||
bool success = true;
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
|
|
@ -352,10 +361,6 @@ bool Region::remap_page_range(size_t page_index, size_t page_count, bool do_comm
|
|||
ScopedSpinLock page_lock(m_page_directory->get_lock());
|
||||
size_t index = page_index;
|
||||
while (index < page_index + page_count) {
|
||||
if (do_commit && !commit(index)) {
|
||||
success = false;
|
||||
break;
|
||||
}
|
||||
if (!map_individual_page_impl(index)) {
|
||||
success = false;
|
||||
break;
|
||||
|
|
@ -455,9 +460,16 @@ PageFaultResponse Region::handle_fault(const PageFault& fault)
|
|||
#endif
|
||||
return handle_inode_fault(page_index_in_region);
|
||||
}
|
||||
|
||||
auto& page_slot = physical_page_slot(page_index_in_region);
|
||||
if (page_slot->is_lazy_committed_page()) {
|
||||
page_slot = static_cast<AnonymousVMObject&>(*m_vmobject).allocate_committed_page(page_index_in_region);
|
||||
remap_page(page_index_in_region);
|
||||
return PageFaultResponse::Continue;
|
||||
}
|
||||
#ifdef MAP_SHARED_ZERO_PAGE_LAZILY
|
||||
if (fault.is_read()) {
|
||||
physical_page_slot(page_index_in_region) = MM.shared_zero_page();
|
||||
page_slot = MM.shared_zero_page();
|
||||
remap_page(page_index_in_region);
|
||||
return PageFaultResponse::Continue;
|
||||
}
|
||||
|
|
@ -472,7 +484,8 @@ PageFaultResponse Region::handle_fault(const PageFault& fault)
|
|||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << "PV(cow) fault in Region{" << this << "}[" << page_index_in_region << "]";
|
||||
#endif
|
||||
if (physical_page(page_index_in_region)->is_shared_zero_page()) {
|
||||
auto* phys_page = physical_page(page_index_in_region);
|
||||
if (phys_page->is_shared_zero_page() || phys_page->is_lazy_committed_page()) {
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << "NP(zero) fault in Region{" << this << "}[" << page_index_in_region << "]";
|
||||
#endif
|
||||
|
|
@ -493,7 +506,7 @@ PageFaultResponse Region::handle_zero_fault(size_t page_index_in_region)
|
|||
|
||||
auto& page_slot = physical_page_slot(page_index_in_region);
|
||||
|
||||
if (!page_slot.is_null() && !page_slot->is_shared_zero_page()) {
|
||||
if (!page_slot.is_null() && !page_slot->is_shared_zero_page() && !page_slot->is_lazy_committed_page()) {
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << "MM: zero_page() but page already present. Fine with me!";
|
||||
#endif
|
||||
|
|
@ -506,16 +519,19 @@ PageFaultResponse Region::handle_zero_fault(size_t page_index_in_region)
|
|||
if (current_thread != nullptr)
|
||||
current_thread->did_zero_fault();
|
||||
|
||||
auto page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
||||
if (page.is_null()) {
|
||||
klog() << "MM: handle_zero_fault was unable to allocate a physical page";
|
||||
return PageFaultResponse::OutOfMemory;
|
||||
if (page_slot->is_lazy_committed_page()) {
|
||||
page_slot = static_cast<AnonymousVMObject&>(*m_vmobject).allocate_committed_page(page_index_in_region);
|
||||
} else {
|
||||
page_slot = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
||||
if (page_slot.is_null()) {
|
||||
klog() << "MM: handle_zero_fault was unable to allocate a physical page";
|
||||
return PageFaultResponse::OutOfMemory;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> ZERO " << page->paddr();
|
||||
dbg() << " >> ZERO " << page_slot->paddr();
|
||||
#endif
|
||||
page_slot = move(page);
|
||||
if (!remap_page(page_index_in_region)) {
|
||||
klog() << "MM: handle_zero_fault was unable to allocate a page table to map " << page_slot;
|
||||
return PageFaultResponse::OutOfMemory;
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ public:
|
|||
|
||||
PageFaultResponse handle_fault(const PageFault&);
|
||||
|
||||
NonnullOwnPtr<Region> clone();
|
||||
OwnPtr<Region> clone();
|
||||
|
||||
bool contains(VirtualAddress vaddr) const
|
||||
{
|
||||
|
|
@ -194,7 +194,7 @@ public:
|
|||
|
||||
void set_inherit_mode(InheritMode inherit_mode) { m_inherit_mode = inherit_mode; }
|
||||
|
||||
bool remap_page_range(size_t page_index, size_t page_count, bool do_commit);
|
||||
bool remap_page_range(size_t page_index, size_t page_count);
|
||||
|
||||
bool is_volatile(VirtualAddress vaddr, size_t size) const;
|
||||
enum class SetVolatileError {
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ NonnullRefPtr<SharedInodeVMObject> SharedInodeVMObject::create_with_inode(Inode&
|
|||
return vmobject;
|
||||
}
|
||||
|
||||
NonnullRefPtr<VMObject> SharedInodeVMObject::clone()
|
||||
RefPtr<VMObject> SharedInodeVMObject::clone()
|
||||
{
|
||||
return adopt(*new SharedInodeVMObject(*this));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ class SharedInodeVMObject final : public InodeVMObject {
|
|||
|
||||
public:
|
||||
static NonnullRefPtr<SharedInodeVMObject> create_with_inode(Inode&);
|
||||
virtual NonnullRefPtr<VMObject> clone() override;
|
||||
virtual RefPtr<VMObject> clone() override;
|
||||
|
||||
private:
|
||||
virtual bool is_shared_inode() const override { return true; }
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class VMObject : public RefCounted<VMObject>
|
|||
public:
|
||||
virtual ~VMObject();
|
||||
|
||||
virtual NonnullRefPtr<VMObject> clone() = 0;
|
||||
virtual RefPtr<VMObject> clone() = 0;
|
||||
|
||||
virtual bool is_anonymous() const { return false; }
|
||||
virtual bool is_purgeable() const { return false; }
|
||||
|
|
|
|||
Loading…
Reference in a new issue