2020-03-08 18:23:58 +00:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2020, Andreas Kling <kling@serenityos.org>
|
|
|
|
*
|
2021-04-22 08:24:48 +00:00
|
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
2020-03-08 18:23:58 +00:00
|
|
|
*/
|
|
|
|
|
2020-03-09 20:29:22 +00:00
|
|
|
#include <AK/Badge.h>
|
2021-01-24 14:28:26 +00:00
|
|
|
#include <AK/Debug.h>
|
2020-03-08 18:23:58 +00:00
|
|
|
#include <AK/HashTable.h>
|
2020-11-08 12:48:16 +00:00
|
|
|
#include <AK/StackInfo.h>
|
2020-10-15 18:46:52 +00:00
|
|
|
#include <AK/TemporaryChange.h>
|
2020-08-16 18:33:56 +00:00
|
|
|
#include <LibCore/ElapsedTimer.h>
|
2020-10-06 16:50:47 +00:00
|
|
|
#include <LibJS/Heap/Allocator.h>
|
2020-03-18 19:03:17 +00:00
|
|
|
#include <LibJS/Heap/Handle.h>
|
2020-03-16 13:20:30 +00:00
|
|
|
#include <LibJS/Heap/Heap.h>
|
|
|
|
#include <LibJS/Heap/HeapBlock.h>
|
2020-03-08 18:23:58 +00:00
|
|
|
#include <LibJS/Interpreter.h>
|
2020-03-16 13:20:30 +00:00
|
|
|
#include <LibJS/Runtime/Object.h>
|
2020-03-16 18:08:59 +00:00
|
|
|
#include <setjmp.h>
|
2020-03-08 18:23:58 +00:00
|
|
|
|
|
|
|
namespace JS {
|
|
|
|
|
2020-09-20 17:24:44 +00:00
|
|
|
Heap::Heap(VM& vm)
|
|
|
|
: m_vm(vm)
|
2020-03-08 18:23:58 +00:00
|
|
|
{
|
2020-10-06 16:50:47 +00:00
|
|
|
m_allocators.append(make<Allocator>(16));
|
|
|
|
m_allocators.append(make<Allocator>(32));
|
|
|
|
m_allocators.append(make<Allocator>(64));
|
|
|
|
m_allocators.append(make<Allocator>(128));
|
|
|
|
m_allocators.append(make<Allocator>(256));
|
|
|
|
m_allocators.append(make<Allocator>(512));
|
|
|
|
m_allocators.append(make<Allocator>(1024));
|
2021-05-12 12:34:33 +00:00
|
|
|
m_allocators.append(make<Allocator>(3072));
|
2020-03-08 18:23:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Heap::~Heap()
|
|
|
|
{
|
2020-03-23 13:11:19 +00:00
|
|
|
collect_garbage(CollectionType::CollectEverything);
|
2020-03-08 18:23:58 +00:00
|
|
|
}
|
|
|
|
|
2020-10-06 16:50:47 +00:00
|
|
|
ALWAYS_INLINE Allocator& Heap::allocator_for_size(size_t cell_size)
|
|
|
|
{
|
|
|
|
for (auto& allocator : m_allocators) {
|
|
|
|
if (allocator->cell_size() >= cell_size)
|
|
|
|
return *allocator;
|
|
|
|
}
|
2021-02-23 19:42:32 +00:00
|
|
|
VERIFY_NOT_REACHED();
|
2020-10-06 16:50:47 +00:00
|
|
|
}
|
|
|
|
|
2020-03-08 18:23:58 +00:00
|
|
|
Cell* Heap::allocate_cell(size_t size)
|
|
|
|
{
|
2020-04-06 10:36:49 +00:00
|
|
|
if (should_collect_on_every_allocation()) {
|
2020-03-16 18:18:46 +00:00
|
|
|
collect_garbage();
|
2020-04-06 10:36:49 +00:00
|
|
|
} else if (m_allocations_since_last_gc > m_max_allocations_between_gc) {
|
|
|
|
m_allocations_since_last_gc = 0;
|
|
|
|
collect_garbage();
|
|
|
|
} else {
|
|
|
|
++m_allocations_since_last_gc;
|
|
|
|
}
|
2020-03-16 18:18:46 +00:00
|
|
|
|
2020-10-06 16:50:47 +00:00
|
|
|
auto& allocator = allocator_for_size(size);
|
|
|
|
return allocator.allocate_cell(*this);
|
2020-03-08 18:23:58 +00:00
|
|
|
}
|
|
|
|
|
2020-08-16 18:33:56 +00:00
|
|
|
void Heap::collect_garbage(CollectionType collection_type, bool print_report)
|
2020-03-08 18:23:58 +00:00
|
|
|
{
|
2021-02-23 19:42:32 +00:00
|
|
|
VERIFY(!m_collecting_garbage);
|
2020-09-21 12:35:19 +00:00
|
|
|
TemporaryChange change(m_collecting_garbage, true);
|
|
|
|
|
2020-08-16 18:33:56 +00:00
|
|
|
Core::ElapsedTimer collection_measurement_timer;
|
|
|
|
collection_measurement_timer.start();
|
2020-03-23 13:11:19 +00:00
|
|
|
if (collection_type == CollectionType::CollectGarbage) {
|
2020-04-19 09:30:47 +00:00
|
|
|
if (m_gc_deferrals) {
|
|
|
|
m_should_gc_when_deferral_ends = true;
|
|
|
|
return;
|
|
|
|
}
|
2020-03-23 13:11:19 +00:00
|
|
|
HashTable<Cell*> roots;
|
|
|
|
gather_roots(roots);
|
|
|
|
mark_live_cells(roots);
|
|
|
|
}
|
2020-08-16 18:33:56 +00:00
|
|
|
sweep_dead_cells(print_report, collection_measurement_timer);
|
2020-03-08 18:23:58 +00:00
|
|
|
}
|
|
|
|
|
2020-03-15 14:12:34 +00:00
|
|
|
void Heap::gather_roots(HashTable<Cell*>& roots)
|
2020-03-08 18:23:58 +00:00
|
|
|
{
|
2020-09-21 11:47:33 +00:00
|
|
|
vm().gather_roots(roots);
|
2020-03-16 18:08:59 +00:00
|
|
|
gather_conservative_roots(roots);
|
|
|
|
|
2020-03-18 19:03:17 +00:00
|
|
|
for (auto* handle : m_handles)
|
|
|
|
roots.set(handle->cell());
|
|
|
|
|
2020-04-19 15:24:56 +00:00
|
|
|
for (auto* list : m_marked_value_lists) {
|
|
|
|
for (auto& value : list->values()) {
|
|
|
|
if (value.is_cell())
|
|
|
|
roots.set(value.as_cell());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-04-18 16:12:33 +00:00
|
|
|
if constexpr (HEAP_DEBUG) {
|
|
|
|
dbgln("gather_roots:");
|
|
|
|
for (auto* root : roots)
|
|
|
|
dbgln(" + {}", root);
|
|
|
|
}
|
2020-03-08 18:23:58 +00:00
|
|
|
}
|
|
|
|
|
2021-01-05 11:26:23 +00:00
|
|
|
__attribute__((no_sanitize("address"))) void Heap::gather_conservative_roots(HashTable<Cell*>& roots)
|
2020-03-16 18:08:59 +00:00
|
|
|
{
|
|
|
|
FlatPtr dummy;
|
|
|
|
|
2021-04-07 13:12:32 +00:00
|
|
|
dbgln_if(HEAP_DEBUG, "gather_conservative_roots:");
|
2020-03-16 18:08:59 +00:00
|
|
|
|
|
|
|
jmp_buf buf;
|
|
|
|
setjmp(buf);
|
|
|
|
|
|
|
|
HashTable<FlatPtr> possible_pointers;
|
|
|
|
|
2020-03-23 12:14:57 +00:00
|
|
|
const FlatPtr* raw_jmp_buf = reinterpret_cast<const FlatPtr*>(buf);
|
|
|
|
|
2020-05-08 18:01:43 +00:00
|
|
|
for (size_t i = 0; i < ((size_t)sizeof(buf)) / sizeof(FlatPtr); i += sizeof(FlatPtr))
|
2020-03-23 12:14:57 +00:00
|
|
|
possible_pointers.set(raw_jmp_buf[i]);
|
2020-03-16 18:08:59 +00:00
|
|
|
|
|
|
|
FlatPtr stack_reference = reinterpret_cast<FlatPtr>(&dummy);
|
2020-11-08 12:48:16 +00:00
|
|
|
auto& stack_info = m_vm.stack_info();
|
2020-03-16 18:08:59 +00:00
|
|
|
|
2020-11-08 12:48:16 +00:00
|
|
|
for (FlatPtr stack_address = stack_reference; stack_address < stack_info.top(); stack_address += sizeof(FlatPtr)) {
|
2020-03-16 18:08:59 +00:00
|
|
|
auto data = *reinterpret_cast<FlatPtr*>(stack_address);
|
|
|
|
possible_pointers.set(data);
|
|
|
|
}
|
|
|
|
|
2020-11-10 19:24:08 +00:00
|
|
|
HashTable<HeapBlock*> all_live_heap_blocks;
|
|
|
|
for_each_block([&](auto& block) {
|
|
|
|
all_live_heap_blocks.set(&block);
|
|
|
|
return IterationDecision::Continue;
|
|
|
|
});
|
|
|
|
|
2020-03-16 18:08:59 +00:00
|
|
|
for (auto possible_pointer : possible_pointers) {
|
|
|
|
if (!possible_pointer)
|
|
|
|
continue;
|
2021-04-07 13:12:32 +00:00
|
|
|
dbgln_if(HEAP_DEBUG, " ? {}", (const void*)possible_pointer);
|
2020-11-10 19:24:08 +00:00
|
|
|
auto* possible_heap_block = HeapBlock::from_cell(reinterpret_cast<const Cell*>(possible_pointer));
|
|
|
|
if (all_live_heap_blocks.contains(possible_heap_block)) {
|
|
|
|
if (auto* cell = possible_heap_block->cell_from_possible_pointer(possible_pointer)) {
|
2021-05-25 16:35:27 +00:00
|
|
|
if (cell->state() == Cell::State::Live) {
|
2021-04-07 13:12:32 +00:00
|
|
|
dbgln_if(HEAP_DEBUG, " ?-> {}", (const void*)cell);
|
2020-11-10 19:24:08 +00:00
|
|
|
roots.set(cell);
|
|
|
|
} else {
|
2021-04-07 13:12:32 +00:00
|
|
|
dbgln_if(HEAP_DEBUG, " #-> {}", (const void*)cell);
|
2020-11-10 19:24:08 +00:00
|
|
|
}
|
2020-03-16 18:08:59 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-09 21:11:22 +00:00
|
|
|
class MarkingVisitor final : public Cell::Visitor {
|
2020-03-08 18:23:58 +00:00
|
|
|
public:
|
2020-05-04 11:30:40 +00:00
|
|
|
MarkingVisitor() { }
|
2020-03-08 18:23:58 +00:00
|
|
|
|
2020-04-16 14:07:50 +00:00
|
|
|
virtual void visit_impl(Cell* cell)
|
2020-03-08 18:23:58 +00:00
|
|
|
{
|
2020-03-09 21:11:22 +00:00
|
|
|
if (cell->is_marked())
|
|
|
|
return;
|
2021-04-07 13:12:32 +00:00
|
|
|
dbgln_if(HEAP_DEBUG, " ! {}", cell);
|
2020-03-09 21:11:22 +00:00
|
|
|
cell->set_marked(true);
|
2020-11-28 13:33:36 +00:00
|
|
|
cell->visit_edges(*this);
|
2020-03-09 21:11:22 +00:00
|
|
|
}
|
|
|
|
};
|
2020-03-08 18:23:58 +00:00
|
|
|
|
2020-03-09 21:11:22 +00:00
|
|
|
void Heap::mark_live_cells(const HashTable<Cell*>& roots)
|
2020-03-08 18:23:58 +00:00
|
|
|
{
|
2021-04-07 13:12:32 +00:00
|
|
|
dbgln_if(HEAP_DEBUG, "mark_live_cells:");
|
2020-03-09 21:11:22 +00:00
|
|
|
MarkingVisitor visitor;
|
2020-04-16 14:07:50 +00:00
|
|
|
for (auto* root : roots)
|
2020-03-09 21:11:22 +00:00
|
|
|
visitor.visit(root);
|
2020-03-08 18:23:58 +00:00
|
|
|
}
|
|
|
|
|
2020-08-16 18:33:56 +00:00
|
|
|
void Heap::sweep_dead_cells(bool print_report, const Core::ElapsedTimer& measurement_timer)
|
2020-03-08 18:23:58 +00:00
|
|
|
{
|
2021-04-07 13:12:32 +00:00
|
|
|
dbgln_if(HEAP_DEBUG, "sweep_dead_cells:");
|
2020-03-21 10:45:50 +00:00
|
|
|
Vector<HeapBlock*, 32> empty_blocks;
|
2020-10-06 16:50:47 +00:00
|
|
|
Vector<HeapBlock*, 32> full_blocks_that_became_usable;
|
2020-03-21 10:45:50 +00:00
|
|
|
|
2020-08-16 18:33:56 +00:00
|
|
|
size_t collected_cells = 0;
|
|
|
|
size_t live_cells = 0;
|
|
|
|
size_t collected_cell_bytes = 0;
|
|
|
|
size_t live_cell_bytes = 0;
|
|
|
|
|
2020-10-06 16:50:47 +00:00
|
|
|
for_each_block([&](auto& block) {
|
2020-03-21 10:45:50 +00:00
|
|
|
bool block_has_live_cells = false;
|
2020-10-06 16:50:47 +00:00
|
|
|
bool block_was_full = block.is_full();
|
2021-05-25 16:35:27 +00:00
|
|
|
block.template for_each_cell_in_state<Cell::State::Live>([&](Cell* cell) {
|
|
|
|
if (!cell->is_marked()) {
|
|
|
|
dbgln_if(HEAP_DEBUG, " ~ {}", cell);
|
|
|
|
block.deallocate(cell);
|
|
|
|
++collected_cells;
|
|
|
|
collected_cell_bytes += block.cell_size();
|
|
|
|
} else {
|
|
|
|
cell->set_marked(false);
|
|
|
|
block_has_live_cells = true;
|
|
|
|
++live_cells;
|
|
|
|
live_cell_bytes += block.cell_size();
|
2020-03-08 18:23:58 +00:00
|
|
|
}
|
|
|
|
});
|
2020-03-21 10:45:50 +00:00
|
|
|
if (!block_has_live_cells)
|
2020-10-06 16:50:47 +00:00
|
|
|
empty_blocks.append(&block);
|
|
|
|
else if (block_was_full != block.is_full())
|
|
|
|
full_blocks_that_became_usable.append(&block);
|
|
|
|
return IterationDecision::Continue;
|
|
|
|
});
|
2020-03-21 10:45:50 +00:00
|
|
|
|
|
|
|
for (auto* block : empty_blocks) {
|
2021-04-07 13:12:32 +00:00
|
|
|
dbgln_if(HEAP_DEBUG, " - HeapBlock empty @ {}: cell_size={}", block, block->cell_size());
|
2020-10-06 16:50:47 +00:00
|
|
|
allocator_for_size(block->cell_size()).block_did_become_empty({}, *block);
|
2020-03-21 10:45:50 +00:00
|
|
|
}
|
|
|
|
|
2020-10-06 16:50:47 +00:00
|
|
|
for (auto* block : full_blocks_that_became_usable) {
|
2021-04-07 13:12:32 +00:00
|
|
|
dbgln_if(HEAP_DEBUG, " - HeapBlock usable again @ {}: cell_size={}", block, block->cell_size());
|
2020-10-06 16:50:47 +00:00
|
|
|
allocator_for_size(block->cell_size()).block_did_become_usable({}, *block);
|
2020-03-08 18:23:58 +00:00
|
|
|
}
|
2020-10-06 16:50:47 +00:00
|
|
|
|
2021-04-18 16:12:33 +00:00
|
|
|
if constexpr (HEAP_DEBUG) {
|
|
|
|
for_each_block([&](auto& block) {
|
|
|
|
dbgln(" > Live HeapBlock @ {}: cell_size={}", &block, block.cell_size());
|
|
|
|
return IterationDecision::Continue;
|
|
|
|
});
|
|
|
|
}
|
2020-08-16 18:33:56 +00:00
|
|
|
|
|
|
|
int time_spent = measurement_timer.elapsed();
|
|
|
|
|
|
|
|
if (print_report) {
|
2020-10-06 16:50:47 +00:00
|
|
|
size_t live_block_count = 0;
|
|
|
|
for_each_block([&](auto&) {
|
|
|
|
++live_block_count;
|
|
|
|
return IterationDecision::Continue;
|
|
|
|
});
|
|
|
|
|
2020-10-04 14:44:40 +00:00
|
|
|
dbgln("Garbage collection report");
|
|
|
|
dbgln("=============================================");
|
|
|
|
dbgln(" Time spent: {} ms", time_spent);
|
|
|
|
dbgln(" Live cells: {} ({} bytes)", live_cells, live_cell_bytes);
|
|
|
|
dbgln("Collected cells: {} ({} bytes)", collected_cells, collected_cell_bytes);
|
2020-10-06 16:50:47 +00:00
|
|
|
dbgln(" Live blocks: {} ({} bytes)", live_block_count, live_block_count * HeapBlock::block_size);
|
2020-10-04 14:44:40 +00:00
|
|
|
dbgln(" Freed blocks: {} ({} bytes)", empty_blocks.size(), empty_blocks.size() * HeapBlock::block_size);
|
|
|
|
dbgln("=============================================");
|
2020-08-16 18:33:56 +00:00
|
|
|
}
|
2020-03-08 18:23:58 +00:00
|
|
|
}
|
2020-03-18 19:03:17 +00:00
|
|
|
|
|
|
|
void Heap::did_create_handle(Badge<HandleImpl>, HandleImpl& impl)
|
|
|
|
{
|
2021-02-23 19:42:32 +00:00
|
|
|
VERIFY(!m_handles.contains(&impl));
|
2020-03-18 19:03:17 +00:00
|
|
|
m_handles.set(&impl);
|
|
|
|
}
|
|
|
|
|
|
|
|
void Heap::did_destroy_handle(Badge<HandleImpl>, HandleImpl& impl)
|
|
|
|
{
|
2021-02-23 19:42:32 +00:00
|
|
|
VERIFY(m_handles.contains(&impl));
|
2020-03-18 19:03:17 +00:00
|
|
|
m_handles.remove(&impl);
|
|
|
|
}
|
|
|
|
|
2020-04-19 15:24:56 +00:00
|
|
|
void Heap::did_create_marked_value_list(Badge<MarkedValueList>, MarkedValueList& list)
|
|
|
|
{
|
2021-02-23 19:42:32 +00:00
|
|
|
VERIFY(!m_marked_value_lists.contains(&list));
|
2020-04-19 15:24:56 +00:00
|
|
|
m_marked_value_lists.set(&list);
|
|
|
|
}
|
|
|
|
|
|
|
|
void Heap::did_destroy_marked_value_list(Badge<MarkedValueList>, MarkedValueList& list)
|
|
|
|
{
|
2021-02-23 19:42:32 +00:00
|
|
|
VERIFY(m_marked_value_lists.contains(&list));
|
2020-04-19 15:24:56 +00:00
|
|
|
m_marked_value_lists.remove(&list);
|
|
|
|
}
|
|
|
|
|
2020-04-19 09:30:47 +00:00
|
|
|
void Heap::defer_gc(Badge<DeferGC>)
|
|
|
|
{
|
|
|
|
++m_gc_deferrals;
|
|
|
|
}
|
|
|
|
|
|
|
|
void Heap::undefer_gc(Badge<DeferGC>)
|
|
|
|
{
|
2021-02-23 19:42:32 +00:00
|
|
|
VERIFY(m_gc_deferrals > 0);
|
2020-04-19 09:30:47 +00:00
|
|
|
--m_gc_deferrals;
|
|
|
|
|
|
|
|
if (!m_gc_deferrals) {
|
|
|
|
if (m_should_gc_when_deferral_ends)
|
|
|
|
collect_garbage();
|
|
|
|
m_should_gc_when_deferral_ends = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-08 18:23:58 +00:00
|
|
|
}
|