From 55e467c35985ba7bbf9a29bb06915461b3389b68 Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Thu, 9 Nov 2023 09:28:04 +0100 Subject: [PATCH] LibJS/JIT: Add fast path for cached PutById --- AK/Weakable.h | 2 + Userland/Libraries/LibJIT/X86_64/Assembler.h | 34 +++++ .../Libraries/LibJS/Bytecode/Executable.h | 4 + Userland/Libraries/LibJS/JIT/Compiler.cpp | 132 ++++++++++++++++-- Userland/Libraries/LibJS/JIT/Compiler.h | 20 ++- Userland/Libraries/LibJS/Runtime/Object.h | 4 + Userland/Libraries/LibJS/Runtime/Shape.h | 5 +- 7 files changed, 189 insertions(+), 12 deletions(-) diff --git a/AK/Weakable.h b/AK/Weakable.h index 38999445f16..8f841a4df63 100644 --- a/AK/Weakable.h +++ b/AK/Weakable.h @@ -42,6 +42,8 @@ public: void revoke() { m_ptr = nullptr; } + static FlatPtr ptr_offset() { return OFFSET_OF(WeakLink, m_ptr); } + private: template explicit WeakLink(T& weakable) diff --git a/Userland/Libraries/LibJIT/X86_64/Assembler.h b/Userland/Libraries/LibJIT/X86_64/Assembler.h index a0cc5c69150..eb1feaaca74 100644 --- a/Userland/Libraries/LibJIT/X86_64/Assembler.h +++ b/Userland/Libraries/LibJIT/X86_64/Assembler.h @@ -619,6 +619,23 @@ struct X86_64Assembler { } } + void shift_left(Operand dest, Optional count) + { + VERIFY(dest.type == Operand::Type::Reg); + if (count.has_value()) { + VERIFY(count->type == Operand::Type::Imm); + VERIFY(count->fits_in_u8()); + emit_rex_for_slash(dest, REX_W::Yes); + emit8(0xc1); + emit_modrm_slash(4, dest); + emit8(count->offset_or_immediate); + } else { + emit_rex_for_slash(dest, REX_W::Yes); + emit8(0xd3); + emit_modrm_slash(4, dest); + } + } + void shift_left32(Operand dest, Optional count) { VERIFY(dest.type == Operand::Type::Reg); @@ -653,6 +670,23 @@ struct X86_64Assembler { } } + void arithmetic_right_shift(Operand dest, Optional count) + { + VERIFY(dest.type == Operand::Type::Reg); + if (count.has_value()) { + VERIFY(count->type == Operand::Type::Imm); + VERIFY(count->fits_in_u8()); + emit_rex_for_slash(dest, REX_W::Yes); + emit8(0xc1); + emit_modrm_slash(7, dest); + emit8(count->offset_or_immediate); + } else { + emit_rex_for_slash(dest, REX_W::Yes); + emit8(0xd3); + emit_modrm_slash(7, dest); + } + } + void arithmetic_right_shift32(Operand dest, Optional count) { VERIFY(dest.type == Operand::Type::Reg); diff --git a/Userland/Libraries/LibJS/Bytecode/Executable.h b/Userland/Libraries/LibJS/Bytecode/Executable.h index 34959336d27..d2fad828587 100644 --- a/Userland/Libraries/LibJS/Bytecode/Executable.h +++ b/Userland/Libraries/LibJS/Bytecode/Executable.h @@ -24,6 +24,10 @@ class NativeExecutable; namespace JS::Bytecode { struct PropertyLookupCache { + static FlatPtr shape_offset() { return OFFSET_OF(PropertyLookupCache, shape); } + static FlatPtr property_offset_offset() { return OFFSET_OF(PropertyLookupCache, property_offset); } + static FlatPtr unique_shape_serial_number_offset() { return OFFSET_OF(PropertyLookupCache, unique_shape_serial_number); } + WeakPtr shape; Optional property_offset; u64 unique_shape_serial_number { 0 }; diff --git a/Userland/Libraries/LibJS/JIT/Compiler.cpp b/Userland/Libraries/LibJS/JIT/Compiler.cpp index 8a214e83b02..eb7d96d566f 100644 --- a/Userland/Libraries/LibJS/JIT/Compiler.cpp +++ b/Userland/Libraries/LibJS/JIT/Compiler.cpp @@ -302,22 +302,22 @@ void Compiler::jump_if_int32(Assembler::Reg reg, Assembler::Label& label) } template -void Compiler::branch_if_int32(Assembler::Reg reg, Codegen codegen) +void Compiler::branch_if_type(Assembler::Reg reg, u16 type_tag, Codegen codegen) { // GPR0 = reg >> 48; m_assembler.mov(Assembler::Operand::Register(GPR0), Assembler::Operand::Register(reg)); m_assembler.shift_right(Assembler::Operand::Register(GPR0), Assembler::Operand::Imm(48)); - Assembler::Label not_int32_case {}; + Assembler::Label not_type_case {}; m_assembler.jump_if( Assembler::Operand::Register(GPR0), Assembler::Condition::NotEqualTo, - Assembler::Operand::Imm(INT32_TAG), - not_int32_case); + Assembler::Operand::Imm(type_tag), + not_type_case); codegen(); - not_int32_case.link(m_assembler); + not_type_case.link(m_assembler); } template @@ -1347,9 +1347,125 @@ static Value cxx_put_by_id(VM& vm, Value base, DeprecatedFlyString const& proper return value; } +void Compiler::extract_object_pointer(Assembler::Reg dst_object, Assembler::Reg src_value) +{ + // This is basically Value::as_object() where src_value is the Value. + m_assembler.mov( + Assembler::Operand::Register(dst_object), + Assembler::Operand::Register(src_value)); + m_assembler.shift_left( + Assembler::Operand::Register(dst_object), + Assembler::Operand::Imm(16)); + m_assembler.arithmetic_right_shift( + Assembler::Operand::Register(dst_object), + Assembler::Operand::Imm(16)); +} + void Compiler::compile_put_by_id(Bytecode::Op::PutById const& op) { + auto& cache = m_bytecode_executable.property_lookup_caches[op.cache_index()]; + load_vm_register(ARG1, op.base()); + m_assembler.mov( + Assembler::Operand::Register(ARG5), + Assembler::Operand::Imm(bit_cast(&cache))); + + Assembler::Label end; + Assembler::Label slow_case; + if (op.kind() == Bytecode::Op::PropertyKind::KeyValue) { + + branch_if_object(ARG1, [&] { + extract_object_pointer(GPR0, ARG1); + + // if (cache.shape != &object->shape()) goto slow_case; + m_assembler.mov( + Assembler::Operand::Register(GPR2), + Assembler::Operand::Mem64BaseAndOffset(GPR0, Object::shape_offset())); + m_assembler.mov( + Assembler::Operand::Register(GPR1), + Assembler::Operand::Mem64BaseAndOffset(ARG5, Bytecode::PropertyLookupCache::shape_offset())); + + m_assembler.jump_if( + Assembler::Operand::Register(GPR1), + Assembler::Condition::EqualTo, + Assembler::Operand::Imm(0), + slow_case); + + m_assembler.mov( + Assembler::Operand::Register(GPR1), + Assembler::Operand::Mem64BaseAndOffset(GPR1, AK::WeakLink::ptr_offset())); + + m_assembler.jump_if( + Assembler::Operand::Register(GPR2), + Assembler::Condition::NotEqualTo, + Assembler::Operand::Register(GPR1), + slow_case); + + // (!object->shape().is_unique() || object->shape().unique_shape_serial_number() == cache.unique_shape_serial_number)) { + Assembler::Label fast_case; + + // GPR1 = object->shape().is_unique() + m_assembler.mov8( + Assembler::Operand::Register(GPR1), + Assembler::Operand::Mem64BaseAndOffset(GPR2, Shape::is_unique_offset())); + + m_assembler.jump_if( + Assembler::Operand::Register(GPR1), + Assembler::Condition::EqualTo, + Assembler::Operand::Imm(0), + fast_case); + + // GPR1 = object->shape().unique_shape_serial_number() + m_assembler.mov( + Assembler::Operand::Register(GPR1), + Assembler::Operand::Mem64BaseAndOffset(GPR2, Shape::unique_shape_serial_number_offset())); + + // GPR2 = cache.unique_shape_serial_number + m_assembler.mov( + Assembler::Operand::Register(GPR2), + Assembler::Operand::Mem64BaseAndOffset(ARG5, Bytecode::PropertyLookupCache::unique_shape_serial_number_offset())); + + // if (GPR1 != GPR2) goto slow_case; + m_assembler.jump_if( + Assembler::Operand::Register(GPR1), + Assembler::Condition::NotEqualTo, + Assembler::Operand::Register(GPR2), + slow_case); + + fast_case.link(m_assembler); + + // object->put_direct(*cache.property_offset, value); + // GPR0 = object + // GPR1 = *cache.property_offset * sizeof(Value) + m_assembler.mov( + Assembler::Operand::Register(GPR1), + Assembler::Operand::Mem64BaseAndOffset(ARG5, Bytecode::PropertyLookupCache::property_offset_offset() + decltype(cache.property_offset)::value_offset())); + m_assembler.mul32( + Assembler::Operand::Register(GPR1), + Assembler::Operand::Imm(sizeof(Value)), + slow_case); + + // GPR0 = object->m_storage.outline_buffer + m_assembler.mov( + Assembler::Operand::Register(GPR0), + Assembler::Operand::Mem64BaseAndOffset(GPR0, Object::storage_offset() + Vector::outline_buffer_offset())); + + // GPR0 = &object->m_storage.outline_buffer[*cache.property_offset] + m_assembler.add( + Assembler::Operand::Register(GPR0), + Assembler::Operand::Register(GPR1)); + + // *GPR0 = value + load_accumulator(GPR1); + m_assembler.mov( + Assembler::Operand::Mem64BaseAndOffset(GPR0, 0), + Assembler::Operand::Register(GPR1)); + + m_assembler.jump(end); + }); + } + + slow_case.link(m_assembler); m_assembler.mov( Assembler::Operand::Register(ARG2), Assembler::Operand::Imm(bit_cast(&m_bytecode_executable.get_identifier(op.property())))); @@ -1357,12 +1473,12 @@ void Compiler::compile_put_by_id(Bytecode::Op::PutById const& op) m_assembler.mov( Assembler::Operand::Register(ARG4), Assembler::Operand::Imm(to_underlying(op.kind()))); - m_assembler.mov( - Assembler::Operand::Register(ARG5), - Assembler::Operand::Imm(bit_cast(&m_bytecode_executable.property_lookup_caches[op.cache_index()]))); + native_call((void*)cxx_put_by_id); store_accumulator(RET); check_exception(); + + end.link(m_assembler); } static Value cxx_put_by_value(VM& vm, Value base, Value property, Value value, Bytecode::Op::PropertyKind kind) diff --git a/Userland/Libraries/LibJS/JIT/Compiler.h b/Userland/Libraries/LibJS/JIT/Compiler.h index 3dbacbb4e93..2ca2865f312 100644 --- a/Userland/Libraries/LibJS/JIT/Compiler.h +++ b/Userland/Libraries/LibJS/JIT/Compiler.h @@ -27,7 +27,7 @@ private: # if ARCH(X86_64) static constexpr auto GPR0 = Assembler::Reg::RAX; static constexpr auto GPR1 = Assembler::Reg::RCX; - static constexpr auto GPR2 = Assembler::Reg::R12; + static constexpr auto GPR2 = Assembler::Reg::R13; static constexpr auto ARG0 = Assembler::Reg::RDI; static constexpr auto ARG1 = Assembler::Reg::RSI; static constexpr auto ARG2 = Assembler::Reg::RDX; @@ -38,7 +38,7 @@ private: static constexpr auto STACK_POINTER = Assembler::Reg::RSP; static constexpr auto REGISTER_ARRAY_BASE = Assembler::Reg::RBX; static constexpr auto LOCALS_ARRAY_BASE = Assembler::Reg::R14; - static constexpr auto CACHED_ACCUMULATOR = Assembler::Reg::R13; + static constexpr auto CACHED_ACCUMULATOR = Assembler::Reg::R12; static constexpr auto RUNNING_EXECUTION_CONTEXT_BASE = Assembler::Reg::R15; # endif @@ -171,7 +171,21 @@ private: void jump_if_int32(Assembler::Reg, Assembler::Label&); template - void branch_if_int32(Assembler::Reg, Codegen); + void branch_if_type(Assembler::Reg, u16 type_tag, Codegen); + + template + void branch_if_int32(Assembler::Reg reg, Codegen codegen) + { + branch_if_type(reg, INT32_TAG, codegen); + } + + template + void branch_if_object(Assembler::Reg reg, Codegen codegen) + { + branch_if_type(reg, OBJECT_TAG, codegen); + } + + void extract_object_pointer(Assembler::Reg dst_object, Assembler::Reg src_value); template void branch_if_both_int32(Assembler::Reg, Assembler::Reg, Codegen); diff --git a/Userland/Libraries/LibJS/Runtime/Object.h b/Userland/Libraries/LibJS/Runtime/Object.h index 5892147d911..f6a79f572d9 100644 --- a/Userland/Libraries/LibJS/Runtime/Object.h +++ b/Userland/Libraries/LibJS/Runtime/Object.h @@ -195,6 +195,8 @@ public: Value get_direct(size_t index) const { return m_storage[index]; } void put_direct(size_t index, Value value) { m_storage[index] = value; } + static FlatPtr storage_offset() { return OFFSET_OF(Object, m_storage); } + IndexedProperties const& indexed_properties() const { return m_indexed_properties; } IndexedProperties& indexed_properties() { return m_indexed_properties; } void set_indexed_property_elements(Vector&& values) { m_indexed_properties = IndexedProperties(move(values)); } @@ -202,6 +204,8 @@ public: Shape& shape() { return *m_shape; } Shape const& shape() const { return *m_shape; } + static FlatPtr shape_offset() { return OFFSET_OF(Object, m_shape); } + void ensure_shape_is_unique(); template diff --git a/Userland/Libraries/LibJS/Runtime/Shape.h b/Userland/Libraries/LibJS/Runtime/Shape.h index d517ce19e5a..45a0a4cf181 100644 --- a/Userland/Libraries/LibJS/Runtime/Shape.h +++ b/Userland/Libraries/LibJS/Runtime/Shape.h @@ -57,6 +57,8 @@ public: void add_property_without_transition(PropertyKey const&, PropertyAttributes); bool is_unique() const { return m_unique; } + static FlatPtr is_unique_offset() { return OFFSET_OF(Shape, m_unique); } + Shape* create_unique_clone() const; Realm& realm() const { return m_realm; } @@ -80,6 +82,7 @@ public: void reconfigure_property_in_unique_shape(StringOrSymbol const& property_key, PropertyAttributes attributes); [[nodiscard]] u64 unique_shape_serial_number() const { return m_unique_shape_serial_number; } + static FlatPtr unique_shape_serial_number_offset() { return OFFSET_OF(Shape, m_unique_shape_serial_number); } private: explicit Shape(Realm&); @@ -106,7 +109,7 @@ private: PropertyAttributes m_attributes { 0 }; TransitionType m_transition_type : 6 { TransitionType::Invalid }; - bool m_unique : 1 { false }; + bool m_unique { false }; // Since unique shapes never change identity, inline caches use this incrementing serial number // to know whether its property table has been modified since last time we checked.