mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2024-11-25 17:10:23 +00:00
d0e614c045
The interrupts enabled check in the Kernel mutex is there so that we don't lock mutexes within a spinlock, because mutexes reenable interrupts and that will mess up the spinlock in more ways than one if the thread moves processors. This check is guarded behind a debug flag because it's too hard to fix all the problems at once, but we regressed and weren't even getting to init stage 2 with it enabled. With this commit, we get to stage 2 again. In early boot, there are no interrupts enabled and spinlocks used, so we can sort of kind of safely ignore the interrupt state. There might be a better solution with another boot state flag that checks whether APs are up (because they have interrupts enabled from the start) but that seems overkill.
377 lines
13 KiB
C++
377 lines
13 KiB
C++
/*
|
|
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
|
|
* Copyright (c) 2022, Idan Horowitz <idan.horowitz@serenityos.org>
|
|
*
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*/
|
|
|
|
#include <Kernel/Debug.h>
|
|
#include <Kernel/KSyms.h>
|
|
#include <Kernel/Locking/LockLocation.h>
|
|
#include <Kernel/Locking/Mutex.h>
|
|
#include <Kernel/Locking/Spinlock.h>
|
|
#include <Kernel/Thread.h>
|
|
|
|
extern bool g_in_early_boot;
|
|
|
|
namespace Kernel {
|
|
|
|
void Mutex::lock(Mode mode, [[maybe_unused]] LockLocation const& location)
|
|
{
|
|
// NOTE: This may be called from an interrupt handler (not an IRQ handler)
|
|
// and also from within critical sections!
|
|
VERIFY(!Processor::current_in_irq());
|
|
if constexpr (LOCK_IN_CRITICAL_DEBUG) {
|
|
// There are no interrupts enabled in early boot.
|
|
if (!g_in_early_boot)
|
|
VERIFY_INTERRUPTS_ENABLED();
|
|
}
|
|
VERIFY(mode != Mode::Unlocked);
|
|
auto* current_thread = Thread::current();
|
|
|
|
SpinlockLocker lock(m_lock);
|
|
bool did_block = false;
|
|
Mode current_mode = m_mode;
|
|
switch (current_mode) {
|
|
case Mode::Unlocked: {
|
|
dbgln_if(LOCK_TRACE_DEBUG, "Mutex::lock @ ({}) {}: acquire {}, currently unlocked", this, m_name, mode_to_string(mode));
|
|
m_mode = mode;
|
|
VERIFY(!m_holder);
|
|
VERIFY(m_shared_holders == 0);
|
|
if (mode == Mode::Exclusive) {
|
|
m_holder = current_thread;
|
|
} else {
|
|
VERIFY(mode == Mode::Shared);
|
|
++m_shared_holders;
|
|
#if LOCK_SHARED_UPGRADE_DEBUG
|
|
m_shared_holders_map.set(current_thread, 1);
|
|
#endif
|
|
}
|
|
VERIFY(m_times_locked == 0);
|
|
m_times_locked++;
|
|
|
|
#if LOCK_DEBUG
|
|
if (current_thread) {
|
|
current_thread->holding_lock(*this, 1, location);
|
|
}
|
|
#endif
|
|
return;
|
|
}
|
|
case Mode::Exclusive: {
|
|
VERIFY(m_holder);
|
|
if (m_holder != current_thread) {
|
|
block(*current_thread, mode, lock, 1);
|
|
did_block = true;
|
|
// If we blocked then m_mode should have been updated to what we requested
|
|
VERIFY(m_mode == mode);
|
|
}
|
|
|
|
if (m_mode == Mode::Exclusive) {
|
|
VERIFY(m_holder == current_thread);
|
|
VERIFY(m_shared_holders == 0);
|
|
} else if (did_block && mode == Mode::Shared) {
|
|
// Only if we blocked trying to acquire a shared lock the lock would have been converted
|
|
VERIFY(!m_holder);
|
|
VERIFY(m_shared_holders > 0);
|
|
}
|
|
|
|
if constexpr (LOCK_TRACE_DEBUG) {
|
|
if (mode == Mode::Exclusive)
|
|
dbgln("Mutex::lock @ {} ({}): acquire {}, currently exclusive, holding: {}", this, m_name, mode_to_string(mode), m_times_locked);
|
|
else
|
|
dbgln("Mutex::lock @ {} ({}): acquire exclusive (requested {}), currently exclusive, holding: {}", this, m_name, mode_to_string(mode), m_times_locked);
|
|
}
|
|
|
|
VERIFY(m_times_locked > 0);
|
|
if (!did_block) {
|
|
// if we didn't block we must still be an exclusive lock
|
|
VERIFY(m_mode == Mode::Exclusive);
|
|
m_times_locked++;
|
|
}
|
|
|
|
#if LOCK_DEBUG
|
|
current_thread->holding_lock(*this, 1, location);
|
|
#endif
|
|
return;
|
|
}
|
|
case Mode::Shared: {
|
|
VERIFY(m_behavior == MutexBehavior::Regular);
|
|
VERIFY(!m_holder);
|
|
if (mode == Mode::Exclusive) {
|
|
dbgln_if(LOCK_TRACE_DEBUG, "Mutex::lock @ {} ({}): blocking for exclusive access, currently shared, locks held {}", this, m_name, m_times_locked);
|
|
#if LOCK_SHARED_UPGRADE_DEBUG
|
|
VERIFY(m_shared_holders_map.size() != 1 || m_shared_holders_map.begin()->key != current_thread);
|
|
#endif
|
|
// WARNING: The following block will deadlock if the current thread is the only shared locker of this Mutex
|
|
// and is asking to upgrade the lock to be exclusive without first releasing the shared lock. We have no
|
|
// allocation-free way to detect such a scenario, so if you suspect that this is the cause of your deadlock,
|
|
// try turning on LOCK_SHARED_UPGRADE_DEBUG.
|
|
block(*current_thread, mode, lock, 1);
|
|
did_block = true;
|
|
VERIFY(m_mode == mode);
|
|
}
|
|
|
|
dbgln_if(LOCK_TRACE_DEBUG, "Mutex::lock @ {} ({}): acquire {}, currently shared, locks held {}", this, m_name, mode_to_string(mode), m_times_locked);
|
|
|
|
VERIFY(m_times_locked > 0);
|
|
if (m_mode == Mode::Shared) {
|
|
VERIFY(!m_holder);
|
|
VERIFY(!did_block);
|
|
} else if (did_block) {
|
|
VERIFY(mode == Mode::Exclusive);
|
|
VERIFY(m_holder == current_thread);
|
|
VERIFY(m_shared_holders == 0);
|
|
}
|
|
|
|
if (!did_block) {
|
|
// if we didn't block we must still be a shared lock
|
|
VERIFY(m_mode == Mode::Shared);
|
|
m_times_locked++;
|
|
VERIFY(m_shared_holders > 0);
|
|
++m_shared_holders;
|
|
#if LOCK_SHARED_UPGRADE_DEBUG
|
|
auto it = m_shared_holders_map.find(current_thread);
|
|
if (it != m_shared_holders_map.end())
|
|
it->value++;
|
|
else
|
|
m_shared_holders_map.set(current_thread, 1);
|
|
#endif
|
|
}
|
|
|
|
#if LOCK_DEBUG
|
|
current_thread->holding_lock(*this, 1, location);
|
|
#endif
|
|
return;
|
|
}
|
|
default:
|
|
VERIFY_NOT_REACHED();
|
|
}
|
|
}
|
|
|
|
void Mutex::unlock()
|
|
{
|
|
// NOTE: This may be called from an interrupt handler (not an IRQ handler)
|
|
// and also from within critical sections!
|
|
VERIFY(!Processor::current_in_irq());
|
|
if constexpr (LOCK_IN_CRITICAL_DEBUG) {
|
|
// There are no interrupts enabled in early boot.
|
|
if (!g_in_early_boot)
|
|
VERIFY_INTERRUPTS_ENABLED();
|
|
}
|
|
auto* current_thread = Thread::current();
|
|
SpinlockLocker lock(m_lock);
|
|
Mode current_mode = m_mode;
|
|
if constexpr (LOCK_TRACE_DEBUG) {
|
|
if (current_mode == Mode::Shared)
|
|
dbgln("Mutex::unlock @ {} ({}): release {}, locks held: {}", this, m_name, mode_to_string(current_mode), m_times_locked);
|
|
else
|
|
dbgln("Mutex::unlock @ {} ({}): release {}, holding: {}", this, m_name, mode_to_string(current_mode), m_times_locked);
|
|
}
|
|
|
|
VERIFY(current_mode != Mode::Unlocked);
|
|
|
|
VERIFY(m_times_locked > 0);
|
|
m_times_locked--;
|
|
|
|
switch (current_mode) {
|
|
case Mode::Exclusive:
|
|
VERIFY(m_holder == current_thread);
|
|
VERIFY(m_shared_holders == 0);
|
|
if (m_times_locked == 0)
|
|
m_holder = nullptr;
|
|
break;
|
|
case Mode::Shared: {
|
|
VERIFY(!m_holder);
|
|
VERIFY(m_shared_holders > 0);
|
|
--m_shared_holders;
|
|
#if LOCK_SHARED_UPGRADE_DEBUG
|
|
auto it = m_shared_holders_map.find(current_thread);
|
|
if (it->value > 1)
|
|
it->value--;
|
|
else
|
|
m_shared_holders_map.remove(it);
|
|
#endif
|
|
break;
|
|
}
|
|
default:
|
|
VERIFY_NOT_REACHED();
|
|
}
|
|
|
|
#if LOCK_DEBUG
|
|
if (current_thread) {
|
|
current_thread->holding_lock(*this, -1, {});
|
|
}
|
|
#endif
|
|
|
|
if (m_times_locked == 0) {
|
|
VERIFY(current_mode == Mode::Exclusive ? !m_holder : m_shared_holders == 0);
|
|
|
|
m_mode = Mode::Unlocked;
|
|
unblock_waiters(current_mode);
|
|
}
|
|
}
|
|
|
|
void Mutex::block(Thread& current_thread, Mode mode, SpinlockLocker<Spinlock>& lock, u32 requested_locks)
|
|
{
|
|
if constexpr (LOCK_IN_CRITICAL_DEBUG) {
|
|
// There are no interrupts enabled in early boot.
|
|
if (!g_in_early_boot)
|
|
VERIFY_INTERRUPTS_ENABLED();
|
|
}
|
|
m_blocked_thread_lists.with([&](auto& lists) {
|
|
auto append_to_list = [&]<typename L>(L& list) {
|
|
VERIFY(!list.contains(current_thread));
|
|
list.append(current_thread);
|
|
};
|
|
|
|
if (m_behavior == MutexBehavior::BigLock)
|
|
append_to_list(lists.exclusive_big_lock);
|
|
else
|
|
append_to_list(lists.list_for_mode(mode));
|
|
});
|
|
|
|
dbgln_if(LOCK_TRACE_DEBUG, "Mutex::lock @ {} ({}) waiting...", this, m_name);
|
|
current_thread.block(*this, lock, requested_locks);
|
|
dbgln_if(LOCK_TRACE_DEBUG, "Mutex::lock @ {} ({}) waited", this, m_name);
|
|
|
|
m_blocked_thread_lists.with([&](auto& lists) {
|
|
auto remove_from_list = [&]<typename L>(L& list) {
|
|
VERIFY(list.contains(current_thread));
|
|
list.remove(current_thread);
|
|
};
|
|
|
|
if (m_behavior == MutexBehavior::BigLock)
|
|
remove_from_list(lists.exclusive_big_lock);
|
|
else
|
|
remove_from_list(lists.list_for_mode(mode));
|
|
});
|
|
}
|
|
|
|
void Mutex::unblock_waiters(Mode previous_mode)
|
|
{
|
|
VERIFY(m_times_locked == 0);
|
|
VERIFY(m_mode == Mode::Unlocked);
|
|
|
|
m_blocked_thread_lists.with([&](auto& lists) {
|
|
auto unblock_shared = [&]() {
|
|
if (lists.shared.is_empty())
|
|
return false;
|
|
VERIFY(m_behavior == MutexBehavior::Regular);
|
|
m_mode = Mode::Shared;
|
|
for (auto& thread : lists.shared) {
|
|
auto requested_locks = thread.unblock_from_mutex(*this);
|
|
m_shared_holders += requested_locks;
|
|
#if LOCK_SHARED_UPGRADE_DEBUG
|
|
auto set_result = m_shared_holders_map.set(&thread, requested_locks);
|
|
VERIFY(set_result == AK::HashSetResult::InsertedNewEntry);
|
|
#endif
|
|
m_times_locked += requested_locks;
|
|
}
|
|
return true;
|
|
};
|
|
auto unblock_exclusive = [&]<typename L>(L& list) {
|
|
if (auto* next_exclusive_thread = list.first()) {
|
|
m_mode = Mode::Exclusive;
|
|
m_times_locked = next_exclusive_thread->unblock_from_mutex(*this);
|
|
m_holder = next_exclusive_thread;
|
|
return true;
|
|
}
|
|
return false;
|
|
};
|
|
|
|
if (m_behavior == MutexBehavior::BigLock) {
|
|
unblock_exclusive(lists.exclusive_big_lock);
|
|
} else if (previous_mode == Mode::Exclusive) {
|
|
if (!unblock_shared())
|
|
unblock_exclusive(lists.exclusive);
|
|
} else {
|
|
if (!unblock_exclusive(lists.exclusive))
|
|
unblock_shared();
|
|
}
|
|
});
|
|
}
|
|
|
|
auto Mutex::force_unlock_exclusive_if_locked(u32& lock_count_to_restore) -> Mode
|
|
{
|
|
VERIFY(m_behavior == MutexBehavior::BigLock);
|
|
// NOTE: This may be called from an interrupt handler (not an IRQ handler)
|
|
// and also from within critical sections!
|
|
VERIFY(!Processor::current_in_irq());
|
|
|
|
auto* current_thread = Thread::current();
|
|
SpinlockLocker lock(m_lock);
|
|
auto current_mode = m_mode;
|
|
switch (current_mode) {
|
|
case Mode::Exclusive: {
|
|
if (m_holder != current_thread) {
|
|
lock_count_to_restore = 0;
|
|
return Mode::Unlocked;
|
|
}
|
|
|
|
dbgln_if(LOCK_RESTORE_DEBUG, "Mutex::force_unlock_exclusive_if_locked @ {}: unlocking exclusive with lock count: {}", this, m_times_locked);
|
|
#if LOCK_DEBUG
|
|
m_holder->holding_lock(*this, -(int)m_times_locked, {});
|
|
#endif
|
|
m_holder = nullptr;
|
|
VERIFY(m_times_locked > 0);
|
|
lock_count_to_restore = m_times_locked;
|
|
m_times_locked = 0;
|
|
m_mode = Mode::Unlocked;
|
|
unblock_waiters(Mode::Exclusive);
|
|
break;
|
|
}
|
|
case Mode::Unlocked: {
|
|
lock_count_to_restore = 0;
|
|
break;
|
|
}
|
|
default:
|
|
VERIFY_NOT_REACHED();
|
|
}
|
|
return current_mode;
|
|
}
|
|
|
|
void Mutex::restore_exclusive_lock(u32 lock_count, [[maybe_unused]] LockLocation const& location)
|
|
{
|
|
VERIFY(m_behavior == MutexBehavior::BigLock);
|
|
VERIFY(lock_count > 0);
|
|
VERIFY(!Processor::current_in_irq());
|
|
|
|
auto* current_thread = Thread::current();
|
|
bool did_block = false;
|
|
SpinlockLocker lock(m_lock);
|
|
[[maybe_unused]] auto previous_mode = m_mode;
|
|
if (m_mode == Mode::Exclusive && m_holder != current_thread) {
|
|
block(*current_thread, Mode::Exclusive, lock, lock_count);
|
|
did_block = true;
|
|
// If we blocked then m_mode should have been updated to what we requested
|
|
VERIFY(m_mode == Mode::Exclusive);
|
|
}
|
|
|
|
dbgln_if(LOCK_RESTORE_DEBUG, "Mutex::restore_exclusive_lock @ {}: restoring exclusive with lock count {}, was {}", this, lock_count, mode_to_string(previous_mode));
|
|
|
|
VERIFY(m_mode != Mode::Shared);
|
|
VERIFY(m_shared_holders == 0);
|
|
if (did_block) {
|
|
VERIFY(m_times_locked > 0);
|
|
VERIFY(m_holder == current_thread);
|
|
} else {
|
|
if (m_mode == Mode::Unlocked) {
|
|
m_mode = Mode::Exclusive;
|
|
VERIFY(m_times_locked == 0);
|
|
m_times_locked = lock_count;
|
|
VERIFY(!m_holder);
|
|
m_holder = current_thread;
|
|
} else {
|
|
VERIFY(m_mode == Mode::Exclusive);
|
|
VERIFY(m_holder == current_thread);
|
|
VERIFY(m_times_locked > 0);
|
|
m_times_locked += lock_count;
|
|
}
|
|
}
|
|
|
|
#if LOCK_DEBUG
|
|
m_holder->holding_lock(*this, (int)lock_count, location);
|
|
#endif
|
|
}
|
|
|
|
}
|