AK+Kernel: Make automatically locking RefPtr & co a kernel-only thing

Some time ago, automatic locking was added to the AK smart pointers to
paper over various race conditions in the kernel. Until we've actually
solved the issues in the kernel, we're stuck with the locking.

However, we don't need to punish single-threaded userspace programs with
the high cost of locking. This patch moves the thread-safe variants of
RefPtr, NonnullRefPtr, WeakPtr and RefCounted into Kernel/Library/.

AK/NonnullRefPtr.h

@@ -6,15 +6,14 @@
 
 #pragma once
 
-#include <AK/Assertions.h>
-#include <AK/Atomic.h>
-#include <AK/Format.h>
-#include <AK/Traits.h>
-#include <AK/Types.h>
 #ifdef KERNEL
-#    include <Kernel/Arch/x86/Processor.h>
+#    include <Kernel/Library/ThreadSafeNonnullRefPtr.h>
-#    include <Kernel/Arch/x86/ScopedCritical.h>
+#else
-#endif
+#    include <AK/Assertions.h>
+#    include <AK/Atomic.h>
+#    include <AK/Format.h>
+#    include <AK/Traits.h>
+#    include <AK/Types.h>
 
 namespace AK {
 
@@ -51,52 +50,55 @@ public:
 
     enum AdoptTag { Adopt };
 
-    ALWAYS_INLINE NonnullRefPtr(const T& object)
+    ALWAYS_INLINE NonnullRefPtr(T const& object)
-        : m_bits((FlatPtr)&object)
+        : m_ptr(const_cast<T*>(&object))
     {
-        VERIFY(!(m_bits & 1));
+        m_ptr->ref();
-        const_cast<T&>(object).ref();
     }
+
     template<typename U>
-    ALWAYS_INLINE NonnullRefPtr(const U& object) requires(IsConvertible<U*, T*>)
+    ALWAYS_INLINE NonnullRefPtr(U const& object) requires(IsConvertible<U*, T*>)
-        : m_bits((FlatPtr) static_cast<const T*>(&object))
+        : m_ptr(const_cast<T*>(static_cast<T const*>(&object)))
     {
-        VERIFY(!(m_bits & 1));
+        m_ptr->ref();
-        const_cast<T&>(static_cast<const T&>(object)).ref();
     }
+
     ALWAYS_INLINE NonnullRefPtr(AdoptTag, T& object)
-        : m_bits((FlatPtr)&object)
+        : m_ptr(&object)
     {
-        VERIFY(!(m_bits & 1));
     }
+
     ALWAYS_INLINE NonnullRefPtr(NonnullRefPtr&& other)
-        : m_bits((FlatPtr)&other.leak_ref())
+        : m_ptr(&other.leak_ref())
     {
-        VERIFY(!(m_bits & 1));
     }
+
     template<typename U>
     ALWAYS_INLINE NonnullRefPtr(NonnullRefPtr<U>&& other) requires(IsConvertible<U*, T*>)
-        : m_bits((FlatPtr)&other.leak_ref())
+        : m_ptr(static_cast<T*>(&other.leak_ref()))
     {
-        VERIFY(!(m_bits & 1));
     }
-    ALWAYS_INLINE NonnullRefPtr(const NonnullRefPtr& other)
+
-        : m_bits((FlatPtr)other.add_ref())
+    ALWAYS_INLINE NonnullRefPtr(NonnullRefPtr const& other)
+        : m_ptr(const_cast<T*>(other.ptr()))
     {
-        VERIFY(!(m_bits & 1));
+        m_ptr->ref();
     }
+
     template<typename U>
-    ALWAYS_INLINE NonnullRefPtr(const NonnullRefPtr<U>& other) requires(IsConvertible<U*, T*>)
+    ALWAYS_INLINE NonnullRefPtr(NonnullRefPtr<U> const& other) requires(IsConvertible<U*, T*>)
-        : m_bits((FlatPtr)other.add_ref())
+        : m_ptr(const_cast<T*>(static_cast<T const*>(other.ptr())))
     {
-        VERIFY(!(m_bits & 1));
+        m_ptr->ref();
     }
+
     ALWAYS_INLINE ~NonnullRefPtr()
     {
-        assign(nullptr);
+        unref_if_not_null(m_ptr);
-#ifdef SANITIZE_PTRS
+        m_ptr = nullptr;
-        m_bits.store(explode_byte(0xb0), AK::MemoryOrder::memory_order_relaxed);
+#    ifdef SANITIZE_PTRS
-#endif
+        m_ptr = reinterpret_cast<T*>(explode_byte(0xb0));
+#    endif
     }
 
     template<typename U>
@@ -111,44 +113,46 @@ public:
     NonnullRefPtr(const RefPtr<T>&) = delete;
     NonnullRefPtr& operator=(const RefPtr<T>&) = delete;
 
-    NonnullRefPtr& operator=(const NonnullRefPtr& other)
+    NonnullRefPtr& operator=(NonnullRefPtr const& other)
     {
-        if (this != &other)
+        NonnullRefPtr tmp { other };
-            assign(other.add_ref());
+        swap(tmp);
         return *this;
     }
 
     template<typename U>
-    NonnullRefPtr& operator=(const NonnullRefPtr<U>& other) requires(IsConvertible<U*, T*>)
+    NonnullRefPtr& operator=(NonnullRefPtr<U> const& other) requires(IsConvertible<U*, T*>)
     {
-        assign(other.add_ref());
+        NonnullRefPtr tmp { other };
+        swap(tmp);
         return *this;
     }
 
     ALWAYS_INLINE NonnullRefPtr& operator=(NonnullRefPtr&& other)
     {
-        if (this != &other)
+        NonnullRefPtr tmp { move(other) };
-            assign(&other.leak_ref());
+        swap(tmp);
         return *this;
     }
 
     template<typename U>
     NonnullRefPtr& operator=(NonnullRefPtr<U>&& other) requires(IsConvertible<U*, T*>)
     {
-        assign(&other.leak_ref());
+        NonnullRefPtr tmp { move(other) };
+        swap(tmp);
         return *this;
     }
 
-    NonnullRefPtr& operator=(const T& object)
+    NonnullRefPtr& operator=(T const& object)
     {
-        const_cast<T&>(object).ref();
+        NonnullRefPtr tmp { object };
-        assign(const_cast<T*>(&object));
+        swap(tmp);
         return *this;
     }
 
     [[nodiscard]] ALWAYS_INLINE T& leak_ref()
     {
-        T* ptr = exchange(nullptr);
+        T* ptr = exchange(m_ptr, nullptr);
         VERIFY(ptr);
         return *ptr;
     }
@@ -203,113 +207,24 @@ public:
 
     void swap(NonnullRefPtr& other)
     {
-        if (this == &other)
+        AK::swap(m_ptr, other.m_ptr);
-            return;
-
-        // NOTE: swap is not atomic!
-        T* other_ptr = other.exchange(nullptr);
-        T* ptr = exchange(other_ptr);
-        other.exchange(ptr);
     }
 
     template<typename U>
     void swap(NonnullRefPtr<U>& other) requires(IsConvertible<U*, T*>)
     {
-        // NOTE: swap is not atomic!
+        AK::swap(m_ptr, other.m_ptr);
-        U* other_ptr = other.exchange(nullptr);
-        T* ptr = exchange(other_ptr);
-        other.exchange(ptr);
     }
 
 private:
     NonnullRefPtr() = delete;
-
-    ALWAYS_INLINE T* as_ptr() const
-    {
-        return (T*)(m_bits.load(AK::MemoryOrder::memory_order_relaxed) & ~(FlatPtr)1);
-    }
-
     ALWAYS_INLINE RETURNS_NONNULL T* as_nonnull_ptr() const
     {
-        T* ptr = (T*)(m_bits.load(AK::MemoryOrder::memory_order_relaxed) & ~(FlatPtr)1);
+        VERIFY(m_ptr);
-        VERIFY(ptr);
+        return m_ptr;
-        return ptr;
     }
 
-    template<typename F>
+    T* m_ptr { nullptr };
-    void do_while_locked(F f) const
-    {
-#ifdef KERNEL
-        // We don't want to be pre-empted while we have the lock bit set
-        Kernel::ScopedCritical critical;
-#endif
-        FlatPtr bits;
-        for (;;) {
-            bits = m_bits.fetch_or(1, AK::MemoryOrder::memory_order_acq_rel);
-            if (!(bits & 1))
-                break;
-#ifdef KERNEL
-            Kernel::Processor::wait_check();
-#endif
-        }
-        VERIFY(!(bits & 1));
-        f((T*)bits);
-        m_bits.store(bits, AK::MemoryOrder::memory_order_release);
-    }
-
-    ALWAYS_INLINE void assign(T* new_ptr)
-    {
-        T* prev_ptr = exchange(new_ptr);
-        unref_if_not_null(prev_ptr);
-    }
-
-    ALWAYS_INLINE T* exchange(T* new_ptr)
-    {
-        VERIFY(!((FlatPtr)new_ptr & 1));
-#ifdef KERNEL
-        // We don't want to be pre-empted while we have the lock bit set
-        Kernel::ScopedCritical critical;
-#endif
-        // Only exchange while not locked
-        FlatPtr expected = m_bits.load(AK::MemoryOrder::memory_order_relaxed);
-        for (;;) {
-            expected &= ~(FlatPtr)1; // only if lock bit is not set
-            if (m_bits.compare_exchange_strong(expected, (FlatPtr)new_ptr, AK::MemoryOrder::memory_order_acq_rel))
-                break;
-#ifdef KERNEL
-            Kernel::Processor::wait_check();
-#endif
-        }
-        VERIFY(!(expected & 1));
-        return (T*)expected;
-    }
-
-    T* add_ref() const
-    {
-#ifdef KERNEL
-        // We don't want to be pre-empted while we have the lock bit set
-        Kernel::ScopedCritical critical;
-#endif
-        // Lock the pointer
-        FlatPtr expected = m_bits.load(AK::MemoryOrder::memory_order_relaxed);
-        for (;;) {
-            expected &= ~(FlatPtr)1; // only if lock bit is not set
-            if (m_bits.compare_exchange_strong(expected, expected | 1, AK::MemoryOrder::memory_order_acq_rel))
-                break;
-#ifdef KERNEL
-            Kernel::Processor::wait_check();
-#endif
-        }
-
-        // Add a reference now that we locked the pointer
-        ref_if_not_null((T*)expected);
-
-        // Unlock the pointer again
-        m_bits.store(expected, AK::MemoryOrder::memory_order_release);
-        return (T*)expected;
-    }
-
-    mutable Atomic<FlatPtr> m_bits { 0 };
 };
 
 template<typename T>
@@ -357,3 +272,5 @@ struct Traits<NonnullRefPtr<T>> : public GenericTraits<NonnullRefPtr<T>> {
 using AK::adopt_ref;
 using AK::make_ref_counted;
 using AK::NonnullRefPtr;
+
+#endif

AK/RefCounted.h

@@ -6,12 +6,15 @@
 
 #pragma once
 
-#include <AK/Assertions.h>
+#ifdef KERNEL
-#include <AK/Atomic.h>
+#    include <Kernel/Library/ThreadSafeRefCounted.h>
-#include <AK/Checked.h>
+#else
-#include <AK/Noncopyable.h>
+
-#include <AK/Platform.h>
+#    include <AK/Assertions.h>
-#include <AK/StdLibExtras.h>
+#    include <AK/Checked.h>
+#    include <AK/Noncopyable.h>
+#    include <AK/Platform.h>
+#    include <AK/StdLibExtras.h>
 
 namespace AK {
 
@@ -49,43 +52,32 @@ public:
 
     void ref() const
     {
-        auto old_ref_count = m_ref_count.fetch_add(1, AK::MemoryOrder::memory_order_relaxed);
+        VERIFY(m_ref_count > 0);
-        VERIFY(old_ref_count > 0);
+        VERIFY(!Checked<RefCountType>::addition_would_overflow(m_ref_count, 1));
-        VERIFY(!Checked<RefCountType>::addition_would_overflow(old_ref_count, 1));
+        ++m_ref_count;
     }
 
     [[nodiscard]] bool try_ref() const
     {
-        RefCountType expected = m_ref_count.load(AK::MemoryOrder::memory_order_relaxed);
+        if (m_ref_count == 0)
-        for (;;) {
+            return false;
-            if (expected == 0)
+        ref();
-                return false;
+        return true;
-            VERIFY(!Checked<RefCountType>::addition_would_overflow(expected, 1));
-            if (m_ref_count.compare_exchange_strong(expected, expected + 1, AK::MemoryOrder::memory_order_acquire))
-                return true;
-        }
     }
 
-    [[nodiscard]] RefCountType ref_count() const
+    [[nodiscard]] RefCountType ref_count() const { return m_ref_count; }
-    {
-        return m_ref_count.load(AK::MemoryOrder::memory_order_relaxed);
-    }
 
 protected:
     RefCountedBase() = default;
-    ~RefCountedBase()
+    ~RefCountedBase() { VERIFY(!m_ref_count); }
-    {
-        VERIFY(m_ref_count.load(AK::MemoryOrder::memory_order_relaxed) == 0);
-    }
 
     RefCountType deref_base() const
     {
-        auto old_ref_count = m_ref_count.fetch_sub(1, AK::MemoryOrder::memory_order_acq_rel);
+        VERIFY(m_ref_count);
-        VERIFY(old_ref_count > 0);
+        return --m_ref_count;
-        return old_ref_count - 1;
     }
 
-    mutable Atomic<RefCountType> m_ref_count { 1 };
+    RefCountType mutable m_ref_count { 1 };
 };
 
 template<typename T>
@@ -109,3 +101,5 @@ public:
 
 using AK::RefCounted;
 using AK::RefCountedBase;
+
+#endif

AK/RefPtr.h

@@ -6,114 +6,23 @@
 
 #pragma once
 
-#include <AK/Assertions.h>
-#include <AK/Atomic.h>
-#include <AK/Format.h>
-#include <AK/NonnullRefPtr.h>
-#include <AK/StdLibExtras.h>
-#include <AK/Traits.h>
-#include <AK/Types.h>
 #ifdef KERNEL
-#    include <Kernel/API/KResult.h>
+#    include <Kernel/Library/ThreadSafeRefPtr.h>
-#    include <Kernel/Arch/x86/Processor.h>
+#else
-#    include <Kernel/Arch/x86/ScopedCritical.h>
+
-#endif
+#    include <AK/Assertions.h>
+#    include <AK/Atomic.h>
+#    include <AK/Format.h>
+#    include <AK/NonnullRefPtr.h>
+#    include <AK/StdLibExtras.h>
+#    include <AK/Traits.h>
+#    include <AK/Types.h>
 
 namespace AK {
 
 template<typename T>
 class OwnPtr;
 
-template<typename T>
-struct RefPtrTraits {
-    ALWAYS_INLINE static T* as_ptr(FlatPtr bits)
-    {
-        return (T*)(bits & ~(FlatPtr)1);
-    }
-
-    ALWAYS_INLINE static FlatPtr as_bits(T* ptr)
-    {
-        VERIFY(!((FlatPtr)ptr & 1));
-        return (FlatPtr)ptr;
-    }
-
-    template<typename U, typename PtrTraits>
-    ALWAYS_INLINE static FlatPtr convert_from(FlatPtr bits)
-    {
-        if (PtrTraits::is_null(bits))
-            return default_null_value;
-        return as_bits(PtrTraits::as_ptr(bits));
-    }
-
-    ALWAYS_INLINE static bool is_null(FlatPtr bits)
-    {
-        return !(bits & ~(FlatPtr)1);
-    }
-
-    ALWAYS_INLINE static FlatPtr exchange(Atomic<FlatPtr>& atomic_var, FlatPtr new_value)
-    {
-        // Only exchange when lock is not held
-        VERIFY(!(new_value & 1));
-        FlatPtr expected = atomic_var.load(AK::MemoryOrder::memory_order_relaxed);
-        for (;;) {
-            expected &= ~(FlatPtr)1; // only if lock bit is not set
-            if (atomic_var.compare_exchange_strong(expected, new_value, AK::MemoryOrder::memory_order_acq_rel))
-                break;
-#ifdef KERNEL
-            Kernel::Processor::wait_check();
-#endif
-        }
-        return expected;
-    }
-
-    ALWAYS_INLINE static bool exchange_if_null(Atomic<FlatPtr>& atomic_var, FlatPtr new_value)
-    {
-        // Only exchange when lock is not held
-        VERIFY(!(new_value & 1));
-        for (;;) {
-            FlatPtr expected = default_null_value; // only if lock bit is not set
-            if (atomic_var.compare_exchange_strong(expected, new_value, AK::MemoryOrder::memory_order_acq_rel))
-                break;
-            if (!is_null(expected))
-                return false;
-#ifdef KERNEL
-            Kernel::Processor::wait_check();
-#endif
-        }
-        return true;
-    }
-
-    ALWAYS_INLINE static FlatPtr lock(Atomic<FlatPtr>& atomic_var)
-    {
-        // This sets the lock bit atomically, preventing further modifications.
-        // This is important when e.g. copying a RefPtr where the source
-        // might be released and freed too quickly. This allows us
-        // to temporarily lock the pointer so we can add a reference, then
-        // unlock it
-        FlatPtr bits;
-        for (;;) {
-            bits = atomic_var.fetch_or(1, AK::MemoryOrder::memory_order_acq_rel);
-            if (!(bits & 1))
-                break;
-#ifdef KERNEL
-            Kernel::Processor::wait_check();
-#endif
-        }
-        VERIFY(!(bits & 1));
-        return bits;
-    }
-
-    ALWAYS_INLINE static void unlock(Atomic<FlatPtr>& atomic_var, FlatPtr new_value)
-    {
-        VERIFY(!(new_value & 1));
-        atomic_var.store(new_value, AK::MemoryOrder::memory_order_release);
-    }
-
-    static constexpr FlatPtr default_null_value = 0;
-
-    using NullType = std::nullptr_t;
-};
-
 template<typename T, typename PtrTraits>
 class RefPtr {
     template<typename U, typename P>
@@ -127,149 +36,154 @@ public:
     };
 
     RefPtr() = default;
-    RefPtr(const T* ptr)
+    RefPtr(T const* ptr)
-        : m_bits(PtrTraits::as_bits(const_cast<T*>(ptr)))
+        : m_ptr(const_cast<T*>(ptr))
     {
-        ref_if_not_null(const_cast<T*>(ptr));
+        ref_if_not_null(m_ptr);
     }
-    RefPtr(const T& object)
+
-        : m_bits(PtrTraits::as_bits(const_cast<T*>(&object)))
+    RefPtr(T const& object)
+        : m_ptr(const_cast<T*>(&object))
     {
-        T* ptr = const_cast<T*>(&object);
+        m_ptr->ref();
-        VERIFY(ptr);
-        VERIFY(!is_null());
-        ptr->ref();
     }
+
     RefPtr(AdoptTag, T& object)
-        : m_bits(PtrTraits::as_bits(&object))
+        : m_ptr(&object)
     {
-        VERIFY(!is_null());
     }
+
     RefPtr(RefPtr&& other)
-        : m_bits(other.leak_ref_raw())
+        : m_ptr(other.leak_ref())
     {
     }
-    ALWAYS_INLINE RefPtr(const NonnullRefPtr<T>& other)
+
-        : m_bits(PtrTraits::as_bits(const_cast<T*>(other.add_ref())))
+    ALWAYS_INLINE RefPtr(NonnullRefPtr<T> const& other)
+        : m_ptr(const_cast<T*>(other.ptr()))
     {
+        m_ptr->ref();
     }
+
     template<typename U>
-    ALWAYS_INLINE RefPtr(const NonnullRefPtr<U>& other) requires(IsConvertible<U*, T*>)
+    ALWAYS_INLINE RefPtr(NonnullRefPtr<U> const& other) requires(IsConvertible<U*, T*>)
-        : m_bits(PtrTraits::as_bits(const_cast<U*>(other.add_ref())))
+        : m_ptr(const_cast<T*>(static_cast<T const*>(other.ptr())))
     {
+        m_ptr->ref();
     }
+
     template<typename U>
     ALWAYS_INLINE RefPtr(NonnullRefPtr<U>&& other) requires(IsConvertible<U*, T*>)
-        : m_bits(PtrTraits::as_bits(&other.leak_ref()))
+        : m_ptr(static_cast<T*>(&other.leak_ref()))
     {
-        VERIFY(!is_null());
     }
+
     template<typename U, typename P = RefPtrTraits<U>>
     RefPtr(RefPtr<U, P>&& other) requires(IsConvertible<U*, T*>)
-        : m_bits(PtrTraits::template convert_from<U, P>(other.leak_ref_raw()))
+        : m_ptr(static_cast<T*>(other.leak_ref()))
     {
     }
-    RefPtr(const RefPtr& other)
+
-        : m_bits(other.add_ref_raw())
+    RefPtr(RefPtr const& other)
+        : m_ptr(other.m_ptr)
     {
+        ref_if_not_null(m_ptr);
     }
+
     template<typename U, typename P = RefPtrTraits<U>>
-    RefPtr(const RefPtr<U, P>& other) requires(IsConvertible<U*, T*>)
+    RefPtr(RefPtr<U, P> const& other) requires(IsConvertible<U*, T*>)
-        : m_bits(other.add_ref_raw())
+        : m_ptr(const_cast<T*>(static_cast<T const*>(other.ptr())))
     {
+        ref_if_not_null(m_ptr);
     }
+
     ALWAYS_INLINE ~RefPtr()
     {
         clear();
-#ifdef SANITIZE_PTRS
+#    ifdef SANITIZE_PTRS
-        m_bits.store(explode_byte(0xe0), AK::MemoryOrder::memory_order_relaxed);
+        m_ptr = reinterpret_cast<T*>(explode_byte(0xe0));
-#endif
+#    endif
     }
 
     template<typename U>
-    RefPtr(const OwnPtr<U>&) = delete;
+    RefPtr(OwnPtr<U> const&) = delete;
     template<typename U>
-    RefPtr& operator=(const OwnPtr<U>&) = delete;
+    RefPtr& operator=(OwnPtr<U> const&) = delete;
 
     void swap(RefPtr& other)
     {
-        if (this == &other)
+        AK::swap(m_ptr, other.m_ptr);
-            return;
-
-        // NOTE: swap is not atomic!
-        FlatPtr other_bits = PtrTraits::exchange(other.m_bits, PtrTraits::default_null_value);
-        FlatPtr bits = PtrTraits::exchange(m_bits, other_bits);
-        PtrTraits::exchange(other.m_bits, bits);
     }
 
     template<typename U, typename P = RefPtrTraits<U>>
     void swap(RefPtr<U, P>& other) requires(IsConvertible<U*, T*>)
     {
-        // NOTE: swap is not atomic!
+        AK::swap(m_ptr, other.m_ptr);
-        FlatPtr other_bits = P::exchange(other.m_bits, P::default_null_value);
-        FlatPtr bits = PtrTraits::exchange(m_bits, PtrTraits::template convert_from<U, P>(other_bits));
-        P::exchange(other.m_bits, P::template convert_from<U, P>(bits));
     }
 
     ALWAYS_INLINE RefPtr& operator=(RefPtr&& other)
     {
-        if (this != &other)
+        RefPtr tmp { move(other) };
-            assign_raw(other.leak_ref_raw());
+        swap(tmp);
         return *this;
     }
 
     template<typename U, typename P = RefPtrTraits<U>>
     ALWAYS_INLINE RefPtr& operator=(RefPtr<U, P>&& other) requires(IsConvertible<U*, T*>)
     {
-        assign_raw(PtrTraits::template convert_from<U, P>(other.leak_ref_raw()));
+        RefPtr tmp { move(other) };
+        swap(tmp);
         return *this;
     }
 
     template<typename U>
     ALWAYS_INLINE RefPtr& operator=(NonnullRefPtr<U>&& other) requires(IsConvertible<U*, T*>)
     {
-        assign_raw(PtrTraits::as_bits(&other.leak_ref()));
+        RefPtr tmp { move(other) };
+        swap(tmp);
         return *this;
     }
 
-    ALWAYS_INLINE RefPtr& operator=(const NonnullRefPtr<T>& other)
+    ALWAYS_INLINE RefPtr& operator=(NonnullRefPtr<T> const& other)
     {
-        assign_raw(PtrTraits::as_bits(other.add_ref()));
+        RefPtr tmp { other };
+        swap(tmp);
         return *this;
     }
 
     template<typename U>
-    ALWAYS_INLINE RefPtr& operator=(const NonnullRefPtr<U>& other) requires(IsConvertible<U*, T*>)
+    ALWAYS_INLINE RefPtr& operator=(NonnullRefPtr<U> const& other) requires(IsConvertible<U*, T*>)
     {
-        assign_raw(PtrTraits::as_bits(other.add_ref()));
+        RefPtr tmp { other };
+        swap(tmp);
         return *this;
     }
 
-    ALWAYS_INLINE RefPtr& operator=(const RefPtr& other)
+    ALWAYS_INLINE RefPtr& operator=(RefPtr const& other)
     {
-        if (this != &other)
+        RefPtr tmp { other };
-            assign_raw(other.add_ref_raw());
+        swap(tmp);
         return *this;
     }
 
     template<typename U>
-    ALWAYS_INLINE RefPtr& operator=(const RefPtr<U>& other) requires(IsConvertible<U*, T*>)
+    ALWAYS_INLINE RefPtr& operator=(RefPtr<U> const& other) requires(IsConvertible<U*, T*>)
     {
-        assign_raw(other.add_ref_raw());
+        RefPtr tmp { other };
+        swap(tmp);
         return *this;
     }
 
-    ALWAYS_INLINE RefPtr& operator=(const T* ptr)
+    ALWAYS_INLINE RefPtr& operator=(T const* ptr)
     {
-        ref_if_not_null(const_cast<T*>(ptr));
+        RefPtr tmp { ptr };
-        assign_raw(PtrTraits::as_bits(const_cast<T*>(ptr)));
+        swap(tmp);
         return *this;
     }
 
-    ALWAYS_INLINE RefPtr& operator=(const T& object)
+    ALWAYS_INLINE RefPtr& operator=(T const& object)
     {
-        const_cast<T&>(object).ref();
+        RefPtr tmp { object };
-        assign_raw(PtrTraits::as_bits(const_cast<T*>(&object)));
+        swap(tmp);
         return *this;
     }
 
@@ -283,7 +197,8 @@ public:
     {
         if (this == &other)
             return is_null();
-        return PtrTraits::exchange_if_null(m_bits, other.leak_ref_raw());
+        *this = move(other);
+        return true;
     }
 
     template<typename U, typename P = RefPtrTraits<U>>
@@ -291,27 +206,28 @@ public:
     {
         if (this == &other)
             return is_null();
-        return PtrTraits::exchange_if_null(m_bits, PtrTraits::template convert_from<U, P>(other.leak_ref_raw()));
+        *this = move(other);
+        return true;
     }
 
     ALWAYS_INLINE void clear()
     {
-        assign_raw(PtrTraits::default_null_value);
+        unref_if_not_null(m_ptr);
+        m_ptr = nullptr;
     }
 
-    bool operator!() const { return PtrTraits::is_null(m_bits.load(AK::MemoryOrder::memory_order_relaxed)); }
+    bool operator!() const { return !m_ptr; }
 
     [[nodiscard]] T* leak_ref()
     {
-        FlatPtr bits = PtrTraits::exchange(m_bits, PtrTraits::default_null_value);
+        return exchange(m_ptr, nullptr);
-        return PtrTraits::as_ptr(bits);
     }
 
     NonnullRefPtr<T> release_nonnull()
     {
-        FlatPtr bits = PtrTraits::exchange(m_bits, PtrTraits::default_null_value);
+        auto* ptr = leak_ref();
-        VERIFY(!PtrTraits::is_null(bits));
+        VERIFY(ptr);
-        return NonnullRefPtr<T>(NonnullRefPtr<T>::Adopt, *PtrTraits::as_ptr(bits));
+        return NonnullRefPtr<T>(NonnullRefPtr<T>::Adopt, *ptr);
     }
 
     ALWAYS_INLINE T* ptr() { return as_ptr(); }
@@ -357,88 +273,21 @@ public:
     bool operator==(T* other) { return as_ptr() == other; }
     bool operator!=(T* other) { return as_ptr() != other; }
 
-    ALWAYS_INLINE bool is_null() const { return PtrTraits::is_null(m_bits.load(AK::MemoryOrder::memory_order_relaxed)); }
+    ALWAYS_INLINE bool is_null() const { return !m_ptr; }
-
-    template<typename U = T, typename EnableIf<IsSame<U, T> && !IsNullPointer<typename PtrTraits::NullType>>::Type* = nullptr>
-    typename PtrTraits::NullType null_value() const
-    {
-        // make sure we are holding a null value
-        FlatPtr bits = m_bits.load(AK::MemoryOrder::memory_order_relaxed);
-        VERIFY(PtrTraits::is_null(bits));
-        return PtrTraits::to_null_value(bits);
-    }
-    template<typename U = T, typename EnableIf<IsSame<U, T> && !IsNullPointer<typename PtrTraits::NullType>>::Type* = nullptr>
-    void set_null_value(typename PtrTraits::NullType value)
-    {
-        // make sure that new null value would be interpreted as a null value
-        FlatPtr bits = PtrTraits::from_null_value(value);
-        VERIFY(PtrTraits::is_null(bits));
-        assign_raw(bits);
-    }
 
 private:
-    template<typename F>
-    void do_while_locked(F f) const
-    {
-#ifdef KERNEL
-        // We don't want to be pre-empted while we have the lock bit set
-        Kernel::ScopedCritical critical;
-#endif
-        FlatPtr bits = PtrTraits::lock(m_bits);
-        T* ptr = PtrTraits::as_ptr(bits);
-        f(ptr);
-        PtrTraits::unlock(m_bits, bits);
-    }
-
-    [[nodiscard]] ALWAYS_INLINE FlatPtr leak_ref_raw()
-    {
-        return PtrTraits::exchange(m_bits, PtrTraits::default_null_value);
-    }
-
-    [[nodiscard]] ALWAYS_INLINE FlatPtr add_ref_raw() const
-    {
-#ifdef KERNEL
-        // We don't want to be pre-empted while we have the lock bit set
-        Kernel::ScopedCritical critical;
-#endif
-        // This prevents a race condition between thread A and B:
-        // 1. Thread A copies RefPtr, e.g. through assignment or copy constructor,
-        //    gets the pointer from source, but is pre-empted before adding
-        //    another reference
-        // 2. Thread B calls clear, leak_ref, or release_nonnull on source, and
-        //    then drops the last reference, causing the object to be deleted
-        // 3. Thread A finishes step #1 by attempting to add a reference to
-        //    the object that was already deleted in step #2
-        FlatPtr bits = PtrTraits::lock(m_bits);
-        if (T* ptr = PtrTraits::as_ptr(bits))
-            ptr->ref();
-        PtrTraits::unlock(m_bits, bits);
-        return bits;
-    }
-
-    ALWAYS_INLINE void assign_raw(FlatPtr bits)
-    {
-        FlatPtr prev_bits = PtrTraits::exchange(m_bits, bits);
-        unref_if_not_null(PtrTraits::as_ptr(prev_bits));
-    }
-
     ALWAYS_INLINE T* as_ptr() const
     {
-        return PtrTraits::as_ptr(m_bits.load(AK::MemoryOrder::memory_order_relaxed));
+        return m_ptr;
     }
 
     ALWAYS_INLINE T* as_nonnull_ptr() const
     {
-        return as_nonnull_ptr(m_bits.load(AK::MemoryOrder::memory_order_relaxed));
+        VERIFY(m_ptr);
+        return m_ptr;
     }
 
-    ALWAYS_INLINE T* as_nonnull_ptr(FlatPtr bits) const
+    T* m_ptr { nullptr };
-    {
-        VERIFY(!PtrTraits::is_null(bits));
-        return PtrTraits::as_ptr(bits);
-    }
-
-    mutable Atomic<FlatPtr> m_bits { PtrTraits::default_null_value };
 };
 
 template<typename T>
@@ -496,17 +345,6 @@ inline RefPtr<T> try_make_ref_counted(Args&&... args)
     return adopt_ref_if_nonnull(new (nothrow) T { forward<Args>(args)... });
 }
 
-#ifdef KERNEL
-template<typename T>
-inline Kernel::KResultOr<NonnullRefPtr<T>> adopt_nonnull_ref_or_enomem(T* object)
-{
-    auto result = adopt_ref_if_nonnull(object);
-    if (!result)
-        return ENOMEM;
-    return result.release_nonnull();
-}
-#endif
-
 }
 
 using AK::adopt_ref_if_nonnull;
@@ -514,6 +352,4 @@ using AK::RefPtr;
 using AK::static_ptr_cast;
 using AK::try_make_ref_counted;
 
-#ifdef KERNEL
-using AK::adopt_nonnull_ref_or_enomem;
 #endif

AK/WeakPtr.h

@@ -6,7 +6,11 @@
 
 #pragma once
 
-#include <AK/Weakable.h>
+#ifdef KERNEL
+#    include <Kernel/Library/ThreadSafeWeakPtr.h>
+#else
+
+#    include <AK/Weakable.h>
 
 namespace AK {
 
@@ -65,21 +69,16 @@ public:
     }
 
     template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
-    WeakPtr(const RefPtr<U>& object)
+    WeakPtr(RefPtr<U> const& object)
     {
-        object.do_while_locked([&](U* obj) {
+        if (object)
-            if (obj)
+            m_link = object->template make_weak_ptr<U>().take_link();
-                m_link = obj->template make_weak_ptr<U>().take_link();
-        });
     }
 
     template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
-    WeakPtr(const NonnullRefPtr<U>& object)
+    WeakPtr(NonnullRefPtr<U> const& object)
     {
-        object.do_while_locked([&](U* obj) {
+        m_link = object->template make_weak_ptr<U>().take_link();
-            if (obj)
-                m_link = obj->template make_weak_ptr<U>().take_link();
-        });
     }
 
     template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
@@ -102,61 +101,36 @@ public:
     template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
     WeakPtr& operator=(const RefPtr<U>& object)
     {
-        object.do_while_locked([&](U* obj) {
+        if (object)
-            if (obj)
+            m_link = object->template make_weak_ptr<U>().take_link();
-                m_link = obj->template make_weak_ptr<U>().take_link();
+        else
-            else
+            m_link = nullptr;
-                m_link = nullptr;
-        });
         return *this;
     }
 
     template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
     WeakPtr& operator=(const NonnullRefPtr<U>& object)
     {
-        object.do_while_locked([&](U* obj) {
+        m_link = object->template make_weak_ptr<U>().take_link();
-            if (obj)
-                m_link = obj->template make_weak_ptr<U>().take_link();
-            else
-                m_link = nullptr;
-        });
         return *this;
     }
 
     [[nodiscard]] RefPtr<T> strong_ref() const
     {
-        // This only works with RefCounted objects, but it is the only
+        return RefPtr<T> { ptr() };
-        // safe way to get a strong reference from a WeakPtr. Any code
-        // that uses objects not derived from RefCounted will have to
-        // use unsafe_ptr(), but as the name suggests, it is not safe...
-        RefPtr<T> ref;
-        // Using do_while_locked protects against a race with clear()!
-        m_link.do_while_locked([&](WeakLink* link) {
-            if (link)
-                ref = link->template strong_ref<T>();
-        });
-        return ref;
     }
 
-#ifndef KERNEL
-    // A lot of user mode code is single-threaded. But for kernel mode code
-    // this is generally not true as everything is multi-threaded. So make
-    // these shortcuts and aliases only available to non-kernel code.
     T* ptr() const { return unsafe_ptr(); }
     T* operator->() { return unsafe_ptr(); }
     const T* operator->() const { return unsafe_ptr(); }
     operator const T*() const { return unsafe_ptr(); }
     operator T*() { return unsafe_ptr(); }
-#endif
 
     [[nodiscard]] T* unsafe_ptr() const
     {
-        T* ptr = nullptr;
+        if (m_link)
-        m_link.do_while_locked([&](WeakLink* link) {
+            return m_link->template unsafe_ptr<T>();
-            if (link)
+        return nullptr;
-                ptr = link->unsafe_ptr<T>();
-        });
-        return ptr;
     }
 
     operator bool() const { return m_link ? !m_link->is_null() : false; }
@@ -219,12 +193,7 @@ template<typename T>
 struct Formatter<WeakPtr<T>> : Formatter<const T*> {
     void format(FormatBuilder& builder, const WeakPtr<T>& value)
     {
-#ifdef KERNEL
-        auto ref = value.strong_ref();
-        Formatter<const T*>::format(builder, ref.ptr());
-#else
         Formatter<const T*>::format(builder, value.ptr());
-#endif
     }
 };
 
@@ -240,3 +209,4 @@ WeakPtr<T> try_make_weak_ptr(const T* ptr)
 }
 
 using AK::WeakPtr;
+#endif

Kernel/Library/ThreadSafeNonnullRefPtr.h

@@ -0,0 +1,359 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Assertions.h>
+#include <AK/Atomic.h>
+#include <AK/Format.h>
+#include <AK/Traits.h>
+#include <AK/Types.h>
+#ifdef KERNEL
+#    include <Kernel/Arch/x86/Processor.h>
+#    include <Kernel/Arch/x86/ScopedCritical.h>
+#endif
+
+namespace AK {
+
+template<typename T>
+class OwnPtr;
+template<typename T, typename PtrTraits>
+class RefPtr;
+
+template<typename T>
+ALWAYS_INLINE void ref_if_not_null(T* ptr)
+{
+    if (ptr)
+        ptr->ref();
+}
+
+template<typename T>
+ALWAYS_INLINE void unref_if_not_null(T* ptr)
+{
+    if (ptr)
+        ptr->unref();
+}
+
+template<typename T>
+class NonnullRefPtr {
+    template<typename U, typename P>
+    friend class RefPtr;
+    template<typename U>
+    friend class NonnullRefPtr;
+    template<typename U>
+    friend class WeakPtr;
+
+public:
+    using ElementType = T;
+
+    enum AdoptTag { Adopt };
+
+    ALWAYS_INLINE NonnullRefPtr(const T& object)
+        : m_bits((FlatPtr)&object)
+    {
+        VERIFY(!(m_bits & 1));
+        const_cast<T&>(object).ref();
+    }
+    template<typename U>
+    ALWAYS_INLINE NonnullRefPtr(const U& object) requires(IsConvertible<U*, T*>)
+        : m_bits((FlatPtr) static_cast<const T*>(&object))
+    {
+        VERIFY(!(m_bits & 1));
+        const_cast<T&>(static_cast<const T&>(object)).ref();
+    }
+    ALWAYS_INLINE NonnullRefPtr(AdoptTag, T& object)
+        : m_bits((FlatPtr)&object)
+    {
+        VERIFY(!(m_bits & 1));
+    }
+    ALWAYS_INLINE NonnullRefPtr(NonnullRefPtr&& other)
+        : m_bits((FlatPtr)&other.leak_ref())
+    {
+        VERIFY(!(m_bits & 1));
+    }
+    template<typename U>
+    ALWAYS_INLINE NonnullRefPtr(NonnullRefPtr<U>&& other) requires(IsConvertible<U*, T*>)
+        : m_bits((FlatPtr)&other.leak_ref())
+    {
+        VERIFY(!(m_bits & 1));
+    }
+    ALWAYS_INLINE NonnullRefPtr(const NonnullRefPtr& other)
+        : m_bits((FlatPtr)other.add_ref())
+    {
+        VERIFY(!(m_bits & 1));
+    }
+    template<typename U>
+    ALWAYS_INLINE NonnullRefPtr(const NonnullRefPtr<U>& other) requires(IsConvertible<U*, T*>)
+        : m_bits((FlatPtr)other.add_ref())
+    {
+        VERIFY(!(m_bits & 1));
+    }
+    ALWAYS_INLINE ~NonnullRefPtr()
+    {
+        assign(nullptr);
+#ifdef SANITIZE_PTRS
+        m_bits.store(explode_byte(0xb0), AK::MemoryOrder::memory_order_relaxed);
+#endif
+    }
+
+    template<typename U>
+    NonnullRefPtr(const OwnPtr<U>&) = delete;
+    template<typename U>
+    NonnullRefPtr& operator=(const OwnPtr<U>&) = delete;
+
+    template<typename U>
+    NonnullRefPtr(const RefPtr<U>&) = delete;
+    template<typename U>
+    NonnullRefPtr& operator=(const RefPtr<U>&) = delete;
+    NonnullRefPtr(const RefPtr<T>&) = delete;
+    NonnullRefPtr& operator=(const RefPtr<T>&) = delete;
+
+    NonnullRefPtr& operator=(const NonnullRefPtr& other)
+    {
+        if (this != &other)
+            assign(other.add_ref());
+        return *this;
+    }
+
+    template<typename U>
+    NonnullRefPtr& operator=(const NonnullRefPtr<U>& other) requires(IsConvertible<U*, T*>)
+    {
+        assign(other.add_ref());
+        return *this;
+    }
+
+    ALWAYS_INLINE NonnullRefPtr& operator=(NonnullRefPtr&& other)
+    {
+        if (this != &other)
+            assign(&other.leak_ref());
+        return *this;
+    }
+
+    template<typename U>
+    NonnullRefPtr& operator=(NonnullRefPtr<U>&& other) requires(IsConvertible<U*, T*>)
+    {
+        assign(&other.leak_ref());
+        return *this;
+    }
+
+    NonnullRefPtr& operator=(const T& object)
+    {
+        const_cast<T&>(object).ref();
+        assign(const_cast<T*>(&object));
+        return *this;
+    }
+
+    [[nodiscard]] ALWAYS_INLINE T& leak_ref()
+    {
+        T* ptr = exchange(nullptr);
+        VERIFY(ptr);
+        return *ptr;
+    }
+
+    ALWAYS_INLINE RETURNS_NONNULL T* ptr()
+    {
+        return as_nonnull_ptr();
+    }
+    ALWAYS_INLINE RETURNS_NONNULL const T* ptr() const
+    {
+        return as_nonnull_ptr();
+    }
+
+    ALWAYS_INLINE RETURNS_NONNULL T* operator->()
+    {
+        return as_nonnull_ptr();
+    }
+    ALWAYS_INLINE RETURNS_NONNULL const T* operator->() const
+    {
+        return as_nonnull_ptr();
+    }
+
+    ALWAYS_INLINE T& operator*()
+    {
+        return *as_nonnull_ptr();
+    }
+    ALWAYS_INLINE const T& operator*() const
+    {
+        return *as_nonnull_ptr();
+    }
+
+    ALWAYS_INLINE RETURNS_NONNULL operator T*()
+    {
+        return as_nonnull_ptr();
+    }
+    ALWAYS_INLINE RETURNS_NONNULL operator const T*() const
+    {
+        return as_nonnull_ptr();
+    }
+
+    ALWAYS_INLINE operator T&()
+    {
+        return *as_nonnull_ptr();
+    }
+    ALWAYS_INLINE operator const T&() const
+    {
+        return *as_nonnull_ptr();
+    }
+
+    operator bool() const = delete;
+    bool operator!() const = delete;
+
+    void swap(NonnullRefPtr& other)
+    {
+        if (this == &other)
+            return;
+
+        // NOTE: swap is not atomic!
+        T* other_ptr = other.exchange(nullptr);
+        T* ptr = exchange(other_ptr);
+        other.exchange(ptr);
+    }
+
+    template<typename U>
+    void swap(NonnullRefPtr<U>& other) requires(IsConvertible<U*, T*>)
+    {
+        // NOTE: swap is not atomic!
+        U* other_ptr = other.exchange(nullptr);
+        T* ptr = exchange(other_ptr);
+        other.exchange(ptr);
+    }
+
+private:
+    NonnullRefPtr() = delete;
+
+    ALWAYS_INLINE T* as_ptr() const
+    {
+        return (T*)(m_bits.load(AK::MemoryOrder::memory_order_relaxed) & ~(FlatPtr)1);
+    }
+
+    ALWAYS_INLINE RETURNS_NONNULL T* as_nonnull_ptr() const
+    {
+        T* ptr = (T*)(m_bits.load(AK::MemoryOrder::memory_order_relaxed) & ~(FlatPtr)1);
+        VERIFY(ptr);
+        return ptr;
+    }
+
+    template<typename F>
+    void do_while_locked(F f) const
+    {
+#ifdef KERNEL
+        // We don't want to be pre-empted while we have the lock bit set
+        Kernel::ScopedCritical critical;
+#endif
+        FlatPtr bits;
+        for (;;) {
+            bits = m_bits.fetch_or(1, AK::MemoryOrder::memory_order_acq_rel);
+            if (!(bits & 1))
+                break;
+#ifdef KERNEL
+            Kernel::Processor::wait_check();
+#endif
+        }
+        VERIFY(!(bits & 1));
+        f((T*)bits);
+        m_bits.store(bits, AK::MemoryOrder::memory_order_release);
+    }
+
+    ALWAYS_INLINE void assign(T* new_ptr)
+    {
+        T* prev_ptr = exchange(new_ptr);
+        unref_if_not_null(prev_ptr);
+    }
+
+    ALWAYS_INLINE T* exchange(T* new_ptr)
+    {
+        VERIFY(!((FlatPtr)new_ptr & 1));
+#ifdef KERNEL
+        // We don't want to be pre-empted while we have the lock bit set
+        Kernel::ScopedCritical critical;
+#endif
+        // Only exchange while not locked
+        FlatPtr expected = m_bits.load(AK::MemoryOrder::memory_order_relaxed);
+        for (;;) {
+            expected &= ~(FlatPtr)1; // only if lock bit is not set
+            if (m_bits.compare_exchange_strong(expected, (FlatPtr)new_ptr, AK::MemoryOrder::memory_order_acq_rel))
+                break;
+#ifdef KERNEL
+            Kernel::Processor::wait_check();
+#endif
+        }
+        VERIFY(!(expected & 1));
+        return (T*)expected;
+    }
+
+    T* add_ref() const
+    {
+#ifdef KERNEL
+        // We don't want to be pre-empted while we have the lock bit set
+        Kernel::ScopedCritical critical;
+#endif
+        // Lock the pointer
+        FlatPtr expected = m_bits.load(AK::MemoryOrder::memory_order_relaxed);
+        for (;;) {
+            expected &= ~(FlatPtr)1; // only if lock bit is not set
+            if (m_bits.compare_exchange_strong(expected, expected | 1, AK::MemoryOrder::memory_order_acq_rel))
+                break;
+#ifdef KERNEL
+            Kernel::Processor::wait_check();
+#endif
+        }
+
+        // Add a reference now that we locked the pointer
+        ref_if_not_null((T*)expected);
+
+        // Unlock the pointer again
+        m_bits.store(expected, AK::MemoryOrder::memory_order_release);
+        return (T*)expected;
+    }
+
+    mutable Atomic<FlatPtr> m_bits { 0 };
+};
+
+template<typename T>
+inline NonnullRefPtr<T> adopt_ref(T& object)
+{
+    return NonnullRefPtr<T>(NonnullRefPtr<T>::Adopt, object);
+}
+
+template<typename T>
+struct Formatter<NonnullRefPtr<T>> : Formatter<const T*> {
+    void format(FormatBuilder& builder, const NonnullRefPtr<T>& value)
+    {
+        Formatter<const T*>::format(builder, value.ptr());
+    }
+};
+
+template<typename T, typename U>
+inline void swap(NonnullRefPtr<T>& a, NonnullRefPtr<U>& b) requires(IsConvertible<U*, T*>)
+{
+    a.swap(b);
+}
+
+template<typename T, class... Args>
+requires(IsConstructible<T, Args...>) inline NonnullRefPtr<T> make_ref_counted(Args&&... args)
+{
+    return NonnullRefPtr<T>(NonnullRefPtr<T>::Adopt, *new T(forward<Args>(args)...));
+}
+
+// FIXME: Remove once P0960R3 is available in Clang.
+template<typename T, class... Args>
+inline NonnullRefPtr<T> make_ref_counted(Args&&... args)
+{
+    return NonnullRefPtr<T>(NonnullRefPtr<T>::Adopt, *new T { forward<Args>(args)... });
+}
+}
+
+template<typename T>
+struct Traits<NonnullRefPtr<T>> : public GenericTraits<NonnullRefPtr<T>> {
+    using PeekType = T*;
+    using ConstPeekType = const T*;
+    static unsigned hash(const NonnullRefPtr<T>& p) { return ptr_hash(p.ptr()); }
+    static bool equals(const NonnullRefPtr<T>& a, const NonnullRefPtr<T>& b) { return a.ptr() == b.ptr(); }
+};
+
+using AK::adopt_ref;
+using AK::make_ref_counted;
+using AK::NonnullRefPtr;

Kernel/Library/ThreadSafeRefCounted.h

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Assertions.h>
+#include <AK/Atomic.h>
+#include <AK/Checked.h>
+#include <AK/Noncopyable.h>
+#include <AK/Platform.h>
+#include <AK/StdLibExtras.h>
+
+namespace AK {
+
+template<class T>
+constexpr auto call_will_be_destroyed_if_present(const T* object) -> decltype(const_cast<T*>(object)->will_be_destroyed(), TrueType {})
+{
+    const_cast<T*>(object)->will_be_destroyed();
+    return {};
+}
+
+constexpr auto call_will_be_destroyed_if_present(...) -> FalseType
+{
+    return {};
+}
+
+template<class T>
+constexpr auto call_one_ref_left_if_present(const T* object) -> decltype(const_cast<T*>(object)->one_ref_left(), TrueType {})
+{
+    const_cast<T*>(object)->one_ref_left();
+    return {};
+}
+
+constexpr auto call_one_ref_left_if_present(...) -> FalseType
+{
+    return {};
+}
+
+class RefCountedBase {
+    AK_MAKE_NONCOPYABLE(RefCountedBase);
+    AK_MAKE_NONMOVABLE(RefCountedBase);
+
+public:
+    using RefCountType = unsigned int;
+    using AllowOwnPtr = FalseType;
+
+    void ref() const
+    {
+        auto old_ref_count = m_ref_count.fetch_add(1, AK::MemoryOrder::memory_order_relaxed);
+        VERIFY(old_ref_count > 0);
+        VERIFY(!Checked<RefCountType>::addition_would_overflow(old_ref_count, 1));
+    }
+
+    [[nodiscard]] bool try_ref() const
+    {
+        RefCountType expected = m_ref_count.load(AK::MemoryOrder::memory_order_relaxed);
+        for (;;) {
+            if (expected == 0)
+                return false;
+            VERIFY(!Checked<RefCountType>::addition_would_overflow(expected, 1));
+            if (m_ref_count.compare_exchange_strong(expected, expected + 1, AK::MemoryOrder::memory_order_acquire))
+                return true;
+        }
+    }
+
+    [[nodiscard]] RefCountType ref_count() const
+    {
+        return m_ref_count.load(AK::MemoryOrder::memory_order_relaxed);
+    }
+
+protected:
+    RefCountedBase() = default;
+    ~RefCountedBase()
+    {
+        VERIFY(m_ref_count.load(AK::MemoryOrder::memory_order_relaxed) == 0);
+    }
+
+    RefCountType deref_base() const
+    {
+        auto old_ref_count = m_ref_count.fetch_sub(1, AK::MemoryOrder::memory_order_acq_rel);
+        VERIFY(old_ref_count > 0);
+        return old_ref_count - 1;
+    }
+
+    mutable Atomic<RefCountType> m_ref_count { 1 };
+};
+
+template<typename T>
+class RefCounted : public RefCountedBase {
+public:
+    bool unref() const
+    {
+        auto new_ref_count = deref_base();
+        if (new_ref_count == 0) {
+            call_will_be_destroyed_if_present(static_cast<const T*>(this));
+            delete static_cast<const T*>(this);
+            return true;
+        } else if (new_ref_count == 1) {
+            call_one_ref_left_if_present(static_cast<const T*>(this));
+        }
+        return false;
+    }
+};
+
+}
+
+using AK::RefCounted;
+using AK::RefCountedBase;

Kernel/Library/ThreadSafeRefPtr.h

@@ -0,0 +1,519 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Assertions.h>
+#include <AK/Atomic.h>
+#include <AK/Format.h>
+#include <AK/NonnullRefPtr.h>
+#include <AK/StdLibExtras.h>
+#include <AK/Traits.h>
+#include <AK/Types.h>
+#ifdef KERNEL
+#    include <Kernel/API/KResult.h>
+#    include <Kernel/Arch/x86/Processor.h>
+#    include <Kernel/Arch/x86/ScopedCritical.h>
+#endif
+
+namespace AK {
+
+template<typename T>
+class OwnPtr;
+
+template<typename T>
+struct RefPtrTraits {
+    ALWAYS_INLINE static T* as_ptr(FlatPtr bits)
+    {
+        return (T*)(bits & ~(FlatPtr)1);
+    }
+
+    ALWAYS_INLINE static FlatPtr as_bits(T* ptr)
+    {
+        VERIFY(!((FlatPtr)ptr & 1));
+        return (FlatPtr)ptr;
+    }
+
+    template<typename U, typename PtrTraits>
+    ALWAYS_INLINE static FlatPtr convert_from(FlatPtr bits)
+    {
+        if (PtrTraits::is_null(bits))
+            return default_null_value;
+        return as_bits(PtrTraits::as_ptr(bits));
+    }
+
+    ALWAYS_INLINE static bool is_null(FlatPtr bits)
+    {
+        return !(bits & ~(FlatPtr)1);
+    }
+
+    ALWAYS_INLINE static FlatPtr exchange(Atomic<FlatPtr>& atomic_var, FlatPtr new_value)
+    {
+        // Only exchange when lock is not held
+        VERIFY(!(new_value & 1));
+        FlatPtr expected = atomic_var.load(AK::MemoryOrder::memory_order_relaxed);
+        for (;;) {
+            expected &= ~(FlatPtr)1; // only if lock bit is not set
+            if (atomic_var.compare_exchange_strong(expected, new_value, AK::MemoryOrder::memory_order_acq_rel))
+                break;
+#ifdef KERNEL
+            Kernel::Processor::wait_check();
+#endif
+        }
+        return expected;
+    }
+
+    ALWAYS_INLINE static bool exchange_if_null(Atomic<FlatPtr>& atomic_var, FlatPtr new_value)
+    {
+        // Only exchange when lock is not held
+        VERIFY(!(new_value & 1));
+        for (;;) {
+            FlatPtr expected = default_null_value; // only if lock bit is not set
+            if (atomic_var.compare_exchange_strong(expected, new_value, AK::MemoryOrder::memory_order_acq_rel))
+                break;
+            if (!is_null(expected))
+                return false;
+#ifdef KERNEL
+            Kernel::Processor::wait_check();
+#endif
+        }
+        return true;
+    }
+
+    ALWAYS_INLINE static FlatPtr lock(Atomic<FlatPtr>& atomic_var)
+    {
+        // This sets the lock bit atomically, preventing further modifications.
+        // This is important when e.g. copying a RefPtr where the source
+        // might be released and freed too quickly. This allows us
+        // to temporarily lock the pointer so we can add a reference, then
+        // unlock it
+        FlatPtr bits;
+        for (;;) {
+            bits = atomic_var.fetch_or(1, AK::MemoryOrder::memory_order_acq_rel);
+            if (!(bits & 1))
+                break;
+#ifdef KERNEL
+            Kernel::Processor::wait_check();
+#endif
+        }
+        VERIFY(!(bits & 1));
+        return bits;
+    }
+
+    ALWAYS_INLINE static void unlock(Atomic<FlatPtr>& atomic_var, FlatPtr new_value)
+    {
+        VERIFY(!(new_value & 1));
+        atomic_var.store(new_value, AK::MemoryOrder::memory_order_release);
+    }
+
+    static constexpr FlatPtr default_null_value = 0;
+
+    using NullType = std::nullptr_t;
+};
+
+template<typename T, typename PtrTraits>
+class RefPtr {
+    template<typename U, typename P>
+    friend class RefPtr;
+    template<typename U>
+    friend class WeakPtr;
+
+public:
+    enum AdoptTag {
+        Adopt
+    };
+
+    RefPtr() = default;
+    RefPtr(const T* ptr)
+        : m_bits(PtrTraits::as_bits(const_cast<T*>(ptr)))
+    {
+        ref_if_not_null(const_cast<T*>(ptr));
+    }
+    RefPtr(const T& object)
+        : m_bits(PtrTraits::as_bits(const_cast<T*>(&object)))
+    {
+        T* ptr = const_cast<T*>(&object);
+        VERIFY(ptr);
+        VERIFY(!is_null());
+        ptr->ref();
+    }
+    RefPtr(AdoptTag, T& object)
+        : m_bits(PtrTraits::as_bits(&object))
+    {
+        VERIFY(!is_null());
+    }
+    RefPtr(RefPtr&& other)
+        : m_bits(other.leak_ref_raw())
+    {
+    }
+    ALWAYS_INLINE RefPtr(const NonnullRefPtr<T>& other)
+        : m_bits(PtrTraits::as_bits(const_cast<T*>(other.add_ref())))
+    {
+    }
+    template<typename U>
+    ALWAYS_INLINE RefPtr(const NonnullRefPtr<U>& other) requires(IsConvertible<U*, T*>)
+        : m_bits(PtrTraits::as_bits(const_cast<U*>(other.add_ref())))
+    {
+    }
+    template<typename U>
+    ALWAYS_INLINE RefPtr(NonnullRefPtr<U>&& other) requires(IsConvertible<U*, T*>)
+        : m_bits(PtrTraits::as_bits(&other.leak_ref()))
+    {
+        VERIFY(!is_null());
+    }
+    template<typename U, typename P = RefPtrTraits<U>>
+    RefPtr(RefPtr<U, P>&& other) requires(IsConvertible<U*, T*>)
+        : m_bits(PtrTraits::template convert_from<U, P>(other.leak_ref_raw()))
+    {
+    }
+    RefPtr(const RefPtr& other)
+        : m_bits(other.add_ref_raw())
+    {
+    }
+    template<typename U, typename P = RefPtrTraits<U>>
+    RefPtr(const RefPtr<U, P>& other) requires(IsConvertible<U*, T*>)
+        : m_bits(other.add_ref_raw())
+    {
+    }
+    ALWAYS_INLINE ~RefPtr()
+    {
+        clear();
+#ifdef SANITIZE_PTRS
+        m_bits.store(explode_byte(0xe0), AK::MemoryOrder::memory_order_relaxed);
+#endif
+    }
+
+    template<typename U>
+    RefPtr(const OwnPtr<U>&) = delete;
+    template<typename U>
+    RefPtr& operator=(const OwnPtr<U>&) = delete;
+
+    void swap(RefPtr& other)
+    {
+        if (this == &other)
+            return;
+
+        // NOTE: swap is not atomic!
+        FlatPtr other_bits = PtrTraits::exchange(other.m_bits, PtrTraits::default_null_value);
+        FlatPtr bits = PtrTraits::exchange(m_bits, other_bits);
+        PtrTraits::exchange(other.m_bits, bits);
+    }
+
+    template<typename U, typename P = RefPtrTraits<U>>
+    void swap(RefPtr<U, P>& other) requires(IsConvertible<U*, T*>)
+    {
+        // NOTE: swap is not atomic!
+        FlatPtr other_bits = P::exchange(other.m_bits, P::default_null_value);
+        FlatPtr bits = PtrTraits::exchange(m_bits, PtrTraits::template convert_from<U, P>(other_bits));
+        P::exchange(other.m_bits, P::template convert_from<U, P>(bits));
+    }
+
+    ALWAYS_INLINE RefPtr& operator=(RefPtr&& other)
+    {
+        if (this != &other)
+            assign_raw(other.leak_ref_raw());
+        return *this;
+    }
+
+    template<typename U, typename P = RefPtrTraits<U>>
+    ALWAYS_INLINE RefPtr& operator=(RefPtr<U, P>&& other) requires(IsConvertible<U*, T*>)
+    {
+        assign_raw(PtrTraits::template convert_from<U, P>(other.leak_ref_raw()));
+        return *this;
+    }
+
+    template<typename U>
+    ALWAYS_INLINE RefPtr& operator=(NonnullRefPtr<U>&& other) requires(IsConvertible<U*, T*>)
+    {
+        assign_raw(PtrTraits::as_bits(&other.leak_ref()));
+        return *this;
+    }
+
+    ALWAYS_INLINE RefPtr& operator=(const NonnullRefPtr<T>& other)
+    {
+        assign_raw(PtrTraits::as_bits(other.add_ref()));
+        return *this;
+    }
+
+    template<typename U>
+    ALWAYS_INLINE RefPtr& operator=(const NonnullRefPtr<U>& other) requires(IsConvertible<U*, T*>)
+    {
+        assign_raw(PtrTraits::as_bits(other.add_ref()));
+        return *this;
+    }
+
+    ALWAYS_INLINE RefPtr& operator=(const RefPtr& other)
+    {
+        if (this != &other)
+            assign_raw(other.add_ref_raw());
+        return *this;
+    }
+
+    template<typename U>
+    ALWAYS_INLINE RefPtr& operator=(const RefPtr<U>& other) requires(IsConvertible<U*, T*>)
+    {
+        assign_raw(other.add_ref_raw());
+        return *this;
+    }
+
+    ALWAYS_INLINE RefPtr& operator=(const T* ptr)
+    {
+        ref_if_not_null(const_cast<T*>(ptr));
+        assign_raw(PtrTraits::as_bits(const_cast<T*>(ptr)));
+        return *this;
+    }
+
+    ALWAYS_INLINE RefPtr& operator=(const T& object)
+    {
+        const_cast<T&>(object).ref();
+        assign_raw(PtrTraits::as_bits(const_cast<T*>(&object)));
+        return *this;
+    }
+
+    RefPtr& operator=(std::nullptr_t)
+    {
+        clear();
+        return *this;
+    }
+
+    ALWAYS_INLINE bool assign_if_null(RefPtr&& other)
+    {
+        if (this == &other)
+            return is_null();
+        return PtrTraits::exchange_if_null(m_bits, other.leak_ref_raw());
+    }
+
+    template<typename U, typename P = RefPtrTraits<U>>
+    ALWAYS_INLINE bool assign_if_null(RefPtr<U, P>&& other)
+    {
+        if (this == &other)
+            return is_null();
+        return PtrTraits::exchange_if_null(m_bits, PtrTraits::template convert_from<U, P>(other.leak_ref_raw()));
+    }
+
+    ALWAYS_INLINE void clear()
+    {
+        assign_raw(PtrTraits::default_null_value);
+    }
+
+    bool operator!() const { return PtrTraits::is_null(m_bits.load(AK::MemoryOrder::memory_order_relaxed)); }
+
+    [[nodiscard]] T* leak_ref()
+    {
+        FlatPtr bits = PtrTraits::exchange(m_bits, PtrTraits::default_null_value);
+        return PtrTraits::as_ptr(bits);
+    }
+
+    NonnullRefPtr<T> release_nonnull()
+    {
+        FlatPtr bits = PtrTraits::exchange(m_bits, PtrTraits::default_null_value);
+        VERIFY(!PtrTraits::is_null(bits));
+        return NonnullRefPtr<T>(NonnullRefPtr<T>::Adopt, *PtrTraits::as_ptr(bits));
+    }
+
+    ALWAYS_INLINE T* ptr() { return as_ptr(); }
+    ALWAYS_INLINE const T* ptr() const { return as_ptr(); }
+
+    ALWAYS_INLINE T* operator->()
+    {
+        return as_nonnull_ptr();
+    }
+
+    ALWAYS_INLINE const T* operator->() const
+    {
+        return as_nonnull_ptr();
+    }
+
+    ALWAYS_INLINE T& operator*()
+    {
+        return *as_nonnull_ptr();
+    }
+
+    ALWAYS_INLINE const T& operator*() const
+    {
+        return *as_nonnull_ptr();
+    }
+
+    ALWAYS_INLINE operator const T*() const { return as_ptr(); }
+    ALWAYS_INLINE operator T*() { return as_ptr(); }
+
+    ALWAYS_INLINE operator bool() { return !is_null(); }
+
+    bool operator==(std::nullptr_t) const { return is_null(); }
+    bool operator!=(std::nullptr_t) const { return !is_null(); }
+
+    bool operator==(const RefPtr& other) const { return as_ptr() == other.as_ptr(); }
+    bool operator!=(const RefPtr& other) const { return as_ptr() != other.as_ptr(); }
+
+    bool operator==(RefPtr& other) { return as_ptr() == other.as_ptr(); }
+    bool operator!=(RefPtr& other) { return as_ptr() != other.as_ptr(); }
+
+    bool operator==(const T* other) const { return as_ptr() == other; }
+    bool operator!=(const T* other) const { return as_ptr() != other; }
+
+    bool operator==(T* other) { return as_ptr() == other; }
+    bool operator!=(T* other) { return as_ptr() != other; }
+
+    ALWAYS_INLINE bool is_null() const { return PtrTraits::is_null(m_bits.load(AK::MemoryOrder::memory_order_relaxed)); }
+
+    template<typename U = T, typename EnableIf<IsSame<U, T> && !IsNullPointer<typename PtrTraits::NullType>>::Type* = nullptr>
+    typename PtrTraits::NullType null_value() const
+    {
+        // make sure we are holding a null value
+        FlatPtr bits = m_bits.load(AK::MemoryOrder::memory_order_relaxed);
+        VERIFY(PtrTraits::is_null(bits));
+        return PtrTraits::to_null_value(bits);
+    }
+    template<typename U = T, typename EnableIf<IsSame<U, T> && !IsNullPointer<typename PtrTraits::NullType>>::Type* = nullptr>
+    void set_null_value(typename PtrTraits::NullType value)
+    {
+        // make sure that new null value would be interpreted as a null value
+        FlatPtr bits = PtrTraits::from_null_value(value);
+        VERIFY(PtrTraits::is_null(bits));
+        assign_raw(bits);
+    }
+
+private:
+    template<typename F>
+    void do_while_locked(F f) const
+    {
+#ifdef KERNEL
+        // We don't want to be pre-empted while we have the lock bit set
+        Kernel::ScopedCritical critical;
+#endif
+        FlatPtr bits = PtrTraits::lock(m_bits);
+        T* ptr = PtrTraits::as_ptr(bits);
+        f(ptr);
+        PtrTraits::unlock(m_bits, bits);
+    }
+
+    [[nodiscard]] ALWAYS_INLINE FlatPtr leak_ref_raw()
+    {
+        return PtrTraits::exchange(m_bits, PtrTraits::default_null_value);
+    }
+
+    [[nodiscard]] ALWAYS_INLINE FlatPtr add_ref_raw() const
+    {
+#ifdef KERNEL
+        // We don't want to be pre-empted while we have the lock bit set
+        Kernel::ScopedCritical critical;
+#endif
+        // This prevents a race condition between thread A and B:
+        // 1. Thread A copies RefPtr, e.g. through assignment or copy constructor,
+        //    gets the pointer from source, but is pre-empted before adding
+        //    another reference
+        // 2. Thread B calls clear, leak_ref, or release_nonnull on source, and
+        //    then drops the last reference, causing the object to be deleted
+        // 3. Thread A finishes step #1 by attempting to add a reference to
+        //    the object that was already deleted in step #2
+        FlatPtr bits = PtrTraits::lock(m_bits);
+        if (T* ptr = PtrTraits::as_ptr(bits))
+            ptr->ref();
+        PtrTraits::unlock(m_bits, bits);
+        return bits;
+    }
+
+    ALWAYS_INLINE void assign_raw(FlatPtr bits)
+    {
+        FlatPtr prev_bits = PtrTraits::exchange(m_bits, bits);
+        unref_if_not_null(PtrTraits::as_ptr(prev_bits));
+    }
+
+    ALWAYS_INLINE T* as_ptr() const
+    {
+        return PtrTraits::as_ptr(m_bits.load(AK::MemoryOrder::memory_order_relaxed));
+    }
+
+    ALWAYS_INLINE T* as_nonnull_ptr() const
+    {
+        return as_nonnull_ptr(m_bits.load(AK::MemoryOrder::memory_order_relaxed));
+    }
+
+    ALWAYS_INLINE T* as_nonnull_ptr(FlatPtr bits) const
+    {
+        VERIFY(!PtrTraits::is_null(bits));
+        return PtrTraits::as_ptr(bits);
+    }
+
+    mutable Atomic<FlatPtr> m_bits { PtrTraits::default_null_value };
+};
+
+template<typename T>
+struct Formatter<RefPtr<T>> : Formatter<const T*> {
+    void format(FormatBuilder& builder, const RefPtr<T>& value)
+    {
+        Formatter<const T*>::format(builder, value.ptr());
+    }
+};
+
+template<typename T>
+struct Traits<RefPtr<T>> : public GenericTraits<RefPtr<T>> {
+    using PeekType = T*;
+    using ConstPeekType = const T*;
+    static unsigned hash(const RefPtr<T>& p) { return ptr_hash(p.ptr()); }
+    static bool equals(const RefPtr<T>& a, const RefPtr<T>& b) { return a.ptr() == b.ptr(); }
+};
+
+template<typename T, typename U>
+inline NonnullRefPtr<T> static_ptr_cast(const NonnullRefPtr<U>& ptr)
+{
+    return NonnullRefPtr<T>(static_cast<const T&>(*ptr));
+}
+
+template<typename T, typename U, typename PtrTraits = RefPtrTraits<T>>
+inline RefPtr<T> static_ptr_cast(const RefPtr<U>& ptr)
+{
+    return RefPtr<T, PtrTraits>(static_cast<const T*>(ptr.ptr()));
+}
+
+template<typename T, typename PtrTraitsT, typename U, typename PtrTraitsU>
+inline void swap(RefPtr<T, PtrTraitsT>& a, RefPtr<U, PtrTraitsU>& b) requires(IsConvertible<U*, T*>)
+{
+    a.swap(b);
+}
+
+template<typename T>
+inline RefPtr<T> adopt_ref_if_nonnull(T* object)
+{
+    if (object)
+        return RefPtr<T>(RefPtr<T>::Adopt, *object);
+    return {};
+}
+
+template<typename T, class... Args>
+requires(IsConstructible<T, Args...>) inline RefPtr<T> try_make_ref_counted(Args&&... args)
+{
+    return adopt_ref_if_nonnull(new (nothrow) T(forward<Args>(args)...));
+}
+
+// FIXME: Remove once P0960R3 is available in Clang.
+template<typename T, class... Args>
+inline RefPtr<T> try_make_ref_counted(Args&&... args)
+{
+    return adopt_ref_if_nonnull(new (nothrow) T { forward<Args>(args)... });
+}
+
+#ifdef KERNEL
+template<typename T>
+inline Kernel::KResultOr<NonnullRefPtr<T>> adopt_nonnull_ref_or_enomem(T* object)
+{
+    auto result = adopt_ref_if_nonnull(object);
+    if (!result)
+        return ENOMEM;
+    return result.release_nonnull();
+}
+#endif
+
+}
+
+using AK::adopt_ref_if_nonnull;
+using AK::RefPtr;
+using AK::static_ptr_cast;
+using AK::try_make_ref_counted;
+
+#ifdef KERNEL
+using AK::adopt_nonnull_ref_or_enomem;
+#endif

Kernel/Library/ThreadSafeWeakPtr.h

@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Weakable.h>
+
+namespace AK {
+
+template<typename T>
+class WeakPtr {
+    template<typename U>
+    friend class Weakable;
+
+public:
+    WeakPtr() = default;
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr(const WeakPtr<U>& other)
+        : m_link(other.m_link)
+    {
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr(WeakPtr<U>&& other)
+        : m_link(other.take_link())
+    {
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr& operator=(WeakPtr<U>&& other)
+    {
+        m_link = other.take_link();
+        return *this;
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr& operator=(const WeakPtr<U>& other)
+    {
+        if ((const void*)this != (const void*)&other)
+            m_link = other.m_link;
+        return *this;
+    }
+
+    WeakPtr& operator=(std::nullptr_t)
+    {
+        clear();
+        return *this;
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr(const U& object)
+        : m_link(object.template make_weak_ptr<U>().take_link())
+    {
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr(const U* object)
+    {
+        if (object)
+            m_link = object->template make_weak_ptr<U>().take_link();
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr(const RefPtr<U>& object)
+    {
+        object.do_while_locked([&](U* obj) {
+            if (obj)
+                m_link = obj->template make_weak_ptr<U>().take_link();
+        });
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr(const NonnullRefPtr<U>& object)
+    {
+        object.do_while_locked([&](U* obj) {
+            if (obj)
+                m_link = obj->template make_weak_ptr<U>().take_link();
+        });
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr& operator=(const U& object)
+    {
+        m_link = object.template make_weak_ptr<U>().take_link();
+        return *this;
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr& operator=(const U* object)
+    {
+        if (object)
+            m_link = object->template make_weak_ptr<U>().take_link();
+        else
+            m_link = nullptr;
+        return *this;
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr& operator=(const RefPtr<U>& object)
+    {
+        object.do_while_locked([&](U* obj) {
+            if (obj)
+                m_link = obj->template make_weak_ptr<U>().take_link();
+            else
+                m_link = nullptr;
+        });
+        return *this;
+    }
+
+    template<typename U, typename EnableIf<IsBaseOf<T, U>>::Type* = nullptr>
+    WeakPtr& operator=(const NonnullRefPtr<U>& object)
+    {
+        object.do_while_locked([&](U* obj) {
+            if (obj)
+                m_link = obj->template make_weak_ptr<U>().take_link();
+            else
+                m_link = nullptr;
+        });
+        return *this;
+    }
+
+    [[nodiscard]] RefPtr<T> strong_ref() const
+    {
+        // This only works with RefCounted objects, but it is the only
+        // safe way to get a strong reference from a WeakPtr. Any code
+        // that uses objects not derived from RefCounted will have to
+        // use unsafe_ptr(), but as the name suggests, it is not safe...
+        RefPtr<T> ref;
+        // Using do_while_locked protects against a race with clear()!
+        m_link.do_while_locked([&](WeakLink* link) {
+            if (link)
+                ref = link->template strong_ref<T>();
+        });
+        return ref;
+    }
+
+#ifndef KERNEL
+    // A lot of user mode code is single-threaded. But for kernel mode code
+    // this is generally not true as everything is multi-threaded. So make
+    // these shortcuts and aliases only available to non-kernel code.
+    T* ptr() const { return unsafe_ptr(); }
+    T* operator->() { return unsafe_ptr(); }
+    const T* operator->() const { return unsafe_ptr(); }
+    operator const T*() const { return unsafe_ptr(); }
+    operator T*() { return unsafe_ptr(); }
+#endif
+
+    [[nodiscard]] T* unsafe_ptr() const
+    {
+        T* ptr = nullptr;
+        m_link.do_while_locked([&](WeakLink* link) {
+            if (link)
+                ptr = link->unsafe_ptr<T>();
+        });
+        return ptr;
+    }
+
+    operator bool() const { return m_link ? !m_link->is_null() : false; }
+
+    [[nodiscard]] bool is_null() const { return !m_link || m_link->is_null(); }
+    void clear() { m_link = nullptr; }
+
+    [[nodiscard]] RefPtr<WeakLink> take_link() { return move(m_link); }
+
+private:
+    WeakPtr(const RefPtr<WeakLink>& link)
+        : m_link(link)
+    {
+    }
+
+    RefPtr<WeakLink> m_link;
+};
+
+template<typename T>
+template<typename U>
+inline WeakPtr<U> Weakable<T>::make_weak_ptr() const
+{
+    if constexpr (IsBaseOf<RefCountedBase, T>) {
+        // Checking m_being_destroyed isn't sufficient when dealing with
+        // a RefCounted type.The reference count will drop to 0 before the
+        // destructor is invoked and revoke_weak_ptrs is called. So, try
+        // to add a ref (which should fail if the ref count is at 0) so
+        // that we prevent the destructor and revoke_weak_ptrs from being
+        // triggered until we're done.
+        if (!static_cast<const T*>(this)->try_ref())
+            return {};
+    } else {
+        // For non-RefCounted types this means a weak reference can be
+        // obtained until the ~Weakable destructor is invoked!
+        if (m_being_destroyed.load(AK::MemoryOrder::memory_order_acquire))
+            return {};
+    }
+    if (!m_link) {
+        // There is a small chance that we create a new WeakLink and throw
+        // it away because another thread beat us to it. But the window is
+        // pretty small and the overhead isn't terrible.
+        m_link.assign_if_null(adopt_ref(*new WeakLink(const_cast<T&>(static_cast<const T&>(*this)))));
+    }
+
+    WeakPtr<U> weak_ptr(m_link);
+
+    if constexpr (IsBaseOf<RefCountedBase, T>) {
+        // Now drop the reference we temporarily added
+        if (static_cast<const T*>(this)->unref()) {
+            // We just dropped the last reference, which should have called
+            // revoke_weak_ptrs, which should have invalidated our weak_ptr
+            VERIFY(!weak_ptr.strong_ref());
+            return {};
+        }
+    }
+    return weak_ptr;
+}
+
+template<typename T>
+struct Formatter<WeakPtr<T>> : Formatter<const T*> {
+    void format(FormatBuilder& builder, const WeakPtr<T>& value)
+    {
+#ifdef KERNEL
+        auto ref = value.strong_ref();
+        Formatter<const T*>::format(builder, ref.ptr());
+#else
+        Formatter<const T*>::format(builder, value.ptr());
+#endif
+    }
+};
+
+template<typename T>
+WeakPtr<T> try_make_weak_ptr(const T* ptr)
+{
+    if (ptr) {
+        return ptr->template make_weak_ptr<T>();
+    }
+    return {};
+}
+
+}
+
+using AK::WeakPtr;