Kernel: Implement more x86_64 context switching functionality

Kernel/Arch/x86/common/Processor.cpp

@@ -40,9 +40,11 @@ Atomic<u32> Processor::s_idle_cpu_mask { 0 };
 extern "C" void thread_context_first_enter(void);
 extern "C" void exit_kernel_thread(void);
 
-// The compiler can't see the calls to this function inside assembly.
-// Declare it, to avoid dead code warnings.
+// The compiler can't see the calls to these functions inside assembly.
+// Declare them, to avoid dead code warnings.
 extern "C" void context_first_init(Thread* from_thread, Thread* to_thread, TrapFrame* trap) __attribute__((used));
+extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __attribute__((used));
+extern "C" u32 do_init_context(Thread* thread, u32 flags) __attribute__((used));
 
 UNMAP_AFTER_INIT static void sse_init()
 {
@@ -1165,4 +1167,69 @@ extern "C" void context_first_init([[maybe_unused]] Thread* from_thread, [[maybe
     Scheduler::leave_on_first_switch(flags & ~0x200);
 }
 
+extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread)
+{
+    VERIFY(from_thread == to_thread || from_thread->state() != Thread::Running);
+    VERIFY(to_thread->state() == Thread::Running);
+
+    bool has_fxsr = Processor::current().has_feature(CPUFeature::FXSR);
+    Processor::set_current_thread(*to_thread);
+
+    auto& from_regs = from_thread->regs();
+    auto& to_regs = to_thread->regs();
+
+    if (has_fxsr)
+        asm volatile("fxsave %0"
+                     : "=m"(from_thread->fpu_state()));
+    else
+        asm volatile("fnsave %0"
+                     : "=m"(from_thread->fpu_state()));
+
+#if ARCH(I386)
+    from_regs.fs = get_fs();
+    from_regs.gs = get_gs();
+    set_fs(to_regs.fs);
+    set_gs(to_regs.gs);
+#endif
+
+    if (from_thread->process().is_traced())
+        read_debug_registers_into(from_thread->debug_register_state());
+
+    if (to_thread->process().is_traced()) {
+        write_debug_registers_from(to_thread->debug_register_state());
+    } else {
+        clear_debug_registers();
+    }
+
+    auto& processor = Processor::current();
+#if ARCH(I386)
+    auto& tls_descriptor = processor.get_gdt_entry(GDT_SELECTOR_TLS);
+    tls_descriptor.set_base(to_thread->thread_specific_data());
+    tls_descriptor.set_limit(to_thread->thread_specific_region_size());
+#endif
+
+    if (from_regs.cr3 != to_regs.cr3)
+        write_cr3(to_regs.cr3);
+
+    to_thread->set_cpu(processor.get_id());
+    processor.restore_in_critical(to_thread->saved_critical());
+
+    if (has_fxsr)
+        asm volatile("fxrstor %0" ::"m"(to_thread->fpu_state()));
+    else
+        asm volatile("frstor %0" ::"m"(to_thread->fpu_state()));
+
+    // TODO: ioperm?
+}
+
+extern "C" u32 do_init_context(Thread* thread, u32 flags)
+{
+    VERIFY_INTERRUPTS_DISABLED();
+#if ARCH(I386)
+    thread->regs().eflags = flags;
+#else
+    thread->regs().rflags = flags;
+#endif
+    return Processor::current().init_context(*thread, true);
+}
 }

Kernel/Arch/x86/i386/CPU.cpp

@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include <AK/Assertions.h>
-#include <AK/Types.h>
-
-#include <Kernel/Arch/x86/CPU.h>
-#include <Kernel/Arch/x86/Processor.h>
-#include <Kernel/Arch/x86/TrapFrame.h>
-#include <Kernel/KSyms.h>
-#include <Kernel/Process.h>
-#include <Kernel/Thread.h>
-
-namespace Kernel {
-
-// The compiler can't see the calls to these functions inside assembly.
-// Declare them, to avoid dead code warnings.
-extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __attribute__((used));
-extern "C" u32 do_init_context(Thread* thread, u32 flags) __attribute__((used));
-
-extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread)
-{
-    VERIFY(from_thread == to_thread || from_thread->state() != Thread::Running);
-    VERIFY(to_thread->state() == Thread::Running);
-
-    bool has_fxsr = Processor::current().has_feature(CPUFeature::FXSR);
-    Processor::set_current_thread(*to_thread);
-
-    auto& from_regs = from_thread->regs();
-    auto& to_regs = to_thread->regs();
-
-    if (has_fxsr)
-        asm volatile("fxsave %0"
-                     : "=m"(from_thread->fpu_state()));
-    else
-        asm volatile("fnsave %0"
-                     : "=m"(from_thread->fpu_state()));
-
-    from_regs.fs = get_fs();
-    from_regs.gs = get_gs();
-    set_fs(to_regs.fs);
-    set_gs(to_regs.gs);
-
-    if (from_thread->process().is_traced())
-        read_debug_registers_into(from_thread->debug_register_state());
-
-    if (to_thread->process().is_traced()) {
-        write_debug_registers_from(to_thread->debug_register_state());
-    } else {
-        clear_debug_registers();
-    }
-
-    auto& processor = Processor::current();
-    auto& tls_descriptor = processor.get_gdt_entry(GDT_SELECTOR_TLS);
-    tls_descriptor.set_base(to_thread->thread_specific_data());
-    tls_descriptor.set_limit(to_thread->thread_specific_region_size());
-
-    if (from_regs.cr3 != to_regs.cr3)
-        write_cr3(to_regs.cr3);
-
-    to_thread->set_cpu(processor.get_id());
-    processor.restore_in_critical(to_thread->saved_critical());
-
-    if (has_fxsr)
-        asm volatile("fxrstor %0" ::"m"(to_thread->fpu_state()));
-    else
-        asm volatile("frstor %0" ::"m"(to_thread->fpu_state()));
-
-    // TODO: ioperm?
-}
-
-extern "C" u32 do_init_context(Thread* thread, u32 flags)
-{
-    VERIFY_INTERRUPTS_DISABLED();
-    thread->regs().eflags = flags;
-    return Processor::current().init_context(*thread, true);
-}
-
-}

Kernel/Arch/x86/i386/Processor.cpp

@@ -193,8 +193,7 @@ void Processor::switch_context(Thread*& from_thread, Thread*& to_thread)
     // Switch to new thread context, passing from_thread and to_thread
     // through to the new context using registers edx and eax
     asm volatile(
-        // NOTE: changing how much we push to the stack affects
-        //       SWITCH_CONTEXT_TO_STACK_SIZE and thread_context_first_enter()!
+        // NOTE: changing how much we push to the stack affects thread_context_first_enter()!
         "pushfl \n"
         "pushl %%ebx \n"
         "pushl %%esi \n"

Kernel/Arch/x86/x86_64/CPU.cpp

@@ -1,37 +0,0 @@
-/*
- * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include <AK/Assertions.h>
-#include <AK/Types.h>
-
-#include <Kernel/Arch/x86/CPU.h>
-#include <Kernel/Arch/x86/Processor.h>
-#include <Kernel/Arch/x86/TrapFrame.h>
-#include <Kernel/KSyms.h>
-#include <Kernel/Process.h>
-#include <Kernel/Thread.h>
-
-namespace Kernel {
-
-// The compiler can't see the calls to these functions inside assembly.
-// Declare them, to avoid dead code warnings.
-extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __attribute__((used));
-extern "C" u32 do_init_context(Thread* thread, u32 flags) __attribute__((used));
-
-extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread)
-{
-    (void)from_thread;
-    (void)to_thread;
-    TODO();
-}
-
-extern "C" u32 do_init_context(Thread* thread, u32 flags)
-{
-    (void)thread;
-    (void)flags;
-    TODO();
-}
-}

Kernel/Arch/x86/x86_64/Processor.cpp

@@ -91,9 +91,8 @@ u32 Processor::init_context(Thread& thread, bool leave_crit)
     auto& regs = thread.regs();
     bool return_to_user = (regs.cs & 3) != 0;
 
-    stack_top -= 2 * sizeof(u64);
-    *reinterpret_cast<u64*>(kernel_stack_top - 2 * sizeof(u64)) = regs.rsp;
-    *reinterpret_cast<u64*>(kernel_stack_top - 3 * sizeof(u64)) = FlatPtr(&exit_kernel_thread);
+    stack_top -= 1 * sizeof(u64);
+    *reinterpret_cast<u64*>(kernel_stack_top - 2 * sizeof(u64)) = FlatPtr(&exit_kernel_thread);
 
     stack_top -= sizeof(RegisterState);
 
@@ -167,7 +166,71 @@ void Processor::switch_context(Thread*& from_thread, Thread*& to_thread)
     dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context --> switching out of: {} {}", VirtualAddress(from_thread), *from_thread);
     from_thread->save_critical(m_in_critical);
 
-    PANIC("Context switching not implemented.");
+    // clang-format off
+    // Switch to new thread context, passing from_thread and to_thread
+    // through to the new context using registers rdx and rax
+    asm volatile(
+        // NOTE: changing how much we push to the stack affects thread_context_first_enter()!
+        "pushfq \n"
+        "pushq %%rbx \n"
+        "pushq %%rcx \n"
+        "pushq %%rbp \n"
+        "pushq %%rsi \n"
+        "pushq %%rdi \n"
+        "pushq %%r8 \n"
+        "pushq %%r9 \n"
+        "pushq %%r10 \n"
+        "pushq %%r11 \n"
+        "pushq %%r12 \n"
+        "pushq %%r13 \n"
+        "pushq %%r14 \n"
+        "pushq %%r15 \n"
+        "movq %%rsp, %[from_rsp] \n"
+        "movabs $1f, %%rbx \n"
+        "movq %%rbx, %[from_rip] \n"
+        "movq %[to_rsp0], %%rbx \n"
+        "movl %%ebx, %[tss_rsp0l] \n"
+        "shrq $32, %%rbx \n"
+        "movl %%ebx, %[tss_rsp0h] \n"
+        "movq %[to_rsp], %%rsp \n"
+        "pushq %[to_thread] \n"
+        "pushq %[from_thread] \n"
+        "pushq %[to_rip] \n"
+        "cld \n"
+        "movq 16(%%rsp), %%rsi \n"
+        "movq 8(%%rsp), %%rdi \n"
+        "jmp enter_thread_context \n"
+        "1: \n"
+        "popq %%rdx \n"
+        "popq %%rax \n"
+        "popq %%r15 \n"
+        "popq %%r14 \n"
+        "popq %%r13 \n"
+        "popq %%r12 \n"
+        "popq %%r11 \n"
+        "popq %%r10 \n"
+        "popq %%r9 \n"
+        "popq %%r8 \n"
+        "popq %%rdi \n"
+        "popq %%rsi \n"
+        "popq %%rbp \n"
+        "popq %%rcx \n"
+        "popq %%rbx \n"
+        "popfq \n"
+        : [from_rsp] "=m" (from_thread->regs().rsp),
+        [from_rip] "=m" (from_thread->regs().rip),
+        [tss_rsp0l] "=m" (m_tss.rsp0l),
+        [tss_rsp0h] "=m" (m_tss.rsp0h),
+        "=d" (from_thread), // needed so that from_thread retains the correct value
+        "=a" (to_thread) // needed so that to_thread retains the correct value
+        : [to_rsp] "g" (to_thread->regs().rsp),
+        [to_rsp0] "g" (to_thread->regs().rsp0),
+        [to_rip] "c" (to_thread->regs().rip),
+        [from_thread] "d" (from_thread),
+        [to_thread] "a" (to_thread)
+        : "memory", "rbx"
+    );
+    // clang-format on
 
     dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context <-- from {} {} to {} {}", VirtualAddress(from_thread), *from_thread, VirtualAddress(to_thread), *to_thread);
 

Kernel/CMakeLists.txt

@@ -277,7 +277,6 @@ set(KERNEL_SOURCES
     ${KERNEL_SOURCES}
     ${CMAKE_CURRENT_SOURCE_DIR}/Arch/x86/${KERNEL_ARCH}/ASM_wrapper.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/Arch/x86/${KERNEL_ARCH}/Boot/boot.S
-    ${CMAKE_CURRENT_SOURCE_DIR}/Arch/x86/${KERNEL_ARCH}/CPU.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/Arch/x86/${KERNEL_ARCH}/InterruptEntry.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/Arch/x86/${KERNEL_ARCH}/Processor.cpp
 )

Kernel/Process.cpp

@@ -185,7 +185,7 @@ RefPtr<Process> Process::create_kernel_process(RefPtr<Thread>& first_thread, Str
     first_thread->regs().esp = FlatPtr(entry_data); // entry function argument is expected to be in regs.esp
 #else
     first_thread->regs().rip = (FlatPtr)entry;
-    first_thread->regs().rsp = FlatPtr(entry_data); // entry function argument is expected to be in regs.rsp
+    first_thread->regs().rdi = FlatPtr(entry_data); // entry function argument is expected to be in regs.rdi
 #endif
 
     if (process->pid() != 0) {

Kernel/Scheduler.cpp

@@ -354,8 +354,13 @@ bool Scheduler::context_switch(Thread* thread)
             from_thread->set_state(Thread::Runnable);
 
 #ifdef LOG_EVERY_CONTEXT_SWITCH
+#    if ARCH(I386)
         dbgln("Scheduler[{}]: {} -> {} [prio={}] {:04x}:{:08x}", Processor::id(), from_thread->tid().value(),
             thread->tid().value(), thread->priority(), thread->regs().cs, thread->regs().eip);
+#    else
+        dbgln("Scheduler[{}]: {} -> {} [prio={}] {:04x}:{:16x}", Processor::id(), from_thread->tid().value(),
+            thread->tid().value(), thread->priority(), thread->regs().cs, thread->regs().rip);
+#    endif
 #endif
     }
 
@@ -375,14 +380,19 @@ bool Scheduler::context_switch(Thread* thread)
     enter_current(*from_thread, false);
     VERIFY(thread == Thread::current());
 
-#if ARCH(I386)
     if (thread->process().is_user_process()) {
-        auto iopl = get_iopl_from_eflags(Thread::current()->get_register_dump_from_stack().eflags);
+        FlatPtr flags;
+        auto& regs = Thread::current()->get_register_dump_from_stack();
+#if ARCH(I386)
+        flags = regs.eflags;
+#else
+        flags = regs.rflags;
+#endif
+        auto iopl = get_iopl_from_eflags(flags);
         if (iopl != 0) {
             PANIC("Switched to thread {} with non-zero IOPL={}", Thread::current()->tid().value(), iopl);
         }
     }
-#endif
 
     return true;
 }