Kernel: Add x2APIC support

This allows addressing all cores on more modern processors. For now,
we still have a hardcoded limit of 64 due to s_processors being a
static array.

Kernel/ACPI/Definitions.h

@@ -277,6 +277,15 @@ struct [[gnu::packed]] ProcessorLocalAPIC {
     u32 flags;
 };
 
+// https://uefi.org/specs/ACPI/6.4/05_ACPI_Software_Programming_Model/ACPI_Software_Programming_Model.html#processor-local-x2apic-structure
+struct [[gnu::packed]] ProcessorLocalX2APIC {
+    MADTEntryHeader h;
+    u16 reserved;
+    u32 apic_id;
+    u32 flags;
+    u32 acpi_processor_id;
+};
+
 // https://uefi.org/specs/ACPI/6.4/05_ACPI_Software_Programming_Model/ACPI_Software_Programming_Model.html#interrupt-source-override-structure
 struct [[gnu::packed]] InterruptSourceOverride {
     MADTEntryHeader h;

Kernel/Arch/x86/Processor.h

@@ -102,9 +102,9 @@ struct DeferredCallEntry {
 };
 
 class Processor;
-// Note: We only support 8 processors at most at the moment,
-// so allocate 8 slots of inline capacity in the container.
-using ProcessorContainer = Array<Processor*, 8>;
+// Note: We only support 64 processors at most at the moment,
+// so allocate 64 slots of inline capacity in the container.
+using ProcessorContainer = Array<Processor*, 64>;
 
 class Processor {
     friend class ProcessorInfo;
@@ -190,6 +190,8 @@ public:
         s_idle_cpu_mask.fetch_and(~(1u << m_cpu), AK::MemoryOrder::memory_order_relaxed);
     }
 
+    static Processor& by_id(u32);
+
     static u32 count()
     {
         // NOTE: because this value never changes once all APs are booted,

Kernel/Arch/x86/common/Processor.cpp

@@ -582,6 +582,11 @@ ProcessorContainer& Processor::processors()
     return s_processors;
 }
 
+Processor& Processor::by_id(u32 id)
+{
+    return *s_processors[id];
+}
+
 void Processor::enter_trap(TrapFrame& trap, bool raise_irq)
 {
     VERIFY_INTERRUPTS_DISABLED();

Kernel/Interrupts/APIC.cpp

@@ -34,7 +34,9 @@
 #define APIC_ENABLED (1 << 8)
 
 #define APIC_BASE_MSR 0x1b
+#define APIC_REGS_MSR_BASE 0x800
 
+#define APIC_REG_ID 0x20
 #define APIC_REG_EOI 0xb0
 #define APIC_REG_LD 0xd0
 #define APIC_REG_DF 0xe0
@@ -145,27 +147,39 @@ PhysicalAddress APIC::get_base()
 void APIC::set_base(const PhysicalAddress& base)
 {
     MSR msr(APIC_BASE_MSR);
-    msr.set(base.get() | 0x800);
+    u64 flags = 1 << 11;
+    if (m_is_x2)
+        flags |= 1 << 10;
+    msr.set(base.get() | flags);
 }
 
 void APIC::write_register(u32 offset, u32 value)
 {
-    *reinterpret_cast<volatile u32*>(m_apic_base->vaddr().offset(offset).as_ptr()) = value;
+    if (m_is_x2) {
+        MSR msr(APIC_REGS_MSR_BASE + (offset >> 4));
+        msr.set(value);
+    } else {
+        *reinterpret_cast<volatile u32*>(m_apic_base->vaddr().offset(offset).as_ptr()) = value;
+    }
 }
 
 u32 APIC::read_register(u32 offset)
 {
+    if (m_is_x2) {
+        MSR msr(APIC_REGS_MSR_BASE + (offset >> 4));
+        return (u32)msr.get();
+    }
     return *reinterpret_cast<volatile u32*>(m_apic_base->vaddr().offset(offset).as_ptr());
 }
 
 void APIC::set_lvt(u32 offset, u8 interrupt)
 {
-    write_register(offset, (read_register(offset) & 0xffffffff) | interrupt);
+    write_register(offset, read_register(offset) | interrupt);
 }
 
 void APIC::set_siv(u32 offset, u8 interrupt)
 {
-    write_register(offset, (read_register(offset) & 0xffffffff) | interrupt | APIC_ENABLED);
+    write_register(offset, read_register(offset) | interrupt | APIC_ENABLED);
 }
 
 void APIC::wait_for_pending_icr()
@@ -177,8 +191,13 @@ void APIC::wait_for_pending_icr()
 
 void APIC::write_icr(const ICRReg& icr)
 {
-    write_register(APIC_REG_ICR_HIGH, icr.high());
-    write_register(APIC_REG_ICR_LOW, icr.low());
+    if (m_is_x2) {
+        MSR msr(APIC_REGS_MSR_BASE + (APIC_REG_ICR_LOW >> 4));
+        msr.set(icr.x2_value());
+    } else {
+        write_register(APIC_REG_ICR_HIGH, icr.x_high());
+        write_register(APIC_REG_ICR_LOW, icr.x_low());
+    }
 }
 
 #define APIC_LVT_TIMER_ONESHOT 0
@@ -224,15 +243,19 @@ UNMAP_AFTER_INIT bool APIC::init_bsp()
     CPUID id(1);
     if ((id.edx() & (1 << 9)) == 0)
         return false;
+    if (id.ecx() & (1 << 21))
+        m_is_x2 = true;
 
     PhysicalAddress apic_base = get_base();
-    dbgln_if(APIC_DEBUG, "Initializing APIC, base: {}", apic_base);
+    dbgln_if(APIC_DEBUG, "Initializing {}APIC, base: {}", m_is_x2 ? "x2" : "x", apic_base);
     set_base(apic_base);
 
-    m_apic_base = MM.allocate_kernel_region(apic_base.page_base(), PAGE_SIZE, {}, Memory::Region::Access::ReadWrite);
-    if (!m_apic_base) {
-        dbgln("APIC: Failed to allocate memory for APIC base");
-        return false;
+    if (!m_is_x2) {
+        m_apic_base = MM.allocate_kernel_region(apic_base.page_base(), PAGE_SIZE, {}, Memory::Region::Access::ReadWrite);
+        if (!m_apic_base) {
+            dbgln("APIC: Failed to allocate memory for APIC base");
+            return false;
+        }
     }
 
     auto rsdp = ACPI::StaticParsing::find_rsdp();
@@ -254,10 +277,17 @@ UNMAP_AFTER_INIT bool APIC::init_bsp()
         size_t entry_length = madt_entry->length;
         if (madt_entry->type == (u8)ACPI::Structures::MADTEntryType::LocalAPIC) {
             auto* plapic_entry = (const ACPI::Structures::MADTEntries::ProcessorLocalAPIC*)madt_entry;
-            dbgln_if(APIC_DEBUG, "APIC: AP found @ MADT entry {}, processor ID: {}, APIC ID: {}, flags: {:#08x}", entry_index, plapic_entry->acpi_processor_id, plapic_entry->apic_id, plapic_entry->flags);
+            dbgln_if(APIC_DEBUG, "APIC: AP found @ MADT entry {}, processor ID: {}, xAPIC ID: {}, flags: {:#08x}", entry_index, plapic_entry->acpi_processor_id, plapic_entry->apic_id, plapic_entry->flags);
             m_processor_cnt++;
             if ((plapic_entry->flags & 0x1) != 0)
                 m_processor_enabled_cnt++;
+        } else if (madt_entry->type == (u8)ACPI::Structures::MADTEntryType::Local_x2APIC) {
+            // Only used for APID IDs >= 255
+            auto* plx2apic_entry = (const ACPI::Structures::MADTEntries::ProcessorLocalX2APIC*)madt_entry;
+            dbgln_if(APIC_DEBUG, "APIC: AP found @ MADT entry {}, processor ID: {}, x2APIC ID: {}, flags: {:#08x}", entry_index, plx2apic_entry->acpi_processor_id, plx2apic_entry->apic_id, plx2apic_entry->flags);
+            m_processor_cnt++;
+            if ((plx2apic_entry->flags & 0x1) != 0)
+                m_processor_enabled_cnt++;
         }
         madt_entry = (ACPI::Structures::MADTEntryHeader*)(VirtualAddress(madt_entry).offset(entry_length).get());
         entries_length -= entry_length;
@@ -357,13 +387,13 @@ UNMAP_AFTER_INIT void APIC::do_boot_aps()
     dbgln_if(APIC_DEBUG, "APIC: Starting {} AP(s)", aps_to_enable);
 
     // INIT
-    write_icr(ICRReg(0, ICRReg::INIT, ICRReg::Physical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf));
+    write_icr({ 0, 0, ICRReg::INIT, ICRReg::Physical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf });
 
     IO::delay(10 * 1000);
 
     for (int i = 0; i < 2; i++) {
         // SIPI
-        write_icr(ICRReg(0x08, ICRReg::StartUp, ICRReg::Physical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf)); // start execution at P8000
+        write_icr({ 0x08, 0, ICRReg::StartUp, ICRReg::Physical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf }); // start execution at P8000
 
         IO::delay(200);
     }
@@ -405,17 +435,28 @@ UNMAP_AFTER_INIT void APIC::boot_aps()
 
 UNMAP_AFTER_INIT void APIC::enable(u32 cpu)
 {
-    if (cpu >= 8) {
-        // TODO: x2apic support?
-        PANIC("SMP support is currently limited to 8 CPUs!");
-    }
+    VERIFY(m_is_x2 || cpu < 8);
+
+    u32 apic_id;
+    if (m_is_x2) {
+        dbgln_if(APIC_DEBUG, "Enable x2APIC on CPU #{}", cpu);
 
-    // Use the CPU# as logical apic id
-    VERIFY(cpu <= 0xff);
-    write_register(APIC_REG_LD, (read_register(APIC_REG_LD) & 0x00ffffff) | (cpu << 24)); // TODO: only if not in x2apic mode
+        // We need to enable x2 mode on each core independently
+        set_base(get_base());
 
-    // read it back to make sure it's actually set
-    auto apic_id = read_register(APIC_REG_LD) >> 24;
+        apic_id = read_register(APIC_REG_ID);
+    } else {
+        dbgln_if(APIC_DEBUG, "Setting logical xAPIC ID for CPU #{}", cpu);
+
+        // Use the CPU# as logical apic id
+        VERIFY(cpu <= 8);
+        write_register(APIC_REG_LD, (read_register(APIC_REG_LD) & 0x00ffffff) | (cpu << 24));
+
+        // read it back to make sure it's actually set
+        apic_id = read_register(APIC_REG_LD) >> 24;
+    }
+
+    dbgln_if(APIC_DEBUG, "CPU #{} apic id: {}", cpu, apic_id);
     Processor::current().info().set_apic_id(apic_id);
 
     dbgln_if(APIC_DEBUG, "Enabling local APIC for CPU #{}, logical APIC ID: {}", cpu, apic_id);
@@ -423,20 +464,23 @@ UNMAP_AFTER_INIT void APIC::enable(u32 cpu)
     if (cpu == 0) {
         SpuriousInterruptHandler::initialize(IRQ_APIC_SPURIOUS);
 
-        // set error interrupt vector
-        set_lvt(APIC_REG_LVT_ERR, IRQ_APIC_ERR);
         APICErrInterruptHandler::initialize(IRQ_APIC_ERR);
 
         // register IPI interrupt vector
         APICIPIInterruptHandler::initialize(IRQ_APIC_IPI);
     }
 
+    if (!m_is_x2) {
+        // local destination mode (flat mode), not supported in x2 mode
+        write_register(APIC_REG_DF, 0xf0000000);
+    }
+
+    // set error interrupt vector
+    set_lvt(APIC_REG_LVT_ERR, IRQ_APIC_ERR);
+
     // set spurious interrupt vector
     set_siv(APIC_REG_SIV, IRQ_APIC_SPURIOUS);
 
-    // local destination mode (flat mode)
-    write_register(APIC_REG_DF, 0xf0000000);
-
     write_register(APIC_REG_LVT_TIMER, APIC_LVT(0, 0) | APIC_LVT_MASKED);
     write_register(APIC_REG_LVT_THERMAL, APIC_LVT(0, 0) | APIC_LVT_MASKED);
     write_register(APIC_REG_LVT_PERFORMANCE_COUNTER, APIC_LVT(0, 0) | APIC_LVT_MASKED);
@@ -485,21 +529,21 @@ void APIC::broadcast_ipi()
 {
     dbgln_if(APIC_SMP_DEBUG, "SMP: Broadcast IPI from CPU #{}", Processor::current_id());
     wait_for_pending_icr();
-    write_icr(ICRReg(IRQ_APIC_IPI + IRQ_VECTOR_BASE, ICRReg::Fixed, ICRReg::Logical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf));
+    write_icr({ IRQ_APIC_IPI + IRQ_VECTOR_BASE, 0xffffffff, ICRReg::Fixed, ICRReg::Logical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf });
 }
 
 void APIC::send_ipi(u32 cpu)
 {
     dbgln_if(APIC_SMP_DEBUG, "SMP: Send IPI from CPU #{} to CPU #{}", Processor::current_id(), cpu);
     VERIFY(cpu != Processor::current_id());
-    VERIFY(cpu < 8);
+    VERIFY(cpu < Processor::count());
     wait_for_pending_icr();
-    write_icr(ICRReg(IRQ_APIC_IPI + IRQ_VECTOR_BASE, ICRReg::Fixed, ICRReg::Logical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::NoShorthand, cpu));
+    write_icr({ IRQ_APIC_IPI + IRQ_VECTOR_BASE, m_is_x2 ? Processor::by_id(cpu).info().apic_id() : cpu, ICRReg::Fixed, m_is_x2 ? ICRReg::Physical : ICRReg::Logical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::NoShorthand });
 }
 
 UNMAP_AFTER_INIT APICTimer* APIC::initialize_timers(HardwareTimerBase& calibration_timer)
 {
-    if (!m_apic_base)
+    if (!m_apic_base && !m_is_x2)
         return nullptr;
 
     // We should only initialize and calibrate the APIC timer once on the BSP!

Kernel/Interrupts/APIC.h

@@ -47,11 +47,7 @@ public:
     u32 get_timer_divisor();
 
 private:
-    class ICRReg {
-        u32 m_low { 0 };
-        u32 m_high { 0 };
-
-    public:
+    struct ICRReg {
         enum DeliveryMode {
             Fixed = 0x0,
             LowPriority = 0x1,
@@ -79,14 +75,17 @@ private:
             AllExcludingSelf = 0x3,
         };
 
-        ICRReg(u8 vector, DeliveryMode delivery_mode, DestinationMode destination_mode, Level level, TriggerMode trigger_mode, DestinationShorthand destinationShort, u8 destination = 0)
-            : m_low(vector | (delivery_mode << 8) | (destination_mode << 11) | (level << 14) | (static_cast<u32>(trigger_mode) << 15) | (destinationShort << 18))
-            , m_high((u32)destination << 24)
-        {
-        }
+        u8 vector { 0 };
+        u32 destination { 0 };
+        DeliveryMode delivery_mode { DeliveryMode::Fixed };
+        DestinationMode destination_mode { DestinationMode::Physical };
+        Level level { Level::DeAssert };
+        TriggerMode trigger_mode { TriggerMode::Edge };
+        DestinationShorthand destination_short { DestinationShorthand::NoShorthand };
 
-        u32 low() const { return m_low; }
-        u32 high() const { return m_high; }
+        u32 x_low() const { return (u32)vector | (delivery_mode << 8) | (destination_mode << 11) | (level << 14) | (static_cast<u32>(trigger_mode) << 15) | (destination_short << 18); }
+        u32 x_high() const { return destination << 24; }
+        u64 x2_value() const { return ((u64)destination << 32) | x_low(); }
     };
 
     OwnPtr<Memory::Region> m_apic_base;
@@ -97,9 +96,10 @@ private:
     u32 m_processor_cnt { 0 };
     u32 m_processor_enabled_cnt { 0 };
     APICTimer* m_apic_timer { nullptr };
+    bool m_is_x2 { false };
 
     static PhysicalAddress get_base();
-    static void set_base(const PhysicalAddress& base);
+    void set_base(const PhysicalAddress& base);
     void write_register(u32 offset, u32 value);
     u32 read_register(u32 offset);
     void set_lvt(u32 offset, u8 interrupt);