Processor.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452
  1. /*
  2. * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #pragma once
  7. #include <AK/Array.h>
  8. #include <AK/Concepts.h>
  9. #include <AK/Function.h>
  10. #include <AK/Types.h>
  11. #include <Kernel/Arch/x86/ASM_wrapper.h>
  12. #include <Kernel/Arch/x86/CPUID.h>
  13. #include <Kernel/Arch/x86/DescriptorTable.h>
  14. #include <Kernel/Arch/x86/PageDirectory.h>
  15. #include <Kernel/Arch/x86/TSS.h>
  16. #include <Kernel/Forward.h>
  17. namespace Kernel {
  18. class ProcessorInfo;
  19. struct ProcessorMessageEntry;
  20. enum class ProcessorSpecificDataID {
  21. MemoryManager,
  22. __Count,
  23. };
  24. #if ARCH(X86_64)
  25. # define MSR_FS_BASE 0xc0000100
  26. # define MSR_GS_BASE 0xc0000101
  27. #endif
  28. // FIXME: Find a better place for these
  29. extern "C" void thread_context_first_enter(void);
  30. extern "C" void exit_kernel_thread(void);
  31. extern "C" void do_assume_context(Thread* thread, u32 flags);
  32. struct [[gnu::aligned(16)]] FPUState
  33. {
  34. u8 buffer[512];
  35. };
  36. struct ProcessorMessage {
  37. using CallbackFunction = Function<void()>;
  38. enum Type {
  39. FlushTlb,
  40. Callback,
  41. };
  42. Type type;
  43. Atomic<u32> refs;
  44. union {
  45. ProcessorMessage* next; // only valid while in the pool
  46. alignas(CallbackFunction) u8 callback_storage[sizeof(CallbackFunction)];
  47. struct {
  48. Memory::PageDirectory const* page_directory;
  49. u8* ptr;
  50. size_t page_count;
  51. } flush_tlb;
  52. };
  53. volatile bool async;
  54. ProcessorMessageEntry* per_proc_entries;
  55. CallbackFunction& callback_value()
  56. {
  57. return *bit_cast<CallbackFunction*>(&callback_storage);
  58. }
  59. void invoke_callback()
  60. {
  61. VERIFY(type == Type::Callback);
  62. callback_value()();
  63. }
  64. };
  65. struct ProcessorMessageEntry {
  66. ProcessorMessageEntry* next;
  67. ProcessorMessage* msg;
  68. };
  69. struct DeferredCallEntry {
  70. using HandlerFunction = Function<void()>;
  71. DeferredCallEntry* next;
  72. alignas(HandlerFunction) u8 handler_storage[sizeof(HandlerFunction)];
  73. bool was_allocated;
  74. HandlerFunction& handler_value()
  75. {
  76. return *bit_cast<HandlerFunction*>(&handler_storage);
  77. }
  78. void invoke_handler()
  79. {
  80. handler_value()();
  81. }
  82. };
  83. class Processor;
  84. // Note: We only support 64 processors at most at the moment,
  85. // so allocate 64 slots of inline capacity in the container.
  86. using ProcessorContainer = Array<Processor*, 64>;
  87. class Processor {
  88. friend class ProcessorInfo;
  89. AK_MAKE_NONCOPYABLE(Processor);
  90. AK_MAKE_NONMOVABLE(Processor);
  91. Processor* m_self;
  92. DescriptorTablePointer m_gdtr;
  93. Descriptor m_gdt[256];
  94. u32 m_gdt_length;
  95. u32 m_cpu;
  96. FlatPtr m_in_irq;
  97. volatile u32 m_in_critical;
  98. static Atomic<u32> s_idle_cpu_mask;
  99. TSS m_tss;
  100. static FPUState s_clean_fpu_state;
  101. CPUFeature m_features;
  102. static Atomic<u32> g_total_processors;
  103. u8 m_physical_address_bit_width;
  104. ProcessorInfo* m_info;
  105. Thread* m_current_thread;
  106. Thread* m_idle_thread;
  107. Atomic<ProcessorMessageEntry*> m_message_queue;
  108. bool m_invoke_scheduler_async;
  109. bool m_scheduler_initialized;
  110. bool m_in_scheduler;
  111. Atomic<bool> m_halt_requested;
  112. DeferredCallEntry* m_pending_deferred_calls; // in reverse order
  113. DeferredCallEntry* m_free_deferred_call_pool_entry;
  114. DeferredCallEntry m_deferred_call_pool[5];
  115. void* m_processor_specific_data[(size_t)ProcessorSpecificDataID::__Count];
  116. void gdt_init();
  117. void write_raw_gdt_entry(u16 selector, u32 low, u32 high);
  118. void write_gdt_entry(u16 selector, Descriptor& descriptor);
  119. static ProcessorContainer& processors();
  120. static void smp_return_to_pool(ProcessorMessage& msg);
  121. static ProcessorMessage& smp_get_from_pool();
  122. static void smp_cleanup_message(ProcessorMessage& msg);
  123. bool smp_enqueue_message(ProcessorMessage&);
  124. static void smp_unicast_message(u32 cpu, ProcessorMessage& msg, bool async);
  125. static void smp_broadcast_message(ProcessorMessage& msg);
  126. static void smp_broadcast_wait_sync(ProcessorMessage& msg);
  127. static void smp_broadcast_halt();
  128. void deferred_call_pool_init();
  129. void deferred_call_execute_pending();
  130. DeferredCallEntry* deferred_call_get_free();
  131. void deferred_call_return_to_pool(DeferredCallEntry*);
  132. void deferred_call_queue_entry(DeferredCallEntry*);
  133. void cpu_detect();
  134. void cpu_setup();
  135. String features_string() const;
  136. public:
  137. Processor() = default;
  138. void early_initialize(u32 cpu);
  139. void initialize(u32 cpu);
  140. void detect_hypervisor();
  141. void detect_hypervisor_hyperv(CPUID const& hypervisor_leaf_range);
  142. void idle_begin()
  143. {
  144. s_idle_cpu_mask.fetch_or(1u << m_cpu, AK::MemoryOrder::memory_order_relaxed);
  145. }
  146. void idle_end()
  147. {
  148. s_idle_cpu_mask.fetch_and(~(1u << m_cpu), AK::MemoryOrder::memory_order_relaxed);
  149. }
  150. static Processor& by_id(u32);
  151. static u32 count()
  152. {
  153. // NOTE: because this value never changes once all APs are booted,
  154. // we can safely bypass loading it atomically.
  155. return *g_total_processors.ptr();
  156. }
  157. ALWAYS_INLINE static void pause()
  158. {
  159. asm volatile("pause");
  160. }
  161. ALWAYS_INLINE static void wait_check()
  162. {
  163. Processor::pause();
  164. if (Processor::is_smp_enabled())
  165. Processor::current().smp_process_pending_messages();
  166. }
  167. [[noreturn]] static void halt();
  168. static void flush_entire_tlb_local()
  169. {
  170. write_cr3(read_cr3());
  171. }
  172. static void flush_tlb_local(VirtualAddress vaddr, size_t page_count);
  173. static void flush_tlb(Memory::PageDirectory const*, VirtualAddress, size_t);
  174. Descriptor& get_gdt_entry(u16 selector);
  175. void flush_gdt();
  176. const DescriptorTablePointer& get_gdtr();
  177. static size_t processor_count() { return processors().size(); }
  178. template<IteratorFunction<Processor&> Callback>
  179. static inline IterationDecision for_each(Callback callback)
  180. {
  181. auto& procs = processors();
  182. size_t count = procs.size();
  183. for (size_t i = 0; i < count; i++) {
  184. if (callback(*procs[i]) == IterationDecision::Break)
  185. return IterationDecision::Break;
  186. }
  187. return IterationDecision::Continue;
  188. }
  189. template<VoidFunction<Processor&> Callback>
  190. static inline IterationDecision for_each(Callback callback)
  191. {
  192. auto& procs = processors();
  193. size_t count = procs.size();
  194. for (size_t i = 0; i < count; i++) {
  195. if (procs[i] != nullptr)
  196. callback(*procs[i]);
  197. }
  198. return IterationDecision::Continue;
  199. }
  200. ALWAYS_INLINE u8 physical_address_bit_width() const { return m_physical_address_bit_width; }
  201. ALWAYS_INLINE ProcessorInfo& info() { return *m_info; }
  202. static bool is_smp_enabled();
  203. ALWAYS_INLINE static Processor& current()
  204. {
  205. return *(Processor*)read_gs_ptr(__builtin_offsetof(Processor, m_self));
  206. }
  207. ALWAYS_INLINE static bool is_initialized()
  208. {
  209. return
  210. #if ARCH(I386)
  211. get_gs() == GDT_SELECTOR_PROC &&
  212. #endif
  213. read_gs_ptr(__builtin_offsetof(Processor, m_self)) != 0;
  214. }
  215. template<typename T>
  216. T* get_specific()
  217. {
  218. return static_cast<T*>(m_processor_specific_data[static_cast<size_t>(T::processor_specific_data_id())]);
  219. }
  220. void set_specific(ProcessorSpecificDataID specific_id, void* ptr)
  221. {
  222. m_processor_specific_data[static_cast<size_t>(specific_id)] = ptr;
  223. }
  224. ALWAYS_INLINE void set_idle_thread(Thread& idle_thread)
  225. {
  226. m_idle_thread = &idle_thread;
  227. }
  228. ALWAYS_INLINE static Thread* current_thread()
  229. {
  230. // If we were to use Processor::current here, we'd have to
  231. // disable interrupts to prevent a race where we may get pre-empted
  232. // right after getting the Processor structure and then get moved
  233. // to another processor, which would lead us to get the wrong thread.
  234. // To avoid having to disable interrupts, we can just read the field
  235. // directly in an atomic fashion, similar to Processor::current.
  236. return (Thread*)read_gs_ptr(__builtin_offsetof(Processor, m_current_thread));
  237. }
  238. ALWAYS_INLINE static void set_current_thread(Thread& current_thread)
  239. {
  240. // See comment in Processor::current_thread
  241. write_gs_ptr(__builtin_offsetof(Processor, m_current_thread), FlatPtr(&current_thread));
  242. }
  243. ALWAYS_INLINE static Thread* idle_thread()
  244. {
  245. // See comment in Processor::current_thread
  246. return (Thread*)read_gs_ptr(__builtin_offsetof(Processor, m_idle_thread));
  247. }
  248. ALWAYS_INLINE u32 id() const
  249. {
  250. // NOTE: This variant should only be used when iterating over all
  251. // Processor instances, or when it's guaranteed that the thread
  252. // cannot move to another processor in between calling Processor::current
  253. // and Processor::get_id, or if this fact is not important.
  254. // All other cases should use Processor::id instead!
  255. return m_cpu;
  256. }
  257. ALWAYS_INLINE static u32 current_id()
  258. {
  259. // See comment in Processor::current_thread
  260. return read_gs_ptr(__builtin_offsetof(Processor, m_cpu));
  261. }
  262. ALWAYS_INLINE static bool is_bootstrap_processor()
  263. {
  264. return Processor::current_id() == 0;
  265. }
  266. ALWAYS_INLINE static FlatPtr current_in_irq()
  267. {
  268. return read_gs_ptr(__builtin_offsetof(Processor, m_in_irq));
  269. }
  270. ALWAYS_INLINE static void restore_in_critical(u32 critical)
  271. {
  272. write_gs_ptr(__builtin_offsetof(Processor, m_in_critical), critical);
  273. }
  274. ALWAYS_INLINE static void enter_critical()
  275. {
  276. write_gs_ptr(__builtin_offsetof(Processor, m_in_critical), in_critical() + 1);
  277. }
  278. ALWAYS_INLINE static bool current_in_scheduler()
  279. {
  280. return read_gs_value<decltype(m_in_scheduler)>(__builtin_offsetof(Processor, m_in_scheduler));
  281. }
  282. ALWAYS_INLINE static void set_current_in_scheduler(bool value)
  283. {
  284. write_gs_value<decltype(m_in_scheduler)>(__builtin_offsetof(Processor, m_in_scheduler), value);
  285. }
  286. private:
  287. ALWAYS_INLINE void do_leave_critical()
  288. {
  289. VERIFY(m_in_critical > 0);
  290. if (m_in_critical == 1) {
  291. if (!m_in_irq) {
  292. deferred_call_execute_pending();
  293. VERIFY(m_in_critical == 1);
  294. }
  295. m_in_critical = 0;
  296. if (!m_in_irq)
  297. check_invoke_scheduler();
  298. } else {
  299. m_in_critical = m_in_critical - 1;
  300. }
  301. }
  302. public:
  303. ALWAYS_INLINE static void leave_critical()
  304. {
  305. current().do_leave_critical();
  306. }
  307. ALWAYS_INLINE static u32 clear_critical()
  308. {
  309. auto prev_critical = in_critical();
  310. write_gs_ptr(__builtin_offsetof(Processor, m_in_critical), 0);
  311. auto& proc = current();
  312. if (!proc.m_in_irq)
  313. proc.check_invoke_scheduler();
  314. return prev_critical;
  315. }
  316. ALWAYS_INLINE static void restore_critical(u32 prev_critical)
  317. {
  318. // NOTE: This doesn't have to be atomic, and it's also fine if we
  319. // get preempted in between these steps. If we move to another
  320. // processors m_in_critical will move along with us. And if we
  321. // are preempted, we would resume with the same flags.
  322. write_gs_ptr(__builtin_offsetof(Processor, m_in_critical), prev_critical);
  323. }
  324. ALWAYS_INLINE static u32 in_critical()
  325. {
  326. // See comment in Processor::current_thread
  327. return read_gs_ptr(__builtin_offsetof(Processor, m_in_critical));
  328. }
  329. ALWAYS_INLINE static FPUState const& clean_fpu_state() { return s_clean_fpu_state; }
  330. static void smp_enable();
  331. bool smp_process_pending_messages();
  332. static void smp_unicast(u32 cpu, Function<void()>, bool async);
  333. static void smp_broadcast_flush_tlb(Memory::PageDirectory const*, VirtualAddress, size_t);
  334. static u32 smp_wake_n_idle_processors(u32 wake_count);
  335. static void deferred_call_queue(Function<void()> callback);
  336. ALWAYS_INLINE bool has_feature(CPUFeature f) const
  337. {
  338. return (static_cast<u32>(m_features) & static_cast<u32>(f)) != 0;
  339. }
  340. void check_invoke_scheduler();
  341. void invoke_scheduler_async() { m_invoke_scheduler_async = true; }
  342. void enter_trap(TrapFrame& trap, bool raise_irq);
  343. void exit_trap(TrapFrame& trap);
  344. [[noreturn]] void initialize_context_switching(Thread& initial_thread);
  345. NEVER_INLINE void switch_context(Thread*& from_thread, Thread*& to_thread);
  346. [[noreturn]] static void assume_context(Thread& thread, FlatPtr flags);
  347. FlatPtr init_context(Thread& thread, bool leave_crit);
  348. static Vector<FlatPtr> capture_stack_trace(Thread& thread, size_t max_frames = 0);
  349. static StringView platform_string();
  350. };
  351. template<typename T>
  352. class ProcessorSpecific {
  353. public:
  354. static void initialize()
  355. {
  356. Processor::current().set_specific(T::processor_specific_data_id(), new T);
  357. }
  358. static T& get()
  359. {
  360. return *Processor::current().get_specific<T>();
  361. }
  362. };
  363. }