Processor.cpp 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346
  1. /*
  2. * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include <AK/BuiltinWrappers.h>
  7. #include <AK/Format.h>
  8. #include <AK/StdLibExtras.h>
  9. #include <AK/String.h>
  10. #include <AK/Types.h>
  11. #include <Kernel/Interrupts/APIC.h>
  12. #include <Kernel/Process.h>
  13. #include <Kernel/Sections.h>
  14. #include <Kernel/StdLib.h>
  15. #include <Kernel/Thread.h>
  16. #include <Kernel/Arch/Processor.h>
  17. #include <Kernel/Arch/ScopedCritical.h>
  18. #include <Kernel/Arch/x86/CPUID.h>
  19. #include <Kernel/Arch/x86/InterruptDisabler.h>
  20. #include <Kernel/Arch/x86/Interrupts.h>
  21. #include <Kernel/Arch/x86/MSR.h>
  22. #include <Kernel/Arch/x86/ProcessorInfo.h>
  23. #include <Kernel/Arch/x86/SafeMem.h>
  24. #include <Kernel/Arch/x86/TrapFrame.h>
  25. #include <Kernel/Memory/PageDirectory.h>
  26. #include <Kernel/Memory/ScopedAddressSpaceSwitcher.h>
  27. namespace Kernel {
  28. READONLY_AFTER_INIT FPUState Processor::s_clean_fpu_state;
  29. READONLY_AFTER_INIT static ProcessorContainer s_processors {};
  30. READONLY_AFTER_INIT Atomic<u32> Processor::g_total_processors;
  31. READONLY_AFTER_INIT static volatile bool s_smp_enabled;
  32. static Atomic<ProcessorMessage*> s_message_pool;
  33. Atomic<u32> Processor::s_idle_cpu_mask { 0 };
  34. // The compiler can't see the calls to these functions inside assembly.
  35. // Declare them, to avoid dead code warnings.
  36. extern "C" void context_first_init(Thread* from_thread, Thread* to_thread, TrapFrame* trap) __attribute__((used));
  37. extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __attribute__((used));
  38. extern "C" FlatPtr do_init_context(Thread* thread, u32 flags) __attribute__((used));
  39. bool Processor::is_smp_enabled()
  40. {
  41. return s_smp_enabled;
  42. }
  43. UNMAP_AFTER_INIT static void sse_init()
  44. {
  45. write_cr0((read_cr0() & 0xfffffffbu) | 0x2);
  46. write_cr4(read_cr4() | 0x600);
  47. }
  48. void exit_kernel_thread(void)
  49. {
  50. Thread::current()->exit();
  51. }
  52. UNMAP_AFTER_INIT void Processor::cpu_detect()
  53. {
  54. // NOTE: This is called during Processor::early_initialize, we cannot
  55. // safely log at this point because we don't have kmalloc
  56. // initialized yet!
  57. auto set_feature =
  58. [&](CPUFeature f) {
  59. m_features = static_cast<CPUFeature>(static_cast<u32>(m_features) | static_cast<u32>(f));
  60. };
  61. m_features = static_cast<CPUFeature>(0);
  62. CPUID processor_info(0x1);
  63. if (processor_info.edx() & (1 << 4))
  64. set_feature(CPUFeature::TSC);
  65. if (processor_info.edx() & (1 << 6))
  66. set_feature(CPUFeature::PAE);
  67. if (processor_info.edx() & (1 << 13))
  68. set_feature(CPUFeature::PGE);
  69. if (processor_info.edx() & (1 << 23))
  70. set_feature(CPUFeature::MMX);
  71. if (processor_info.edx() & (1 << 24))
  72. set_feature(CPUFeature::FXSR);
  73. if (processor_info.edx() & (1 << 25))
  74. set_feature(CPUFeature::SSE);
  75. if (processor_info.edx() & (1 << 26))
  76. set_feature(CPUFeature::SSE2);
  77. if (processor_info.ecx() & (1 << 0))
  78. set_feature(CPUFeature::SSE3);
  79. if (processor_info.ecx() & (1 << 9))
  80. set_feature(CPUFeature::SSSE3);
  81. if (processor_info.ecx() & (1 << 19))
  82. set_feature(CPUFeature::SSE4_1);
  83. if (processor_info.ecx() & (1 << 20))
  84. set_feature(CPUFeature::SSE4_2);
  85. if (processor_info.ecx() & (1 << 26))
  86. set_feature(CPUFeature::XSAVE);
  87. if (processor_info.ecx() & (1 << 28))
  88. set_feature(CPUFeature::AVX);
  89. if (processor_info.ecx() & (1 << 30))
  90. set_feature(CPUFeature::RDRAND);
  91. if (processor_info.ecx() & (1u << 31))
  92. set_feature(CPUFeature::HYPERVISOR);
  93. if (processor_info.edx() & (1 << 11)) {
  94. u32 stepping = processor_info.eax() & 0xf;
  95. u32 model = (processor_info.eax() >> 4) & 0xf;
  96. u32 family = (processor_info.eax() >> 8) & 0xf;
  97. if (!(family == 6 && model < 3 && stepping < 3))
  98. set_feature(CPUFeature::SEP);
  99. if ((family == 6 && model >= 3) || (family == 0xf && model >= 0xe))
  100. set_feature(CPUFeature::CONSTANT_TSC);
  101. }
  102. u32 max_extended_leaf = CPUID(0x80000000).eax();
  103. if (max_extended_leaf >= 0x80000001) {
  104. CPUID extended_processor_info(0x80000001);
  105. if (extended_processor_info.edx() & (1 << 20))
  106. set_feature(CPUFeature::NX);
  107. if (extended_processor_info.edx() & (1 << 27))
  108. set_feature(CPUFeature::RDTSCP);
  109. if (extended_processor_info.edx() & (1 << 29))
  110. set_feature(CPUFeature::LM);
  111. if (extended_processor_info.edx() & (1 << 11)) {
  112. // Only available in 64 bit mode
  113. set_feature(CPUFeature::SYSCALL);
  114. }
  115. }
  116. if (max_extended_leaf >= 0x80000007) {
  117. CPUID cpuid(0x80000007);
  118. if (cpuid.edx() & (1 << 8)) {
  119. set_feature(CPUFeature::CONSTANT_TSC);
  120. set_feature(CPUFeature::NONSTOP_TSC);
  121. }
  122. }
  123. if (max_extended_leaf >= 0x80000008) {
  124. // CPUID.80000008H:EAX[7:0] reports the physical-address width supported by the processor.
  125. CPUID cpuid(0x80000008);
  126. m_physical_address_bit_width = cpuid.eax() & 0xff;
  127. // CPUID.80000008H:EAX[15:8] reports the linear-address width supported by the processor.
  128. m_virtual_address_bit_width = (cpuid.eax() >> 8) & 0xff;
  129. } else {
  130. // For processors that do not support CPUID function 80000008H, the width is generally 36 if CPUID.01H:EDX.PAE [bit 6] = 1 and 32 otherwise.
  131. m_physical_address_bit_width = has_feature(CPUFeature::PAE) ? 36 : 32;
  132. // Processors that do not support CPUID function 80000008H, support a linear-address width of 32.
  133. m_virtual_address_bit_width = 32;
  134. }
  135. CPUID extended_features(0x7);
  136. if (extended_features.ebx() & (1 << 20))
  137. set_feature(CPUFeature::SMAP);
  138. if (extended_features.ebx() & (1 << 7))
  139. set_feature(CPUFeature::SMEP);
  140. if (extended_features.ecx() & (1 << 2))
  141. set_feature(CPUFeature::UMIP);
  142. if (extended_features.ebx() & (1 << 18))
  143. set_feature(CPUFeature::RDSEED);
  144. }
  145. UNMAP_AFTER_INIT void Processor::cpu_setup()
  146. {
  147. // NOTE: This is called during Processor::early_initialize, we cannot
  148. // safely log at this point because we don't have kmalloc
  149. // initialized yet!
  150. cpu_detect();
  151. if (has_feature(CPUFeature::SSE)) {
  152. // enter_thread_context() assumes that if a x86 CPU supports SSE then it also supports FXSR.
  153. // SSE support without FXSR is an extremely unlikely scenario, so let's be pragmatic about it.
  154. VERIFY(has_feature(CPUFeature::FXSR));
  155. sse_init();
  156. }
  157. write_cr0(read_cr0() | 0x00010000);
  158. if (has_feature(CPUFeature::PGE)) {
  159. // Turn on CR4.PGE so the CPU will respect the G bit in page tables.
  160. write_cr4(read_cr4() | 0x80);
  161. }
  162. if (has_feature(CPUFeature::NX)) {
  163. // Turn on IA32_EFER.NXE
  164. MSR ia32_efer(MSR_IA32_EFER);
  165. ia32_efer.set(ia32_efer.get() | 0x800);
  166. }
  167. if (has_feature(CPUFeature::SMEP)) {
  168. // Turn on CR4.SMEP
  169. write_cr4(read_cr4() | 0x100000);
  170. }
  171. if (has_feature(CPUFeature::SMAP)) {
  172. // Turn on CR4.SMAP
  173. write_cr4(read_cr4() | 0x200000);
  174. }
  175. if (has_feature(CPUFeature::UMIP)) {
  176. write_cr4(read_cr4() | 0x800);
  177. }
  178. if (has_feature(CPUFeature::TSC)) {
  179. write_cr4(read_cr4() | 0x4);
  180. }
  181. if (has_feature(CPUFeature::XSAVE)) {
  182. // Turn on CR4.OSXSAVE
  183. write_cr4(read_cr4() | 0x40000);
  184. // According to the Intel manual: "After reset, all bits (except bit 0) in XCR0 are cleared to zero; XCR0[0] is set to 1."
  185. // Sadly we can't trust this, for example VirtualBox starts with bits 0-4 set, so let's do it ourselves.
  186. write_xcr0(0x1);
  187. if (has_feature(CPUFeature::AVX)) {
  188. // Turn on SSE, AVX and x87 flags
  189. write_xcr0(read_xcr0() | 0x7);
  190. }
  191. }
  192. }
  193. String Processor::features_string() const
  194. {
  195. StringBuilder builder;
  196. auto feature_to_str =
  197. [](CPUFeature f) -> const char* {
  198. switch (f) {
  199. case CPUFeature::NX:
  200. return "nx";
  201. case CPUFeature::PAE:
  202. return "pae";
  203. case CPUFeature::PGE:
  204. return "pge";
  205. case CPUFeature::RDRAND:
  206. return "rdrand";
  207. case CPUFeature::RDSEED:
  208. return "rdseed";
  209. case CPUFeature::SMAP:
  210. return "smap";
  211. case CPUFeature::SMEP:
  212. return "smep";
  213. case CPUFeature::SSE:
  214. return "sse";
  215. case CPUFeature::TSC:
  216. return "tsc";
  217. case CPUFeature::RDTSCP:
  218. return "rdtscp";
  219. case CPUFeature::CONSTANT_TSC:
  220. return "constant_tsc";
  221. case CPUFeature::NONSTOP_TSC:
  222. return "nonstop_tsc";
  223. case CPUFeature::UMIP:
  224. return "umip";
  225. case CPUFeature::SEP:
  226. return "sep";
  227. case CPUFeature::SYSCALL:
  228. return "syscall";
  229. case CPUFeature::MMX:
  230. return "mmx";
  231. case CPUFeature::FXSR:
  232. return "fxsr";
  233. case CPUFeature::SSE2:
  234. return "sse2";
  235. case CPUFeature::SSE3:
  236. return "sse3";
  237. case CPUFeature::SSSE3:
  238. return "ssse3";
  239. case CPUFeature::SSE4_1:
  240. return "sse4.1";
  241. case CPUFeature::SSE4_2:
  242. return "sse4.2";
  243. case CPUFeature::XSAVE:
  244. return "xsave";
  245. case CPUFeature::AVX:
  246. return "avx";
  247. case CPUFeature::LM:
  248. return "lm";
  249. case CPUFeature::HYPERVISOR:
  250. return "hypervisor";
  251. // no default statement here intentionally so that we get
  252. // a warning if a new feature is forgotten to be added here
  253. }
  254. // Shouldn't ever happen
  255. return "???";
  256. };
  257. bool first = true;
  258. for (u32 flag = 1; flag != 0; flag <<= 1) {
  259. if ((static_cast<u32>(m_features) & flag) != 0) {
  260. if (first)
  261. first = false;
  262. else
  263. builder.append(' ');
  264. auto str = feature_to_str(static_cast<CPUFeature>(flag));
  265. builder.append(str, strlen(str));
  266. }
  267. }
  268. return builder.build();
  269. }
  270. UNMAP_AFTER_INIT void Processor::early_initialize(u32 cpu)
  271. {
  272. m_self = this;
  273. m_cpu = cpu;
  274. m_in_irq = 0;
  275. m_in_critical = 0;
  276. m_invoke_scheduler_async = false;
  277. m_scheduler_initialized = false;
  278. m_in_scheduler = true;
  279. m_message_queue = nullptr;
  280. m_idle_thread = nullptr;
  281. m_current_thread = nullptr;
  282. m_info = nullptr;
  283. m_halt_requested = false;
  284. if (cpu == 0) {
  285. s_smp_enabled = false;
  286. g_total_processors.store(1u, AK::MemoryOrder::memory_order_release);
  287. } else {
  288. g_total_processors.fetch_add(1u, AK::MemoryOrder::memory_order_acq_rel);
  289. }
  290. deferred_call_pool_init();
  291. cpu_setup();
  292. gdt_init();
  293. VERIFY(is_initialized()); // sanity check
  294. VERIFY(&current() == this); // sanity check
  295. }
  296. UNMAP_AFTER_INIT void Processor::initialize(u32 cpu)
  297. {
  298. VERIFY(m_self == this);
  299. VERIFY(&current() == this); // sanity check
  300. dmesgln("CPU[{}]: Supported features: {}", current_id(), features_string());
  301. if (!has_feature(CPUFeature::RDRAND))
  302. dmesgln("CPU[{}]: No RDRAND support detected, randomness will be poor", current_id());
  303. dmesgln("CPU[{}]: Physical address bit width: {}", current_id(), m_physical_address_bit_width);
  304. dmesgln("CPU[{}]: Virtual address bit width: {}", current_id(), m_virtual_address_bit_width);
  305. if (cpu == 0)
  306. idt_init();
  307. else
  308. flush_idt();
  309. if (cpu == 0) {
  310. VERIFY((FlatPtr(&s_clean_fpu_state) & 0xF) == 0);
  311. asm volatile("fninit");
  312. if (has_feature(CPUFeature::FXSR))
  313. asm volatile("fxsave %0"
  314. : "=m"(s_clean_fpu_state));
  315. else
  316. asm volatile("fnsave %0"
  317. : "=m"(s_clean_fpu_state));
  318. if (has_feature(CPUFeature::HYPERVISOR))
  319. detect_hypervisor();
  320. }
  321. m_info = new ProcessorInfo(*this);
  322. {
  323. // We need to prevent races between APs starting up at the same time
  324. VERIFY(cpu < s_processors.size());
  325. s_processors[cpu] = this;
  326. }
  327. }
  328. UNMAP_AFTER_INIT void Processor::detect_hypervisor()
  329. {
  330. CPUID hypervisor_leaf_range(0x40000000);
  331. // Get signature of hypervisor.
  332. alignas(sizeof(u32)) char hypervisor_signature_buffer[13];
  333. *reinterpret_cast<u32*>(hypervisor_signature_buffer) = hypervisor_leaf_range.ebx();
  334. *reinterpret_cast<u32*>(hypervisor_signature_buffer + 4) = hypervisor_leaf_range.ecx();
  335. *reinterpret_cast<u32*>(hypervisor_signature_buffer + 8) = hypervisor_leaf_range.edx();
  336. hypervisor_signature_buffer[12] = '\0';
  337. StringView hypervisor_signature(hypervisor_signature_buffer);
  338. dmesgln("CPU[{}]: CPUID hypervisor signature '{}' ({:#x} {:#x} {:#x}), max leaf {:#x}", current_id(), hypervisor_signature, hypervisor_leaf_range.ebx(), hypervisor_leaf_range.ecx(), hypervisor_leaf_range.edx(), hypervisor_leaf_range.eax());
  339. if (hypervisor_signature == "Microsoft Hv"sv)
  340. detect_hypervisor_hyperv(hypervisor_leaf_range);
  341. }
  342. UNMAP_AFTER_INIT void Processor::detect_hypervisor_hyperv(CPUID const& hypervisor_leaf_range)
  343. {
  344. if (hypervisor_leaf_range.eax() < 0x40000001)
  345. return;
  346. CPUID hypervisor_interface(0x40000001);
  347. // Get signature of hypervisor interface.
  348. alignas(sizeof(u32)) char interface_signature_buffer[5];
  349. *reinterpret_cast<u32*>(interface_signature_buffer) = hypervisor_interface.eax();
  350. interface_signature_buffer[4] = '\0';
  351. StringView hyperv_interface_signature(interface_signature_buffer);
  352. dmesgln("CPU[{}]: Hyper-V interface signature '{}' ({:#x})", current_id(), hyperv_interface_signature, hypervisor_interface.eax());
  353. if (hypervisor_leaf_range.eax() < 0x40000001)
  354. return;
  355. CPUID hypervisor_sysid(0x40000002);
  356. dmesgln("CPU[{}]: Hyper-V system identity {}.{}, build number {}", current_id(), hypervisor_sysid.ebx() >> 16, hypervisor_sysid.ebx() & 0xFFFF, hypervisor_sysid.eax());
  357. if (hypervisor_leaf_range.eax() < 0x40000005 || hyperv_interface_signature != "Hv#1"sv)
  358. return;
  359. dmesgln("CPU[{}]: Hyper-V hypervisor detected", current_id());
  360. // TODO: Actually do something with Hyper-V.
  361. }
  362. void Processor::write_raw_gdt_entry(u16 selector, u32 low, u32 high)
  363. {
  364. u16 i = (selector & 0xfffc) >> 3;
  365. u32 prev_gdt_length = m_gdt_length;
  366. if (i >= m_gdt_length) {
  367. m_gdt_length = i + 1;
  368. VERIFY(m_gdt_length <= sizeof(m_gdt) / sizeof(m_gdt[0]));
  369. m_gdtr.limit = (m_gdt_length + 1) * 8 - 1;
  370. }
  371. m_gdt[i].low = low;
  372. m_gdt[i].high = high;
  373. // clear selectors we may have skipped
  374. while (i < prev_gdt_length) {
  375. m_gdt[i].low = 0;
  376. m_gdt[i].high = 0;
  377. i++;
  378. }
  379. }
  380. void Processor::write_gdt_entry(u16 selector, Descriptor& descriptor)
  381. {
  382. write_raw_gdt_entry(selector, descriptor.low, descriptor.high);
  383. }
  384. Descriptor& Processor::get_gdt_entry(u16 selector)
  385. {
  386. u16 i = (selector & 0xfffc) >> 3;
  387. return *(Descriptor*)(&m_gdt[i]);
  388. }
  389. void Processor::flush_gdt()
  390. {
  391. m_gdtr.address = m_gdt;
  392. m_gdtr.limit = (m_gdt_length * 8) - 1;
  393. asm volatile("lgdt %0" ::"m"(m_gdtr)
  394. : "memory");
  395. }
  396. const DescriptorTablePointer& Processor::get_gdtr()
  397. {
  398. return m_gdtr;
  399. }
  400. Vector<FlatPtr> Processor::capture_stack_trace(Thread& thread, size_t max_frames)
  401. {
  402. FlatPtr frame_ptr = 0, ip = 0;
  403. Vector<FlatPtr, 32> stack_trace;
  404. auto walk_stack = [&](FlatPtr stack_ptr) {
  405. static constexpr size_t max_stack_frames = 4096;
  406. bool is_walking_userspace_stack = false;
  407. stack_trace.append(ip);
  408. size_t count = 1;
  409. while (stack_ptr && stack_trace.size() < max_stack_frames) {
  410. FlatPtr retaddr;
  411. count++;
  412. if (max_frames != 0 && count > max_frames)
  413. break;
  414. if (!Memory::is_user_address(VirtualAddress { stack_ptr })) {
  415. if (is_walking_userspace_stack) {
  416. dbgln("SHENANIGANS! Userspace stack points back into kernel memory");
  417. break;
  418. }
  419. } else {
  420. is_walking_userspace_stack = true;
  421. }
  422. if (Memory::is_user_range(VirtualAddress(stack_ptr), sizeof(FlatPtr) * 2)) {
  423. if (copy_from_user(&retaddr, &((FlatPtr*)stack_ptr)[1]).is_error() || !retaddr)
  424. break;
  425. stack_trace.append(retaddr);
  426. if (copy_from_user(&stack_ptr, (FlatPtr*)stack_ptr).is_error())
  427. break;
  428. } else {
  429. void* fault_at;
  430. if (!safe_memcpy(&retaddr, &((FlatPtr*)stack_ptr)[1], sizeof(FlatPtr), fault_at) || !retaddr)
  431. break;
  432. stack_trace.append(retaddr);
  433. if (!safe_memcpy(&stack_ptr, (FlatPtr*)stack_ptr, sizeof(FlatPtr), fault_at))
  434. break;
  435. }
  436. }
  437. };
  438. auto capture_current_thread = [&]() {
  439. frame_ptr = (FlatPtr)__builtin_frame_address(0);
  440. ip = (FlatPtr)__builtin_return_address(0);
  441. walk_stack(frame_ptr);
  442. };
  443. // Since the thread may be running on another processor, there
  444. // is a chance a context switch may happen while we're trying
  445. // to get it. It also won't be entirely accurate and merely
  446. // reflect the status at the last context switch.
  447. SpinlockLocker lock(g_scheduler_lock);
  448. if (&thread == Processor::current_thread()) {
  449. VERIFY(thread.state() == Thread::Running);
  450. // Leave the scheduler lock. If we trigger page faults we may
  451. // need to be preempted. Since this is our own thread it won't
  452. // cause any problems as the stack won't change below this frame.
  453. lock.unlock();
  454. capture_current_thread();
  455. } else if (thread.is_active()) {
  456. VERIFY(thread.cpu() != Processor::current_id());
  457. // If this is the case, the thread is currently running
  458. // on another processor. We can't trust the kernel stack as
  459. // it may be changing at any time. We need to probably send
  460. // an IPI to that processor, have it walk the stack and wait
  461. // until it returns the data back to us
  462. auto& proc = Processor::current();
  463. smp_unicast(
  464. thread.cpu(),
  465. [&]() {
  466. dbgln("CPU[{}] getting stack for cpu #{}", Processor::current_id(), proc.id());
  467. ScopedAddressSpaceSwitcher switcher(thread.process());
  468. VERIFY(&Processor::current() != &proc);
  469. VERIFY(&thread == Processor::current_thread());
  470. // NOTE: Because the other processor is still holding the
  471. // scheduler lock while waiting for this callback to finish,
  472. // the current thread on the target processor cannot change
  473. // TODO: What to do about page faults here? We might deadlock
  474. // because the other processor is still holding the
  475. // scheduler lock...
  476. capture_current_thread();
  477. },
  478. false);
  479. } else {
  480. switch (thread.state()) {
  481. case Thread::Running:
  482. VERIFY_NOT_REACHED(); // should have been handled above
  483. case Thread::Runnable:
  484. case Thread::Stopped:
  485. case Thread::Blocked:
  486. case Thread::Dying:
  487. case Thread::Dead: {
  488. // We need to retrieve ebp from what was last pushed to the kernel
  489. // stack. Before switching out of that thread, it switch_context
  490. // pushed the callee-saved registers, and the last of them happens
  491. // to be ebp.
  492. ScopedAddressSpaceSwitcher switcher(thread.process());
  493. auto& regs = thread.regs();
  494. auto* stack_top = reinterpret_cast<FlatPtr*>(regs.sp());
  495. if (Memory::is_user_range(VirtualAddress(stack_top), sizeof(FlatPtr))) {
  496. if (copy_from_user(&frame_ptr, &((FlatPtr*)stack_top)[0]).is_error())
  497. frame_ptr = 0;
  498. } else {
  499. void* fault_at;
  500. if (!safe_memcpy(&frame_ptr, &((FlatPtr*)stack_top)[0], sizeof(FlatPtr), fault_at))
  501. frame_ptr = 0;
  502. }
  503. ip = regs.ip();
  504. // TODO: We need to leave the scheduler lock here, but we also
  505. // need to prevent the target thread from being run while
  506. // we walk the stack
  507. lock.unlock();
  508. walk_stack(frame_ptr);
  509. break;
  510. }
  511. default:
  512. dbgln("Cannot capture stack trace for thread {} in state {}", thread, thread.state_string());
  513. break;
  514. }
  515. }
  516. return stack_trace;
  517. }
  518. ProcessorContainer& Processor::processors()
  519. {
  520. return s_processors;
  521. }
  522. Processor& Processor::by_id(u32 id)
  523. {
  524. return *s_processors[id];
  525. }
  526. void Processor::enter_trap(TrapFrame& trap, bool raise_irq)
  527. {
  528. VERIFY_INTERRUPTS_DISABLED();
  529. VERIFY(&Processor::current() == this);
  530. trap.prev_irq_level = m_in_irq;
  531. if (raise_irq)
  532. m_in_irq++;
  533. auto* current_thread = Processor::current_thread();
  534. if (current_thread) {
  535. auto& current_trap = current_thread->current_trap();
  536. trap.next_trap = current_trap;
  537. current_trap = &trap;
  538. // The cs register of this trap tells us where we will return back to
  539. auto new_previous_mode = ((trap.regs->cs & 3) != 0) ? Thread::PreviousMode::UserMode : Thread::PreviousMode::KernelMode;
  540. if (current_thread->set_previous_mode(new_previous_mode) && trap.prev_irq_level == 0) {
  541. current_thread->update_time_scheduled(Scheduler::current_time(), new_previous_mode == Thread::PreviousMode::KernelMode, false);
  542. }
  543. } else {
  544. trap.next_trap = nullptr;
  545. }
  546. }
  547. void Processor::exit_trap(TrapFrame& trap)
  548. {
  549. VERIFY_INTERRUPTS_DISABLED();
  550. VERIFY(&Processor::current() == this);
  551. // Temporarily enter a critical section. This is to prevent critical
  552. // sections entered and left within e.g. smp_process_pending_messages
  553. // to trigger a context switch while we're executing this function
  554. // See the comment at the end of the function why we don't use
  555. // ScopedCritical here.
  556. m_in_critical = m_in_critical + 1;
  557. VERIFY(m_in_irq >= trap.prev_irq_level);
  558. m_in_irq = trap.prev_irq_level;
  559. if (s_smp_enabled)
  560. smp_process_pending_messages();
  561. // Process the deferred call queue. Among other things, this ensures
  562. // that any pending thread unblocks happen before we enter the scheduler.
  563. deferred_call_execute_pending();
  564. auto* current_thread = Processor::current_thread();
  565. if (current_thread) {
  566. auto& current_trap = current_thread->current_trap();
  567. current_trap = trap.next_trap;
  568. Thread::PreviousMode new_previous_mode;
  569. if (current_trap) {
  570. VERIFY(current_trap->regs);
  571. // If we have another higher level trap then we probably returned
  572. // from an interrupt or irq handler. The cs register of the
  573. // new/higher level trap tells us what the mode prior to it was
  574. new_previous_mode = ((current_trap->regs->cs & 3) != 0) ? Thread::PreviousMode::UserMode : Thread::PreviousMode::KernelMode;
  575. } else {
  576. // If we don't have a higher level trap then we're back in user mode.
  577. // Which means that the previous mode prior to being back in user mode was kernel mode
  578. new_previous_mode = Thread::PreviousMode::KernelMode;
  579. }
  580. if (current_thread->set_previous_mode(new_previous_mode))
  581. current_thread->update_time_scheduled(Scheduler::current_time(), true, false);
  582. }
  583. VERIFY_INTERRUPTS_DISABLED();
  584. // Leave the critical section without actually enabling interrupts.
  585. // We don't want context switches to happen until we're explicitly
  586. // triggering a switch in check_invoke_scheduler.
  587. m_in_critical = m_in_critical - 1;
  588. if (!m_in_irq && !m_in_critical)
  589. check_invoke_scheduler();
  590. }
  591. void Processor::check_invoke_scheduler()
  592. {
  593. InterruptDisabler disabler;
  594. VERIFY(!m_in_irq);
  595. VERIFY(!m_in_critical);
  596. VERIFY(&Processor::current() == this);
  597. if (m_invoke_scheduler_async && m_scheduler_initialized) {
  598. m_invoke_scheduler_async = false;
  599. Scheduler::invoke_async();
  600. }
  601. }
  602. void Processor::flush_tlb_local(VirtualAddress vaddr, size_t page_count)
  603. {
  604. auto ptr = vaddr.as_ptr();
  605. while (page_count > 0) {
  606. // clang-format off
  607. asm volatile("invlpg %0"
  608. :
  609. : "m"(*ptr)
  610. : "memory");
  611. // clang-format on
  612. ptr += PAGE_SIZE;
  613. page_count--;
  614. }
  615. }
  616. void Processor::flush_tlb(Memory::PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count)
  617. {
  618. if (s_smp_enabled && (!Memory::is_user_address(vaddr) || Process::current().thread_count() > 1))
  619. smp_broadcast_flush_tlb(page_directory, vaddr, page_count);
  620. else
  621. flush_tlb_local(vaddr, page_count);
  622. }
  623. void Processor::smp_return_to_pool(ProcessorMessage& msg)
  624. {
  625. ProcessorMessage* next = nullptr;
  626. for (;;) {
  627. msg.next = next;
  628. if (s_message_pool.compare_exchange_strong(next, &msg, AK::MemoryOrder::memory_order_acq_rel))
  629. break;
  630. Processor::pause();
  631. }
  632. }
  633. ProcessorMessage& Processor::smp_get_from_pool()
  634. {
  635. ProcessorMessage* msg;
  636. // The assumption is that messages are never removed from the pool!
  637. for (;;) {
  638. msg = s_message_pool.load(AK::MemoryOrder::memory_order_consume);
  639. if (!msg) {
  640. if (!Processor::current().smp_process_pending_messages()) {
  641. Processor::pause();
  642. }
  643. continue;
  644. }
  645. // If another processor were to use this message in the meanwhile,
  646. // "msg" is still valid (because it never gets freed). We'd detect
  647. // this because the expected value "msg" and pool would
  648. // no longer match, and the compare_exchange will fail. But accessing
  649. // "msg->next" is always safe here.
  650. if (s_message_pool.compare_exchange_strong(msg, msg->next, AK::MemoryOrder::memory_order_acq_rel)) {
  651. // We successfully "popped" this available message
  652. break;
  653. }
  654. }
  655. VERIFY(msg != nullptr);
  656. return *msg;
  657. }
  658. u32 Processor::smp_wake_n_idle_processors(u32 wake_count)
  659. {
  660. VERIFY_INTERRUPTS_DISABLED();
  661. VERIFY(wake_count > 0);
  662. if (!s_smp_enabled)
  663. return 0;
  664. // Wake at most N - 1 processors
  665. if (wake_count >= Processor::count()) {
  666. wake_count = Processor::count() - 1;
  667. VERIFY(wake_count > 0);
  668. }
  669. u32 current_id = Processor::current_id();
  670. u32 did_wake_count = 0;
  671. auto& apic = APIC::the();
  672. while (did_wake_count < wake_count) {
  673. // Try to get a set of idle CPUs and flip them to busy
  674. u32 idle_mask = s_idle_cpu_mask.load(AK::MemoryOrder::memory_order_relaxed) & ~(1u << current_id);
  675. u32 idle_count = popcount(idle_mask);
  676. if (idle_count == 0)
  677. break; // No (more) idle processor available
  678. u32 found_mask = 0;
  679. for (u32 i = 0; i < idle_count; i++) {
  680. u32 cpu = bit_scan_forward(idle_mask) - 1;
  681. idle_mask &= ~(1u << cpu);
  682. found_mask |= 1u << cpu;
  683. }
  684. idle_mask = s_idle_cpu_mask.fetch_and(~found_mask, AK::MemoryOrder::memory_order_acq_rel) & found_mask;
  685. if (idle_mask == 0)
  686. continue; // All of them were flipped to busy, try again
  687. idle_count = popcount(idle_mask);
  688. for (u32 i = 0; i < idle_count; i++) {
  689. u32 cpu = bit_scan_forward(idle_mask) - 1;
  690. idle_mask &= ~(1u << cpu);
  691. // Send an IPI to that CPU to wake it up. There is a possibility
  692. // someone else woke it up as well, or that it woke up due to
  693. // a timer interrupt. But we tried hard to avoid this...
  694. apic.send_ipi(cpu);
  695. did_wake_count++;
  696. }
  697. }
  698. return did_wake_count;
  699. }
  700. UNMAP_AFTER_INIT void Processor::smp_enable()
  701. {
  702. size_t msg_pool_size = Processor::count() * 100u;
  703. size_t msg_entries_cnt = Processor::count();
  704. auto msgs = new ProcessorMessage[msg_pool_size];
  705. auto msg_entries = new ProcessorMessageEntry[msg_pool_size * msg_entries_cnt];
  706. size_t msg_entry_i = 0;
  707. for (size_t i = 0; i < msg_pool_size; i++, msg_entry_i += msg_entries_cnt) {
  708. auto& msg = msgs[i];
  709. msg.next = i < msg_pool_size - 1 ? &msgs[i + 1] : nullptr;
  710. msg.per_proc_entries = &msg_entries[msg_entry_i];
  711. for (size_t k = 0; k < msg_entries_cnt; k++)
  712. msg_entries[msg_entry_i + k].msg = &msg;
  713. }
  714. s_message_pool.store(&msgs[0], AK::MemoryOrder::memory_order_release);
  715. // Start sending IPI messages
  716. s_smp_enabled = true;
  717. }
  718. void Processor::smp_cleanup_message(ProcessorMessage& msg)
  719. {
  720. switch (msg.type) {
  721. case ProcessorMessage::Callback:
  722. msg.callback_value().~Function();
  723. break;
  724. default:
  725. break;
  726. }
  727. }
  728. bool Processor::smp_process_pending_messages()
  729. {
  730. VERIFY(s_smp_enabled);
  731. bool did_process = false;
  732. enter_critical();
  733. if (auto pending_msgs = m_message_queue.exchange(nullptr, AK::MemoryOrder::memory_order_acq_rel)) {
  734. // We pulled the stack of pending messages in LIFO order, so we need to reverse the list first
  735. auto reverse_list =
  736. [](ProcessorMessageEntry* list) -> ProcessorMessageEntry* {
  737. ProcessorMessageEntry* rev_list = nullptr;
  738. while (list) {
  739. auto next = list->next;
  740. list->next = rev_list;
  741. rev_list = list;
  742. list = next;
  743. }
  744. return rev_list;
  745. };
  746. pending_msgs = reverse_list(pending_msgs);
  747. // now process in the right order
  748. ProcessorMessageEntry* next_msg;
  749. for (auto cur_msg = pending_msgs; cur_msg; cur_msg = next_msg) {
  750. next_msg = cur_msg->next;
  751. auto msg = cur_msg->msg;
  752. dbgln_if(SMP_DEBUG, "SMP[{}]: Processing message {}", current_id(), VirtualAddress(msg));
  753. switch (msg->type) {
  754. case ProcessorMessage::Callback:
  755. msg->invoke_callback();
  756. break;
  757. case ProcessorMessage::FlushTlb:
  758. if (Memory::is_user_address(VirtualAddress(msg->flush_tlb.ptr))) {
  759. // We assume that we don't cross into kernel land!
  760. VERIFY(Memory::is_user_range(VirtualAddress(msg->flush_tlb.ptr), msg->flush_tlb.page_count * PAGE_SIZE));
  761. if (read_cr3() != msg->flush_tlb.page_directory->cr3()) {
  762. // This processor isn't using this page directory right now, we can ignore this request
  763. dbgln_if(SMP_DEBUG, "SMP[{}]: No need to flush {} pages at {}", current_id(), msg->flush_tlb.page_count, VirtualAddress(msg->flush_tlb.ptr));
  764. break;
  765. }
  766. }
  767. flush_tlb_local(VirtualAddress(msg->flush_tlb.ptr), msg->flush_tlb.page_count);
  768. break;
  769. }
  770. bool is_async = msg->async; // Need to cache this value *before* dropping the ref count!
  771. auto prev_refs = msg->refs.fetch_sub(1u, AK::MemoryOrder::memory_order_acq_rel);
  772. VERIFY(prev_refs != 0);
  773. if (prev_refs == 1) {
  774. // All processors handled this. If this is an async message,
  775. // we need to clean it up and return it to the pool
  776. if (is_async) {
  777. smp_cleanup_message(*msg);
  778. smp_return_to_pool(*msg);
  779. }
  780. }
  781. if (m_halt_requested.load(AK::MemoryOrder::memory_order_relaxed))
  782. halt_this();
  783. }
  784. did_process = true;
  785. } else if (m_halt_requested.load(AK::MemoryOrder::memory_order_relaxed)) {
  786. halt_this();
  787. }
  788. leave_critical();
  789. return did_process;
  790. }
  791. bool Processor::smp_enqueue_message(ProcessorMessage& msg)
  792. {
  793. // Note that it's quite possible that the other processor may pop
  794. // the queue at any given time. We rely on the fact that the messages
  795. // are pooled and never get freed!
  796. auto& msg_entry = msg.per_proc_entries[id()];
  797. VERIFY(msg_entry.msg == &msg);
  798. ProcessorMessageEntry* next = nullptr;
  799. for (;;) {
  800. msg_entry.next = next;
  801. if (m_message_queue.compare_exchange_strong(next, &msg_entry, AK::MemoryOrder::memory_order_acq_rel))
  802. break;
  803. Processor::pause();
  804. }
  805. // If the enqueued message was the only message in the queue when posted,
  806. // we return true. This is used by callers when deciding whether to generate an IPI.
  807. return next == nullptr;
  808. }
  809. void Processor::smp_broadcast_message(ProcessorMessage& msg)
  810. {
  811. auto& current_processor = Processor::current();
  812. dbgln_if(SMP_DEBUG, "SMP[{}]: Broadcast message {} to cpus: {} processor: {}", current_processor.id(), VirtualAddress(&msg), count(), VirtualAddress(&current_processor));
  813. msg.refs.store(count() - 1, AK::MemoryOrder::memory_order_release);
  814. VERIFY(msg.refs > 0);
  815. bool need_broadcast = false;
  816. for_each(
  817. [&](Processor& proc) {
  818. if (&proc != &current_processor) {
  819. if (proc.smp_enqueue_message(msg))
  820. need_broadcast = true;
  821. }
  822. });
  823. // Now trigger an IPI on all other APs (unless all targets already had messages queued)
  824. if (need_broadcast)
  825. APIC::the().broadcast_ipi();
  826. }
  827. void Processor::smp_broadcast_wait_sync(ProcessorMessage& msg)
  828. {
  829. auto& cur_proc = Processor::current();
  830. VERIFY(!msg.async);
  831. // If synchronous then we must cleanup and return the message back
  832. // to the pool. Otherwise, the last processor to complete it will return it
  833. while (msg.refs.load(AK::MemoryOrder::memory_order_consume) != 0) {
  834. Processor::pause();
  835. // We need to process any messages that may have been sent to
  836. // us while we're waiting. This also checks if another processor
  837. // may have requested us to halt.
  838. cur_proc.smp_process_pending_messages();
  839. }
  840. smp_cleanup_message(msg);
  841. smp_return_to_pool(msg);
  842. }
  843. void Processor::smp_unicast_message(u32 cpu, ProcessorMessage& msg, bool async)
  844. {
  845. auto& current_processor = Processor::current();
  846. VERIFY(cpu != current_processor.id());
  847. auto& target_processor = processors()[cpu];
  848. msg.async = async;
  849. dbgln_if(SMP_DEBUG, "SMP[{}]: Send message {} to cpu #{} processor: {}", current_processor.id(), VirtualAddress(&msg), cpu, VirtualAddress(&target_processor));
  850. msg.refs.store(1u, AK::MemoryOrder::memory_order_release);
  851. if (target_processor->smp_enqueue_message(msg)) {
  852. APIC::the().send_ipi(cpu);
  853. }
  854. if (!async) {
  855. // If synchronous then we must cleanup and return the message back
  856. // to the pool. Otherwise, the last processor to complete it will return it
  857. while (msg.refs.load(AK::MemoryOrder::memory_order_consume) != 0) {
  858. Processor::pause();
  859. // We need to process any messages that may have been sent to
  860. // us while we're waiting. This also checks if another processor
  861. // may have requested us to halt.
  862. current_processor.smp_process_pending_messages();
  863. }
  864. smp_cleanup_message(msg);
  865. smp_return_to_pool(msg);
  866. }
  867. }
  868. void Processor::smp_unicast(u32 cpu, Function<void()> callback, bool async)
  869. {
  870. auto& msg = smp_get_from_pool();
  871. msg.type = ProcessorMessage::Callback;
  872. new (msg.callback_storage) ProcessorMessage::CallbackFunction(move(callback));
  873. smp_unicast_message(cpu, msg, async);
  874. }
  875. void Processor::smp_broadcast_flush_tlb(Memory::PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count)
  876. {
  877. auto& msg = smp_get_from_pool();
  878. msg.async = false;
  879. msg.type = ProcessorMessage::FlushTlb;
  880. msg.flush_tlb.page_directory = page_directory;
  881. msg.flush_tlb.ptr = vaddr.as_ptr();
  882. msg.flush_tlb.page_count = page_count;
  883. smp_broadcast_message(msg);
  884. // While the other processors handle this request, we'll flush ours
  885. flush_tlb_local(vaddr, page_count);
  886. // Now wait until everybody is done as well
  887. smp_broadcast_wait_sync(msg);
  888. }
  889. void Processor::smp_broadcast_halt()
  890. {
  891. // We don't want to use a message, because this could have been triggered
  892. // by being out of memory and we might not be able to get a message
  893. for_each(
  894. [&](Processor& proc) {
  895. proc.m_halt_requested.store(true, AK::MemoryOrder::memory_order_release);
  896. });
  897. // Now trigger an IPI on all other APs
  898. APIC::the().broadcast_ipi();
  899. }
  900. void Processor::Processor::halt()
  901. {
  902. if (s_smp_enabled)
  903. smp_broadcast_halt();
  904. halt_this();
  905. }
  906. UNMAP_AFTER_INIT void Processor::deferred_call_pool_init()
  907. {
  908. size_t pool_count = sizeof(m_deferred_call_pool) / sizeof(m_deferred_call_pool[0]);
  909. for (size_t i = 0; i < pool_count; i++) {
  910. auto& entry = m_deferred_call_pool[i];
  911. entry.next = i < pool_count - 1 ? &m_deferred_call_pool[i + 1] : nullptr;
  912. new (entry.handler_storage) DeferredCallEntry::HandlerFunction;
  913. entry.was_allocated = false;
  914. }
  915. m_pending_deferred_calls = nullptr;
  916. m_free_deferred_call_pool_entry = &m_deferred_call_pool[0];
  917. }
  918. void Processor::deferred_call_return_to_pool(DeferredCallEntry* entry)
  919. {
  920. VERIFY(m_in_critical);
  921. VERIFY(!entry->was_allocated);
  922. entry->handler_value() = {};
  923. entry->next = m_free_deferred_call_pool_entry;
  924. m_free_deferred_call_pool_entry = entry;
  925. }
  926. DeferredCallEntry* Processor::deferred_call_get_free()
  927. {
  928. VERIFY(m_in_critical);
  929. if (m_free_deferred_call_pool_entry) {
  930. // Fast path, we have an entry in our pool
  931. auto* entry = m_free_deferred_call_pool_entry;
  932. m_free_deferred_call_pool_entry = entry->next;
  933. VERIFY(!entry->was_allocated);
  934. return entry;
  935. }
  936. auto* entry = new DeferredCallEntry;
  937. new (entry->handler_storage) DeferredCallEntry::HandlerFunction;
  938. entry->was_allocated = true;
  939. return entry;
  940. }
  941. void Processor::deferred_call_execute_pending()
  942. {
  943. VERIFY(m_in_critical);
  944. if (!m_pending_deferred_calls)
  945. return;
  946. auto* pending_list = m_pending_deferred_calls;
  947. m_pending_deferred_calls = nullptr;
  948. // We pulled the stack of pending deferred calls in LIFO order, so we need to reverse the list first
  949. auto reverse_list =
  950. [](DeferredCallEntry* list) -> DeferredCallEntry* {
  951. DeferredCallEntry* rev_list = nullptr;
  952. while (list) {
  953. auto next = list->next;
  954. list->next = rev_list;
  955. rev_list = list;
  956. list = next;
  957. }
  958. return rev_list;
  959. };
  960. pending_list = reverse_list(pending_list);
  961. do {
  962. pending_list->invoke_handler();
  963. // Return the entry back to the pool, or free it
  964. auto* next = pending_list->next;
  965. if (pending_list->was_allocated) {
  966. pending_list->handler_value().~Function();
  967. delete pending_list;
  968. } else
  969. deferred_call_return_to_pool(pending_list);
  970. pending_list = next;
  971. } while (pending_list);
  972. }
  973. void Processor::deferred_call_queue_entry(DeferredCallEntry* entry)
  974. {
  975. VERIFY(m_in_critical);
  976. entry->next = m_pending_deferred_calls;
  977. m_pending_deferred_calls = entry;
  978. }
  979. void Processor::deferred_call_queue(Function<void()> callback)
  980. {
  981. // NOTE: If we are called outside of a critical section and outside
  982. // of an irq handler, the function will be executed before we return!
  983. ScopedCritical critical;
  984. auto& cur_proc = Processor::current();
  985. auto* entry = cur_proc.deferred_call_get_free();
  986. entry->handler_value() = move(callback);
  987. cur_proc.deferred_call_queue_entry(entry);
  988. }
  989. UNMAP_AFTER_INIT void Processor::gdt_init()
  990. {
  991. m_gdt_length = 0;
  992. m_gdtr.address = nullptr;
  993. m_gdtr.limit = 0;
  994. write_raw_gdt_entry(0x0000, 0x00000000, 0x00000000);
  995. #if ARCH(I386)
  996. write_raw_gdt_entry(GDT_SELECTOR_CODE0, 0x0000ffff, 0x00cf9a00); // code0
  997. write_raw_gdt_entry(GDT_SELECTOR_DATA0, 0x0000ffff, 0x00cf9200); // data0
  998. write_raw_gdt_entry(GDT_SELECTOR_CODE3, 0x0000ffff, 0x00cffa00); // code3
  999. write_raw_gdt_entry(GDT_SELECTOR_DATA3, 0x0000ffff, 0x00cff200); // data3
  1000. #else
  1001. write_raw_gdt_entry(GDT_SELECTOR_CODE0, 0x0000ffff, 0x00af9a00); // code0
  1002. write_raw_gdt_entry(GDT_SELECTOR_DATA0, 0x0000ffff, 0x00af9200); // data0
  1003. write_raw_gdt_entry(GDT_SELECTOR_DATA3, 0x0000ffff, 0x008ff200); // data3
  1004. write_raw_gdt_entry(GDT_SELECTOR_CODE3, 0x0000ffff, 0x00affa00); // code3
  1005. #endif
  1006. #if ARCH(I386)
  1007. Descriptor tls_descriptor {};
  1008. tls_descriptor.low = tls_descriptor.high = 0;
  1009. tls_descriptor.dpl = 3;
  1010. tls_descriptor.segment_present = 1;
  1011. tls_descriptor.granularity = 0;
  1012. tls_descriptor.operation_size64 = 0;
  1013. tls_descriptor.operation_size32 = 1;
  1014. tls_descriptor.descriptor_type = 1;
  1015. tls_descriptor.type = 2;
  1016. write_gdt_entry(GDT_SELECTOR_TLS, tls_descriptor); // tls3
  1017. Descriptor gs_descriptor {};
  1018. gs_descriptor.set_base(VirtualAddress { this });
  1019. gs_descriptor.set_limit(sizeof(Processor) - 1);
  1020. gs_descriptor.dpl = 0;
  1021. gs_descriptor.segment_present = 1;
  1022. gs_descriptor.granularity = 0;
  1023. gs_descriptor.operation_size64 = 0;
  1024. gs_descriptor.operation_size32 = 1;
  1025. gs_descriptor.descriptor_type = 1;
  1026. gs_descriptor.type = 2;
  1027. write_gdt_entry(GDT_SELECTOR_PROC, gs_descriptor); // gs0
  1028. #endif
  1029. Descriptor tss_descriptor {};
  1030. tss_descriptor.set_base(VirtualAddress { (size_t)&m_tss & 0xffffffff });
  1031. tss_descriptor.set_limit(sizeof(TSS) - 1);
  1032. tss_descriptor.dpl = 0;
  1033. tss_descriptor.segment_present = 1;
  1034. tss_descriptor.granularity = 0;
  1035. tss_descriptor.operation_size64 = 0;
  1036. tss_descriptor.operation_size32 = 1;
  1037. tss_descriptor.descriptor_type = 0;
  1038. tss_descriptor.type = 9;
  1039. write_gdt_entry(GDT_SELECTOR_TSS, tss_descriptor); // tss
  1040. #if ARCH(X86_64)
  1041. Descriptor tss_descriptor_part2 {};
  1042. tss_descriptor_part2.low = (size_t)&m_tss >> 32;
  1043. write_gdt_entry(GDT_SELECTOR_TSS_PART2, tss_descriptor_part2);
  1044. #endif
  1045. flush_gdt();
  1046. load_task_register(GDT_SELECTOR_TSS);
  1047. #if ARCH(X86_64)
  1048. MSR gs_base(MSR_GS_BASE);
  1049. gs_base.set((u64)this);
  1050. #else
  1051. asm volatile(
  1052. "mov %%ax, %%ds\n"
  1053. "mov %%ax, %%es\n"
  1054. "mov %%ax, %%fs\n"
  1055. "mov %%ax, %%ss\n" ::"a"(GDT_SELECTOR_DATA0)
  1056. : "memory");
  1057. set_gs(GDT_SELECTOR_PROC);
  1058. #endif
  1059. #if ARCH(I386)
  1060. // Make sure CS points to the kernel code descriptor.
  1061. // clang-format off
  1062. asm volatile(
  1063. "ljmpl $" __STRINGIFY(GDT_SELECTOR_CODE0) ", $sanity\n"
  1064. "sanity:\n");
  1065. // clang-format on
  1066. #endif
  1067. }
  1068. extern "C" void context_first_init([[maybe_unused]] Thread* from_thread, [[maybe_unused]] Thread* to_thread, [[maybe_unused]] TrapFrame* trap)
  1069. {
  1070. VERIFY(!are_interrupts_enabled());
  1071. VERIFY(is_kernel_mode());
  1072. dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context <-- from {} {} to {} {} (context_first_init)", VirtualAddress(from_thread), *from_thread, VirtualAddress(to_thread), *to_thread);
  1073. VERIFY(to_thread == Thread::current());
  1074. Scheduler::enter_current(*from_thread, true);
  1075. auto in_critical = to_thread->saved_critical();
  1076. VERIFY(in_critical > 0);
  1077. Processor::restore_in_critical(in_critical);
  1078. // Since we got here and don't have Scheduler::context_switch in the
  1079. // call stack (because this is the first time we switched into this
  1080. // context), we need to notify the scheduler so that it can release
  1081. // the scheduler lock. We don't want to enable interrupts at this point
  1082. // as we're still in the middle of a context switch. Doing so could
  1083. // trigger a context switch within a context switch, leading to a crash.
  1084. FlatPtr flags = trap->regs->flags();
  1085. Scheduler::leave_on_first_switch(flags & ~0x200);
  1086. }
  1087. extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread)
  1088. {
  1089. VERIFY(from_thread == to_thread || from_thread->state() != Thread::Running);
  1090. VERIFY(to_thread->state() == Thread::Running);
  1091. bool has_fxsr = Processor::current().has_feature(CPUFeature::FXSR);
  1092. Processor::set_current_thread(*to_thread);
  1093. auto& from_regs = from_thread->regs();
  1094. auto& to_regs = to_thread->regs();
  1095. if (has_fxsr)
  1096. asm volatile("fxsave %0"
  1097. : "=m"(from_thread->fpu_state()));
  1098. else
  1099. asm volatile("fnsave %0"
  1100. : "=m"(from_thread->fpu_state()));
  1101. #if ARCH(I386)
  1102. from_regs.fs = get_fs();
  1103. from_regs.gs = get_gs();
  1104. set_fs(to_regs.fs);
  1105. set_gs(to_regs.gs);
  1106. #endif
  1107. if (from_thread->process().is_traced())
  1108. read_debug_registers_into(from_thread->debug_register_state());
  1109. if (to_thread->process().is_traced()) {
  1110. write_debug_registers_from(to_thread->debug_register_state());
  1111. } else {
  1112. clear_debug_registers();
  1113. }
  1114. auto& processor = Processor::current();
  1115. #if ARCH(I386)
  1116. auto& tls_descriptor = processor.get_gdt_entry(GDT_SELECTOR_TLS);
  1117. tls_descriptor.set_base(to_thread->thread_specific_data());
  1118. tls_descriptor.set_limit(to_thread->thread_specific_region_size());
  1119. #else
  1120. MSR fs_base_msr(MSR_FS_BASE);
  1121. fs_base_msr.set(to_thread->thread_specific_data().get());
  1122. #endif
  1123. if (from_regs.cr3 != to_regs.cr3)
  1124. write_cr3(to_regs.cr3);
  1125. to_thread->set_cpu(processor.id());
  1126. auto in_critical = to_thread->saved_critical();
  1127. VERIFY(in_critical > 0);
  1128. Processor::restore_in_critical(in_critical);
  1129. if (has_fxsr)
  1130. asm volatile("fxrstor %0" ::"m"(to_thread->fpu_state()));
  1131. else
  1132. asm volatile("frstor %0" ::"m"(to_thread->fpu_state()));
  1133. // TODO: ioperm?
  1134. }
  1135. extern "C" FlatPtr do_init_context(Thread* thread, u32 flags)
  1136. {
  1137. VERIFY_INTERRUPTS_DISABLED();
  1138. thread->regs().set_flags(flags);
  1139. return Processor::current().init_context(*thread, true);
  1140. }
  1141. void Processor::assume_context(Thread& thread, FlatPtr flags)
  1142. {
  1143. dbgln_if(CONTEXT_SWITCH_DEBUG, "Assume context for thread {} {}", VirtualAddress(&thread), thread);
  1144. VERIFY_INTERRUPTS_DISABLED();
  1145. Scheduler::prepare_after_exec();
  1146. // in_critical() should be 2 here. The critical section in Process::exec
  1147. // and then the scheduler lock
  1148. VERIFY(Processor::in_critical() == 2);
  1149. do_assume_context(&thread, flags);
  1150. VERIFY_NOT_REACHED();
  1151. }
  1152. u64 Processor::time_spent_idle() const
  1153. {
  1154. return m_idle_thread->time_in_user() + m_idle_thread->time_in_kernel();
  1155. }
  1156. }