Processor.cpp 61 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666
  1. /*
  2. * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
  3. * Copyright (c) 2022, Linus Groh <linusg@serenityos.org>
  4. * Copyright (c) 2022, the SerenityOS developers.
  5. *
  6. * SPDX-License-Identifier: BSD-2-Clause
  7. */
  8. #include <AK/BuiltinWrappers.h>
  9. #include <AK/Format.h>
  10. #include <AK/StdLibExtras.h>
  11. #include <AK/StringBuilder.h>
  12. #include <AK/Types.h>
  13. #include <Kernel/Interrupts/APIC.h>
  14. #include <Kernel/Process.h>
  15. #include <Kernel/Scheduler.h>
  16. #include <Kernel/Sections.h>
  17. #include <Kernel/StdLib.h>
  18. #include <Kernel/Thread.h>
  19. #include <Kernel/Arch/InterruptDisabler.h>
  20. #include <Kernel/Arch/Interrupts.h>
  21. #include <Kernel/Arch/Processor.h>
  22. #include <Kernel/Arch/SafeMem.h>
  23. #include <Kernel/Arch/ScopedCritical.h>
  24. #include <Kernel/Arch/x86/CPUID.h>
  25. #include <Kernel/Arch/x86/MSR.h>
  26. #include <Kernel/Arch/x86/ProcessorInfo.h>
  27. #include <Kernel/Arch/x86/TrapFrame.h>
  28. #include <Kernel/Memory/PageDirectory.h>
  29. #include <Kernel/Memory/ScopedAddressSpaceSwitcher.h>
  30. namespace Kernel {
  31. READONLY_AFTER_INIT FPUState Processor::s_clean_fpu_state;
  32. READONLY_AFTER_INIT static ProcessorContainer s_processors {};
  33. READONLY_AFTER_INIT Atomic<u32> Processor::g_total_processors;
  34. READONLY_AFTER_INIT static bool volatile s_smp_enabled;
  35. static Atomic<ProcessorMessage*> s_message_pool;
  36. Atomic<u32> Processor::s_idle_cpu_mask { 0 };
  37. // The compiler can't see the calls to these functions inside assembly.
  38. // Declare them, to avoid dead code warnings.
  39. extern "C" void context_first_init(Thread* from_thread, Thread* to_thread, TrapFrame* trap) __attribute__((used));
  40. extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __attribute__((used));
  41. extern "C" FlatPtr do_init_context(Thread* thread, u32 flags) __attribute__((used));
  42. extern "C" void syscall_entry();
  43. bool Processor::is_smp_enabled()
  44. {
  45. return s_smp_enabled;
  46. }
  47. UNMAP_AFTER_INIT static void sse_init()
  48. {
  49. write_cr0((read_cr0() & 0xfffffffbu) | 0x2);
  50. write_cr4(read_cr4() | 0x600);
  51. }
  52. void exit_kernel_thread(void)
  53. {
  54. Thread::current()->exit();
  55. }
  56. UNMAP_AFTER_INIT void Processor::cpu_detect()
  57. {
  58. // NOTE: This is called during Processor::early_initialize, we cannot
  59. // safely log at this point because we don't have kmalloc
  60. // initialized yet!
  61. m_features = CPUFeature::Type(0u);
  62. CPUID processor_info(0x1);
  63. auto handle_edx_bit_11_feature = [&] {
  64. u32 stepping = processor_info.eax() & 0xf;
  65. u32 model = (processor_info.eax() >> 4) & 0xf;
  66. u32 family = (processor_info.eax() >> 8) & 0xf;
  67. // FIXME: I have no clue what these mean or where it's from (the Intel manual I've seen just says EDX[11] is SEP).
  68. // If you do, please convert them to constants or add comments!
  69. if (!(family == 6 && model < 3 && stepping < 3))
  70. m_features |= CPUFeature::SEP;
  71. if ((family == 6 && model >= 3) || (family == 0xf && model >= 0xe))
  72. m_features |= CPUFeature::CONSTANT_TSC;
  73. };
  74. if (processor_info.ecx() & (1 << 0))
  75. m_features |= CPUFeature::SSE3;
  76. if (processor_info.ecx() & (1 << 1))
  77. m_features |= CPUFeature::PCLMULQDQ;
  78. if (processor_info.ecx() & (1 << 2))
  79. m_features |= CPUFeature::DTES64;
  80. if (processor_info.ecx() & (1 << 3))
  81. m_features |= CPUFeature::MONITOR;
  82. if (processor_info.ecx() & (1 << 4))
  83. m_features |= CPUFeature::DS_CPL;
  84. if (processor_info.ecx() & (1 << 5))
  85. m_features |= CPUFeature::VMX;
  86. if (processor_info.ecx() & (1 << 6))
  87. m_features |= CPUFeature::SMX;
  88. if (processor_info.ecx() & (1 << 7))
  89. m_features |= CPUFeature::EST;
  90. if (processor_info.ecx() & (1 << 8))
  91. m_features |= CPUFeature::TM2;
  92. if (processor_info.ecx() & (1 << 9))
  93. m_features |= CPUFeature::SSSE3;
  94. if (processor_info.ecx() & (1 << 10))
  95. m_features |= CPUFeature::CNXT_ID;
  96. if (processor_info.ecx() & (1 << 11))
  97. m_features |= CPUFeature::SDBG;
  98. if (processor_info.ecx() & (1 << 12))
  99. m_features |= CPUFeature::FMA;
  100. if (processor_info.ecx() & (1 << 13))
  101. m_features |= CPUFeature::CX16;
  102. if (processor_info.ecx() & (1 << 14))
  103. m_features |= CPUFeature::XTPR;
  104. if (processor_info.ecx() & (1 << 15))
  105. m_features |= CPUFeature::PDCM;
  106. if (processor_info.ecx() & (1 << 17))
  107. m_features |= CPUFeature::PCID;
  108. if (processor_info.ecx() & (1 << 18))
  109. m_features |= CPUFeature::DCA;
  110. if (processor_info.ecx() & (1 << 19))
  111. m_features |= CPUFeature::SSE4_1;
  112. if (processor_info.ecx() & (1 << 20))
  113. m_features |= CPUFeature::SSE4_2;
  114. if (processor_info.ecx() & (1 << 21))
  115. m_features |= CPUFeature::X2APIC;
  116. if (processor_info.ecx() & (1 << 22))
  117. m_features |= CPUFeature::MOVBE;
  118. if (processor_info.ecx() & (1 << 23))
  119. m_features |= CPUFeature::POPCNT;
  120. if (processor_info.ecx() & (1 << 24))
  121. m_features |= CPUFeature::TSC_DEADLINE;
  122. if (processor_info.ecx() & (1 << 25))
  123. m_features |= CPUFeature::AES;
  124. if (processor_info.ecx() & (1 << 26))
  125. m_features |= CPUFeature::XSAVE;
  126. if (processor_info.ecx() & (1 << 27))
  127. m_features |= CPUFeature::OSXSAVE;
  128. if (processor_info.ecx() & (1 << 28))
  129. m_features |= CPUFeature::AVX;
  130. if (processor_info.ecx() & (1 << 29))
  131. m_features |= CPUFeature::F16C;
  132. if (processor_info.ecx() & (1 << 30))
  133. m_features |= CPUFeature::RDRAND;
  134. if (processor_info.ecx() & (1 << 31))
  135. m_features |= CPUFeature::HYPERVISOR;
  136. if (processor_info.edx() & (1 << 0))
  137. m_features |= CPUFeature::FPU;
  138. if (processor_info.edx() & (1 << 1))
  139. m_features |= CPUFeature::VME;
  140. if (processor_info.edx() & (1 << 2))
  141. m_features |= CPUFeature::DE;
  142. if (processor_info.edx() & (1 << 3))
  143. m_features |= CPUFeature::PSE;
  144. if (processor_info.edx() & (1 << 4))
  145. m_features |= CPUFeature::TSC;
  146. if (processor_info.edx() & (1 << 5))
  147. m_features |= CPUFeature::MSR;
  148. if (processor_info.edx() & (1 << 6))
  149. m_features |= CPUFeature::PAE;
  150. if (processor_info.edx() & (1 << 7))
  151. m_features |= CPUFeature::MCE;
  152. if (processor_info.edx() & (1 << 8))
  153. m_features |= CPUFeature::CX8;
  154. if (processor_info.edx() & (1 << 9))
  155. m_features |= CPUFeature::APIC;
  156. if (processor_info.edx() & (1 << 11))
  157. handle_edx_bit_11_feature();
  158. if (processor_info.edx() & (1 << 12))
  159. m_features |= CPUFeature::MTRR;
  160. if (processor_info.edx() & (1 << 13))
  161. m_features |= CPUFeature::PGE;
  162. if (processor_info.edx() & (1 << 14))
  163. m_features |= CPUFeature::MCA;
  164. if (processor_info.edx() & (1 << 15))
  165. m_features |= CPUFeature::CMOV;
  166. if (processor_info.edx() & (1 << 16))
  167. m_features |= CPUFeature::PAT;
  168. if (processor_info.edx() & (1 << 17))
  169. m_features |= CPUFeature::PSE36;
  170. if (processor_info.edx() & (1 << 18))
  171. m_features |= CPUFeature::PSN;
  172. if (processor_info.edx() & (1 << 19))
  173. m_features |= CPUFeature::CLFLUSH;
  174. if (processor_info.edx() & (1 << 21))
  175. m_features |= CPUFeature::DS;
  176. if (processor_info.edx() & (1 << 22))
  177. m_features |= CPUFeature::ACPI;
  178. if (processor_info.edx() & (1 << 23))
  179. m_features |= CPUFeature::MMX;
  180. if (processor_info.edx() & (1 << 24))
  181. m_features |= CPUFeature::FXSR;
  182. if (processor_info.edx() & (1 << 25))
  183. m_features |= CPUFeature::SSE;
  184. if (processor_info.edx() & (1 << 26))
  185. m_features |= CPUFeature::SSE2;
  186. if (processor_info.edx() & (1 << 27))
  187. m_features |= CPUFeature::SS;
  188. if (processor_info.edx() & (1 << 28))
  189. m_features |= CPUFeature::HTT;
  190. if (processor_info.edx() & (1 << 29))
  191. m_features |= CPUFeature::TM;
  192. if (processor_info.edx() & (1 << 30))
  193. m_features |= CPUFeature::IA64;
  194. if (processor_info.edx() & (1 << 31))
  195. m_features |= CPUFeature::PBE;
  196. CPUID extended_features(0x7);
  197. if (extended_features.ebx() & (1 << 0))
  198. m_features |= CPUFeature::FSGSBASE;
  199. if (extended_features.ebx() & (1 << 1))
  200. m_features |= CPUFeature::TSC_ADJUST;
  201. if (extended_features.ebx() & (1 << 2))
  202. m_features |= CPUFeature::SGX;
  203. if (extended_features.ebx() & (1 << 3))
  204. m_features |= CPUFeature::BMI1;
  205. if (extended_features.ebx() & (1 << 4))
  206. m_features |= CPUFeature::HLE;
  207. if (extended_features.ebx() & (1 << 5))
  208. m_features |= CPUFeature::AVX2;
  209. if (extended_features.ebx() & (1 << 6))
  210. m_features |= CPUFeature::FDP_EXCPTN_ONLY;
  211. if (extended_features.ebx() & (1 << 7))
  212. m_features |= CPUFeature::SMEP;
  213. if (extended_features.ebx() & (1 << 8))
  214. m_features |= CPUFeature::BMI2;
  215. if (extended_features.ebx() & (1 << 9))
  216. m_features |= CPUFeature::ERMS;
  217. if (extended_features.ebx() & (1 << 10))
  218. m_features |= CPUFeature::INVPCID;
  219. if (extended_features.ebx() & (1 << 11))
  220. m_features |= CPUFeature::RTM;
  221. if (extended_features.ebx() & (1 << 12))
  222. m_features |= CPUFeature::PQM;
  223. if (extended_features.ebx() & (1 << 13))
  224. m_features |= CPUFeature::ZERO_FCS_FDS;
  225. if (extended_features.ebx() & (1 << 14))
  226. m_features |= CPUFeature::MPX;
  227. if (extended_features.ebx() & (1 << 15))
  228. m_features |= CPUFeature::PQE;
  229. if (extended_features.ebx() & (1 << 16))
  230. m_features |= CPUFeature::AVX512_F;
  231. if (extended_features.ebx() & (1 << 17))
  232. m_features |= CPUFeature::AVX512_DQ;
  233. if (extended_features.ebx() & (1 << 18))
  234. m_features |= CPUFeature::RDSEED;
  235. if (extended_features.ebx() & (1 << 19))
  236. m_features |= CPUFeature::ADX;
  237. if (extended_features.ebx() & (1 << 20))
  238. m_features |= CPUFeature::SMAP;
  239. if (extended_features.ebx() & (1 << 21))
  240. m_features |= CPUFeature::AVX512_IFMA;
  241. if (extended_features.ebx() & (1 << 22))
  242. m_features |= CPUFeature::PCOMMIT;
  243. if (extended_features.ebx() & (1 << 23))
  244. m_features |= CPUFeature::CLFLUSHOPT;
  245. if (extended_features.ebx() & (1 << 24))
  246. m_features |= CPUFeature::CLWB;
  247. if (extended_features.ebx() & (1 << 25))
  248. m_features |= CPUFeature::INTEL_PT;
  249. if (extended_features.ebx() & (1 << 26))
  250. m_features |= CPUFeature::AVX512_PF;
  251. if (extended_features.ebx() & (1 << 27))
  252. m_features |= CPUFeature::AVX512_ER;
  253. if (extended_features.ebx() & (1 << 28))
  254. m_features |= CPUFeature::AVX512_CD;
  255. if (extended_features.ebx() & (1 << 29))
  256. m_features |= CPUFeature::SHA;
  257. if (extended_features.ebx() & (1 << 30))
  258. m_features |= CPUFeature::AVX512_BW;
  259. if (extended_features.ebx() & (1 << 31))
  260. m_features |= CPUFeature::AVX512_VL;
  261. if (extended_features.ecx() & (1 << 0))
  262. m_features |= CPUFeature::PREFETCHWT1;
  263. if (extended_features.ecx() & (1 << 1))
  264. m_features |= CPUFeature::AVX512_VBMI;
  265. if (extended_features.ecx() & (1 << 2))
  266. m_features |= CPUFeature::UMIP;
  267. if (extended_features.ecx() & (1 << 3))
  268. m_features |= CPUFeature::PKU;
  269. if (extended_features.ecx() & (1 << 4))
  270. m_features |= CPUFeature::OSPKE;
  271. if (extended_features.ecx() & (1 << 5))
  272. m_features |= CPUFeature::WAITPKG;
  273. if (extended_features.ecx() & (1 << 6))
  274. m_features |= CPUFeature::AVX512_VBMI2;
  275. if (extended_features.ecx() & (1 << 7))
  276. m_features |= CPUFeature::CET_SS;
  277. if (extended_features.ecx() & (1 << 8))
  278. m_features |= CPUFeature::GFNI;
  279. if (extended_features.ecx() & (1 << 9))
  280. m_features |= CPUFeature::VAES;
  281. if (extended_features.ecx() & (1 << 10))
  282. m_features |= CPUFeature::VPCLMULQDQ;
  283. if (extended_features.ecx() & (1 << 11))
  284. m_features |= CPUFeature::AVX512_VNNI;
  285. if (extended_features.ecx() & (1 << 12))
  286. m_features |= CPUFeature::AVX512_BITALG;
  287. if (extended_features.ecx() & (1 << 13))
  288. m_features |= CPUFeature::TME_EN;
  289. if (extended_features.ecx() & (1 << 14))
  290. m_features |= CPUFeature::AVX512_VPOPCNTDQ;
  291. if (extended_features.ecx() & (1 << 16))
  292. m_features |= CPUFeature::INTEL_5_LEVEL_PAGING;
  293. if (extended_features.ecx() & (1 << 22))
  294. m_features |= CPUFeature::RDPID;
  295. if (extended_features.ecx() & (1 << 23))
  296. m_features |= CPUFeature::KL;
  297. if (extended_features.ecx() & (1 << 25))
  298. m_features |= CPUFeature::CLDEMOTE;
  299. if (extended_features.ecx() & (1 << 27))
  300. m_features |= CPUFeature::MOVDIRI;
  301. if (extended_features.ecx() & (1 << 28))
  302. m_features |= CPUFeature::MOVDIR64B;
  303. if (extended_features.ecx() & (1 << 29))
  304. m_features |= CPUFeature::ENQCMD;
  305. if (extended_features.ecx() & (1 << 30))
  306. m_features |= CPUFeature::SGX_LC;
  307. if (extended_features.ecx() & (1 << 31))
  308. m_features |= CPUFeature::PKS;
  309. if (extended_features.edx() & (1 << 2))
  310. m_features |= CPUFeature::AVX512_4VNNIW;
  311. if (extended_features.edx() & (1 << 3))
  312. m_features |= CPUFeature::AVX512_4FMAPS;
  313. if (extended_features.edx() & (1 << 4))
  314. m_features |= CPUFeature::FSRM;
  315. if (extended_features.edx() & (1 << 8))
  316. m_features |= CPUFeature::AVX512_VP2INTERSECT;
  317. if (extended_features.edx() & (1 << 9))
  318. m_features |= CPUFeature::SRBDS_CTRL;
  319. if (extended_features.edx() & (1 << 10))
  320. m_features |= CPUFeature::MD_CLEAR;
  321. if (extended_features.edx() & (1 << 11))
  322. m_features |= CPUFeature::RTM_ALWAYS_ABORT;
  323. if (extended_features.edx() & (1 << 13))
  324. m_features |= CPUFeature::TSX_FORCE_ABORT;
  325. if (extended_features.edx() & (1 << 14))
  326. m_features |= CPUFeature::SERIALIZE;
  327. if (extended_features.edx() & (1 << 15))
  328. m_features |= CPUFeature::HYBRID;
  329. if (extended_features.edx() & (1 << 16))
  330. m_features |= CPUFeature::TSXLDTRK;
  331. if (extended_features.edx() & (1 << 18))
  332. m_features |= CPUFeature::PCONFIG;
  333. if (extended_features.edx() & (1 << 19))
  334. m_features |= CPUFeature::LBR;
  335. if (extended_features.edx() & (1 << 20))
  336. m_features |= CPUFeature::CET_IBT;
  337. if (extended_features.edx() & (1 << 22))
  338. m_features |= CPUFeature::AMX_BF16;
  339. if (extended_features.edx() & (1 << 23))
  340. m_features |= CPUFeature::AVX512_FP16;
  341. if (extended_features.edx() & (1 << 24))
  342. m_features |= CPUFeature::AMX_TILE;
  343. if (extended_features.edx() & (1 << 25))
  344. m_features |= CPUFeature::AMX_INT8;
  345. if (extended_features.edx() & (1 << 26))
  346. m_features |= CPUFeature::SPEC_CTRL;
  347. if (extended_features.edx() & (1 << 27))
  348. m_features |= CPUFeature::STIBP;
  349. if (extended_features.edx() & (1 << 28))
  350. m_features |= CPUFeature::L1D_FLUSH;
  351. if (extended_features.edx() & (1 << 29))
  352. m_features |= CPUFeature::IA32_ARCH_CAPABILITIES;
  353. if (extended_features.edx() & (1 << 30))
  354. m_features |= CPUFeature::IA32_CORE_CAPABILITIES;
  355. if (extended_features.edx() & (1 << 31))
  356. m_features |= CPUFeature::SSBD;
  357. u32 max_extended_leaf = CPUID(0x80000000).eax();
  358. if (max_extended_leaf >= 0x80000001) {
  359. CPUID extended_processor_info(0x80000001);
  360. if (extended_processor_info.ecx() & (1 << 0))
  361. m_features |= CPUFeature::LAHF_LM;
  362. if (extended_processor_info.ecx() & (1 << 1))
  363. m_features |= CPUFeature::CMP_LEGACY;
  364. if (extended_processor_info.ecx() & (1 << 2))
  365. m_features |= CPUFeature::SVM;
  366. if (extended_processor_info.ecx() & (1 << 3))
  367. m_features |= CPUFeature::EXTAPIC;
  368. if (extended_processor_info.ecx() & (1 << 4))
  369. m_features |= CPUFeature::CR8_LEGACY;
  370. if (extended_processor_info.ecx() & (1 << 5))
  371. m_features |= CPUFeature::ABM;
  372. if (extended_processor_info.ecx() & (1 << 6))
  373. m_features |= CPUFeature::SSE4A;
  374. if (extended_processor_info.ecx() & (1 << 7))
  375. m_features |= CPUFeature::MISALIGNSSE;
  376. if (extended_processor_info.ecx() & (1 << 8))
  377. m_features |= CPUFeature::_3DNOWPREFETCH;
  378. if (extended_processor_info.ecx() & (1 << 9))
  379. m_features |= CPUFeature::OSVW;
  380. if (extended_processor_info.ecx() & (1 << 10))
  381. m_features |= CPUFeature::IBS;
  382. if (extended_processor_info.ecx() & (1 << 11))
  383. m_features |= CPUFeature::XOP;
  384. if (extended_processor_info.ecx() & (1 << 12))
  385. m_features |= CPUFeature::SKINIT;
  386. if (extended_processor_info.ecx() & (1 << 13))
  387. m_features |= CPUFeature::WDT;
  388. if (extended_processor_info.ecx() & (1 << 15))
  389. m_features |= CPUFeature::LWP;
  390. if (extended_processor_info.ecx() & (1 << 16))
  391. m_features |= CPUFeature::FMA4;
  392. if (extended_processor_info.ecx() & (1 << 17))
  393. m_features |= CPUFeature::TCE;
  394. if (extended_processor_info.ecx() & (1 << 19))
  395. m_features |= CPUFeature::NODEID_MSR;
  396. if (extended_processor_info.ecx() & (1 << 21))
  397. m_features |= CPUFeature::TBM;
  398. if (extended_processor_info.ecx() & (1 << 22))
  399. m_features |= CPUFeature::TOPOEXT;
  400. if (extended_processor_info.ecx() & (1 << 23))
  401. m_features |= CPUFeature::PERFCTR_CORE;
  402. if (extended_processor_info.ecx() & (1 << 24))
  403. m_features |= CPUFeature::PERFCTR_NB;
  404. if (extended_processor_info.ecx() & (1 << 26))
  405. m_features |= CPUFeature::DBX;
  406. if (extended_processor_info.ecx() & (1 << 27))
  407. m_features |= CPUFeature::PERFTSC;
  408. if (extended_processor_info.ecx() & (1 << 28))
  409. m_features |= CPUFeature::PCX_L2I;
  410. if (extended_processor_info.edx() & (1 << 11))
  411. m_features |= CPUFeature::SYSCALL; // Only available in 64 bit mode
  412. if (extended_processor_info.edx() & (1 << 19))
  413. m_features |= CPUFeature::MP;
  414. if (extended_processor_info.edx() & (1 << 20))
  415. m_features |= CPUFeature::NX;
  416. if (extended_processor_info.edx() & (1 << 22))
  417. m_features |= CPUFeature::MMXEXT;
  418. if (extended_processor_info.edx() & (1 << 23))
  419. m_features |= CPUFeature::RDTSCP;
  420. if (extended_processor_info.edx() & (1 << 25))
  421. m_features |= CPUFeature::FXSR_OPT;
  422. if (extended_processor_info.edx() & (1 << 26))
  423. m_features |= CPUFeature::PDPE1GB;
  424. if (extended_processor_info.edx() & (1 << 27))
  425. m_features |= CPUFeature::RDTSCP;
  426. if (extended_processor_info.edx() & (1 << 29))
  427. m_features |= CPUFeature::LM;
  428. if (extended_processor_info.edx() & (1 << 30))
  429. m_features |= CPUFeature::_3DNOWEXT;
  430. if (extended_processor_info.edx() & (1 << 31))
  431. m_features |= CPUFeature::_3DNOW;
  432. }
  433. if (max_extended_leaf >= 0x80000007) {
  434. CPUID cpuid(0x80000007);
  435. if (cpuid.edx() & (1 << 8)) {
  436. m_features |= CPUFeature::CONSTANT_TSC;
  437. m_features |= CPUFeature::NONSTOP_TSC;
  438. }
  439. }
  440. #if ARCH(X86_64)
  441. m_has_qemu_hvf_quirk = false;
  442. #endif
  443. if (max_extended_leaf >= 0x80000008) {
  444. // CPUID.80000008H:EAX[7:0] reports the physical-address width supported by the processor.
  445. CPUID cpuid(0x80000008);
  446. m_physical_address_bit_width = cpuid.eax() & 0xff;
  447. // CPUID.80000008H:EAX[15:8] reports the linear-address width supported by the processor.
  448. m_virtual_address_bit_width = (cpuid.eax() >> 8) & 0xff;
  449. } else {
  450. // For processors that do not support CPUID function 80000008H, the width is generally 36 if CPUID.01H:EDX.PAE [bit 6] = 1 and 32 otherwise.
  451. m_physical_address_bit_width = has_feature(CPUFeature::PAE) ? 36 : 32;
  452. // Processors that do not support CPUID function 80000008H, support a linear-address width of 32.
  453. m_virtual_address_bit_width = 32;
  454. #if ARCH(X86_64)
  455. // Workaround QEMU hypervisor.framework bug
  456. // https://gitlab.com/qemu-project/qemu/-/issues/664
  457. //
  458. // We detect this as follows:
  459. // * We're in a hypervisor
  460. // * hypervisor_leaf_range is null under Hypervisor.framework
  461. // * m_physical_address_bit_width is 36 bits
  462. if (has_feature(CPUFeature::HYPERVISOR)) {
  463. CPUID hypervisor_leaf_range(0x40000000);
  464. if (!hypervisor_leaf_range.ebx() && m_physical_address_bit_width == 36) {
  465. m_has_qemu_hvf_quirk = true;
  466. m_virtual_address_bit_width = 48;
  467. }
  468. }
  469. #endif
  470. }
  471. }
  472. UNMAP_AFTER_INIT void Processor::cpu_setup()
  473. {
  474. // NOTE: This is called during Processor::early_initialize, we cannot
  475. // safely log at this point because we don't have kmalloc
  476. // initialized yet!
  477. cpu_detect();
  478. if (has_feature(CPUFeature::SSE)) {
  479. // enter_thread_context() assumes that if a x86 CPU supports SSE then it also supports FXSR.
  480. // SSE support without FXSR is an extremely unlikely scenario, so let's be pragmatic about it.
  481. VERIFY(has_feature(CPUFeature::FXSR));
  482. sse_init();
  483. }
  484. write_cr0(read_cr0() | 0x00010000);
  485. if (has_feature(CPUFeature::PGE)) {
  486. // Turn on CR4.PGE so the CPU will respect the G bit in page tables.
  487. write_cr4(read_cr4() | 0x80);
  488. }
  489. if (has_feature(CPUFeature::NX)) {
  490. // Turn on IA32_EFER.NXE
  491. MSR ia32_efer(MSR_IA32_EFER);
  492. ia32_efer.set(ia32_efer.get() | 0x800);
  493. }
  494. if (has_feature(CPUFeature::PAT)) {
  495. MSR ia32_pat(MSR_IA32_PAT);
  496. // Set PA4 to Write Comine. This allows us to
  497. // use this mode by only setting the bit in the PTE
  498. // and leaving all other bits in the upper levels unset,
  499. // which maps to setting bit 3 of the index, resulting
  500. // in the index value 0 or 4.
  501. u64 pat = ia32_pat.get() & ~(0x7ull << 32);
  502. pat |= 0x1ull << 32; // set WC mode for PA4
  503. ia32_pat.set(pat);
  504. }
  505. if (has_feature(CPUFeature::SMEP)) {
  506. // Turn on CR4.SMEP
  507. write_cr4(read_cr4() | 0x100000);
  508. }
  509. if (has_feature(CPUFeature::SMAP)) {
  510. // Turn on CR4.SMAP
  511. write_cr4(read_cr4() | 0x200000);
  512. }
  513. if (has_feature(CPUFeature::UMIP)) {
  514. write_cr4(read_cr4() | 0x800);
  515. }
  516. if (has_feature(CPUFeature::TSC)) {
  517. write_cr4(read_cr4() | 0x4);
  518. }
  519. if (has_feature(CPUFeature::XSAVE)) {
  520. // Turn on CR4.OSXSAVE
  521. write_cr4(read_cr4() | 0x40000);
  522. // According to the Intel manual: "After reset, all bits (except bit 0) in XCR0 are cleared to zero; XCR0[0] is set to 1."
  523. // Sadly we can't trust this, for example VirtualBox starts with bits 0-4 set, so let's do it ourselves.
  524. write_xcr0(0x1);
  525. if (has_feature(CPUFeature::AVX)) {
  526. // Turn on SSE, AVX and x87 flags
  527. write_xcr0(read_xcr0() | SIMD::StateComponent::AVX | SIMD::StateComponent::SSE | SIMD::StateComponent::X87);
  528. }
  529. }
  530. #if ARCH(X86_64)
  531. // x86_64 processors must support the syscall feature.
  532. VERIFY(has_feature(CPUFeature::SYSCALL));
  533. MSR efer_msr(MSR_EFER);
  534. efer_msr.set(efer_msr.get() | 1u);
  535. // Write code and stack selectors to the STAR MSR. The first value stored in bits 63:48 controls the sysret CS (value + 0x10) and SS (value + 0x8),
  536. // and the value stored in bits 47:32 controls the syscall CS (value) and SS (value + 0x8).
  537. u64 star = 0;
  538. star |= 0x13ul << 48u;
  539. star |= 0x08ul << 32u;
  540. MSR star_msr(MSR_STAR);
  541. star_msr.set(star);
  542. // Write the syscall entry point to the LSTAR MSR.
  543. MSR lstar_msr(MSR_LSTAR);
  544. lstar_msr.set(reinterpret_cast<u64>(&syscall_entry));
  545. // Write the SFMASK MSR. This MSR controls which bits of rflags are masked when a syscall instruction is executed -
  546. // if a bit is set in sfmask, the corresponding bit in rflags is cleared. The value set here clears most of rflags,
  547. // but keeps the reserved and virtualization bits intact. The userspace rflags value is saved in r11 by syscall.
  548. constexpr u64 rflags_mask = 0x257fd5u;
  549. MSR sfmask_msr(MSR_SFMASK);
  550. sfmask_msr.set(rflags_mask);
  551. #endif
  552. // Query OS-enabled CPUID features again, and set the flags if needed.
  553. CPUID processor_info(0x1);
  554. if (processor_info.ecx() & (1 << 27))
  555. m_features |= CPUFeature::OSXSAVE;
  556. CPUID extended_features(0x7);
  557. if (extended_features.ecx() & (1 << 4))
  558. m_features |= CPUFeature::OSPKE;
  559. }
  560. UNMAP_AFTER_INIT void Processor::early_initialize(u32 cpu)
  561. {
  562. m_self = this;
  563. m_cpu = cpu;
  564. m_in_irq = 0;
  565. m_in_critical = 0;
  566. m_invoke_scheduler_async = false;
  567. m_scheduler_initialized = false;
  568. m_in_scheduler = true;
  569. m_message_queue = nullptr;
  570. m_idle_thread = nullptr;
  571. m_current_thread = nullptr;
  572. m_info = nullptr;
  573. m_halt_requested = false;
  574. if (cpu == 0) {
  575. s_smp_enabled = false;
  576. g_total_processors.store(1u, AK::MemoryOrder::memory_order_release);
  577. } else {
  578. g_total_processors.fetch_add(1u, AK::MemoryOrder::memory_order_acq_rel);
  579. }
  580. deferred_call_pool_init();
  581. cpu_setup();
  582. gdt_init();
  583. VERIFY(is_initialized()); // sanity check
  584. VERIFY(&current() == this); // sanity check
  585. }
  586. UNMAP_AFTER_INIT void Processor::initialize(u32 cpu)
  587. {
  588. VERIFY(m_self == this);
  589. VERIFY(&current() == this); // sanity check
  590. m_info = new ProcessorInfo(*this);
  591. dmesgln("CPU[{}]: Supported features: {}", current_id(), m_info->features_string());
  592. if (!has_feature(CPUFeature::RDRAND))
  593. dmesgln("CPU[{}]: No RDRAND support detected, randomness will be poor", current_id());
  594. dmesgln("CPU[{}]: Physical address bit width: {}", current_id(), m_physical_address_bit_width);
  595. dmesgln("CPU[{}]: Virtual address bit width: {}", current_id(), m_virtual_address_bit_width);
  596. #if ARCH(X86_64)
  597. if (m_has_qemu_hvf_quirk)
  598. dmesgln("CPU[{}]: Applied correction for QEMU Hypervisor.framework quirk", current_id());
  599. #endif
  600. if (cpu == 0)
  601. idt_init();
  602. else
  603. flush_idt();
  604. if (cpu == 0) {
  605. VERIFY((FlatPtr(&s_clean_fpu_state) & 0xF) == 0);
  606. asm volatile("fninit");
  607. // Initialize AVX state
  608. if (has_feature(CPUFeature::XSAVE | CPUFeature::AVX)) {
  609. asm volatile("xsave %0\n"
  610. : "=m"(s_clean_fpu_state)
  611. : "a"(static_cast<u32>(SIMD::StateComponent::AVX | SIMD::StateComponent::SSE | SIMD::StateComponent::X87)), "d"(0u));
  612. } else if (has_feature(CPUFeature::FXSR)) {
  613. asm volatile("fxsave %0"
  614. : "=m"(s_clean_fpu_state));
  615. } else {
  616. asm volatile("fnsave %0"
  617. : "=m"(s_clean_fpu_state));
  618. }
  619. if (has_feature(CPUFeature::HYPERVISOR))
  620. detect_hypervisor();
  621. }
  622. {
  623. // We need to prevent races between APs starting up at the same time
  624. VERIFY(cpu < s_processors.size());
  625. s_processors[cpu] = this;
  626. }
  627. }
  628. UNMAP_AFTER_INIT void Processor::detect_hypervisor()
  629. {
  630. CPUID hypervisor_leaf_range(0x40000000);
  631. auto hypervisor_vendor_id_string = m_info->hypervisor_vendor_id_string();
  632. dmesgln("CPU[{}]: CPUID hypervisor signature '{}', max leaf {:#x}", current_id(), hypervisor_vendor_id_string, hypervisor_leaf_range.eax());
  633. if (hypervisor_vendor_id_string == "Microsoft Hv"sv)
  634. detect_hypervisor_hyperv(hypervisor_leaf_range);
  635. }
  636. UNMAP_AFTER_INIT void Processor::detect_hypervisor_hyperv(CPUID const& hypervisor_leaf_range)
  637. {
  638. if (hypervisor_leaf_range.eax() < 0x40000001)
  639. return;
  640. CPUID hypervisor_interface(0x40000001);
  641. // Get signature of hypervisor interface.
  642. alignas(sizeof(u32)) char interface_signature_buffer[5];
  643. *reinterpret_cast<u32*>(interface_signature_buffer) = hypervisor_interface.eax();
  644. interface_signature_buffer[4] = '\0';
  645. StringView hyperv_interface_signature(interface_signature_buffer);
  646. dmesgln("CPU[{}]: Hyper-V interface signature '{}' ({:#x})", current_id(), hyperv_interface_signature, hypervisor_interface.eax());
  647. if (hypervisor_leaf_range.eax() < 0x40000001)
  648. return;
  649. CPUID hypervisor_sysid(0x40000002);
  650. dmesgln("CPU[{}]: Hyper-V system identity {}.{}, build number {}", current_id(), hypervisor_sysid.ebx() >> 16, hypervisor_sysid.ebx() & 0xFFFF, hypervisor_sysid.eax());
  651. if (hypervisor_leaf_range.eax() < 0x40000005 || hyperv_interface_signature != "Hv#1"sv)
  652. return;
  653. dmesgln("CPU[{}]: Hyper-V hypervisor detected", current_id());
  654. // TODO: Actually do something with Hyper-V.
  655. }
  656. void Processor::write_raw_gdt_entry(u16 selector, u32 low, u32 high)
  657. {
  658. u16 i = (selector & 0xfffc) >> 3;
  659. u32 prev_gdt_length = m_gdt_length;
  660. if (i >= m_gdt_length) {
  661. m_gdt_length = i + 1;
  662. VERIFY(m_gdt_length <= sizeof(m_gdt) / sizeof(m_gdt[0]));
  663. m_gdtr.limit = (m_gdt_length + 1) * 8 - 1;
  664. }
  665. m_gdt[i].low = low;
  666. m_gdt[i].high = high;
  667. // clear selectors we may have skipped
  668. for (auto j = prev_gdt_length; j < i; ++j) {
  669. m_gdt[j].low = 0;
  670. m_gdt[j].high = 0;
  671. }
  672. }
  673. void Processor::write_gdt_entry(u16 selector, Descriptor& descriptor)
  674. {
  675. write_raw_gdt_entry(selector, descriptor.low, descriptor.high);
  676. }
  677. Descriptor& Processor::get_gdt_entry(u16 selector)
  678. {
  679. u16 i = (selector & 0xfffc) >> 3;
  680. return *(Descriptor*)(&m_gdt[i]);
  681. }
  682. void Processor::flush_gdt()
  683. {
  684. m_gdtr.address = m_gdt;
  685. m_gdtr.limit = (m_gdt_length * 8) - 1;
  686. asm volatile("lgdt %0" ::"m"(m_gdtr)
  687. : "memory");
  688. }
  689. DescriptorTablePointer const& Processor::get_gdtr()
  690. {
  691. return m_gdtr;
  692. }
  693. ErrorOr<Vector<FlatPtr, 32>> Processor::capture_stack_trace(Thread& thread, size_t max_frames)
  694. {
  695. FlatPtr frame_ptr = 0, ip = 0;
  696. Vector<FlatPtr, 32> stack_trace;
  697. auto walk_stack = [&](FlatPtr stack_ptr) -> ErrorOr<void> {
  698. constexpr size_t max_stack_frames = 4096;
  699. bool is_walking_userspace_stack = false;
  700. TRY(stack_trace.try_append(ip));
  701. size_t count = 1;
  702. while (stack_ptr && stack_trace.size() < max_stack_frames) {
  703. FlatPtr retaddr;
  704. count++;
  705. if (max_frames != 0 && count > max_frames)
  706. break;
  707. if (!Memory::is_user_address(VirtualAddress { stack_ptr })) {
  708. if (is_walking_userspace_stack) {
  709. dbgln("SHENANIGANS! Userspace stack points back into kernel memory");
  710. break;
  711. }
  712. } else {
  713. is_walking_userspace_stack = true;
  714. }
  715. if (Memory::is_user_range(VirtualAddress(stack_ptr), sizeof(FlatPtr) * 2)) {
  716. if (copy_from_user(&retaddr, &((FlatPtr*)stack_ptr)[1]).is_error() || !retaddr)
  717. break;
  718. TRY(stack_trace.try_append(retaddr));
  719. if (copy_from_user(&stack_ptr, (FlatPtr*)stack_ptr).is_error())
  720. break;
  721. } else {
  722. void* fault_at;
  723. if (!safe_memcpy(&retaddr, &((FlatPtr*)stack_ptr)[1], sizeof(FlatPtr), fault_at) || !retaddr)
  724. break;
  725. TRY(stack_trace.try_append(retaddr));
  726. if (!safe_memcpy(&stack_ptr, (FlatPtr*)stack_ptr, sizeof(FlatPtr), fault_at))
  727. break;
  728. }
  729. }
  730. return {};
  731. };
  732. auto capture_current_thread = [&]() {
  733. frame_ptr = (FlatPtr)__builtin_frame_address(0);
  734. ip = (FlatPtr)__builtin_return_address(0);
  735. return walk_stack(frame_ptr);
  736. };
  737. // Since the thread may be running on another processor, there
  738. // is a chance a context switch may happen while we're trying
  739. // to get it. It also won't be entirely accurate and merely
  740. // reflect the status at the last context switch.
  741. SpinlockLocker lock(g_scheduler_lock);
  742. if (&thread == Processor::current_thread()) {
  743. VERIFY(thread.state() == Thread::State::Running);
  744. // Leave the scheduler lock. If we trigger page faults we may
  745. // need to be preempted. Since this is our own thread it won't
  746. // cause any problems as the stack won't change below this frame.
  747. lock.unlock();
  748. TRY(capture_current_thread());
  749. } else if (thread.is_active()) {
  750. VERIFY(thread.cpu() != Processor::current_id());
  751. // If this is the case, the thread is currently running
  752. // on another processor. We can't trust the kernel stack as
  753. // it may be changing at any time. We need to probably send
  754. // an IPI to that processor, have it walk the stack and wait
  755. // until it returns the data back to us
  756. auto& proc = Processor::current();
  757. ErrorOr<void> result;
  758. smp_unicast(
  759. thread.cpu(),
  760. [&]() {
  761. dbgln("CPU[{}] getting stack for cpu #{}", Processor::current_id(), proc.id());
  762. ScopedAddressSpaceSwitcher switcher(thread.process());
  763. VERIFY(&Processor::current() != &proc);
  764. VERIFY(&thread == Processor::current_thread());
  765. // NOTE: Because the other processor is still holding the
  766. // scheduler lock while waiting for this callback to finish,
  767. // the current thread on the target processor cannot change
  768. // TODO: What to do about page faults here? We might deadlock
  769. // because the other processor is still holding the
  770. // scheduler lock...
  771. result = capture_current_thread();
  772. },
  773. false);
  774. TRY(result);
  775. } else {
  776. switch (thread.state()) {
  777. case Thread::State::Running:
  778. VERIFY_NOT_REACHED(); // should have been handled above
  779. case Thread::State::Runnable:
  780. case Thread::State::Stopped:
  781. case Thread::State::Blocked:
  782. case Thread::State::Dying:
  783. case Thread::State::Dead: {
  784. // We need to retrieve ebp from what was last pushed to the kernel
  785. // stack. Before switching out of that thread, it switch_context
  786. // pushed the callee-saved registers, and the last of them happens
  787. // to be ebp.
  788. ScopedAddressSpaceSwitcher switcher(thread.process());
  789. auto& regs = thread.regs();
  790. auto* stack_top = reinterpret_cast<FlatPtr*>(regs.sp());
  791. if (Memory::is_user_range(VirtualAddress(stack_top), sizeof(FlatPtr))) {
  792. if (copy_from_user(&frame_ptr, &((FlatPtr*)stack_top)[0]).is_error())
  793. frame_ptr = 0;
  794. } else {
  795. void* fault_at;
  796. if (!safe_memcpy(&frame_ptr, &((FlatPtr*)stack_top)[0], sizeof(FlatPtr), fault_at))
  797. frame_ptr = 0;
  798. }
  799. ip = regs.ip();
  800. // TODO: We need to leave the scheduler lock here, but we also
  801. // need to prevent the target thread from being run while
  802. // we walk the stack
  803. lock.unlock();
  804. TRY(walk_stack(frame_ptr));
  805. break;
  806. }
  807. default:
  808. dbgln("Cannot capture stack trace for thread {} in state {}", thread, thread.state_string());
  809. break;
  810. }
  811. }
  812. return stack_trace;
  813. }
  814. ProcessorContainer& Processor::processors()
  815. {
  816. return s_processors;
  817. }
  818. Processor& Processor::by_id(u32 id)
  819. {
  820. return *s_processors[id];
  821. }
  822. void Processor::enter_trap(TrapFrame& trap, bool raise_irq)
  823. {
  824. VERIFY_INTERRUPTS_DISABLED();
  825. VERIFY(&Processor::current() == this);
  826. trap.prev_irq_level = m_in_irq;
  827. if (raise_irq)
  828. m_in_irq++;
  829. auto* current_thread = Processor::current_thread();
  830. if (current_thread) {
  831. auto& current_trap = current_thread->current_trap();
  832. trap.next_trap = current_trap;
  833. current_trap = &trap;
  834. // The cs register of this trap tells us where we will return back to
  835. auto new_previous_mode = ((trap.regs->cs & 3) != 0) ? Thread::PreviousMode::UserMode : Thread::PreviousMode::KernelMode;
  836. if (current_thread->set_previous_mode(new_previous_mode) && trap.prev_irq_level == 0) {
  837. current_thread->update_time_scheduled(Scheduler::current_time(), new_previous_mode == Thread::PreviousMode::KernelMode, false);
  838. }
  839. } else {
  840. trap.next_trap = nullptr;
  841. }
  842. }
  843. void Processor::exit_trap(TrapFrame& trap)
  844. {
  845. VERIFY_INTERRUPTS_DISABLED();
  846. VERIFY(&Processor::current() == this);
  847. // Temporarily enter a critical section. This is to prevent critical
  848. // sections entered and left within e.g. smp_process_pending_messages
  849. // to trigger a context switch while we're executing this function
  850. // See the comment at the end of the function why we don't use
  851. // ScopedCritical here.
  852. m_in_critical = m_in_critical + 1;
  853. VERIFY(m_in_irq >= trap.prev_irq_level);
  854. m_in_irq = trap.prev_irq_level;
  855. if (s_smp_enabled)
  856. smp_process_pending_messages();
  857. // Process the deferred call queue. Among other things, this ensures
  858. // that any pending thread unblocks happen before we enter the scheduler.
  859. deferred_call_execute_pending();
  860. auto* current_thread = Processor::current_thread();
  861. if (current_thread) {
  862. auto& current_trap = current_thread->current_trap();
  863. current_trap = trap.next_trap;
  864. Thread::PreviousMode new_previous_mode;
  865. if (current_trap) {
  866. VERIFY(current_trap->regs);
  867. // If we have another higher level trap then we probably returned
  868. // from an interrupt or irq handler. The cs register of the
  869. // new/higher level trap tells us what the mode prior to it was
  870. new_previous_mode = ((current_trap->regs->cs & 3) != 0) ? Thread::PreviousMode::UserMode : Thread::PreviousMode::KernelMode;
  871. } else {
  872. // If we don't have a higher level trap then we're back in user mode.
  873. // Which means that the previous mode prior to being back in user mode was kernel mode
  874. new_previous_mode = Thread::PreviousMode::KernelMode;
  875. }
  876. if (current_thread->set_previous_mode(new_previous_mode))
  877. current_thread->update_time_scheduled(Scheduler::current_time(), true, false);
  878. }
  879. VERIFY_INTERRUPTS_DISABLED();
  880. // Leave the critical section without actually enabling interrupts.
  881. // We don't want context switches to happen until we're explicitly
  882. // triggering a switch in check_invoke_scheduler.
  883. m_in_critical = m_in_critical - 1;
  884. if (!m_in_irq && !m_in_critical)
  885. check_invoke_scheduler();
  886. }
  887. void Processor::check_invoke_scheduler()
  888. {
  889. InterruptDisabler disabler;
  890. VERIFY(!m_in_irq);
  891. VERIFY(!m_in_critical);
  892. VERIFY(&Processor::current() == this);
  893. if (m_invoke_scheduler_async && m_scheduler_initialized) {
  894. m_invoke_scheduler_async = false;
  895. Scheduler::invoke_async();
  896. }
  897. }
  898. void Processor::flush_tlb_local(VirtualAddress vaddr, size_t page_count)
  899. {
  900. auto ptr = vaddr.as_ptr();
  901. while (page_count > 0) {
  902. // clang-format off
  903. asm volatile("invlpg %0"
  904. :
  905. : "m"(*ptr)
  906. : "memory");
  907. // clang-format on
  908. ptr += PAGE_SIZE;
  909. page_count--;
  910. }
  911. }
  912. void Processor::flush_tlb(Memory::PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count)
  913. {
  914. if (s_smp_enabled && (!Memory::is_user_address(vaddr) || Process::current().thread_count() > 1))
  915. smp_broadcast_flush_tlb(page_directory, vaddr, page_count);
  916. else
  917. flush_tlb_local(vaddr, page_count);
  918. }
  919. void Processor::smp_return_to_pool(ProcessorMessage& msg)
  920. {
  921. ProcessorMessage* next = nullptr;
  922. for (;;) {
  923. msg.next = next;
  924. if (s_message_pool.compare_exchange_strong(next, &msg, AK::MemoryOrder::memory_order_acq_rel))
  925. break;
  926. Processor::pause();
  927. }
  928. }
  929. ProcessorMessage& Processor::smp_get_from_pool()
  930. {
  931. ProcessorMessage* msg;
  932. // The assumption is that messages are never removed from the pool!
  933. for (;;) {
  934. msg = s_message_pool.load(AK::MemoryOrder::memory_order_consume);
  935. if (!msg) {
  936. if (!Processor::current().smp_process_pending_messages()) {
  937. Processor::pause();
  938. }
  939. continue;
  940. }
  941. // If another processor were to use this message in the meanwhile,
  942. // "msg" is still valid (because it never gets freed). We'd detect
  943. // this because the expected value "msg" and pool would
  944. // no longer match, and the compare_exchange will fail. But accessing
  945. // "msg->next" is always safe here.
  946. if (s_message_pool.compare_exchange_strong(msg, msg->next, AK::MemoryOrder::memory_order_acq_rel)) {
  947. // We successfully "popped" this available message
  948. break;
  949. }
  950. }
  951. VERIFY(msg != nullptr);
  952. return *msg;
  953. }
  954. u32 Processor::smp_wake_n_idle_processors(u32 wake_count)
  955. {
  956. VERIFY_INTERRUPTS_DISABLED();
  957. VERIFY(wake_count > 0);
  958. if (!s_smp_enabled)
  959. return 0;
  960. // Wake at most N - 1 processors
  961. if (wake_count >= Processor::count()) {
  962. wake_count = Processor::count() - 1;
  963. VERIFY(wake_count > 0);
  964. }
  965. u32 current_id = Processor::current_id();
  966. u32 did_wake_count = 0;
  967. auto& apic = APIC::the();
  968. while (did_wake_count < wake_count) {
  969. // Try to get a set of idle CPUs and flip them to busy
  970. u32 idle_mask = s_idle_cpu_mask.load(AK::MemoryOrder::memory_order_relaxed) & ~(1u << current_id);
  971. u32 idle_count = popcount(idle_mask);
  972. if (idle_count == 0)
  973. break; // No (more) idle processor available
  974. u32 found_mask = 0;
  975. for (u32 i = 0; i < idle_count; i++) {
  976. u32 cpu = bit_scan_forward(idle_mask) - 1;
  977. idle_mask &= ~(1u << cpu);
  978. found_mask |= 1u << cpu;
  979. }
  980. idle_mask = s_idle_cpu_mask.fetch_and(~found_mask, AK::MemoryOrder::memory_order_acq_rel) & found_mask;
  981. if (idle_mask == 0)
  982. continue; // All of them were flipped to busy, try again
  983. idle_count = popcount(idle_mask);
  984. for (u32 i = 0; i < idle_count; i++) {
  985. u32 cpu = bit_scan_forward(idle_mask) - 1;
  986. idle_mask &= ~(1u << cpu);
  987. // Send an IPI to that CPU to wake it up. There is a possibility
  988. // someone else woke it up as well, or that it woke up due to
  989. // a timer interrupt. But we tried hard to avoid this...
  990. apic.send_ipi(cpu);
  991. did_wake_count++;
  992. }
  993. }
  994. return did_wake_count;
  995. }
  996. UNMAP_AFTER_INIT void Processor::smp_enable()
  997. {
  998. size_t msg_pool_size = Processor::count() * 100u;
  999. size_t msg_entries_cnt = Processor::count();
  1000. auto msgs = new ProcessorMessage[msg_pool_size];
  1001. auto msg_entries = new ProcessorMessageEntry[msg_pool_size * msg_entries_cnt];
  1002. size_t msg_entry_i = 0;
  1003. for (size_t i = 0; i < msg_pool_size; i++, msg_entry_i += msg_entries_cnt) {
  1004. auto& msg = msgs[i];
  1005. msg.next = i < msg_pool_size - 1 ? &msgs[i + 1] : nullptr;
  1006. msg.per_proc_entries = &msg_entries[msg_entry_i];
  1007. for (size_t k = 0; k < msg_entries_cnt; k++)
  1008. msg_entries[msg_entry_i + k].msg = &msg;
  1009. }
  1010. s_message_pool.store(&msgs[0], AK::MemoryOrder::memory_order_release);
  1011. // Start sending IPI messages
  1012. s_smp_enabled = true;
  1013. }
  1014. void Processor::smp_cleanup_message(ProcessorMessage& msg)
  1015. {
  1016. switch (msg.type) {
  1017. case ProcessorMessage::Callback:
  1018. msg.callback_value().~Function();
  1019. break;
  1020. default:
  1021. break;
  1022. }
  1023. }
  1024. bool Processor::smp_process_pending_messages()
  1025. {
  1026. VERIFY(s_smp_enabled);
  1027. bool did_process = false;
  1028. enter_critical();
  1029. if (auto pending_msgs = m_message_queue.exchange(nullptr, AK::MemoryOrder::memory_order_acq_rel)) {
  1030. // We pulled the stack of pending messages in LIFO order, so we need to reverse the list first
  1031. auto reverse_list =
  1032. [](ProcessorMessageEntry* list) -> ProcessorMessageEntry* {
  1033. ProcessorMessageEntry* rev_list = nullptr;
  1034. while (list) {
  1035. auto next = list->next;
  1036. list->next = rev_list;
  1037. rev_list = list;
  1038. list = next;
  1039. }
  1040. return rev_list;
  1041. };
  1042. pending_msgs = reverse_list(pending_msgs);
  1043. // now process in the right order
  1044. ProcessorMessageEntry* next_msg;
  1045. for (auto cur_msg = pending_msgs; cur_msg; cur_msg = next_msg) {
  1046. next_msg = cur_msg->next;
  1047. auto msg = cur_msg->msg;
  1048. dbgln_if(SMP_DEBUG, "SMP[{}]: Processing message {}", current_id(), VirtualAddress(msg));
  1049. switch (msg->type) {
  1050. case ProcessorMessage::Callback:
  1051. msg->invoke_callback();
  1052. break;
  1053. case ProcessorMessage::FlushTlb:
  1054. if (Memory::is_user_address(VirtualAddress(msg->flush_tlb.ptr))) {
  1055. // We assume that we don't cross into kernel land!
  1056. VERIFY(Memory::is_user_range(VirtualAddress(msg->flush_tlb.ptr), msg->flush_tlb.page_count * PAGE_SIZE));
  1057. if (read_cr3() != msg->flush_tlb.page_directory->cr3()) {
  1058. // This processor isn't using this page directory right now, we can ignore this request
  1059. dbgln_if(SMP_DEBUG, "SMP[{}]: No need to flush {} pages at {}", current_id(), msg->flush_tlb.page_count, VirtualAddress(msg->flush_tlb.ptr));
  1060. break;
  1061. }
  1062. }
  1063. flush_tlb_local(VirtualAddress(msg->flush_tlb.ptr), msg->flush_tlb.page_count);
  1064. break;
  1065. }
  1066. bool is_async = msg->async; // Need to cache this value *before* dropping the ref count!
  1067. auto prev_refs = msg->refs.fetch_sub(1u, AK::MemoryOrder::memory_order_acq_rel);
  1068. VERIFY(prev_refs != 0);
  1069. if (prev_refs == 1) {
  1070. // All processors handled this. If this is an async message,
  1071. // we need to clean it up and return it to the pool
  1072. if (is_async) {
  1073. smp_cleanup_message(*msg);
  1074. smp_return_to_pool(*msg);
  1075. }
  1076. }
  1077. if (m_halt_requested.load(AK::MemoryOrder::memory_order_relaxed))
  1078. halt_this();
  1079. }
  1080. did_process = true;
  1081. } else if (m_halt_requested.load(AK::MemoryOrder::memory_order_relaxed)) {
  1082. halt_this();
  1083. }
  1084. leave_critical();
  1085. return did_process;
  1086. }
  1087. bool Processor::smp_enqueue_message(ProcessorMessage& msg)
  1088. {
  1089. // Note that it's quite possible that the other processor may pop
  1090. // the queue at any given time. We rely on the fact that the messages
  1091. // are pooled and never get freed!
  1092. auto& msg_entry = msg.per_proc_entries[id()];
  1093. VERIFY(msg_entry.msg == &msg);
  1094. ProcessorMessageEntry* next = nullptr;
  1095. for (;;) {
  1096. msg_entry.next = next;
  1097. if (m_message_queue.compare_exchange_strong(next, &msg_entry, AK::MemoryOrder::memory_order_acq_rel))
  1098. break;
  1099. Processor::pause();
  1100. }
  1101. // If the enqueued message was the only message in the queue when posted,
  1102. // we return true. This is used by callers when deciding whether to generate an IPI.
  1103. return next == nullptr;
  1104. }
  1105. void Processor::smp_broadcast_message(ProcessorMessage& msg)
  1106. {
  1107. auto& current_processor = Processor::current();
  1108. dbgln_if(SMP_DEBUG, "SMP[{}]: Broadcast message {} to cpus: {} processor: {}", current_processor.id(), VirtualAddress(&msg), count(), VirtualAddress(&current_processor));
  1109. msg.refs.store(count() - 1, AK::MemoryOrder::memory_order_release);
  1110. VERIFY(msg.refs > 0);
  1111. bool need_broadcast = false;
  1112. for_each(
  1113. [&](Processor& proc) {
  1114. if (&proc != &current_processor) {
  1115. if (proc.smp_enqueue_message(msg))
  1116. need_broadcast = true;
  1117. }
  1118. });
  1119. // Now trigger an IPI on all other APs (unless all targets already had messages queued)
  1120. if (need_broadcast)
  1121. APIC::the().broadcast_ipi();
  1122. }
  1123. void Processor::smp_broadcast_wait_sync(ProcessorMessage& msg)
  1124. {
  1125. auto& cur_proc = Processor::current();
  1126. VERIFY(!msg.async);
  1127. // If synchronous then we must cleanup and return the message back
  1128. // to the pool. Otherwise, the last processor to complete it will return it
  1129. while (msg.refs.load(AK::MemoryOrder::memory_order_consume) != 0) {
  1130. Processor::pause();
  1131. // We need to process any messages that may have been sent to
  1132. // us while we're waiting. This also checks if another processor
  1133. // may have requested us to halt.
  1134. cur_proc.smp_process_pending_messages();
  1135. }
  1136. smp_cleanup_message(msg);
  1137. smp_return_to_pool(msg);
  1138. }
  1139. void Processor::smp_unicast_message(u32 cpu, ProcessorMessage& msg, bool async)
  1140. {
  1141. auto& current_processor = Processor::current();
  1142. VERIFY(cpu != current_processor.id());
  1143. auto& target_processor = processors()[cpu];
  1144. msg.async = async;
  1145. dbgln_if(SMP_DEBUG, "SMP[{}]: Send message {} to cpu #{} processor: {}", current_processor.id(), VirtualAddress(&msg), cpu, VirtualAddress(&target_processor));
  1146. msg.refs.store(1u, AK::MemoryOrder::memory_order_release);
  1147. if (target_processor->smp_enqueue_message(msg)) {
  1148. APIC::the().send_ipi(cpu);
  1149. }
  1150. if (!async) {
  1151. // If synchronous then we must cleanup and return the message back
  1152. // to the pool. Otherwise, the last processor to complete it will return it
  1153. while (msg.refs.load(AK::MemoryOrder::memory_order_consume) != 0) {
  1154. Processor::pause();
  1155. // We need to process any messages that may have been sent to
  1156. // us while we're waiting. This also checks if another processor
  1157. // may have requested us to halt.
  1158. current_processor.smp_process_pending_messages();
  1159. }
  1160. smp_cleanup_message(msg);
  1161. smp_return_to_pool(msg);
  1162. }
  1163. }
  1164. void Processor::smp_unicast(u32 cpu, Function<void()> callback, bool async)
  1165. {
  1166. auto& msg = smp_get_from_pool();
  1167. msg.type = ProcessorMessage::Callback;
  1168. new (msg.callback_storage) ProcessorMessage::CallbackFunction(move(callback));
  1169. smp_unicast_message(cpu, msg, async);
  1170. }
  1171. void Processor::smp_broadcast_flush_tlb(Memory::PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count)
  1172. {
  1173. auto& msg = smp_get_from_pool();
  1174. msg.async = false;
  1175. msg.type = ProcessorMessage::FlushTlb;
  1176. msg.flush_tlb.page_directory = page_directory;
  1177. msg.flush_tlb.ptr = vaddr.as_ptr();
  1178. msg.flush_tlb.page_count = page_count;
  1179. smp_broadcast_message(msg);
  1180. // While the other processors handle this request, we'll flush ours
  1181. flush_tlb_local(vaddr, page_count);
  1182. // Now wait until everybody is done as well
  1183. smp_broadcast_wait_sync(msg);
  1184. }
  1185. void Processor::smp_broadcast_halt()
  1186. {
  1187. // We don't want to use a message, because this could have been triggered
  1188. // by being out of memory and we might not be able to get a message
  1189. for_each(
  1190. [&](Processor& proc) {
  1191. proc.m_halt_requested.store(true, AK::MemoryOrder::memory_order_release);
  1192. });
  1193. // Now trigger an IPI on all other APs
  1194. APIC::the().broadcast_ipi();
  1195. }
  1196. void Processor::Processor::halt()
  1197. {
  1198. if (s_smp_enabled)
  1199. smp_broadcast_halt();
  1200. halt_this();
  1201. }
  1202. UNMAP_AFTER_INIT void Processor::deferred_call_pool_init()
  1203. {
  1204. size_t pool_count = sizeof(m_deferred_call_pool) / sizeof(m_deferred_call_pool[0]);
  1205. for (size_t i = 0; i < pool_count; i++) {
  1206. auto& entry = m_deferred_call_pool[i];
  1207. entry.next = i < pool_count - 1 ? &m_deferred_call_pool[i + 1] : nullptr;
  1208. new (entry.handler_storage) DeferredCallEntry::HandlerFunction;
  1209. entry.was_allocated = false;
  1210. }
  1211. m_pending_deferred_calls = nullptr;
  1212. m_free_deferred_call_pool_entry = &m_deferred_call_pool[0];
  1213. }
  1214. void Processor::deferred_call_return_to_pool(DeferredCallEntry* entry)
  1215. {
  1216. VERIFY(m_in_critical);
  1217. VERIFY(!entry->was_allocated);
  1218. entry->handler_value() = {};
  1219. entry->next = m_free_deferred_call_pool_entry;
  1220. m_free_deferred_call_pool_entry = entry;
  1221. }
  1222. DeferredCallEntry* Processor::deferred_call_get_free()
  1223. {
  1224. VERIFY(m_in_critical);
  1225. if (m_free_deferred_call_pool_entry) {
  1226. // Fast path, we have an entry in our pool
  1227. auto* entry = m_free_deferred_call_pool_entry;
  1228. m_free_deferred_call_pool_entry = entry->next;
  1229. VERIFY(!entry->was_allocated);
  1230. return entry;
  1231. }
  1232. auto* entry = new DeferredCallEntry;
  1233. new (entry->handler_storage) DeferredCallEntry::HandlerFunction;
  1234. entry->was_allocated = true;
  1235. return entry;
  1236. }
  1237. void Processor::deferred_call_execute_pending()
  1238. {
  1239. VERIFY(m_in_critical);
  1240. if (!m_pending_deferred_calls)
  1241. return;
  1242. auto* pending_list = m_pending_deferred_calls;
  1243. m_pending_deferred_calls = nullptr;
  1244. // We pulled the stack of pending deferred calls in LIFO order, so we need to reverse the list first
  1245. auto reverse_list =
  1246. [](DeferredCallEntry* list) -> DeferredCallEntry* {
  1247. DeferredCallEntry* rev_list = nullptr;
  1248. while (list) {
  1249. auto next = list->next;
  1250. list->next = rev_list;
  1251. rev_list = list;
  1252. list = next;
  1253. }
  1254. return rev_list;
  1255. };
  1256. pending_list = reverse_list(pending_list);
  1257. do {
  1258. pending_list->invoke_handler();
  1259. // Return the entry back to the pool, or free it
  1260. auto* next = pending_list->next;
  1261. if (pending_list->was_allocated) {
  1262. pending_list->handler_value().~Function();
  1263. delete pending_list;
  1264. } else
  1265. deferred_call_return_to_pool(pending_list);
  1266. pending_list = next;
  1267. } while (pending_list);
  1268. }
  1269. void Processor::deferred_call_queue_entry(DeferredCallEntry* entry)
  1270. {
  1271. VERIFY(m_in_critical);
  1272. entry->next = m_pending_deferred_calls;
  1273. m_pending_deferred_calls = entry;
  1274. }
  1275. void Processor::deferred_call_queue(Function<void()> callback)
  1276. {
  1277. // NOTE: If we are called outside of a critical section and outside
  1278. // of an irq handler, the function will be executed before we return!
  1279. ScopedCritical critical;
  1280. auto& cur_proc = Processor::current();
  1281. auto* entry = cur_proc.deferred_call_get_free();
  1282. entry->handler_value() = move(callback);
  1283. cur_proc.deferred_call_queue_entry(entry);
  1284. }
  1285. UNMAP_AFTER_INIT void Processor::gdt_init()
  1286. {
  1287. m_gdt_length = 0;
  1288. m_gdtr.address = nullptr;
  1289. m_gdtr.limit = 0;
  1290. write_raw_gdt_entry(0x0000, 0x00000000, 0x00000000);
  1291. #if ARCH(I386)
  1292. write_raw_gdt_entry(GDT_SELECTOR_CODE0, 0x0000ffff, 0x00cf9a00); // code0
  1293. write_raw_gdt_entry(GDT_SELECTOR_DATA0, 0x0000ffff, 0x00cf9200); // data0
  1294. write_raw_gdt_entry(GDT_SELECTOR_CODE3, 0x0000ffff, 0x00cffa00); // code3
  1295. write_raw_gdt_entry(GDT_SELECTOR_DATA3, 0x0000ffff, 0x00cff200); // data3
  1296. #else
  1297. write_raw_gdt_entry(GDT_SELECTOR_CODE0, 0x0000ffff, 0x00af9a00); // code0
  1298. write_raw_gdt_entry(GDT_SELECTOR_DATA0, 0x0000ffff, 0x00af9200); // data0
  1299. write_raw_gdt_entry(GDT_SELECTOR_DATA3, 0x0000ffff, 0x008ff200); // data3
  1300. write_raw_gdt_entry(GDT_SELECTOR_CODE3, 0x0000ffff, 0x00affa00); // code3
  1301. #endif
  1302. #if ARCH(I386)
  1303. Descriptor tls_descriptor {};
  1304. tls_descriptor.low = tls_descriptor.high = 0;
  1305. tls_descriptor.dpl = 3;
  1306. tls_descriptor.segment_present = 1;
  1307. tls_descriptor.granularity = 0;
  1308. tls_descriptor.operation_size64 = 0;
  1309. tls_descriptor.operation_size32 = 1;
  1310. tls_descriptor.descriptor_type = 1;
  1311. tls_descriptor.type = 2;
  1312. write_gdt_entry(GDT_SELECTOR_TLS, tls_descriptor); // tls3
  1313. Descriptor gs_descriptor {};
  1314. gs_descriptor.set_base(VirtualAddress { this });
  1315. gs_descriptor.set_limit(sizeof(Processor) - 1);
  1316. gs_descriptor.dpl = 0;
  1317. gs_descriptor.segment_present = 1;
  1318. gs_descriptor.granularity = 0;
  1319. gs_descriptor.operation_size64 = 0;
  1320. gs_descriptor.operation_size32 = 1;
  1321. gs_descriptor.descriptor_type = 1;
  1322. gs_descriptor.type = 2;
  1323. write_gdt_entry(GDT_SELECTOR_PROC, gs_descriptor); // gs0
  1324. #endif
  1325. Descriptor tss_descriptor {};
  1326. tss_descriptor.set_base(VirtualAddress { (size_t)&m_tss & 0xffffffff });
  1327. tss_descriptor.set_limit(sizeof(TSS) - 1);
  1328. tss_descriptor.dpl = 0;
  1329. tss_descriptor.segment_present = 1;
  1330. tss_descriptor.granularity = 0;
  1331. tss_descriptor.operation_size64 = 0;
  1332. tss_descriptor.operation_size32 = 1;
  1333. tss_descriptor.descriptor_type = 0;
  1334. tss_descriptor.type = Descriptor::SystemType::AvailableTSS;
  1335. write_gdt_entry(GDT_SELECTOR_TSS, tss_descriptor); // tss
  1336. #if ARCH(X86_64)
  1337. Descriptor tss_descriptor_part2 {};
  1338. tss_descriptor_part2.low = (size_t)&m_tss >> 32;
  1339. write_gdt_entry(GDT_SELECTOR_TSS_PART2, tss_descriptor_part2);
  1340. #endif
  1341. flush_gdt();
  1342. load_task_register(GDT_SELECTOR_TSS);
  1343. #if ARCH(X86_64)
  1344. MSR gs_base(MSR_GS_BASE);
  1345. gs_base.set((u64)this);
  1346. #else
  1347. asm volatile(
  1348. "mov %%ax, %%ds\n"
  1349. "mov %%ax, %%es\n"
  1350. "mov %%ax, %%fs\n"
  1351. "mov %%ax, %%ss\n" ::"a"(GDT_SELECTOR_DATA0)
  1352. : "memory");
  1353. set_gs(GDT_SELECTOR_PROC);
  1354. #endif
  1355. #if ARCH(I386)
  1356. // Make sure CS points to the kernel code descriptor.
  1357. // clang-format off
  1358. asm volatile(
  1359. "ljmpl $" __STRINGIFY(GDT_SELECTOR_CODE0) ", $sanity\n"
  1360. "sanity:\n");
  1361. // clang-format on
  1362. #endif
  1363. }
  1364. extern "C" void context_first_init([[maybe_unused]] Thread* from_thread, [[maybe_unused]] Thread* to_thread, [[maybe_unused]] TrapFrame* trap)
  1365. {
  1366. VERIFY(!are_interrupts_enabled());
  1367. VERIFY(is_kernel_mode());
  1368. dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context <-- from {} {} to {} {} (context_first_init)", VirtualAddress(from_thread), *from_thread, VirtualAddress(to_thread), *to_thread);
  1369. VERIFY(to_thread == Thread::current());
  1370. Scheduler::enter_current(*from_thread);
  1371. auto in_critical = to_thread->saved_critical();
  1372. VERIFY(in_critical > 0);
  1373. Processor::restore_in_critical(in_critical);
  1374. // Since we got here and don't have Scheduler::context_switch in the
  1375. // call stack (because this is the first time we switched into this
  1376. // context), we need to notify the scheduler so that it can release
  1377. // the scheduler lock. We don't want to enable interrupts at this point
  1378. // as we're still in the middle of a context switch. Doing so could
  1379. // trigger a context switch within a context switch, leading to a crash.
  1380. FlatPtr flags = trap->regs->flags();
  1381. Scheduler::leave_on_first_switch(flags & ~0x200);
  1382. }
  1383. extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread)
  1384. {
  1385. VERIFY(from_thread == to_thread || from_thread->state() != Thread::State::Running);
  1386. VERIFY(to_thread->state() == Thread::State::Running);
  1387. bool has_fxsr = Processor::current().has_feature(CPUFeature::FXSR);
  1388. bool has_xsave_avx_support = Processor::current().has_feature(CPUFeature::XSAVE) && Processor::current().has_feature(CPUFeature::AVX);
  1389. Processor::set_current_thread(*to_thread);
  1390. auto& from_regs = from_thread->regs();
  1391. auto& to_regs = to_thread->regs();
  1392. // NOTE: IOPL should never be non-zero in any situation, so let's panic immediately
  1393. // instead of carrying on with elevated I/O privileges.
  1394. VERIFY(get_iopl_from_eflags(to_regs.flags()) == 0);
  1395. if (has_xsave_avx_support) {
  1396. // The specific state components saved correspond to the bits set in the requested-feature bitmap (RFBM), which is the logical-AND of EDX:EAX and XCR0.
  1397. // https://www.moritz.systems/blog/how-debuggers-work-getting-and-setting-x86-registers-part-2/
  1398. asm volatile("xsave %0\n"
  1399. : "=m"(from_thread->fpu_state())
  1400. : "a"(static_cast<u32>(SIMD::StateComponent::AVX | SIMD::StateComponent::SSE | SIMD::StateComponent::X87)), "d"(0u));
  1401. } else if (has_fxsr) {
  1402. asm volatile("fxsave %0"
  1403. : "=m"(from_thread->fpu_state()));
  1404. } else {
  1405. asm volatile("fnsave %0"
  1406. : "=m"(from_thread->fpu_state()));
  1407. }
  1408. #if ARCH(I386)
  1409. from_regs.fs = get_fs();
  1410. from_regs.gs = get_gs();
  1411. set_fs(to_regs.fs);
  1412. set_gs(to_regs.gs);
  1413. #endif
  1414. if (from_thread->process().is_traced())
  1415. read_debug_registers_into(from_thread->debug_register_state());
  1416. if (to_thread->process().is_traced()) {
  1417. write_debug_registers_from(to_thread->debug_register_state());
  1418. } else {
  1419. clear_debug_registers();
  1420. }
  1421. auto& processor = Processor::current();
  1422. #if ARCH(I386)
  1423. auto& tls_descriptor = processor.get_gdt_entry(GDT_SELECTOR_TLS);
  1424. tls_descriptor.set_base(to_thread->thread_specific_data());
  1425. tls_descriptor.set_limit(to_thread->thread_specific_region_size());
  1426. #else
  1427. MSR fs_base_msr(MSR_FS_BASE);
  1428. fs_base_msr.set(to_thread->thread_specific_data().get());
  1429. #endif
  1430. if (from_regs.cr3 != to_regs.cr3)
  1431. write_cr3(to_regs.cr3);
  1432. to_thread->set_cpu(processor.id());
  1433. auto in_critical = to_thread->saved_critical();
  1434. VERIFY(in_critical > 0);
  1435. Processor::restore_in_critical(in_critical);
  1436. if (has_xsave_avx_support)
  1437. asm volatile("xrstor %0" ::"m"(to_thread->fpu_state()), "a"(static_cast<u32>(SIMD::StateComponent::AVX | SIMD::StateComponent::SSE | SIMD::StateComponent::X87)), "d"(0u));
  1438. else if (has_fxsr)
  1439. asm volatile("fxrstor %0" ::"m"(to_thread->fpu_state()));
  1440. else
  1441. asm volatile("frstor %0" ::"m"(to_thread->fpu_state()));
  1442. }
  1443. extern "C" FlatPtr do_init_context(Thread* thread, u32 flags)
  1444. {
  1445. VERIFY_INTERRUPTS_DISABLED();
  1446. thread->regs().set_flags(flags);
  1447. return Processor::current().init_context(*thread, true);
  1448. }
  1449. void Processor::assume_context(Thread& thread, FlatPtr flags)
  1450. {
  1451. dbgln_if(CONTEXT_SWITCH_DEBUG, "Assume context for thread {} {}", VirtualAddress(&thread), thread);
  1452. VERIFY_INTERRUPTS_DISABLED();
  1453. Scheduler::prepare_after_exec();
  1454. // in_critical() should be 2 here. The critical section in Process::exec
  1455. // and then the scheduler lock
  1456. VERIFY(Processor::in_critical() == 2);
  1457. do_assume_context(&thread, flags);
  1458. VERIFY_NOT_REACHED();
  1459. }
  1460. u64 Processor::time_spent_idle() const
  1461. {
  1462. return m_idle_thread->time_in_user() + m_idle_thread->time_in_kernel();
  1463. }
  1464. }