Processor.cpp 60 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650
  1. /*
  2. * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
  3. * Copyright (c) 2022, Linus Groh <linusg@serenityos.org>
  4. * Copyright (c) 2022, the SerenityOS developers.
  5. *
  6. * SPDX-License-Identifier: BSD-2-Clause
  7. */
  8. #include <AK/BuiltinWrappers.h>
  9. #include <AK/Format.h>
  10. #include <AK/StdLibExtras.h>
  11. #include <AK/StringBuilder.h>
  12. #include <AK/Types.h>
  13. #include <Kernel/Interrupts/APIC.h>
  14. #include <Kernel/Process.h>
  15. #include <Kernel/Scheduler.h>
  16. #include <Kernel/Sections.h>
  17. #include <Kernel/StdLib.h>
  18. #include <Kernel/Thread.h>
  19. #include <Kernel/Arch/Processor.h>
  20. #include <Kernel/Arch/ScopedCritical.h>
  21. #include <Kernel/Arch/x86/CPUID.h>
  22. #include <Kernel/Arch/x86/InterruptDisabler.h>
  23. #include <Kernel/Arch/x86/Interrupts.h>
  24. #include <Kernel/Arch/x86/MSR.h>
  25. #include <Kernel/Arch/x86/ProcessorInfo.h>
  26. #include <Kernel/Arch/x86/SafeMem.h>
  27. #include <Kernel/Arch/x86/TrapFrame.h>
  28. #include <Kernel/Memory/PageDirectory.h>
  29. #include <Kernel/Memory/ScopedAddressSpaceSwitcher.h>
  30. namespace Kernel {
  31. READONLY_AFTER_INIT FPUState Processor::s_clean_fpu_state;
  32. READONLY_AFTER_INIT static ProcessorContainer s_processors {};
  33. READONLY_AFTER_INIT Atomic<u32> Processor::g_total_processors;
  34. READONLY_AFTER_INIT static bool volatile s_smp_enabled;
  35. static Atomic<ProcessorMessage*> s_message_pool;
  36. Atomic<u32> Processor::s_idle_cpu_mask { 0 };
  37. // The compiler can't see the calls to these functions inside assembly.
  38. // Declare them, to avoid dead code warnings.
  39. extern "C" void context_first_init(Thread* from_thread, Thread* to_thread, TrapFrame* trap) __attribute__((used));
  40. extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __attribute__((used));
  41. extern "C" FlatPtr do_init_context(Thread* thread, u32 flags) __attribute__((used));
  42. extern "C" void syscall_entry();
  43. bool Processor::is_smp_enabled()
  44. {
  45. return s_smp_enabled;
  46. }
  47. UNMAP_AFTER_INIT static void sse_init()
  48. {
  49. write_cr0((read_cr0() & 0xfffffffbu) | 0x2);
  50. write_cr4(read_cr4() | 0x600);
  51. }
  52. void exit_kernel_thread(void)
  53. {
  54. Thread::current()->exit();
  55. }
  56. UNMAP_AFTER_INIT void Processor::cpu_detect()
  57. {
  58. // NOTE: This is called during Processor::early_initialize, we cannot
  59. // safely log at this point because we don't have kmalloc
  60. // initialized yet!
  61. m_features = CPUFeature::Type(0u);
  62. CPUID processor_info(0x1);
  63. auto handle_edx_bit_11_feature = [&] {
  64. u32 stepping = processor_info.eax() & 0xf;
  65. u32 model = (processor_info.eax() >> 4) & 0xf;
  66. u32 family = (processor_info.eax() >> 8) & 0xf;
  67. // FIXME: I have no clue what these mean or where it's from (the Intel manual I've seen just says EDX[11] is SEP).
  68. // If you do, please convert them to constants or add comments!
  69. if (!(family == 6 && model < 3 && stepping < 3))
  70. m_features |= CPUFeature::SEP;
  71. if ((family == 6 && model >= 3) || (family == 0xf && model >= 0xe))
  72. m_features |= CPUFeature::CONSTANT_TSC;
  73. };
  74. if (processor_info.ecx() & (1 << 0))
  75. m_features |= CPUFeature::SSE3;
  76. if (processor_info.ecx() & (1 << 1))
  77. m_features |= CPUFeature::PCLMULQDQ;
  78. if (processor_info.ecx() & (1 << 2))
  79. m_features |= CPUFeature::DTES64;
  80. if (processor_info.ecx() & (1 << 3))
  81. m_features |= CPUFeature::MONITOR;
  82. if (processor_info.ecx() & (1 << 4))
  83. m_features |= CPUFeature::DS_CPL;
  84. if (processor_info.ecx() & (1 << 5))
  85. m_features |= CPUFeature::VMX;
  86. if (processor_info.ecx() & (1 << 6))
  87. m_features |= CPUFeature::SMX;
  88. if (processor_info.ecx() & (1 << 7))
  89. m_features |= CPUFeature::EST;
  90. if (processor_info.ecx() & (1 << 8))
  91. m_features |= CPUFeature::TM2;
  92. if (processor_info.ecx() & (1 << 9))
  93. m_features |= CPUFeature::SSSE3;
  94. if (processor_info.ecx() & (1 << 10))
  95. m_features |= CPUFeature::CNXT_ID;
  96. if (processor_info.ecx() & (1 << 11))
  97. m_features |= CPUFeature::SDBG;
  98. if (processor_info.ecx() & (1 << 12))
  99. m_features |= CPUFeature::FMA;
  100. if (processor_info.ecx() & (1 << 13))
  101. m_features |= CPUFeature::CX16;
  102. if (processor_info.ecx() & (1 << 14))
  103. m_features |= CPUFeature::XTPR;
  104. if (processor_info.ecx() & (1 << 15))
  105. m_features |= CPUFeature::PDCM;
  106. if (processor_info.ecx() & (1 << 17))
  107. m_features |= CPUFeature::PCID;
  108. if (processor_info.ecx() & (1 << 18))
  109. m_features |= CPUFeature::DCA;
  110. if (processor_info.ecx() & (1 << 19))
  111. m_features |= CPUFeature::SSE4_1;
  112. if (processor_info.ecx() & (1 << 20))
  113. m_features |= CPUFeature::SSE4_2;
  114. if (processor_info.ecx() & (1 << 21))
  115. m_features |= CPUFeature::X2APIC;
  116. if (processor_info.ecx() & (1 << 22))
  117. m_features |= CPUFeature::MOVBE;
  118. if (processor_info.ecx() & (1 << 23))
  119. m_features |= CPUFeature::POPCNT;
  120. if (processor_info.ecx() & (1 << 24))
  121. m_features |= CPUFeature::TSC_DEADLINE;
  122. if (processor_info.ecx() & (1 << 25))
  123. m_features |= CPUFeature::AES;
  124. if (processor_info.ecx() & (1 << 26))
  125. m_features |= CPUFeature::XSAVE;
  126. if (processor_info.ecx() & (1 << 27))
  127. m_features |= CPUFeature::OSXSAVE;
  128. if (processor_info.ecx() & (1 << 28))
  129. m_features |= CPUFeature::AVX;
  130. if (processor_info.ecx() & (1 << 29))
  131. m_features |= CPUFeature::F16C;
  132. if (processor_info.ecx() & (1 << 30))
  133. m_features |= CPUFeature::RDRAND;
  134. if (processor_info.ecx() & (1 << 31))
  135. m_features |= CPUFeature::HYPERVISOR;
  136. if (processor_info.edx() & (1 << 0))
  137. m_features |= CPUFeature::FPU;
  138. if (processor_info.edx() & (1 << 1))
  139. m_features |= CPUFeature::VME;
  140. if (processor_info.edx() & (1 << 2))
  141. m_features |= CPUFeature::DE;
  142. if (processor_info.edx() & (1 << 3))
  143. m_features |= CPUFeature::PSE;
  144. if (processor_info.edx() & (1 << 4))
  145. m_features |= CPUFeature::TSC;
  146. if (processor_info.edx() & (1 << 5))
  147. m_features |= CPUFeature::MSR;
  148. if (processor_info.edx() & (1 << 6))
  149. m_features |= CPUFeature::PAE;
  150. if (processor_info.edx() & (1 << 7))
  151. m_features |= CPUFeature::MCE;
  152. if (processor_info.edx() & (1 << 8))
  153. m_features |= CPUFeature::CX8;
  154. if (processor_info.edx() & (1 << 9))
  155. m_features |= CPUFeature::APIC;
  156. if (processor_info.edx() & (1 << 11))
  157. handle_edx_bit_11_feature();
  158. if (processor_info.edx() & (1 << 12))
  159. m_features |= CPUFeature::MTRR;
  160. if (processor_info.edx() & (1 << 13))
  161. m_features |= CPUFeature::PGE;
  162. if (processor_info.edx() & (1 << 14))
  163. m_features |= CPUFeature::MCA;
  164. if (processor_info.edx() & (1 << 15))
  165. m_features |= CPUFeature::CMOV;
  166. if (processor_info.edx() & (1 << 16))
  167. m_features |= CPUFeature::PAT;
  168. if (processor_info.edx() & (1 << 17))
  169. m_features |= CPUFeature::PSE36;
  170. if (processor_info.edx() & (1 << 18))
  171. m_features |= CPUFeature::PSN;
  172. if (processor_info.edx() & (1 << 19))
  173. m_features |= CPUFeature::CLFLUSH;
  174. if (processor_info.edx() & (1 << 21))
  175. m_features |= CPUFeature::DS;
  176. if (processor_info.edx() & (1 << 22))
  177. m_features |= CPUFeature::ACPI;
  178. if (processor_info.edx() & (1 << 23))
  179. m_features |= CPUFeature::MMX;
  180. if (processor_info.edx() & (1 << 24))
  181. m_features |= CPUFeature::FXSR;
  182. if (processor_info.edx() & (1 << 25))
  183. m_features |= CPUFeature::SSE;
  184. if (processor_info.edx() & (1 << 26))
  185. m_features |= CPUFeature::SSE2;
  186. if (processor_info.edx() & (1 << 27))
  187. m_features |= CPUFeature::SS;
  188. if (processor_info.edx() & (1 << 28))
  189. m_features |= CPUFeature::HTT;
  190. if (processor_info.edx() & (1 << 29))
  191. m_features |= CPUFeature::TM;
  192. if (processor_info.edx() & (1 << 30))
  193. m_features |= CPUFeature::IA64;
  194. if (processor_info.edx() & (1 << 31))
  195. m_features |= CPUFeature::PBE;
  196. CPUID extended_features(0x7);
  197. if (extended_features.ebx() & (1 << 0))
  198. m_features |= CPUFeature::FSGSBASE;
  199. if (extended_features.ebx() & (1 << 1))
  200. m_features |= CPUFeature::TSC_ADJUST;
  201. if (extended_features.ebx() & (1 << 2))
  202. m_features |= CPUFeature::SGX;
  203. if (extended_features.ebx() & (1 << 3))
  204. m_features |= CPUFeature::BMI1;
  205. if (extended_features.ebx() & (1 << 4))
  206. m_features |= CPUFeature::HLE;
  207. if (extended_features.ebx() & (1 << 5))
  208. m_features |= CPUFeature::AVX2;
  209. if (extended_features.ebx() & (1 << 6))
  210. m_features |= CPUFeature::FDP_EXCPTN_ONLY;
  211. if (extended_features.ebx() & (1 << 7))
  212. m_features |= CPUFeature::SMEP;
  213. if (extended_features.ebx() & (1 << 8))
  214. m_features |= CPUFeature::BMI2;
  215. if (extended_features.ebx() & (1 << 9))
  216. m_features |= CPUFeature::ERMS;
  217. if (extended_features.ebx() & (1 << 10))
  218. m_features |= CPUFeature::INVPCID;
  219. if (extended_features.ebx() & (1 << 11))
  220. m_features |= CPUFeature::RTM;
  221. if (extended_features.ebx() & (1 << 12))
  222. m_features |= CPUFeature::PQM;
  223. if (extended_features.ebx() & (1 << 13))
  224. m_features |= CPUFeature::ZERO_FCS_FDS;
  225. if (extended_features.ebx() & (1 << 14))
  226. m_features |= CPUFeature::MPX;
  227. if (extended_features.ebx() & (1 << 15))
  228. m_features |= CPUFeature::PQE;
  229. if (extended_features.ebx() & (1 << 16))
  230. m_features |= CPUFeature::AVX512_F;
  231. if (extended_features.ebx() & (1 << 17))
  232. m_features |= CPUFeature::AVX512_DQ;
  233. if (extended_features.ebx() & (1 << 18))
  234. m_features |= CPUFeature::RDSEED;
  235. if (extended_features.ebx() & (1 << 19))
  236. m_features |= CPUFeature::ADX;
  237. if (extended_features.ebx() & (1 << 20))
  238. m_features |= CPUFeature::SMAP;
  239. if (extended_features.ebx() & (1 << 21))
  240. m_features |= CPUFeature::AVX512_IFMA;
  241. if (extended_features.ebx() & (1 << 22))
  242. m_features |= CPUFeature::PCOMMIT;
  243. if (extended_features.ebx() & (1 << 23))
  244. m_features |= CPUFeature::CLFLUSHOPT;
  245. if (extended_features.ebx() & (1 << 24))
  246. m_features |= CPUFeature::CLWB;
  247. if (extended_features.ebx() & (1 << 25))
  248. m_features |= CPUFeature::INTEL_PT;
  249. if (extended_features.ebx() & (1 << 26))
  250. m_features |= CPUFeature::AVX512_PF;
  251. if (extended_features.ebx() & (1 << 27))
  252. m_features |= CPUFeature::AVX512_ER;
  253. if (extended_features.ebx() & (1 << 28))
  254. m_features |= CPUFeature::AVX512_CD;
  255. if (extended_features.ebx() & (1 << 29))
  256. m_features |= CPUFeature::SHA;
  257. if (extended_features.ebx() & (1 << 30))
  258. m_features |= CPUFeature::AVX512_BW;
  259. if (extended_features.ebx() & (1 << 31))
  260. m_features |= CPUFeature::AVX512_VL;
  261. if (extended_features.ecx() & (1 << 0))
  262. m_features |= CPUFeature::PREFETCHWT1;
  263. if (extended_features.ecx() & (1 << 1))
  264. m_features |= CPUFeature::AVX512_VBMI;
  265. if (extended_features.ecx() & (1 << 2))
  266. m_features |= CPUFeature::UMIP;
  267. if (extended_features.ecx() & (1 << 3))
  268. m_features |= CPUFeature::PKU;
  269. if (extended_features.ecx() & (1 << 4))
  270. m_features |= CPUFeature::OSPKU;
  271. if (extended_features.ecx() & (1 << 5))
  272. m_features |= CPUFeature::WAITPKG;
  273. if (extended_features.ecx() & (1 << 6))
  274. m_features |= CPUFeature::AVX512_VBMI2;
  275. if (extended_features.ecx() & (1 << 7))
  276. m_features |= CPUFeature::CET_SS;
  277. if (extended_features.ecx() & (1 << 8))
  278. m_features |= CPUFeature::GFNI;
  279. if (extended_features.ecx() & (1 << 9))
  280. m_features |= CPUFeature::VAES;
  281. if (extended_features.ecx() & (1 << 10))
  282. m_features |= CPUFeature::VPCLMULQDQ;
  283. if (extended_features.ecx() & (1 << 11))
  284. m_features |= CPUFeature::AVX512_VNNI;
  285. if (extended_features.ecx() & (1 << 12))
  286. m_features |= CPUFeature::AVX512_BITALG;
  287. if (extended_features.ecx() & (1 << 13))
  288. m_features |= CPUFeature::TME_EN;
  289. if (extended_features.ecx() & (1 << 14))
  290. m_features |= CPUFeature::AVX512_VPOPCNTDQ;
  291. if (extended_features.ecx() & (1 << 16))
  292. m_features |= CPUFeature::INTEL_5_LEVEL_PAGING;
  293. if (extended_features.ecx() & (1 << 22))
  294. m_features |= CPUFeature::RDPID;
  295. if (extended_features.ecx() & (1 << 23))
  296. m_features |= CPUFeature::KL;
  297. if (extended_features.ecx() & (1 << 25))
  298. m_features |= CPUFeature::CLDEMOTE;
  299. if (extended_features.ecx() & (1 << 27))
  300. m_features |= CPUFeature::MOVDIRI;
  301. if (extended_features.ecx() & (1 << 28))
  302. m_features |= CPUFeature::MOVDIR64B;
  303. if (extended_features.ecx() & (1 << 29))
  304. m_features |= CPUFeature::ENQCMD;
  305. if (extended_features.ecx() & (1 << 30))
  306. m_features |= CPUFeature::SGX_LC;
  307. if (extended_features.ecx() & (1 << 31))
  308. m_features |= CPUFeature::PKS;
  309. if (extended_features.edx() & (1 << 2))
  310. m_features |= CPUFeature::AVX512_4VNNIW;
  311. if (extended_features.edx() & (1 << 3))
  312. m_features |= CPUFeature::AVX512_4FMAPS;
  313. if (extended_features.edx() & (1 << 4))
  314. m_features |= CPUFeature::FSRM;
  315. if (extended_features.edx() & (1 << 8))
  316. m_features |= CPUFeature::AVX512_VP2INTERSECT;
  317. if (extended_features.edx() & (1 << 9))
  318. m_features |= CPUFeature::SRBDS_CTRL;
  319. if (extended_features.edx() & (1 << 10))
  320. m_features |= CPUFeature::MD_CLEAR;
  321. if (extended_features.edx() & (1 << 11))
  322. m_features |= CPUFeature::RTM_ALWAYS_ABORT;
  323. if (extended_features.edx() & (1 << 13))
  324. m_features |= CPUFeature::TSX_FORCE_ABORT;
  325. if (extended_features.edx() & (1 << 14))
  326. m_features |= CPUFeature::SERIALIZE;
  327. if (extended_features.edx() & (1 << 15))
  328. m_features |= CPUFeature::HYBRID;
  329. if (extended_features.edx() & (1 << 16))
  330. m_features |= CPUFeature::TSXLDTRK;
  331. if (extended_features.edx() & (1 << 18))
  332. m_features |= CPUFeature::PCONFIG;
  333. if (extended_features.edx() & (1 << 19))
  334. m_features |= CPUFeature::LBR;
  335. if (extended_features.edx() & (1 << 20))
  336. m_features |= CPUFeature::CET_IBT;
  337. if (extended_features.edx() & (1 << 22))
  338. m_features |= CPUFeature::AMX_BF16;
  339. if (extended_features.edx() & (1 << 23))
  340. m_features |= CPUFeature::AVX512_FP16;
  341. if (extended_features.edx() & (1 << 24))
  342. m_features |= CPUFeature::AMX_TILE;
  343. if (extended_features.edx() & (1 << 25))
  344. m_features |= CPUFeature::AMX_INT8;
  345. if (extended_features.edx() & (1 << 26))
  346. m_features |= CPUFeature::SPEC_CTRL;
  347. if (extended_features.edx() & (1 << 27))
  348. m_features |= CPUFeature::STIBP;
  349. if (extended_features.edx() & (1 << 28))
  350. m_features |= CPUFeature::L1D_FLUSH;
  351. if (extended_features.edx() & (1 << 29))
  352. m_features |= CPUFeature::IA32_ARCH_CAPABILITIES;
  353. if (extended_features.edx() & (1 << 30))
  354. m_features |= CPUFeature::IA32_CORE_CAPABILITIES;
  355. if (extended_features.edx() & (1 << 31))
  356. m_features |= CPUFeature::SSBD;
  357. u32 max_extended_leaf = CPUID(0x80000000).eax();
  358. if (max_extended_leaf >= 0x80000001) {
  359. CPUID extended_processor_info(0x80000001);
  360. if (extended_processor_info.ecx() & (1 << 0))
  361. m_features |= CPUFeature::LAHF_LM;
  362. if (extended_processor_info.ecx() & (1 << 1))
  363. m_features |= CPUFeature::CMP_LEGACY;
  364. if (extended_processor_info.ecx() & (1 << 2))
  365. m_features |= CPUFeature::SVM;
  366. if (extended_processor_info.ecx() & (1 << 3))
  367. m_features |= CPUFeature::EXTAPIC;
  368. if (extended_processor_info.ecx() & (1 << 4))
  369. m_features |= CPUFeature::CR8_LEGACY;
  370. if (extended_processor_info.ecx() & (1 << 5))
  371. m_features |= CPUFeature::ABM;
  372. if (extended_processor_info.ecx() & (1 << 6))
  373. m_features |= CPUFeature::SSE4A;
  374. if (extended_processor_info.ecx() & (1 << 7))
  375. m_features |= CPUFeature::MISALIGNSSE;
  376. if (extended_processor_info.ecx() & (1 << 8))
  377. m_features |= CPUFeature::_3DNOWPREFETCH;
  378. if (extended_processor_info.ecx() & (1 << 9))
  379. m_features |= CPUFeature::OSVW;
  380. if (extended_processor_info.ecx() & (1 << 10))
  381. m_features |= CPUFeature::IBS;
  382. if (extended_processor_info.ecx() & (1 << 11))
  383. m_features |= CPUFeature::XOP;
  384. if (extended_processor_info.ecx() & (1 << 12))
  385. m_features |= CPUFeature::SKINIT;
  386. if (extended_processor_info.ecx() & (1 << 13))
  387. m_features |= CPUFeature::WDT;
  388. if (extended_processor_info.ecx() & (1 << 15))
  389. m_features |= CPUFeature::LWP;
  390. if (extended_processor_info.ecx() & (1 << 16))
  391. m_features |= CPUFeature::FMA4;
  392. if (extended_processor_info.ecx() & (1 << 17))
  393. m_features |= CPUFeature::TCE;
  394. if (extended_processor_info.ecx() & (1 << 19))
  395. m_features |= CPUFeature::NODEID_MSR;
  396. if (extended_processor_info.ecx() & (1 << 21))
  397. m_features |= CPUFeature::TBM;
  398. if (extended_processor_info.ecx() & (1 << 22))
  399. m_features |= CPUFeature::TOPOEXT;
  400. if (extended_processor_info.ecx() & (1 << 23))
  401. m_features |= CPUFeature::PERFCTR_CORE;
  402. if (extended_processor_info.ecx() & (1 << 24))
  403. m_features |= CPUFeature::PERFCTR_NB;
  404. if (extended_processor_info.ecx() & (1 << 26))
  405. m_features |= CPUFeature::DBX;
  406. if (extended_processor_info.ecx() & (1 << 27))
  407. m_features |= CPUFeature::PERFTSC;
  408. if (extended_processor_info.ecx() & (1 << 28))
  409. m_features |= CPUFeature::PCX_L2I;
  410. if (extended_processor_info.edx() & (1 << 11))
  411. m_features |= CPUFeature::SYSCALL; // Only available in 64 bit mode
  412. if (extended_processor_info.edx() & (1 << 19))
  413. m_features |= CPUFeature::MP;
  414. if (extended_processor_info.edx() & (1 << 20))
  415. m_features |= CPUFeature::NX;
  416. if (extended_processor_info.edx() & (1 << 22))
  417. m_features |= CPUFeature::MMXEXT;
  418. if (extended_processor_info.edx() & (1 << 23))
  419. m_features |= CPUFeature::RDTSCP;
  420. if (extended_processor_info.edx() & (1 << 25))
  421. m_features |= CPUFeature::FXSR_OPT;
  422. if (extended_processor_info.edx() & (1 << 26))
  423. m_features |= CPUFeature::PDPE1GB;
  424. if (extended_processor_info.edx() & (1 << 27))
  425. m_features |= CPUFeature::RDTSCP;
  426. if (extended_processor_info.edx() & (1 << 29))
  427. m_features |= CPUFeature::LM;
  428. if (extended_processor_info.edx() & (1 << 30))
  429. m_features |= CPUFeature::_3DNOWEXT;
  430. if (extended_processor_info.edx() & (1 << 31))
  431. m_features |= CPUFeature::_3DNOW;
  432. }
  433. if (max_extended_leaf >= 0x80000007) {
  434. CPUID cpuid(0x80000007);
  435. if (cpuid.edx() & (1 << 8)) {
  436. m_features |= CPUFeature::CONSTANT_TSC;
  437. m_features |= CPUFeature::NONSTOP_TSC;
  438. }
  439. }
  440. #if ARCH(X86_64)
  441. m_has_qemu_hvf_quirk = false;
  442. #endif
  443. if (max_extended_leaf >= 0x80000008) {
  444. // CPUID.80000008H:EAX[7:0] reports the physical-address width supported by the processor.
  445. CPUID cpuid(0x80000008);
  446. m_physical_address_bit_width = cpuid.eax() & 0xff;
  447. // CPUID.80000008H:EAX[15:8] reports the linear-address width supported by the processor.
  448. m_virtual_address_bit_width = (cpuid.eax() >> 8) & 0xff;
  449. } else {
  450. // For processors that do not support CPUID function 80000008H, the width is generally 36 if CPUID.01H:EDX.PAE [bit 6] = 1 and 32 otherwise.
  451. m_physical_address_bit_width = has_feature(CPUFeature::PAE) ? 36 : 32;
  452. // Processors that do not support CPUID function 80000008H, support a linear-address width of 32.
  453. m_virtual_address_bit_width = 32;
  454. #if ARCH(X86_64)
  455. // Workaround QEMU hypervisor.framework bug
  456. // https://gitlab.com/qemu-project/qemu/-/issues/664
  457. //
  458. // We detect this as follows:
  459. // * We're in a hypervisor
  460. // * hypervisor_leaf_range is null under Hypervisor.framework
  461. // * m_physical_address_bit_width is 36 bits
  462. if (has_feature(CPUFeature::HYPERVISOR)) {
  463. CPUID hypervisor_leaf_range(0x40000000);
  464. if (!hypervisor_leaf_range.ebx() && m_physical_address_bit_width == 36) {
  465. m_has_qemu_hvf_quirk = true;
  466. m_virtual_address_bit_width = 48;
  467. }
  468. }
  469. #endif
  470. }
  471. }
  472. UNMAP_AFTER_INIT void Processor::cpu_setup()
  473. {
  474. // NOTE: This is called during Processor::early_initialize, we cannot
  475. // safely log at this point because we don't have kmalloc
  476. // initialized yet!
  477. cpu_detect();
  478. if (has_feature(CPUFeature::SSE)) {
  479. // enter_thread_context() assumes that if a x86 CPU supports SSE then it also supports FXSR.
  480. // SSE support without FXSR is an extremely unlikely scenario, so let's be pragmatic about it.
  481. VERIFY(has_feature(CPUFeature::FXSR));
  482. sse_init();
  483. }
  484. write_cr0(read_cr0() | 0x00010000);
  485. if (has_feature(CPUFeature::PGE)) {
  486. // Turn on CR4.PGE so the CPU will respect the G bit in page tables.
  487. write_cr4(read_cr4() | 0x80);
  488. }
  489. if (has_feature(CPUFeature::NX)) {
  490. // Turn on IA32_EFER.NXE
  491. MSR ia32_efer(MSR_IA32_EFER);
  492. ia32_efer.set(ia32_efer.get() | 0x800);
  493. }
  494. if (has_feature(CPUFeature::PAT)) {
  495. MSR ia32_pat(MSR_IA32_PAT);
  496. // Set PA4 to Write Comine. This allows us to
  497. // use this mode by only setting the bit in the PTE
  498. // and leaving all other bits in the upper levels unset,
  499. // which maps to setting bit 3 of the index, resulting
  500. // in the index value 0 or 4.
  501. u64 pat = ia32_pat.get() & ~(0x7ull << 32);
  502. pat |= 0x1ull << 32; // set WC mode for PA4
  503. ia32_pat.set(pat);
  504. }
  505. if (has_feature(CPUFeature::SMEP)) {
  506. // Turn on CR4.SMEP
  507. write_cr4(read_cr4() | 0x100000);
  508. }
  509. if (has_feature(CPUFeature::SMAP)) {
  510. // Turn on CR4.SMAP
  511. write_cr4(read_cr4() | 0x200000);
  512. }
  513. if (has_feature(CPUFeature::UMIP)) {
  514. write_cr4(read_cr4() | 0x800);
  515. }
  516. if (has_feature(CPUFeature::TSC)) {
  517. write_cr4(read_cr4() | 0x4);
  518. }
  519. if (has_feature(CPUFeature::XSAVE)) {
  520. // Turn on CR4.OSXSAVE
  521. write_cr4(read_cr4() | 0x40000);
  522. // According to the Intel manual: "After reset, all bits (except bit 0) in XCR0 are cleared to zero; XCR0[0] is set to 1."
  523. // Sadly we can't trust this, for example VirtualBox starts with bits 0-4 set, so let's do it ourselves.
  524. write_xcr0(0x1);
  525. if (has_feature(CPUFeature::AVX)) {
  526. // Turn on SSE, AVX and x87 flags
  527. write_xcr0(read_xcr0() | 0x7);
  528. }
  529. }
  530. #if ARCH(X86_64)
  531. // x86_64 processors must support the syscall feature.
  532. VERIFY(has_feature(CPUFeature::SYSCALL));
  533. MSR efer_msr(MSR_EFER);
  534. efer_msr.set(efer_msr.get() | 1u);
  535. // Write code and stack selectors to the STAR MSR. The first value stored in bits 63:48 controls the sysret CS (value + 0x10) and SS (value + 0x8),
  536. // and the value stored in bits 47:32 controls the syscall CS (value) and SS (value + 0x8).
  537. u64 star = 0;
  538. star |= 0x13ul << 48u;
  539. star |= 0x08ul << 32u;
  540. MSR star_msr(MSR_STAR);
  541. star_msr.set(star);
  542. // Write the syscall entry point to the LSTAR MSR.
  543. MSR lstar_msr(MSR_LSTAR);
  544. lstar_msr.set(reinterpret_cast<u64>(&syscall_entry));
  545. // Write the SFMASK MSR. This MSR controls which bits of rflags are masked when a syscall instruction is executed -
  546. // if a bit is set in sfmask, the corresponding bit in rflags is cleared. The value set here clears most of rflags,
  547. // but keeps the reserved and virtualization bits intact. The userspace rflags value is saved in r11 by syscall.
  548. constexpr u64 rflags_mask = 0x257fd5u;
  549. MSR sfmask_msr(MSR_SFMASK);
  550. sfmask_msr.set(rflags_mask);
  551. #endif
  552. }
  553. UNMAP_AFTER_INIT void Processor::early_initialize(u32 cpu)
  554. {
  555. m_self = this;
  556. m_cpu = cpu;
  557. m_in_irq = 0;
  558. m_in_critical = 0;
  559. m_invoke_scheduler_async = false;
  560. m_scheduler_initialized = false;
  561. m_in_scheduler = true;
  562. m_message_queue = nullptr;
  563. m_idle_thread = nullptr;
  564. m_current_thread = nullptr;
  565. m_info = nullptr;
  566. m_halt_requested = false;
  567. if (cpu == 0) {
  568. s_smp_enabled = false;
  569. g_total_processors.store(1u, AK::MemoryOrder::memory_order_release);
  570. } else {
  571. g_total_processors.fetch_add(1u, AK::MemoryOrder::memory_order_acq_rel);
  572. }
  573. deferred_call_pool_init();
  574. cpu_setup();
  575. gdt_init();
  576. VERIFY(is_initialized()); // sanity check
  577. VERIFY(&current() == this); // sanity check
  578. }
  579. UNMAP_AFTER_INIT void Processor::initialize(u32 cpu)
  580. {
  581. VERIFY(m_self == this);
  582. VERIFY(&current() == this); // sanity check
  583. m_info = new ProcessorInfo(*this);
  584. dmesgln("CPU[{}]: Supported features: {}", current_id(), m_info->features_string());
  585. if (!has_feature(CPUFeature::RDRAND))
  586. dmesgln("CPU[{}]: No RDRAND support detected, randomness will be poor", current_id());
  587. dmesgln("CPU[{}]: Physical address bit width: {}", current_id(), m_physical_address_bit_width);
  588. dmesgln("CPU[{}]: Virtual address bit width: {}", current_id(), m_virtual_address_bit_width);
  589. #if ARCH(X86_64)
  590. if (m_has_qemu_hvf_quirk)
  591. dmesgln("CPU[{}]: Applied correction for QEMU Hypervisor.framework quirk", current_id());
  592. #endif
  593. if (cpu == 0)
  594. idt_init();
  595. else
  596. flush_idt();
  597. if (cpu == 0) {
  598. VERIFY((FlatPtr(&s_clean_fpu_state) & 0xF) == 0);
  599. asm volatile("fninit");
  600. if (has_feature(CPUFeature::FXSR))
  601. asm volatile("fxsave %0"
  602. : "=m"(s_clean_fpu_state));
  603. else
  604. asm volatile("fnsave %0"
  605. : "=m"(s_clean_fpu_state));
  606. if (has_feature(CPUFeature::HYPERVISOR))
  607. detect_hypervisor();
  608. }
  609. {
  610. // We need to prevent races between APs starting up at the same time
  611. VERIFY(cpu < s_processors.size());
  612. s_processors[cpu] = this;
  613. }
  614. }
  615. UNMAP_AFTER_INIT void Processor::detect_hypervisor()
  616. {
  617. CPUID hypervisor_leaf_range(0x40000000);
  618. // Get signature of hypervisor.
  619. alignas(sizeof(u32)) char hypervisor_signature_buffer[13];
  620. *reinterpret_cast<u32*>(hypervisor_signature_buffer) = hypervisor_leaf_range.ebx();
  621. *reinterpret_cast<u32*>(hypervisor_signature_buffer + 4) = hypervisor_leaf_range.ecx();
  622. *reinterpret_cast<u32*>(hypervisor_signature_buffer + 8) = hypervisor_leaf_range.edx();
  623. hypervisor_signature_buffer[12] = '\0';
  624. StringView hypervisor_signature(hypervisor_signature_buffer);
  625. dmesgln("CPU[{}]: CPUID hypervisor signature '{}' ({:#x} {:#x} {:#x}), max leaf {:#x}", current_id(), hypervisor_signature, hypervisor_leaf_range.ebx(), hypervisor_leaf_range.ecx(), hypervisor_leaf_range.edx(), hypervisor_leaf_range.eax());
  626. if (hypervisor_signature == "Microsoft Hv"sv)
  627. detect_hypervisor_hyperv(hypervisor_leaf_range);
  628. }
  629. UNMAP_AFTER_INIT void Processor::detect_hypervisor_hyperv(CPUID const& hypervisor_leaf_range)
  630. {
  631. if (hypervisor_leaf_range.eax() < 0x40000001)
  632. return;
  633. CPUID hypervisor_interface(0x40000001);
  634. // Get signature of hypervisor interface.
  635. alignas(sizeof(u32)) char interface_signature_buffer[5];
  636. *reinterpret_cast<u32*>(interface_signature_buffer) = hypervisor_interface.eax();
  637. interface_signature_buffer[4] = '\0';
  638. StringView hyperv_interface_signature(interface_signature_buffer);
  639. dmesgln("CPU[{}]: Hyper-V interface signature '{}' ({:#x})", current_id(), hyperv_interface_signature, hypervisor_interface.eax());
  640. if (hypervisor_leaf_range.eax() < 0x40000001)
  641. return;
  642. CPUID hypervisor_sysid(0x40000002);
  643. dmesgln("CPU[{}]: Hyper-V system identity {}.{}, build number {}", current_id(), hypervisor_sysid.ebx() >> 16, hypervisor_sysid.ebx() & 0xFFFF, hypervisor_sysid.eax());
  644. if (hypervisor_leaf_range.eax() < 0x40000005 || hyperv_interface_signature != "Hv#1"sv)
  645. return;
  646. dmesgln("CPU[{}]: Hyper-V hypervisor detected", current_id());
  647. // TODO: Actually do something with Hyper-V.
  648. }
  649. void Processor::write_raw_gdt_entry(u16 selector, u32 low, u32 high)
  650. {
  651. u16 i = (selector & 0xfffc) >> 3;
  652. u32 prev_gdt_length = m_gdt_length;
  653. if (i >= m_gdt_length) {
  654. m_gdt_length = i + 1;
  655. VERIFY(m_gdt_length <= sizeof(m_gdt) / sizeof(m_gdt[0]));
  656. m_gdtr.limit = (m_gdt_length + 1) * 8 - 1;
  657. }
  658. m_gdt[i].low = low;
  659. m_gdt[i].high = high;
  660. // clear selectors we may have skipped
  661. for (auto j = prev_gdt_length; j < i; ++j) {
  662. m_gdt[j].low = 0;
  663. m_gdt[j].high = 0;
  664. }
  665. }
  666. void Processor::write_gdt_entry(u16 selector, Descriptor& descriptor)
  667. {
  668. write_raw_gdt_entry(selector, descriptor.low, descriptor.high);
  669. }
  670. Descriptor& Processor::get_gdt_entry(u16 selector)
  671. {
  672. u16 i = (selector & 0xfffc) >> 3;
  673. return *(Descriptor*)(&m_gdt[i]);
  674. }
  675. void Processor::flush_gdt()
  676. {
  677. m_gdtr.address = m_gdt;
  678. m_gdtr.limit = (m_gdt_length * 8) - 1;
  679. asm volatile("lgdt %0" ::"m"(m_gdtr)
  680. : "memory");
  681. }
  682. DescriptorTablePointer const& Processor::get_gdtr()
  683. {
  684. return m_gdtr;
  685. }
  686. ErrorOr<Vector<FlatPtr, 32>> Processor::capture_stack_trace(Thread& thread, size_t max_frames)
  687. {
  688. FlatPtr frame_ptr = 0, ip = 0;
  689. Vector<FlatPtr, 32> stack_trace;
  690. auto walk_stack = [&](FlatPtr stack_ptr) -> ErrorOr<void> {
  691. constexpr size_t max_stack_frames = 4096;
  692. bool is_walking_userspace_stack = false;
  693. TRY(stack_trace.try_append(ip));
  694. size_t count = 1;
  695. while (stack_ptr && stack_trace.size() < max_stack_frames) {
  696. FlatPtr retaddr;
  697. count++;
  698. if (max_frames != 0 && count > max_frames)
  699. break;
  700. if (!Memory::is_user_address(VirtualAddress { stack_ptr })) {
  701. if (is_walking_userspace_stack) {
  702. dbgln("SHENANIGANS! Userspace stack points back into kernel memory");
  703. break;
  704. }
  705. } else {
  706. is_walking_userspace_stack = true;
  707. }
  708. if (Memory::is_user_range(VirtualAddress(stack_ptr), sizeof(FlatPtr) * 2)) {
  709. if (copy_from_user(&retaddr, &((FlatPtr*)stack_ptr)[1]).is_error() || !retaddr)
  710. break;
  711. TRY(stack_trace.try_append(retaddr));
  712. if (copy_from_user(&stack_ptr, (FlatPtr*)stack_ptr).is_error())
  713. break;
  714. } else {
  715. void* fault_at;
  716. if (!safe_memcpy(&retaddr, &((FlatPtr*)stack_ptr)[1], sizeof(FlatPtr), fault_at) || !retaddr)
  717. break;
  718. TRY(stack_trace.try_append(retaddr));
  719. if (!safe_memcpy(&stack_ptr, (FlatPtr*)stack_ptr, sizeof(FlatPtr), fault_at))
  720. break;
  721. }
  722. }
  723. return {};
  724. };
  725. auto capture_current_thread = [&]() {
  726. frame_ptr = (FlatPtr)__builtin_frame_address(0);
  727. ip = (FlatPtr)__builtin_return_address(0);
  728. return walk_stack(frame_ptr);
  729. };
  730. // Since the thread may be running on another processor, there
  731. // is a chance a context switch may happen while we're trying
  732. // to get it. It also won't be entirely accurate and merely
  733. // reflect the status at the last context switch.
  734. SpinlockLocker lock(g_scheduler_lock);
  735. if (&thread == Processor::current_thread()) {
  736. VERIFY(thread.state() == Thread::State::Running);
  737. // Leave the scheduler lock. If we trigger page faults we may
  738. // need to be preempted. Since this is our own thread it won't
  739. // cause any problems as the stack won't change below this frame.
  740. lock.unlock();
  741. TRY(capture_current_thread());
  742. } else if (thread.is_active()) {
  743. VERIFY(thread.cpu() != Processor::current_id());
  744. // If this is the case, the thread is currently running
  745. // on another processor. We can't trust the kernel stack as
  746. // it may be changing at any time. We need to probably send
  747. // an IPI to that processor, have it walk the stack and wait
  748. // until it returns the data back to us
  749. auto& proc = Processor::current();
  750. ErrorOr<void> result;
  751. smp_unicast(
  752. thread.cpu(),
  753. [&]() {
  754. dbgln("CPU[{}] getting stack for cpu #{}", Processor::current_id(), proc.id());
  755. ScopedAddressSpaceSwitcher switcher(thread.process());
  756. VERIFY(&Processor::current() != &proc);
  757. VERIFY(&thread == Processor::current_thread());
  758. // NOTE: Because the other processor is still holding the
  759. // scheduler lock while waiting for this callback to finish,
  760. // the current thread on the target processor cannot change
  761. // TODO: What to do about page faults here? We might deadlock
  762. // because the other processor is still holding the
  763. // scheduler lock...
  764. result = capture_current_thread();
  765. },
  766. false);
  767. TRY(result);
  768. } else {
  769. switch (thread.state()) {
  770. case Thread::State::Running:
  771. VERIFY_NOT_REACHED(); // should have been handled above
  772. case Thread::State::Runnable:
  773. case Thread::State::Stopped:
  774. case Thread::State::Blocked:
  775. case Thread::State::Dying:
  776. case Thread::State::Dead: {
  777. // We need to retrieve ebp from what was last pushed to the kernel
  778. // stack. Before switching out of that thread, it switch_context
  779. // pushed the callee-saved registers, and the last of them happens
  780. // to be ebp.
  781. ScopedAddressSpaceSwitcher switcher(thread.process());
  782. auto& regs = thread.regs();
  783. auto* stack_top = reinterpret_cast<FlatPtr*>(regs.sp());
  784. if (Memory::is_user_range(VirtualAddress(stack_top), sizeof(FlatPtr))) {
  785. if (copy_from_user(&frame_ptr, &((FlatPtr*)stack_top)[0]).is_error())
  786. frame_ptr = 0;
  787. } else {
  788. void* fault_at;
  789. if (!safe_memcpy(&frame_ptr, &((FlatPtr*)stack_top)[0], sizeof(FlatPtr), fault_at))
  790. frame_ptr = 0;
  791. }
  792. ip = regs.ip();
  793. // TODO: We need to leave the scheduler lock here, but we also
  794. // need to prevent the target thread from being run while
  795. // we walk the stack
  796. lock.unlock();
  797. TRY(walk_stack(frame_ptr));
  798. break;
  799. }
  800. default:
  801. dbgln("Cannot capture stack trace for thread {} in state {}", thread, thread.state_string());
  802. break;
  803. }
  804. }
  805. return stack_trace;
  806. }
  807. ProcessorContainer& Processor::processors()
  808. {
  809. return s_processors;
  810. }
  811. Processor& Processor::by_id(u32 id)
  812. {
  813. return *s_processors[id];
  814. }
  815. void Processor::enter_trap(TrapFrame& trap, bool raise_irq)
  816. {
  817. VERIFY_INTERRUPTS_DISABLED();
  818. VERIFY(&Processor::current() == this);
  819. trap.prev_irq_level = m_in_irq;
  820. if (raise_irq)
  821. m_in_irq++;
  822. auto* current_thread = Processor::current_thread();
  823. if (current_thread) {
  824. auto& current_trap = current_thread->current_trap();
  825. trap.next_trap = current_trap;
  826. current_trap = &trap;
  827. // The cs register of this trap tells us where we will return back to
  828. auto new_previous_mode = ((trap.regs->cs & 3) != 0) ? Thread::PreviousMode::UserMode : Thread::PreviousMode::KernelMode;
  829. if (current_thread->set_previous_mode(new_previous_mode) && trap.prev_irq_level == 0) {
  830. current_thread->update_time_scheduled(Scheduler::current_time(), new_previous_mode == Thread::PreviousMode::KernelMode, false);
  831. }
  832. } else {
  833. trap.next_trap = nullptr;
  834. }
  835. }
  836. void Processor::exit_trap(TrapFrame& trap)
  837. {
  838. VERIFY_INTERRUPTS_DISABLED();
  839. VERIFY(&Processor::current() == this);
  840. // Temporarily enter a critical section. This is to prevent critical
  841. // sections entered and left within e.g. smp_process_pending_messages
  842. // to trigger a context switch while we're executing this function
  843. // See the comment at the end of the function why we don't use
  844. // ScopedCritical here.
  845. m_in_critical = m_in_critical + 1;
  846. VERIFY(m_in_irq >= trap.prev_irq_level);
  847. m_in_irq = trap.prev_irq_level;
  848. if (s_smp_enabled)
  849. smp_process_pending_messages();
  850. // Process the deferred call queue. Among other things, this ensures
  851. // that any pending thread unblocks happen before we enter the scheduler.
  852. deferred_call_execute_pending();
  853. auto* current_thread = Processor::current_thread();
  854. if (current_thread) {
  855. auto& current_trap = current_thread->current_trap();
  856. current_trap = trap.next_trap;
  857. Thread::PreviousMode new_previous_mode;
  858. if (current_trap) {
  859. VERIFY(current_trap->regs);
  860. // If we have another higher level trap then we probably returned
  861. // from an interrupt or irq handler. The cs register of the
  862. // new/higher level trap tells us what the mode prior to it was
  863. new_previous_mode = ((current_trap->regs->cs & 3) != 0) ? Thread::PreviousMode::UserMode : Thread::PreviousMode::KernelMode;
  864. } else {
  865. // If we don't have a higher level trap then we're back in user mode.
  866. // Which means that the previous mode prior to being back in user mode was kernel mode
  867. new_previous_mode = Thread::PreviousMode::KernelMode;
  868. }
  869. if (current_thread->set_previous_mode(new_previous_mode))
  870. current_thread->update_time_scheduled(Scheduler::current_time(), true, false);
  871. }
  872. VERIFY_INTERRUPTS_DISABLED();
  873. // Leave the critical section without actually enabling interrupts.
  874. // We don't want context switches to happen until we're explicitly
  875. // triggering a switch in check_invoke_scheduler.
  876. m_in_critical = m_in_critical - 1;
  877. if (!m_in_irq && !m_in_critical)
  878. check_invoke_scheduler();
  879. }
  880. void Processor::check_invoke_scheduler()
  881. {
  882. InterruptDisabler disabler;
  883. VERIFY(!m_in_irq);
  884. VERIFY(!m_in_critical);
  885. VERIFY(&Processor::current() == this);
  886. if (m_invoke_scheduler_async && m_scheduler_initialized) {
  887. m_invoke_scheduler_async = false;
  888. Scheduler::invoke_async();
  889. }
  890. }
  891. void Processor::flush_tlb_local(VirtualAddress vaddr, size_t page_count)
  892. {
  893. auto ptr = vaddr.as_ptr();
  894. while (page_count > 0) {
  895. // clang-format off
  896. asm volatile("invlpg %0"
  897. :
  898. : "m"(*ptr)
  899. : "memory");
  900. // clang-format on
  901. ptr += PAGE_SIZE;
  902. page_count--;
  903. }
  904. }
  905. void Processor::flush_tlb(Memory::PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count)
  906. {
  907. if (s_smp_enabled && (!Memory::is_user_address(vaddr) || Process::current().thread_count() > 1))
  908. smp_broadcast_flush_tlb(page_directory, vaddr, page_count);
  909. else
  910. flush_tlb_local(vaddr, page_count);
  911. }
  912. void Processor::smp_return_to_pool(ProcessorMessage& msg)
  913. {
  914. ProcessorMessage* next = nullptr;
  915. for (;;) {
  916. msg.next = next;
  917. if (s_message_pool.compare_exchange_strong(next, &msg, AK::MemoryOrder::memory_order_acq_rel))
  918. break;
  919. Processor::pause();
  920. }
  921. }
  922. ProcessorMessage& Processor::smp_get_from_pool()
  923. {
  924. ProcessorMessage* msg;
  925. // The assumption is that messages are never removed from the pool!
  926. for (;;) {
  927. msg = s_message_pool.load(AK::MemoryOrder::memory_order_consume);
  928. if (!msg) {
  929. if (!Processor::current().smp_process_pending_messages()) {
  930. Processor::pause();
  931. }
  932. continue;
  933. }
  934. // If another processor were to use this message in the meanwhile,
  935. // "msg" is still valid (because it never gets freed). We'd detect
  936. // this because the expected value "msg" and pool would
  937. // no longer match, and the compare_exchange will fail. But accessing
  938. // "msg->next" is always safe here.
  939. if (s_message_pool.compare_exchange_strong(msg, msg->next, AK::MemoryOrder::memory_order_acq_rel)) {
  940. // We successfully "popped" this available message
  941. break;
  942. }
  943. }
  944. VERIFY(msg != nullptr);
  945. return *msg;
  946. }
  947. u32 Processor::smp_wake_n_idle_processors(u32 wake_count)
  948. {
  949. VERIFY_INTERRUPTS_DISABLED();
  950. VERIFY(wake_count > 0);
  951. if (!s_smp_enabled)
  952. return 0;
  953. // Wake at most N - 1 processors
  954. if (wake_count >= Processor::count()) {
  955. wake_count = Processor::count() - 1;
  956. VERIFY(wake_count > 0);
  957. }
  958. u32 current_id = Processor::current_id();
  959. u32 did_wake_count = 0;
  960. auto& apic = APIC::the();
  961. while (did_wake_count < wake_count) {
  962. // Try to get a set of idle CPUs and flip them to busy
  963. u32 idle_mask = s_idle_cpu_mask.load(AK::MemoryOrder::memory_order_relaxed) & ~(1u << current_id);
  964. u32 idle_count = popcount(idle_mask);
  965. if (idle_count == 0)
  966. break; // No (more) idle processor available
  967. u32 found_mask = 0;
  968. for (u32 i = 0; i < idle_count; i++) {
  969. u32 cpu = bit_scan_forward(idle_mask) - 1;
  970. idle_mask &= ~(1u << cpu);
  971. found_mask |= 1u << cpu;
  972. }
  973. idle_mask = s_idle_cpu_mask.fetch_and(~found_mask, AK::MemoryOrder::memory_order_acq_rel) & found_mask;
  974. if (idle_mask == 0)
  975. continue; // All of them were flipped to busy, try again
  976. idle_count = popcount(idle_mask);
  977. for (u32 i = 0; i < idle_count; i++) {
  978. u32 cpu = bit_scan_forward(idle_mask) - 1;
  979. idle_mask &= ~(1u << cpu);
  980. // Send an IPI to that CPU to wake it up. There is a possibility
  981. // someone else woke it up as well, or that it woke up due to
  982. // a timer interrupt. But we tried hard to avoid this...
  983. apic.send_ipi(cpu);
  984. did_wake_count++;
  985. }
  986. }
  987. return did_wake_count;
  988. }
  989. UNMAP_AFTER_INIT void Processor::smp_enable()
  990. {
  991. size_t msg_pool_size = Processor::count() * 100u;
  992. size_t msg_entries_cnt = Processor::count();
  993. auto msgs = new ProcessorMessage[msg_pool_size];
  994. auto msg_entries = new ProcessorMessageEntry[msg_pool_size * msg_entries_cnt];
  995. size_t msg_entry_i = 0;
  996. for (size_t i = 0; i < msg_pool_size; i++, msg_entry_i += msg_entries_cnt) {
  997. auto& msg = msgs[i];
  998. msg.next = i < msg_pool_size - 1 ? &msgs[i + 1] : nullptr;
  999. msg.per_proc_entries = &msg_entries[msg_entry_i];
  1000. for (size_t k = 0; k < msg_entries_cnt; k++)
  1001. msg_entries[msg_entry_i + k].msg = &msg;
  1002. }
  1003. s_message_pool.store(&msgs[0], AK::MemoryOrder::memory_order_release);
  1004. // Start sending IPI messages
  1005. s_smp_enabled = true;
  1006. }
  1007. void Processor::smp_cleanup_message(ProcessorMessage& msg)
  1008. {
  1009. switch (msg.type) {
  1010. case ProcessorMessage::Callback:
  1011. msg.callback_value().~Function();
  1012. break;
  1013. default:
  1014. break;
  1015. }
  1016. }
  1017. bool Processor::smp_process_pending_messages()
  1018. {
  1019. VERIFY(s_smp_enabled);
  1020. bool did_process = false;
  1021. enter_critical();
  1022. if (auto pending_msgs = m_message_queue.exchange(nullptr, AK::MemoryOrder::memory_order_acq_rel)) {
  1023. // We pulled the stack of pending messages in LIFO order, so we need to reverse the list first
  1024. auto reverse_list =
  1025. [](ProcessorMessageEntry* list) -> ProcessorMessageEntry* {
  1026. ProcessorMessageEntry* rev_list = nullptr;
  1027. while (list) {
  1028. auto next = list->next;
  1029. list->next = rev_list;
  1030. rev_list = list;
  1031. list = next;
  1032. }
  1033. return rev_list;
  1034. };
  1035. pending_msgs = reverse_list(pending_msgs);
  1036. // now process in the right order
  1037. ProcessorMessageEntry* next_msg;
  1038. for (auto cur_msg = pending_msgs; cur_msg; cur_msg = next_msg) {
  1039. next_msg = cur_msg->next;
  1040. auto msg = cur_msg->msg;
  1041. dbgln_if(SMP_DEBUG, "SMP[{}]: Processing message {}", current_id(), VirtualAddress(msg));
  1042. switch (msg->type) {
  1043. case ProcessorMessage::Callback:
  1044. msg->invoke_callback();
  1045. break;
  1046. case ProcessorMessage::FlushTlb:
  1047. if (Memory::is_user_address(VirtualAddress(msg->flush_tlb.ptr))) {
  1048. // We assume that we don't cross into kernel land!
  1049. VERIFY(Memory::is_user_range(VirtualAddress(msg->flush_tlb.ptr), msg->flush_tlb.page_count * PAGE_SIZE));
  1050. if (read_cr3() != msg->flush_tlb.page_directory->cr3()) {
  1051. // This processor isn't using this page directory right now, we can ignore this request
  1052. dbgln_if(SMP_DEBUG, "SMP[{}]: No need to flush {} pages at {}", current_id(), msg->flush_tlb.page_count, VirtualAddress(msg->flush_tlb.ptr));
  1053. break;
  1054. }
  1055. }
  1056. flush_tlb_local(VirtualAddress(msg->flush_tlb.ptr), msg->flush_tlb.page_count);
  1057. break;
  1058. }
  1059. bool is_async = msg->async; // Need to cache this value *before* dropping the ref count!
  1060. auto prev_refs = msg->refs.fetch_sub(1u, AK::MemoryOrder::memory_order_acq_rel);
  1061. VERIFY(prev_refs != 0);
  1062. if (prev_refs == 1) {
  1063. // All processors handled this. If this is an async message,
  1064. // we need to clean it up and return it to the pool
  1065. if (is_async) {
  1066. smp_cleanup_message(*msg);
  1067. smp_return_to_pool(*msg);
  1068. }
  1069. }
  1070. if (m_halt_requested.load(AK::MemoryOrder::memory_order_relaxed))
  1071. halt_this();
  1072. }
  1073. did_process = true;
  1074. } else if (m_halt_requested.load(AK::MemoryOrder::memory_order_relaxed)) {
  1075. halt_this();
  1076. }
  1077. leave_critical();
  1078. return did_process;
  1079. }
  1080. bool Processor::smp_enqueue_message(ProcessorMessage& msg)
  1081. {
  1082. // Note that it's quite possible that the other processor may pop
  1083. // the queue at any given time. We rely on the fact that the messages
  1084. // are pooled and never get freed!
  1085. auto& msg_entry = msg.per_proc_entries[id()];
  1086. VERIFY(msg_entry.msg == &msg);
  1087. ProcessorMessageEntry* next = nullptr;
  1088. for (;;) {
  1089. msg_entry.next = next;
  1090. if (m_message_queue.compare_exchange_strong(next, &msg_entry, AK::MemoryOrder::memory_order_acq_rel))
  1091. break;
  1092. Processor::pause();
  1093. }
  1094. // If the enqueued message was the only message in the queue when posted,
  1095. // we return true. This is used by callers when deciding whether to generate an IPI.
  1096. return next == nullptr;
  1097. }
  1098. void Processor::smp_broadcast_message(ProcessorMessage& msg)
  1099. {
  1100. auto& current_processor = Processor::current();
  1101. dbgln_if(SMP_DEBUG, "SMP[{}]: Broadcast message {} to cpus: {} processor: {}", current_processor.id(), VirtualAddress(&msg), count(), VirtualAddress(&current_processor));
  1102. msg.refs.store(count() - 1, AK::MemoryOrder::memory_order_release);
  1103. VERIFY(msg.refs > 0);
  1104. bool need_broadcast = false;
  1105. for_each(
  1106. [&](Processor& proc) {
  1107. if (&proc != &current_processor) {
  1108. if (proc.smp_enqueue_message(msg))
  1109. need_broadcast = true;
  1110. }
  1111. });
  1112. // Now trigger an IPI on all other APs (unless all targets already had messages queued)
  1113. if (need_broadcast)
  1114. APIC::the().broadcast_ipi();
  1115. }
  1116. void Processor::smp_broadcast_wait_sync(ProcessorMessage& msg)
  1117. {
  1118. auto& cur_proc = Processor::current();
  1119. VERIFY(!msg.async);
  1120. // If synchronous then we must cleanup and return the message back
  1121. // to the pool. Otherwise, the last processor to complete it will return it
  1122. while (msg.refs.load(AK::MemoryOrder::memory_order_consume) != 0) {
  1123. Processor::pause();
  1124. // We need to process any messages that may have been sent to
  1125. // us while we're waiting. This also checks if another processor
  1126. // may have requested us to halt.
  1127. cur_proc.smp_process_pending_messages();
  1128. }
  1129. smp_cleanup_message(msg);
  1130. smp_return_to_pool(msg);
  1131. }
  1132. void Processor::smp_unicast_message(u32 cpu, ProcessorMessage& msg, bool async)
  1133. {
  1134. auto& current_processor = Processor::current();
  1135. VERIFY(cpu != current_processor.id());
  1136. auto& target_processor = processors()[cpu];
  1137. msg.async = async;
  1138. dbgln_if(SMP_DEBUG, "SMP[{}]: Send message {} to cpu #{} processor: {}", current_processor.id(), VirtualAddress(&msg), cpu, VirtualAddress(&target_processor));
  1139. msg.refs.store(1u, AK::MemoryOrder::memory_order_release);
  1140. if (target_processor->smp_enqueue_message(msg)) {
  1141. APIC::the().send_ipi(cpu);
  1142. }
  1143. if (!async) {
  1144. // If synchronous then we must cleanup and return the message back
  1145. // to the pool. Otherwise, the last processor to complete it will return it
  1146. while (msg.refs.load(AK::MemoryOrder::memory_order_consume) != 0) {
  1147. Processor::pause();
  1148. // We need to process any messages that may have been sent to
  1149. // us while we're waiting. This also checks if another processor
  1150. // may have requested us to halt.
  1151. current_processor.smp_process_pending_messages();
  1152. }
  1153. smp_cleanup_message(msg);
  1154. smp_return_to_pool(msg);
  1155. }
  1156. }
  1157. void Processor::smp_unicast(u32 cpu, Function<void()> callback, bool async)
  1158. {
  1159. auto& msg = smp_get_from_pool();
  1160. msg.type = ProcessorMessage::Callback;
  1161. new (msg.callback_storage) ProcessorMessage::CallbackFunction(move(callback));
  1162. smp_unicast_message(cpu, msg, async);
  1163. }
  1164. void Processor::smp_broadcast_flush_tlb(Memory::PageDirectory const* page_directory, VirtualAddress vaddr, size_t page_count)
  1165. {
  1166. auto& msg = smp_get_from_pool();
  1167. msg.async = false;
  1168. msg.type = ProcessorMessage::FlushTlb;
  1169. msg.flush_tlb.page_directory = page_directory;
  1170. msg.flush_tlb.ptr = vaddr.as_ptr();
  1171. msg.flush_tlb.page_count = page_count;
  1172. smp_broadcast_message(msg);
  1173. // While the other processors handle this request, we'll flush ours
  1174. flush_tlb_local(vaddr, page_count);
  1175. // Now wait until everybody is done as well
  1176. smp_broadcast_wait_sync(msg);
  1177. }
  1178. void Processor::smp_broadcast_halt()
  1179. {
  1180. // We don't want to use a message, because this could have been triggered
  1181. // by being out of memory and we might not be able to get a message
  1182. for_each(
  1183. [&](Processor& proc) {
  1184. proc.m_halt_requested.store(true, AK::MemoryOrder::memory_order_release);
  1185. });
  1186. // Now trigger an IPI on all other APs
  1187. APIC::the().broadcast_ipi();
  1188. }
  1189. void Processor::Processor::halt()
  1190. {
  1191. if (s_smp_enabled)
  1192. smp_broadcast_halt();
  1193. halt_this();
  1194. }
  1195. UNMAP_AFTER_INIT void Processor::deferred_call_pool_init()
  1196. {
  1197. size_t pool_count = sizeof(m_deferred_call_pool) / sizeof(m_deferred_call_pool[0]);
  1198. for (size_t i = 0; i < pool_count; i++) {
  1199. auto& entry = m_deferred_call_pool[i];
  1200. entry.next = i < pool_count - 1 ? &m_deferred_call_pool[i + 1] : nullptr;
  1201. new (entry.handler_storage) DeferredCallEntry::HandlerFunction;
  1202. entry.was_allocated = false;
  1203. }
  1204. m_pending_deferred_calls = nullptr;
  1205. m_free_deferred_call_pool_entry = &m_deferred_call_pool[0];
  1206. }
  1207. void Processor::deferred_call_return_to_pool(DeferredCallEntry* entry)
  1208. {
  1209. VERIFY(m_in_critical);
  1210. VERIFY(!entry->was_allocated);
  1211. entry->handler_value() = {};
  1212. entry->next = m_free_deferred_call_pool_entry;
  1213. m_free_deferred_call_pool_entry = entry;
  1214. }
  1215. DeferredCallEntry* Processor::deferred_call_get_free()
  1216. {
  1217. VERIFY(m_in_critical);
  1218. if (m_free_deferred_call_pool_entry) {
  1219. // Fast path, we have an entry in our pool
  1220. auto* entry = m_free_deferred_call_pool_entry;
  1221. m_free_deferred_call_pool_entry = entry->next;
  1222. VERIFY(!entry->was_allocated);
  1223. return entry;
  1224. }
  1225. auto* entry = new DeferredCallEntry;
  1226. new (entry->handler_storage) DeferredCallEntry::HandlerFunction;
  1227. entry->was_allocated = true;
  1228. return entry;
  1229. }
  1230. void Processor::deferred_call_execute_pending()
  1231. {
  1232. VERIFY(m_in_critical);
  1233. if (!m_pending_deferred_calls)
  1234. return;
  1235. auto* pending_list = m_pending_deferred_calls;
  1236. m_pending_deferred_calls = nullptr;
  1237. // We pulled the stack of pending deferred calls in LIFO order, so we need to reverse the list first
  1238. auto reverse_list =
  1239. [](DeferredCallEntry* list) -> DeferredCallEntry* {
  1240. DeferredCallEntry* rev_list = nullptr;
  1241. while (list) {
  1242. auto next = list->next;
  1243. list->next = rev_list;
  1244. rev_list = list;
  1245. list = next;
  1246. }
  1247. return rev_list;
  1248. };
  1249. pending_list = reverse_list(pending_list);
  1250. do {
  1251. pending_list->invoke_handler();
  1252. // Return the entry back to the pool, or free it
  1253. auto* next = pending_list->next;
  1254. if (pending_list->was_allocated) {
  1255. pending_list->handler_value().~Function();
  1256. delete pending_list;
  1257. } else
  1258. deferred_call_return_to_pool(pending_list);
  1259. pending_list = next;
  1260. } while (pending_list);
  1261. }
  1262. void Processor::deferred_call_queue_entry(DeferredCallEntry* entry)
  1263. {
  1264. VERIFY(m_in_critical);
  1265. entry->next = m_pending_deferred_calls;
  1266. m_pending_deferred_calls = entry;
  1267. }
  1268. void Processor::deferred_call_queue(Function<void()> callback)
  1269. {
  1270. // NOTE: If we are called outside of a critical section and outside
  1271. // of an irq handler, the function will be executed before we return!
  1272. ScopedCritical critical;
  1273. auto& cur_proc = Processor::current();
  1274. auto* entry = cur_proc.deferred_call_get_free();
  1275. entry->handler_value() = move(callback);
  1276. cur_proc.deferred_call_queue_entry(entry);
  1277. }
  1278. UNMAP_AFTER_INIT void Processor::gdt_init()
  1279. {
  1280. m_gdt_length = 0;
  1281. m_gdtr.address = nullptr;
  1282. m_gdtr.limit = 0;
  1283. write_raw_gdt_entry(0x0000, 0x00000000, 0x00000000);
  1284. #if ARCH(I386)
  1285. write_raw_gdt_entry(GDT_SELECTOR_CODE0, 0x0000ffff, 0x00cf9a00); // code0
  1286. write_raw_gdt_entry(GDT_SELECTOR_DATA0, 0x0000ffff, 0x00cf9200); // data0
  1287. write_raw_gdt_entry(GDT_SELECTOR_CODE3, 0x0000ffff, 0x00cffa00); // code3
  1288. write_raw_gdt_entry(GDT_SELECTOR_DATA3, 0x0000ffff, 0x00cff200); // data3
  1289. #else
  1290. write_raw_gdt_entry(GDT_SELECTOR_CODE0, 0x0000ffff, 0x00af9a00); // code0
  1291. write_raw_gdt_entry(GDT_SELECTOR_DATA0, 0x0000ffff, 0x00af9200); // data0
  1292. write_raw_gdt_entry(GDT_SELECTOR_DATA3, 0x0000ffff, 0x008ff200); // data3
  1293. write_raw_gdt_entry(GDT_SELECTOR_CODE3, 0x0000ffff, 0x00affa00); // code3
  1294. #endif
  1295. #if ARCH(I386)
  1296. Descriptor tls_descriptor {};
  1297. tls_descriptor.low = tls_descriptor.high = 0;
  1298. tls_descriptor.dpl = 3;
  1299. tls_descriptor.segment_present = 1;
  1300. tls_descriptor.granularity = 0;
  1301. tls_descriptor.operation_size64 = 0;
  1302. tls_descriptor.operation_size32 = 1;
  1303. tls_descriptor.descriptor_type = 1;
  1304. tls_descriptor.type = 2;
  1305. write_gdt_entry(GDT_SELECTOR_TLS, tls_descriptor); // tls3
  1306. Descriptor gs_descriptor {};
  1307. gs_descriptor.set_base(VirtualAddress { this });
  1308. gs_descriptor.set_limit(sizeof(Processor) - 1);
  1309. gs_descriptor.dpl = 0;
  1310. gs_descriptor.segment_present = 1;
  1311. gs_descriptor.granularity = 0;
  1312. gs_descriptor.operation_size64 = 0;
  1313. gs_descriptor.operation_size32 = 1;
  1314. gs_descriptor.descriptor_type = 1;
  1315. gs_descriptor.type = 2;
  1316. write_gdt_entry(GDT_SELECTOR_PROC, gs_descriptor); // gs0
  1317. #endif
  1318. Descriptor tss_descriptor {};
  1319. tss_descriptor.set_base(VirtualAddress { (size_t)&m_tss & 0xffffffff });
  1320. tss_descriptor.set_limit(sizeof(TSS) - 1);
  1321. tss_descriptor.dpl = 0;
  1322. tss_descriptor.segment_present = 1;
  1323. tss_descriptor.granularity = 0;
  1324. tss_descriptor.operation_size64 = 0;
  1325. tss_descriptor.operation_size32 = 1;
  1326. tss_descriptor.descriptor_type = 0;
  1327. tss_descriptor.type = Descriptor::SystemType::AvailableTSS;
  1328. write_gdt_entry(GDT_SELECTOR_TSS, tss_descriptor); // tss
  1329. #if ARCH(X86_64)
  1330. Descriptor tss_descriptor_part2 {};
  1331. tss_descriptor_part2.low = (size_t)&m_tss >> 32;
  1332. write_gdt_entry(GDT_SELECTOR_TSS_PART2, tss_descriptor_part2);
  1333. #endif
  1334. flush_gdt();
  1335. load_task_register(GDT_SELECTOR_TSS);
  1336. #if ARCH(X86_64)
  1337. MSR gs_base(MSR_GS_BASE);
  1338. gs_base.set((u64)this);
  1339. #else
  1340. asm volatile(
  1341. "mov %%ax, %%ds\n"
  1342. "mov %%ax, %%es\n"
  1343. "mov %%ax, %%fs\n"
  1344. "mov %%ax, %%ss\n" ::"a"(GDT_SELECTOR_DATA0)
  1345. : "memory");
  1346. set_gs(GDT_SELECTOR_PROC);
  1347. #endif
  1348. #if ARCH(I386)
  1349. // Make sure CS points to the kernel code descriptor.
  1350. // clang-format off
  1351. asm volatile(
  1352. "ljmpl $" __STRINGIFY(GDT_SELECTOR_CODE0) ", $sanity\n"
  1353. "sanity:\n");
  1354. // clang-format on
  1355. #endif
  1356. }
  1357. extern "C" void context_first_init([[maybe_unused]] Thread* from_thread, [[maybe_unused]] Thread* to_thread, [[maybe_unused]] TrapFrame* trap)
  1358. {
  1359. VERIFY(!are_interrupts_enabled());
  1360. VERIFY(is_kernel_mode());
  1361. dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context <-- from {} {} to {} {} (context_first_init)", VirtualAddress(from_thread), *from_thread, VirtualAddress(to_thread), *to_thread);
  1362. VERIFY(to_thread == Thread::current());
  1363. Scheduler::enter_current(*from_thread);
  1364. auto in_critical = to_thread->saved_critical();
  1365. VERIFY(in_critical > 0);
  1366. Processor::restore_in_critical(in_critical);
  1367. // Since we got here and don't have Scheduler::context_switch in the
  1368. // call stack (because this is the first time we switched into this
  1369. // context), we need to notify the scheduler so that it can release
  1370. // the scheduler lock. We don't want to enable interrupts at this point
  1371. // as we're still in the middle of a context switch. Doing so could
  1372. // trigger a context switch within a context switch, leading to a crash.
  1373. FlatPtr flags = trap->regs->flags();
  1374. Scheduler::leave_on_first_switch(flags & ~0x200);
  1375. }
  1376. extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread)
  1377. {
  1378. VERIFY(from_thread == to_thread || from_thread->state() != Thread::State::Running);
  1379. VERIFY(to_thread->state() == Thread::State::Running);
  1380. bool has_fxsr = Processor::current().has_feature(CPUFeature::FXSR);
  1381. Processor::set_current_thread(*to_thread);
  1382. auto& from_regs = from_thread->regs();
  1383. auto& to_regs = to_thread->regs();
  1384. // NOTE: IOPL should never be non-zero in any situation, so let's panic immediately
  1385. // instead of carrying on with elevated I/O privileges.
  1386. VERIFY(get_iopl_from_eflags(to_regs.flags()) == 0);
  1387. if (has_fxsr)
  1388. asm volatile("fxsave %0"
  1389. : "=m"(from_thread->fpu_state()));
  1390. else
  1391. asm volatile("fnsave %0"
  1392. : "=m"(from_thread->fpu_state()));
  1393. #if ARCH(I386)
  1394. from_regs.fs = get_fs();
  1395. from_regs.gs = get_gs();
  1396. set_fs(to_regs.fs);
  1397. set_gs(to_regs.gs);
  1398. #endif
  1399. if (from_thread->process().is_traced())
  1400. read_debug_registers_into(from_thread->debug_register_state());
  1401. if (to_thread->process().is_traced()) {
  1402. write_debug_registers_from(to_thread->debug_register_state());
  1403. } else {
  1404. clear_debug_registers();
  1405. }
  1406. auto& processor = Processor::current();
  1407. #if ARCH(I386)
  1408. auto& tls_descriptor = processor.get_gdt_entry(GDT_SELECTOR_TLS);
  1409. tls_descriptor.set_base(to_thread->thread_specific_data());
  1410. tls_descriptor.set_limit(to_thread->thread_specific_region_size());
  1411. #else
  1412. MSR fs_base_msr(MSR_FS_BASE);
  1413. fs_base_msr.set(to_thread->thread_specific_data().get());
  1414. #endif
  1415. if (from_regs.cr3 != to_regs.cr3)
  1416. write_cr3(to_regs.cr3);
  1417. to_thread->set_cpu(processor.id());
  1418. auto in_critical = to_thread->saved_critical();
  1419. VERIFY(in_critical > 0);
  1420. Processor::restore_in_critical(in_critical);
  1421. if (has_fxsr)
  1422. asm volatile("fxrstor %0" ::"m"(to_thread->fpu_state()));
  1423. else
  1424. asm volatile("frstor %0" ::"m"(to_thread->fpu_state()));
  1425. }
  1426. extern "C" FlatPtr do_init_context(Thread* thread, u32 flags)
  1427. {
  1428. VERIFY_INTERRUPTS_DISABLED();
  1429. thread->regs().set_flags(flags);
  1430. return Processor::current().init_context(*thread, true);
  1431. }
  1432. void Processor::assume_context(Thread& thread, FlatPtr flags)
  1433. {
  1434. dbgln_if(CONTEXT_SWITCH_DEBUG, "Assume context for thread {} {}", VirtualAddress(&thread), thread);
  1435. VERIFY_INTERRUPTS_DISABLED();
  1436. Scheduler::prepare_after_exec();
  1437. // in_critical() should be 2 here. The critical section in Process::exec
  1438. // and then the scheduler lock
  1439. VERIFY(Processor::in_critical() == 2);
  1440. do_assume_context(&thread, flags);
  1441. VERIFY_NOT_REACHED();
  1442. }
  1443. u64 Processor::time_spent_idle() const
  1444. {
  1445. return m_idle_thread->time_in_user() + m_idle_thread->time_in_kernel();
  1446. }
  1447. }