NVMeController.cpp 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447
  1. /*
  2. * Copyright (c) 2021, Pankaj R <pankydev8@gmail.com>
  3. * Copyright (c) 2022, the SerenityOS developers.
  4. *
  5. * SPDX-License-Identifier: BSD-2-Clause
  6. */
  7. #include <AK/Format.h>
  8. #include <AK/Types.h>
  9. #include <Kernel/Arch/Delay.h>
  10. #include <Kernel/Arch/Interrupts.h>
  11. #include <Kernel/Arch/SafeMem.h>
  12. #include <Kernel/Boot/CommandLine.h>
  13. #include <Kernel/Bus/PCI/API.h>
  14. #include <Kernel/Bus/PCI/BarMapping.h>
  15. #include <Kernel/Devices/Device.h>
  16. #include <Kernel/Devices/Storage/NVMe/NVMeController.h>
  17. #include <Kernel/Devices/Storage/StorageManagement.h>
  18. #include <Kernel/Library/LockRefPtr.h>
  19. #include <Kernel/Sections.h>
  20. namespace Kernel {
  21. UNMAP_AFTER_INIT ErrorOr<NonnullRefPtr<NVMeController>> NVMeController::try_initialize(Kernel::PCI::DeviceIdentifier const& device_identifier, bool is_queue_polled)
  22. {
  23. auto controller = TRY(adopt_nonnull_ref_or_enomem(new NVMeController(device_identifier, StorageManagement::generate_relative_nvme_controller_id({}))));
  24. TRY(controller->initialize(is_queue_polled));
  25. return controller;
  26. }
  27. UNMAP_AFTER_INIT NVMeController::NVMeController(const PCI::DeviceIdentifier& device_identifier, u32 hardware_relative_controller_id)
  28. : PCI::Device(const_cast<PCI::DeviceIdentifier&>(device_identifier))
  29. , StorageController(hardware_relative_controller_id)
  30. {
  31. }
  32. UNMAP_AFTER_INIT ErrorOr<void> NVMeController::initialize(bool is_queue_polled)
  33. {
  34. // Nr of queues = one queue per core
  35. auto nr_of_queues = Processor::count();
  36. auto queue_type = is_queue_polled ? QueueType::Polled : QueueType::IRQ;
  37. PCI::enable_memory_space(device_identifier());
  38. PCI::enable_bus_mastering(device_identifier());
  39. m_bar = TRY(PCI::get_bar_address(device_identifier(), PCI::HeaderType0BaseRegister::BAR0));
  40. static_assert(sizeof(ControllerRegister) == REG_SQ0TDBL_START);
  41. static_assert(sizeof(NVMeSubmission) == (1 << SQ_WIDTH));
  42. // Map only until doorbell register for the controller
  43. // Queues will individually map the doorbell register respectively
  44. m_controller_regs = TRY(Memory::map_typed_writable<ControllerRegister volatile>(m_bar));
  45. auto caps = m_controller_regs->cap;
  46. m_ready_timeout = Duration::from_milliseconds((CAP_TO(caps) + 1) * 500); // CAP.TO is in 500ms units
  47. calculate_doorbell_stride();
  48. if (queue_type == QueueType::IRQ) {
  49. // IO queues + 1 admin queue
  50. m_irq_type = TRY(reserve_irqs(nr_of_queues + 1, true));
  51. }
  52. TRY(create_admin_queue(queue_type));
  53. VERIFY(m_admin_queue_ready == true);
  54. VERIFY(IO_QUEUE_SIZE < MQES(caps));
  55. dbgln_if(NVME_DEBUG, "NVMe: IO queue depth is: {}", IO_QUEUE_SIZE);
  56. TRY(identify_and_init_controller());
  57. // Create an IO queue per core
  58. for (u32 cpuid = 0; cpuid < nr_of_queues; ++cpuid) {
  59. // qid is zero is used for admin queue
  60. TRY(create_io_queue(cpuid + 1, queue_type));
  61. }
  62. TRY(identify_and_init_namespaces());
  63. return {};
  64. }
  65. bool NVMeController::wait_for_ready(bool expected_ready_bit_value)
  66. {
  67. constexpr size_t one_ms_io_delay = 1000;
  68. auto wait_iterations = m_ready_timeout.to_milliseconds();
  69. u32 expected_rdy = expected_ready_bit_value ? 1 : 0;
  70. while (((m_controller_regs->csts >> CSTS_RDY_BIT) & 0x1) != expected_rdy) {
  71. microseconds_delay(one_ms_io_delay);
  72. if (--wait_iterations == 0) {
  73. if (((m_controller_regs->csts >> CSTS_RDY_BIT) & 0x1) != expected_rdy) {
  74. dbgln_if(NVME_DEBUG, "NVMEController: CSTS.RDY still not set to {} after {} ms", expected_rdy, m_ready_timeout.to_milliseconds());
  75. return false;
  76. }
  77. break;
  78. }
  79. }
  80. return true;
  81. }
  82. ErrorOr<void> NVMeController::reset_controller()
  83. {
  84. if ((m_controller_regs->cc & (1 << CC_EN_BIT)) != 0) {
  85. // If the EN bit is already set, we need to wait
  86. // until the RDY bit is 1, otherwise the behavior is undefined
  87. if (!wait_for_ready(true))
  88. return Error::from_errno(ETIMEDOUT);
  89. }
  90. auto cc = m_controller_regs->cc;
  91. cc = cc & ~(1 << CC_EN_BIT);
  92. m_controller_regs->cc = cc;
  93. full_memory_barrier();
  94. // Wait until the RDY bit is cleared
  95. if (!wait_for_ready(false))
  96. return Error::from_errno(ETIMEDOUT);
  97. return {};
  98. }
  99. ErrorOr<void> NVMeController::start_controller()
  100. {
  101. if (!(m_controller_regs->cc & (1 << CC_EN_BIT))) {
  102. // If the EN bit is not already set, we need to wait
  103. // until the RDY bit is 0, otherwise the behavior is undefined
  104. if (!wait_for_ready(false))
  105. return Error::from_errno(ETIMEDOUT);
  106. }
  107. auto cc = m_controller_regs->cc;
  108. cc = cc | (1 << CC_EN_BIT);
  109. cc = cc | (CQ_WIDTH << CC_IOCQES_BIT);
  110. cc = cc | (SQ_WIDTH << CC_IOSQES_BIT);
  111. m_controller_regs->cc = cc;
  112. full_memory_barrier();
  113. // Wait until the RDY bit is set
  114. if (!wait_for_ready(true))
  115. return Error::from_errno(ETIMEDOUT);
  116. return {};
  117. }
  118. UNMAP_AFTER_INIT u32 NVMeController::get_admin_q_dept()
  119. {
  120. u32 aqa = m_controller_regs->aqa;
  121. // Queue depth is 0 based
  122. u32 q_depth = min(ACQ_SIZE(aqa), ASQ_SIZE(aqa)) + 1;
  123. dbgln_if(NVME_DEBUG, "NVMe: Admin queue depth is {}", q_depth);
  124. return q_depth;
  125. }
  126. UNMAP_AFTER_INIT ErrorOr<void> NVMeController::identify_and_init_namespaces()
  127. {
  128. RefPtr<Memory::PhysicalPage> prp_dma_buffer;
  129. OwnPtr<Memory::Region> prp_dma_region;
  130. auto namespace_data_struct = TRY(ByteBuffer::create_zeroed(NVMe_IDENTIFY_SIZE));
  131. u32 active_namespace_list[NVMe_IDENTIFY_SIZE / sizeof(u32)];
  132. {
  133. auto buffer = TRY(MM.allocate_dma_buffer_page("Identify PRP"sv, Memory::Region::Access::ReadWrite, prp_dma_buffer));
  134. prp_dma_region = move(buffer);
  135. }
  136. // Get the active namespace
  137. {
  138. NVMeSubmission sub {};
  139. u16 status = 0;
  140. sub.op = OP_ADMIN_IDENTIFY;
  141. sub.identify.data_ptr.prp1 = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(prp_dma_buffer->paddr().as_ptr()));
  142. sub.identify.cns = NVMe_CNS_ID_ACTIVE_NS & 0xff;
  143. status = submit_admin_command(sub, true);
  144. if (status) {
  145. dmesgln_pci(*this, "Failed to identify active namespace command");
  146. return EFAULT;
  147. }
  148. if (void* fault_at; !safe_memcpy(active_namespace_list, prp_dma_region->vaddr().as_ptr(), NVMe_IDENTIFY_SIZE, fault_at)) {
  149. return EFAULT;
  150. }
  151. }
  152. // Get the NAMESPACE attributes
  153. {
  154. NVMeSubmission sub {};
  155. IdentifyNamespace id_ns {};
  156. u16 status = 0;
  157. for (auto nsid : active_namespace_list) {
  158. memset(prp_dma_region->vaddr().as_ptr(), 0, NVMe_IDENTIFY_SIZE);
  159. // Invalid NS
  160. if (nsid == 0)
  161. break;
  162. sub.op = OP_ADMIN_IDENTIFY;
  163. sub.identify.data_ptr.prp1 = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(prp_dma_buffer->paddr().as_ptr()));
  164. sub.identify.cns = NVMe_CNS_ID_NS & 0xff;
  165. sub.identify.nsid = nsid;
  166. status = submit_admin_command(sub, true);
  167. if (status) {
  168. dmesgln_pci(*this, "Failed identify namespace with nsid {}", nsid);
  169. return EFAULT;
  170. }
  171. static_assert(sizeof(IdentifyNamespace) == NVMe_IDENTIFY_SIZE);
  172. if (void* fault_at; !safe_memcpy(&id_ns, prp_dma_region->vaddr().as_ptr(), NVMe_IDENTIFY_SIZE, fault_at)) {
  173. return EFAULT;
  174. }
  175. auto val = get_ns_features(id_ns);
  176. auto block_counts = val.get<0>();
  177. auto block_size = 1 << val.get<1>();
  178. dbgln_if(NVME_DEBUG, "NVMe: Block count is {} and Block size is {}", block_counts, block_size);
  179. m_namespaces.append(TRY(NVMeNameSpace::try_create(*this, m_queues, nsid, block_counts, block_size)));
  180. m_device_count++;
  181. dbgln_if(NVME_DEBUG, "NVMe: Initialized namespace with NSID: {}", nsid);
  182. }
  183. }
  184. return {};
  185. }
  186. ErrorOr<void> NVMeController::identify_and_init_controller()
  187. {
  188. RefPtr<Memory::PhysicalPage> prp_dma_buffer;
  189. OwnPtr<Memory::Region> prp_dma_region;
  190. IdentifyController ctrl {};
  191. {
  192. auto buffer = TRY(MM.allocate_dma_buffer_page("Identify PRP"sv, Memory::Region::Access::ReadWrite, prp_dma_buffer));
  193. prp_dma_region = move(buffer);
  194. }
  195. // Check if the controller supports shadow doorbell
  196. {
  197. NVMeSubmission sub {};
  198. u16 status = 0;
  199. sub.op = OP_ADMIN_IDENTIFY;
  200. sub.identify.data_ptr.prp1 = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(prp_dma_buffer->paddr().as_ptr()));
  201. sub.identify.cns = NVMe_CNS_ID_CTRL & 0xff;
  202. status = submit_admin_command(sub, true);
  203. if (status) {
  204. dmesgln_pci(*this, "Failed to identify active namespace command");
  205. return EFAULT;
  206. }
  207. if (void* fault_at; !safe_memcpy(&ctrl, prp_dma_region->vaddr().as_ptr(), NVMe_IDENTIFY_SIZE, fault_at)) {
  208. return EFAULT;
  209. }
  210. }
  211. if (ctrl.oacs & ID_CTRL_SHADOW_DBBUF_MASK) {
  212. OwnPtr<Memory::Region> dbbuf_dma_region;
  213. OwnPtr<Memory::Region> eventidx_dma_region;
  214. {
  215. auto buffer = TRY(MM.allocate_dma_buffer_page("shadow dbbuf"sv, Memory::Region::Access::ReadWrite, m_dbbuf_shadow_page));
  216. dbbuf_dma_region = move(buffer);
  217. memset(dbbuf_dma_region->vaddr().as_ptr(), 0, PAGE_SIZE);
  218. }
  219. {
  220. auto buffer = TRY(MM.allocate_dma_buffer_page("eventidx"sv, Memory::Region::Access::ReadWrite, m_dbbuf_eventidx_page));
  221. eventidx_dma_region = move(buffer);
  222. memset(eventidx_dma_region->vaddr().as_ptr(), 0, PAGE_SIZE);
  223. }
  224. {
  225. NVMeSubmission sub {};
  226. sub.op = OP_ADMIN_DBBUF_CONFIG;
  227. sub.dbbuf_cmd.data_ptr.prp1 = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(m_dbbuf_shadow_page->paddr().as_ptr()));
  228. sub.dbbuf_cmd.data_ptr.prp2 = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(m_dbbuf_eventidx_page->paddr().as_ptr()));
  229. submit_admin_command(sub, true);
  230. }
  231. dbgln_if(NVME_DEBUG, "Shadow doorbell Enabled!");
  232. }
  233. return {};
  234. }
  235. UNMAP_AFTER_INIT Tuple<u64, u8> NVMeController::get_ns_features(IdentifyNamespace& identify_data_struct)
  236. {
  237. auto flbas = identify_data_struct.flbas & FLBA_SIZE_MASK;
  238. auto namespace_size = identify_data_struct.nsze;
  239. auto lba_format = identify_data_struct.lbaf[flbas];
  240. auto lba_size = (lba_format & LBA_SIZE_MASK) >> 16;
  241. return Tuple<u64, u8>(namespace_size, lba_size);
  242. }
  243. LockRefPtr<StorageDevice> NVMeController::device(u32 index) const
  244. {
  245. return m_namespaces.at(index);
  246. }
  247. size_t NVMeController::devices_count() const
  248. {
  249. return m_device_count;
  250. }
  251. ErrorOr<void> NVMeController::reset()
  252. {
  253. TRY(reset_controller());
  254. TRY(start_controller());
  255. return {};
  256. }
  257. ErrorOr<void> NVMeController::shutdown()
  258. {
  259. return Error::from_errno(ENOTIMPL);
  260. }
  261. void NVMeController::complete_current_request([[maybe_unused]] AsyncDeviceRequest::RequestResult result)
  262. {
  263. VERIFY_NOT_REACHED();
  264. }
  265. UNMAP_AFTER_INIT ErrorOr<void> NVMeController::create_admin_queue(QueueType queue_type)
  266. {
  267. auto qdepth = get_admin_q_dept();
  268. OwnPtr<Memory::Region> cq_dma_region;
  269. Vector<NonnullRefPtr<Memory::PhysicalPage>> cq_dma_pages;
  270. OwnPtr<Memory::Region> sq_dma_region;
  271. Vector<NonnullRefPtr<Memory::PhysicalPage>> sq_dma_pages;
  272. auto cq_size = round_up_to_power_of_two(CQ_SIZE(qdepth), 4096);
  273. auto sq_size = round_up_to_power_of_two(SQ_SIZE(qdepth), 4096);
  274. auto maybe_error = reset_controller();
  275. if (maybe_error.is_error()) {
  276. dmesgln_pci(*this, "Failed to reset the NVMe controller");
  277. return maybe_error;
  278. }
  279. {
  280. auto buffer = TRY(MM.allocate_dma_buffer_pages(cq_size, "Admin CQ queue"sv, Memory::Region::Access::ReadWrite, cq_dma_pages));
  281. cq_dma_region = move(buffer);
  282. }
  283. // Phase bit is important to determine completion, so zero out the space
  284. // so that we don't get any garbage phase bit value
  285. memset(cq_dma_region->vaddr().as_ptr(), 0, cq_size);
  286. {
  287. auto buffer = TRY(MM.allocate_dma_buffer_pages(sq_size, "Admin SQ queue"sv, Memory::Region::Access::ReadWrite, sq_dma_pages));
  288. sq_dma_region = move(buffer);
  289. }
  290. auto doorbell_regs = TRY(Memory::map_typed_writable<DoorbellRegister volatile>(m_bar.offset(REG_SQ0TDBL_START)));
  291. Doorbell doorbell = {
  292. .mmio_reg = move(doorbell_regs),
  293. .dbbuf_shadow = {},
  294. .dbbuf_eventidx = {},
  295. };
  296. m_controller_regs->acq = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(cq_dma_pages.first()->paddr().as_ptr()));
  297. m_controller_regs->asq = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(sq_dma_pages.first()->paddr().as_ptr()));
  298. Optional<u8> irq;
  299. if (queue_type == QueueType::IRQ)
  300. irq = TRY(allocate_irq(0)); // Admin queue always uses the 0th index when using MSIx
  301. maybe_error = start_controller();
  302. if (maybe_error.is_error()) {
  303. dmesgln_pci(*this, "Failed to restart the NVMe controller");
  304. return maybe_error;
  305. }
  306. set_admin_queue_ready_flag();
  307. m_admin_queue = TRY(NVMeQueue::try_create(*this, 0, irq, qdepth, move(cq_dma_region), move(sq_dma_region), move(doorbell), queue_type));
  308. dbgln_if(NVME_DEBUG, "NVMe: Admin queue created");
  309. return {};
  310. }
  311. UNMAP_AFTER_INIT ErrorOr<void> NVMeController::create_io_queue(u8 qid, QueueType queue_type)
  312. {
  313. OwnPtr<Memory::Region> cq_dma_region;
  314. Vector<NonnullRefPtr<Memory::PhysicalPage>> cq_dma_pages;
  315. OwnPtr<Memory::Region> sq_dma_region;
  316. Vector<NonnullRefPtr<Memory::PhysicalPage>> sq_dma_pages;
  317. auto cq_size = round_up_to_power_of_two(CQ_SIZE(IO_QUEUE_SIZE), 4096);
  318. auto sq_size = round_up_to_power_of_two(SQ_SIZE(IO_QUEUE_SIZE), 4096);
  319. {
  320. auto buffer = TRY(MM.allocate_dma_buffer_pages(cq_size, "IO CQ queue"sv, Memory::Region::Access::ReadWrite, cq_dma_pages));
  321. cq_dma_region = move(buffer);
  322. }
  323. // Phase bit is important to determine completion, so zero out the space
  324. // so that we don't get any garbage phase bit value
  325. memset(cq_dma_region->vaddr().as_ptr(), 0, cq_size);
  326. {
  327. auto buffer = TRY(MM.allocate_dma_buffer_pages(sq_size, "IO SQ queue"sv, Memory::Region::Access::ReadWrite, sq_dma_pages));
  328. sq_dma_region = move(buffer);
  329. }
  330. {
  331. NVMeSubmission sub {};
  332. sub.op = OP_ADMIN_CREATE_COMPLETION_QUEUE;
  333. sub.create_cq.prp1 = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(cq_dma_pages.first()->paddr().as_ptr()));
  334. sub.create_cq.cqid = qid;
  335. // The queue size is 0 based
  336. sub.create_cq.qsize = AK::convert_between_host_and_little_endian(IO_QUEUE_SIZE - 1);
  337. auto flags = (queue_type == QueueType::IRQ) ? QUEUE_IRQ_ENABLED : QUEUE_IRQ_DISABLED;
  338. flags |= QUEUE_PHY_CONTIGUOUS;
  339. // When using MSIx interrupts, qid is used as an index into the interrupt table
  340. if (m_irq_type.has_value() && m_irq_type.value() != PCI::InterruptType::PIN)
  341. sub.create_cq.irq_vector = qid;
  342. else
  343. sub.create_cq.irq_vector = 0;
  344. sub.create_cq.cq_flags = AK::convert_between_host_and_little_endian(flags & 0xFFFF);
  345. submit_admin_command(sub, true);
  346. }
  347. {
  348. NVMeSubmission sub {};
  349. sub.op = OP_ADMIN_CREATE_SUBMISSION_QUEUE;
  350. sub.create_sq.prp1 = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(sq_dma_pages.first()->paddr().as_ptr()));
  351. sub.create_sq.sqid = qid;
  352. // The queue size is 0 based
  353. sub.create_sq.qsize = AK::convert_between_host_and_little_endian(IO_QUEUE_SIZE - 1);
  354. auto flags = QUEUE_PHY_CONTIGUOUS;
  355. sub.create_sq.cqid = qid;
  356. sub.create_sq.sq_flags = AK::convert_between_host_and_little_endian(flags);
  357. submit_admin_command(sub, true);
  358. }
  359. auto queue_doorbell_offset = (2 * qid) * (4 << m_dbl_stride);
  360. auto doorbell_regs = TRY(Memory::map_typed_writable<DoorbellRegister volatile>(m_bar.offset(REG_SQ0TDBL_START + queue_doorbell_offset)));
  361. Memory::TypedMapping<DoorbellRegister> shadow_doorbell_regs {};
  362. Memory::TypedMapping<DoorbellRegister> eventidx_doorbell_regs {};
  363. if (!m_dbbuf_shadow_page.is_null()) {
  364. shadow_doorbell_regs = TRY(Memory::map_typed_writable<DoorbellRegister>(m_dbbuf_shadow_page->paddr().offset(queue_doorbell_offset)));
  365. eventidx_doorbell_regs = TRY(Memory::map_typed_writable<DoorbellRegister>(m_dbbuf_eventidx_page->paddr().offset(queue_doorbell_offset)));
  366. }
  367. Doorbell doorbell = {
  368. .mmio_reg = move(doorbell_regs),
  369. .dbbuf_shadow = move(shadow_doorbell_regs),
  370. .dbbuf_eventidx = move(eventidx_doorbell_regs),
  371. };
  372. auto irq = TRY(allocate_irq(qid));
  373. m_queues.append(TRY(NVMeQueue::try_create(*this, qid, irq, IO_QUEUE_SIZE, move(cq_dma_region), move(sq_dma_region), move(doorbell), queue_type)));
  374. dbgln_if(NVME_DEBUG, "NVMe: Created IO Queue with QID{}", m_queues.size());
  375. return {};
  376. }
  377. }