NVMeQueue.cpp 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178
  1. /*
  2. * Copyright (c) 2021, Pankaj R <pankydev8@gmail.com>
  3. *
  4. * SPDX-License-Identifier: BSD-2-Clause
  5. */
  6. #include "NVMeQueue.h"
  7. #include "Kernel/StdLib.h"
  8. #include "NVMeQueue.h"
  9. #include <Kernel/Arch/x86/IO.h>
  10. #include <Kernel/Storage/NVMe/NVMeController.h>
  11. #include <Kernel/Storage/NVMe/NVMeInterruptQueue.h>
  12. #include <Kernel/Storage/NVMe/NVMePollQueue.h>
  13. namespace Kernel {
  14. ErrorOr<NonnullRefPtr<NVMeQueue>> NVMeQueue::try_create(u16 qid, Optional<u8> irq, u32 q_depth, OwnPtr<Memory::Region> cq_dma_region, NonnullRefPtrVector<Memory::PhysicalPage> cq_dma_page, OwnPtr<Memory::Region> sq_dma_region, NonnullRefPtrVector<Memory::PhysicalPage> sq_dma_page, Memory::TypedMapping<volatile DoorbellRegister> db_regs)
  15. {
  16. // Note: Allocate DMA region for RW operation. For now the requests don't exceed more than 4096 bytes (Storage device takes care of it)
  17. RefPtr<Memory::PhysicalPage> rw_dma_page;
  18. auto rw_dma_region = TRY(MM.allocate_dma_buffer_page("NVMe Queue Read/Write DMA"sv, Memory::Region::Access::ReadWrite, rw_dma_page));
  19. if (!irq.has_value()) {
  20. auto queue = TRY(adopt_nonnull_ref_or_enomem(new (nothrow) NVMePollQueue(move(rw_dma_region), *rw_dma_page, qid, q_depth, move(cq_dma_region), cq_dma_page, move(sq_dma_region), sq_dma_page, move(db_regs))));
  21. return queue;
  22. }
  23. auto queue = TRY(adopt_nonnull_ref_or_enomem(new (nothrow) NVMeInterruptQueue(move(rw_dma_region), *rw_dma_page, qid, irq.value(), q_depth, move(cq_dma_region), cq_dma_page, move(sq_dma_region), sq_dma_page, move(db_regs))));
  24. return queue;
  25. }
  26. UNMAP_AFTER_INIT NVMeQueue::NVMeQueue(NonnullOwnPtr<Memory::Region> rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u32 q_depth, OwnPtr<Memory::Region> cq_dma_region, NonnullRefPtrVector<Memory::PhysicalPage> cq_dma_page, OwnPtr<Memory::Region> sq_dma_region, NonnullRefPtrVector<Memory::PhysicalPage> sq_dma_page, Memory::TypedMapping<volatile DoorbellRegister> db_regs)
  27. : m_current_request(nullptr)
  28. , m_rw_dma_region(move(rw_dma_region))
  29. , m_qid(qid)
  30. , m_admin_queue(qid == 0)
  31. , m_qdepth(q_depth)
  32. , m_cq_dma_region(move(cq_dma_region))
  33. , m_cq_dma_page(cq_dma_page)
  34. , m_sq_dma_region(move(sq_dma_region))
  35. , m_sq_dma_page(sq_dma_page)
  36. , m_db_regs(move(db_regs))
  37. , m_rw_dma_page(rw_dma_page)
  38. {
  39. m_sqe_array = { reinterpret_cast<NVMeSubmission*>(m_sq_dma_region->vaddr().as_ptr()), m_qdepth };
  40. m_cqe_array = { reinterpret_cast<NVMeCompletion*>(m_cq_dma_region->vaddr().as_ptr()), m_qdepth };
  41. }
  42. bool NVMeQueue::cqe_available()
  43. {
  44. return PHASE_TAG(m_cqe_array[m_cq_head].status) == m_cq_valid_phase;
  45. }
  46. void NVMeQueue::update_cqe_head()
  47. {
  48. // To prevent overflow, use a temp variable
  49. u32 temp_cq_head = m_cq_head + 1;
  50. if (temp_cq_head == m_qdepth) {
  51. m_cq_head = 0;
  52. m_cq_valid_phase ^= 1;
  53. } else {
  54. m_cq_head = temp_cq_head;
  55. }
  56. }
  57. u32 NVMeQueue::process_cq()
  58. {
  59. u32 nr_of_processed_cqes = 0;
  60. while (cqe_available()) {
  61. u16 status;
  62. u16 cmdid;
  63. ++nr_of_processed_cqes;
  64. status = CQ_STATUS_FIELD(m_cqe_array[m_cq_head].status);
  65. cmdid = m_cqe_array[m_cq_head].command_id;
  66. dbgln_if(NVME_DEBUG, "NVMe: Completion with status {:x} and command identifier {}. CQ_HEAD: {}", status, cmdid, m_cq_head);
  67. // TODO: We don't use AsyncBlockDevice requests for admin queue as it is only applicable for a block device (NVMe namespace)
  68. // But admin commands precedes namespace creation. Unify requests to avoid special conditions
  69. if (m_admin_queue == false) {
  70. // As the block layer calls are now sync (as we wait on each requests),
  71. // everything is operated on a single request similar to BMIDE driver.
  72. // TODO: Remove this constraint eventually.
  73. VERIFY(cmdid == m_prev_sq_tail);
  74. if (m_current_request) {
  75. complete_current_request(status);
  76. }
  77. }
  78. update_cqe_head();
  79. }
  80. if (nr_of_processed_cqes) {
  81. update_cq_doorbell();
  82. }
  83. return nr_of_processed_cqes;
  84. }
  85. void NVMeQueue::submit_sqe(NVMeSubmission& sub)
  86. {
  87. SpinlockLocker lock(m_sq_lock);
  88. // For now let's use sq tail as a unique command id.
  89. sub.cmdid = m_sq_tail;
  90. m_prev_sq_tail = m_sq_tail;
  91. memcpy(&m_sqe_array[m_sq_tail], &sub, sizeof(NVMeSubmission));
  92. {
  93. u32 temp_sq_tail = m_sq_tail + 1;
  94. if (temp_sq_tail == m_qdepth)
  95. m_sq_tail = 0;
  96. else
  97. m_sq_tail = temp_sq_tail;
  98. }
  99. dbgln_if(NVME_DEBUG, "NVMe: Submission with command identifier {}. SQ_TAIL: {}", sub.cmdid, m_sq_tail);
  100. full_memory_barrier();
  101. update_sq_doorbell();
  102. }
  103. u16 NVMeQueue::submit_sync_sqe(NVMeSubmission& sub)
  104. {
  105. // For now let's use sq tail as a unique command id.
  106. u16 cqe_cid;
  107. u16 cid = m_sq_tail;
  108. submit_sqe(sub);
  109. do {
  110. int index;
  111. {
  112. SpinlockLocker lock(m_cq_lock);
  113. index = m_cq_head - 1;
  114. if (index < 0)
  115. index = m_qdepth - 1;
  116. }
  117. cqe_cid = m_cqe_array[index].command_id;
  118. IO::delay(1);
  119. } while (cid != cqe_cid);
  120. auto status = CQ_STATUS_FIELD(m_cqe_array[m_cq_head].status);
  121. return status;
  122. }
  123. void NVMeQueue::read(AsyncBlockDeviceRequest& request, u16 nsid, u64 index, u32 count)
  124. {
  125. NVMeSubmission sub {};
  126. SpinlockLocker m_lock(m_request_lock);
  127. m_current_request = request;
  128. sub.op = OP_NVME_READ;
  129. sub.rw.nsid = nsid;
  130. sub.rw.slba = AK::convert_between_host_and_little_endian(index);
  131. // No. of lbas is 0 based
  132. sub.rw.length = AK::convert_between_host_and_little_endian((count - 1) & 0xFFFF);
  133. sub.rw.data_ptr.prp1 = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(m_rw_dma_page->paddr().as_ptr()));
  134. full_memory_barrier();
  135. submit_sqe(sub);
  136. }
  137. void NVMeQueue::write(AsyncBlockDeviceRequest& request, u16 nsid, u64 index, u32 count)
  138. {
  139. NVMeSubmission sub {};
  140. SpinlockLocker m_lock(m_request_lock);
  141. m_current_request = request;
  142. if (auto result = m_current_request->read_from_buffer(m_current_request->buffer(), m_rw_dma_region->vaddr().as_ptr(), 512 * m_current_request->block_count()); result.is_error()) {
  143. complete_current_request(AsyncDeviceRequest::MemoryFault);
  144. return;
  145. }
  146. sub.op = OP_NVME_WRITE;
  147. sub.rw.nsid = nsid;
  148. sub.rw.slba = AK::convert_between_host_and_little_endian(index);
  149. // No. of lbas is 0 based
  150. sub.rw.length = AK::convert_between_host_and_little_endian((count - 1) & 0xFFFF);
  151. sub.rw.data_ptr.prp1 = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(m_rw_dma_page->paddr().as_ptr()));
  152. full_memory_barrier();
  153. submit_sqe(sub);
  154. }
  155. UNMAP_AFTER_INIT NVMeQueue::~NVMeQueue()
  156. {
  157. }
  158. }