diff --git a/Kernel/CMakeLists.txt b/Kernel/CMakeLists.txt index 3a41fda5177..4a9cfe39fe0 100644 --- a/Kernel/CMakeLists.txt +++ b/Kernel/CMakeLists.txt @@ -103,6 +103,8 @@ set(KERNEL_SOURCES Storage/Partition/PartitionTable.cpp Storage/NVMe/NVMeController.cpp Storage/NVMe/NVMeNameSpace.cpp + Storage/NVMe/NVMeInterruptQueue.cpp + Storage/NVMe/NVMePollQueue.cpp Storage/NVMe/NVMeQueue.cpp Storage/StorageDevice.cpp Storage/RamdiskController.cpp diff --git a/Kernel/Storage/NVMe/NVMeController.cpp b/Kernel/Storage/NVMe/NVMeController.cpp index 0397273eeab..aab71dece35 100644 --- a/Kernel/Storage/NVMe/NVMeController.cpp +++ b/Kernel/Storage/NVMe/NVMeController.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -19,10 +20,10 @@ namespace Kernel { Atomic NVMeController::controller_id {}; -UNMAP_AFTER_INIT ErrorOr> NVMeController::try_initialize(const Kernel::PCI::DeviceIdentifier& device_identifier) +UNMAP_AFTER_INIT ErrorOr> NVMeController::try_initialize(const Kernel::PCI::DeviceIdentifier& device_identifier, bool is_queue_polled) { auto controller = TRY(adopt_nonnull_ref_or_enomem(new NVMeController(device_identifier))); - TRY(controller->initialize()); + TRY(controller->initialize(is_queue_polled)); NVMeController::controller_id++; return controller; } @@ -33,11 +34,11 @@ UNMAP_AFTER_INIT NVMeController::NVMeController(const PCI::DeviceIdentifier& dev { } -UNMAP_AFTER_INIT ErrorOr NVMeController::initialize() +UNMAP_AFTER_INIT ErrorOr NVMeController::initialize(bool is_queue_polled) { // Nr of queues = one queue per core auto nr_of_queues = Processor::count(); - auto irq = m_pci_device_id.interrupt_line().value(); + auto irq = is_queue_polled ? Optional {} : m_pci_device_id.interrupt_line().value(); PCI::enable_memory_space(m_pci_device_id.address()); PCI::enable_bus_mastering(m_pci_device_id.address()); @@ -62,7 +63,7 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::initialize() // Create an IO queue per core for (u32 cpuid = 0; cpuid < nr_of_queues; ++cpuid) { // qid is zero is used for admin queue - TRY(create_io_queue(irq, cpuid + 1)); + TRY(create_io_queue(cpuid + 1, irq)); } TRY(identify_and_init_namespaces()); return {}; @@ -253,7 +254,7 @@ void NVMeController::complete_current_request([[maybe_unused]] AsyncDeviceReques VERIFY_NOT_REACHED(); } -UNMAP_AFTER_INIT ErrorOr NVMeController::create_admin_queue(u8 irq) +UNMAP_AFTER_INIT ErrorOr NVMeController::create_admin_queue(Optional irq) { auto qdepth = get_admin_q_dept(); OwnPtr cq_dma_region; @@ -281,8 +282,6 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_admin_queue(u8 irq) } auto doorbell_regs = TRY(Memory::map_typed_writable(PhysicalAddress(m_bar + REG_SQ0TDBL_START))); - m_admin_queue = TRY(NVMeQueue::try_create(0, irq, qdepth, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs))); - m_controller_regs->acq = reinterpret_cast(AK::convert_between_host_and_little_endian(cq_dma_pages.first().paddr().as_ptr())); m_controller_regs->asq = reinterpret_cast(AK::convert_between_host_and_little_endian(sq_dma_pages.first().paddr().as_ptr())); @@ -291,12 +290,13 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_admin_queue(u8 irq) return EFAULT; } set_admin_queue_ready_flag(); - m_admin_queue->enable_interrupts(); + m_admin_queue = TRY(NVMeQueue::try_create(0, irq, qdepth, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs))); + dbgln_if(NVME_DEBUG, "NVMe: Admin queue created"); return {}; } -UNMAP_AFTER_INIT ErrorOr NVMeController::create_io_queue(u8 irq, u8 qid) +UNMAP_AFTER_INIT ErrorOr NVMeController::create_io_queue(u8 qid, Optional irq) { OwnPtr cq_dma_region; NonnullRefPtrVector cq_dma_pages; @@ -326,7 +326,8 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_io_queue(u8 irq, u8 qid) sub.create_cq.cqid = qid; // The queue size is 0 based sub.create_cq.qsize = AK::convert_between_host_and_little_endian(IO_QUEUE_SIZE - 1); - auto flags = QUEUE_IRQ_ENABLED | QUEUE_PHY_CONTIGUOUS; + auto flags = irq.has_value() ? QUEUE_IRQ_ENABLED : QUEUE_IRQ_DISABLED; + flags |= QUEUE_PHY_CONTIGUOUS; // TODO: Eventually move to MSI. // For now using pin based interrupts. Clear the first 16 bits // to use pin-based interrupts. @@ -340,7 +341,7 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_io_queue(u8 irq, u8 qid) sub.create_sq.sqid = qid; // The queue size is 0 based sub.create_sq.qsize = AK::convert_between_host_and_little_endian(IO_QUEUE_SIZE - 1); - auto flags = QUEUE_IRQ_ENABLED | QUEUE_PHY_CONTIGUOUS; + auto flags = QUEUE_PHY_CONTIGUOUS; sub.create_sq.cqid = qid; sub.create_sq.sq_flags = AK::convert_between_host_and_little_endian(flags); submit_admin_command(sub, true); @@ -350,7 +351,6 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_io_queue(u8 irq, u8 qid) auto doorbell_regs = TRY(Memory::map_typed_writable(PhysicalAddress(m_bar + queue_doorbell_offset))); m_queues.append(TRY(NVMeQueue::try_create(qid, irq, IO_QUEUE_SIZE, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs)))); - m_queues.last().enable_interrupts(); dbgln_if(NVME_DEBUG, "NVMe: Created IO Queue with QID{}", m_queues.size()); return {}; } diff --git a/Kernel/Storage/NVMe/NVMeController.h b/Kernel/Storage/NVMe/NVMeController.h index bc85786c100..d881e48f527 100644 --- a/Kernel/Storage/NVMe/NVMeController.h +++ b/Kernel/Storage/NVMe/NVMeController.h @@ -26,8 +26,8 @@ namespace Kernel { class NVMeController : public PCI::Device , public StorageController { public: - static ErrorOr> try_initialize(PCI::DeviceIdentifier const&); - ErrorOr initialize(); + static ErrorOr> try_initialize(PCI::DeviceIdentifier const&, bool is_queue_polled); + ErrorOr initialize(bool is_queue_polled); explicit NVMeController(PCI::DeviceIdentifier const&); RefPtr device(u32 index) const override; size_t devices_count() const override; @@ -58,8 +58,8 @@ public: private: ErrorOr identify_and_init_namespaces(); Tuple get_ns_features(IdentifyNamespace& identify_data_struct); - ErrorOr create_admin_queue(u8 irq); - ErrorOr create_io_queue(u8 irq, u8 qid); + ErrorOr create_admin_queue(Optional irq); + ErrorOr create_io_queue(u8 qid, Optional irq); void calculate_doorbell_stride() { m_dbl_stride = (m_controller_regs->cap >> CAP_DBL_SHIFT) & CAP_DBL_MASK; diff --git a/Kernel/Storage/NVMe/NVMeDefinitions.h b/Kernel/Storage/NVMe/NVMeDefinitions.h index 5ba4911e468..b7ce1416ced 100644 --- a/Kernel/Storage/NVMe/NVMeDefinitions.h +++ b/Kernel/Storage/NVMe/NVMeDefinitions.h @@ -125,6 +125,7 @@ enum IOCommandOpcode { // FLAGS static constexpr u8 QUEUE_PHY_CONTIGUOUS = (1 << 0); static constexpr u8 QUEUE_IRQ_ENABLED = (1 << 1); +static constexpr u8 QUEUE_IRQ_DISABLED = (0 << 1); struct [[gnu::packed]] NVMeCompletion { LittleEndian cmd_spec; diff --git a/Kernel/Storage/NVMe/NVMeInterruptQueue.cpp b/Kernel/Storage/NVMe/NVMeInterruptQueue.cpp new file mode 100644 index 00000000000..f3e5752c661 --- /dev/null +++ b/Kernel/Storage/NVMe/NVMeInterruptQueue.cpp @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2022, Pankaj R + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "NVMeInterruptQueue.h" +#include "Kernel/Devices/BlockDevice.h" +#include "NVMeDefinitions.h" +#include + +namespace Kernel { + +UNMAP_AFTER_INIT NVMeInterruptQueue::NVMeInterruptQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs) + : NVMeQueue(move(rw_dma_region), rw_dma_page, qid, q_depth, move(cq_dma_region), cq_dma_page, move(sq_dma_region), sq_dma_page, move(db_regs)) + , IRQHandler(irq) +{ + enable_irq(); +} + +bool NVMeInterruptQueue::handle_irq(const RegisterState&) +{ + SpinlockLocker lock(m_request_lock); + return process_cq() ? true : false; +} + +void NVMeInterruptQueue::submit_sqe(NVMeSubmission& sub) +{ + NVMeQueue::submit_sqe(sub); +} + +void NVMeInterruptQueue::complete_current_request(u16 status) +{ + VERIFY(m_request_lock.is_locked()); + + g_io_work->queue([this, status]() { + SpinlockLocker lock(m_request_lock); + auto current_request = m_current_request; + m_current_request.clear(); + if (status) { + lock.unlock(); + current_request->complete(AsyncBlockDeviceRequest::Failure); + return; + } + if (current_request->request_type() == AsyncBlockDeviceRequest::RequestType::Read) { + if (auto result = current_request->write_to_buffer(current_request->buffer(), m_rw_dma_region->vaddr().as_ptr(), 512 * current_request->block_count()); result.is_error()) { + lock.unlock(); + current_request->complete(AsyncDeviceRequest::MemoryFault); + return; + } + } + lock.unlock(); + current_request->complete(AsyncDeviceRequest::Success); + return; + }); +} +} diff --git a/Kernel/Storage/NVMe/NVMeInterruptQueue.h b/Kernel/Storage/NVMe/NVMeInterruptQueue.h new file mode 100644 index 00000000000..9b562efb583 --- /dev/null +++ b/Kernel/Storage/NVMe/NVMeInterruptQueue.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022, Pankaj R + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include + +namespace Kernel { + +class NVMeInterruptQueue : public NVMeQueue + , public IRQHandler { +public: + NVMeInterruptQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs); + void submit_sqe(NVMeSubmission& submission) override; + virtual ~NVMeInterruptQueue() override {}; + +private: + virtual void complete_current_request(u16 status) override; + bool handle_irq(RegisterState const&) override; +}; +} diff --git a/Kernel/Storage/NVMe/NVMePollQueue.cpp b/Kernel/Storage/NVMe/NVMePollQueue.cpp new file mode 100644 index 00000000000..8b1340a9b45 --- /dev/null +++ b/Kernel/Storage/NVMe/NVMePollQueue.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022, Pankaj R + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "NVMePollQueue.h" +#include "Kernel/Arch/x86/IO.h" +#include "Kernel/Devices/BlockDevice.h" +#include "NVMeDefinitions.h" + +namespace Kernel { +UNMAP_AFTER_INIT NVMePollQueue::NVMePollQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs) + : NVMeQueue(move(rw_dma_region), rw_dma_page, qid, q_depth, move(cq_dma_region), cq_dma_page, move(sq_dma_region), sq_dma_page, move(db_regs)) +{ +} + +void NVMePollQueue::submit_sqe(NVMeSubmission& sub) +{ + NVMeQueue::submit_sqe(sub); + SpinlockLocker lock_cq(m_cq_lock); + while (!process_cq()) { + IO::delay(1); + } +} + +void NVMePollQueue::complete_current_request(u16 status) +{ + auto current_request = m_current_request; + m_current_request.clear(); + if (status) { + current_request->complete(AsyncBlockDeviceRequest::Failure); + return; + } + if (current_request->request_type() == AsyncBlockDeviceRequest::RequestType::Read) { + if (auto result = current_request->write_to_buffer(current_request->buffer(), m_rw_dma_region->vaddr().as_ptr(), 512 * current_request->block_count()); result.is_error()) { + current_request->complete(AsyncDeviceRequest::MemoryFault); + return; + } + } + current_request->complete(AsyncDeviceRequest::Success); + return; +} +} diff --git a/Kernel/Storage/NVMe/NVMePollQueue.h b/Kernel/Storage/NVMe/NVMePollQueue.h new file mode 100644 index 00000000000..458f11c14f3 --- /dev/null +++ b/Kernel/Storage/NVMe/NVMePollQueue.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2022, Pankaj R + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include + +namespace Kernel { + +class NVMePollQueue : public NVMeQueue { +public: + NVMePollQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs); + void submit_sqe(NVMeSubmission& submission) override; + virtual ~NVMePollQueue() override {}; + +private: + virtual void complete_current_request(u16 status) override; +}; +} diff --git a/Kernel/Storage/NVMe/NVMeQueue.cpp b/Kernel/Storage/NVMe/NVMeQueue.cpp index 5289f6446d1..bf531f9224a 100644 --- a/Kernel/Storage/NVMe/NVMeQueue.cpp +++ b/Kernel/Storage/NVMe/NVMeQueue.cpp @@ -6,36 +6,38 @@ #include "NVMeQueue.h" #include "Kernel/StdLib.h" +#include "NVMeQueue.h" #include -#include #include -#include +#include +#include namespace Kernel { - -ErrorOr> NVMeQueue::try_create(u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs) +ErrorOr> NVMeQueue::try_create(u16 qid, Optional irq, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs) { // Note: Allocate DMA region for RW operation. For now the requests don't exceed more than 4096 bytes (Storage device takes care of it) RefPtr rw_dma_page; auto rw_dma_region = TRY(MM.allocate_dma_buffer_page("NVMe Queue Read/Write DMA"sv, Memory::Region::Access::ReadWrite, rw_dma_page)); - auto queue = TRY(adopt_nonnull_ref_or_enomem(new (nothrow) NVMeQueue(move(rw_dma_region), *rw_dma_page, qid, irq, q_depth, move(cq_dma_region), cq_dma_page, move(sq_dma_region), sq_dma_page, move(db_regs)))); + if (!irq.has_value()) { + auto queue = TRY(adopt_nonnull_ref_or_enomem(new (nothrow) NVMePollQueue(move(rw_dma_region), *rw_dma_page, qid, q_depth, move(cq_dma_region), cq_dma_page, move(sq_dma_region), sq_dma_page, move(db_regs)))); + return queue; + } + auto queue = TRY(adopt_nonnull_ref_or_enomem(new (nothrow) NVMeInterruptQueue(move(rw_dma_region), *rw_dma_page, qid, irq.value(), q_depth, move(cq_dma_region), cq_dma_page, move(sq_dma_region), sq_dma_page, move(db_regs)))); return queue; } -UNMAP_AFTER_INIT NVMeQueue::NVMeQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs) - : IRQHandler(irq) +UNMAP_AFTER_INIT NVMeQueue::NVMeQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs) + : m_current_request(nullptr) + , m_rw_dma_region(move(rw_dma_region)) , m_qid(qid) , m_admin_queue(qid == 0) - , m_irq(irq) , m_qdepth(q_depth) , m_cq_dma_region(move(cq_dma_region)) , m_cq_dma_page(cq_dma_page) , m_sq_dma_region(move(sq_dma_region)) , m_sq_dma_page(sq_dma_page) - , m_rw_dma_region(move(rw_dma_region)) , m_db_regs(move(db_regs)) , m_rw_dma_page(rw_dma_page) - , m_current_request(nullptr) { m_sqe_array = { reinterpret_cast(m_sq_dma_region->vaddr().as_ptr()), m_qdepth }; @@ -59,7 +61,7 @@ void NVMeQueue::update_cqe_head() } } -bool NVMeQueue::handle_irq(const RegisterState&) +u32 NVMeQueue::process_cq() { u32 nr_of_processed_cqes = 0; while (cqe_available()) { @@ -76,7 +78,6 @@ bool NVMeQueue::handle_irq(const RegisterState&) // everything is operated on a single request similar to BMIDE driver. // TODO: Remove this constraint eventually. VERIFY(cmdid == m_prev_sq_tail); - SpinlockLocker lock(m_request_lock); if (m_current_request) { complete_current_request(status); } @@ -86,7 +87,7 @@ bool NVMeQueue::handle_irq(const RegisterState&) if (nr_of_processed_cqes) { update_cq_doorbell(); } - return nr_of_processed_cqes ? true : false; + return nr_of_processed_cqes; } void NVMeQueue::submit_sqe(NVMeSubmission& sub) @@ -126,7 +127,7 @@ u16 NVMeQueue::submit_sync_sqe(NVMeSubmission& sub) index = m_qdepth - 1; } cqe_cid = m_cqe_array[index].command_id; - Scheduler::yield(); + IO::delay(1); } while (cid != cqe_cid); auto status = CQ_STATUS_FIELD(m_cqe_array[m_cq_head].status); @@ -171,29 +172,7 @@ void NVMeQueue::write(AsyncBlockDeviceRequest& request, u16 nsid, u64 index, u32 submit_sqe(sub); } -void NVMeQueue::complete_current_request(u16 status) +UNMAP_AFTER_INIT NVMeQueue::~NVMeQueue() { - VERIFY(m_request_lock.is_locked()); - - g_io_work->queue([this, status]() { - SpinlockLocker lock(m_request_lock); - auto current_request = m_current_request; - m_current_request.clear(); - if (status) { - lock.unlock(); - current_request->complete(AsyncBlockDeviceRequest::Failure); - return; - } - if (current_request->request_type() == AsyncBlockDeviceRequest::RequestType::Read) { - if (auto result = current_request->write_to_buffer(current_request->buffer(), m_rw_dma_region->vaddr().as_ptr(), 512 * current_request->block_count()); result.is_error()) { - lock.unlock(); - current_request->complete(AsyncDeviceRequest::MemoryFault); - return; - } - } - lock.unlock(); - current_request->complete(AsyncDeviceRequest::Success); - return; - }); } } diff --git a/Kernel/Storage/NVMe/NVMeQueue.h b/Kernel/Storage/NVMe/NVMeQueue.h index 85dbd7ddc38..66ca555da71 100644 --- a/Kernel/Storage/NVMe/NVMeQueue.h +++ b/Kernel/Storage/NVMe/NVMeQueue.h @@ -27,45 +27,47 @@ struct DoorbellRegister { }; class AsyncBlockDeviceRequest; -class NVMeQueue : public IRQHandler - , public RefCounted { +class NVMeQueue : public RefCounted { public: - static ErrorOr> try_create(u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs); + static ErrorOr> try_create(u16 qid, Optional irq, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs); bool is_admin_queue() { return m_admin_queue; }; - void submit_sqe(NVMeSubmission&); u16 submit_sync_sqe(NVMeSubmission&); void read(AsyncBlockDeviceRequest& request, u16 nsid, u64 index, u32 count); void write(AsyncBlockDeviceRequest& request, u16 nsid, u64 index, u32 count); - void enable_interrupts() { enable_irq(); }; - void disable_interrupts() { disable_irq(); }; + virtual void submit_sqe(NVMeSubmission&); + virtual ~NVMeQueue(); + +protected: + u32 process_cq(); + void update_sq_doorbell() + { + m_db_regs->sq_tail = m_sq_tail; + } + NVMeQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs); private: - NVMeQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, NonnullRefPtrVector cq_dma_page, OwnPtr sq_dma_region, NonnullRefPtrVector sq_dma_page, Memory::TypedMapping db_regs); - - virtual bool handle_irq(const RegisterState&) override; - bool cqe_available(); void update_cqe_head(); - void complete_current_request(u16 status); + virtual void complete_current_request(u16 status) = 0; void update_cq_doorbell() { m_db_regs->cq_head = m_cq_head; } - void update_sq_doorbell() - { - m_db_regs->sq_tail = m_sq_tail; - } +protected: + Spinlock m_cq_lock { LockRank::Interrupts }; + RefPtr m_current_request; + NonnullOwnPtr m_rw_dma_region; + Spinlock m_request_lock; +private: u16 m_qid {}; u8 m_cq_valid_phase { 1 }; u16 m_sq_tail {}; u16 m_prev_sq_tail {}; u16 m_cq_head {}; bool m_admin_queue { false }; - u8 m_irq {}; u32 m_qdepth {}; - Spinlock m_cq_lock { LockRank::Interrupts }; Spinlock m_sq_lock { LockRank::Interrupts }; OwnPtr m_cq_dma_region; NonnullRefPtrVector m_cq_dma_page; @@ -73,10 +75,7 @@ private: OwnPtr m_sq_dma_region; NonnullRefPtrVector m_sq_dma_page; Span m_cqe_array; - NonnullOwnPtr m_rw_dma_region; Memory::TypedMapping m_db_regs; NonnullRefPtr m_rw_dma_page; - Spinlock m_request_lock; - RefPtr m_current_request; }; } diff --git a/Kernel/Storage/StorageManagement.cpp b/Kernel/Storage/StorageManagement.cpp index cfcdc5b2748..1b67a2febd8 100644 --- a/Kernel/Storage/StorageManagement.cpp +++ b/Kernel/Storage/StorageManagement.cpp @@ -44,7 +44,7 @@ bool StorageManagement::boot_argument_contains_partition_uuid() return m_boot_argument.starts_with(partition_uuid_prefix); } -UNMAP_AFTER_INIT void StorageManagement::enumerate_controllers(bool force_pio) +UNMAP_AFTER_INIT void StorageManagement::enumerate_controllers(bool force_pio, bool nvme_poll) { VERIFY(m_controllers.is_empty()); @@ -60,10 +60,10 @@ UNMAP_AFTER_INIT void StorageManagement::enumerate_controllers(bool force_pio) static constexpr PCI::HardwareID vmd_device = { 0x8086, 0x9a0b }; if (device_identifier.hardware_id() == vmd_device) { auto controller = PCI::VolumeManagementDevice::must_create(device_identifier); - PCI::Access::the().add_host_controller_and_enumerate_attached_devices(move(controller), [this](PCI::DeviceIdentifier const& device_identifier) -> void { + PCI::Access::the().add_host_controller_and_enumerate_attached_devices(move(controller), [this, nvme_poll](PCI::DeviceIdentifier const& device_identifier) -> void { auto subclass_code = static_cast(device_identifier.subclass_code().value()); if (subclass_code == SubclassID::NVMeController) { - auto controller = NVMeController::try_initialize(device_identifier); + auto controller = NVMeController::try_initialize(device_identifier, nvme_poll); if (controller.is_error()) { dmesgln("Unable to initialize NVMe controller: {}", controller.error()); } else { @@ -84,7 +84,7 @@ UNMAP_AFTER_INIT void StorageManagement::enumerate_controllers(bool force_pio) m_controllers.append(AHCIController::initialize(device_identifier)); } if (subclass_code == SubclassID::NVMeController) { - auto controller = NVMeController::try_initialize(device_identifier); + auto controller = NVMeController::try_initialize(device_identifier, nvme_poll); if (controller.is_error()) { dmesgln("Unable to initialize NVMe controller: {}", controller.error()); } else { @@ -274,11 +274,11 @@ NonnullRefPtr StorageManagement::root_filesystem() const return file_system; } -UNMAP_AFTER_INIT void StorageManagement::initialize(StringView root_device, bool force_pio) +UNMAP_AFTER_INIT void StorageManagement::initialize(StringView root_device, bool force_pio, bool poll) { VERIFY(s_device_minor_number == 0); m_boot_argument = root_device; - enumerate_controllers(force_pio); + enumerate_controllers(force_pio, poll); enumerate_storage_devices(); enumerate_disk_partitions(); if (!boot_argument_contains_partition_uuid()) { diff --git a/Kernel/Storage/StorageManagement.h b/Kernel/Storage/StorageManagement.h index 29c7e6be197..32c1678dc2d 100644 --- a/Kernel/Storage/StorageManagement.h +++ b/Kernel/Storage/StorageManagement.h @@ -23,7 +23,7 @@ class StorageManagement { public: StorageManagement(); static bool initialized(); - void initialize(StringView boot_argument, bool force_pio); + void initialize(StringView boot_argument, bool force_pio, bool nvme_poll); static StorageManagement& the(); NonnullRefPtr root_filesystem() const; @@ -36,7 +36,7 @@ public: private: bool boot_argument_contains_partition_uuid(); - void enumerate_controllers(bool force_pio); + void enumerate_controllers(bool force_pio, bool nvme_poll); void enumerate_storage_devices(); void enumerate_disk_partitions(); diff --git a/Kernel/init.cpp b/Kernel/init.cpp index 09039903860..ae344405811 100644 --- a/Kernel/init.cpp +++ b/Kernel/init.cpp @@ -330,7 +330,7 @@ void init_stage2(void*) (void)SB16::try_detect_and_create(); AC97::detect(); - StorageManagement::the().initialize(kernel_command_line().root_device(), kernel_command_line().is_force_pio()); + StorageManagement::the().initialize(kernel_command_line().root_device(), kernel_command_line().is_force_pio(), kernel_command_line().is_nvme_polling_enabled()); if (VirtualFileSystem::the().mount_root(StorageManagement::the().root_filesystem()).is_error()) { PANIC("VirtualFileSystem::mount_root failed"); }