* BSD LICENSE
*
* Copyright (c) Intel Corporation. All rights reserved.
* Copyright (c) 2017, Western Digital Corporation or its affiliates.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "nvme_internal.h"
struct nvme_qpair_string {
uint16_t value;
const char *str;
};
static const struct nvme_qpair_string admin_opcode[] = {
{ NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" },
{ NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" },
{ NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" },
{ NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" },
{ NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" },
{ NVME_OPC_IDENTIFY, "IDENTIFY" },
{ NVME_OPC_ABORT, "ABORT" },
{ NVME_OPC_SET_FEATURES, "SET FEATURES" },
{ NVME_OPC_GET_FEATURES, "GET FEATURES" },
{ NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" },
{ NVME_OPC_NS_MANAGEMENT, "NAMESPACE MANAGEMENT" },
{ NVME_OPC_FIRMWARE_COMMIT, "FIRMWARE COMMIT" },
{ NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" },
{ NVME_OPC_NS_ATTACHMENT, "NAMESPACE ATTACHMENT" },
{ NVME_OPC_FORMAT_NVM, "FORMAT NVM" },
{ NVME_OPC_SECURITY_SEND, "SECURITY SEND" },
{ NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" },
{ 0xFFFF, "ADMIN COMMAND" }
};
static const struct nvme_qpair_string io_opcode[] = {
{ NVME_OPC_FLUSH, "FLUSH" },
{ NVME_OPC_WRITE, "WRITE" },
{ NVME_OPC_READ, "READ" },
{ NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" },
{ NVME_OPC_COMPARE, "COMPARE" },
{ NVME_OPC_WRITE_ZEROES, "WRITE ZEROES" },
{ NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" },
{ NVME_OPC_RESERVATION_REGISTER, "RESERVATION REGISTER" },
{ NVME_OPC_RESERVATION_REPORT, "RESERVATION REPORT" },
{ NVME_OPC_RESERVATION_ACQUIRE, "RESERVATION ACQUIRE" },
{ NVME_OPC_RESERVATION_RELEASE, "RESERVATION RELEASE" },
{ 0xFFFF, "IO COMMAND" }
};
static const struct nvme_qpair_string generic_status[] = {
{ NVME_SC_SUCCESS, "SUCCESS" },
{ NVME_SC_INVALID_OPCODE, "INVALID OPCODE" },
{ NVME_SC_INVALID_FIELD, "INVALID FIELD" },
{ NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" },
{ NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" },
{ NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" },
{ NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" },
{ NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" },
{ NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" },
{ NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" },
{ NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" },
{ NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" },
{ NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" },
{ NVME_SC_INVALID_SGL_SEG_DESCRIPTOR, "INVALID SGL SEGMENT DESCRIPTOR" },
{ NVME_SC_INVALID_NUM_SGL_DESCIRPTORS, "INVALID NUMBER OF SGL DESCRIPTORS" },
{ NVME_SC_DATA_SGL_LENGTH_INVALID, "DATA SGL LENGTH INVALID" },
{ NVME_SC_METADATA_SGL_LENGTH_INVALID, "METADATA SGL LENGTH INVALID" },
{ NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID, "SGL DESCRIPTOR TYPE INVALID" },
{ NVME_SC_INVALID_CONTROLLER_MEM_BUF, "INVALID CONTROLLER MEMORY BUFFER" },
{ NVME_SC_INVALID_PRP_OFFSET, "INVALID PRP OFFSET" },
{ NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED, "ATOMIC WRITE UNIT EXCEEDED" },
{ NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" },
{ NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" },
{ NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" },
{ NVME_SC_RESERVATION_CONFLICT, "RESERVATION CONFLICT" },
{ NVME_SC_FORMAT_IN_PROGRESS, "FORMAT IN PROGRESS" },
{ 0xFFFF, "GENERIC" }
};
static const struct nvme_qpair_string command_specific_status[] = {
{ NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" },
{ NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" },
{ NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" },
{ NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" },
{ NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED,"ASYNC LIMIT EXCEEDED" },
{ NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" },
{ NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" },
{ NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" },
{ NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" },
{ NVME_SC_INVALID_FORMAT, "INVALID FORMAT" },
{ NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET,"FIRMWARE REQUIRES CONVENTIONAL RESET" },
{ NVME_SC_INVALID_QUEUE_DELETION, "INVALID QUEUE DELETION" },
{ NVME_SC_FEATURE_ID_NOT_SAVEABLE, "FEATURE ID NOT SAVEABLE" },
{ NVME_SC_FEATURE_NOT_CHANGEABLE, "FEATURE NOT CHANGEABLE" },
{ NVME_SC_FEATURE_NOT_NAMESPACE_SPECIFIC,"FEATURE NOT NAMESPACE SPECIFIC" },
{ NVME_SC_FIRMWARE_REQ_NVM_RESET, "FIRMWARE REQUIRES NVM RESET" },
{ NVME_SC_FIRMWARE_REQ_RESET, "FIRMWARE REQUIRES RESET" },
{ NVME_SC_FIRMWARE_REQ_MAX_TIME_VIOLATION,"FIRMWARE REQUIRES MAX TIME VIOLATION" },
{ NVME_SC_FIRMWARE_ACTIVATION_PROHIBITED,"FIRMWARE ACTIVATION PROHIBITED" },
{ NVME_SC_OVERLAPPING_RANGE, "OVERLAPPING RANGE" },
{ NVME_SC_NAMESPACE_INSUFFICIENT_CAPACITY,"NAMESPACE INSUFFICIENT CAPACITY" },
{ NVME_SC_NAMESPACE_ID_UNAVAILABLE, "NAMESPACE ID UNAVAILABLE" },
{ NVME_SC_NAMESPACE_ALREADY_ATTACHED, "NAMESPACE ALREADY ATTACHED" },
{ NVME_SC_NAMESPACE_IS_PRIVATE, "NAMESPACE IS PRIVATE" },
{ NVME_SC_NAMESPACE_NOT_ATTACHED, "NAMESPACE NOT ATTACHED" },
{ NVME_SC_THINPROVISIONING_NOT_SUPPORTED,"THINPROVISIONING NOT SUPPORTED" },
{ NVME_SC_CONTROLLER_LIST_INVALID, "CONTROLLER LIST INVALID" },
{ NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" },
{ NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" },
{ NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" },
{ 0xFFFF, "COMMAND SPECIFIC" }
};
static const struct nvme_qpair_string media_error_status[] = {
{ NVME_SC_WRITE_FAULTS, "WRITE FAULTS" },
{ NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" },
{ NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" },
{ NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" },
{ NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" },
{ NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" },
{ NVME_SC_ACCESS_DENIED, "ACCESS DENIED" },
{ NVME_SC_DEALLOCATED_OR_UNWRITTEN_BLOCK, "DEALLOCATED OR UNWRITTEN BLOCK" },
{ 0xFFFF, "MEDIA ERROR" }
};
static inline bool nvme_qpair_is_admin_queue(struct nvme_qpair *qpair)
{
return qpair->id == 0;
}
static inline bool nvme_qpair_is_io_queue(struct nvme_qpair *qpair)
{
return qpair->id != 0;
}
static const char*nvme_qpair_get_string(const struct nvme_qpair_string *strings,
uint16_t value)
{
const struct nvme_qpair_string *entry;
entry = strings;
while (entry->value != 0xFFFF) {
if (entry->value == value)
return entry->str;
entry++;
}
return entry->str;
}
static void nvme_qpair_admin_qpair_print_command(struct nvme_qpair *qpair,
struct nvme_cmd *cmd)
{
nvme_info("%s (%02x) sqid:%d cid:%d nsid:%x cdw10:%08x cdw11:%08x\n",
nvme_qpair_get_string(admin_opcode, cmd->opc), cmd->opc,
qpair->id, cmd->cid,
cmd->nsid, cmd->cdw10, cmd->cdw11);
}
static void nvme_qpair_io_qpair_print_command(struct nvme_qpair *qpair,
struct nvme_cmd *cmd)
{
nvme_assert(qpair != NULL, "print_command: qpair == NULL\n");
nvme_assert(cmd != NULL, "print_command: cmd == NULL\n");
switch ((int)cmd->opc) {
case NVME_OPC_WRITE:
case NVME_OPC_READ:
case NVME_OPC_WRITE_UNCORRECTABLE:
case NVME_OPC_COMPARE:
nvme_info("%s sqid:%d cid:%d nsid:%d lba:%llu len:%d\n",
nvme_qpair_get_string(io_opcode, cmd->opc),
qpair->id, cmd->cid, cmd->nsid,
((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10,
(cmd->cdw12 & 0xFFFF) + 1);
break;
case NVME_OPC_FLUSH:
case NVME_OPC_DATASET_MANAGEMENT:
nvme_info("%s sqid:%d cid:%d nsid:%d\n",
nvme_qpair_get_string(io_opcode, cmd->opc),
qpair->id, cmd->cid, cmd->nsid);
break;
default:
nvme_info("%s (%02x) sqid:%d cid:%d nsid:%d\n",
nvme_qpair_get_string(io_opcode, cmd->opc),
cmd->opc, qpair->id, cmd->cid, cmd->nsid);
break;
}
}
static void nvme_qpair_print_command(struct nvme_qpair *qpair,
struct nvme_cmd *cmd)
{
nvme_assert(qpair != NULL, "qpair can not be NULL");
nvme_assert(cmd != NULL, "cmd can not be NULL");
if (nvme_qpair_is_admin_queue(qpair))
return nvme_qpair_admin_qpair_print_command(qpair, cmd);
return nvme_qpair_io_qpair_print_command(qpair, cmd);
}
static const char *get_status_string(uint16_t sct, uint16_t sc)
{
const struct nvme_qpair_string *entry;
switch (sct) {
case NVME_SCT_GENERIC:
entry = generic_status;
break;
case NVME_SCT_COMMAND_SPECIFIC:
entry = command_specific_status;
break;
case NVME_SCT_MEDIA_ERROR:
entry = media_error_status;
break;
case NVME_SCT_VENDOR_SPECIFIC:
return "VENDOR SPECIFIC";
default:
return "RESERVED";
}
return nvme_qpair_get_string(entry, sc);
}
static void nvme_qpair_print_completion(struct nvme_qpair *qpair,
struct nvme_cpl *cpl)
{
nvme_info("Cpl: %s (%02x/%02x) sqid:%d cid:%d "
"cdw0:%x sqhd:%04x p:%x m:%x dnr:%x\n",
get_status_string(cpl->status.sct, cpl->status.sc),
cpl->status.sct,
cpl->status.sc,
cpl->sqid,
cpl->cid,
cpl->cdw0,
cpl->sqhd,
cpl->status.p,
cpl->status.m,
cpl->status.dnr);
}
static bool nvme_qpair_completion_retry(const struct nvme_cpl *cpl)
{
* TODO: spec is not clear how commands that are aborted due
* to TLER will be marked. So for now, it seems
* NAMESPACE_NOT_READY is the only case where we should
* look at the DNR bit.
*/
switch ((int)cpl->status.sct) {
case NVME_SCT_GENERIC:
switch ((int)cpl->status.sc) {
case NVME_SC_NAMESPACE_NOT_READY:
case NVME_SC_FORMAT_IN_PROGRESS:
if (cpl->status.dnr)
return false;
return true;
case NVME_SC_INVALID_OPCODE:
case NVME_SC_INVALID_FIELD:
case NVME_SC_COMMAND_ID_CONFLICT:
case NVME_SC_DATA_TRANSFER_ERROR:
case NVME_SC_ABORTED_POWER_LOSS:
case NVME_SC_INTERNAL_DEVICE_ERROR:
case NVME_SC_ABORTED_BY_REQUEST:
case NVME_SC_ABORTED_SQ_DELETION:
case NVME_SC_ABORTED_FAILED_FUSED:
case NVME_SC_ABORTED_MISSING_FUSED:
case NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
case NVME_SC_COMMAND_SEQUENCE_ERROR:
case NVME_SC_LBA_OUT_OF_RANGE:
case NVME_SC_CAPACITY_EXCEEDED:
default:
return false;
}
case NVME_SCT_COMMAND_SPECIFIC:
case NVME_SCT_MEDIA_ERROR:
case NVME_SCT_VENDOR_SPECIFIC:
default:
return false;
}
}
static void nvme_qpair_construct_tracker(struct nvme_tracker *tr,
uint16_t cid, uint64_t phys_addr)
{
tr->prp_sgl_bus_addr = phys_addr + offsetof(struct nvme_tracker, u.prp);
tr->cid = cid;
tr->active = false;
}
static inline void nvme_qpair_copy_command(struct nvme_cmd *dst,
const struct nvme_cmd *src)
{
#if defined(__AVX__)
__m256i *d256 = (__m256i *)dst;
const __m256i *s256 = (const __m256i *)src;
_mm256_store_si256(&d256[0], _mm256_load_si256(&s256[0]));
_mm256_store_si256(&d256[1], _mm256_load_si256(&s256[1]));
#elif defined(__SSE2__)
__m128i *d128 = (__m128i *)dst;
const __m128i *s128 = (const __m128i *)src;
_mm_store_si128(&d128[0], _mm_load_si128(&s128[0]));
_mm_store_si128(&d128[1], _mm_load_si128(&s128[1]));
_mm_store_si128(&d128[2], _mm_load_si128(&s128[2]));
_mm_store_si128(&d128[3], _mm_load_si128(&s128[3]));
#else
*dst = *src;
#endif
}
static void nvme_qpair_submit_tracker(struct nvme_qpair *qpair,
struct nvme_tracker *tr)
{
struct nvme_request *req = tr->req;
* Set the tracker active and copy its command
* to the submission queue.
*/
nvme_debug("qpair %d: Submit command, tail %d, cid %d / %d\n",
qpair->id,
(int)qpair->sq_tail,
(int)tr->cid,
(int)tr->req->cmd.cid);
qpair->tr[tr->cid].active = true;
nvme_qpair_copy_command(&qpair->cmd[qpair->sq_tail], &req->cmd);
if (++qpair->sq_tail == qpair->entries)
qpair->sq_tail = 0;
nvme_wmb();
nvme_mmio_write_4(qpair->sq_tdbl, qpair->sq_tail);
}
static void nvme_qpair_complete_tracker(struct nvme_qpair *qpair,
struct nvme_tracker *tr,
struct nvme_cpl *cpl,
bool print_on_error)
{
struct nvme_request *req = tr->req;
bool retry, error;
if (!req) {
nvme_crit("tracker has no request\n");
qpair->tr[cpl->cid].active = false;
goto done;
}
error = nvme_cpl_is_error(cpl);
retry = error && nvme_qpair_completion_retry(cpl) &&
(req->retries < NVME_MAX_RETRY_COUNT);
if (error && print_on_error) {
nvme_qpair_print_command(qpair, &req->cmd);
nvme_qpair_print_completion(qpair, cpl);
}
qpair->tr[cpl->cid].active = false;
if (cpl->cid != req->cmd.cid)
nvme_crit("cpl and command CID mismatch (%d / %d)\n",
(int)cpl->cid, (int)req->cmd.cid);
if (retry) {
req->retries++;
nvme_qpair_submit_tracker(qpair, tr);
return;
}
if (req->cb_fn)
req->cb_fn(req->cb_arg, cpl);
nvme_request_free_locked(req);
done:
tr->req = NULL;
LIST_REMOVE(tr, list);
LIST_INSERT_HEAD(&qpair->free_tr, tr, list);
}
static void nvme_qpair_submit_queued_requests(struct nvme_qpair *qpair)
{
STAILQ_HEAD(, nvme_request) req_queue;
STAILQ_INIT(&req_queue);
pthread_mutex_lock(&qpair->lock);
STAILQ_CONCAT(&req_queue, &qpair->queued_req);
* If the controller is in the middle of a reset, don't
* try to submit queued requests - let the reset logic
* handle that instead.
*/
while (!qpair->ctrlr->resetting && LIST_FIRST(&qpair->free_tr)
&& !STAILQ_EMPTY(&req_queue)) {
struct nvme_request *req = STAILQ_FIRST(&req_queue);
STAILQ_REMOVE_HEAD(&req_queue, stailq);
pthread_mutex_unlock(&qpair->lock);
nvme_qpair_submit_request(qpair, req);
pthread_mutex_lock(&qpair->lock);
}
STAILQ_CONCAT(&qpair->queued_req, &req_queue);
pthread_mutex_unlock(&qpair->lock);
}
static void nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair,
struct nvme_tracker *tr,
uint32_t sct,
uint32_t sc,
uint32_t dnr,
bool print_on_error)
{
struct nvme_cpl cpl;
memset(&cpl, 0, sizeof(cpl));
cpl.sqid = qpair->id;
cpl.cid = tr->cid;
cpl.status.sct = sct;
cpl.status.sc = sc;
cpl.status.dnr = dnr;
nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
}
static void nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
struct nvme_request *req,
uint32_t sct, uint32_t sc,
bool print_on_error)
{
struct nvme_cpl cpl;
bool error;
memset(&cpl, 0, sizeof(cpl));
cpl.sqid = qpair->id;
cpl.status.sct = sct;
cpl.status.sc = sc;
error = nvme_cpl_is_error(&cpl);
if (error && print_on_error) {
nvme_qpair_print_command(qpair, &req->cmd);
nvme_qpair_print_completion(qpair, &cpl);
}
if (req->cb_fn)
req->cb_fn(req->cb_arg, &cpl);
nvme_request_free_locked(req);
}
static void nvme_qpair_abort_aers(struct nvme_qpair *qpair)
{
struct nvme_tracker *tr;
tr = LIST_FIRST(&qpair->outstanding_tr);
while (tr != NULL) {
nvme_assert(tr->req != NULL,
"tr->req == NULL in abort_aers\n");
if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) {
nvme_qpair_manual_complete_tracker(qpair, tr,
NVME_SCT_GENERIC,
NVME_SC_ABORTED_SQ_DELETION,
0, false);
tr = LIST_FIRST(&qpair->outstanding_tr);
continue;
}
tr = LIST_NEXT(tr, list);
}
}
static inline void _nvme_qpair_admin_qpair_destroy(struct nvme_qpair *qpair)
{
nvme_qpair_abort_aers(qpair);
}
static inline void _nvme_qpair_req_bad_phys(struct nvme_qpair *qpair,
struct nvme_tracker *tr)
{
* Bad vtophys translation, so abort this request
* and return immediately, without retry.
*/
nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
NVME_SC_INVALID_FIELD,
1, true);
}
* Build PRP list describing physically contiguous payload buffer.
*/
static int _nvme_qpair_build_contig_request(struct nvme_qpair *qpair,
struct nvme_request *req,
struct nvme_tracker *tr)
{
uint64_t phys_addr;
void *seg_addr;
uint32_t nseg, cur_nseg, modulo, unaligned;
void *md_payload;
void *payload = req->payload.u.contig + req->payload_offset;
phys_addr = nvme_mem_vtophys(payload);
if (phys_addr == NVME_VTOPHYS_ERROR) {
_nvme_qpair_req_bad_phys(qpair, tr);
return -1;
}
nseg = req->payload_size >> PAGE_SHIFT;
modulo = req->payload_size & (PAGE_SIZE - 1);
unaligned = phys_addr & (PAGE_SIZE - 1);
if (modulo || unaligned)
nseg += 1 + ((modulo + unaligned - 1) >> PAGE_SHIFT);
if (req->payload.md) {
md_payload = req->payload.md + req->md_offset;
tr->req->cmd.mptr = nvme_mem_vtophys(md_payload);
if (tr->req->cmd.mptr == NVME_VTOPHYS_ERROR) {
_nvme_qpair_req_bad_phys(qpair, tr);
return -1;
}
}
tr->req->cmd.psdt = NVME_PSDT_PRP;
tr->req->cmd.dptr.prp.prp1 = phys_addr;
if (nseg == 2) {
seg_addr = payload + PAGE_SIZE - unaligned;
tr->req->cmd.dptr.prp.prp2 = nvme_mem_vtophys(seg_addr);
} else if (nseg > 2) {
cur_nseg = 1;
tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;
while (cur_nseg < nseg) {
seg_addr = payload + cur_nseg * PAGE_SIZE - unaligned;
phys_addr = nvme_mem_vtophys(seg_addr);
if (phys_addr == NVME_VTOPHYS_ERROR) {
_nvme_qpair_req_bad_phys(qpair, tr);
return -1;
}
tr->u.prp[cur_nseg - 1] = phys_addr;
cur_nseg++;
}
}
return 0;
}
* Build SGL list describing scattered payload buffer.
*/
static int _nvme_qpair_build_hw_sgl_request(struct nvme_qpair *qpair,
struct nvme_request *req,
struct nvme_tracker *tr)
{
struct nvme_sgl_descriptor *sgl;
uint64_t phys_addr;
uint32_t remaining_transfer_len, length, nseg = 0;
int ret;
* Build scattered payloads.
*/
nvme_assert(req->payload_size != 0,
"cannot build SGL for zero-length transfer\n");
nvme_assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL,
"sgl payload type required\n");
nvme_assert(req->payload.u.sgl.reset_sgl_fn != NULL,
"sgl reset callback required\n");
nvme_assert(req->payload.u.sgl.next_sge_fn != NULL,
"sgl callback required\n");
req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg,
req->payload_offset);
sgl = tr->u.sgl;
req->cmd.psdt = NVME_PSDT_SGL_MPTR_SGL;
req->cmd.dptr.sgl1.unkeyed.subtype = 0;
remaining_transfer_len = req->payload_size;
while (remaining_transfer_len > 0) {
if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {
_nvme_qpair_req_bad_phys(qpair, tr);
return -1;
}
ret = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg,
&phys_addr, &length);
if (ret != 0) {
_nvme_qpair_req_bad_phys(qpair, tr);
return ret;
}
length = nvme_min(remaining_transfer_len, length);
remaining_transfer_len -= length;
sgl->unkeyed.type = NVME_SGL_TYPE_DATA_BLOCK;
sgl->unkeyed.length = length;
sgl->address = phys_addr;
sgl->unkeyed.subtype = 0;
sgl++;
nseg++;
}
if (nseg == 1) {
* The whole transfer can be described by a single Scatter
* Gather List descriptor. Use the special case described
* by the spec where SGL1's type is Data Block.
* This means the SGL in the tracker is not used at all,
* so copy the first (and only) SGL element into SGL1.
*/
req->cmd.dptr.sgl1.unkeyed.type = NVME_SGL_TYPE_DATA_BLOCK;
req->cmd.dptr.sgl1.address = tr->u.sgl[0].address;
req->cmd.dptr.sgl1.unkeyed.length = tr->u.sgl[0].unkeyed.length;
} else {
req->cmd.dptr.sgl1.unkeyed.type = NVME_SGL_TYPE_LAST_SEGMENT;
req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr;
req->cmd.dptr.sgl1.unkeyed.length =
nseg * sizeof(struct nvme_sgl_descriptor);
}
return 0;
}
* Build Physical Region Page list describing scattered payload buffer.
*/
static int _nvme_qpair_build_prps_sgl_request(struct nvme_qpair *qpair,
struct nvme_request *req,
struct nvme_tracker *tr)
{
uint64_t phys_addr, prp2 = 0;
uint32_t data_transferred, remaining_transfer_len, length;
uint32_t nseg, cur_nseg, total_nseg = 0, last_nseg = 0;
uint32_t modulo, unaligned, sge_count = 0;
int ret;
* Build scattered payloads.
*/
nvme_assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL,
"sgl payload type required\n");
nvme_assert(req->payload.u.sgl.reset_sgl_fn != NULL,
"sgl reset callback required\n");
req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg,
req->payload_offset);
remaining_transfer_len = req->payload_size;
while (remaining_transfer_len > 0) {
nvme_assert(req->payload.u.sgl.next_sge_fn != NULL,
"sgl callback required\n");
ret = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg,
&phys_addr, &length);
if (ret != 0) {
_nvme_qpair_req_bad_phys(qpair, tr);
return -1;
}
nvme_assert((phys_addr & 0x3) == 0, "address must be dword aligned\n");
nvme_assert((length >= remaining_transfer_len) || ((phys_addr + length) % PAGE_SIZE) == 0,
"All SGEs except last must end on a page boundary\n");
nvme_assert((sge_count == 0) || (phys_addr % PAGE_SIZE) == 0,
"All SGEs except first must start on a page boundary\n");
data_transferred = nvme_min(remaining_transfer_len, length);
nseg = data_transferred >> PAGE_SHIFT;
modulo = data_transferred & (PAGE_SIZE - 1);
unaligned = phys_addr & (PAGE_SIZE - 1);
if (modulo || unaligned)
nseg += 1 + ((modulo + unaligned - 1) >> PAGE_SHIFT);
if (total_nseg == 0) {
req->cmd.psdt = NVME_PSDT_PRP;
req->cmd.dptr.prp.prp1 = phys_addr;
}
total_nseg += nseg;
sge_count++;
remaining_transfer_len -= data_transferred;
if (total_nseg == 2) {
if (sge_count == 1)
tr->req->cmd.dptr.prp.prp2 = phys_addr +
PAGE_SIZE - unaligned;
else if (sge_count == 2)
tr->req->cmd.dptr.prp.prp2 = phys_addr;
prp2 = tr->req->cmd.dptr.prp.prp2;
} else if (total_nseg > 2) {
if (sge_count == 1)
cur_nseg = 1;
else
cur_nseg = 0;
tr->req->cmd.dptr.prp.prp2 =
(uint64_t)tr->prp_sgl_bus_addr;
while (cur_nseg < nseg) {
if (prp2) {
tr->u.prp[0] = prp2;
tr->u.prp[last_nseg + 1] = phys_addr +
cur_nseg * PAGE_SIZE - unaligned;
} else {
tr->u.prp[last_nseg] = phys_addr +
cur_nseg * PAGE_SIZE - unaligned;
}
last_nseg++;
cur_nseg++;
}
}
}
return 0;
}
static void _nvme_qpair_admin_qpair_enable(struct nvme_qpair *qpair)
{
struct nvme_tracker *tr, *tr_temp;
* Manually abort each outstanding admin command. Do not retry
* admin commands found here, since they will be left over from
* a controller reset and its likely the context in which the
* command was issued no longer applies.
*/
LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, tr_temp) {
nvme_info("Aborting outstanding admin command\n");
nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
NVME_SC_ABORTED_BY_REQUEST,
1 , true);
}
qpair->enabled = true;
}
static void _nvme_qpair_io_qpair_enable(struct nvme_qpair *qpair)
{
struct nvme_tracker *tr, *temp;
struct nvme_request *req;
qpair->enabled = true;
qpair->ctrlr->enabled_io_qpairs++;
while (!STAILQ_EMPTY(&qpair->queued_req)) {
req = STAILQ_FIRST(&qpair->queued_req);
STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
nvme_info("Aborting queued I/O command\n");
nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC,
NVME_SC_ABORTED_BY_REQUEST,
true);
}
LIST_FOREACH_SAFE(tr, &qpair->outstanding_tr, list, temp) {
nvme_info("Aborting outstanding I/O command\n");
nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
NVME_SC_ABORTED_BY_REQUEST,
0, true);
}
}
static inline void _nvme_qpair_admin_qpair_disable(struct nvme_qpair *qpair)
{
qpair->enabled = false;
nvme_qpair_abort_aers(qpair);
}
static inline void _nvme_qpair_io_qpair_disable(struct nvme_qpair *qpair)
{
qpair->enabled = false;
qpair->ctrlr->enabled_io_qpairs--;
}
* Reserve room for the submission queue
* in the controller memory buffer
*/
static int nvme_ctrlr_reserve_sq_in_cmb(struct nvme_ctrlr *ctrlr,
uint16_t entries,
uint64_t aligned, uint64_t *offset)
{
uint64_t round_offset;
const uint64_t length = entries * sizeof(struct nvme_cmd);
round_offset = ctrlr->cmb_current_offset;
round_offset = (round_offset + (aligned - 1)) & ~(aligned - 1);
if (round_offset + length > ctrlr->cmb_size)
return -1;
*offset = round_offset;
ctrlr->cmb_current_offset = round_offset + length;
return 0;
}
* Initialize a queue pair on the host side.
*/
int nvme_qpair_construct(struct nvme_ctrlr *ctrlr, struct nvme_qpair *qpair,
enum nvme_qprio qprio,
uint16_t entries, uint16_t trackers)
{
volatile uint32_t *doorbell_base;
struct nvme_tracker *tr;
uint64_t offset;
unsigned long phys_addr = 0;
uint16_t i;
int ret;
nvme_assert(entries != 0, "Invalid number of entries\n");
nvme_assert(trackers != 0, "Invalid trackers\n");
pthread_mutex_init(&qpair->lock, NULL);
qpair->entries = entries;
qpair->trackers = trackers;
qpair->qprio = qprio;
qpair->sq_in_cmb = false;
qpair->ctrlr = ctrlr;
if (ctrlr->opts.use_cmb_sqs) {
* Reserve room for the submission queue in ctrlr
* memory buffer.
*/
ret = nvme_ctrlr_reserve_sq_in_cmb(ctrlr, entries,
PAGE_SIZE,
&offset);
if (ret == 0) {
qpair->cmd = ctrlr->cmb_bar_virt_addr + offset;
qpair->cmd_bus_addr = ctrlr->cmb_bar_phys_addr + offset;
qpair->sq_in_cmb = true;
nvme_debug("Allocated qpair %d cmd in cmb at %p / 0x%llx\n",
qpair->id,
qpair->cmd, qpair->cmd_bus_addr);
}
}
if (qpair->sq_in_cmb == false) {
qpair->cmd =
nvme_mem_alloc_node(sizeof(struct nvme_cmd) * entries,
PAGE_SIZE, NVME_NODE_ID_ANY,
(unsigned long *) &qpair->cmd_bus_addr);
if (!qpair->cmd) {
nvme_err("Allocate qpair commands failed\n");
goto fail;
}
memset(qpair->cmd, 0, sizeof(struct nvme_cmd) * entries);
nvme_debug("Allocated qpair %d cmd %p / 0x%llx\n",
qpair->id,
qpair->cmd, qpair->cmd_bus_addr);
}
qpair->cpl = nvme_mem_alloc_node(sizeof(struct nvme_cpl) * entries,
PAGE_SIZE, NVME_NODE_ID_ANY,
(unsigned long *) &qpair->cpl_bus_addr);
if (!qpair->cpl) {
nvme_err("Allocate qpair completions failed\n");
goto fail;
}
memset(qpair->cpl, 0, sizeof(struct nvme_cpl) * entries);
nvme_debug("Allocated qpair %d cpl at %p / 0x%llx\n",
qpair->id,
qpair->cpl,
qpair->cpl_bus_addr);
doorbell_base = &ctrlr->regs->doorbell[0].sq_tdbl;
qpair->sq_tdbl = doorbell_base +
(2 * qpair->id + 0) * ctrlr->doorbell_stride_u32;
qpair->cq_hdbl = doorbell_base +
(2 * qpair->id + 1) * ctrlr->doorbell_stride_u32;
LIST_INIT(&qpair->free_tr);
LIST_INIT(&qpair->outstanding_tr);
STAILQ_INIT(&qpair->free_req);
STAILQ_INIT(&qpair->queued_req);
if (nvme_request_pool_construct(qpair)) {
nvme_err("Create request pool failed\n");
goto fail;
}
* Reserve space for all of the trackers in a single allocation.
* struct nvme_tracker must be padded so that its size is already
* a power of 2. This ensures the PRP list embedded in the nvme_tracker
* object will not span a 4KB boundary, while allowing access to
* trackers in tr[] via normal array indexing.
*/
qpair->tr = nvme_mem_alloc_node(sizeof(struct nvme_tracker) * trackers,
sizeof(struct nvme_tracker),
NVME_NODE_ID_ANY, &phys_addr);
if (!qpair->tr) {
nvme_err("Allocate request trackers failed\n");
goto fail;
}
memset(qpair->tr, 0, sizeof(struct nvme_tracker) * trackers);
nvme_debug("Allocated qpair %d trackers at %p / 0x%lx\n",
qpair->id, qpair->tr, phys_addr);
for (i = 0; i < trackers; i++) {
tr = &qpair->tr[i];
nvme_qpair_construct_tracker(tr, i, phys_addr);
LIST_INSERT_HEAD(&qpair->free_tr, tr, list);
phys_addr += sizeof(struct nvme_tracker);
}
nvme_qpair_reset(qpair);
return 0;
fail:
nvme_qpair_destroy(qpair);
return -1;
}
void nvme_qpair_destroy(struct nvme_qpair *qpair)
{
if (!qpair->ctrlr)
return;
if (nvme_qpair_is_admin_queue(qpair))
_nvme_qpair_admin_qpair_destroy(qpair);
if (qpair->cmd && !qpair->sq_in_cmb) {
nvme_free(qpair->cmd);
qpair->cmd = NULL;
}
if (qpair->cpl) {
nvme_free(qpair->cpl);
qpair->cpl = NULL;
}
if (qpair->tr) {
nvme_free(qpair->tr);
qpair->tr = NULL;
}
nvme_request_pool_destroy(qpair);
qpair->ctrlr = NULL;
pthread_mutex_destroy(&qpair->lock);
}
static bool nvme_qpair_enabled(struct nvme_qpair *qpair)
{
if (!qpair->enabled && !qpair->ctrlr->resetting)
nvme_qpair_enable(qpair);
return qpair->enabled;
}
int nvme_qpair_submit_request(struct nvme_qpair *qpair,
struct nvme_request *req)
{
struct nvme_tracker *tr;
struct nvme_request *child_req, *tmp;
struct nvme_ctrlr *ctrlr = qpair->ctrlr;
bool child_req_failed = false;
int ret = 0;
if (ctrlr->failed) {
nvme_request_free(req);
return ENXIO;
}
nvme_qpair_enabled(qpair);
if (req->child_reqs) {
* This is a splitted (parent) request. Submit all of the
* children but not the parent request itself, since the
* parent is the original unsplit request.
*/
TAILQ_FOREACH_SAFE(child_req, &req->children, child_tailq, tmp) {
if (!child_req_failed) {
ret = nvme_qpair_submit_request(qpair, child_req);
if (ret != 0)
child_req_failed = true;
} else {
* one child_req fails */
nvme_request_remove_child(req, child_req);
nvme_request_free(child_req);
}
}
return ret;
}
pthread_mutex_lock(&qpair->lock);
tr = LIST_FIRST(&qpair->free_tr);
if (tr == NULL || !qpair->enabled || !STAILQ_EMPTY(&qpair->queued_req)) {
* No tracker is available, the qpair is disabled due
* to an in-progress controller-level reset, or
* there are already queued requests.
*
* Put the request on the qpair's request queue to be
* processed when a tracker frees up via a command
* completion or when the controller reset is completed.
*/
STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
pthread_mutex_unlock(&qpair->lock);
if (tr)
nvme_qpair_submit_queued_requests(qpair);
return 0;
}
LIST_REMOVE(tr, list);
LIST_INSERT_HEAD(&qpair->outstanding_tr, tr, list);
tr->req = req;
req->cmd.cid = tr->cid;
if (req->payload_size == 0) {
ret = 0;
} else if (req->payload.type == NVME_PAYLOAD_TYPE_CONTIG) {
ret = _nvme_qpair_build_contig_request(qpair, req, tr);
} else if (req->payload.type == NVME_PAYLOAD_TYPE_SGL) {
if (ctrlr->flags & NVME_CTRLR_SGL_SUPPORTED)
ret = _nvme_qpair_build_hw_sgl_request(qpair, req, tr);
else
ret = _nvme_qpair_build_prps_sgl_request(qpair, req, tr);
} else {
nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
NVME_SC_INVALID_FIELD,
1 , true);
ret = -EINVAL;
}
if (ret == 0)
nvme_qpair_submit_tracker(qpair, tr);
pthread_mutex_unlock(&qpair->lock);
return ret;
}
* Poll for completion of NVMe commands submitted to the
* specified I/O queue pair.
*/
unsigned int nvme_qpair_poll(struct nvme_qpair *qpair,
unsigned int max_completions)
{
struct nvme_tracker *tr;
struct nvme_cpl *cpl;
uint32_t num_completions = 0;
if (!nvme_qpair_enabled(qpair))
* qpair is not enabled, likely because a controller reset is
* is in progress. Ignore the interrupt - any I/O that was
* associated with this interrupt will get retried when the
* reset is complete.
*/
return 0;
if ((max_completions == 0) ||
(max_completions > (qpair->entries - 1U)))
* max_completions == 0 means unlimited, but complete at most
* one queue depth batch of I/O at a time so that the completion
* queue doorbells don't wrap around.
*/
max_completions = qpair->entries - 1;
pthread_mutex_lock(&qpair->lock);
while (1) {
cpl = &qpair->cpl[qpair->cq_head];
if (cpl->status.p != qpair->phase)
break;
tr = &qpair->tr[cpl->cid];
if (tr->active) {
nvme_qpair_complete_tracker(qpair, tr, cpl, true);
} else {
nvme_info("cpl does not map to outstanding cmd\n");
nvme_qpair_print_completion(qpair, cpl);
nvme_panic("received completion for unknown cmd\n");
}
if (++qpair->cq_head == qpair->entries) {
qpair->cq_head = 0;
qpair->phase = !qpair->phase;
}
if (++num_completions == max_completions)
break;
}
if (num_completions > 0)
nvme_mmio_write_4(qpair->cq_hdbl, qpair->cq_head);
pthread_mutex_unlock(&qpair->lock);
if (!STAILQ_EMPTY(&qpair->queued_req))
nvme_qpair_submit_queued_requests(qpair);
return num_completions;
}
void nvme_qpair_reset(struct nvme_qpair *qpair)
{
pthread_mutex_lock(&qpair->lock);
qpair->sq_tail = qpair->cq_head = 0;
* First time through the completion queue, HW will set phase
* bit on completions to 1. So set this to 1 here, indicating
* we're looking for a 1 to know which entries have completed.
* we'll toggle the bit each time when the completion queue rolls over.
*/
qpair->phase = 1;
memset(qpair->cmd, 0, qpair->entries * sizeof(struct nvme_cmd));
memset(qpair->cpl, 0, qpair->entries * sizeof(struct nvme_cpl));
pthread_mutex_unlock(&qpair->lock);
}
void nvme_qpair_enable(struct nvme_qpair *qpair)
{
pthread_mutex_lock(&qpair->lock);
if (nvme_qpair_is_io_queue(qpair))
_nvme_qpair_io_qpair_enable(qpair);
else
_nvme_qpair_admin_qpair_enable(qpair);
pthread_mutex_unlock(&qpair->lock);
}
void nvme_qpair_disable(struct nvme_qpair *qpair)
{
pthread_mutex_lock(&qpair->lock);
if (nvme_qpair_is_io_queue(qpair))
_nvme_qpair_io_qpair_disable(qpair);
else
_nvme_qpair_admin_qpair_disable(qpair);
pthread_mutex_unlock(&qpair->lock);
}
void nvme_qpair_fail(struct nvme_qpair *qpair)
{
struct nvme_tracker *tr;
struct nvme_request *req;
pthread_mutex_lock(&qpair->lock);
while (!STAILQ_EMPTY(&qpair->queued_req)) {
nvme_notice("Failing queued I/O command\n");
req = STAILQ_FIRST(&qpair->queued_req);
STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC,
NVME_SC_ABORTED_BY_REQUEST,
true);
}
while (!LIST_EMPTY(&qpair->outstanding_tr)) {
* Do not remove the tracker. The abort_tracker path
* will do that for us.
*/
nvme_notice("Failing outstanding I/O command\n");
tr = LIST_FIRST(&qpair->outstanding_tr);
nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC,
NVME_SC_ABORTED_BY_REQUEST,
1, true);
}
pthread_mutex_unlock(&qpair->lock);
}
