From a5caef861e2e693cee635af2cdfd35bc6f782da5 Mon Sep 17 00:00:00 2001
From: suweifeng <suweifeng1@huawei.com>
Date: Mon, 19 Aug 2024 08:13:04 +0000
Subject: [PATCH 12/12] adapt for spdk-24.01
Adapt for spdk-24.01:
1, fix compile error
2, adapt for spdk change in bdev_nvme_self.c
Signed-off-by: suweifeng <suweifeng1@huawei.com>
include/spdk/nvme.h | 2 -
include/spdk/thread.h | 16 +-
lib/bdev/bdev.c | 36 +-
lib/bdev/bdev_self.c | 3 +-
lib/event/reactor.c | 5 -
lib/nvme/nvme_ctrlr.c | 7 +-
lib/nvme/nvme_pcie.c | 4 +-
lib/thread/thread.c | 3 +
module/bdev/nvme/bdev_nvme.c | 51 +-
module/bdev/nvme/bdev_nvme.h | 996 +++++++++++-----------
module/bdev/nvme/bdev_nvme_self.c | 1282 +++++++++++++++--------------
module/bdev/nvme/bdev_nvme_self.h | 86 +-
12 files changed, 1305 insertions(+), 1186 deletions(-)
@@ -4280,7 +4280,6 @@ struct spdk_nvme_qpair;
int32_t nvme_ctrlr_get_info(const char *ctrlName, struct nvme_ctrlr_info **ppCtrlr);
struct spdk_nvme_ctrlr *spdk_nvme_ctrlr_get_by_name(const char *ctrlname);
struct spdk_nvme_ctrlr *spdk_nvme_ctrlr_get_by_ctrlr(const struct nvme_bdev_ctrlr *nvme_bdev_ctrlr);
-struct nvme_bdev_ctrlr *nvme_ctrlr_get_by_name(const char *ctrlname);
void nvme_ctrlr_clear_iostat_by_name(const char *ctrlname);
void nvme_ctrlr_clear_iostat_all(void);
struct nvme_bdev_ctrlr *bdev_nvme_get_ctrlr_by_bdev_desc(void *bdev);
@@ -4301,7 +4300,6 @@ bool spdk_nvme_ctrlr_is_ns_manage_supported(struct spdk_nvme_ctrlr *ctrlr);
bool spdk_nvme_ctrlr_is_format_supported(struct spdk_nvme_ctrlr *ctrlr);
bool spdk_nvme_ctrlr_is_format_all_ns(struct spdk_nvme_ctrlr *ctrlr);
bool spdk_nvme_ctrlr_is_directive_supported(struct spdk_nvme_ctrlr *ctrlr);
-bool spdk_nvme_ctrlr_is_streams_supported(struct spdk_nvme_ctrlr *ctrlr);
int32_t spdk_nvme_ctrlr_identify_directives(struct spdk_nvme_ctrlr *ctrlr, uint16_t nsid,
void *payload);
int32_t spdk_nvme_ctrlr_enable_streams(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid);
@@ -45,6 +45,20 @@ enum spdk_thread_poller_rc {
struct spdk_thread;
#ifdef SPDK_CONFIG_APP_RW
+enum spdk_thread_state {
+ SPDK_THREAD_STATE_INITIALIZED,
+ /* The thread is processing poller and message by spdk_thread_poll(). */
+ SPDK_THREAD_STATE_RUNNING,
+
+ /* The thread is in the process of termination. It reaps unregistering
+ * * poller are releasing I/O channel.
+ * */
+ SPDK_THREAD_STATE_EXITING,
+
+ /* The thread is exited. It is ready to call spdk_thread_destroy(). */
+ SPDK_THREAD_STATE_EXITED,
+};
+
struct spdk_iodev_thread_info {
struct spdk_thread *thread;
volatile int32_t state;
@@ -54,8 +68,6 @@ extern struct spdk_iodev_thread_info lcore_thread_info[RTE_MAX_LCORE];
void spdk_reactors_use(bool useOrNot);
-bool spdk_get_reactor_type(void);
-
void spdk_set_thread_exited(struct spdk_thread *thread);
uint32_t spdk_get_channel_ref(void *io_ch);
@@ -4,6 +4,11 @@
* Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
*/
+#ifdef SPDK_CONFIG_APP_RW
+#define SECUREC_NEED_ERRNO_TYPE
+#include <securec.h>
+#endif
+
#include "spdk/stdinc.h"
#include "spdk/bdev.h"
@@ -28,7 +33,6 @@
#ifdef SPDK_CONFIG_APP_RW
#include "spdk/stdinc.h"
#include "spdk/barrier.h"
-#include <securec.h>
#include "spdk_internal/bdev_stat.h"
#endif
@@ -3453,10 +3457,10 @@ _bdev_io_submit(void *ctx)
struct spdk_bdev_io_stat *stat = &bdev_ch->stat;
if (bdev_ch->io_outstanding > 0) {
stat->pre_ticks = stat->cur_ticks;
- stat->cur_ticks = tsc;
+ stat->cur_ticks = bdev_io->internal.submit_tsc;
stat->io_ticks += stat->cur_ticks - stat->pre_ticks;
} else {
- stat->cur_ticks = tsc;
+ stat->cur_ticks = bdev_io->internal.submit_tsc;
}
#endif
@@ -5320,6 +5324,12 @@ bdev_read_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch
bdev_io->u.bdev.memory_domain = NULL;
bdev_io->u.bdev.memory_domain_ctx = NULL;
bdev_io->u.bdev.accel_sequence = NULL;
+#else
+ struct libstorage_io *io = (struct libstorage_io *)cb_arg;
+ bdev_build_contig_io(SPDK_BDEV_IO_TYPE_READ_NVME, buf, md_buf, offset_blocks, num_blocks,
+ io, bdev_io);
+ cb_arg = &io->location;
+#endif
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
@@ -5415,10 +5425,12 @@ bdev_readv_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *c
bdev_io->u.bdev.md_buf = md_buf;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io->u.bdev.offset_blocks = offset_blocks;
-#else
+#ifdef SPDK_CONFIG_APP_RW
struct libstorage_io *io = (struct libstorage_io *)cb_arg;
- bdev_build_contig_io(SPDK_BDEV_IO_TYPE_READ_NVME, buf, md_buf, offset_blocks, num_blocks,
- io, bdev_io);
+ bdev_io->type = SPDK_BDEV_IO_TYPE_READV_NVME;
+ bdev_io->u.bdev.nbytes = io->nbytes;
+ bdev_io->driver_ctx[SPDK_BDEV_IO_ACTION_PI] = io->pi_action;
+ bdev_io->driver_ctx[SPDK_BDEV_IO_ACTION_FUA] = io->fua;
cb_arg = &io->location;
#endif
bdev_io_init(bdev_io, bdev, cb_arg, cb);
@@ -6057,11 +6069,9 @@ spdk_bdev_comparev_and_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = __io_ch_to_bdev_ch(ch);
-#ifndef SPDK_CONFIG_APP_RW
if (!desc->write) {
return -EBADF;
}
-#endif
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
@@ -6086,12 +6096,6 @@ spdk_bdev_comparev_and_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io
bdev_io->u.bdev.md_buf = NULL;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io->u.bdev.offset_blocks = offset_blocks;
-#else
- LIBSTORAGE_IO_T *io = (struct libstorage_io *)cb_arg;
- bdev_build_contig_io(SPDK_BDEV_IO_TYPE_WRITE_NVME, buf, md_buf, offset_blocks, num_blocks,
- io, bdev_io);
- cb_arg = &io->location;
-#endif
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io->u.bdev.memory_domain = NULL;
bdev_io->u.bdev.memory_domain_ctx = NULL;
@@ -10405,9 +10409,9 @@ spdk_bdev_io_get_pool(size_t nbytes)
}
if (nbytes <= SPDK_BDEV_SMALL_BUF_MAX_SIZE + SPDK_BDEV_SMALL_BUF_WITH_MAX_MD) {
- pool = g_bdev_mgr.buf_small_pool;
+ pool = NULL;
} else {
- pool = g_bdev_mgr.buf_large_pool;
+ pool = NULL;
}
return pool;
@@ -11,6 +11,8 @@
* GNU General Public License for more details.
*/
+#define SECUREC_NEED_ERRNO_TYPE
+#include <securec.h>
#include "bdev_internal.h"
#include "spdk/stdinc.h"
@@ -20,7 +22,6 @@
#include "spdk/nvme_spec.h"
#include "spdk/log.h"
-#include <securec.h>
#include "spdk_internal/bdev_stat.h"
pthread_mutex_t *g_io_stat_map_mutex = NULL;
@@ -1027,10 +1027,6 @@ reactor_run(void *arg)
}
}
}
-#ifdef SPDK_CONFIG_APP_RW
- /* When all thread in reactor is finish, inform libstorage to release resource. */
- lcore_thread_info[reactor->lcore].state = SPDK_THREAD_STATE_EXITED;
-#endif
return 0;
}
@@ -1095,7 +1091,6 @@ spdk_reactors_start(void)
spdk_env_thread_wait_all();
g_reactor_state = SPDK_REACTOR_STATE_SHUTDOWN;
-#endif
}
static void
@@ -5366,6 +5366,7 @@ nvme_ctrlr_destruct_ublock(struct spdk_nvme_ctrlr *ctrlr)
{
struct spdk_nvme_qpair *qpair = NULL;
struct spdk_nvme_qpair *tmp = NULL;
+ struct spdk_nvme_ns *ns, *tmp_ns;
SPDK_DEBUGLOG(nvme, "Prepare to destruct SSD: %s\n", ctrlr->trid.traddr);
TAILQ_FOREACH_SAFE(qpair, &ctrlr->active_io_qpairs, tailq, tmp) {
@@ -5373,7 +5374,11 @@ nvme_ctrlr_destruct_ublock(struct spdk_nvme_ctrlr *ctrlr)
}
nvme_ctrlr_free_doorbell_buffer(ctrlr);
- nvme_ctrlr_destruct_namespaces(ctrlr);
+ RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) {
+ nvme_ctrlr_destruct_namespace(ctrlr, ns->id);
+ RB_REMOVE(nvme_ns_tree, &ctrlr->ns, ns);
+ spdk_free(ns);
+ }
spdk_bit_array_free(&ctrlr->free_io_qids);
nvme_transport_ctrlr_destruct(ctrlr);
}
@@ -606,7 +606,6 @@ nvme_pcie_ctrlr_unmap_pmr(struct nvme_pcie_ctrlr *pctrlr)
return rc;
}
-#ifndef SPDK_CONFIG_APP_RW
static int
nvme_pcie_ctrlr_config_pmr(struct spdk_nvme_ctrlr *ctrlr, bool enable)
{
@@ -678,7 +677,6 @@ nvme_pcie_ctrlr_config_pmr(struct spdk_nvme_ctrlr *ctrlr, bool enable)
return 0;
}
-#endif
static int
nvme_pcie_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr)
@@ -872,6 +870,7 @@ nvme_pcie_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
enum_ctx.has_pci_addr = true;
}
+#ifndef SPDK_CONFIG_APP_RW
/* Only the primary process can monitor hotplug. */
if (spdk_process_is_primary()) {
if (_nvme_pcie_hotplug_monitor(probe_ctx) > 0) {
@@ -881,6 +880,7 @@ nvme_pcie_ctrlr_scan(struct spdk_nvme_probe_ctx *probe_ctx,
return 0;
}
}
+#endif
if (enum_ctx.has_pci_addr == false) {
return spdk_pci_enumerate(spdk_pci_nvme_get_driver(),
@@ -90,7 +90,9 @@ struct spdk_poller {
char name[SPDK_MAX_POLLER_NAME_LEN + 1];
};
+#ifndef SPDK_CONFIG_APP_RW
enum spdk_thread_state {
+ SPDK_THREAD_STATE_INITIALIZED,
/* The thread is processing poller and message by spdk_thread_poll(). */
SPDK_THREAD_STATE_RUNNING,
@@ -102,6 +104,7 @@ enum spdk_thread_state {
/* The thread is exited. It is ready to call spdk_thread_destroy(). */
SPDK_THREAD_STATE_EXITED,
};
+#endif
struct spdk_thread {
uint64_t tsc_last;
@@ -44,6 +44,7 @@
static int bdev_nvme_config_json(struct spdk_json_write_ctx *w);
+#ifndef SPDK_CONFIG_APP_RW
struct nvme_bdev_io {
/** array of iovecs to transfer. */
struct iovec *iovs;
@@ -101,6 +102,7 @@ struct nvme_bdev_io {
/* Current tsc at submit time. */
uint64_t submit_tsc;
};
+#endif
struct nvme_probe_skip_entry {
struct spdk_nvme_transport_id trid;
@@ -149,10 +151,16 @@ static struct spdk_poller *g_hotplug_probe_poller;
static struct spdk_nvme_probe_ctx *g_hotplug_probe_ctx;
#ifdef SPDK_CONFIG_APP_RW
bool g_useCUSE = false;
+struct nvme_probe_ctx {
+ size_t count;
+ struct spdk_nvme_transport_id trids[NVME_MAX_CONTROLLERS];
+ struct spdk_nvme_host_id hostids[NVME_MAX_CONTROLLERS];
+ const char *names[NVME_MAX_CONTROLLERS];
+ uint32_t prchk_flags[NVME_MAX_CONTROLLERS];
+ const char *hostnqn;
+};
#endif
-static void nvme_ctrlr_populate_namespaces(struct nvme_ctrlr *nvme_ctrlr,
- struct nvme_async_probe_ctx *ctx);
static void nvme_ctrlr_populate_namespaces_done(struct nvme_ctrlr *nvme_ctrlr,
struct nvme_async_probe_ctx *ctx);
static int bdev_nvme_library_init(void);
@@ -203,7 +211,7 @@ static int bdev_nvme_failover_ctrlr(struct nvme_ctrlr *nvme_ctrlr);
void remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr);
static int nvme_ctrlr_read_ana_log_page(struct nvme_ctrlr *nvme_ctrlr);
-static struct nvme_ns *nvme_ns_alloc(void);
+struct nvme_ns *nvme_ns_alloc(void);
static void nvme_ns_free(struct nvme_ns *ns);
static int
@@ -336,7 +344,7 @@ nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns)
return RB_NEXT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
}
-static struct nvme_ctrlr *
+struct nvme_ctrlr *
nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid)
{
struct nvme_bdev_ctrlr *nbdev_ctrlr;
@@ -1644,6 +1652,7 @@ bdev_nvme_poll(void *arg)
return num_completions > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;
}
+#ifndef SPDK_CONFIG_APP_RW
static int bdev_nvme_poll_adminq(void *arg);
static void
@@ -1654,7 +1663,6 @@ bdev_nvme_change_adminq_poll_period(struct nvme_ctrlr *nvme_ctrlr, uint64_t new_
nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq,
nvme_ctrlr, new_period_us);
}
-#endif
static int
bdev_nvme_poll_adminq(void *arg)
@@ -1684,6 +1692,7 @@ bdev_nvme_poll_adminq(void *arg)
return rc == 0 ? SPDK_POLLER_IDLE : SPDK_POLLER_BUSY;
}
+#endif
static void
nvme_bdev_free(void *io_device)
@@ -1931,8 +1940,10 @@ nvme_ctrlr_disconnect(struct nvme_ctrlr *nvme_ctrlr, nvme_ctrlr_disconnected_cb
assert(nvme_ctrlr->disconnected_cb == NULL);
nvme_ctrlr->disconnected_cb = cb_fn;
+#ifndef SPDK_CONFIG_APP_RW
/* During disconnection, reduce the period to poll adminq more often. */
bdev_nvme_change_adminq_poll_period(nvme_ctrlr, 0);
+#endif
}
enum bdev_nvme_op_after_reset {
@@ -3415,7 +3426,7 @@ bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf)
group->poller = SPDK_POLLER_REGISTER(bdev_nvme_poll, group, g_opts.nvme_ioq_poll_period_us);
- if (group->poller == NULL && spdk_get_reactor_type()) {
+ if (group->poller == NULL) {
spdk_nvme_poll_group_destroy(group->group);
return -1;
}
@@ -4344,7 +4355,7 @@ timeout_cb(void *cb_arg, struct spdk_nvme_ctrlr *ctrlr,
}
}
-static struct nvme_ns *
+struct nvme_ns *
nvme_ns_alloc(void)
{
struct nvme_ns *nvme_ns;
@@ -4492,7 +4503,7 @@ nvme_bdev_add_ns(struct nvme_bdev *bdev, struct nvme_ns *nvme_ns)
return 0;
}
-static void
+void
nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns)
{
struct spdk_nvme_ns *ns;
@@ -5358,9 +5369,10 @@ nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr,
} else {
bdev_nvme_get_default_ctrlr_opts(&nvme_ctrlr->opts);
}
-
+#ifndef SPDK_CONFIG_APP_RW
nvme_ctrlr->adminq_timer_poller = SPDK_POLLER_REGISTER(bdev_nvme_poll_adminq, nvme_ctrlr,
g_opts.nvme_adminq_poll_period_us);
+#endif
if (g_opts.timeout_us > 0) {
/* Register timeout callback. Timeout values for IO vs. admin reqs can be different. */
@@ -8337,15 +8349,20 @@ bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w)
#ifdef SPDK_CONFIG_APP_RW
void *nvme_channel_get_group(void *io_ch)
{
- struct nvme_io_channel *nvme_io_ch = io_ch;
- return nvme_io_ch->group;
+ struct nvme_bdev_channel *bdev_ch = io_ch;
+ struct nvme_io_path *io_path;
+ io_path = STAILQ_FIRST(&bdev_ch->io_path_list);
+
+ return io_path->qpair->group;
}
int nvme_channel_get_state(void *io_ch)
{
- struct nvme_io_channel *nvme_ch = io_ch;
- return nvme_ch->state;
+ struct nvme_bdev_channel *bdev_ch = io_ch;
+
+ return bdev_ch->state;
}
+
struct nvme_bdev_io *nvme_bdev_io_update_args(struct nvme_bdev_io *bio, struct iovec *iov,
int iovcnt)
{
@@ -8361,7 +8378,7 @@ struct nvme_probe_ctx *bdev_nvme_create_probe_ctx(struct spdk_nvme_transport_id
{
struct nvme_probe_ctx *probe_ctx = calloc(1, sizeof(*probe_ctx));
if (probe_ctx == NULL) {
- SPDK_ERRLOG("Failed to allocate probe_ctx\n");
+ SPDK_ERRLOG("Failed to allocate probe_ctx");
return NULL;
}
@@ -8380,7 +8397,7 @@ probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
SPDK_DEBUGLOG(nvme, "Probing device %s\n", trid->traddr);
- if (nvme_bdev_ctrlr_get(trid)) {
+ if (nvme_ctrlr_get(trid)) {
SPDK_ERRLOG("A controller with the provided trid (traddr: %s) already exists.\n",
trid->traddr);
return false;
@@ -8441,7 +8458,7 @@ int bdev_probe_ctrlr(void)
retry_count = spdk_conf_section_get_intval(sp, "RetryCount");
if (retry_count >= 0) {
- g_opts.retry_count = retry_count;
+ g_opts.transport_retry_count = retry_count;
}
if (retry_count > 255) {
SPDK_WARNLOG("RetryCount:%d should not be greater than 255, set it to 255 this time\n",
@@ -8544,7 +8561,7 @@ int bdev_probe_ctrlr(void)
continue;
}
- if (!nvme_bdev_ctrlr_get(&probe_ctx->trids[i])) {
+ if (!nvme_ctrlr_get(&probe_ctx->trids[i])) {
SPDK_ERRLOG("NVMe SSD \"%s\" could not be found.\n", probe_ctx->trids[i].traddr);
SPDK_ERRLOG("Check PCIe BDF and that it is attached to UIO/VFIO driver.\n");
}
@@ -1,465 +1,531 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (C) 2016 Intel Corporation. All rights reserved.
- * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
- * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
- * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved.
- */
-
-#ifndef SPDK_BDEV_NVME_H
-#define SPDK_BDEV_NVME_H
-
-#include "spdk/stdinc.h"
-
-#include "spdk/queue.h"
-#include "spdk/nvme.h"
-#include "spdk/bdev_module.h"
-#include "spdk/jsonrpc.h"
-
-TAILQ_HEAD(nvme_bdev_ctrlrs, nvme_bdev_ctrlr);
-extern struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs;
-extern pthread_mutex_t g_bdev_nvme_mutex;
-extern bool g_bdev_nvme_module_finish;
-extern struct spdk_thread *g_bdev_nvme_init_thread;
-
-#define NVME_MAX_CONTROLLERS 1024
-
-enum bdev_nvme_multipath_policy {
- BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE,
- BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE,
-};
-
-enum bdev_nvme_multipath_selector {
- BDEV_NVME_MP_SELECTOR_ROUND_ROBIN = 1,
- BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH,
-};
-
-typedef void (*spdk_bdev_create_nvme_fn)(void *ctx, size_t bdev_count, int rc);
-typedef void (*spdk_bdev_nvme_start_discovery_fn)(void *ctx, int status);
-typedef void (*spdk_bdev_nvme_stop_discovery_fn)(void *ctx);
-
-struct nvme_ctrlr_opts {
- uint32_t prchk_flags;
- int32_t ctrlr_loss_timeout_sec;
- uint32_t reconnect_delay_sec;
- uint32_t fast_io_fail_timeout_sec;
- bool from_discovery_service;
- /* Path to the file containing PSK, used for dumping configuration. */
- char psk_path[PATH_MAX];
-};
-
-struct nvme_async_probe_ctx {
- struct spdk_nvme_probe_ctx *probe_ctx;
- const char *base_name;
- const char **names;
- uint32_t max_bdevs;
- uint32_t reported_bdevs;
- struct spdk_poller *poller;
- struct spdk_nvme_transport_id trid;
- struct nvme_ctrlr_opts bdev_opts;
- struct spdk_nvme_ctrlr_opts drv_opts;
- spdk_bdev_create_nvme_fn cb_fn;
- void *cb_ctx;
- uint32_t populates_in_progress;
- bool ctrlr_attached;
- bool probe_done;
- bool namespaces_populated;
-};
-
-struct nvme_ns {
- uint32_t id;
- struct spdk_nvme_ns *ns;
- struct nvme_ctrlr *ctrlr;
- struct nvme_bdev *bdev;
- uint32_t ana_group_id;
- enum spdk_nvme_ana_state ana_state;
- bool ana_state_updating;
- bool ana_transition_timedout;
- struct spdk_poller *anatt_timer;
- struct nvme_async_probe_ctx *probe_ctx;
- TAILQ_ENTRY(nvme_ns) tailq;
- RB_ENTRY(nvme_ns) node;
-
- /**
- * record io path stat before destroyed. Allocation of stat is
- * decided by option io_path_stat of RPC
- * bdev_nvme_set_options
- */
- struct spdk_bdev_io_stat *stat;
-};
-
-struct nvme_bdev_io;
-struct nvme_bdev_ctrlr;
-struct nvme_bdev;
-struct nvme_io_path;
-
-struct nvme_path_id {
- struct spdk_nvme_transport_id trid;
- struct spdk_nvme_host_id hostid;
- TAILQ_ENTRY(nvme_path_id) link;
- uint64_t last_failed_tsc;
-};
-
-typedef void (*bdev_nvme_ctrlr_op_cb)(void *cb_arg, int rc);
-typedef void (*nvme_ctrlr_disconnected_cb)(struct nvme_ctrlr *nvme_ctrlr);
-
-struct nvme_ctrlr {
- /**
- * points to pinned, physically contiguous memory region;
- * contains 4KB IDENTIFY structure for controller which is
- * target for CONTROLLER IDENTIFY command during initialization
- */
- struct spdk_nvme_ctrlr *ctrlr;
- struct nvme_path_id *active_path_id;
- int ref;
-
- uint32_t resetting : 1;
- uint32_t reconnect_is_delayed : 1;
- uint32_t in_failover : 1;
- uint32_t pending_failover : 1;
- uint32_t fast_io_fail_timedout : 1;
- uint32_t destruct : 1;
- uint32_t ana_log_page_updating : 1;
- uint32_t io_path_cache_clearing : 1;
- uint32_t dont_retry : 1;
- uint32_t disabled : 1;
-
- struct nvme_ctrlr_opts opts;
-
- RB_HEAD(nvme_ns_tree, nvme_ns) namespaces;
-
- struct spdk_opal_dev *opal_dev;
-
- struct spdk_poller *adminq_timer_poller;
- struct spdk_thread *thread;
-
- bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn;
- void *ctrlr_op_cb_arg;
- /* Poller used to check for reset/detach completion */
- struct spdk_poller *reset_detach_poller;
- struct spdk_nvme_detach_ctx *detach_ctx;
-
- uint64_t reset_start_tsc;
- struct spdk_poller *reconnect_delay_timer;
-
- nvme_ctrlr_disconnected_cb disconnected_cb;
-
- /** linked list pointer for device list */
- TAILQ_ENTRY(nvme_ctrlr) tailq;
- struct nvme_bdev_ctrlr *nbdev_ctrlr;
-
- TAILQ_HEAD(nvme_paths, nvme_path_id) trids;
-
- uint32_t max_ana_log_page_size;
- struct spdk_nvme_ana_page *ana_log_page;
- struct spdk_nvme_ana_group_descriptor *copied_ana_desc;
-
- struct nvme_async_probe_ctx *probe_ctx;
-
- pthread_mutex_t mutex;
-};
-
-struct nvme_bdev_ctrlr {
- char *name;
- TAILQ_HEAD(, nvme_ctrlr) ctrlrs;
- TAILQ_HEAD(, nvme_bdev) bdevs;
- TAILQ_ENTRY(nvme_bdev_ctrlr) tailq;
-};
-
-struct nvme_error_stat {
- uint32_t status_type[8];
- uint32_t status[4][256];
-};
-
-struct nvme_bdev {
- struct spdk_bdev disk;
- uint32_t nsid;
- struct nvme_bdev_ctrlr *nbdev_ctrlr;
- pthread_mutex_t mutex;
- int ref;
- enum bdev_nvme_multipath_policy mp_policy;
- enum bdev_nvme_multipath_selector mp_selector;
- uint32_t rr_min_io;
- TAILQ_HEAD(, nvme_ns) nvme_ns_list;
- bool opal;
- TAILQ_ENTRY(nvme_bdev) tailq;
- struct nvme_error_stat *err_stat;
-};
-
-struct nvme_qpair {
- struct nvme_ctrlr *ctrlr;
- struct spdk_nvme_qpair *qpair;
- struct nvme_poll_group *group;
- struct nvme_ctrlr_channel *ctrlr_ch;
-
- /* The following is used to update io_path cache of nvme_bdev_channels. */
- TAILQ_HEAD(, nvme_io_path) io_path_list;
-
- TAILQ_ENTRY(nvme_qpair) tailq;
-};
-
-struct nvme_ctrlr_channel {
- struct nvme_qpair *qpair;
- TAILQ_HEAD(, spdk_bdev_io) pending_resets;
-
- struct spdk_io_channel_iter *reset_iter;
- struct spdk_poller *connect_poller;
-};
-
-struct nvme_io_path {
- struct nvme_ns *nvme_ns;
- struct nvme_qpair *qpair;
- STAILQ_ENTRY(nvme_io_path) stailq;
-
- /* The following are used to update io_path cache of the nvme_bdev_channel. */
- struct nvme_bdev_channel *nbdev_ch;
- TAILQ_ENTRY(nvme_io_path) tailq;
-
- /* allocation of stat is decided by option io_path_stat of RPC bdev_nvme_set_options */
- struct spdk_bdev_io_stat *stat;
-};
-
-struct nvme_bdev_channel {
- struct nvme_io_path *current_io_path;
- enum bdev_nvme_multipath_policy mp_policy;
- enum bdev_nvme_multipath_selector mp_selector;
- uint32_t rr_min_io;
- uint32_t rr_counter;
- STAILQ_HEAD(, nvme_io_path) io_path_list;
- TAILQ_HEAD(retry_io_head, spdk_bdev_io) retry_io_list;
- struct spdk_poller *retry_io_poller;
-};
-
-struct nvme_poll_group {
- struct spdk_nvme_poll_group *group;
- struct spdk_io_channel *accel_channel;
- struct spdk_poller *poller;
- bool collect_spin_stat;
- uint64_t spin_ticks;
- uint64_t start_ticks;
- uint64_t end_ticks;
- TAILQ_HEAD(, nvme_qpair) qpair_list;
-};
-
-void nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path);
-
-struct nvme_ctrlr *nvme_ctrlr_get_by_name(const char *name);
-
-struct nvme_ctrlr *nvme_bdev_ctrlr_get_ctrlr_by_id(struct nvme_bdev_ctrlr *nbdev_ctrlr,
- uint16_t cntlid);
-
-struct nvme_bdev_ctrlr *nvme_bdev_ctrlr_get_by_name(const char *name);
-
-typedef void (*nvme_bdev_ctrlr_for_each_fn)(struct nvme_bdev_ctrlr *nbdev_ctrlr, void *ctx);
-
-void nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx);
-
-void nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid,
- struct spdk_json_write_ctx *w);
-
-void nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr);
-
-struct nvme_ns *nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid);
-struct nvme_ns *nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr);
-struct nvme_ns *nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns);
-
-struct nvme_bdev_io;
-struct nvme_probe_ctx;
-
-enum spdk_bdev_timeout_action {
- SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE = 0,
- SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET,
- SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT,
-};
-
-struct spdk_bdev_nvme_opts {
- enum spdk_bdev_timeout_action action_on_timeout;
- uint64_t timeout_us;
- uint64_t timeout_admin_us;
- uint32_t keep_alive_timeout_ms;
- /* The number of attempts per I/O in the transport layer before an I/O fails. */
- uint32_t transport_retry_count;
- uint32_t arbitration_burst;
- uint32_t low_priority_weight;
- uint32_t medium_priority_weight;
- uint32_t high_priority_weight;
- uint64_t nvme_adminq_poll_period_us;
- uint64_t nvme_ioq_poll_period_us;
- uint32_t io_queue_requests;
- bool delay_cmd_submit;
- /* The number of attempts per I/O in the bdev layer before an I/O fails. */
- int32_t bdev_retry_count;
- uint8_t transport_ack_timeout;
- int32_t ctrlr_loss_timeout_sec;
- uint32_t reconnect_delay_sec;
- uint32_t fast_io_fail_timeout_sec;
- bool disable_auto_failback;
- bool generate_uuids;
- /* Type of Service - RDMA only */
- uint8_t transport_tos;
- bool nvme_error_stat;
- uint32_t rdma_srq_size;
- bool io_path_stat;
- bool allow_accel_sequence;
- uint32_t rdma_max_cq_size;
-};
-
-struct spdk_nvme_qpair *bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch);
-void bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts);
-int bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts);
-int bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx);
-
-void bdev_nvme_get_default_ctrlr_opts(struct nvme_ctrlr_opts *opts);
-
-int bdev_nvme_create(struct spdk_nvme_transport_id *trid,
- const char *base_name,
- const char **names,
- uint32_t count,
- spdk_bdev_create_nvme_fn cb_fn,
- void *cb_ctx,
- struct spdk_nvme_ctrlr_opts *drv_opts,
- struct nvme_ctrlr_opts *bdev_opts,
- bool multipath);
-
-int bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid, const char *base_name,
- struct spdk_nvme_ctrlr_opts *drv_opts, struct nvme_ctrlr_opts *bdev_opts,
- uint64_t timeout, bool from_mdns,
- spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx);
-int bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn,
- void *cb_ctx);
-void bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w);
-
-int bdev_nvme_start_mdns_discovery(const char *base_name,
- const char *svcname,
- struct spdk_nvme_ctrlr_opts *drv_opts,
- struct nvme_ctrlr_opts *bdev_opts);
-int bdev_nvme_stop_mdns_discovery(const char *name);
-void bdev_nvme_get_mdns_discovery_info(struct spdk_jsonrpc_request *request);
-void bdev_nvme_mdns_discovery_config_json(struct spdk_json_write_ctx *w);
-
-struct spdk_nvme_ctrlr *bdev_nvme_get_ctrlr(struct spdk_bdev *bdev);
-
-typedef void (*bdev_nvme_delete_done_fn)(void *ctx, int rc);
-
-/**
- * Delete NVMe controller with all bdevs on top of it, or delete the specified path
- * if there is any alternative path. Requires to pass name of NVMe controller.
- *
- * \param name NVMe controller name
- * \param path_id The specified path to remove (optional)
- * \param delete_done Callback function on delete complete (optional)
- * \param delete_done_ctx Context passed to callback (optional)
- * \return zero on success,
- * -EINVAL on wrong parameters or
- * -ENODEV if controller is not found or
- * -ENOMEM on no memory
- */
-int bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id,
- bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx);
-
-enum nvme_ctrlr_op {
- NVME_CTRLR_OP_RESET = 1,
- NVME_CTRLR_OP_ENABLE,
- NVME_CTRLR_OP_DISABLE,
-};
-
-/**
- * Perform specified operation on an NVMe controller.
- *
- * NOTE: The callback function is always called after this function returns except for
- * out of memory cases.
- *
- * \param nvme_ctrlr The specified NVMe controller to operate
- * \param op Operation code
- * \param cb_fn Function to be called back after operation completes
- * \param cb_arg Argument for callback function
- */
-void nvme_ctrlr_op_rpc(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op,
- bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg);
-
-/**
- * Perform specified operation on all NVMe controllers in an NVMe bdev controller.
- *
- * NOTE: The callback function is always called after this function returns except for
- * out of memory cases.
- *
- * \param nbdev_ctrlr The specified NVMe bdev controller to operate
- * \param op Operation code
- * \param cb_fn Function to be called back after operation completes
- * \param cb_arg Argument for callback function
- */
-void nvme_bdev_ctrlr_op_rpc(struct nvme_bdev_ctrlr *nbdev_ctrlr, enum nvme_ctrlr_op op,
- bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg);
-
-typedef void (*bdev_nvme_set_preferred_path_cb)(void *cb_arg, int rc);
-
-/**
- * Set the preferred I/O path for an NVMe bdev in multipath mode.
- *
- * NOTE: This function does not support NVMe bdevs in failover mode.
- *
- * \param name NVMe bdev name
- * \param cntlid NVMe-oF controller ID
- * \param cb_fn Function to be called back after completion.
- * \param cb_arg Argument for callback function.
- */
-void bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid,
- bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg);
-
-typedef void (*bdev_nvme_set_multipath_policy_cb)(void *cb_arg, int rc);
-
-/**
- * Set multipath policy of the NVMe bdev.
- *
- * \param name NVMe bdev name
- * \param policy Multipath policy (active-passive or active-active)
- * \param selector Multipath selector (round_robin, queue_depth)
- * \param rr_min_io Number of IO to route to a path before switching to another for round-robin
- * \param cb_fn Function to be called back after completion.
- */
-void bdev_nvme_set_multipath_policy(const char *name,
- enum bdev_nvme_multipath_policy policy,
- enum bdev_nvme_multipath_selector selector,
- uint32_t rr_min_io,
- bdev_nvme_set_multipath_policy_cb cb_fn,
- void *cb_arg);
-
-#ifdef SPDK_CONFIG_APP_RW
-void
-bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl);
-
-void
-bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset);
-
-int
-bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length);
-
-bool
-probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
- struct spdk_nvme_ctrlr_opts *opts);
-
-void
-attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
- struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts);
-
-void
-remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr);
-
-void
-nvme_ctrlr_populate_namespace(struct nvme_bdev_ctrlr *ctrlr, struct nvme_bdev_ns *ns,
- struct nvme_async_probe_ctx *ctx);
-
-void
-nvme_ctrlr_depopulate_namespace(struct nvme_bdev_ctrlr *ctrlr, struct nvme_bdev_ns *ns);
-
-int
-bdev_probe_ctrlr(void);
-
-struct nvme_bdev_io *
-nvme_bdev_io_update_args(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt);
-
-struct nvme_probe_ctx *
-bdev_nvme_create_probe_ctx(struct spdk_nvme_transport_id *trid, const char *base_name,
- const char *hostnqn);
-#endif
-
-#endif /* SPDK_BDEV_NVME_H */
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C) 2016 Intel Corporation. All rights reserved.
+ * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
+ * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * Copyright (c) 2022 Dell Inc, or its subsidiaries. All rights reserved.
+ */
+
+#ifndef SPDK_BDEV_NVME_H
+#define SPDK_BDEV_NVME_H
+
+#include "spdk/stdinc.h"
+
+#include "spdk/queue.h"
+#include "spdk/nvme.h"
+#include "spdk/bdev_module.h"
+#include "spdk/jsonrpc.h"
+
+struct nvme_bdev_io;
+struct nvme_probe_ctx;
+
+TAILQ_HEAD(nvme_bdev_ctrlrs, nvme_bdev_ctrlr);
+extern struct nvme_bdev_ctrlrs g_nvme_bdev_ctrlrs;
+extern pthread_mutex_t g_bdev_nvme_mutex;
+extern bool g_bdev_nvme_module_finish;
+extern struct spdk_thread *g_bdev_nvme_init_thread;
+
+enum bdev_nvme_multipath_policy {
+ BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE,
+ BDEV_NVME_MP_POLICY_ACTIVE_ACTIVE,
+};
+
+enum bdev_nvme_multipath_selector {
+ BDEV_NVME_MP_SELECTOR_ROUND_ROBIN = 1,
+ BDEV_NVME_MP_SELECTOR_QUEUE_DEPTH,
+};
+
+typedef void (*spdk_bdev_create_nvme_fn)(void *ctx, size_t bdev_count, int rc);
+typedef void (*spdk_bdev_nvme_start_discovery_fn)(void *ctx, int status);
+typedef void (*spdk_bdev_nvme_stop_discovery_fn)(void *ctx);
+
+struct nvme_ctrlr_opts {
+ uint32_t prchk_flags;
+ int32_t ctrlr_loss_timeout_sec;
+ uint32_t reconnect_delay_sec;
+ uint32_t fast_io_fail_timeout_sec;
+ bool from_discovery_service;
+ /* Path to the file containing PSK, used for dumping configuration. */
+ char psk_path[PATH_MAX];
+};
+
+struct nvme_async_probe_ctx {
+ struct spdk_nvme_probe_ctx *probe_ctx;
+ const char *base_name;
+ const char **names;
+ uint32_t max_bdevs;
+ uint32_t reported_bdevs;
+ struct spdk_poller *poller;
+ struct spdk_nvme_transport_id trid;
+ struct nvme_ctrlr_opts bdev_opts;
+ struct spdk_nvme_ctrlr_opts drv_opts;
+ spdk_bdev_create_nvme_fn cb_fn;
+ void *cb_ctx;
+ uint32_t populates_in_progress;
+ bool ctrlr_attached;
+ bool probe_done;
+ bool namespaces_populated;
+};
+
+struct nvme_ns {
+ uint32_t id;
+ struct spdk_nvme_ns *ns;
+ struct nvme_ctrlr *ctrlr;
+ struct nvme_bdev *bdev;
+ uint32_t ana_group_id;
+ enum spdk_nvme_ana_state ana_state;
+ bool ana_state_updating;
+ bool ana_transition_timedout;
+ struct spdk_poller *anatt_timer;
+ struct nvme_async_probe_ctx *probe_ctx;
+ TAILQ_ENTRY(nvme_ns) tailq;
+ RB_ENTRY(nvme_ns) node;
+
+ /**
+ * record io path stat before destroyed. Allocation of stat is
+ * decided by option io_path_stat of RPC
+ * bdev_nvme_set_options
+ */
+ struct spdk_bdev_io_stat *stat;
+};
+
+struct nvme_bdev_io;
+struct nvme_bdev_ctrlr;
+struct nvme_bdev;
+struct nvme_io_path;
+
+struct nvme_path_id {
+ struct spdk_nvme_transport_id trid;
+ struct spdk_nvme_host_id hostid;
+ TAILQ_ENTRY(nvme_path_id) link;
+ uint64_t last_failed_tsc;
+};
+
+typedef void (*bdev_nvme_ctrlr_op_cb)(void *cb_arg, int rc);
+typedef void (*nvme_ctrlr_disconnected_cb)(struct nvme_ctrlr *nvme_ctrlr);
+
+struct nvme_ctrlr {
+ /**
+ * points to pinned, physically contiguous memory region;
+ * contains 4KB IDENTIFY structure for controller which is
+ * target for CONTROLLER IDENTIFY command during initialization
+ */
+ struct spdk_nvme_ctrlr *ctrlr;
+ struct nvme_path_id *active_path_id;
+ int ref;
+
+ uint32_t resetting : 1;
+ uint32_t reconnect_is_delayed : 1;
+ uint32_t in_failover : 1;
+ uint32_t pending_failover : 1;
+ uint32_t fast_io_fail_timedout : 1;
+ uint32_t destruct : 1;
+ uint32_t ana_log_page_updating : 1;
+ uint32_t io_path_cache_clearing : 1;
+ uint32_t dont_retry : 1;
+ uint32_t disabled : 1;
+
+ struct nvme_ctrlr_opts opts;
+
+ RB_HEAD(nvme_ns_tree, nvme_ns) namespaces;
+
+ struct spdk_opal_dev *opal_dev;
+
+ struct spdk_poller *adminq_timer_poller;
+ struct spdk_thread *thread;
+
+ bdev_nvme_ctrlr_op_cb ctrlr_op_cb_fn;
+ void *ctrlr_op_cb_arg;
+ /* Poller used to check for reset/detach completion */
+ struct spdk_poller *reset_detach_poller;
+ struct spdk_nvme_detach_ctx *detach_ctx;
+
+ uint64_t reset_start_tsc;
+ struct spdk_poller *reconnect_delay_timer;
+
+ nvme_ctrlr_disconnected_cb disconnected_cb;
+
+ /** linked list pointer for device list */
+ TAILQ_ENTRY(nvme_ctrlr) tailq;
+ struct nvme_bdev_ctrlr *nbdev_ctrlr;
+
+ TAILQ_HEAD(nvme_paths, nvme_path_id) trids;
+
+ uint32_t max_ana_log_page_size;
+ struct spdk_nvme_ana_page *ana_log_page;
+ struct spdk_nvme_ana_group_descriptor *copied_ana_desc;
+
+ struct nvme_async_probe_ctx *probe_ctx;
+
+ pthread_mutex_t mutex;
+};
+
+struct nvme_bdev_ctrlr {
+ char *name;
+ TAILQ_HEAD(, nvme_ctrlr) ctrlrs;
+ TAILQ_HEAD(, nvme_bdev) bdevs;
+ TAILQ_ENTRY(nvme_bdev_ctrlr) tailq;
+};
+
+struct nvme_error_stat {
+ uint32_t status_type[8];
+ uint32_t status[4][256];
+};
+
+struct nvme_bdev {
+ struct spdk_bdev disk;
+ uint32_t nsid;
+ struct nvme_bdev_ctrlr *nbdev_ctrlr;
+ pthread_mutex_t mutex;
+ int ref;
+ enum bdev_nvme_multipath_policy mp_policy;
+ enum bdev_nvme_multipath_selector mp_selector;
+ uint32_t rr_min_io;
+ TAILQ_HEAD(, nvme_ns) nvme_ns_list;
+ bool opal;
+ TAILQ_ENTRY(nvme_bdev) tailq;
+ struct nvme_error_stat *err_stat;
+};
+
+struct nvme_qpair {
+ struct nvme_ctrlr *ctrlr;
+ struct spdk_nvme_qpair *qpair;
+ struct nvme_poll_group *group;
+ struct nvme_ctrlr_channel *ctrlr_ch;
+
+ /* The following is used to update io_path cache of nvme_bdev_channels. */
+ TAILQ_HEAD(, nvme_io_path) io_path_list;
+
+ TAILQ_ENTRY(nvme_qpair) tailq;
+};
+
+struct nvme_ctrlr_channel {
+ struct nvme_qpair *qpair;
+ TAILQ_HEAD(, spdk_bdev_io) pending_resets;
+
+ struct spdk_io_channel_iter *reset_iter;
+ struct spdk_poller *connect_poller;
+};
+
+struct nvme_io_path {
+ struct nvme_ns *nvme_ns;
+ struct nvme_qpair *qpair;
+ STAILQ_ENTRY(nvme_io_path) stailq;
+
+ /* The following are used to update io_path cache of the nvme_bdev_channel. */
+ struct nvme_bdev_channel *nbdev_ch;
+ TAILQ_ENTRY(nvme_io_path) tailq;
+
+ /* allocation of stat is decided by option io_path_stat of RPC bdev_nvme_set_options */
+ struct spdk_bdev_io_stat *stat;
+};
+
+struct nvme_bdev_channel {
+ struct nvme_io_path *current_io_path;
+ enum bdev_nvme_multipath_policy mp_policy;
+ enum bdev_nvme_multipath_selector mp_selector;
+ uint32_t rr_min_io;
+ uint32_t rr_counter;
+ STAILQ_HEAD(, nvme_io_path) io_path_list;
+ TAILQ_HEAD(retry_io_head, spdk_bdev_io) retry_io_list;
+ struct spdk_poller *retry_io_poller;
+ int state;
+};
+
+struct nvme_poll_group {
+ struct spdk_nvme_poll_group *group;
+ struct spdk_io_channel *accel_channel;
+ struct spdk_poller *poller;
+ bool collect_spin_stat;
+ uint64_t spin_ticks;
+ uint64_t start_ticks;
+ uint64_t end_ticks;
+#ifdef SPDK_CONFIG_APP_RW
+ uint64_t save_start_ticks;
+ uint64_t num_poll_timeout;
+#endif
+ TAILQ_HEAD(, nvme_qpair) qpair_list;
+};
+
+void nvme_io_path_info_json(struct spdk_json_write_ctx *w, struct nvme_io_path *io_path);
+
+struct nvme_ctrlr *nvme_ctrlr_get_by_name(const char *name);
+
+struct nvme_ctrlr *nvme_bdev_ctrlr_get_ctrlr_by_id(struct nvme_bdev_ctrlr *nbdev_ctrlr,
+ uint16_t cntlid);
+
+struct nvme_bdev_ctrlr *nvme_bdev_ctrlr_get_by_name(const char *name);
+
+typedef void (*nvme_bdev_ctrlr_for_each_fn)(struct nvme_bdev_ctrlr *nbdev_ctrlr, void *ctx);
+
+void nvme_bdev_ctrlr_for_each(nvme_bdev_ctrlr_for_each_fn fn, void *ctx);
+
+void nvme_bdev_dump_trid_json(const struct spdk_nvme_transport_id *trid,
+ struct spdk_json_write_ctx *w);
+
+void nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrlr);
+
+struct nvme_ns *nvme_ctrlr_get_ns(struct nvme_ctrlr *nvme_ctrlr, uint32_t nsid);
+struct nvme_ns *nvme_ctrlr_get_first_active_ns(struct nvme_ctrlr *nvme_ctrlr);
+struct nvme_ns *nvme_ctrlr_get_next_active_ns(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *ns);
+
+enum spdk_bdev_timeout_action {
+ SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE = 0,
+ SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET,
+ SPDK_BDEV_NVME_TIMEOUT_ACTION_ABORT,
+};
+
+struct spdk_bdev_nvme_opts {
+ enum spdk_bdev_timeout_action action_on_timeout;
+ uint64_t timeout_us;
+ uint64_t timeout_admin_us;
+ uint32_t keep_alive_timeout_ms;
+ /* The number of attempts per I/O in the transport layer before an I/O fails. */
+ uint32_t transport_retry_count;
+ uint32_t arbitration_burst;
+ uint32_t low_priority_weight;
+ uint32_t medium_priority_weight;
+ uint32_t high_priority_weight;
+ uint64_t nvme_adminq_poll_period_us;
+ uint64_t nvme_ioq_poll_period_us;
+ uint32_t io_queue_requests;
+ bool delay_cmd_submit;
+ /* The number of attempts per I/O in the bdev layer before an I/O fails. */
+ int32_t bdev_retry_count;
+ uint8_t transport_ack_timeout;
+ int32_t ctrlr_loss_timeout_sec;
+ uint32_t reconnect_delay_sec;
+ uint32_t fast_io_fail_timeout_sec;
+ bool disable_auto_failback;
+ bool generate_uuids;
+ /* Type of Service - RDMA only */
+ uint8_t transport_tos;
+ bool nvme_error_stat;
+ uint32_t rdma_srq_size;
+ bool io_path_stat;
+ bool allow_accel_sequence;
+ uint32_t rdma_max_cq_size;
+};
+
+struct spdk_nvme_qpair *bdev_nvme_get_io_qpair(struct spdk_io_channel *ctrlr_io_ch);
+void bdev_nvme_get_opts(struct spdk_bdev_nvme_opts *opts);
+int bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts);
+int bdev_nvme_set_hotplug(bool enabled, uint64_t period_us, spdk_msg_fn cb, void *cb_ctx);
+
+void bdev_nvme_get_default_ctrlr_opts(struct nvme_ctrlr_opts *opts);
+
+int bdev_nvme_create(struct spdk_nvme_transport_id *trid,
+ const char *base_name,
+ const char **names,
+ uint32_t count,
+ spdk_bdev_create_nvme_fn cb_fn,
+ void *cb_ctx,
+ struct spdk_nvme_ctrlr_opts *drv_opts,
+ struct nvme_ctrlr_opts *bdev_opts,
+ bool multipath);
+
+int bdev_nvme_start_discovery(struct spdk_nvme_transport_id *trid, const char *base_name,
+ struct spdk_nvme_ctrlr_opts *drv_opts, struct nvme_ctrlr_opts *bdev_opts,
+ uint64_t timeout, bool from_mdns,
+ spdk_bdev_nvme_start_discovery_fn cb_fn, void *cb_ctx);
+int bdev_nvme_stop_discovery(const char *name, spdk_bdev_nvme_stop_discovery_fn cb_fn,
+ void *cb_ctx);
+void bdev_nvme_get_discovery_info(struct spdk_json_write_ctx *w);
+
+int bdev_nvme_start_mdns_discovery(const char *base_name,
+ const char *svcname,
+ struct spdk_nvme_ctrlr_opts *drv_opts,
+ struct nvme_ctrlr_opts *bdev_opts);
+int bdev_nvme_stop_mdns_discovery(const char *name);
+void bdev_nvme_get_mdns_discovery_info(struct spdk_jsonrpc_request *request);
+void bdev_nvme_mdns_discovery_config_json(struct spdk_json_write_ctx *w);
+
+struct spdk_nvme_ctrlr *bdev_nvme_get_ctrlr(struct spdk_bdev *bdev);
+
+typedef void (*bdev_nvme_delete_done_fn)(void *ctx, int rc);
+
+/**
+ * Delete NVMe controller with all bdevs on top of it, or delete the specified path
+ * if there is any alternative path. Requires to pass name of NVMe controller.
+ *
+ * \param name NVMe controller name
+ * \param path_id The specified path to remove (optional)
+ * \param delete_done Callback function on delete complete (optional)
+ * \param delete_done_ctx Context passed to callback (optional)
+ * \return zero on success,
+ * -EINVAL on wrong parameters or
+ * -ENODEV if controller is not found or
+ * -ENOMEM on no memory
+ */
+int bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id,
+ bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx);
+
+enum nvme_ctrlr_op {
+ NVME_CTRLR_OP_RESET = 1,
+ NVME_CTRLR_OP_ENABLE,
+ NVME_CTRLR_OP_DISABLE,
+};
+
+/**
+ * Perform specified operation on an NVMe controller.
+ *
+ * NOTE: The callback function is always called after this function returns except for
+ * out of memory cases.
+ *
+ * \param nvme_ctrlr The specified NVMe controller to operate
+ * \param op Operation code
+ * \param cb_fn Function to be called back after operation completes
+ * \param cb_arg Argument for callback function
+ */
+void nvme_ctrlr_op_rpc(struct nvme_ctrlr *nvme_ctrlr, enum nvme_ctrlr_op op,
+ bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg);
+
+/**
+ * Perform specified operation on all NVMe controllers in an NVMe bdev controller.
+ *
+ * NOTE: The callback function is always called after this function returns except for
+ * out of memory cases.
+ *
+ * \param nbdev_ctrlr The specified NVMe bdev controller to operate
+ * \param op Operation code
+ * \param cb_fn Function to be called back after operation completes
+ * \param cb_arg Argument for callback function
+ */
+void nvme_bdev_ctrlr_op_rpc(struct nvme_bdev_ctrlr *nbdev_ctrlr, enum nvme_ctrlr_op op,
+ bdev_nvme_ctrlr_op_cb cb_fn, void *cb_arg);
+
+typedef void (*bdev_nvme_set_preferred_path_cb)(void *cb_arg, int rc);
+
+/**
+ * Set the preferred I/O path for an NVMe bdev in multipath mode.
+ *
+ * NOTE: This function does not support NVMe bdevs in failover mode.
+ *
+ * \param name NVMe bdev name
+ * \param cntlid NVMe-oF controller ID
+ * \param cb_fn Function to be called back after completion.
+ * \param cb_arg Argument for callback function.
+ */
+void bdev_nvme_set_preferred_path(const char *name, uint16_t cntlid,
+ bdev_nvme_set_preferred_path_cb cb_fn, void *cb_arg);
+
+typedef void (*bdev_nvme_set_multipath_policy_cb)(void *cb_arg, int rc);
+
+/**
+ * Set multipath policy of the NVMe bdev.
+ *
+ * \param name NVMe bdev name
+ * \param policy Multipath policy (active-passive or active-active)
+ * \param selector Multipath selector (round_robin, queue_depth)
+ * \param rr_min_io Number of IO to route to a path before switching to another for round-robin
+ * \param cb_fn Function to be called back after completion.
+ */
+void bdev_nvme_set_multipath_policy(const char *name,
+ enum bdev_nvme_multipath_policy policy,
+ enum bdev_nvme_multipath_selector selector,
+ uint32_t rr_min_io,
+ bdev_nvme_set_multipath_policy_cb cb_fn,
+ void *cb_arg);
+
+#ifdef SPDK_CONFIG_APP_RW
+
+struct nvme_bdev_io {
+ /** array of iovecs to transfer. */
+ struct iovec *iovs;
+
+ /** Number of iovecs in iovs array. */
+ int iovcnt;
+
+ /** Current iovec position. */
+ int iovpos;
+
+ /** Offset in current iovec. */
+ uint32_t iov_offset;
+
+ /** I/O path the current I/O or admin passthrough is submitted on, or the I/O path
+ * being reset in a reset I/O.
+ */
+ struct nvme_io_path *io_path;
+
+ /** array of iovecs to transfer. */
+ struct iovec *fused_iovs;
+
+ /** Number of iovecs in iovs array. */
+ int fused_iovcnt;
+
+ /** Current iovec position. */
+ int fused_iovpos;
+
+ /** Offset in current iovec. */
+ uint32_t fused_iov_offset;
+
+ /** Saved status for admin passthru completion event, PI error verification, or intermediate compare-and-write status */
+ struct spdk_nvme_cpl cpl;
+
+ /** Extended IO opts passed by the user to bdev layer and mapped to NVME format */
+ struct spdk_nvme_ns_cmd_ext_io_opts ext_opts;
+
+ /** Keeps track if first of fused commands was submitted */
+ bool first_fused_submitted;
+
+ /** Keeps track if first of fused commands was completed */
+ bool first_fused_completed;
+
+ /** Temporary pointer to zone report buffer */
+ struct spdk_nvme_zns_zone_report *zone_report_buf;
+
+ /** Keep track of how many zones that have been copied to the spdk_bdev_zone_info struct */
+ uint64_t handled_zones;
+
+ /** Expiration value in ticks to retry the current I/O. */
+ uint64_t retry_ticks;
+
+ /* How many times the current I/O was retried. */
+ int32_t retry_count;
+
+ /* Current tsc at submit time. */
+ uint64_t submit_tsc;
+};
+
+void
+bdev_nvme_queued_done(void *ref, const struct spdk_nvme_cpl *cpl);
+
+void
+bdev_nvme_queued_reset_sgl(void *ref, uint32_t sgl_offset);
+
+int
+bdev_nvme_queued_next_sge(void *ref, void **address, uint32_t *length);
+
+bool
+probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr_opts *opts);
+
+void
+attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+ struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts);
+
+void
+remove_cb(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr);
+
+
+void
+nvme_ctrlr_populate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns);
+
+void
+nvme_ctrlr_depopulate_namespace(struct nvme_ctrlr *nvme_ctrlr, struct nvme_ns *nvme_ns);
+
+int
+bdev_probe_ctrlr(void);
+
+struct nvme_bdev_io *
+nvme_bdev_io_update_args(struct nvme_bdev_io *bio, struct iovec *iov, int iovcnt);
+
+struct nvme_probe_ctx *
+bdev_nvme_create_probe_ctx(struct spdk_nvme_transport_id *trid, const char *base_name,
+ const char *hostnqn);
+
+struct nvme_ctrlr *nvme_ctrlr_get(const struct spdk_nvme_transport_id *trid);
+
+struct nvme_ns *nvme_ns_alloc(void);
+#endif
+
+#endif /* SPDK_BDEV_NVME_H */
@@ -1,632 +1,650 @@
-/*
- * Copyright (C) 2021. Huawei Technologies Co., Ltd. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
-#include "bdev_nvme.h"
-
-#include "spdk/json.h"
-#include "spdk/likely.h"
-#include "spdk/bdev_module.h"
-#include "spdk/nvme.h"
-
-#include "spdk_internal/bdev_stat.h"
-#include "bdev_nvme_self.h"
-#include "common.h"
-#include <securec.h>
-
-void bdev_update_ch_timeout(struct nvme_bdev_poll_group *group)
-{
- uint64_t current_ticks = 0;
- uint64_t poll_ticks = 0;
- int64_t poll_time = 0;
-
- current_ticks = spdk_get_ticks();
-
- if (spdk_unlikely(g_polltime_threshold)) {
- if (group->save_start_ticks) {
- poll_ticks = current_ticks - group->save_start_ticks;
- poll_time = (poll_ticks * 1000ULL) / spdk_get_ticks_hz();
- if (poll_time >= g_polltime_threshold) {
- group->num_poll_timeout++;
- SPDK_WARNLOG("group[%p] poll timeout in %ldms", group, poll_time);
- }
- }
- group->save_start_ticks = current_ticks;
- }
-}
-
-int _bdev_nvme_submit_request_self(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
-{
- struct nvme_io_channel *nvme_ch = spdk_io_channel_get_ctx(ch);
-
- if (nvme_ch->qpair == NULL) {
- /* The device is currently resetting */
- return -1;
- }
-
- switch (bdev_io->type) {
- case SPDK_BDEV_IO_TYPE_READ_NVME:
- SPDK_DEBUGLOG(bdev_nvme, "read %lu lbas with offset %#lx\n", bdev_io->u.contig.num_blocks,
- bdev_io->u.contig.offset_blocks);
- return bdev_nvme_queue_cmd_with_md((struct nvme_bdev *)bdev_io->bdev->ctxt, nvme_ch->qpair,
- bdev_io->driver_ctx, bdev_io->u.contig.buf,
- bdev_io->u.contig.md_buf, SPDK_BDEV_IO_TYPE_READ,
- bdev_io->u.contig.num_blocks, bdev_io->u.contig.offset_blocks);
- case SPDK_BDEV_IO_TYPE_WRITE_NVME:
- SPDK_DEBUGLOG(bdev_nvme, "write %lu lbas with offset %#lx\n", bdev_io->u.contig.num_blocks,
- bdev_io->u.contig.offset_blocks);
- return bdev_nvme_queue_cmd_with_md((struct nvme_bdev *)bdev_io->bdev->ctxt, nvme_ch->qpair,
- bdev_io->driver_ctx, bdev_io->u.contig.buf,
- bdev_io->u.contig.md_buf, SPDK_BDEV_IO_TYPE_WRITE,
- bdev_io->u.contig.num_blocks, bdev_io->u.contig.offset_blocks);
- case SPDK_BDEV_IO_TYPE_READV_NVME:
- SPDK_DEBUGLOG(bdev_nvme, "readv %lu lbas with offset %#lx\n", bdev_io->u.bdev.num_blocks,
- bdev_io->u.bdev.offset_blocks);
- return bdev_nvme_queue_cmd_v_with_md((struct nvme_bdev *)bdev_io->bdev->ctxt, nvme_ch->qpair,
- bdev_io->driver_ctx, SPDK_BDEV_IO_TYPE_READ,
- bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
- bdev_io->u.bdev.num_blocks, bdev_io->u.bdev.offset_blocks);
- case SPDK_BDEV_IO_TYPE_WRITEV_NVME:
- SPDK_DEBUGLOG(bdev_nvme, "writev %lu lbas with offset %#lx\n", bdev_io->u.bdev.num_blocks,
- bdev_io->u.bdev.offset_blocks);
- return bdev_nvme_queue_cmd_v_with_md((struct nvme_bdev *)bdev_io->bdev->ctxt, nvme_ch->qpair,
- bdev_io->driver_ctx, SPDK_BDEV_IO_TYPE_WRITE,
- bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
- bdev_io->u.bdev.num_blocks, bdev_io->u.bdev.offset_blocks);
- case SPDK_BDEV_IO_TYPE_UNMAP_BLOCKS:
- return bdev_nvme_unmap_blocks((struct nvme_bdev *)bdev_io->bdev->ctxt,
- ch,
- (void *)bdev_io->driver_ctx,
- (struct spdk_nvme_dsm_range *)bdev_io->u.contig.buf,
- bdev_io->u.contig.num_blocks);
- default:
- return -EINVAL;
- }
- return 0;
-}
-
-int bdev_nvme_dump_info_json_self(void *ctx, struct spdk_json_write_ctx *w)
-{
- return 0;
-}
-
-uint16_t bdev_nvme_get_io_channel_id(struct spdk_io_channel *ch)
-{
- struct nvme_io_channel *nvme_ch = spdk_io_channel_get_ctx(ch);
- uint16_t channel_id;
- struct spdk_nvme_qpair *qpair = nvme_ch->qpair;
- channel_id = spdk_nvme_get_qpair_id(qpair);
- return channel_id;
-}
-
-uint64_t bdev_nvme_get_timeout_count(struct spdk_io_channel *ch)
-{
- struct nvme_io_channel *nvme_ch = spdk_io_channel_get_ctx(ch);
- return nvme_ch->group->num_poll_timeout;
-}
-
-int32_t nvme_ctrlr_get_info(const char *ctrlName, struct nvme_ctrlr_info **ppCtrlr)
-{
- uint32_t num_ctrlr = 0, i = 0;
- struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = NULL;
- struct nvme_ctrlr_info *pCtrlrInfo = NULL;
- const struct spdk_nvme_ctrlr_data *cdata = NULL;
- struct spdk_nvme_ctrlr_opts *opts = NULL;
-
- struct spdk_pci_device *pci_dev = NULL;
- int rc;
-
- TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
- num_ctrlr++;
- }
- if (num_ctrlr == 0) {
- SPDK_NOTICELOG("No any nvme controller.\n");
- return 0;
- }
- if (ctrlName != NULL) {
- num_ctrlr = 1;
- }
- pCtrlrInfo = calloc(num_ctrlr, sizeof(struct nvme_ctrlr_info));
- if (pCtrlrInfo == NULL) {
- SPDK_ERRLOG("Failed to alloc memory for getting controller infomation.\n");
- return -1;
- }
- TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
- if (i >= num_ctrlr) { /* prevent having controllers be added or deleted */
- i++;
- continue;
- }
- if (ctrlName != NULL) {
- if (strcmp(nvme_bdev_ctrlr->name, ctrlName) != 0) {
- continue;
- }
- }
- cdata = spdk_nvme_ctrlr_get_data(nvme_bdev_ctrlr->ctrlr);
- opts = spdk_nvme_ctrlr_get_opts(nvme_bdev_ctrlr->ctrlr);
- pci_dev = spdk_nvme_ctrlr_get_pci_device(nvme_bdev_ctrlr->ctrlr);
- if (pci_dev == NULL) {
- SPDK_ERRLOG("Failed to get pci device\n");
- break;
- }
- rc = strcpy_s(pCtrlrInfo[i].ctrlName, sizeof(pCtrlrInfo[i].ctrlName), nvme_bdev_ctrlr->name);
- if (rc != 0) {
- SPDK_ERRLOG("String copy failed\n");
- }
- rc = strcpy_s(pCtrlrInfo[i].pciAddr, sizeof(pCtrlrInfo[i].pciAddr),
- nvme_bdev_ctrlr->connected_trid->traddr);
- if (rc != 0) {
- SPDK_ERRLOG("String copy failed\n");
- }
-
- rc = memcpy_s(pCtrlrInfo[i].sn, sizeof(pCtrlrInfo[i].sn), cdata->sn, 20);
- if (rc != 0) {
- SPDK_ERRLOG("Memory copy failed\n");
- }
-
- rc = memcpy_s(pCtrlrInfo[i].fr, sizeof(pCtrlrInfo[i].fr), cdata->fr, 8);
- if (rc != 0) {
- SPDK_ERRLOG("Memory copy failed\n");
- }
-
- rc = memcpy_s(pCtrlrInfo[i].mn, sizeof(pCtrlrInfo[i].mn), cdata->mn, 40);
- if (rc != 0) {
- SPDK_ERRLOG("Memory copy failed\n");
- }
-
- pCtrlrInfo[i].trtype = (uint16_t)nvme_bdev_ctrlr->connected_trid->trtype;
- pCtrlrInfo[i].tnvmcap = cdata->tnvmcap[0];
- pCtrlrInfo[i].unvmcap = cdata->unvmcap[0];
- pCtrlrInfo[i].support_ns = cdata->oacs.ns_manage;
- pCtrlrInfo[i].directives = cdata->oacs.directives;
- pCtrlrInfo[i].dsm = cdata->oncs.dsm;
- pCtrlrInfo[i].max_num_ns = cdata->nn;
- pCtrlrInfo[i].num_io_queues = opts->num_io_queues;
- pCtrlrInfo[i].io_queue_size = opts->io_queue_size;
- pCtrlrInfo[i].device_id = spdk_pci_device_get_device_id(pci_dev);
- pCtrlrInfo[i].subdevice_id = spdk_pci_device_get_subdevice_id(pci_dev);
- pCtrlrInfo[i].vid = cdata->vid;
- pCtrlrInfo[i].ssvid = cdata->ssvid;
- pCtrlrInfo[i].ctrlid = cdata->cntlid;
- pCtrlrInfo[i].version = spdk_nvme_ctrlr_get_regs_vs(nvme_bdev_ctrlr->ctrlr).raw;
- i++;
- if (ctrlName != NULL) {
- break;
- }
- }
- if (i != num_ctrlr) {
- SPDK_ERRLOG("It has controller been added or deleted when fetched infomation, please try again later.\n");
- free(pCtrlrInfo);
- return -1;
- }
- *ppCtrlr = pCtrlrInfo;
- return num_ctrlr;
-}
-
-struct nvme_bdev_ctrlr *nvme_ctrlr_get_by_name(const char *name)
-{
- struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = NULL;
-
- if (name == NULL) {
- return NULL;
- }
-
- TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
- if (strcmp(name, nvme_bdev_ctrlr->name) == 0) {
- return nvme_bdev_ctrlr;
- }
- }
-
- return NULL;
-}
-
-struct spdk_nvme_ctrlr *spdk_nvme_ctrlr_get_by_name(const char *ctrlname)
-{
- struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = NULL;
-
- TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
- if (strcmp(nvme_bdev_ctrlr->name, ctrlname) == 0) {
- return nvme_bdev_ctrlr->ctrlr;
- }
- }
-
- return NULL;
-}
-
-struct spdk_nvme_ctrlr *spdk_nvme_ctrlr_get_by_ctrlr(const struct nvme_bdev_ctrlr *nvme_bdev_ctrlr)
-{
- if (nvme_bdev_ctrlr == NULL) {
- return NULL;
- }
- return nvme_bdev_ctrlr->ctrlr;
-}
-
-void nvme_ctrlr_clear_iostat_by_name(const char *ctrlname)
-{
- int i;
- size_t size = strnlen(ctrlname, 24);
-
- for (i = 0; i < STAT_MAX_NUM; i++) {
- if (strncmp(g_io_stat_map[i].bdev_name, ctrlname, size) == 0) {
- if ((g_io_stat_map[i].bdev_name[size] == 'n') && isdigit(g_io_stat_map[i].bdev_name[size + 1])) {
- g_io_stat_map[i].channel_id = 0;
- memset(g_io_stat_map[i].bdev_name, 0, sizeof(g_io_stat_map[i].bdev_name));
- g_io_stat_map[i].num_read_ops = 0;
- g_io_stat_map[i].num_write_ops = 0;
- g_io_stat_map[i].bytes_read = 0;
- g_io_stat_map[i].bytes_written = 0;
- g_io_stat_map[i].io_outstanding = 0;
- g_io_stat_map[i].read_latency_ticks = 0;
- g_io_stat_map[i].write_latency_ticks = 0;
- g_io_stat_map[i].io_ticks = 0;
- /* used flag set false in last avoid race in channel create */
- g_io_stat_map[i].used = false;
- }
- }
- }
-}
-
-void nvme_ctrlr_clear_iostat_all(void)
-{
- struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = NULL;
-
- TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
- nvme_ctrlr_clear_iostat_by_name(nvme_bdev_ctrlr->name);
- }
-}
-
-struct spdk_nvme_ns *bdev_nvme_get_ns(struct nvme_bdev *nbdev)
-{
- return nbdev->nvme_ns->ns;
-}
-
-void bdev_nvme_update_block_by_nvme_ctrlr(struct spdk_nvme_ctrlr *ctrlr)
-{
- uint32_t i;
- struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = NULL;
- struct nvme_bdev_ns *ns = NULL;
- struct nvme_bdev *nvme_bdev = NULL, *tmp = NULL;
-
-
- pthread_mutex_lock(&g_bdev_nvme_mutex);
- TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
- if (nvme_bdev_ctrlr->ctrlr != ctrlr) {
- continue;
- }
-
- pthread_mutex_unlock(&g_bdev_nvme_mutex);
- for (i = 0; i < nvme_bdev_ctrlr->num_ns; i++) {
- ns = nvme_bdev_ctrlr->namespaces[i];
- TAILQ_FOREACH_SAFE(nvme_bdev, &ns->bdevs, tailq, tmp) {
- nvme_bdev->disk.blocklen = spdk_nvme_ns_get_sector_size(nvme_bdev->nvme_ns->ns);
- nvme_bdev->disk.blockcnt = spdk_nvme_ns_get_num_sectors(nvme_bdev->nvme_ns->ns);
- }
- }
- return;
- }
- pthread_mutex_unlock(&g_bdev_nvme_mutex);
-}
-
-int bdev_nvme_update_ns(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, uint32_t nsid)
-{
- struct nvme_bdev_ns *ns = NULL;
- bool ns_active = spdk_nvme_ctrlr_is_active_ns(nvme_bdev_ctrlr->ctrlr, nsid);
-
- if (nvme_bdev_ctrlr == NULL || nsid > nvme_bdev_ctrlr->num_ns) {
- SPDK_ERRLOG("Parameter error. nsid[%u], the max nsid is[%u]\n", nsid, nvme_bdev_ctrlr->num_ns);
- return -1;
- }
-
- ns = nvme_bdev_ctrlr->namespaces[nsid - 1];
- ns->type = NVME_BDEV_NS_STANDARD;
-
- if (!ns->populated && ns_active) {
- SPDK_NOTICELOG("NSID %u to be added\n", nsid);
- ns->id = nsid;
- ns->ctrlr = nvme_bdev_ctrlr;
- TAILQ_INIT(&ns->bdevs);
- /* add a new bdev device in this ns */
- nvme_ctrlr_populate_namespace(nvme_bdev_ctrlr, ns, NULL);
- return 0;
- }
-
- if (ns->populated && !ns_active) {
- SPDK_NOTICELOG("NSID %u is removed\n", nsid);
- nvme_ctrlr_depopulate_namespace(nvme_bdev_ctrlr, ns);
- return 0;
- }
-
- return 0;
-}
-
-bool spdk_bdev_can_remove(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, uint32_t nsid)
-{
- struct nvme_bdev_ns *ns = NULL;
- struct nvme_bdev *bdev = NULL, *tmp = NULL;
- bool empty = false;
-
- ns = nvme_bdev_ctrlr->namespaces[nsid - 1];
- if (ns == NULL) {
- return true;
- }
-
- TAILQ_FOREACH_SAFE(bdev, &ns->bdevs, tailq, tmp) {
- pthread_mutex_lock(&bdev->disk.internal.mutex);
- empty = TAILQ_EMPTY(&bdev->disk.internal.open_descs);
- /* for each bdev in ns, we need to check if any descs is in tailq */
- if (empty) {
- /* one bdev is empty, check next until all bdev is checked */
- bdev->disk.internal.ns_status = SPDK_BDEV_NS_STATUS_REMOVING;
- pthread_mutex_unlock(&bdev->disk.internal.mutex);
- } else {
- /* means at least one bdev is used, so we just quit this process
- and mark the status is false. */
- pthread_mutex_unlock(&bdev->disk.internal.mutex);
- break;
- }
- }
- return empty;
-}
-
-void spdk_bdev_set_ns_normal(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, uint32_t nsid)
-{
- struct nvme_bdev_ns *ns = NULL;
- struct nvme_bdev *bdev = NULL, *tmp = NULL;
-
- ns = nvme_bdev_ctrlr->namespaces[nsid - 1];
- if (ns == NULL) {
- return;
- }
-
- TAILQ_FOREACH_SAFE(bdev, &ns->bdevs, tailq, tmp) {
- pthread_mutex_lock(&bdev->disk.internal.mutex);
- /* set the ns_status to ready case ns delete fail */
- if (bdev->disk.internal.ns_status == SPDK_BDEV_NS_STATUS_REMOVING) {
- bdev->disk.internal.ns_status = SPDK_BDEV_NS_STATUS_READY;
- }
- pthread_mutex_unlock(&bdev->disk.internal.mutex);
- }
-}
-
-static void check_error_type(int rc, bool read, void *qpair)
-{
- if (rc == -ENOMEM) {
- SPDK_NOTICELOG("%s failed: rc = %d\n", read ? "read" : "write", rc);
- } else if (rc < 0) {
- SPDK_ERRLOG("%s failed: rc = %d, qpair is %p\n", read ? "read" : "write", rc, qpair);
- }
-}
-
-int bdev_nvme_queue_cmd_with_md(struct nvme_bdev *bdev, struct spdk_nvme_qpair *qpair,
- void *driver_ctx,
- void *buffer, void *metadata, int direction, uint64_t lba_count, uint64_t lba)
-{
- int rc;
- uint32_t io_flags = 0;
- uint8_t *bdev_io_action = (uint8_t *)driver_ctx;
- /* filter bit 0&1 of io->pi_action to get pi_action */
- uint8_t pi_action = bdev_io_action[SPDK_BDEV_IO_ACTION_PI] & 0x03;
- uint8_t dif_flag = bdev_io_action[SPDK_BDEV_IO_ACTION_PI];
- uint8_t fua = bdev_io_action[SPDK_BDEV_IO_ACTION_FUA];
- uint32_t pi_type;
-
- spdk_bdev_set_io_location(driver_ctx, (uint8_t)LOCAL_LIBSTORAGE_BDEV_NVME);
-
- if (pi_action > IO_NO_PROTECTION) {
- pi_type = spdk_nvme_ns_get_pi_type(bdev->nvme_ns->ns);
- if (dif_flag & FLAG_PRCHK) {
- io_flags |= SPDK_NVME_IO_FLAGS_PRCHK_GUARD;
- }
- /* type3 not support ref tag */
- if (!(dif_flag & FLAG_NO_REF) && (pi_type != SPDK_NVME_FMT_NVM_PROTECTION_TYPE3)) {
- io_flags |= SPDK_NVME_IO_FLAGS_PRCHK_REFTAG;
- }
- if (pi_action == IO_HALF_WAY_PROTECTION) {
- io_flags |= SPDK_NVME_IO_FLAGS_PRACT;
- }
- }
-
- if (fua) {
- io_flags |= SPDK_NVME_IO_FLAGS_FORCE_UNIT_ACCESS;
- }
-
- if (direction == SPDK_BDEV_IO_TYPE_READ) {
- rc = spdk_nvme_ns_cmd_read_with_md(bdev->nvme_ns->ns, qpair, buffer, metadata, lba,
- lba_count, bdev_nvme_queued_done, driver_ctx, io_flags, 0, 0);
- } else {
- rc = spdk_nvme_ns_cmd_write_with_md(bdev->nvme_ns->ns, qpair, buffer, metadata, lba,
- lba_count, bdev_nvme_queued_done, driver_ctx, io_flags, 0, 0);
- }
-
- check_error_type(rc, direction == SPDK_BDEV_IO_TYPE_READ, qpair);
- return rc;
-}
-
-int bdev_nvme_queue_cmd_v_with_md(struct nvme_bdev *bdev, struct spdk_nvme_qpair *qpair,
- void *driver_ctx,
- int direction, struct iovec *iov, int iovcnt, uint64_t lba_count, uint64_t lba)
-{
- int rc;
- struct nvme_bdev_io *bio = NULL;
- uint32_t io_flags = 0;
- uint8_t *bdev_io_action = (uint8_t *)driver_ctx;
- /* filter bit 0&1 of io->pi_action to get pi_action */
- uint8_t pi_action = bdev_io_action[SPDK_BDEV_IO_ACTION_PI] & 0x03;
- uint8_t dif_flag = bdev_io_action[SPDK_BDEV_IO_ACTION_PI];
- uint8_t fua = bdev_io_action[SPDK_BDEV_IO_ACTION_FUA];
- uint32_t pi_type;
-
- spdk_bdev_set_io_location(driver_ctx, (uint8_t)LOCAL_LIBSTORAGE_BDEV_NVME);
-
- if (pi_action > IO_NO_PROTECTION) {
- pi_type = spdk_nvme_ns_get_pi_type(bdev->nvme_ns->ns);
- if (dif_flag & FLAG_PRCHK) {
- io_flags |= SPDK_NVME_IO_FLAGS_PRCHK_GUARD;
- }
- /* type3 not support ref tag */
- if (!(dif_flag & FLAG_NO_REF) && (pi_type != SPDK_NVME_FMT_NVM_PROTECTION_TYPE3)) {
- io_flags |= SPDK_NVME_IO_FLAGS_PRCHK_REFTAG;
- }
- if (pi_action == IO_HALF_WAY_PROTECTION) {
- io_flags |= SPDK_NVME_IO_FLAGS_PRACT;
- }
- }
-
- if (fua) {
- io_flags |= SPDK_NVME_IO_FLAGS_FORCE_UNIT_ACCESS;
- }
-
- bio = nvme_bdev_io_update_args((struct nvme_bdev_io *)driver_ctx, iov, iovcnt);
-
- if (direction == SPDK_BDEV_IO_TYPE_READ) {
- rc = spdk_nvme_ns_cmd_readv(bdev->nvme_ns->ns, qpair, lba,
- lba_count, bdev_nvme_queued_done, bio, io_flags,
- bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge);
- } else {
- rc = spdk_nvme_ns_cmd_writev(bdev->nvme_ns->ns, qpair, lba, lba_count,
- bdev_nvme_queued_done, bio, io_flags,
- bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge);
- }
-
- check_error_type(rc, direction == SPDK_BDEV_IO_TYPE_READ, qpair);
- return rc;
-}
-
-struct nvme_bdev_ctrlr *bdev_nvme_get_ctrlr_by_bdev_desc(void *bdev_desc)
-{
- struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(bdev_desc);
- struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev->ctxt;
- if (nbdev == NULL) {
- return NULL;
- }
- return nbdev->nvme_ns->ctrlr;
-}
-
-int bdev_nvme_unmap_blocks(struct nvme_bdev *nbdev, struct spdk_io_channel *ch, void *driver_ctx,
- struct spdk_nvme_dsm_range *unmap_d, uint32_t unmap_count)
-{
- struct nvme_io_channel *nvme_ch = spdk_io_channel_get_ctx(ch);
- uint32_t i;
-
- if (unmap_count == 0 || unmap_count > SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES) {
- SPDK_ERRLOG("Invalid parameter, unmap count: %u\n", unmap_count);
- return -EINVAL;
- }
-
- if (unmap_d == NULL) {
- return -EINVAL;
- }
-
- for (i = 0; i < unmap_count; i++) {
- if (unmap_d[i].length > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS) {
- SPDK_ERRLOG("Invalid parameter, unmap block count: %u\n", unmap_d[i].length);
- return -EINVAL;
- }
- unmap_d[i].attributes.raw = 0;
- }
-
- spdk_bdev_set_io_location(driver_ctx, (uint8_t)LOCAL_LIBSTORAGE_BDEV_NVME);
- return spdk_nvme_ns_cmd_dataset_management(nbdev->nvme_ns->ns, nvme_ch->qpair,
- SPDK_NVME_DSM_ATTR_DEALLOCATE,
- unmap_d, unmap_count,
- bdev_nvme_queued_done, driver_ctx);
-}
-
-void spdk_bdev_nvme_remove_cb(void *cb_ctx, void *ctrlr)
-{
- remove_cb(cb_ctx, (struct spdk_nvme_ctrlr *)ctrlr);
-}
-
-void spdk_bdev_fail_ctrlr(void *cb_ctx, void *ctrlr)
-{
- spdk_nvme_ctrlr_fail_hotplug((struct spdk_nvme_ctrlr *)ctrlr);
- remove_cb(cb_ctx, (struct spdk_nvme_ctrlr *)ctrlr);
-}
-
-struct spdk_nvme_ctrlr *spdk_nvme_bdev_ctrlr_get(char *pci_trid)
-{
- struct nvme_bdev_ctrlr *nvme_bdev_ctrlr;
-
- TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
- if (strcmp(nvme_bdev_ctrlr->connected_trid->traddr, pci_trid) == 0) {
- return nvme_bdev_ctrlr->ctrlr;
- }
- }
-
- return NULL;
-}
-
-int spdk_bdev_nvme_create_self(struct spdk_nvme_transport_id *trid,
- const char *base_name,
- const char **names, size_t *count,
- const char *hostnqn)
-{
- struct nvme_probe_ctx *probe_ctx;
- struct nvme_bdev_ctrlr *nvme_bdev_ctrlr;
- struct nvme_bdev_ns *ns;
- struct nvme_bdev *nvme_bdev;
- struct nvme_bdev *tmp = NULL;
- uint32_t i, nsid;
- size_t j;
-
- if (nvme_bdev_ctrlr_get(trid) != NULL) {
- SPDK_ERRLOG("A controller with the trid (traddr: %s) already exists.\n", trid->traddr);
- return -1;
- }
-
- probe_ctx = bdev_nvme_create_probe_ctx(trid, base_name, hostnqn);
- if (probe_ctx == NULL) {
- SPDK_ERRLOG("Failed to create probe_ctx\n");
- return -1;
- }
-
- if (spdk_nvme_probe(trid, probe_ctx, probe_cb, attach_cb, remove_cb)) {
- SPDK_ERRLOG("Failed to probe for new devices\n");
- free(probe_ctx);
- return -1;
- }
-
- nvme_bdev_ctrlr = nvme_bdev_ctrlr_get(trid);
- if (!nvme_bdev_ctrlr) {
- SPDK_ERRLOG("Failed to find new NVMe controller\n");
- free(probe_ctx);
- return -1;
- }
-
- /*
- * Report the new bdevs that were created in this call.
- * There can be more than one bdev per NVMe controller since one bdev is created per namespace.
- */
- j = 0;
- for (i = 0; i < nvme_bdev_ctrlr->num_ns; i++) {
- nsid = i + 1;
- ns = nvme_bdev_ctrlr->namespaces[nsid - 1];
-
- if (!ns->populated) {
- continue;
- }
- assert(ns->id == nsid);
- TAILQ_FOREACH_SAFE(nvme_bdev, &ns->bdevs, tailq, tmp) {
- if (j < *count) {
- j++;
- names[j] = nvme_bdev->disk.name;
- } else {
- SPDK_ERRLOG("Maximum number of namespaces is %zu.", *count);
- free(probe_ctx);
- return -1;
- }
- }
- }
-
- *count = j;
-
- free(probe_ctx);
- return 0;
-}
+/*
+ * Copyright (C) 2021. Huawei Technologies Co., Ltd. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#define SECUREC_NEED_ERRNO_TYPE
+#include <securec.h>
+#include "bdev_nvme.h"
+
+#include "spdk/json.h"
+#include "spdk/likely.h"
+#include "spdk/bdev_module.h"
+#include "bdev_nvme.h"
+#include "spdk/nvme.h"
+#include "spdk/tree.h"
+
+#include "spdk_internal/bdev_stat.h"
+#include "bdev_nvme_self.h"
+
+void bdev_update_ch_timeout(struct nvme_poll_group *group)
+{
+ uint64_t current_ticks = 0;
+ uint64_t poll_ticks = 0;
+ int64_t poll_time = 0;
+
+ current_ticks = spdk_get_ticks();
+
+ if (spdk_unlikely(g_polltime_threshold)) {
+ if (group->save_start_ticks) {
+ poll_ticks = current_ticks - group->save_start_ticks;
+ poll_time = (poll_ticks * 1000ULL) / spdk_get_ticks_hz();
+ if (poll_time >= g_polltime_threshold) {
+ group->num_poll_timeout++;
+ SPDK_NOTICELOG("group[%p] poll timeout in %ldms", group, poll_time);
+ }
+ }
+ group->save_start_ticks = current_ticks;
+ }
+}
+
+int _bdev_nvme_submit_request_self(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io)
+{
+ struct nvme_bdev_io *nbdev_io = (struct nvme_bdev_io *)bdev_io->driver_ctx;
+
+ if (nbdev_io->io_path->qpair->qpair == NULL) {
+ /* The device is currently resetting */
+ return -1;
+ }
+
+ switch (bdev_io->type) {
+ case SPDK_BDEV_IO_TYPE_READ_NVME:
+ SPDK_DEBUGLOG(bdev_nvme, "read %lu lbas with offset %#lx\n", bdev_io->u.contig.num_blocks,
+ bdev_io->u.contig.offset_blocks);
+ return bdev_nvme_queue_cmd_with_md(nbdev_io->io_path->nvme_ns->ns, nbdev_io->io_path->qpair->qpair,
+ bdev_io->driver_ctx, bdev_io->u.contig.buf,
+ bdev_io->u.contig.md_buf, SPDK_BDEV_IO_TYPE_READ,
+ bdev_io->u.contig.num_blocks, bdev_io->u.contig.offset_blocks);
+ case SPDK_BDEV_IO_TYPE_WRITE_NVME:
+ SPDK_DEBUGLOG(bdev_nvme, "write %lu lbas with offset %#lx\n", bdev_io->u.contig.num_blocks,
+ bdev_io->u.contig.offset_blocks);
+ return bdev_nvme_queue_cmd_with_md(nbdev_io->io_path->nvme_ns->ns, nbdev_io->io_path->qpair->qpair,
+ bdev_io->driver_ctx, bdev_io->u.contig.buf,
+ bdev_io->u.contig.md_buf, SPDK_BDEV_IO_TYPE_WRITE,
+ bdev_io->u.contig.num_blocks, bdev_io->u.contig.offset_blocks);
+ case SPDK_BDEV_IO_TYPE_READV_NVME:
+ SPDK_DEBUGLOG(bdev_nvme, "readv %lu lbas with offset %#lx\n", bdev_io->u.bdev.num_blocks,
+ bdev_io->u.bdev.offset_blocks);
+ return bdev_nvme_queue_cmd_v_with_md(nbdev_io->io_path->nvme_ns->ns, nbdev_io->io_path->qpair->qpair,
+ bdev_io->driver_ctx, SPDK_BDEV_IO_TYPE_READ,
+ bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
+ bdev_io->u.bdev.num_blocks, bdev_io->u.bdev.offset_blocks);
+ case SPDK_BDEV_IO_TYPE_WRITEV_NVME:
+ SPDK_DEBUGLOG(bdev_nvme, "writev %lu lbas with offset %#lx\n", bdev_io->u.bdev.num_blocks,
+ bdev_io->u.bdev.offset_blocks);
+ return bdev_nvme_queue_cmd_v_with_md(nbdev_io->io_path->nvme_ns->ns, nbdev_io->io_path->qpair->qpair,
+ bdev_io->driver_ctx, SPDK_BDEV_IO_TYPE_WRITE,
+ bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt,
+ bdev_io->u.bdev.num_blocks, bdev_io->u.bdev.offset_blocks);
+ case SPDK_BDEV_IO_TYPE_UNMAP_BLOCKS:
+ return bdev_nvme_unmap_blocks(nbdev_io->io_path->nvme_ns->ns,
+ nbdev_io->io_path->qpair->qpair,
+ (void *)bdev_io->driver_ctx,
+ (struct spdk_nvme_dsm_range *)bdev_io->u.contig.buf,
+ bdev_io->u.contig.num_blocks);
+ default:
+ return -EINVAL;
+ }
+}
+
+int bdev_nvme_dump_info_json_self(void *ctx, struct spdk_json_write_ctx *w)
+{
+ return 0;
+}
+
+int bdev_nvme_get_io_channel_id(struct spdk_io_channel *ch)
+{
+ uint16_t channel_id = -1;
+ struct nvme_io_path *io_path = NULL;
+ struct nvme_bdev_channel *bdev_ch = spdk_io_channel_get_ctx(ch);
+ io_path = STAILQ_FIRST(&bdev_ch->io_path_list);
+ if (io_path == NULL) {
+ return channel_id;
+ }
+ struct spdk_nvme_qpair *qpair = io_path->qpair->qpair;
+ if (qpair != NULL) {
+ channel_id = spdk_nvme_get_qpair_id(qpair);
+ }
+ return channel_id;
+}
+
+uint64_t bdev_nvme_get_timeout_count(struct spdk_io_channel *ch)
+{
+ struct nvme_ctrlr_channel *ctrlr_ch;
+ if (ch == NULL) {
+ return 0;
+ }
+ ctrlr_ch = spdk_io_channel_get_ctx(ch);
+ if (ctrlr_ch->qpair == NULL || ctrlr_ch->qpair->group == NULL) {
+ return 0;
+ }
+
+ return ctrlr_ch->qpair->group->num_poll_timeout;
+}
+
+int32_t nvme_ctrlr_get_info(const char *ctrlName, struct nvme_ctrlr_info **ppCtrlr)
+{
+ uint32_t num_ctrlr = 0, i = 0;
+ struct nvme_bdev_ctrlr *nbdev_ctrlr = NULL;
+ struct nvme_ctrlr *nvme_ctrlr = NULL;
+ struct nvme_ctrlr_info *pCtrlrInfo = NULL;
+ const struct spdk_nvme_ctrlr_data *cdata = NULL;
+ const struct spdk_nvme_ctrlr_opts *opts = NULL;
+
+ struct spdk_pci_device *pci_dev = NULL;
+ int rc;
+
+ TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
+ num_ctrlr++;
+ }
+ if (num_ctrlr == 0) {
+ SPDK_NOTICELOG("No any nvme controller.\n");
+ return 0;
+ }
+ if (ctrlName != NULL) {
+ num_ctrlr = 1;
+ }
+ pCtrlrInfo = calloc(num_ctrlr, sizeof(struct nvme_ctrlr_info));
+ if (pCtrlrInfo == NULL) {
+ SPDK_ERRLOG("Failed to alloc memory for getting controller infomation.\n");
+ return -1;
+ }
+ TAILQ_FOREACH(nbdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
+ if (i >= num_ctrlr) { /* prevent having controllers be added or deleted */
+ i++;
+ continue;
+ }
+ if (ctrlName != NULL) {
+ if (strcmp(nbdev_ctrlr->name, ctrlName) != 0) {
+ continue;
+ }
+ }
+ nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
+ cdata = spdk_nvme_ctrlr_get_data(nvme_ctrlr->ctrlr);
+ opts = spdk_nvme_ctrlr_get_opts(nvme_ctrlr->ctrlr);
+ pci_dev = spdk_nvme_ctrlr_get_pci_device(nvme_ctrlr->ctrlr);
+ if (pci_dev == NULL) {
+ SPDK_ERRLOG("Failed to get pci device\n");
+ break;
+ }
+ rc = strcpy_s(pCtrlrInfo[i].ctrlName, sizeof(pCtrlrInfo[i].ctrlName), nbdev_ctrlr->name);
+ if (rc != 0) {
+ SPDK_ERRLOG("String copy failed\n");
+ }
+ rc = strcpy_s(pCtrlrInfo[i].pciAddr, sizeof(pCtrlrInfo[i].pciAddr),
+ nvme_ctrlr->active_path_id->trid.traddr);
+ if (rc != 0) {
+ SPDK_ERRLOG("String copy failed\n");
+ }
+
+ rc = memcpy_s(pCtrlrInfo[i].sn, sizeof(pCtrlrInfo[i].sn), cdata->sn, 20);
+ if (rc != 0) {
+ SPDK_ERRLOG("Memory copy failed\n");
+ }
+
+ rc = memcpy_s(pCtrlrInfo[i].fr, sizeof(pCtrlrInfo[i].fr), cdata->fr, 8);
+ if (rc != 0) {
+ SPDK_ERRLOG("Memory copy failed\n");
+ }
+
+ rc = memcpy_s(pCtrlrInfo[i].mn, sizeof(pCtrlrInfo[i].mn), cdata->mn, 40);
+ if (rc != 0) {
+ SPDK_ERRLOG("Memory copy failed\n");
+ }
+
+ pCtrlrInfo[i].trtype = (uint16_t)nvme_ctrlr->active_path_id->trid.trtype;
+ pCtrlrInfo[i].tnvmcap = cdata->tnvmcap[0];
+ pCtrlrInfo[i].unvmcap = cdata->unvmcap[0];
+ pCtrlrInfo[i].support_ns = cdata->oacs.ns_manage;
+ pCtrlrInfo[i].directives = cdata->oacs.directives;
+ pCtrlrInfo[i].dsm = cdata->oncs.dsm;
+ pCtrlrInfo[i].max_num_ns = cdata->nn;
+ pCtrlrInfo[i].num_io_queues = opts->num_io_queues;
+ pCtrlrInfo[i].io_queue_size = opts->io_queue_size;
+ pCtrlrInfo[i].device_id = spdk_pci_device_get_device_id(pci_dev);
+ pCtrlrInfo[i].subdevice_id = spdk_pci_device_get_subdevice_id(pci_dev);
+ pCtrlrInfo[i].vid = cdata->vid;
+ pCtrlrInfo[i].ssvid = cdata->ssvid;
+ pCtrlrInfo[i].ctrlid = cdata->cntlid;
+ pCtrlrInfo[i].version = spdk_nvme_ctrlr_get_regs_vs(nvme_ctrlr->ctrlr).raw;
+ i++;
+ if (ctrlName != NULL) {
+ break;
+ }
+ }
+ if (i != num_ctrlr) {
+ SPDK_ERRLOG("It has controller been added or deleted when fetched infomation, please try again later.\n");
+ free(pCtrlrInfo);
+ return -1;
+ }
+ *ppCtrlr = pCtrlrInfo;
+ return num_ctrlr;
+}
+
+struct spdk_nvme_ctrlr *spdk_nvme_ctrlr_get_by_name(const char *ctrlname)
+{
+ struct nvme_ctrlr *nvme_ctrlr = NULL;
+
+ nvme_ctrlr = nvme_ctrlr_get_by_name(ctrlname);
+ if (nvme_ctrlr == NULL) {
+ return NULL;
+ }
+
+ return nvme_ctrlr->ctrlr;
+}
+
+struct spdk_nvme_ctrlr *spdk_nvme_ctrlr_get_by_ctrlr(const struct nvme_bdev_ctrlr *nbdev_ctrlr)
+{
+ struct nvme_ctrlr *nvme_ctrlr = NULL;
+
+ if (nbdev_ctrlr == NULL) {
+ return NULL;
+ }
+ nvme_ctrlr = TAILQ_FIRST(&nbdev_ctrlr->ctrlrs);
+
+ return nvme_ctrlr->ctrlr;
+}
+
+void nvme_ctrlr_clear_iostat_by_name(const char *ctrlname)
+{
+ int i;
+ size_t size = strnlen(ctrlname, 24);
+
+ for (i = 0; i < STAT_MAX_NUM; i++) {
+ if (strncmp(g_io_stat_map[i].bdev_name, ctrlname, size) == 0) {
+ if ((g_io_stat_map[i].bdev_name[size] == 'n') && isdigit(g_io_stat_map[i].bdev_name[size + 1])) {
+ g_io_stat_map[i].channel_id = 0;
+ (void)memset_s(g_io_stat_map[i].bdev_name, sizeof(g_io_stat_map[i].bdev_name), 0, sizeof(g_io_stat_map[i].bdev_name));
+ g_io_stat_map[i].num_read_ops = 0;
+ g_io_stat_map[i].num_write_ops = 0;
+ g_io_stat_map[i].bytes_read = 0;
+ g_io_stat_map[i].bytes_written = 0;
+ g_io_stat_map[i].io_outstanding = 0;
+ g_io_stat_map[i].read_latency_ticks = 0;
+ g_io_stat_map[i].write_latency_ticks = 0;
+ g_io_stat_map[i].io_ticks = 0;
+ /* used flag set false in last avoid race in channel create */
+ g_io_stat_map[i].used = false;
+ }
+ }
+ }
+}
+
+void nvme_ctrlr_clear_iostat_all(void)
+{
+ struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = NULL;
+
+ TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
+ nvme_ctrlr_clear_iostat_by_name(nvme_bdev_ctrlr->name);
+ }
+}
+
+struct spdk_nvme_ns *bdev_nvme_get_ns(struct nvme_bdev *nbdev)
+{
+ struct spdk_nvme_ctrlr *ctrlr;
+ if (nbdev == NULL) {
+ return NULL;
+ }
+
+ ctrlr = spdk_nvme_ctrlr_get_by_ctrlr(nbdev->nbdev_ctrlr);
+ if (ctrlr == NULL) {
+ return NULL;
+ }
+
+ return spdk_nvme_ctrlr_get_ns(ctrlr, nbdev->nsid);
+}
+
+void bdev_nvme_update_block_by_nvme_ctrlr(struct spdk_nvme_ctrlr *ctrlr)
+{
+ uint32_t nsid;
+ struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = NULL;
+ struct nvme_bdev *nvme_bdev = NULL;
+ struct spdk_nvme_ns *ns;
+
+ pthread_mutex_lock(&g_bdev_nvme_mutex);
+ TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
+ if (spdk_nvme_ctrlr_get_by_ctrlr(nvme_bdev_ctrlr) != ctrlr) {
+ continue;
+ }
+
+ for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr);
+ nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
+ ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+ TAILQ_FOREACH(nvme_bdev, &nvme_bdev_ctrlr->bdevs, tailq) {
+ if (nvme_bdev->nsid == nsid) {
+ nvme_bdev->disk.blocklen = spdk_nvme_ns_get_sector_size(ns);
+ nvme_bdev->disk.blockcnt = spdk_nvme_ns_get_num_sectors(ns);
+ }
+ }
+ }
+
+ pthread_mutex_unlock(&g_bdev_nvme_mutex);
+ return;
+ }
+ pthread_mutex_unlock(&g_bdev_nvme_mutex);
+}
+
+static int
+nvme_ns_cmp(struct nvme_ns *ns1, struct nvme_ns *ns2)
+{
+ return ns1->id < ns2->id ? -1 : ns1->id > ns2->id;
+}
+
+RB_GENERATE_STATIC(nvme_ns_tree, nvme_ns, node, nvme_ns_cmp);
+
+int bdev_nvme_update_ns(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, uint32_t nsid)
+{
+ struct nvme_ns *ns = NULL;
+ bool ns_active = false;
+ struct spdk_nvme_ctrlr *ctrlr = NULL;
+ struct nvme_ctrlr *nvme_ctrlr = NULL;
+
+
+ ctrlr = spdk_nvme_ctrlr_get_by_ctrlr(nvme_bdev_ctrlr);
+
+ if (nvme_bdev_ctrlr == NULL || ctrlr == NULL || nsid > spdk_nvme_ctrlr_get_num_ns(ctrlr)) {
+ SPDK_ERRLOG("Parameter error. nsid[%u], the max nsid is[%u]\n", nsid,
+ spdk_nvme_ctrlr_get_num_ns(ctrlr));
+ return -1;
+ }
+
+ nvme_ctrlr = TAILQ_FIRST(&nvme_bdev_ctrlr->ctrlrs);
+ if (nvme_ctrlr == NULL) {
+ SPDK_ERRLOG("Can not get nvme_ctrlr, update abort.\n");
+ return -1;
+ }
+
+ ns_active = spdk_nvme_ctrlr_is_active_ns(ctrlr, nsid);
+
+ ns = nvme_ctrlr_get_ns(nvme_ctrlr, nsid);
+ if (ns == NULL && ns_active) {
+ ns = nvme_ns_alloc();
+ if (ns == NULL) {
+ SPDK_ERRLOG("Failed to allocate namespace\n");
+ return -1;
+ }
+ SPDK_NOTICELOG("NSID %u to be added\n", nsid);
+ ns->id = nsid;
+ ns->ctrlr = nvme_ctrlr;
+ ns->bdev = NULL;
+ RB_INSERT(nvme_ns_tree, &nvme_ctrlr->namespaces, ns);
+ /* add a new bdev device in this ns */
+ nvme_ctrlr_populate_namespace(nvme_ctrlr, ns);
+ }
+
+ if (ns != NULL && !ns_active) {
+ SPDK_NOTICELOG("NSID %u is removed\n", nsid);
+ nvme_ctrlr_depopulate_namespace(nvme_ctrlr, ns);
+ return 0;
+ }
+
+ return 0;
+}
+
+bool spdk_bdev_can_remove(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, uint32_t nsid)
+{
+ struct nvme_bdev *bdev = NULL;
+ bool empty = false;
+
+ pthread_mutex_lock(&g_bdev_nvme_mutex);
+ TAILQ_FOREACH(bdev, &nvme_bdev_ctrlr->bdevs, tailq) {
+ if (bdev->nsid == nsid) {
+ break;
+ }
+ }
+ pthread_mutex_unlock(&g_bdev_nvme_mutex);
+
+ if (bdev == NULL) {
+ return true;
+ }
+
+ spdk_spin_lock(&bdev->disk.internal.spinlock);
+ empty = TAILQ_EMPTY(&bdev->disk.internal.open_descs);
+
+ if (empty) {
+ /* one bdev is empty, check next until all bdev is checked */
+ bdev->disk.internal.ns_status = SPDK_BDEV_NS_STATUS_REMOVING;
+ }
+ /* means at least one bdev is used, so we just quit this process
+ and mark the status is false. */
+ spdk_spin_unlock(&bdev->disk.internal.spinlock);
+
+ return empty;
+}
+
+void spdk_bdev_set_ns_normal(struct nvme_bdev_ctrlr *nvme_bdev_ctrlr, uint32_t nsid)
+{
+ struct nvme_bdev *bdev = NULL;
+
+ pthread_mutex_lock(&g_bdev_nvme_mutex);
+ TAILQ_FOREACH(bdev, &nvme_bdev_ctrlr->bdevs, tailq) {
+ if (bdev->nsid == nsid) {
+ break;
+ }
+ }
+ pthread_mutex_unlock(&g_bdev_nvme_mutex);
+
+ if (bdev == NULL) {
+ return;
+ }
+
+ spdk_spin_lock(&bdev->disk.internal.spinlock);
+ if (bdev->disk.internal.ns_status == SPDK_BDEV_NS_STATUS_REMOVING) {
+ bdev->disk.internal.ns_status = SPDK_BDEV_NS_STATUS_READY;
+ }
+ spdk_spin_unlock(&bdev->disk.internal.spinlock);
+}
+
+static void check_error_type(int rc, bool read, void *qpair)
+{
+ if (rc == -ENOMEM) {
+ SPDK_NOTICELOG("%s failed: rc = %d\n", read ? "read" : "write", rc);
+ } else if (rc < 0) {
+ SPDK_ERRLOG("%s failed: rc = %d, qpair is %p\n", read ? "read" : "write", rc, qpair);
+ }
+}
+
+int bdev_nvme_queue_cmd_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+ void *driver_ctx,
+ void *buffer, void *metadata, int direction, uint64_t lba_count, uint64_t lba)
+{
+ int rc;
+ uint32_t io_flags = 0;
+ uint8_t* bdev_io_action = (uint8_t *)driver_ctx;
+ /* filter bit 0&1 of io->pi_action to get pi_action */
+ uint8_t pi_action = bdev_io_action[SPDK_BDEV_IO_ACTION_PI] & 0x03;
+ uint8_t dif_flag = bdev_io_action[SPDK_BDEV_IO_ACTION_PI];
+ uint8_t fua = bdev_io_action[SPDK_BDEV_IO_ACTION_FUA];
+ uint16_t streamId = bdev_io_action[SPDK_BDEV_IO_STREAM_ID_0];
+ uint32_t pi_type;
+
+ streamId |= ((uint16_t)bdev_io_action[SPDK_BDEV_IO_STREAM_ID_1] << 8);
+ spdk_bdev_set_io_location(driver_ctx, (uint8_t)LOCAL_LIBSTORAGE_BDEV_NVME);
+
+ if (streamId > 0) {
+ io_flags |= SPDK_NVME_IO_FLAGS_STREAMS_DIRECTIVE;
+ }
+
+ if (pi_action > IO_NO_PROTECTION) {
+ pi_type = spdk_nvme_ns_get_pi_type(ns);
+ if (dif_flag & FLAG_PRCHK) {
+ io_flags |= SPDK_NVME_IO_FLAGS_PRCHK_GUARD;
+ }
+ /* type3 not support ref tag */
+ if (!(dif_flag & FLAG_NO_REF) && (pi_type != SPDK_NVME_FMT_NVM_PROTECTION_TYPE3)) {
+ io_flags |= SPDK_NVME_IO_FLAGS_PRCHK_REFTAG;
+ }
+ if (pi_action == IO_HALF_WAY_PROTECTION) {
+ io_flags |= SPDK_NVME_IO_FLAGS_PRACT;
+ }
+ }
+
+ if (fua) {
+ io_flags |= SPDK_NVME_IO_FLAGS_FORCE_UNIT_ACCESS;
+ }
+
+ if (direction == SPDK_BDEV_IO_TYPE_READ) {
+ rc = spdk_nvme_ns_cmd_read_with_md(ns, qpair, buffer, metadata, lba,
+ lba_count, bdev_nvme_queued_done, driver_ctx, io_flags, 0, 0);
+ } else {
+ rc = spdk_nvme_ns_cmd_write_with_md(ns, qpair, buffer, metadata, lba,
+ lba_count, bdev_nvme_queued_done, driver_ctx, io_flags, 0, streamId);
+ }
+
+ check_error_type(rc, direction == SPDK_BDEV_IO_TYPE_READ, qpair);
+ return rc;
+}
+
+int bdev_nvme_queue_cmd_v_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+ void *driver_ctx,
+ int direction, struct iovec *iov, int iovcnt, uint64_t lba_count, uint64_t lba)
+{
+ int rc;
+ struct nvme_bdev_io *bio = NULL;
+ uint32_t io_flags = 0;
+ uint8_t* bdev_io_action = (uint8_t *)driver_ctx;
+ /* filter bit 0&1 of io->pi_action to get pi_action */
+ uint8_t pi_action = bdev_io_action[SPDK_BDEV_IO_ACTION_PI] & 0x03;
+ uint8_t dif_flag = bdev_io_action[SPDK_BDEV_IO_ACTION_PI];
+ uint8_t fua = bdev_io_action[SPDK_BDEV_IO_ACTION_FUA];
+ uint16_t streamId = bdev_io_action[SPDK_BDEV_IO_STREAM_ID_0];
+ uint32_t pi_type;
+
+ streamId |= ((uint16_t)bdev_io_action[SPDK_BDEV_IO_STREAM_ID_1] << 8);
+ spdk_bdev_set_io_location(driver_ctx, (uint8_t)LOCAL_LIBSTORAGE_BDEV_NVME);
+
+ if (streamId > 0) {
+ io_flags |= SPDK_NVME_IO_FLAGS_STREAMS_DIRECTIVE;
+ }
+
+ if (pi_action > IO_NO_PROTECTION) {
+ pi_type = spdk_nvme_ns_get_pi_type(ns);
+ if (dif_flag & FLAG_PRCHK) {
+ io_flags |= SPDK_NVME_IO_FLAGS_PRCHK_GUARD;
+ }
+ /* type3 not support ref tag */
+ if (!(dif_flag & FLAG_NO_REF) && (pi_type != SPDK_NVME_FMT_NVM_PROTECTION_TYPE3)) {
+ io_flags |= SPDK_NVME_IO_FLAGS_PRCHK_REFTAG;
+ }
+ if (pi_action == IO_HALF_WAY_PROTECTION) {
+ io_flags |= SPDK_NVME_IO_FLAGS_PRACT;
+ }
+ }
+
+ if (fua) {
+ io_flags |= SPDK_NVME_IO_FLAGS_FORCE_UNIT_ACCESS;
+ }
+
+ bio = nvme_bdev_io_update_args((struct nvme_bdev_io *)driver_ctx, iov, iovcnt);
+
+ if (direction == SPDK_BDEV_IO_TYPE_READ) {
+ rc = spdk_nvme_ns_cmd_readv(ns, qpair, lba,
+ lba_count, bdev_nvme_queued_done, bio, io_flags,
+ bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge);
+ } else {
+ rc = spdk_nvme_ns_cmd_writev(ns, qpair, lba, lba_count,
+ bdev_nvme_queued_done, bio, io_flags,
+ bdev_nvme_queued_reset_sgl, bdev_nvme_queued_next_sge);
+ }
+
+ check_error_type(rc, direction == SPDK_BDEV_IO_TYPE_READ, qpair);
+ return rc;
+}
+
+struct nvme_bdev_ctrlr *bdev_nvme_get_ctrlr_by_bdev_desc(void *bdev_desc)
+{
+ struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(bdev_desc);
+ struct nvme_bdev *nbdev = (struct nvme_bdev *)bdev->ctxt;
+ if (nbdev == NULL) {
+ return NULL;
+ }
+ return nbdev->nbdev_ctrlr;
+}
+
+int bdev_nvme_unmap_blocks(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *driver_ctx,
+ struct spdk_nvme_dsm_range *unmap_d, uint32_t unmap_count)
+{
+ uint32_t i;
+
+ if (unmap_count == 0 || unmap_count > SPDK_NVME_DATASET_MANAGEMENT_MAX_RANGES) {
+ SPDK_ERRLOG("Invalid parameter, unmap count: %u\n", unmap_count);
+ return -EINVAL;
+ }
+
+ if (unmap_d == NULL) {
+ return -EINVAL;
+ }
+
+ for (i = 0; i < unmap_count; i++) {
+ unmap_d[i].attributes.raw = 0;
+ }
+
+ spdk_bdev_set_io_location(driver_ctx, (uint8_t)LOCAL_LIBSTORAGE_BDEV_NVME);
+ return spdk_nvme_ns_cmd_dataset_management(ns, qpair,
+ SPDK_NVME_DSM_ATTR_DEALLOCATE,
+ unmap_d, unmap_count,
+ bdev_nvme_queued_done, driver_ctx);
+}
+
+void spdk_bdev_nvme_remove_cb(void *cb_ctx, void *ctrlr)
+{
+ remove_cb(cb_ctx, (struct spdk_nvme_ctrlr *)ctrlr);
+}
+
+void spdk_bdev_fail_ctrlr(void *cb_ctx, void *ctrlr)
+{
+ spdk_nvme_ctrlr_fail_hotplug((struct spdk_nvme_ctrlr *)ctrlr);
+ remove_cb(cb_ctx, (struct spdk_nvme_ctrlr *)ctrlr);
+}
+
+struct spdk_nvme_ctrlr *spdk_nvme_bdev_ctrlr_get(char *pci_trid)
+{
+ struct nvme_bdev_ctrlr *nvme_bdev_ctrlr = NULL;
+ struct nvme_ctrlr *nvme_ctrlr;
+
+ TAILQ_FOREACH(nvme_bdev_ctrlr, &g_nvme_bdev_ctrlrs, tailq) {
+ TAILQ_FOREACH(nvme_ctrlr, &nvme_bdev_ctrlr->ctrlrs, tailq) {
+ if (strcmp(nvme_ctrlr->active_path_id->trid.traddr, pci_trid) == 0) {
+ return nvme_ctrlr->ctrlr;
+ }
+ }
+ }
+
+ return NULL;
+}
+
+int spdk_bdev_nvme_create_self(struct spdk_nvme_transport_id *trid,
+ const char *base_name,
+ const char **names, size_t *count,
+ const char *hostnqn)
+{
+ struct nvme_probe_ctx *probe_ctx = NULL;
+
+ if (nvme_ctrlr_get(trid) != NULL) {
+ SPDK_ERRLOG("A controller with the trid (traddr: %s) already exists.\n", trid->traddr);
+ return -1;
+ }
+
+ probe_ctx = bdev_nvme_create_probe_ctx(trid, base_name, hostnqn);
+ if (probe_ctx == NULL) {
+ SPDK_ERRLOG("Failed to create probe_ctx\n");
+ return -1;
+ }
+
+ if (spdk_nvme_probe(trid, probe_ctx, probe_cb, attach_cb, remove_cb)) {
+ SPDK_ERRLOG("Failed to probe for new devices\n");
+ free(probe_ctx);
+ return -1;
+ }
+
+ free(probe_ctx);
+ return 0;
+}
@@ -1,43 +1,43 @@
-/*
- * Copyright (C) 2021. Huawei Technologies Co., Ltd. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 and
- * only version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
-
-void
-bdev_update_ch_timeout(struct nvme_bdev_poll_group *group);
-
-int
-_bdev_nvme_submit_request_self(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io);
-
-int
-bdev_nvme_dump_info_json_self(void *ctx, struct spdk_json_write_ctx *w);
-
-uint16_t
-bdev_nvme_get_io_channel_id(struct spdk_io_channel *ch);
-
-uint64_t
-bdev_nvme_get_timeout_count(struct spdk_io_channel *ch);
-
-int
-bdev_nvme_queue_cmd_with_md(struct nvme_bdev *bdev, struct spdk_nvme_qpair *qpair, void *driver_ctx,
- void *buffer, void *metadata, int direction, uint64_t lba_count, uint64_t lba);
-
-int
-bdev_nvme_queue_cmd_v_with_md(struct nvme_bdev *bdev, struct spdk_nvme_qpair *qpair,
- void *driver_ctx,
- int direction, struct iovec *iov, int iovcnt, uint64_t lba_count, uint64_t lba);
-
-struct nvme_bdev_ctrlr *
-bdev_nvme_get_ctrlr_by_bdev_desc(void *bdev_desc);
-
-int
-bdev_nvme_unmap_blocks(struct nvme_bdev *nbdev, struct spdk_io_channel *ch, void *driver_ctx,
- struct spdk_nvme_dsm_range *unmap_d, uint32_t unmap_count);
+/*
+ * Copyright (C) 2021. Huawei Technologies Co., Ltd. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+void
+bdev_update_ch_timeout(struct nvme_poll_group *group);
+
+int
+_bdev_nvme_submit_request_self(struct nvme_bdev_channel *nbdev_ch, struct spdk_bdev_io *bdev_io);
+
+int
+bdev_nvme_dump_info_json_self(void *ctx, struct spdk_json_write_ctx *w);
+
+int
+bdev_nvme_get_io_channel_id(struct spdk_io_channel *ch);
+
+uint64_t
+bdev_nvme_get_timeout_count(struct spdk_io_channel *ch);
+
+int
+bdev_nvme_queue_cmd_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *driver_ctx,
+ void *buffer, void *metadata, int direction, uint64_t lba_count, uint64_t lba);
+
+int
+bdev_nvme_queue_cmd_v_with_md(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,
+ void *driver_ctx,
+ int direction, struct iovec *iov, int iovcnt, uint64_t lba_count, uint64_t lba);
+
+struct nvme_bdev_ctrlr *
+bdev_nvme_get_ctrlr_by_bdev_desc(void *bdev_desc);
+
+int
+bdev_nvme_unmap_blocks(struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair, void *driver_ctx,
+ struct spdk_nvme_dsm_range *unmap_d, uint32_t unmap_count);
--
2.43.0