@@ -13107,14 +13107,6 @@ F: drivers/nvme/host/
F: include/linux/nvme.h
F: include/uapi/linux/nvme_ioctl.h
-NVM EXPRESS TCP OFFLOAD TRANSPORT DRIVERS
-M: Shai Malin <smalin@marvell.com>
-M: Ariel Elior <aelior@marvell.com>
-L: linux-nvme@lists.infradead.org
-S: Supported
-F: drivers/nvme/host/tcp-offload.c
-F: drivers/nvme/host/tcp-offload.h
-
NVM EXPRESS FC TRANSPORT DRIVERS
M: James Smart <james.smart@broadcom.com>
L: linux-nvme@lists.infradead.org
@@ -84,19 +84,3 @@ config NVME_TCP
from https://github.com/linux-nvme/nvme-cli.
If unsure, say N.
-
-config NVME_TCP_OFFLOAD
- tristate "NVM Express over Fabrics TCP offload common layer"
- depends on BLOCK
- depends on INET
- select NVME_CORE
- select NVME_FABRICS
- help
- This provides support for the NVMe over Fabrics protocol using
- the TCP offload transport. This allows you to use remote block devices
- exported using the NVMe protocol set.
-
- To configure a NVMe over Fabrics controller use the nvme-cli tool
- from https://github.com/linux-nvme/nvme-cli.
-
- If unsure, say N.
@@ -8,7 +8,6 @@ obj-$(CONFIG_NVME_FABRICS) += nvme-fabrics.o
obj-$(CONFIG_NVME_RDMA) += nvme-rdma.o
obj-$(CONFIG_NVME_FC) += nvme-fc.o
obj-$(CONFIG_NVME_TCP) += nvme-tcp.o
-obj-$(CONFIG_NVME_TCP_OFFLOAD) += nvme-tcp-offload.o
nvme-core-y := core.o ioctl.o
nvme-core-$(CONFIG_TRACING) += trace.o
@@ -27,5 +26,3 @@ nvme-rdma-y += rdma.o
nvme-fc-y += fc.o
nvme-tcp-y += tcp.o
-
-nvme-tcp-offload-y += tcp-offload.o
@@ -860,8 +860,8 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
return ret;
}
-int nvmf_check_required_opts(struct nvmf_ctrl_options *opts,
- unsigned int required_opts)
+static int nvmf_check_required_opts(struct nvmf_ctrl_options *opts,
+ unsigned int required_opts)
{
if ((opts->mask & required_opts) != required_opts) {
int i;
@@ -879,7 +879,6 @@ int nvmf_check_required_opts(struct nvmf_ctrl_options *opts,
return 0;
}
-EXPORT_SYMBOL_GPL(nvmf_check_required_opts);
bool nvmf_ip_options_match(struct nvme_ctrl *ctrl,
struct nvmf_ctrl_options *opts)
@@ -943,6 +942,13 @@ void nvmf_free_options(struct nvmf_ctrl_options *opts)
}
EXPORT_SYMBOL_GPL(nvmf_free_options);
+#define NVMF_REQUIRED_OPTS (NVMF_OPT_TRANSPORT | NVMF_OPT_NQN)
+#define NVMF_ALLOWED_OPTS (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \
+ NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \
+ NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\
+ NVMF_OPT_DISABLE_SQFLOW |\
+ NVMF_OPT_FAIL_FAST_TMO)
+
static struct nvme_ctrl *
nvmf_create_ctrl(struct device *dev, const char *buf)
{
@@ -68,13 +68,6 @@ enum {
NVMF_OPT_FAIL_FAST_TMO = 1 << 20,
};
-#define NVMF_REQUIRED_OPTS (NVMF_OPT_TRANSPORT | NVMF_OPT_NQN)
-#define NVMF_ALLOWED_OPTS (NVMF_OPT_QUEUE_SIZE | NVMF_OPT_NR_IO_QUEUES | \
- NVMF_OPT_KATO | NVMF_OPT_HOSTNQN | \
- NVMF_OPT_HOST_ID | NVMF_OPT_DUP_CONNECT |\
- NVMF_OPT_DISABLE_SQFLOW |\
- NVMF_OPT_FAIL_FAST_TMO)
-
/**
* struct nvmf_ctrl_options - Used to hold the options specified
* with the parsing opts enum.
@@ -193,7 +186,5 @@ int nvmf_get_address(struct nvme_ctrl *ctrl, char *buf, int size);
bool nvmf_should_reconnect(struct nvme_ctrl *ctrl);
bool nvmf_ip_options_match(struct nvme_ctrl *ctrl,
struct nvmf_ctrl_options *opts);
-int nvmf_check_required_opts(struct nvmf_ctrl_options *opts,
- unsigned int required_opts);
#endif /* _NVME_FABRICS_H */
deleted file mode 100644
@@ -1,1318 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright 2021 Marvell. All rights reserved.
- */
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-/* Kernel includes */
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-/* Driver includes */
-#include "tcp-offload.h"
-
-static LIST_HEAD(nvme_tcp_ofld_devices);
-static DEFINE_MUTEX(nvme_tcp_ofld_devices_mutex);
-static LIST_HEAD(nvme_tcp_ofld_ctrl_list);
-static DEFINE_MUTEX(nvme_tcp_ofld_ctrl_mutex);
-static struct blk_mq_ops nvme_tcp_ofld_admin_mq_ops;
-static struct blk_mq_ops nvme_tcp_ofld_mq_ops;
-
-static inline struct nvme_tcp_ofld_ctrl *to_tcp_ofld_ctrl(struct nvme_ctrl *nctrl)
-{
- return container_of(nctrl, struct nvme_tcp_ofld_ctrl, nctrl);
-}
-
-static inline int nvme_tcp_ofld_qid(struct nvme_tcp_ofld_queue *queue)
-{
- return queue - queue->ctrl->queues;
-}
-
-/**
- * nvme_tcp_ofld_register_dev() - NVMeTCP Offload Library registration
- * function.
- * @dev: NVMeTCP offload device instance to be registered to the
- * common tcp offload instance.
- *
- * API function that registers the type of vendor specific driver
- * being implemented to the common NVMe over TCP offload library. Part of
- * the overall init sequence of starting up an offload driver.
- */
-int nvme_tcp_ofld_register_dev(struct nvme_tcp_ofld_dev *dev)
-{
- struct nvme_tcp_ofld_ops *ops = dev->ops;
-
- if (!ops->claim_dev ||
- !ops->setup_ctrl ||
- !ops->release_ctrl ||
- !ops->create_queue ||
- !ops->drain_queue ||
- !ops->destroy_queue ||
- !ops->poll_queue ||
- !ops->send_req)
- return -EINVAL;
-
- mutex_lock(&nvme_tcp_ofld_devices_mutex);
- list_add_tail(&dev->entry, &nvme_tcp_ofld_devices);
- mutex_unlock(&nvme_tcp_ofld_devices_mutex);
-
- return 0;
-}
-EXPORT_SYMBOL_GPL(nvme_tcp_ofld_register_dev);
-
-/**
- * nvme_tcp_ofld_unregister_dev() - NVMeTCP Offload Library unregistration
- * function.
- * @dev: NVMeTCP offload device instance to be unregistered from the
- * common tcp offload instance.
- *
- * API function that unregisters the type of vendor specific driver being
- * implemented from the common NVMe over TCP offload library.
- * Part of the overall exit sequence of unloading the implemented driver.
- */
-void nvme_tcp_ofld_unregister_dev(struct nvme_tcp_ofld_dev *dev)
-{
- mutex_lock(&nvme_tcp_ofld_devices_mutex);
- list_del(&dev->entry);
- mutex_unlock(&nvme_tcp_ofld_devices_mutex);
-}
-EXPORT_SYMBOL_GPL(nvme_tcp_ofld_unregister_dev);
-
-/**
- * nvme_tcp_ofld_error_recovery() - NVMeTCP Offload library error recovery.
- * function.
- * @nctrl: NVMe controller instance to change to resetting.
- *
- * API function that change the controller state to resseting.
- * Part of the overall controller reset sequence.
- */
-void nvme_tcp_ofld_error_recovery(struct nvme_ctrl *nctrl)
-{
- if (!nvme_change_ctrl_state(nctrl, NVME_CTRL_RESETTING))
- return;
-
- queue_work(nvme_reset_wq, &to_tcp_ofld_ctrl(nctrl)->err_work);
-}
-EXPORT_SYMBOL_GPL(nvme_tcp_ofld_error_recovery);
-
-/**
- * nvme_tcp_ofld_report_queue_err() - NVMeTCP Offload report error event
- * callback function. Pointed to by nvme_tcp_ofld_queue->report_err.
- * @queue: NVMeTCP offload queue instance on which the error has occurred.
- *
- * API function that allows the vendor specific offload driver to reports errors
- * to the common offload layer, to invoke error recovery.
- */
-int nvme_tcp_ofld_report_queue_err(struct nvme_tcp_ofld_queue *queue)
-{
- pr_err("nvme-tcp-offload queue error\n");
- nvme_tcp_ofld_error_recovery(&queue->ctrl->nctrl);
-
- return 0;
-}
-
-/**
- * nvme_tcp_ofld_req_done() - NVMeTCP Offload request done callback
- * function. Pointed to by nvme_tcp_ofld_req->done.
- * Handles both NVME_TCP_F_DATA_SUCCESS flag and NVMe CQ.
- * @req: NVMeTCP offload request to complete.
- * @result: The nvme_result.
- * @status: The completion status.
- *
- * API function that allows the vendor specific offload driver to report request
- * completions to the common offload layer.
- */
-void nvme_tcp_ofld_req_done(struct nvme_tcp_ofld_req *req,
- union nvme_result *result,
- __le16 status)
-{
- struct request *rq = blk_mq_rq_from_pdu(req);
-
- if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), *result))
- nvme_complete_rq(rq);
-}
-
-/**
- * nvme_tcp_ofld_async_req_done() - NVMeTCP Offload request done callback
- * function for async request. Pointed to by nvme_tcp_ofld_req->done.
- * Handles both NVME_TCP_F_DATA_SUCCESS flag and NVMe CQ.
- * @req: NVMeTCP offload request to complete.
- * @result: The nvme_result.
- * @status: The completion status.
- *
- * API function that allows the vendor specific offload driver to report request
- * completions to the common offload layer.
- */
-void nvme_tcp_ofld_async_req_done(struct nvme_tcp_ofld_req *req,
- union nvme_result *result, __le16 status)
-{
- struct nvme_tcp_ofld_queue *queue = req->queue;
- struct nvme_tcp_ofld_ctrl *ctrl = queue->ctrl;
-
- nvme_complete_async_event(&ctrl->nctrl, status, result);
-}
-
-static struct nvme_tcp_ofld_dev *
-nvme_tcp_ofld_lookup_dev(struct nvme_tcp_ofld_ctrl *ctrl)
-{
- struct nvme_tcp_ofld_dev *dev;
-
- mutex_lock(&nvme_tcp_ofld_devices_mutex);
- list_for_each_entry(dev, &nvme_tcp_ofld_devices, entry) {
- if (dev->ops->claim_dev(dev, ctrl))
- goto out;
- }
-
- dev = NULL;
-out:
- mutex_unlock(&nvme_tcp_ofld_devices_mutex);
-
- return dev;
-}
-
-static struct blk_mq_tag_set *
-nvme_tcp_ofld_alloc_tagset(struct nvme_ctrl *nctrl, bool admin)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
- struct blk_mq_tag_set *set;
- int rc;
-
- if (admin) {
- set = &ctrl->admin_tag_set;
- memset(set, 0, sizeof(*set));
- set->ops = &nvme_tcp_ofld_admin_mq_ops;
- set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
- set->reserved_tags = NVMF_RESERVED_TAGS;
- set->numa_node = nctrl->numa_node;
- set->flags = BLK_MQ_F_BLOCKING;
- set->cmd_size = sizeof(struct nvme_tcp_ofld_req);
- set->driver_data = ctrl;
- set->nr_hw_queues = 1;
- set->timeout = NVME_ADMIN_TIMEOUT;
- } else {
- set = &ctrl->tag_set;
- memset(set, 0, sizeof(*set));
- set->ops = &nvme_tcp_ofld_mq_ops;
- set->queue_depth = nctrl->sqsize + 1;
- set->reserved_tags = NVMF_RESERVED_TAGS;
- set->numa_node = nctrl->numa_node;
- set->flags = BLK_MQ_F_SHOULD_MERGE;
- set->cmd_size = sizeof(struct nvme_tcp_ofld_req);
- set->driver_data = ctrl;
- set->nr_hw_queues = nctrl->queue_count - 1;
- set->timeout = NVME_IO_TIMEOUT;
- set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
- }
-
- rc = blk_mq_alloc_tag_set(set);
- if (rc)
- return ERR_PTR(rc);
-
- return set;
-}
-
-static void __nvme_tcp_ofld_stop_queue(struct nvme_tcp_ofld_queue *queue)
-{
- queue->dev->ops->drain_queue(queue);
-}
-
-static void nvme_tcp_ofld_stop_queue(struct nvme_ctrl *nctrl, int qid)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
- struct nvme_tcp_ofld_queue *queue = &ctrl->queues[qid];
-
- mutex_lock(&queue->queue_lock);
- if (test_and_clear_bit(NVME_TCP_OFLD_Q_LIVE, &queue->flags))
- __nvme_tcp_ofld_stop_queue(queue);
- mutex_unlock(&queue->queue_lock);
-}
-
-static void nvme_tcp_ofld_stop_io_queues(struct nvme_ctrl *ctrl)
-{
- int i;
-
- for (i = 1; i < ctrl->queue_count; i++)
- nvme_tcp_ofld_stop_queue(ctrl, i);
-}
-
-static void __nvme_tcp_ofld_free_queue(struct nvme_tcp_ofld_queue *queue)
-{
- queue->dev->ops->destroy_queue(queue);
-}
-
-static void nvme_tcp_ofld_free_queue(struct nvme_ctrl *nctrl, int qid)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
- struct nvme_tcp_ofld_queue *queue = &ctrl->queues[qid];
-
- if (test_and_clear_bit(NVME_TCP_OFLD_Q_ALLOCATED, &queue->flags)) {
- __nvme_tcp_ofld_free_queue(queue);
- mutex_destroy(&queue->queue_lock);
- }
-}
-
-static void
-nvme_tcp_ofld_free_io_queues(struct nvme_ctrl *nctrl)
-{
- int i;
-
- for (i = 1; i < nctrl->queue_count; i++)
- nvme_tcp_ofld_free_queue(nctrl, i);
-}
-
-static void nvme_tcp_ofld_destroy_io_queues(struct nvme_ctrl *nctrl, bool remove)
-{
- nvme_tcp_ofld_stop_io_queues(nctrl);
- if (remove) {
- blk_cleanup_queue(nctrl->connect_q);
- blk_mq_free_tag_set(nctrl->tagset);
- }
- nvme_tcp_ofld_free_io_queues(nctrl);
-}
-
-static void nvme_tcp_ofld_destroy_admin_queue(struct nvme_ctrl *nctrl, bool remove)
-{
- nvme_tcp_ofld_stop_queue(nctrl, 0);
- if (remove) {
- blk_cleanup_queue(nctrl->admin_q);
- blk_cleanup_queue(nctrl->fabrics_q);
- blk_mq_free_tag_set(nctrl->admin_tagset);
- }
- nvme_tcp_ofld_free_queue(nctrl, 0);
-}
-
-static int nvme_tcp_ofld_start_queue(struct nvme_ctrl *nctrl, int qid)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
- struct nvme_tcp_ofld_queue *queue = &ctrl->queues[qid];
- int rc;
-
- queue = &ctrl->queues[qid];
- if (qid) {
- queue->cmnd_capsule_len = nctrl->ioccsz * 16;
- rc = nvmf_connect_io_queue(nctrl, qid, false);
- } else {
- queue->cmnd_capsule_len = sizeof(struct nvme_command) + NVME_TCP_ADMIN_CCSZ;
- rc = nvmf_connect_admin_queue(nctrl);
- }
-
- if (!rc) {
- set_bit(NVME_TCP_OFLD_Q_LIVE, &queue->flags);
- } else {
- if (test_bit(NVME_TCP_OFLD_Q_ALLOCATED, &queue->flags))
- __nvme_tcp_ofld_stop_queue(queue);
- dev_err(nctrl->device,
- "failed to connect queue: %d ret=%d\n", qid, rc);
- }
-
- return rc;
-}
-
-static int nvme_tcp_ofld_configure_admin_queue(struct nvme_ctrl *nctrl,
- bool new)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
- struct nvme_tcp_ofld_queue *queue = &ctrl->queues[0];
- int rc;
-
- mutex_init(&queue->queue_lock);
-
- rc = ctrl->dev->ops->create_queue(queue, 0, NVME_AQ_DEPTH);
- if (rc)
- return rc;
-
- set_bit(NVME_TCP_OFLD_Q_ALLOCATED, &queue->flags);
- if (new) {
- nctrl->admin_tagset =
- nvme_tcp_ofld_alloc_tagset(nctrl, true);
- if (IS_ERR(nctrl->admin_tagset)) {
- rc = PTR_ERR(nctrl->admin_tagset);
- nctrl->admin_tagset = NULL;
- goto out_free_queue;
- }
-
- nctrl->fabrics_q = blk_mq_init_queue(nctrl->admin_tagset);
- if (IS_ERR(nctrl->fabrics_q)) {
- rc = PTR_ERR(nctrl->fabrics_q);
- nctrl->fabrics_q = NULL;
- goto out_free_tagset;
- }
-
- nctrl->admin_q = blk_mq_init_queue(nctrl->admin_tagset);
- if (IS_ERR(nctrl->admin_q)) {
- rc = PTR_ERR(nctrl->admin_q);
- nctrl->admin_q = NULL;
- goto out_cleanup_fabrics_q;
- }
- }
-
- rc = nvme_tcp_ofld_start_queue(nctrl, 0);
- if (rc)
- goto out_cleanup_queue;
-
- rc = nvme_enable_ctrl(nctrl);
- if (rc)
- goto out_stop_queue;
-
- blk_mq_unquiesce_queue(nctrl->admin_q);
-
- rc = nvme_init_ctrl_finish(nctrl);
- if (rc)
- goto out_quiesce_queue;
-
- return 0;
-
-out_quiesce_queue:
- blk_mq_quiesce_queue(nctrl->admin_q);
- blk_sync_queue(nctrl->admin_q);
-out_stop_queue:
- nvme_tcp_ofld_stop_queue(nctrl, 0);
- nvme_cancel_admin_tagset(nctrl);
-out_cleanup_queue:
- if (new)
- blk_cleanup_queue(nctrl->admin_q);
-out_cleanup_fabrics_q:
- if (new)
- blk_cleanup_queue(nctrl->fabrics_q);
-out_free_tagset:
- if (new)
- blk_mq_free_tag_set(nctrl->admin_tagset);
-out_free_queue:
- nvme_tcp_ofld_free_queue(nctrl, 0);
-
- return rc;
-}
-
-static unsigned int nvme_tcp_ofld_nr_io_queues(struct nvme_ctrl *nctrl)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
- struct nvme_tcp_ofld_dev *dev = ctrl->dev;
- u32 hw_vectors = dev->num_hw_vectors;
- u32 nr_write_queues, nr_poll_queues;
- u32 nr_io_queues, nr_total_queues;
-
- nr_io_queues = min3(nctrl->opts->nr_io_queues, num_online_cpus(),
- hw_vectors);
- nr_write_queues = min3(nctrl->opts->nr_write_queues, num_online_cpus(),
- hw_vectors);
- nr_poll_queues = min3(nctrl->opts->nr_poll_queues, num_online_cpus(),
- hw_vectors);
-
- nr_total_queues = nr_io_queues + nr_write_queues + nr_poll_queues;
-
- return nr_total_queues;
-}
-
-static void
-nvme_tcp_ofld_set_io_queues(struct nvme_ctrl *nctrl, unsigned int nr_io_queues)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
- struct nvmf_ctrl_options *opts = nctrl->opts;
-
- if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
- /*
- * separate read/write queues
- * hand out dedicated default queues only after we have
- * sufficient read queues.
- */
- ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
- nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
- ctrl->io_queues[HCTX_TYPE_DEFAULT] =
- min(opts->nr_write_queues, nr_io_queues);
- nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
- } else {
- /*
- * shared read/write queues
- * either no write queues were requested, or we don't have
- * sufficient queue count to have dedicated default queues.
- */
- ctrl->io_queues[HCTX_TYPE_DEFAULT] =
- min(opts->nr_io_queues, nr_io_queues);
- nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
- }
-
- if (opts->nr_poll_queues && nr_io_queues) {
- /* map dedicated poll queues only if we have queues left */
- ctrl->io_queues[HCTX_TYPE_POLL] =
- min(opts->nr_poll_queues, nr_io_queues);
- }
-}
-
-static int nvme_tcp_ofld_create_io_queues(struct nvme_ctrl *nctrl)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
- int i, rc;
-
- for (i = 1; i < nctrl->queue_count; i++) {
- mutex_init(&ctrl->queues[i].queue_lock);
-
- rc = ctrl->dev->ops->create_queue(&ctrl->queues[i],
- i, nctrl->sqsize + 1);
- if (rc)
- goto out_free_queues;
-
- set_bit(NVME_TCP_OFLD_Q_ALLOCATED, &ctrl->queues[i].flags);
- }
-
- return 0;
-
-out_free_queues:
- for (i--; i >= 1; i--)
- nvme_tcp_ofld_free_queue(nctrl, i);
-
- return rc;
-}
-
-static int nvme_tcp_ofld_alloc_io_queues(struct nvme_ctrl *nctrl)
-{
- unsigned int nr_io_queues;
- int rc;
-
- nr_io_queues = nvme_tcp_ofld_nr_io_queues(nctrl);
- rc = nvme_set_queue_count(nctrl, &nr_io_queues);
- if (rc)
- return rc;
-
- nctrl->queue_count = nr_io_queues + 1;
- if (nctrl->queue_count < 2) {
- dev_err(nctrl->device,
- "unable to set any I/O queues\n");
-
- return -ENOMEM;
- }
-
- dev_info(nctrl->device, "creating %d I/O queues.\n", nr_io_queues);
- nvme_tcp_ofld_set_io_queues(nctrl, nr_io_queues);
-
- return nvme_tcp_ofld_create_io_queues(nctrl);
-}
-
-static int nvme_tcp_ofld_start_io_queues(struct nvme_ctrl *nctrl)
-{
- int i, rc = 0;
-
- for (i = 1; i < nctrl->queue_count; i++) {
- rc = nvme_tcp_ofld_start_queue(nctrl, i);
- if (rc)
- goto out_stop_queues;
- }
-
- return 0;
-
-out_stop_queues:
- for (i--; i >= 1; i--)
- nvme_tcp_ofld_stop_queue(nctrl, i);
-
- return rc;
-}
-
-static int
-nvme_tcp_ofld_configure_io_queues(struct nvme_ctrl *nctrl, bool new)
-{
- int rc = nvme_tcp_ofld_alloc_io_queues(nctrl);
-
- if (rc)
- return rc;
-
- if (new) {
- nctrl->tagset = nvme_tcp_ofld_alloc_tagset(nctrl, false);
- if (IS_ERR(nctrl->tagset)) {
- rc = PTR_ERR(nctrl->tagset);
- nctrl->tagset = NULL;
- goto out_free_io_queues;
- }
-
- nctrl->connect_q = blk_mq_init_queue(nctrl->tagset);
- if (IS_ERR(nctrl->connect_q)) {
- rc = PTR_ERR(nctrl->connect_q);
- nctrl->connect_q = NULL;
- goto out_free_tag_set;
- }
- }
-
- rc = nvme_tcp_ofld_start_io_queues(nctrl);
- if (rc)
- goto out_cleanup_connect_q;
-
- if (!new) {
- nvme_start_queues(nctrl);
- if (!nvme_wait_freeze_timeout(nctrl, NVME_IO_TIMEOUT)) {
- /*
- * If we timed out waiting for freeze we are likely to
- * be stuck. Fail the controller initialization just
- * to be safe.
- */
- rc = -ENODEV;
- goto out_wait_freeze_timed_out;
- }
- blk_mq_update_nr_hw_queues(nctrl->tagset, nctrl->queue_count - 1);
- nvme_unfreeze(nctrl);
- }
-
- return 0;
-
-out_wait_freeze_timed_out:
- nvme_stop_queues(nctrl);
- nvme_sync_io_queues(nctrl);
- nvme_tcp_ofld_stop_io_queues(nctrl);
-out_cleanup_connect_q:
- nvme_cancel_tagset(nctrl);
- if (new)
- blk_cleanup_queue(nctrl->connect_q);
-out_free_tag_set:
- if (new)
- blk_mq_free_tag_set(nctrl->tagset);
-out_free_io_queues:
- nvme_tcp_ofld_free_io_queues(nctrl);
-
- return rc;
-}
-
-static void nvme_tcp_ofld_reconnect_or_remove(struct nvme_ctrl *nctrl)
-{
- /* If we are resetting/deleting then do nothing */
- if (nctrl->state != NVME_CTRL_CONNECTING) {
- WARN_ON_ONCE(nctrl->state == NVME_CTRL_NEW ||
- nctrl->state == NVME_CTRL_LIVE);
-
- return;
- }
-
- if (nvmf_should_reconnect(nctrl)) {
- dev_info(nctrl->device, "Reconnecting in %d seconds...\n",
- nctrl->opts->reconnect_delay);
- queue_delayed_work(nvme_wq,
- &to_tcp_ofld_ctrl(nctrl)->connect_work,
- nctrl->opts->reconnect_delay * HZ);
- } else {
- dev_info(nctrl->device, "Removing controller...\n");
- nvme_delete_ctrl(nctrl);
- }
-}
-
-static int
-nvme_tcp_ofld_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
- unsigned int hctx_idx)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = data;
-
- hctx->driver_data = &ctrl->queues[0];
-
- return 0;
-}
-
-static int nvme_tcp_ofld_setup_ctrl(struct nvme_ctrl *nctrl, bool new)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
- struct nvmf_ctrl_options *opts = nctrl->opts;
- int rc = 0;
-
- rc = ctrl->dev->ops->setup_ctrl(ctrl);
- if (rc)
- return rc;
-
- rc = nvme_tcp_ofld_configure_admin_queue(nctrl, new);
- if (rc)
- goto out_release_ctrl;
-
- if (nctrl->icdoff) {
- dev_err(nctrl->device, "icdoff is not supported!\n");
- rc = -EINVAL;
- goto destroy_admin;
- }
-
- if (!(nctrl->sgls & ((1 << 0) | (1 << 1)))) {
- dev_err(nctrl->device, "Mandatory sgls are not supported!\n");
- goto destroy_admin;
- }
-
- if (opts->queue_size > nctrl->sqsize + 1)
- dev_warn(nctrl->device,
- "queue_size %zu > ctrl sqsize %u, clamping down\n",
- opts->queue_size, nctrl->sqsize + 1);
-
- if (nctrl->sqsize + 1 > nctrl->maxcmd) {
- dev_warn(nctrl->device,
- "sqsize %u > ctrl maxcmd %u, clamping down\n",
- nctrl->sqsize + 1, nctrl->maxcmd);
- nctrl->sqsize = nctrl->maxcmd - 1;
- }
-
- if (nctrl->queue_count > 1) {
- rc = nvme_tcp_ofld_configure_io_queues(nctrl, new);
- if (rc)
- goto destroy_admin;
- }
-
- if (!nvme_change_ctrl_state(nctrl, NVME_CTRL_LIVE)) {
- /*
- * state change failure is ok if we started ctrl delete,
- * unless we're during creation of a new controller to
- * avoid races with teardown flow.
- */
- WARN_ON_ONCE(nctrl->state != NVME_CTRL_DELETING &&
- nctrl->state != NVME_CTRL_DELETING_NOIO);
- WARN_ON_ONCE(new);
- rc = -EINVAL;
- goto destroy_io;
- }
-
- nvme_start_ctrl(nctrl);
-
- return 0;
-
-destroy_io:
- if (nctrl->queue_count > 1) {
- nvme_stop_queues(nctrl);
- nvme_sync_io_queues(nctrl);
- nvme_tcp_ofld_stop_io_queues(nctrl);
- nvme_cancel_tagset(nctrl);
- nvme_tcp_ofld_destroy_io_queues(nctrl, new);
- }
-destroy_admin:
- blk_mq_quiesce_queue(nctrl->admin_q);
- blk_sync_queue(nctrl->admin_q);
- nvme_tcp_ofld_stop_queue(nctrl, 0);
- nvme_cancel_admin_tagset(nctrl);
- nvme_tcp_ofld_destroy_admin_queue(nctrl, new);
-out_release_ctrl:
- ctrl->dev->ops->release_ctrl(ctrl);
-
- return rc;
-}
-
-static int
-nvme_tcp_ofld_check_dev_opts(struct nvmf_ctrl_options *opts,
- struct nvme_tcp_ofld_ops *ofld_ops)
-{
- unsigned int nvme_tcp_ofld_opt_mask = NVMF_ALLOWED_OPTS |
- ofld_ops->allowed_opts | ofld_ops->required_opts;
- struct nvmf_ctrl_options dev_opts_mask;
-
- if (opts->mask & ~nvme_tcp_ofld_opt_mask) {
- pr_warn("One or more nvmf options missing from ofld drvr %s.\n",
- ofld_ops->name);
-
- dev_opts_mask.mask = nvme_tcp_ofld_opt_mask;
-
- return nvmf_check_required_opts(&dev_opts_mask, opts->mask);
- }
-
- return 0;
-}
-
-static void nvme_tcp_ofld_free_ctrl(struct nvme_ctrl *nctrl)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
- struct nvme_tcp_ofld_dev *dev = ctrl->dev;
-
- if (list_empty(&ctrl->list))
- goto free_ctrl;
-
- ctrl->dev->ops->release_ctrl(ctrl);
-
- mutex_lock(&nvme_tcp_ofld_ctrl_mutex);
- list_del(&ctrl->list);
- mutex_unlock(&nvme_tcp_ofld_ctrl_mutex);
-
- nvmf_free_options(nctrl->opts);
-free_ctrl:
- module_put(dev->ops->module);
- kfree(ctrl->queues);
- kfree(ctrl);
-}
-
-static void nvme_tcp_ofld_set_sg_null(struct nvme_command *c)
-{
- struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
-
- sg->addr = 0;
- sg->length = 0;
- sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | NVME_SGL_FMT_TRANSPORT_A;
-}
-
-inline void nvme_tcp_ofld_set_sg_inline(struct nvme_tcp_ofld_queue *queue,
- struct nvme_command *c, u32 data_len)
-{
- struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
-
- sg->addr = cpu_to_le64(queue->ctrl->nctrl.icdoff);
- sg->length = cpu_to_le32(data_len);
- sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
-}
-
-static void nvme_tcp_ofld_map_data(struct nvme_command *c, u32 data_len)
-{
- struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
-
- sg->addr = 0;
- sg->length = cpu_to_le32(data_len);
- sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | NVME_SGL_FMT_TRANSPORT_A;
-}
-
-static void nvme_tcp_ofld_submit_async_event(struct nvme_ctrl *arg)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(arg);
- struct nvme_tcp_ofld_queue *queue = &ctrl->queues[0];
- struct nvme_tcp_ofld_dev *dev = queue->dev;
- struct nvme_tcp_ofld_ops *ops = dev->ops;
-
- ctrl->async_req.nvme_cmd.common.opcode = nvme_admin_async_event;
- ctrl->async_req.nvme_cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
- ctrl->async_req.nvme_cmd.common.flags |= NVME_CMD_SGL_METABUF;
-
- nvme_tcp_ofld_set_sg_null(&ctrl->async_req.nvme_cmd);
-
- ctrl->async_req.async = true;
- ctrl->async_req.queue = queue;
- ctrl->async_req.done = nvme_tcp_ofld_async_req_done;
-
- ops->send_req(&ctrl->async_req);
-}
-
-static void
-nvme_tcp_ofld_teardown_admin_queue(struct nvme_ctrl *nctrl, bool remove)
-{
- blk_mq_quiesce_queue(nctrl->admin_q);
- blk_sync_queue(nctrl->admin_q);
-
- nvme_tcp_ofld_stop_queue(nctrl, 0);
- nvme_cancel_admin_tagset(nctrl);
-
- if (remove)
- blk_mq_unquiesce_queue(nctrl->admin_q);
-
- nvme_tcp_ofld_destroy_admin_queue(nctrl, remove);
-}
-
-static void
-nvme_tcp_ofld_teardown_io_queues(struct nvme_ctrl *nctrl, bool remove)
-{
- if (nctrl->queue_count <= 1)
- return;
-
- blk_mq_quiesce_queue(nctrl->admin_q);
- nvme_start_freeze(nctrl);
- nvme_stop_queues(nctrl);
- nvme_sync_io_queues(nctrl);
- nvme_tcp_ofld_stop_io_queues(nctrl);
- nvme_cancel_tagset(nctrl);
-
- if (remove)
- nvme_start_queues(nctrl);
-
- nvme_tcp_ofld_destroy_io_queues(nctrl, remove);
-}
-
-static void nvme_tcp_ofld_reconnect_ctrl_work(struct work_struct *work)
-{
- struct nvme_tcp_ofld_ctrl *ctrl =
- container_of(to_delayed_work(work),
- struct nvme_tcp_ofld_ctrl,
- connect_work);
- struct nvme_ctrl *nctrl = &ctrl->nctrl;
-
- ++nctrl->nr_reconnects;
-
- if (nvme_tcp_ofld_setup_ctrl(nctrl, false))
- goto requeue;
-
- dev_info(nctrl->device, "Successfully reconnected (%d attempt)\n",
- nctrl->nr_reconnects);
-
- nctrl->nr_reconnects = 0;
-
- return;
-
-requeue:
- dev_info(nctrl->device, "Failed reconnect attempt %d\n",
- nctrl->nr_reconnects);
- nvme_tcp_ofld_reconnect_or_remove(nctrl);
-}
-
-static void nvme_tcp_ofld_error_recovery_work(struct work_struct *work)
-{
- struct nvme_tcp_ofld_ctrl *ctrl =
- container_of(work, struct nvme_tcp_ofld_ctrl, err_work);
- struct nvme_ctrl *nctrl = &ctrl->nctrl;
-
- nvme_stop_keep_alive(nctrl);
- nvme_tcp_ofld_teardown_io_queues(nctrl, false);
- /* unquiesce to fail fast pending requests */
- nvme_start_queues(nctrl);
- nvme_tcp_ofld_teardown_admin_queue(nctrl, false);
- blk_mq_unquiesce_queue(nctrl->admin_q);
-
- if (!nvme_change_ctrl_state(nctrl, NVME_CTRL_CONNECTING)) {
- /* state change failure is ok if we started nctrl delete */
- WARN_ON_ONCE(nctrl->state != NVME_CTRL_DELETING &&
- nctrl->state != NVME_CTRL_DELETING_NOIO);
-
- return;
- }
-
- nvme_tcp_ofld_reconnect_or_remove(nctrl);
-}
-
-static void
-nvme_tcp_ofld_teardown_ctrl(struct nvme_ctrl *nctrl, bool shutdown)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = to_tcp_ofld_ctrl(nctrl);
-
- cancel_work_sync(&ctrl->err_work);
- cancel_delayed_work_sync(&ctrl->connect_work);
- nvme_tcp_ofld_teardown_io_queues(nctrl, shutdown);
- blk_mq_quiesce_queue(nctrl->admin_q);
- if (shutdown)
- nvme_shutdown_ctrl(nctrl);
- else
- nvme_disable_ctrl(nctrl);
- nvme_tcp_ofld_teardown_admin_queue(nctrl, shutdown);
-}
-
-static void nvme_tcp_ofld_delete_ctrl(struct nvme_ctrl *nctrl)
-{
- nvme_tcp_ofld_teardown_ctrl(nctrl, true);
-}
-
-static void nvme_tcp_ofld_reset_ctrl_work(struct work_struct *work)
-{
- struct nvme_ctrl *nctrl =
- container_of(work, struct nvme_ctrl, reset_work);
-
- nvme_stop_ctrl(nctrl);
- nvme_tcp_ofld_teardown_ctrl(nctrl, false);
-
- if (!nvme_change_ctrl_state(nctrl, NVME_CTRL_CONNECTING)) {
- /* state change failure is ok if we started ctrl delete */
- WARN_ON_ONCE(nctrl->state != NVME_CTRL_DELETING &&
- nctrl->state != NVME_CTRL_DELETING_NOIO);
-
- return;
- }
-
- if (nvme_tcp_ofld_setup_ctrl(nctrl, false))
- goto out_fail;
-
- return;
-
-out_fail:
- ++nctrl->nr_reconnects;
- nvme_tcp_ofld_reconnect_or_remove(nctrl);
-}
-
-static int
-nvme_tcp_ofld_init_request(struct blk_mq_tag_set *set,
- struct request *rq,
- unsigned int hctx_idx,
- unsigned int numa_node)
-{
- struct nvme_tcp_ofld_req *req = blk_mq_rq_to_pdu(rq);
- struct nvme_tcp_ofld_ctrl *ctrl = set->driver_data;
- int qid;
-
- qid = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
- req->queue = &ctrl->queues[qid];
- nvme_req(rq)->ctrl = &ctrl->nctrl;
- nvme_req(rq)->cmd = &req->nvme_cmd;
- req->done = nvme_tcp_ofld_req_done;
-
- return 0;
-}
-
-inline size_t nvme_tcp_ofld_inline_data_size(struct nvme_tcp_ofld_queue *queue)
-{
- return queue->cmnd_capsule_len - sizeof(struct nvme_command);
-}
-EXPORT_SYMBOL_GPL(nvme_tcp_ofld_inline_data_size);
-
-static blk_status_t
-nvme_tcp_ofld_queue_rq(struct blk_mq_hw_ctx *hctx,
- const struct blk_mq_queue_data *bd)
-{
- struct nvme_tcp_ofld_req *req = blk_mq_rq_to_pdu(bd->rq);
- struct nvme_tcp_ofld_queue *queue = hctx->driver_data;
- struct nvme_tcp_ofld_ctrl *ctrl = queue->ctrl;
- struct nvme_ns *ns = hctx->queue->queuedata;
- struct nvme_tcp_ofld_dev *dev = queue->dev;
- struct nvme_tcp_ofld_ops *ops = dev->ops;
- struct nvme_command *nvme_cmd;
- struct request *rq = bd->rq;
- bool queue_ready;
- u32 data_len;
- int rc;
-
- queue_ready = test_bit(NVME_TCP_OFLD_Q_LIVE, &queue->flags);
-
- req->async = false;
-
- if (!nvme_check_ready(&ctrl->nctrl, rq, queue_ready))
- return nvme_fail_nonready_command(&ctrl->nctrl, rq);
-
- rc = nvme_setup_cmd(ns, rq);
- if (unlikely(rc))
- return rc;
-
- blk_mq_start_request(rq);
-
- nvme_cmd = &req->nvme_cmd;
- nvme_cmd->common.flags |= NVME_CMD_SGL_METABUF;
-
- data_len = blk_rq_nr_phys_segments(rq) ? blk_rq_payload_bytes(rq) : 0;
- if (!data_len)
- nvme_tcp_ofld_set_sg_null(&req->nvme_cmd);
- else if ((rq_data_dir(rq) == WRITE) &&
- data_len <= nvme_tcp_ofld_inline_data_size(queue))
- nvme_tcp_ofld_set_sg_inline(queue, nvme_cmd, data_len);
- else
- nvme_tcp_ofld_map_data(nvme_cmd, data_len);
-
- rc = ops->send_req(req);
- if (unlikely(rc))
- return rc;
-
- return BLK_STS_OK;
-}
-
-static void
-nvme_tcp_ofld_exit_request(struct blk_mq_tag_set *set,
- struct request *rq, unsigned int hctx_idx)
-{
- /*
- * Nothing is allocated in nvme_tcp_ofld_init_request,
- * hence empty.
- */
-}
-
-static int
-nvme_tcp_ofld_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
- unsigned int hctx_idx)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = data;
-
- hctx->driver_data = &ctrl->queues[hctx_idx + 1];
-
- return 0;
-}
-
-static int nvme_tcp_ofld_map_queues(struct blk_mq_tag_set *set)
-{
- struct nvme_tcp_ofld_ctrl *ctrl = set->driver_data;
- struct nvmf_ctrl_options *opts = ctrl->nctrl.opts;
-
- if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
- /* separate read/write queues */
- set->map[HCTX_TYPE_DEFAULT].nr_queues =
- ctrl->io_queues[HCTX_TYPE_DEFAULT];
- set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
- set->map[HCTX_TYPE_READ].nr_queues =
- ctrl->io_queues[HCTX_TYPE_READ];
- set->map[HCTX_TYPE_READ].queue_offset =
- ctrl->io_queues[HCTX_TYPE_DEFAULT];
- } else {
- /* shared read/write queues */
- set->map[HCTX_TYPE_DEFAULT].nr_queues =
- ctrl->io_queues[HCTX_TYPE_DEFAULT];
- set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
- set->map[HCTX_TYPE_READ].nr_queues =
- ctrl->io_queues[HCTX_TYPE_DEFAULT];
- set->map[HCTX_TYPE_READ].queue_offset = 0;
- }
- blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
- blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
-
- if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
- /* map dedicated poll queues only if we have queues left */
- set->map[HCTX_TYPE_POLL].nr_queues =
- ctrl->io_queues[HCTX_TYPE_POLL];
- set->map[HCTX_TYPE_POLL].queue_offset =
- ctrl->io_queues[HCTX_TYPE_DEFAULT] +
- ctrl->io_queues[HCTX_TYPE_READ];
- blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
- }
-
- dev_info(ctrl->nctrl.device,
- "mapped %d/%d/%d default/read/poll queues.\n",
- ctrl->io_queues[HCTX_TYPE_DEFAULT],
- ctrl->io_queues[HCTX_TYPE_READ],
- ctrl->io_queues[HCTX_TYPE_POLL]);
-
- return 0;
-}
-
-static int nvme_tcp_ofld_poll(struct blk_mq_hw_ctx *hctx)
-{
- struct nvme_tcp_ofld_queue *queue = hctx->driver_data;
- struct nvme_tcp_ofld_dev *dev = queue->dev;
- struct nvme_tcp_ofld_ops *ops = dev->ops;
-
- return ops->poll_queue(queue);
-}
-
-static void nvme_tcp_ofld_complete_timed_out(struct request *rq)
-{
- struct nvme_tcp_ofld_req *req = blk_mq_rq_to_pdu(rq);
- struct nvme_ctrl *nctrl = &req->queue->ctrl->nctrl;
-
- nvme_tcp_ofld_stop_queue(nctrl, nvme_tcp_ofld_qid(req->queue));
- if (blk_mq_request_started(rq) && !blk_mq_request_completed(rq)) {
- nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD;
- blk_mq_complete_request(rq);
- }
-}
-
-static enum blk_eh_timer_return nvme_tcp_ofld_timeout(struct request *rq, bool reserved)
-{
- struct nvme_tcp_ofld_req *req = blk_mq_rq_to_pdu(rq);
- struct nvme_tcp_ofld_ctrl *ctrl = req->queue->ctrl;
-
- dev_warn(ctrl->nctrl.device,
- "queue %d: timeout request %#x type %d\n",
- nvme_tcp_ofld_qid(req->queue), rq->tag, req->nvme_cmd.common.opcode);
-
- if (ctrl->nctrl.state != NVME_CTRL_LIVE) {
- /*
- * If we are resetting, connecting or deleting we should
- * complete immediately because we may block controller
- * teardown or setup sequence
- * - ctrl disable/shutdown fabrics requests
- * - connect requests
- * - initialization admin requests
- * - I/O requests that entered after unquiescing and
- * the controller stopped responding
- *
- * All other requests should be cancelled by the error
- * recovery work, so it's fine that we fail it here.
- */
- nvme_tcp_ofld_complete_timed_out(rq);
-
- return BLK_EH_DONE;
- }
-
- nvme_tcp_ofld_error_recovery(&ctrl->nctrl);
-
- return BLK_EH_RESET_TIMER;
-}
-
-static struct blk_mq_ops nvme_tcp_ofld_mq_ops = {
- .queue_rq = nvme_tcp_ofld_queue_rq,
- .complete = nvme_complete_rq,
- .init_request = nvme_tcp_ofld_init_request,
- .exit_request = nvme_tcp_ofld_exit_request,
- .init_hctx = nvme_tcp_ofld_init_hctx,
- .timeout = nvme_tcp_ofld_timeout,
- .map_queues = nvme_tcp_ofld_map_queues,
- .poll = nvme_tcp_ofld_poll,
-};
-
-static struct blk_mq_ops nvme_tcp_ofld_admin_mq_ops = {
- .queue_rq = nvme_tcp_ofld_queue_rq,
- .complete = nvme_complete_rq,
- .init_request = nvme_tcp_ofld_init_request,
- .exit_request = nvme_tcp_ofld_exit_request,
- .init_hctx = nvme_tcp_ofld_init_admin_hctx,
- .timeout = nvme_tcp_ofld_timeout,
-};
-
-static const struct nvme_ctrl_ops nvme_tcp_ofld_ctrl_ops = {
- .name = "tcp_offload",
- .module = THIS_MODULE,
- .flags = NVME_F_FABRICS,
- .reg_read32 = nvmf_reg_read32,
- .reg_read64 = nvmf_reg_read64,
- .reg_write32 = nvmf_reg_write32,
- .free_ctrl = nvme_tcp_ofld_free_ctrl,
- .submit_async_event = nvme_tcp_ofld_submit_async_event,
- .delete_ctrl = nvme_tcp_ofld_delete_ctrl,
- .get_address = nvmf_get_address,
-};
-
-static bool
-nvme_tcp_ofld_existing_controller(struct nvmf_ctrl_options *opts)
-{
- struct nvme_tcp_ofld_ctrl *ctrl;
- bool found = false;
-
- mutex_lock(&nvme_tcp_ofld_ctrl_mutex);
- list_for_each_entry(ctrl, &nvme_tcp_ofld_ctrl_list, list) {
- found = nvmf_ip_options_match(&ctrl->nctrl, opts);
- if (found)
- break;
- }
- mutex_unlock(&nvme_tcp_ofld_ctrl_mutex);
-
- return found;
-}
-
-static struct nvme_ctrl *
-nvme_tcp_ofld_create_ctrl(struct device *ndev, struct nvmf_ctrl_options *opts)
-{
- struct nvme_tcp_ofld_queue *queue;
- struct nvme_tcp_ofld_ctrl *ctrl;
- struct nvme_tcp_ofld_dev *dev;
- struct nvme_ctrl *nctrl;
- int i, rc = 0;
-
- ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
- if (!ctrl)
- return ERR_PTR(-ENOMEM);
-
- INIT_LIST_HEAD(&ctrl->list);
- nctrl = &ctrl->nctrl;
- nctrl->opts = opts;
- nctrl->queue_count = opts->nr_io_queues + opts->nr_write_queues +
- opts->nr_poll_queues + 1;
- nctrl->sqsize = opts->queue_size - 1;
- nctrl->kato = opts->kato;
- INIT_DELAYED_WORK(&ctrl->connect_work,
- nvme_tcp_ofld_reconnect_ctrl_work);
- INIT_WORK(&ctrl->err_work, nvme_tcp_ofld_error_recovery_work);
- INIT_WORK(&nctrl->reset_work, nvme_tcp_ofld_reset_ctrl_work);
- if (!(opts->mask & NVMF_OPT_TRSVCID)) {
- opts->trsvcid =
- kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
- if (!opts->trsvcid) {
- rc = -ENOMEM;
- goto out_free_ctrl;
- }
- opts->mask |= NVMF_OPT_TRSVCID;
- }
-
- rc = inet_pton_with_scope(&init_net, AF_UNSPEC, opts->traddr,
- opts->trsvcid,
- &ctrl->conn_params.remote_ip_addr);
- if (rc) {
- pr_err("malformed address passed: %s:%s\n",
- opts->traddr, opts->trsvcid);
- goto out_free_ctrl;
- }
-
- if (opts->mask & NVMF_OPT_HOST_TRADDR) {
- rc = inet_pton_with_scope(&init_net, AF_UNSPEC,
- opts->host_traddr, NULL,
- &ctrl->conn_params.local_ip_addr);
- if (rc) {
- pr_err("malformed src address passed: %s\n",
- opts->host_traddr);
- goto out_free_ctrl;
- }
- }
-
- if (!opts->duplicate_connect &&
- nvme_tcp_ofld_existing_controller(opts)) {
- rc = -EALREADY;
- goto out_free_ctrl;
- }
-
- /* Find device that can reach the dest addr */
- dev = nvme_tcp_ofld_lookup_dev(ctrl);
- if (!dev) {
- pr_info("no device found for addr %s:%s.\n",
- opts->traddr, opts->trsvcid);
- rc = -EINVAL;
- goto out_free_ctrl;
- }
-
- /* Increase driver refcnt */
- if (!try_module_get(dev->ops->module)) {
- pr_err("try_module_get failed\n");
- dev = NULL;
- goto out_free_ctrl;
- }
-
- rc = nvme_tcp_ofld_check_dev_opts(opts, dev->ops);
- if (rc)
- goto out_module_put;
-
- ctrl->dev = dev;
-
- if (ctrl->dev->ops->max_hw_sectors)
- nctrl->max_hw_sectors = ctrl->dev->ops->max_hw_sectors;
- if (ctrl->dev->ops->max_segments)
- nctrl->max_segments = ctrl->dev->ops->max_segments;
-
- ctrl->queues = kcalloc(nctrl->queue_count,
- sizeof(struct nvme_tcp_ofld_queue),
- GFP_KERNEL);
- if (!ctrl->queues) {
- rc = -ENOMEM;
- goto out_module_put;
- }
-
- for (i = 0; i < nctrl->queue_count; ++i) {
- queue = &ctrl->queues[i];
- queue->ctrl = ctrl;
- queue->dev = dev;
- queue->report_err = nvme_tcp_ofld_report_queue_err;
- }
-
- rc = nvme_init_ctrl(nctrl, ndev, &nvme_tcp_ofld_ctrl_ops, 0);
- if (rc)
- goto out_free_queues;
-
- if (!nvme_change_ctrl_state(nctrl, NVME_CTRL_CONNECTING)) {
- WARN_ON_ONCE(1);
- rc = -EINTR;
- goto out_uninit_ctrl;
- }
-
- rc = nvme_tcp_ofld_setup_ctrl(nctrl, true);
- if (rc)
- goto out_uninit_ctrl;
-
- dev_info(nctrl->device, "new ctrl: NQN \"%s\", addr %pISp\n",
- opts->subsysnqn, &ctrl->conn_params.remote_ip_addr);
-
- mutex_lock(&nvme_tcp_ofld_ctrl_mutex);
- list_add_tail(&ctrl->list, &nvme_tcp_ofld_ctrl_list);
- mutex_unlock(&nvme_tcp_ofld_ctrl_mutex);
-
- return nctrl;
-
-out_uninit_ctrl:
- nvme_uninit_ctrl(nctrl);
- nvme_put_ctrl(nctrl);
-out_free_queues:
- kfree(ctrl->queues);
-out_module_put:
- module_put(dev->ops->module);
-out_free_ctrl:
- kfree(ctrl);
-
- return ERR_PTR(rc);
-}
-
-static struct nvmf_transport_ops nvme_tcp_ofld_transport = {
- .name = "tcp_offload",
- .module = THIS_MODULE,
- .required_opts = NVMF_OPT_TRADDR,
- .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_NR_WRITE_QUEUES |
- NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
- NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_HDR_DIGEST |
- NVMF_OPT_DATA_DIGEST | NVMF_OPT_NR_POLL_QUEUES |
- NVMF_OPT_TOS,
- .create_ctrl = nvme_tcp_ofld_create_ctrl,
-};
-
-static int __init nvme_tcp_ofld_init_module(void)
-{
- nvmf_register_transport(&nvme_tcp_ofld_transport);
-
- return 0;
-}
-
-static void __exit nvme_tcp_ofld_cleanup_module(void)
-{
- struct nvme_tcp_ofld_ctrl *ctrl;
-
- nvmf_unregister_transport(&nvme_tcp_ofld_transport);
-
- mutex_lock(&nvme_tcp_ofld_ctrl_mutex);
- list_for_each_entry(ctrl, &nvme_tcp_ofld_ctrl_list, list)
- nvme_delete_ctrl(&ctrl->nctrl);
- mutex_unlock(&nvme_tcp_ofld_ctrl_mutex);
- flush_workqueue(nvme_delete_wq);
-}
-
-module_init(nvme_tcp_ofld_init_module);
-module_exit(nvme_tcp_ofld_cleanup_module);
-MODULE_LICENSE("GPL v2");
deleted file mode 100644
@@ -1,206 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright 2021 Marvell. All rights reserved.
- */
-
-/* Linux includes */
-#include <linux/dma-mapping.h>
-#include <linux/scatterlist.h>
-#include <linux/types.h>
-#include <linux/nvme-tcp.h>
-
-/* Driver includes */
-#include "nvme.h"
-#include "fabrics.h"
-
-/* Forward declarations */
-struct nvme_tcp_ofld_ops;
-
-/* Representation of a vendor-specific device. This is the struct used to
- * register to the offload layer by the vendor-specific driver during its probe
- * function.
- * Allocated by vendor-specific driver.
- */
-struct nvme_tcp_ofld_dev {
- struct list_head entry;
- struct net_device *ndev;
- struct nvme_tcp_ofld_ops *ops;
-
- /* Vendor specific driver context */
- int num_hw_vectors;
-};
-
-/* Per IO struct holding the nvme_request and command
- * Allocated by blk-mq.
- */
-struct nvme_tcp_ofld_req {
- struct nvme_request req;
- struct nvme_command nvme_cmd;
- struct list_head queue_entry;
- struct nvme_tcp_ofld_queue *queue;
-
- /* Vendor specific driver context */
- void *private_data;
-
- /* async flag is used to distinguish between async and IO flow
- * in common send_req() of nvme_tcp_ofld_ops.
- */
- bool async;
-
- void (*done)(struct nvme_tcp_ofld_req *req,
- union nvme_result *result,
- __le16 status);
-};
-
-enum nvme_tcp_ofld_queue_flags {
- NVME_TCP_OFLD_Q_ALLOCATED = 0,
- NVME_TCP_OFLD_Q_LIVE = 1,
-};
-
-/* Allocated by nvme_tcp_ofld */
-struct nvme_tcp_ofld_queue {
- /* Offload device associated to this queue */
- struct nvme_tcp_ofld_dev *dev;
- struct nvme_tcp_ofld_ctrl *ctrl;
- unsigned long flags;
- size_t cmnd_capsule_len;
-
- /* mutex used during stop_queue */
- struct mutex queue_lock;
-
- u8 hdr_digest;
- u8 data_digest;
- u8 tos;
-
- /* Vendor specific driver context */
- void *private_data;
-
- /* Error callback function */
- int (*report_err)(struct nvme_tcp_ofld_queue *queue);
-};
-
-/* Connectivity (routing) params used for establishing a connection */
-struct nvme_tcp_ofld_ctrl_con_params {
- struct sockaddr_storage remote_ip_addr;
-
- /* If NVMF_OPT_HOST_TRADDR is provided it will be set in local_ip_addr
- * in nvme_tcp_ofld_create_ctrl().
- * If NVMF_OPT_HOST_TRADDR is not provided the local_ip_addr will be
- * initialized by claim_dev().
- */
- struct sockaddr_storage local_ip_addr;
-};
-
-/* Allocated by nvme_tcp_ofld */
-struct nvme_tcp_ofld_ctrl {
- struct nvme_ctrl nctrl;
- struct list_head list;
- struct nvme_tcp_ofld_dev *dev;
-
- /* admin and IO queues */
- struct blk_mq_tag_set tag_set;
- struct blk_mq_tag_set admin_tag_set;
- struct nvme_tcp_ofld_queue *queues;
-
- struct work_struct err_work;
- struct delayed_work connect_work;
-
- /*
- * Each entry in the array indicates the number of queues of
- * corresponding type.
- */
- u32 io_queues[HCTX_MAX_TYPES];
-
- /* Connectivity params */
- struct nvme_tcp_ofld_ctrl_con_params conn_params;
-
- struct nvme_tcp_ofld_req async_req;
-
- /* Vendor specific driver context */
- void *private_data;
-};
-
-struct nvme_tcp_ofld_ops {
- const char *name;
- struct module *module;
-
- /* For vendor-specific driver to report what opts it supports.
- * It could be different than the ULP supported opts due to hardware
- * limitations. Also it could be different among different vendor
- * drivers.
- */
- int required_opts; /* bitmap using enum nvmf_parsing_opts */
- int allowed_opts; /* bitmap using enum nvmf_parsing_opts */
-
- /* For vendor-specific max num of segments and IO sizes */
- u32 max_hw_sectors;
- u32 max_segments;
-
- /**
- * claim_dev: Return True if addr is reachable via offload device.
- * @dev: The offload device to check.
- * @ctrl: The offload ctrl have the conn_params field. The
- * conn_params is to be filled with routing params by the lower
- * driver.
- */
- int (*claim_dev)(struct nvme_tcp_ofld_dev *dev,
- struct nvme_tcp_ofld_ctrl *ctrl);
-
- /**
- * setup_ctrl: Setup device specific controller structures.
- * @ctrl: The offload ctrl.
- */
- int (*setup_ctrl)(struct nvme_tcp_ofld_ctrl *ctrl);
-
- /**
- * release_ctrl: Release/Free device specific controller structures.
- * @ctrl: The offload ctrl.
- */
- int (*release_ctrl)(struct nvme_tcp_ofld_ctrl *ctrl);
-
- /**
- * create_queue: Create offload queue and establish TCP + NVMeTCP
- * (icreq+icresp) connection. Return true on successful connection.
- * Based on nvme_tcp_alloc_queue.
- * @queue: The queue itself - used as input and output.
- * @qid: The queue ID associated with the requested queue.
- * @q_size: The queue depth.
- */
- int (*create_queue)(struct nvme_tcp_ofld_queue *queue, int qid,
- size_t queue_size);
-
- /**
- * drain_queue: Drain a given queue - blocking function call.
- * Return from this function ensures that no additional
- * completions will arrive on this queue and that the HW will
- * not access host memory.
- * @queue: The queue to drain.
- */
- void (*drain_queue)(struct nvme_tcp_ofld_queue *queue);
-
- /**
- * destroy_queue: Close the TCP + NVMeTCP connection of a given queue
- * and make sure its no longer active (no completions will arrive on the
- * queue).
- * @queue: The queue to destroy.
- */
- void (*destroy_queue)(struct nvme_tcp_ofld_queue *queue);
-
- /**
- * poll_queue: Poll a given queue for completions.
- * @queue: The queue to poll.
- */
- int (*poll_queue)(struct nvme_tcp_ofld_queue *queue);
-
- /**
- * send_req: Dispatch a request. Returns the execution status.
- * @req: Ptr to request to be sent.
- */
- int (*send_req)(struct nvme_tcp_ofld_req *req);
-};
-
-/* Exported functions for lower vendor specific offload drivers */
-int nvme_tcp_ofld_register_dev(struct nvme_tcp_ofld_dev *dev);
-void nvme_tcp_ofld_unregister_dev(struct nvme_tcp_ofld_dev *dev);
-void nvme_tcp_ofld_error_recovery(struct nvme_ctrl *nctrl);
-inline size_t nvme_tcp_ofld_inline_data_size(struct nvme_tcp_ofld_queue *queue);
This reverts commits: - 762411542050dbe27c7c96f13c57f93da5d9b89a nvme: NVME_TCP_OFFLOAD should not default to m - 5ff5622ea1f16d535f1be4e478e712ef48fe183b: Merge branch 'NVMeTCP-Offload-ULP' As requested on the mailing-list: https://lore.kernel.org/netdev/SJ0PR18MB3882C20793EA35A3E8DAE300CC379@SJ0PR18MB3882.namprd18.prod.outlook.com/ This patch will revert the nvme-tcp-offload ULP from net-next. The nvme-tcp-offload ULP series will continue to be considered only on linux-nvme@lists.infradead.org. Signed-off-by: Prabhakar Kushwaha <pkushwaha@marvell.com> Signed-off-by: Michal Kalderon <mkalderon@marvell.com> Signed-off-by: Ariel Elior <aelior@marvell.com> Signed-off-by: Shai Malin <smalin@marvell.com> --- MAINTAINERS | 8 - drivers/nvme/host/Kconfig | 16 - drivers/nvme/host/Makefile | 3 - drivers/nvme/host/fabrics.c | 12 +- drivers/nvme/host/fabrics.h | 9 - drivers/nvme/host/tcp-offload.c | 1318 ------------------------------- drivers/nvme/host/tcp-offload.h | 206 ----- 7 files changed, 9 insertions(+), 1563 deletions(-) delete mode 100644 drivers/nvme/host/tcp-offload.c delete mode 100644 drivers/nvme/host/tcp-offload.h