drivers/nvme/target/loop.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe over Fabrics loopback device.
   4  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
   5  */
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7 #include <linux/scatterlist.h>
   8 #include <linux/blk-mq.h>
   9 #include <linux/nvme.h>
  10 #include <linux/module.h>
  11 #include <linux/parser.h>
  12 #include "nvmet.h"
  13 #include "../host/nvme.h"
  14 #include "../host/fabrics.h"
  15
  16 #define NVME_LOOP_MAX_SEGMENTS          256
  17
  18 struct nvme_loop_iod {
  19         struct nvme_request     nvme_req;
  20         struct nvme_command     cmd;
  21         struct nvme_completion  cqe;
  22         struct nvmet_req        req;
  23         struct nvme_loop_queue  *queue;
  24         struct work_struct      work;
  25         struct sg_table         sg_table;
  26         struct scatterlist      first_sgl[];
  27 };
  28
  29 struct nvme_loop_ctrl {
  30         struct nvme_loop_queue  *queues;
  31
  32         struct blk_mq_tag_set   admin_tag_set;
  33
  34         struct list_head        list;
  35         struct blk_mq_tag_set   tag_set;
  36         struct nvme_loop_iod    async_event_iod;
  37         struct nvme_ctrl        ctrl;
  38
  39         struct nvmet_ctrl       *target_ctrl;
  40         struct nvmet_port       *port;
  41 };
  42
  43 static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl)
  44 {
  45         return container_of(ctrl, struct nvme_loop_ctrl, ctrl);
  46 }
  47
  48 enum nvme_loop_queue_flags {
  49         NVME_LOOP_Q_LIVE        = 0,
  50 };
  51
  52 struct nvme_loop_queue {
  53         struct nvmet_cq         nvme_cq;
  54         struct nvmet_sq         nvme_sq;
  55         struct nvme_loop_ctrl   *ctrl;
  56         unsigned long           flags;
  57 };
  58
  59 static LIST_HEAD(nvme_loop_ports);
  60 static DEFINE_MUTEX(nvme_loop_ports_mutex);
  61
  62 static LIST_HEAD(nvme_loop_ctrl_list);
  63 static DEFINE_MUTEX(nvme_loop_ctrl_mutex);
  64
  65 static void nvme_loop_queue_response(struct nvmet_req *nvme_req);
  66 static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl);
  67
  68 static const struct nvmet_fabrics_ops nvme_loop_ops;
  69
  70 static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue)
  71 {
  72         return queue - queue->ctrl->queues;
  73 }
  74
  75 static void nvme_loop_complete_rq(struct request *req)
  76 {
  77         struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
  78
  79         nvme_cleanup_cmd(req);
  80         sg_free_table_chained(&iod->sg_table, SG_CHUNK_SIZE);
  81         nvme_complete_rq(req);
  82 }
  83
  84 static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue)
  85 {
  86         u32 queue_idx = nvme_loop_queue_idx(queue);
  87
  88         if (queue_idx == 0)
  89                 return queue->ctrl->admin_tag_set.tags[queue_idx];
  90         return queue->ctrl->tag_set.tags[queue_idx - 1];
  91 }
  92
  93 static void nvme_loop_queue_response(struct nvmet_req *req)
  94 {
  95         struct nvme_loop_queue *queue =
  96                 container_of(req->sq, struct nvme_loop_queue, nvme_sq);
  97         struct nvme_completion *cqe = req->cqe;
  98
  99         /*
 100          * AEN requests are special as they don't time out and can
 101          * survive any kind of queue freeze and often don't respond to
 102          * aborts.  We don't even bother to allocate a struct request
 103          * for them but rather special case them here.
 104          */
 105         if (unlikely(nvme_loop_queue_idx(queue) == 0 &&
 106                         cqe->command_id >= NVME_AQ_BLK_MQ_DEPTH)) {
 107                 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
 108                                 &cqe->result);
 109         } else {
 110                 struct request *rq;
 111
 112                 rq = blk_mq_tag_to_rq(nvme_loop_tagset(queue), cqe->command_id);
 113                 if (!rq) {
 114                         dev_err(queue->ctrl->ctrl.device,
 115                                 "tag 0x%x on queue %d not found\n",
 116                                 cqe->command_id, nvme_loop_queue_idx(queue));
 117                         return;
 118                 }
 119
 120                 nvme_end_request(rq, cqe->status, cqe->result);
 121         }
 122 }
 123
 124 static void nvme_loop_execute_work(struct work_struct *work)
 125 {
 126         struct nvme_loop_iod *iod =
 127                 container_of(work, struct nvme_loop_iod, work);
 128
 129         nvmet_req_execute(&iod->req);
 130 }
 131
 132 static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
 133                 const struct blk_mq_queue_data *bd)
 134 {
 135         struct nvme_ns *ns = hctx->queue->queuedata;
 136         struct nvme_loop_queue *queue = hctx->driver_data;
 137         struct request *req = bd->rq;
 138         struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
 139         bool queue_ready = test_bit(NVME_LOOP_Q_LIVE, &queue->flags);
 140         blk_status_t ret;
 141
 142         if (!nvmf_check_ready(&queue->ctrl->ctrl, req, queue_ready))
 143                 return nvmf_fail_nonready_command(&queue->ctrl->ctrl, req);
 144
 145         ret = nvme_setup_cmd(ns, req, &iod->cmd);
 146         if (ret)
 147                 return ret;
 148
 149         blk_mq_start_request(req);
 150         iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
 151         iod->req.port = queue->ctrl->port;
 152         if (!nvmet_req_init(&iod->req, &queue->nvme_cq,
 153                         &queue->nvme_sq, &nvme_loop_ops))
 154                 return BLK_STS_OK;
 155
 156         if (blk_rq_nr_phys_segments(req)) {
 157                 iod->sg_table.sgl = iod->first_sgl;
 158                 if (sg_alloc_table_chained(&iod->sg_table,
 159                                 blk_rq_nr_phys_segments(req),
 160                                 iod->sg_table.sgl, SG_CHUNK_SIZE))
 161                         return BLK_STS_RESOURCE;
 162
 163                 iod->req.sg = iod->sg_table.sgl;
 164                 iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl);
 165                 iod->req.transfer_len = blk_rq_payload_bytes(req);
 166         }
 167
 168         schedule_work(&iod->work);
 169         return BLK_STS_OK;
 170 }
 171
 172 static void nvme_loop_submit_async_event(struct nvme_ctrl *arg)
 173 {
 174         struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg);
 175         struct nvme_loop_queue *queue = &ctrl->queues[0];
 176         struct nvme_loop_iod *iod = &ctrl->async_event_iod;
 177
 178         memset(&iod->cmd, 0, sizeof(iod->cmd));
 179         iod->cmd.common.opcode = nvme_admin_async_event;
 180         iod->cmd.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
 181         iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
 182
 183         if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq,
 184                         &nvme_loop_ops)) {
 185                 dev_err(ctrl->ctrl.device, "failed async event work\n");
 186                 return;
 187         }
 188
 189         schedule_work(&iod->work);
 190 }
 191
 192 static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,
 193                 struct nvme_loop_iod *iod, unsigned int queue_idx)
 194 {
 195         iod->req.cmd = &iod->cmd;
 196         iod->req.cqe = &iod->cqe;
 197         iod->queue = &ctrl->queues[queue_idx];
 198         INIT_WORK(&iod->work, nvme_loop_execute_work);
 199         return 0;
 200 }
 201
 202 static int nvme_loop_init_request(struct blk_mq_tag_set *set,
 203                 struct request *req, unsigned int hctx_idx,
 204                 unsigned int numa_node)
 205 {
 206         struct nvme_loop_ctrl *ctrl = set->driver_data;
 207
 208         nvme_req(req)->ctrl = &ctrl->ctrl;
 209         return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req),
 210                         (set == &ctrl->tag_set) ? hctx_idx + 1 : 0);
 211 }
 212
 213 static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 214                 unsigned int hctx_idx)
 215 {
 216         struct nvme_loop_ctrl *ctrl = data;
 217         struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];
 218
 219         BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
 220
 221         hctx->driver_data = queue;
 222         return 0;
 223 }
 224
 225 static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 226                 unsigned int hctx_idx)
 227 {
 228         struct nvme_loop_ctrl *ctrl = data;
 229         struct nvme_loop_queue *queue = &ctrl->queues[0];
 230
 231         BUG_ON(hctx_idx != 0);
 232
 233         hctx->driver_data = queue;
 234         return 0;
 235 }
 236
 237 static const struct blk_mq_ops nvme_loop_mq_ops = {
 238         .queue_rq       = nvme_loop_queue_rq,
 239         .complete       = nvme_loop_complete_rq,
 240         .init_request   = nvme_loop_init_request,
 241         .init_hctx      = nvme_loop_init_hctx,
 242 };
 243
 244 static const struct blk_mq_ops nvme_loop_admin_mq_ops = {
 245         .queue_rq       = nvme_loop_queue_rq,
 246         .complete       = nvme_loop_complete_rq,
 247         .init_request   = nvme_loop_init_request,
 248         .init_hctx      = nvme_loop_init_admin_hctx,
 249 };
 250
 251 static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)
 252 {
 253         clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
 254         nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
 255         blk_cleanup_queue(ctrl->ctrl.admin_q);
 256         blk_mq_free_tag_set(&ctrl->admin_tag_set);
 257 }
 258
 259 static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl)
 260 {
 261         struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);
 262
 263         if (list_empty(&ctrl->list))
 264                 goto free_ctrl;
 265
 266         mutex_lock(&nvme_loop_ctrl_mutex);
 267         list_del(&ctrl->list);
 268         mutex_unlock(&nvme_loop_ctrl_mutex);
 269
 270         if (nctrl->tagset) {
 271                 blk_cleanup_queue(ctrl->ctrl.connect_q);
 272                 blk_mq_free_tag_set(&ctrl->tag_set);
 273         }
 274         kfree(ctrl->queues);
 275         nvmf_free_options(nctrl->opts);
 276 free_ctrl:
 277         kfree(ctrl);
 278 }
 279
 280 static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)
 281 {
 282         int i;
 283
 284         for (i = 1; i < ctrl->ctrl.queue_count; i++) {
 285                 clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);
 286                 nvmet_sq_destroy(&ctrl->queues[i].nvme_sq);
 287         }
 288 }
 289
 290 static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl)
 291 {
 292         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
 293         unsigned int nr_io_queues;
 294         int ret, i;
 295
 296         nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
 297         ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
 298         if (ret || !nr_io_queues)
 299                 return ret;
 300
 301         dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);
 302
 303         for (i = 1; i <= nr_io_queues; i++) {
 304                 ctrl->queues[i].ctrl = ctrl;
 305                 ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq);
 306                 if (ret)
 307                         goto out_destroy_queues;
 308
 309                 ctrl->ctrl.queue_count++;
 310         }
 311
 312         return 0;
 313
 314 out_destroy_queues:
 315         nvme_loop_destroy_io_queues(ctrl);
 316         return ret;
 317 }
 318
 319 static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl)
 320 {
 321         int i, ret;
 322
 323         for (i = 1; i < ctrl->ctrl.queue_count; i++) {
 324                 ret = nvmf_connect_io_queue(&ctrl->ctrl, i, false);
 325                 if (ret)
 326                         return ret;
 327                 set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[i].flags);
 328         }
 329
 330         return 0;
 331 }
 332
 333 static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
 334 {
 335         int error;
 336
 337         memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
 338         ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops;
 339         ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
 340         ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */
 341         ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;
 342         ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) +
 343                 SG_CHUNK_SIZE * sizeof(struct scatterlist);
 344         ctrl->admin_tag_set.driver_data = ctrl;
 345         ctrl->admin_tag_set.nr_hw_queues = 1;
 346         ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;
 347         ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED;
 348
 349         ctrl->queues[0].ctrl = ctrl;
 350         error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);
 351         if (error)
 352                 return error;
 353         ctrl->ctrl.queue_count = 1;
 354
 355         error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
 356         if (error)
 357                 goto out_free_sq;
 358         ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set;
 359
 360         ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
 361         if (IS_ERR(ctrl->ctrl.admin_q)) {
 362                 error = PTR_ERR(ctrl->ctrl.admin_q);
 363                 goto out_free_tagset;
 364         }
 365
 366         error = nvmf_connect_admin_queue(&ctrl->ctrl);
 367         if (error)
 368                 goto out_cleanup_queue;
 369
 370         set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
 371
 372         error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->ctrl.cap);
 373         if (error) {
 374                 dev_err(ctrl->ctrl.device,
 375                         "prop_get NVME_REG_CAP failed\n");
 376                 goto out_cleanup_queue;
 377         }
 378
 379         ctrl->ctrl.sqsize =
 380                 min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize);
 381
 382         error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
 383         if (error)
 384                 goto out_cleanup_queue;
 385
 386         ctrl->ctrl.max_hw_sectors =
 387                 (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9);
 388
 389         error = nvme_init_identify(&ctrl->ctrl);
 390         if (error)
 391                 goto out_cleanup_queue;
 392
 393         return 0;
 394
 395 out_cleanup_queue:
 396         blk_cleanup_queue(ctrl->ctrl.admin_q);
 397 out_free_tagset:
 398         blk_mq_free_tag_set(&ctrl->admin_tag_set);
 399 out_free_sq:
 400         nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
 401         return error;
 402 }
 403
 404 static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
 405 {
 406         if (ctrl->ctrl.queue_count > 1) {
 407                 nvme_stop_queues(&ctrl->ctrl);
 408                 blk_mq_tagset_busy_iter(&ctrl->tag_set,
 409                                         nvme_cancel_request, &ctrl->ctrl);
 410                 nvme_loop_destroy_io_queues(ctrl);
 411         }
 412
 413         if (ctrl->ctrl.state == NVME_CTRL_LIVE)
 414                 nvme_shutdown_ctrl(&ctrl->ctrl);
 415
 416         blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
 417         blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
 418                                 nvme_cancel_request, &ctrl->ctrl);
 419         blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
 420         nvme_loop_destroy_admin_queue(ctrl);
 421 }
 422
 423 static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl)
 424 {
 425         nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl));
 426 }
 427
 428 static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl)
 429 {
 430         struct nvme_loop_ctrl *ctrl;
 431
 432         mutex_lock(&nvme_loop_ctrl_mutex);
 433         list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) {
 434                 if (ctrl->ctrl.cntlid == nctrl->cntlid)
 435                         nvme_delete_ctrl(&ctrl->ctrl);
 436         }
 437         mutex_unlock(&nvme_loop_ctrl_mutex);
 438 }
 439
 440 static void nvme_loop_reset_ctrl_work(struct work_struct *work)
 441 {
 442         struct nvme_loop_ctrl *ctrl =
 443                 container_of(work, struct nvme_loop_ctrl, ctrl.reset_work);
 444         bool changed;
 445         int ret;
 446
 447         nvme_stop_ctrl(&ctrl->ctrl);
 448         nvme_loop_shutdown_ctrl(ctrl);
 449
 450         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
 451                 /* state change failure should never happen */
 452                 WARN_ON_ONCE(1);
 453                 return;
 454         }
 455
 456         ret = nvme_loop_configure_admin_queue(ctrl);
 457         if (ret)
 458                 goto out_disable;
 459
 460         ret = nvme_loop_init_io_queues(ctrl);
 461         if (ret)
 462                 goto out_destroy_admin;
 463
 464         ret = nvme_loop_connect_io_queues(ctrl);
 465         if (ret)
 466                 goto out_destroy_io;
 467
 468         blk_mq_update_nr_hw_queues(&ctrl->tag_set,
 469                         ctrl->ctrl.queue_count - 1);
 470
 471         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
 472         WARN_ON_ONCE(!changed);
 473
 474         nvme_start_ctrl(&ctrl->ctrl);
 475
 476         return;
 477
 478 out_destroy_io:
 479         nvme_loop_destroy_io_queues(ctrl);
 480 out_destroy_admin:
 481         nvme_loop_destroy_admin_queue(ctrl);
 482 out_disable:
 483         dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
 484         nvme_uninit_ctrl(&ctrl->ctrl);
 485         nvme_put_ctrl(&ctrl->ctrl);
 486 }
 487
 488 static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = {
 489         .name                   = "loop",
 490         .module                 = THIS_MODULE,
 491         .flags                  = NVME_F_FABRICS,
 492         .reg_read32             = nvmf_reg_read32,
 493         .reg_read64             = nvmf_reg_read64,
 494         .reg_write32            = nvmf_reg_write32,
 495         .free_ctrl              = nvme_loop_free_ctrl,
 496         .submit_async_event     = nvme_loop_submit_async_event,
 497         .delete_ctrl            = nvme_loop_delete_ctrl_host,
 498         .get_address            = nvmf_get_address,
 499 };
 500
 501 static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
 502 {
 503         int ret;
 504
 505         ret = nvme_loop_init_io_queues(ctrl);
 506         if (ret)
 507                 return ret;
 508
 509         memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
 510         ctrl->tag_set.ops = &nvme_loop_mq_ops;
 511         ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
 512         ctrl->tag_set.reserved_tags = 1; /* fabric connect */
 513         ctrl->tag_set.numa_node = NUMA_NO_NODE;
 514         ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
 515         ctrl->tag_set.cmd_size = sizeof(struct nvme_loop_iod) +
 516                 SG_CHUNK_SIZE * sizeof(struct scatterlist);
 517         ctrl->tag_set.driver_data = ctrl;
 518         ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1;
 519         ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
 520         ctrl->ctrl.tagset = &ctrl->tag_set;
 521
 522         ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
 523         if (ret)
 524                 goto out_destroy_queues;
 525
 526         ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
 527         if (IS_ERR(ctrl->ctrl.connect_q)) {
 528                 ret = PTR_ERR(ctrl->ctrl.connect_q);
 529                 goto out_free_tagset;
 530         }
 531
 532         ret = nvme_loop_connect_io_queues(ctrl);
 533         if (ret)
 534                 goto out_cleanup_connect_q;
 535
 536         return 0;
 537
 538 out_cleanup_connect_q:
 539         blk_cleanup_queue(ctrl->ctrl.connect_q);
 540 out_free_tagset:
 541         blk_mq_free_tag_set(&ctrl->tag_set);
 542 out_destroy_queues:
 543         nvme_loop_destroy_io_queues(ctrl);
 544         return ret;
 545 }
 546
 547 static struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)
 548 {
 549         struct nvmet_port *p, *found = NULL;
 550
 551         mutex_lock(&nvme_loop_ports_mutex);
 552         list_for_each_entry(p, &nvme_loop_ports, entry) {
 553                 /* if no transport address is specified use the first port */
 554                 if ((ctrl->opts->mask & NVMF_OPT_TRADDR) &&
 555                     strcmp(ctrl->opts->traddr, p->disc_addr.traddr))
 556                         continue;
 557                 found = p;
 558                 break;
 559         }
 560         mutex_unlock(&nvme_loop_ports_mutex);
 561         return found;
 562 }
 563
 564 static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev,
 565                 struct nvmf_ctrl_options *opts)
 566 {
 567         struct nvme_loop_ctrl *ctrl;
 568         bool changed;
 569         int ret;
 570
 571         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
 572         if (!ctrl)
 573                 return ERR_PTR(-ENOMEM);
 574         ctrl->ctrl.opts = opts;
 575         INIT_LIST_HEAD(&ctrl->list);
 576
 577         INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work);
 578
 579         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops,
 580                                 0 /* no quirks, we're perfect! */);
 581         if (ret)
 582                 goto out_put_ctrl;
 583
 584         ret = -ENOMEM;
 585
 586         ctrl->ctrl.sqsize = opts->queue_size - 1;
 587         ctrl->ctrl.kato = opts->kato;
 588         ctrl->port = nvme_loop_find_port(&ctrl->ctrl);
 589
 590         ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues),
 591                         GFP_KERNEL);
 592         if (!ctrl->queues)
 593                 goto out_uninit_ctrl;
 594
 595         ret = nvme_loop_configure_admin_queue(ctrl);
 596         if (ret)
 597                 goto out_free_queues;
 598
 599         if (opts->queue_size > ctrl->ctrl.maxcmd) {
 600                 /* warn if maxcmd is lower than queue_size */
 601                 dev_warn(ctrl->ctrl.device,
 602                         "queue_size %zu > ctrl maxcmd %u, clamping down\n",
 603                         opts->queue_size, ctrl->ctrl.maxcmd);
 604                 opts->queue_size = ctrl->ctrl.maxcmd;
 605         }
 606
 607         if (opts->nr_io_queues) {
 608                 ret = nvme_loop_create_io_queues(ctrl);
 609                 if (ret)
 610                         goto out_remove_admin_queue;
 611         }
 612
 613         nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0);
 614
 615         dev_info(ctrl->ctrl.device,
 616                  "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn);
 617
 618         nvme_get_ctrl(&ctrl->ctrl);
 619
 620         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
 621         WARN_ON_ONCE(!changed);
 622
 623         mutex_lock(&nvme_loop_ctrl_mutex);
 624         list_add_tail(&ctrl->list, &nvme_loop_ctrl_list);
 625         mutex_unlock(&nvme_loop_ctrl_mutex);
 626
 627         nvme_start_ctrl(&ctrl->ctrl);
 628
 629         return &ctrl->ctrl;
 630
 631 out_remove_admin_queue:
 632         nvme_loop_destroy_admin_queue(ctrl);
 633 out_free_queues:
 634         kfree(ctrl->queues);
 635 out_uninit_ctrl:
 636         nvme_uninit_ctrl(&ctrl->ctrl);
 637 out_put_ctrl:
 638         nvme_put_ctrl(&ctrl->ctrl);
 639         if (ret > 0)
 640                 ret = -EIO;
 641         return ERR_PTR(ret);
 642 }
 643
 644 static int nvme_loop_add_port(struct nvmet_port *port)
 645 {
 646         mutex_lock(&nvme_loop_ports_mutex);
 647         list_add_tail(&port->entry, &nvme_loop_ports);
 648         mutex_unlock(&nvme_loop_ports_mutex);
 649         return 0;
 650 }
 651
 652 static void nvme_loop_remove_port(struct nvmet_port *port)
 653 {
 654         mutex_lock(&nvme_loop_ports_mutex);
 655         list_del_init(&port->entry);
 656         mutex_unlock(&nvme_loop_ports_mutex);
 657
 658         /*
 659          * Ensure any ctrls that are in the process of being
 660          * deleted are in fact deleted before we return
 661          * and free the port. This is to prevent active
 662          * ctrls from using a port after it's freed.
 663          */
 664         flush_workqueue(nvme_delete_wq);
 665 }
 666
 667 static const struct nvmet_fabrics_ops nvme_loop_ops = {
 668         .owner          = THIS_MODULE,
 669         .type           = NVMF_TRTYPE_LOOP,
 670         .add_port       = nvme_loop_add_port,
 671         .remove_port    = nvme_loop_remove_port,
 672         .queue_response = nvme_loop_queue_response,
 673         .delete_ctrl    = nvme_loop_delete_ctrl,
 674 };
 675
 676 static struct nvmf_transport_ops nvme_loop_transport = {
 677         .name           = "loop",
 678         .module         = THIS_MODULE,
 679         .create_ctrl    = nvme_loop_create_ctrl,
 680         .allowed_opts   = NVMF_OPT_TRADDR,
 681 };
 682
 683 static int __init nvme_loop_init_module(void)
 684 {
 685         int ret;
 686
 687         ret = nvmet_register_transport(&nvme_loop_ops);
 688         if (ret)
 689                 return ret;
 690
 691         ret = nvmf_register_transport(&nvme_loop_transport);
 692         if (ret)
 693                 nvmet_unregister_transport(&nvme_loop_ops);
 694
 695         return ret;
 696 }
 697
 698 static void __exit nvme_loop_cleanup_module(void)
 699 {
 700         struct nvme_loop_ctrl *ctrl, *next;
 701
 702         nvmf_unregister_transport(&nvme_loop_transport);
 703         nvmet_unregister_transport(&nvme_loop_ops);
 704
 705         mutex_lock(&nvme_loop_ctrl_mutex);
 706         list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list)
 707                 nvme_delete_ctrl(&ctrl->ctrl);
 708         mutex_unlock(&nvme_loop_ctrl_mutex);
 709
 710         flush_workqueue(nvme_delete_wq);
 711 }
 712
 713 module_init(nvme_loop_init_module);
 714 module_exit(nvme_loop_cleanup_module);
 715
 716 MODULE_LICENSE("GPL v2");
 717 MODULE_ALIAS("nvmet-transport-254"); /* 254 == NVMF_TRTYPE_LOOP */