drivers/nvme/host/multipath.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2017-2018 Christoph Hellwig.
   4  */
   5
   6 #include <linux/moduleparam.h>
   7 #include <trace/events/block.h>
   8 #include "nvme.h"
   9
  10 static bool multipath = true;
  11 module_param(multipath, bool, 0444);
  12 MODULE_PARM_DESC(multipath,
  13         "turn on native support for multiple controllers per subsystem");
  14
  15 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
  16 {
  17         struct nvme_ns_head *h;
  18
  19         lockdep_assert_held(&subsys->lock);
  20         list_for_each_entry(h, &subsys->nsheads, entry)
  21                 if (h->disk)
  22                         blk_mq_unfreeze_queue(h->disk->queue);
  23 }
  24
  25 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys)
  26 {
  27         struct nvme_ns_head *h;
  28
  29         lockdep_assert_held(&subsys->lock);
  30         list_for_each_entry(h, &subsys->nsheads, entry)
  31                 if (h->disk)
  32                         blk_mq_freeze_queue_wait(h->disk->queue);
  33 }
  34
  35 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
  36 {
  37         struct nvme_ns_head *h;
  38
  39         lockdep_assert_held(&subsys->lock);
  40         list_for_each_entry(h, &subsys->nsheads, entry)
  41                 if (h->disk)
  42                         blk_freeze_queue_start(h->disk->queue);
  43 }
  44
  45 /*
  46  * If multipathing is enabled we need to always use the subsystem instance
  47  * number for numbering our devices to avoid conflicts between subsystems that
  48  * have multiple controllers and thus use the multipath-aware subsystem node
  49  * and those that have a single controller and use the controller node
  50  * directly.
  51  */
  52 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
  53                         struct nvme_ctrl *ctrl, int *flags)
  54 {
  55         if (!multipath) {
  56                 sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
  57         } else if (ns->head->disk) {
  58                 sprintf(disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
  59                                 ctrl->instance, ns->head->instance);
  60                 *flags = GENHD_FL_HIDDEN;
  61         } else {
  62                 sprintf(disk_name, "nvme%dn%d", ctrl->subsys->instance,
  63                                 ns->head->instance);
  64         }
  65 }
  66
  67 void nvme_failover_req(struct request *req)
  68 {
  69         struct nvme_ns *ns = req->q->queuedata;
  70         u16 status = nvme_req(req)->status;
  71         unsigned long flags;
  72
  73         spin_lock_irqsave(&ns->head->requeue_lock, flags);
  74         blk_steal_bios(&ns->head->requeue_list, req);
  75         spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
  76         blk_mq_end_request(req, 0);
  77
  78         switch (status & 0x7ff) {
  79         case NVME_SC_ANA_TRANSITION:
  80         case NVME_SC_ANA_INACCESSIBLE:
  81         case NVME_SC_ANA_PERSISTENT_LOSS:
  82                 /*
  83                  * If we got back an ANA error we know the controller is alive,
  84                  * but not ready to serve this namespaces.  The spec suggests
  85                  * we should update our general state here, but due to the fact
  86                  * that the admin and I/O queues are not serialized that is
  87                  * fundamentally racy.  So instead just clear the current path,
  88                  * mark the the path as pending and kick of a re-read of the ANA
  89                  * log page ASAP.
  90                  */
  91                 nvme_mpath_clear_current_path(ns);
  92                 if (ns->ctrl->ana_log_buf) {
  93                         set_bit(NVME_NS_ANA_PENDING, &ns->flags);
  94                         queue_work(nvme_wq, &ns->ctrl->ana_work);
  95                 }
  96                 break;
  97         case NVME_SC_HOST_PATH_ERROR:
  98                 /*
  99                  * Temporary transport disruption in talking to the controller.
 100                  * Try to send on a new path.
 101                  */
 102                 nvme_mpath_clear_current_path(ns);
 103                 break;
 104         default:
 105                 /*
 106                  * Reset the controller for any non-ANA error as we don't know
 107                  * what caused the error.
 108                  */
 109                 nvme_reset_ctrl(ns->ctrl);
 110                 break;
 111         }
 112
 113         kblockd_schedule_work(&ns->head->requeue_work);
 114 }
 115
 116 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 117 {
 118         struct nvme_ns *ns;
 119
 120         down_read(&ctrl->namespaces_rwsem);
 121         list_for_each_entry(ns, &ctrl->namespaces, list) {
 122                 if (ns->head->disk)
 123                         kblockd_schedule_work(&ns->head->requeue_work);
 124         }
 125         up_read(&ctrl->namespaces_rwsem);
 126 }
 127
 128 static const char *nvme_ana_state_names[] = {
 129         [0]                             = "invalid state",
 130         [NVME_ANA_OPTIMIZED]            = "optimized",
 131         [NVME_ANA_NONOPTIMIZED]         = "non-optimized",
 132         [NVME_ANA_INACCESSIBLE]         = "inaccessible",
 133         [NVME_ANA_PERSISTENT_LOSS]      = "persistent-loss",
 134         [NVME_ANA_CHANGE]               = "change",
 135 };
 136
 137 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
 138 {
 139         struct nvme_ns_head *head = ns->head;
 140         bool changed = false;
 141         int node;
 142
 143         if (!head)
 144                 goto out;
 145
 146         for_each_node(node) {
 147                 if (ns == rcu_access_pointer(head->current_path[node])) {
 148                         rcu_assign_pointer(head->current_path[node], NULL);
 149                         changed = true;
 150                 }
 151         }
 152 out:
 153         return changed;
 154 }
 155
 156 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
 157 {
 158         struct nvme_ns *ns;
 159
 160         mutex_lock(&ctrl->scan_lock);
 161         list_for_each_entry(ns, &ctrl->namespaces, list)
 162                 if (nvme_mpath_clear_current_path(ns))
 163                         kblockd_schedule_work(&ns->head->requeue_work);
 164         mutex_unlock(&ctrl->scan_lock);
 165 }
 166
 167 static bool nvme_path_is_disabled(struct nvme_ns *ns)
 168 {
 169         return ns->ctrl->state != NVME_CTRL_LIVE ||
 170                 test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
 171                 test_bit(NVME_NS_REMOVING, &ns->flags);
 172 }
 173
 174 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 175 {
 176         int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
 177         struct nvme_ns *found = NULL, *fallback = NULL, *ns;
 178
 179         list_for_each_entry_rcu(ns, &head->list, siblings) {
 180                 if (nvme_path_is_disabled(ns))
 181                         continue;
 182
 183                 if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
 184                         distance = node_distance(node, ns->ctrl->numa_node);
 185                 else
 186                         distance = LOCAL_DISTANCE;
 187
 188                 switch (ns->ana_state) {
 189                 case NVME_ANA_OPTIMIZED:
 190                         if (distance < found_distance) {
 191                                 found_distance = distance;
 192                                 found = ns;
 193                         }
 194                         break;
 195                 case NVME_ANA_NONOPTIMIZED:
 196                         if (distance < fallback_distance) {
 197                                 fallback_distance = distance;
 198                                 fallback = ns;
 199                         }
 200                         break;
 201                 default:
 202                         break;
 203                 }
 204         }
 205
 206         if (!found)
 207                 found = fallback;
 208         if (found)
 209                 rcu_assign_pointer(head->current_path[node], found);
 210         return found;
 211 }
 212
 213 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head,
 214                 struct nvme_ns *ns)
 215 {
 216         ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,
 217                         siblings);
 218         if (ns)
 219                 return ns;
 220         return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings);
 221 }
 222
 223 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 224                 int node, struct nvme_ns *old)
 225 {
 226         struct nvme_ns *ns, *found, *fallback = NULL;
 227
 228         if (list_is_singular(&head->list)) {
 229                 if (nvme_path_is_disabled(old))
 230                         return NULL;
 231                 return old;
 232         }
 233
 234         for (ns = nvme_next_ns(head, old);
 235              ns != old;
 236              ns = nvme_next_ns(head, ns)) {
 237                 if (nvme_path_is_disabled(ns))
 238                         continue;
 239
 240                 if (ns->ana_state == NVME_ANA_OPTIMIZED) {
 241                         found = ns;
 242                         goto out;
 243                 }
 244                 if (ns->ana_state == NVME_ANA_NONOPTIMIZED)
 245                         fallback = ns;
 246         }
 247
 248         if (!fallback)
 249                 return NULL;
 250         found = fallback;
 251 out:
 252         rcu_assign_pointer(head->current_path[node], found);
 253         return found;
 254 }
 255
 256 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 257 {
 258         return ns->ctrl->state == NVME_CTRL_LIVE &&
 259                 ns->ana_state == NVME_ANA_OPTIMIZED;
 260 }
 261
 262 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 263 {
 264         int node = numa_node_id();
 265         struct nvme_ns *ns;
 266
 267         ns = srcu_dereference(head->current_path[node], &head->srcu);
 268         if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR && ns)
 269                 ns = nvme_round_robin_path(head, node, ns);
 270         if (unlikely(!ns || !nvme_path_is_optimized(ns)))
 271                 ns = __nvme_find_path(head, node);
 272         return ns;
 273 }
 274
 275 static bool nvme_available_path(struct nvme_ns_head *head)
 276 {
 277         struct nvme_ns *ns;
 278
 279         list_for_each_entry_rcu(ns, &head->list, siblings) {
 280                 switch (ns->ctrl->state) {
 281                 case NVME_CTRL_LIVE:
 282                 case NVME_CTRL_RESETTING:
 283                 case NVME_CTRL_CONNECTING:
 284                         /* fallthru */
 285                         return true;
 286                 default:
 287                         break;
 288                 }
 289         }
 290         return false;
 291 }
 292
 293 static blk_qc_t nvme_ns_head_make_request(struct request_queue *q,
 294                 struct bio *bio)
 295 {
 296         struct nvme_ns_head *head = q->queuedata;
 297         struct device *dev = disk_to_dev(head->disk);
 298         struct nvme_ns *ns;
 299         blk_qc_t ret = BLK_QC_T_NONE;
 300         int srcu_idx;
 301
 302         /*
 303          * The namespace might be going away and the bio might
 304          * be moved to a different queue via blk_steal_bios(),
 305          * so we need to use the bio_split pool from the original
 306          * queue to allocate the bvecs from.
 307          */
 308         blk_queue_split(q, &bio);
 309
 310         srcu_idx = srcu_read_lock(&head->srcu);
 311         ns = nvme_find_path(head);
 312         if (likely(ns)) {
 313                 bio->bi_disk = ns->disk;
 314                 bio->bi_opf |= REQ_NVME_MPATH;
 315                 trace_block_bio_remap(bio->bi_disk->queue, bio,
 316                                       disk_devt(ns->head->disk),
 317                                       bio->bi_iter.bi_sector);
 318                 ret = direct_make_request(bio);
 319         } else if (nvme_available_path(head)) {
 320                 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n");
 321
 322                 spin_lock_irq(&head->requeue_lock);
 323                 bio_list_add(&head->requeue_list, bio);
 324                 spin_unlock_irq(&head->requeue_lock);
 325         } else {
 326                 dev_warn_ratelimited(dev, "no available path - failing I/O\n");
 327
 328                 bio->bi_status = BLK_STS_IOERR;
 329                 bio_endio(bio);
 330         }
 331
 332         srcu_read_unlock(&head->srcu, srcu_idx);
 333         return ret;
 334 }
 335
 336 static void nvme_requeue_work(struct work_struct *work)
 337 {
 338         struct nvme_ns_head *head =
 339                 container_of(work, struct nvme_ns_head, requeue_work);
 340         struct bio *bio, *next;
 341
 342         spin_lock_irq(&head->requeue_lock);
 343         next = bio_list_get(&head->requeue_list);
 344         spin_unlock_irq(&head->requeue_lock);
 345
 346         while ((bio = next) != NULL) {
 347                 next = bio->bi_next;
 348                 bio->bi_next = NULL;
 349
 350                 /*
 351                  * Reset disk to the mpath node and resubmit to select a new
 352                  * path.
 353                  */
 354                 bio->bi_disk = head->disk;
 355                 generic_make_request(bio);
 356         }
 357 }
 358
 359 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 360 {
 361         struct request_queue *q;
 362         bool vwc = false;
 363
 364         mutex_init(&head->lock);
 365         bio_list_init(&head->requeue_list);
 366         spin_lock_init(&head->requeue_lock);
 367         INIT_WORK(&head->requeue_work, nvme_requeue_work);
 368
 369         /*
 370          * Add a multipath node if the subsystems supports multiple controllers.
 371          * We also do this for private namespaces as the namespace sharing data could
 372          * change after a rescan.
 373          */
 374         if (!(ctrl->subsys->cmic & (1 << 1)) || !multipath)
 375                 return 0;
 376
 377         q = blk_alloc_queue_node(GFP_KERNEL, ctrl->numa_node);
 378         if (!q)
 379                 goto out;
 380         q->queuedata = head;
 381         blk_queue_make_request(q, nvme_ns_head_make_request);
 382         blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
 383         /* set to a default value for 512 until disk is validated */
 384         blk_queue_logical_block_size(q, 512);
 385         blk_set_stacking_limits(&q->limits);
 386
 387         /* we need to propagate up the VMC settings */
 388         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
 389                 vwc = true;
 390         blk_queue_write_cache(q, vwc, vwc);
 391
 392         head->disk = alloc_disk(0);
 393         if (!head->disk)
 394                 goto out_cleanup_queue;
 395         head->disk->fops = &nvme_ns_head_ops;
 396         head->disk->private_data = head;
 397         head->disk->queue = q;
 398         head->disk->flags = GENHD_FL_EXT_DEVT;
 399         sprintf(head->disk->disk_name, "nvme%dn%d",
 400                         ctrl->subsys->instance, head->instance);
 401         return 0;
 402
 403 out_cleanup_queue:
 404         blk_cleanup_queue(q);
 405 out:
 406         return -ENOMEM;
 407 }
 408
 409 static void nvme_mpath_set_live(struct nvme_ns *ns)
 410 {
 411         struct nvme_ns_head *head = ns->head;
 412
 413         lockdep_assert_held(&ns->head->lock);
 414
 415         if (!head->disk)
 416                 return;
 417
 418         if (!(head->disk->flags & GENHD_FL_UP))
 419                 device_add_disk(&head->subsys->dev, head->disk,
 420                                 nvme_ns_id_attr_groups);
 421
 422         if (nvme_path_is_optimized(ns)) {
 423                 int node, srcu_idx;
 424
 425                 srcu_idx = srcu_read_lock(&head->srcu);
 426                 for_each_node(node)
 427                         __nvme_find_path(head, node);
 428                 srcu_read_unlock(&head->srcu, srcu_idx);
 429         }
 430
 431         synchronize_srcu(&ns->head->srcu);
 432         kblockd_schedule_work(&ns->head->requeue_work);
 433 }
 434
 435 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
 436                 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
 437                         void *))
 438 {
 439         void *base = ctrl->ana_log_buf;
 440         size_t offset = sizeof(struct nvme_ana_rsp_hdr);
 441         int error, i;
 442
 443         lockdep_assert_held(&ctrl->ana_lock);
 444
 445         for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
 446                 struct nvme_ana_group_desc *desc = base + offset;
 447                 u32 nr_nsids = le32_to_cpu(desc->nnsids);
 448                 size_t nsid_buf_size = nr_nsids * sizeof(__le32);
 449
 450                 if (WARN_ON_ONCE(desc->grpid == 0))
 451                         return -EINVAL;
 452                 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
 453                         return -EINVAL;
 454                 if (WARN_ON_ONCE(desc->state == 0))
 455                         return -EINVAL;
 456                 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
 457                         return -EINVAL;
 458
 459                 offset += sizeof(*desc);
 460                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
 461                         return -EINVAL;
 462
 463                 error = cb(ctrl, desc, data);
 464                 if (error)
 465                         return error;
 466
 467                 offset += nsid_buf_size;
 468                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
 469                         return -EINVAL;
 470         }
 471
 472         return 0;
 473 }
 474
 475 static inline bool nvme_state_is_live(enum nvme_ana_state state)
 476 {
 477         return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
 478 }
 479
 480 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
 481                 struct nvme_ns *ns)
 482 {
 483         mutex_lock(&ns->head->lock);
 484         ns->ana_grpid = le32_to_cpu(desc->grpid);
 485         ns->ana_state = desc->state;
 486         clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
 487
 488         if (nvme_state_is_live(ns->ana_state))
 489                 nvme_mpath_set_live(ns);
 490         mutex_unlock(&ns->head->lock);
 491 }
 492
 493 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
 494                 struct nvme_ana_group_desc *desc, void *data)
 495 {
 496         u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
 497         unsigned *nr_change_groups = data;
 498         struct nvme_ns *ns;
 499
 500         dev_dbg(ctrl->device, "ANA group %d: %s.\n",
 501                         le32_to_cpu(desc->grpid),
 502                         nvme_ana_state_names[desc->state]);
 503
 504         if (desc->state == NVME_ANA_CHANGE)
 505                 (*nr_change_groups)++;
 506
 507         if (!nr_nsids)
 508                 return 0;
 509
 510         down_write(&ctrl->namespaces_rwsem);
 511         list_for_each_entry(ns, &ctrl->namespaces, list) {
 512                 unsigned nsid = le32_to_cpu(desc->nsids[n]);
 513
 514                 if (ns->head->ns_id < nsid)
 515                         continue;
 516                 if (ns->head->ns_id == nsid)
 517                         nvme_update_ns_ana_state(desc, ns);
 518                 if (++n == nr_nsids)
 519                         break;
 520         }
 521         up_write(&ctrl->namespaces_rwsem);
 522         return 0;
 523 }
 524
 525 static int nvme_read_ana_log(struct nvme_ctrl *ctrl, bool groups_only)
 526 {
 527         u32 nr_change_groups = 0;
 528         int error;
 529
 530         mutex_lock(&ctrl->ana_lock);
 531         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA,
 532                         groups_only ? NVME_ANA_LOG_RGO : 0,
 533                         ctrl->ana_log_buf, ctrl->ana_log_size, 0);
 534         if (error) {
 535                 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
 536                 goto out_unlock;
 537         }
 538
 539         error = nvme_parse_ana_log(ctrl, &nr_change_groups,
 540                         nvme_update_ana_state);
 541         if (error)
 542                 goto out_unlock;
 543
 544         /*
 545          * In theory we should have an ANATT timer per group as they might enter
 546          * the change state at different times.  But that is a lot of overhead
 547          * just to protect against a target that keeps entering new changes
 548          * states while never finishing previous ones.  But we'll still
 549          * eventually time out once all groups are in change state, so this
 550          * isn't a big deal.
 551          *
 552          * We also double the ANATT value to provide some slack for transports
 553          * or AEN processing overhead.
 554          */
 555         if (nr_change_groups)
 556                 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
 557         else
 558                 del_timer_sync(&ctrl->anatt_timer);
 559 out_unlock:
 560         mutex_unlock(&ctrl->ana_lock);
 561         return error;
 562 }
 563
 564 static void nvme_ana_work(struct work_struct *work)
 565 {
 566         struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
 567
 568         nvme_read_ana_log(ctrl, false);
 569 }
 570
 571 static void nvme_anatt_timeout(struct timer_list *t)
 572 {
 573         struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
 574
 575         dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
 576         nvme_reset_ctrl(ctrl);
 577 }
 578
 579 void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 580 {
 581         if (!nvme_ctrl_use_ana(ctrl))
 582                 return;
 583         del_timer_sync(&ctrl->anatt_timer);
 584         cancel_work_sync(&ctrl->ana_work);
 585 }
 586
 587 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store)  \
 588         struct device_attribute subsys_attr_##_name =   \
 589                 __ATTR(_name, _mode, _show, _store)
 590
 591 static const char *nvme_iopolicy_names[] = {
 592         [NVME_IOPOLICY_NUMA]    = "numa",
 593         [NVME_IOPOLICY_RR]      = "round-robin",
 594 };
 595
 596 static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
 597                 struct device_attribute *attr, char *buf)
 598 {
 599         struct nvme_subsystem *subsys =
 600                 container_of(dev, struct nvme_subsystem, dev);
 601
 602         return sprintf(buf, "%s\n",
 603                         nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
 604 }
 605
 606 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 607                 struct device_attribute *attr, const char *buf, size_t count)
 608 {
 609         struct nvme_subsystem *subsys =
 610                 container_of(dev, struct nvme_subsystem, dev);
 611         int i;
 612
 613         for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
 614                 if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
 615                         WRITE_ONCE(subsys->iopolicy, i);
 616                         return count;
 617                 }
 618         }
 619
 620         return -EINVAL;
 621 }
 622 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR,
 623                       nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store);
 624
 625 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
 626                 char *buf)
 627 {
 628         return sprintf(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
 629 }
 630 DEVICE_ATTR_RO(ana_grpid);
 631
 632 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
 633                 char *buf)
 634 {
 635         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 636
 637         return sprintf(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
 638 }
 639 DEVICE_ATTR_RO(ana_state);
 640
 641 static int nvme_set_ns_ana_state(struct nvme_ctrl *ctrl,
 642                 struct nvme_ana_group_desc *desc, void *data)
 643 {
 644         struct nvme_ns *ns = data;
 645
 646         if (ns->ana_grpid == le32_to_cpu(desc->grpid)) {
 647                 nvme_update_ns_ana_state(desc, ns);
 648                 return -ENXIO; /* just break out of the loop */
 649         }
 650
 651         return 0;
 652 }
 653
 654 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id)
 655 {
 656         if (nvme_ctrl_use_ana(ns->ctrl)) {
 657                 mutex_lock(&ns->ctrl->ana_lock);
 658                 ns->ana_grpid = le32_to_cpu(id->anagrpid);
 659                 nvme_parse_ana_log(ns->ctrl, ns, nvme_set_ns_ana_state);
 660                 mutex_unlock(&ns->ctrl->ana_lock);
 661         } else {
 662                 mutex_lock(&ns->head->lock);
 663                 ns->ana_state = NVME_ANA_OPTIMIZED;
 664                 nvme_mpath_set_live(ns);
 665                 mutex_unlock(&ns->head->lock);
 666         }
 667 }
 668
 669 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 670 {
 671         if (!head->disk)
 672                 return;
 673         if (head->disk->flags & GENHD_FL_UP)
 674                 del_gendisk(head->disk);
 675         blk_set_queue_dying(head->disk->queue);
 676         /* make sure all pending bios are cleaned up */
 677         kblockd_schedule_work(&head->requeue_work);
 678         flush_work(&head->requeue_work);
 679         blk_cleanup_queue(head->disk->queue);
 680         put_disk(head->disk);
 681 }
 682
 683 int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 684 {
 685         int error;
 686
 687         /* check if multipath is enabled and we have the capability */
 688         if (!multipath || !ctrl->subsys || !(ctrl->subsys->cmic & (1 << 3)))
 689                 return 0;
 690
 691         ctrl->anacap = id->anacap;
 692         ctrl->anatt = id->anatt;
 693         ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
 694         ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
 695
 696         mutex_init(&ctrl->ana_lock);
 697         timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
 698         ctrl->ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
 699                 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc);
 700         ctrl->ana_log_size += ctrl->max_namespaces * sizeof(__le32);
 701
 702         if (ctrl->ana_log_size > ctrl->max_hw_sectors << SECTOR_SHIFT) {
 703                 dev_err(ctrl->device,
 704                         "ANA log page size (%zd) larger than MDTS (%d).\n",
 705                         ctrl->ana_log_size,
 706                         ctrl->max_hw_sectors << SECTOR_SHIFT);
 707                 dev_err(ctrl->device, "disabling ANA support.\n");
 708                 return 0;
 709         }
 710
 711         INIT_WORK(&ctrl->ana_work, nvme_ana_work);
 712         ctrl->ana_log_buf = kmalloc(ctrl->ana_log_size, GFP_KERNEL);
 713         if (!ctrl->ana_log_buf) {
 714                 error = -ENOMEM;
 715                 goto out;
 716         }
 717
 718         error = nvme_read_ana_log(ctrl, false);
 719         if (error)
 720                 goto out_free_ana_log_buf;
 721         return 0;
 722 out_free_ana_log_buf:
 723         kfree(ctrl->ana_log_buf);
 724         ctrl->ana_log_buf = NULL;
 725 out:
 726         return error;
 727 }
 728
 729 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 730 {
 731         kfree(ctrl->ana_log_buf);
 732         ctrl->ana_log_buf = NULL;
 733 }
 734