drivers/nvme/host/multipath.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2017-2018 Christoph Hellwig.
   4  */
   5
   6 #include <linux/backing-dev.h>
   7 #include <linux/moduleparam.h>
   8 #include <linux/vmalloc.h>
   9 #include <trace/events/block.h>
  10 #include "nvme.h"
  11
  12 bool multipath = true;
  13 module_param(multipath, bool, 0444);
  14 MODULE_PARM_DESC(multipath,
  15         "turn on native support for multiple controllers per subsystem");
  16
  17 static const char *nvme_iopolicy_names[] = {
  18         [NVME_IOPOLICY_NUMA]    = "numa",
  19         [NVME_IOPOLICY_RR]      = "round-robin",
  20 };
  21
  22 static int iopolicy = NVME_IOPOLICY_NUMA;
  23
  24 static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp)
  25 {
  26         if (!val)
  27                 return -EINVAL;
  28         if (!strncmp(val, "numa", 4))
  29                 iopolicy = NVME_IOPOLICY_NUMA;
  30         else if (!strncmp(val, "round-robin", 11))
  31                 iopolicy = NVME_IOPOLICY_RR;
  32         else
  33                 return -EINVAL;
  34
  35         return 0;
  36 }
  37
  38 static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp)
  39 {
  40         return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]);
  41 }
  42
  43 module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy,
  44         &iopolicy, 0644);
  45 MODULE_PARM_DESC(iopolicy,
  46         "Default multipath I/O policy; 'numa' (default) or 'round-robin'");
  47
  48 void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
  49 {
  50         subsys->iopolicy = iopolicy;
  51 }
  52
  53 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
  54 {
  55         struct nvme_ns_head *h;
  56
  57         lockdep_assert_held(&subsys->lock);
  58         list_for_each_entry(h, &subsys->nsheads, entry)
  59                 if (h->disk)
  60                         blk_mq_unfreeze_queue(h->disk->queue);
  61 }
  62
  63 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys)
  64 {
  65         struct nvme_ns_head *h;
  66
  67         lockdep_assert_held(&subsys->lock);
  68         list_for_each_entry(h, &subsys->nsheads, entry)
  69                 if (h->disk)
  70                         blk_mq_freeze_queue_wait(h->disk->queue);
  71 }
  72
  73 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
  74 {
  75         struct nvme_ns_head *h;
  76
  77         lockdep_assert_held(&subsys->lock);
  78         list_for_each_entry(h, &subsys->nsheads, entry)
  79                 if (h->disk)
  80                         blk_freeze_queue_start(h->disk->queue);
  81 }
  82
  83 void nvme_failover_req(struct request *req)
  84 {
  85         struct nvme_ns *ns = req->q->queuedata;
  86         u16 status = nvme_req(req)->status & 0x7ff;
  87         unsigned long flags;
  88         struct bio *bio;
  89
  90         nvme_mpath_clear_current_path(ns);
  91
  92         /*
  93          * If we got back an ANA error, we know the controller is alive but not
  94          * ready to serve this namespace.  Kick of a re-read of the ANA
  95          * information page, and just try any other available path for now.
  96          */
  97         if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {
  98                 set_bit(NVME_NS_ANA_PENDING, &ns->flags);
  99                 queue_work(nvme_wq, &ns->ctrl->ana_work);
 100         }
 101
 102         spin_lock_irqsave(&ns->head->requeue_lock, flags);
 103         for (bio = req->bio; bio; bio = bio->bi_next) {
 104                 bio_set_dev(bio, ns->head->disk->part0);
 105                 if (bio->bi_opf & REQ_POLLED) {
 106                         bio->bi_opf &= ~REQ_POLLED;
 107                         bio->bi_cookie = BLK_QC_T_NONE;
 108                 }
 109                 /*
 110                  * The alternate request queue that we may end up submitting
 111                  * the bio to may be frozen temporarily, in this case REQ_NOWAIT
 112                  * will fail the I/O immediately with EAGAIN to the issuer.
 113                  * We are not in the issuer context which cannot block. Clear
 114                  * the flag to avoid spurious EAGAIN I/O failures.
 115                  */
 116                 bio->bi_opf &= ~REQ_NOWAIT;
 117         }
 118         blk_steal_bios(&ns->head->requeue_list, req);
 119         spin_unlock_irqrestore(&ns->head->requeue_lock, flags);
 120
 121         blk_mq_end_request(req, 0);
 122         kblockd_schedule_work(&ns->head->requeue_work);
 123 }
 124
 125 void nvme_mpath_start_request(struct request *rq)
 126 {
 127         struct nvme_ns *ns = rq->q->queuedata;
 128         struct gendisk *disk = ns->head->disk;
 129
 130         if (!blk_queue_io_stat(disk->queue) || blk_rq_is_passthrough(rq))
 131                 return;
 132
 133         nvme_req(rq)->flags |= NVME_MPATH_IO_STATS;
 134         nvme_req(rq)->start_time = bdev_start_io_acct(disk->part0, req_op(rq),
 135                                                       jiffies);
 136 }
 137 EXPORT_SYMBOL_GPL(nvme_mpath_start_request);
 138
 139 void nvme_mpath_end_request(struct request *rq)
 140 {
 141         struct nvme_ns *ns = rq->q->queuedata;
 142
 143         if (!(nvme_req(rq)->flags & NVME_MPATH_IO_STATS))
 144                 return;
 145         bdev_end_io_acct(ns->head->disk->part0, req_op(rq),
 146                          blk_rq_bytes(rq) >> SECTOR_SHIFT,
 147                          nvme_req(rq)->start_time);
 148 }
 149
 150 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
 151 {
 152         struct nvme_ns *ns;
 153
 154         down_read(&ctrl->namespaces_rwsem);
 155         list_for_each_entry(ns, &ctrl->namespaces, list) {
 156                 if (!ns->head->disk)
 157                         continue;
 158                 kblockd_schedule_work(&ns->head->requeue_work);
 159                 if (nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE)
 160                         disk_uevent(ns->head->disk, KOBJ_CHANGE);
 161         }
 162         up_read(&ctrl->namespaces_rwsem);
 163 }
 164
 165 static const char *nvme_ana_state_names[] = {
 166         [0]                             = "invalid state",
 167         [NVME_ANA_OPTIMIZED]            = "optimized",
 168         [NVME_ANA_NONOPTIMIZED]         = "non-optimized",
 169         [NVME_ANA_INACCESSIBLE]         = "inaccessible",
 170         [NVME_ANA_PERSISTENT_LOSS]      = "persistent-loss",
 171         [NVME_ANA_CHANGE]               = "change",
 172 };
 173
 174 bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
 175 {
 176         struct nvme_ns_head *head = ns->head;
 177         bool changed = false;
 178         int node;
 179
 180         if (!head)
 181                 goto out;
 182
 183         for_each_node(node) {
 184                 if (ns == rcu_access_pointer(head->current_path[node])) {
 185                         rcu_assign_pointer(head->current_path[node], NULL);
 186                         changed = true;
 187                 }
 188         }
 189 out:
 190         return changed;
 191 }
 192
 193 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
 194 {
 195         struct nvme_ns *ns;
 196
 197         down_read(&ctrl->namespaces_rwsem);
 198         list_for_each_entry(ns, &ctrl->namespaces, list) {
 199                 nvme_mpath_clear_current_path(ns);
 200                 kblockd_schedule_work(&ns->head->requeue_work);
 201         }
 202         up_read(&ctrl->namespaces_rwsem);
 203 }
 204
 205 void nvme_mpath_revalidate_paths(struct nvme_ns *ns)
 206 {
 207         struct nvme_ns_head *head = ns->head;
 208         sector_t capacity = get_capacity(head->disk);
 209         int node;
 210         int srcu_idx;
 211
 212         srcu_idx = srcu_read_lock(&head->srcu);
 213         list_for_each_entry_rcu(ns, &head->list, siblings) {
 214                 if (capacity != get_capacity(ns->disk))
 215                         clear_bit(NVME_NS_READY, &ns->flags);
 216         }
 217         srcu_read_unlock(&head->srcu, srcu_idx);
 218
 219         for_each_node(node)
 220                 rcu_assign_pointer(head->current_path[node], NULL);
 221         kblockd_schedule_work(&head->requeue_work);
 222 }
 223
 224 static bool nvme_path_is_disabled(struct nvme_ns *ns)
 225 {
 226         enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl);
 227
 228         /*
 229          * We don't treat NVME_CTRL_DELETING as a disabled path as I/O should
 230          * still be able to complete assuming that the controller is connected.
 231          * Otherwise it will fail immediately and return to the requeue list.
 232          */
 233         if (state != NVME_CTRL_LIVE && state != NVME_CTRL_DELETING)
 234                 return true;
 235         if (test_bit(NVME_NS_ANA_PENDING, &ns->flags) ||
 236             !test_bit(NVME_NS_READY, &ns->flags))
 237                 return true;
 238         return false;
 239 }
 240
 241 static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 242 {
 243         int found_distance = INT_MAX, fallback_distance = INT_MAX, distance;
 244         struct nvme_ns *found = NULL, *fallback = NULL, *ns;
 245
 246         list_for_each_entry_rcu(ns, &head->list, siblings) {
 247                 if (nvme_path_is_disabled(ns))
 248                         continue;
 249
 250                 if (ns->ctrl->numa_node != NUMA_NO_NODE &&
 251                     READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_NUMA)
 252                         distance = node_distance(node, ns->ctrl->numa_node);
 253                 else
 254                         distance = LOCAL_DISTANCE;
 255
 256                 switch (ns->ana_state) {
 257                 case NVME_ANA_OPTIMIZED:
 258                         if (distance < found_distance) {
 259                                 found_distance = distance;
 260                                 found = ns;
 261                         }
 262                         break;
 263                 case NVME_ANA_NONOPTIMIZED:
 264                         if (distance < fallback_distance) {
 265                                 fallback_distance = distance;
 266                                 fallback = ns;
 267                         }
 268                         break;
 269                 default:
 270                         break;
 271                 }
 272         }
 273
 274         if (!found)
 275                 found = fallback;
 276         if (found)
 277                 rcu_assign_pointer(head->current_path[node], found);
 278         return found;
 279 }
 280
 281 static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head,
 282                 struct nvme_ns *ns)
 283 {
 284         ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns,
 285                         siblings);
 286         if (ns)
 287                 return ns;
 288         return list_first_or_null_rcu(&head->list, struct nvme_ns, siblings);
 289 }
 290
 291 static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head,
 292                 int node, struct nvme_ns *old)
 293 {
 294         struct nvme_ns *ns, *found = NULL;
 295
 296         if (list_is_singular(&head->list)) {
 297                 if (nvme_path_is_disabled(old))
 298                         return NULL;
 299                 return old;
 300         }
 301
 302         for (ns = nvme_next_ns(head, old);
 303              ns && ns != old;
 304              ns = nvme_next_ns(head, ns)) {
 305                 if (nvme_path_is_disabled(ns))
 306                         continue;
 307
 308                 if (ns->ana_state == NVME_ANA_OPTIMIZED) {
 309                         found = ns;
 310                         goto out;
 311                 }
 312                 if (ns->ana_state == NVME_ANA_NONOPTIMIZED)
 313                         found = ns;
 314         }
 315
 316         /*
 317          * The loop above skips the current path for round-robin semantics.
 318          * Fall back to the current path if either:
 319          *  - no other optimized path found and current is optimized,
 320          *  - no other usable path found and current is usable.
 321          */
 322         if (!nvme_path_is_disabled(old) &&
 323             (old->ana_state == NVME_ANA_OPTIMIZED ||
 324              (!found && old->ana_state == NVME_ANA_NONOPTIMIZED)))
 325                 return old;
 326
 327         if (!found)
 328                 return NULL;
 329 out:
 330         rcu_assign_pointer(head->current_path[node], found);
 331         return found;
 332 }
 333
 334 static inline bool nvme_path_is_optimized(struct nvme_ns *ns)
 335 {
 336         return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE &&
 337                 ns->ana_state == NVME_ANA_OPTIMIZED;
 338 }
 339
 340 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
 341 {
 342         int node = numa_node_id();
 343         struct nvme_ns *ns;
 344
 345         ns = srcu_dereference(head->current_path[node], &head->srcu);
 346         if (unlikely(!ns))
 347                 return __nvme_find_path(head, node);
 348
 349         if (READ_ONCE(head->subsys->iopolicy) == NVME_IOPOLICY_RR)
 350                 return nvme_round_robin_path(head, node, ns);
 351         if (unlikely(!nvme_path_is_optimized(ns)))
 352                 return __nvme_find_path(head, node);
 353         return ns;
 354 }
 355
 356 static bool nvme_available_path(struct nvme_ns_head *head)
 357 {
 358         struct nvme_ns *ns;
 359
 360         list_for_each_entry_rcu(ns, &head->list, siblings) {
 361                 if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))
 362                         continue;
 363                 switch (nvme_ctrl_state(ns->ctrl)) {
 364                 case NVME_CTRL_LIVE:
 365                 case NVME_CTRL_RESETTING:
 366                 case NVME_CTRL_CONNECTING:
 367                         /* fallthru */
 368                         return true;
 369                 default:
 370                         break;
 371                 }
 372         }
 373         return false;
 374 }
 375
 376 static void nvme_ns_head_submit_bio(struct bio *bio)
 377 {
 378         struct nvme_ns_head *head = bio->bi_bdev->bd_disk->private_data;
 379         struct device *dev = disk_to_dev(head->disk);
 380         struct nvme_ns *ns;
 381         int srcu_idx;
 382
 383         /*
 384          * The namespace might be going away and the bio might be moved to a
 385          * different queue via blk_steal_bios(), so we need to use the bio_split
 386          * pool from the original queue to allocate the bvecs from.
 387          */
 388         bio = bio_split_to_limits(bio);
 389         if (!bio)
 390                 return;
 391
 392         srcu_idx = srcu_read_lock(&head->srcu);
 393         ns = nvme_find_path(head);
 394         if (likely(ns)) {
 395                 bio_set_dev(bio, ns->disk->part0);
 396                 bio->bi_opf |= REQ_NVME_MPATH;
 397                 trace_block_bio_remap(bio, disk_devt(ns->head->disk),
 398                                       bio->bi_iter.bi_sector);
 399                 submit_bio_noacct(bio);
 400         } else if (nvme_available_path(head)) {
 401                 dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n");
 402
 403                 spin_lock_irq(&head->requeue_lock);
 404                 bio_list_add(&head->requeue_list, bio);
 405                 spin_unlock_irq(&head->requeue_lock);
 406         } else {
 407                 dev_warn_ratelimited(dev, "no available path - failing I/O\n");
 408
 409                 bio_io_error(bio);
 410         }
 411
 412         srcu_read_unlock(&head->srcu, srcu_idx);
 413 }
 414
 415 static int nvme_ns_head_open(struct gendisk *disk, blk_mode_t mode)
 416 {
 417         if (!nvme_tryget_ns_head(disk->private_data))
 418                 return -ENXIO;
 419         return 0;
 420 }
 421
 422 static void nvme_ns_head_release(struct gendisk *disk)
 423 {
 424         nvme_put_ns_head(disk->private_data);
 425 }
 426
 427 #ifdef CONFIG_BLK_DEV_ZONED
 428 static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector,
 429                 unsigned int nr_zones, report_zones_cb cb, void *data)
 430 {
 431         struct nvme_ns_head *head = disk->private_data;
 432         struct nvme_ns *ns;
 433         int srcu_idx, ret = -EWOULDBLOCK;
 434
 435         srcu_idx = srcu_read_lock(&head->srcu);
 436         ns = nvme_find_path(head);
 437         if (ns)
 438                 ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data);
 439         srcu_read_unlock(&head->srcu, srcu_idx);
 440         return ret;
 441 }
 442 #else
 443 #define nvme_ns_head_report_zones       NULL
 444 #endif /* CONFIG_BLK_DEV_ZONED */
 445
 446 const struct block_device_operations nvme_ns_head_ops = {
 447         .owner          = THIS_MODULE,
 448         .submit_bio     = nvme_ns_head_submit_bio,
 449         .open           = nvme_ns_head_open,
 450         .release        = nvme_ns_head_release,
 451         .ioctl          = nvme_ns_head_ioctl,
 452         .compat_ioctl   = blkdev_compat_ptr_ioctl,
 453         .getgeo         = nvme_getgeo,
 454         .report_zones   = nvme_ns_head_report_zones,
 455         .pr_ops         = &nvme_pr_ops,
 456 };
 457
 458 static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev)
 459 {
 460         return container_of(cdev, struct nvme_ns_head, cdev);
 461 }
 462
 463 static int nvme_ns_head_chr_open(struct inode *inode, struct file *file)
 464 {
 465         if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev)))
 466                 return -ENXIO;
 467         return 0;
 468 }
 469
 470 static int nvme_ns_head_chr_release(struct inode *inode, struct file *file)
 471 {
 472         nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev));
 473         return 0;
 474 }
 475
 476 static const struct file_operations nvme_ns_head_chr_fops = {
 477         .owner          = THIS_MODULE,
 478         .open           = nvme_ns_head_chr_open,
 479         .release        = nvme_ns_head_chr_release,
 480         .unlocked_ioctl = nvme_ns_head_chr_ioctl,
 481         .compat_ioctl   = compat_ptr_ioctl,
 482         .uring_cmd      = nvme_ns_head_chr_uring_cmd,
 483         .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
 484 };
 485
 486 static int nvme_add_ns_head_cdev(struct nvme_ns_head *head)
 487 {
 488         int ret;
 489
 490         head->cdev_device.parent = &head->subsys->dev;
 491         ret = dev_set_name(&head->cdev_device, "ng%dn%d",
 492                            head->subsys->instance, head->instance);
 493         if (ret)
 494                 return ret;
 495         ret = nvme_cdev_add(&head->cdev, &head->cdev_device,
 496                             &nvme_ns_head_chr_fops, THIS_MODULE);
 497         return ret;
 498 }
 499
 500 static void nvme_requeue_work(struct work_struct *work)
 501 {
 502         struct nvme_ns_head *head =
 503                 container_of(work, struct nvme_ns_head, requeue_work);
 504         struct bio *bio, *next;
 505
 506         spin_lock_irq(&head->requeue_lock);
 507         next = bio_list_get(&head->requeue_list);
 508         spin_unlock_irq(&head->requeue_lock);
 509
 510         while ((bio = next) != NULL) {
 511                 next = bio->bi_next;
 512                 bio->bi_next = NULL;
 513
 514                 submit_bio_noacct(bio);
 515         }
 516 }
 517
 518 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 519 {
 520         struct queue_limits lim;
 521         bool vwc = false;
 522
 523         mutex_init(&head->lock);
 524         bio_list_init(&head->requeue_list);
 525         spin_lock_init(&head->requeue_lock);
 526         INIT_WORK(&head->requeue_work, nvme_requeue_work);
 527
 528         /*
 529          * Add a multipath node if the subsystems supports multiple controllers.
 530          * We also do this for private namespaces as the namespace sharing flag
 531          * could change after a rescan.
 532          */
 533         if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
 534             !nvme_is_unique_nsid(ctrl, head) || !multipath)
 535                 return 0;
 536
 537         blk_set_stacking_limits(&lim);
 538         lim.dma_alignment = 3;
 539         if (head->ids.csi != NVME_CSI_ZNS)
 540                 lim.max_zone_append_sectors = 0;
 541
 542         head->disk = blk_alloc_disk(&lim, ctrl->numa_node);
 543         if (IS_ERR(head->disk))
 544                 return PTR_ERR(head->disk);
 545         head->disk->fops = &nvme_ns_head_ops;
 546         head->disk->private_data = head;
 547         sprintf(head->disk->disk_name, "nvme%dn%d",
 548                         ctrl->subsys->instance, head->instance);
 549
 550         blk_queue_flag_set(QUEUE_FLAG_NONROT, head->disk->queue);
 551         blk_queue_flag_set(QUEUE_FLAG_NOWAIT, head->disk->queue);
 552         blk_queue_flag_set(QUEUE_FLAG_IO_STAT, head->disk->queue);
 553         /*
 554          * This assumes all controllers that refer to a namespace either
 555          * support poll queues or not.  That is not a strict guarantee,
 556          * but if the assumption is wrong the effect is only suboptimal
 557          * performance but not correctness problem.
 558          */
 559         if (ctrl->tagset->nr_maps > HCTX_TYPE_POLL &&
 560             ctrl->tagset->map[HCTX_TYPE_POLL].nr_queues)
 561                 blk_queue_flag_set(QUEUE_FLAG_POLL, head->disk->queue);
 562
 563         /* we need to propagate up the VMC settings */
 564         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
 565                 vwc = true;
 566         blk_queue_write_cache(head->disk->queue, vwc, vwc);
 567         return 0;
 568 }
 569
 570 static void nvme_mpath_set_live(struct nvme_ns *ns)
 571 {
 572         struct nvme_ns_head *head = ns->head;
 573         int rc;
 574
 575         if (!head->disk)
 576                 return;
 577
 578         /*
 579          * test_and_set_bit() is used because it is protecting against two nvme
 580          * paths simultaneously calling device_add_disk() on the same namespace
 581          * head.
 582          */
 583         if (!test_and_set_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {
 584                 rc = device_add_disk(&head->subsys->dev, head->disk,
 585                                      nvme_ns_attr_groups);
 586                 if (rc) {
 587                         clear_bit(NVME_NSHEAD_DISK_LIVE, &ns->flags);
 588                         return;
 589                 }
 590                 nvme_add_ns_head_cdev(head);
 591         }
 592
 593         mutex_lock(&head->lock);
 594         if (nvme_path_is_optimized(ns)) {
 595                 int node, srcu_idx;
 596
 597                 srcu_idx = srcu_read_lock(&head->srcu);
 598                 for_each_node(node)
 599                         __nvme_find_path(head, node);
 600                 srcu_read_unlock(&head->srcu, srcu_idx);
 601         }
 602         mutex_unlock(&head->lock);
 603
 604         synchronize_srcu(&head->srcu);
 605         kblockd_schedule_work(&head->requeue_work);
 606 }
 607
 608 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,
 609                 int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,
 610                         void *))
 611 {
 612         void *base = ctrl->ana_log_buf;
 613         size_t offset = sizeof(struct nvme_ana_rsp_hdr);
 614         int error, i;
 615
 616         lockdep_assert_held(&ctrl->ana_lock);
 617
 618         for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {
 619                 struct nvme_ana_group_desc *desc = base + offset;
 620                 u32 nr_nsids;
 621                 size_t nsid_buf_size;
 622
 623                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))
 624                         return -EINVAL;
 625
 626                 nr_nsids = le32_to_cpu(desc->nnsids);
 627                 nsid_buf_size = flex_array_size(desc, nsids, nr_nsids);
 628
 629                 if (WARN_ON_ONCE(desc->grpid == 0))
 630                         return -EINVAL;
 631                 if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))
 632                         return -EINVAL;
 633                 if (WARN_ON_ONCE(desc->state == 0))
 634                         return -EINVAL;
 635                 if (WARN_ON_ONCE(desc->state > NVME_ANA_CHANGE))
 636                         return -EINVAL;
 637
 638                 offset += sizeof(*desc);
 639                 if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))
 640                         return -EINVAL;
 641
 642                 error = cb(ctrl, desc, data);
 643                 if (error)
 644                         return error;
 645
 646                 offset += nsid_buf_size;
 647         }
 648
 649         return 0;
 650 }
 651
 652 static inline bool nvme_state_is_live(enum nvme_ana_state state)
 653 {
 654         return state == NVME_ANA_OPTIMIZED || state == NVME_ANA_NONOPTIMIZED;
 655 }
 656
 657 static void nvme_update_ns_ana_state(struct nvme_ana_group_desc *desc,
 658                 struct nvme_ns *ns)
 659 {
 660         ns->ana_grpid = le32_to_cpu(desc->grpid);
 661         ns->ana_state = desc->state;
 662         clear_bit(NVME_NS_ANA_PENDING, &ns->flags);
 663         /*
 664          * nvme_mpath_set_live() will trigger I/O to the multipath path device
 665          * and in turn to this path device.  However we cannot accept this I/O
 666          * if the controller is not live.  This may deadlock if called from
 667          * nvme_mpath_init_identify() and the ctrl will never complete
 668          * initialization, preventing I/O from completing.  For this case we
 669          * will reprocess the ANA log page in nvme_mpath_update() once the
 670          * controller is ready.
 671          */
 672         if (nvme_state_is_live(ns->ana_state) &&
 673             nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE)
 674                 nvme_mpath_set_live(ns);
 675 }
 676
 677 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,
 678                 struct nvme_ana_group_desc *desc, void *data)
 679 {
 680         u32 nr_nsids = le32_to_cpu(desc->nnsids), n = 0;
 681         unsigned *nr_change_groups = data;
 682         struct nvme_ns *ns;
 683
 684         dev_dbg(ctrl->device, "ANA group %d: %s.\n",
 685                         le32_to_cpu(desc->grpid),
 686                         nvme_ana_state_names[desc->state]);
 687
 688         if (desc->state == NVME_ANA_CHANGE)
 689                 (*nr_change_groups)++;
 690
 691         if (!nr_nsids)
 692                 return 0;
 693
 694         down_read(&ctrl->namespaces_rwsem);
 695         list_for_each_entry(ns, &ctrl->namespaces, list) {
 696                 unsigned nsid;
 697 again:
 698                 nsid = le32_to_cpu(desc->nsids[n]);
 699                 if (ns->head->ns_id < nsid)
 700                         continue;
 701                 if (ns->head->ns_id == nsid)
 702                         nvme_update_ns_ana_state(desc, ns);
 703                 if (++n == nr_nsids)
 704                         break;
 705                 if (ns->head->ns_id > nsid)
 706                         goto again;
 707         }
 708         up_read(&ctrl->namespaces_rwsem);
 709         return 0;
 710 }
 711
 712 static int nvme_read_ana_log(struct nvme_ctrl *ctrl)
 713 {
 714         u32 nr_change_groups = 0;
 715         int error;
 716
 717         mutex_lock(&ctrl->ana_lock);
 718         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM,
 719                         ctrl->ana_log_buf, ctrl->ana_log_size, 0);
 720         if (error) {
 721                 dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);
 722                 goto out_unlock;
 723         }
 724
 725         error = nvme_parse_ana_log(ctrl, &nr_change_groups,
 726                         nvme_update_ana_state);
 727         if (error)
 728                 goto out_unlock;
 729
 730         /*
 731          * In theory we should have an ANATT timer per group as they might enter
 732          * the change state at different times.  But that is a lot of overhead
 733          * just to protect against a target that keeps entering new changes
 734          * states while never finishing previous ones.  But we'll still
 735          * eventually time out once all groups are in change state, so this
 736          * isn't a big deal.
 737          *
 738          * We also double the ANATT value to provide some slack for transports
 739          * or AEN processing overhead.
 740          */
 741         if (nr_change_groups)
 742                 mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);
 743         else
 744                 del_timer_sync(&ctrl->anatt_timer);
 745 out_unlock:
 746         mutex_unlock(&ctrl->ana_lock);
 747         return error;
 748 }
 749
 750 static void nvme_ana_work(struct work_struct *work)
 751 {
 752         struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);
 753
 754         if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
 755                 return;
 756
 757         nvme_read_ana_log(ctrl);
 758 }
 759
 760 void nvme_mpath_update(struct nvme_ctrl *ctrl)
 761 {
 762         u32 nr_change_groups = 0;
 763
 764         if (!ctrl->ana_log_buf)
 765                 return;
 766
 767         mutex_lock(&ctrl->ana_lock);
 768         nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state);
 769         mutex_unlock(&ctrl->ana_lock);
 770 }
 771
 772 static void nvme_anatt_timeout(struct timer_list *t)
 773 {
 774         struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);
 775
 776         dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");
 777         nvme_reset_ctrl(ctrl);
 778 }
 779
 780 void nvme_mpath_stop(struct nvme_ctrl *ctrl)
 781 {
 782         if (!nvme_ctrl_use_ana(ctrl))
 783                 return;
 784         del_timer_sync(&ctrl->anatt_timer);
 785         cancel_work_sync(&ctrl->ana_work);
 786 }
 787
 788 #define SUBSYS_ATTR_RW(_name, _mode, _show, _store)  \
 789         struct device_attribute subsys_attr_##_name =   \
 790                 __ATTR(_name, _mode, _show, _store)
 791
 792 static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
 793                 struct device_attribute *attr, char *buf)
 794 {
 795         struct nvme_subsystem *subsys =
 796                 container_of(dev, struct nvme_subsystem, dev);
 797
 798         return sysfs_emit(buf, "%s\n",
 799                           nvme_iopolicy_names[READ_ONCE(subsys->iopolicy)]);
 800 }
 801
 802 static ssize_t nvme_subsys_iopolicy_store(struct device *dev,
 803                 struct device_attribute *attr, const char *buf, size_t count)
 804 {
 805         struct nvme_subsystem *subsys =
 806                 container_of(dev, struct nvme_subsystem, dev);
 807         int i;
 808
 809         for (i = 0; i < ARRAY_SIZE(nvme_iopolicy_names); i++) {
 810                 if (sysfs_streq(buf, nvme_iopolicy_names[i])) {
 811                         WRITE_ONCE(subsys->iopolicy, i);
 812                         return count;
 813                 }
 814         }
 815
 816         return -EINVAL;
 817 }
 818 SUBSYS_ATTR_RW(iopolicy, S_IRUGO | S_IWUSR,
 819                       nvme_subsys_iopolicy_show, nvme_subsys_iopolicy_store);
 820
 821 static ssize_t ana_grpid_show(struct device *dev, struct device_attribute *attr,
 822                 char *buf)
 823 {
 824         return sysfs_emit(buf, "%d\n", nvme_get_ns_from_dev(dev)->ana_grpid);
 825 }
 826 DEVICE_ATTR_RO(ana_grpid);
 827
 828 static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
 829                 char *buf)
 830 {
 831         struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
 832
 833         return sysfs_emit(buf, "%s\n", nvme_ana_state_names[ns->ana_state]);
 834 }
 835 DEVICE_ATTR_RO(ana_state);
 836
 837 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl,
 838                 struct nvme_ana_group_desc *desc, void *data)
 839 {
 840         struct nvme_ana_group_desc *dst = data;
 841
 842         if (desc->grpid != dst->grpid)
 843                 return 0;
 844
 845         *dst = *desc;
 846         return -ENXIO; /* just break out of the loop */
 847 }
 848
 849 void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid)
 850 {
 851         if (nvme_ctrl_use_ana(ns->ctrl)) {
 852                 struct nvme_ana_group_desc desc = {
 853                         .grpid = anagrpid,
 854                         .state = 0,
 855                 };
 856
 857                 mutex_lock(&ns->ctrl->ana_lock);
 858                 ns->ana_grpid = le32_to_cpu(anagrpid);
 859                 nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc);
 860                 mutex_unlock(&ns->ctrl->ana_lock);
 861                 if (desc.state) {
 862                         /* found the group desc: update */
 863                         nvme_update_ns_ana_state(&desc, ns);
 864                 } else {
 865                         /* group desc not found: trigger a re-read */
 866                         set_bit(NVME_NS_ANA_PENDING, &ns->flags);
 867                         queue_work(nvme_wq, &ns->ctrl->ana_work);
 868                 }
 869         } else {
 870                 ns->ana_state = NVME_ANA_OPTIMIZED;
 871                 nvme_mpath_set_live(ns);
 872         }
 873
 874         if (blk_queue_stable_writes(ns->queue) && ns->head->disk)
 875                 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES,
 876                                    ns->head->disk->queue);
 877 #ifdef CONFIG_BLK_DEV_ZONED
 878         if (blk_queue_is_zoned(ns->queue) && ns->head->disk)
 879                 ns->head->disk->nr_zones = ns->disk->nr_zones;
 880 #endif
 881 }
 882
 883 void nvme_mpath_shutdown_disk(struct nvme_ns_head *head)
 884 {
 885         if (!head->disk)
 886                 return;
 887         kblockd_schedule_work(&head->requeue_work);
 888         if (test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) {
 889                 nvme_cdev_del(&head->cdev, &head->cdev_device);
 890                 del_gendisk(head->disk);
 891         }
 892 }
 893
 894 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 895 {
 896         if (!head->disk)
 897                 return;
 898         /* make sure all pending bios are cleaned up */
 899         kblockd_schedule_work(&head->requeue_work);
 900         flush_work(&head->requeue_work);
 901         put_disk(head->disk);
 902 }
 903
 904 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)
 905 {
 906         mutex_init(&ctrl->ana_lock);
 907         timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);
 908         INIT_WORK(&ctrl->ana_work, nvme_ana_work);
 909 }
 910
 911 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 912 {
 913         size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT;
 914         size_t ana_log_size;
 915         int error = 0;
 916
 917         /* check if multipath is enabled and we have the capability */
 918         if (!multipath || !ctrl->subsys ||
 919             !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA))
 920                 return 0;
 921
 922         if (!ctrl->max_namespaces ||
 923             ctrl->max_namespaces > le32_to_cpu(id->nn)) {
 924                 dev_err(ctrl->device,
 925                         "Invalid MNAN value %u\n", ctrl->max_namespaces);
 926                 return -EINVAL;
 927         }
 928
 929         ctrl->anacap = id->anacap;
 930         ctrl->anatt = id->anatt;
 931         ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);
 932         ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);
 933
 934         ana_log_size = sizeof(struct nvme_ana_rsp_hdr) +
 935                 ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) +
 936                 ctrl->max_namespaces * sizeof(__le32);
 937         if (ana_log_size > max_transfer_size) {
 938                 dev_err(ctrl->device,
 939                         "ANA log page size (%zd) larger than MDTS (%zd).\n",
 940                         ana_log_size, max_transfer_size);
 941                 dev_err(ctrl->device, "disabling ANA support.\n");
 942                 goto out_uninit;
 943         }
 944         if (ana_log_size > ctrl->ana_log_size) {
 945                 nvme_mpath_stop(ctrl);
 946                 nvme_mpath_uninit(ctrl);
 947                 ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL);
 948                 if (!ctrl->ana_log_buf)
 949                         return -ENOMEM;
 950         }
 951         ctrl->ana_log_size = ana_log_size;
 952         error = nvme_read_ana_log(ctrl);
 953         if (error)
 954                 goto out_uninit;
 955         return 0;
 956
 957 out_uninit:
 958         nvme_mpath_uninit(ctrl);
 959         return error;
 960 }
 961
 962 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
 963 {
 964         kvfree(ctrl->ana_log_buf);
 965         ctrl->ana_log_buf = NULL;
 966         ctrl->ana_log_size = 0;
 967 }