drivers/net/ethernet/google/gve/gve_main.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2 /* Google virtual Ethernet (gve) driver
   3  *
   4  * Copyright (C) 2015-2019 Google, Inc.
   5  */
   6
   7 #include <linux/cpumask.h>
   8 #include <linux/etherdevice.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/module.h>
  11 #include <linux/pci.h>
  12 #include <linux/sched.h>
  13 #include <linux/timer.h>
  14 #include <linux/workqueue.h>
  15 #include <net/sch_generic.h>
  16 #include "gve.h"
  17 #include "gve_adminq.h"
  18 #include "gve_register.h"
  19
  20 #define GVE_DEFAULT_RX_COPYBREAK        (256)
  21
  22 #define DEFAULT_MSG_LEVEL       (NETIF_MSG_DRV | NETIF_MSG_LINK)
  23 #define GVE_VERSION             "1.0.0"
  24 #define GVE_VERSION_PREFIX      "GVE-"
  25
  26 const char gve_version_str[] = GVE_VERSION;
  27 static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
  28
  29 static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s)
  30 {
  31         struct gve_priv *priv = netdev_priv(dev);
  32         unsigned int start;
  33         int ring;
  34
  35         if (priv->rx) {
  36                 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) {
  37                         do {
  38                                 start =
  39                                   u64_stats_fetch_begin(&priv->rx[ring].statss);
  40                                 s->rx_packets += priv->rx[ring].rpackets;
  41                                 s->rx_bytes += priv->rx[ring].rbytes;
  42                         } while (u64_stats_fetch_retry(&priv->rx[ring].statss,
  43                                                        start));
  44                 }
  45         }
  46         if (priv->tx) {
  47                 for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) {
  48                         do {
  49                                 start =
  50                                   u64_stats_fetch_begin(&priv->tx[ring].statss);
  51                                 s->tx_packets += priv->tx[ring].pkt_done;
  52                                 s->tx_bytes += priv->tx[ring].bytes_done;
  53                         } while (u64_stats_fetch_retry(&priv->tx[ring].statss,
  54                                                        start));
  55                 }
  56         }
  57 }
  58
  59 static int gve_alloc_counter_array(struct gve_priv *priv)
  60 {
  61         priv->counter_array =
  62                 dma_alloc_coherent(&priv->pdev->dev,
  63                                    priv->num_event_counters *
  64                                    sizeof(*priv->counter_array),
  65                                    &priv->counter_array_bus, GFP_KERNEL);
  66         if (!priv->counter_array)
  67                 return -ENOMEM;
  68
  69         return 0;
  70 }
  71
  72 static void gve_free_counter_array(struct gve_priv *priv)
  73 {
  74         dma_free_coherent(&priv->pdev->dev,
  75                           priv->num_event_counters *
  76                           sizeof(*priv->counter_array),
  77                           priv->counter_array, priv->counter_array_bus);
  78         priv->counter_array = NULL;
  79 }
  80
  81 static irqreturn_t gve_mgmnt_intr(int irq, void *arg)
  82 {
  83         struct gve_priv *priv = arg;
  84
  85         queue_work(priv->gve_wq, &priv->service_task);
  86         return IRQ_HANDLED;
  87 }
  88
  89 static irqreturn_t gve_intr(int irq, void *arg)
  90 {
  91         struct gve_notify_block *block = arg;
  92         struct gve_priv *priv = block->priv;
  93
  94         iowrite32be(GVE_IRQ_MASK, gve_irq_doorbell(priv, block));
  95         napi_schedule_irqoff(&block->napi);
  96         return IRQ_HANDLED;
  97 }
  98
  99 static int gve_napi_poll(struct napi_struct *napi, int budget)
 100 {
 101         struct gve_notify_block *block;
 102         __be32 __iomem *irq_doorbell;
 103         bool reschedule = false;
 104         struct gve_priv *priv;
 105
 106         block = container_of(napi, struct gve_notify_block, napi);
 107         priv = block->priv;
 108
 109         if (block->tx)
 110                 reschedule |= gve_tx_poll(block, budget);
 111         if (block->rx)
 112                 reschedule |= gve_rx_poll(block, budget);
 113
 114         if (reschedule)
 115                 return budget;
 116
 117         napi_complete(napi);
 118         irq_doorbell = gve_irq_doorbell(priv, block);
 119         iowrite32be(GVE_IRQ_ACK | GVE_IRQ_EVENT, irq_doorbell);
 120
 121         /* Double check we have no extra work.
 122          * Ensure unmask synchronizes with checking for work.
 123          */
 124         dma_rmb();
 125         if (block->tx)
 126                 reschedule |= gve_tx_poll(block, -1);
 127         if (block->rx)
 128                 reschedule |= gve_rx_poll(block, -1);
 129         if (reschedule && napi_reschedule(napi))
 130                 iowrite32be(GVE_IRQ_MASK, irq_doorbell);
 131
 132         return 0;
 133 }
 134
 135 static int gve_alloc_notify_blocks(struct gve_priv *priv)
 136 {
 137         int num_vecs_requested = priv->num_ntfy_blks + 1;
 138         char *name = priv->dev->name;
 139         unsigned int active_cpus;
 140         int vecs_enabled;
 141         int i, j;
 142         int err;
 143
 144         priv->msix_vectors = kvzalloc(num_vecs_requested *
 145                                       sizeof(*priv->msix_vectors), GFP_KERNEL);
 146         if (!priv->msix_vectors)
 147                 return -ENOMEM;
 148         for (i = 0; i < num_vecs_requested; i++)
 149                 priv->msix_vectors[i].entry = i;
 150         vecs_enabled = pci_enable_msix_range(priv->pdev, priv->msix_vectors,
 151                                              GVE_MIN_MSIX, num_vecs_requested);
 152         if (vecs_enabled < 0) {
 153                 dev_err(&priv->pdev->dev, "Could not enable min msix %d/%d\n",
 154                         GVE_MIN_MSIX, vecs_enabled);
 155                 err = vecs_enabled;
 156                 goto abort_with_msix_vectors;
 157         }
 158         if (vecs_enabled != num_vecs_requested) {
 159                 int new_num_ntfy_blks = (vecs_enabled - 1) & ~0x1;
 160                 int vecs_per_type = new_num_ntfy_blks / 2;
 161                 int vecs_left = new_num_ntfy_blks % 2;
 162
 163                 priv->num_ntfy_blks = new_num_ntfy_blks;
 164                 priv->tx_cfg.max_queues = min_t(int, priv->tx_cfg.max_queues,
 165                                                 vecs_per_type);
 166                 priv->rx_cfg.max_queues = min_t(int, priv->rx_cfg.max_queues,
 167                                                 vecs_per_type + vecs_left);
 168                 dev_err(&priv->pdev->dev,
 169                         "Could not enable desired msix, only enabled %d, adjusting tx max queues to %d, and rx max queues to %d\n",
 170                         vecs_enabled, priv->tx_cfg.max_queues,
 171                         priv->rx_cfg.max_queues);
 172                 if (priv->tx_cfg.num_queues > priv->tx_cfg.max_queues)
 173                         priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
 174                 if (priv->rx_cfg.num_queues > priv->rx_cfg.max_queues)
 175                         priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
 176         }
 177         /* Half the notification blocks go to TX and half to RX */
 178         active_cpus = min_t(int, priv->num_ntfy_blks / 2, num_online_cpus());
 179
 180         /* Setup Management Vector  - the last vector */
 181         snprintf(priv->mgmt_msix_name, sizeof(priv->mgmt_msix_name), "%s-mgmnt",
 182                  name);
 183         err = request_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector,
 184                           gve_mgmnt_intr, 0, priv->mgmt_msix_name, priv);
 185         if (err) {
 186                 dev_err(&priv->pdev->dev, "Did not receive management vector.\n");
 187                 goto abort_with_msix_enabled;
 188         }
 189         priv->ntfy_blocks =
 190                 dma_alloc_coherent(&priv->pdev->dev,
 191                                    priv->num_ntfy_blks *
 192                                    sizeof(*priv->ntfy_blocks),
 193                                    &priv->ntfy_block_bus, GFP_KERNEL);
 194         if (!priv->ntfy_blocks) {
 195                 err = -ENOMEM;
 196                 goto abort_with_mgmt_vector;
 197         }
 198         /* Setup the other blocks - the first n-1 vectors */
 199         for (i = 0; i < priv->num_ntfy_blks; i++) {
 200                 struct gve_notify_block *block = &priv->ntfy_blocks[i];
 201                 int msix_idx = i;
 202
 203                 snprintf(block->name, sizeof(block->name), "%s-ntfy-block.%d",
 204                          name, i);
 205                 block->priv = priv;
 206                 err = request_irq(priv->msix_vectors[msix_idx].vector,
 207                                   gve_intr, 0, block->name, block);
 208                 if (err) {
 209                         dev_err(&priv->pdev->dev,
 210                                 "Failed to receive msix vector %d\n", i);
 211                         goto abort_with_some_ntfy_blocks;
 212                 }
 213                 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
 214                                       get_cpu_mask(i % active_cpus));
 215         }
 216         return 0;
 217 abort_with_some_ntfy_blocks:
 218         for (j = 0; j < i; j++) {
 219                 struct gve_notify_block *block = &priv->ntfy_blocks[j];
 220                 int msix_idx = j;
 221
 222                 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
 223                                       NULL);
 224                 free_irq(priv->msix_vectors[msix_idx].vector, block);
 225         }
 226         dma_free_coherent(&priv->pdev->dev, priv->num_ntfy_blks *
 227                           sizeof(*priv->ntfy_blocks),
 228                           priv->ntfy_blocks, priv->ntfy_block_bus);
 229         priv->ntfy_blocks = NULL;
 230 abort_with_mgmt_vector:
 231         free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
 232 abort_with_msix_enabled:
 233         pci_disable_msix(priv->pdev);
 234 abort_with_msix_vectors:
 235         kvfree(priv->msix_vectors);
 236         priv->msix_vectors = NULL;
 237         return err;
 238 }
 239
 240 static void gve_free_notify_blocks(struct gve_priv *priv)
 241 {
 242         int i;
 243
 244         /* Free the irqs */
 245         for (i = 0; i < priv->num_ntfy_blks; i++) {
 246                 struct gve_notify_block *block = &priv->ntfy_blocks[i];
 247                 int msix_idx = i;
 248
 249                 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector,
 250                                       NULL);
 251                 free_irq(priv->msix_vectors[msix_idx].vector, block);
 252         }
 253         dma_free_coherent(&priv->pdev->dev,
 254                           priv->num_ntfy_blks * sizeof(*priv->ntfy_blocks),
 255                           priv->ntfy_blocks, priv->ntfy_block_bus);
 256         priv->ntfy_blocks = NULL;
 257         free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv);
 258         pci_disable_msix(priv->pdev);
 259         kvfree(priv->msix_vectors);
 260         priv->msix_vectors = NULL;
 261 }
 262
 263 static int gve_setup_device_resources(struct gve_priv *priv)
 264 {
 265         int err;
 266
 267         err = gve_alloc_counter_array(priv);
 268         if (err)
 269                 return err;
 270         err = gve_alloc_notify_blocks(priv);
 271         if (err)
 272                 goto abort_with_counter;
 273         err = gve_adminq_configure_device_resources(priv,
 274                                                     priv->counter_array_bus,
 275                                                     priv->num_event_counters,
 276                                                     priv->ntfy_block_bus,
 277                                                     priv->num_ntfy_blks);
 278         if (unlikely(err)) {
 279                 dev_err(&priv->pdev->dev,
 280                         "could not setup device_resources: err=%d\n", err);
 281                 err = -ENXIO;
 282                 goto abort_with_ntfy_blocks;
 283         }
 284         gve_set_device_resources_ok(priv);
 285         return 0;
 286 abort_with_ntfy_blocks:
 287         gve_free_notify_blocks(priv);
 288 abort_with_counter:
 289         gve_free_counter_array(priv);
 290         return err;
 291 }
 292
 293 static void gve_trigger_reset(struct gve_priv *priv);
 294
 295 static void gve_teardown_device_resources(struct gve_priv *priv)
 296 {
 297         int err;
 298
 299         /* Tell device its resources are being freed */
 300         if (gve_get_device_resources_ok(priv)) {
 301                 err = gve_adminq_deconfigure_device_resources(priv);
 302                 if (err) {
 303                         dev_err(&priv->pdev->dev,
 304                                 "Could not deconfigure device resources: err=%d\n",
 305                                 err);
 306                         gve_trigger_reset(priv);
 307                 }
 308         }
 309         gve_free_counter_array(priv);
 310         gve_free_notify_blocks(priv);
 311         gve_clear_device_resources_ok(priv);
 312 }
 313
 314 static void gve_add_napi(struct gve_priv *priv, int ntfy_idx)
 315 {
 316         struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 317
 318         netif_napi_add(priv->dev, &block->napi, gve_napi_poll,
 319                        NAPI_POLL_WEIGHT);
 320 }
 321
 322 static void gve_remove_napi(struct gve_priv *priv, int ntfy_idx)
 323 {
 324         struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 325
 326         netif_napi_del(&block->napi);
 327 }
 328
 329 static int gve_register_qpls(struct gve_priv *priv)
 330 {
 331         int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
 332         int err;
 333         int i;
 334
 335         for (i = 0; i < num_qpls; i++) {
 336                 err = gve_adminq_register_page_list(priv, &priv->qpls[i]);
 337                 if (err) {
 338                         netif_err(priv, drv, priv->dev,
 339                                   "failed to register queue page list %d\n",
 340                                   priv->qpls[i].id);
 341                         /* This failure will trigger a reset - no need to clean
 342                          * up
 343                          */
 344                         return err;
 345                 }
 346         }
 347         return 0;
 348 }
 349
 350 static int gve_unregister_qpls(struct gve_priv *priv)
 351 {
 352         int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
 353         int err;
 354         int i;
 355
 356         for (i = 0; i < num_qpls; i++) {
 357                 err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id);
 358                 /* This failure will trigger a reset - no need to clean up */
 359                 if (err) {
 360                         netif_err(priv, drv, priv->dev,
 361                                   "Failed to unregister queue page list %d\n",
 362                                   priv->qpls[i].id);
 363                         return err;
 364                 }
 365         }
 366         return 0;
 367 }
 368
 369 static int gve_create_rings(struct gve_priv *priv)
 370 {
 371         int err;
 372         int i;
 373
 374         for (i = 0; i < priv->tx_cfg.num_queues; i++) {
 375                 err = gve_adminq_create_tx_queue(priv, i);
 376                 if (err) {
 377                         netif_err(priv, drv, priv->dev, "failed to create tx queue %d\n",
 378                                   i);
 379                         /* This failure will trigger a reset - no need to clean
 380                          * up
 381                          */
 382                         return err;
 383                 }
 384                 netif_dbg(priv, drv, priv->dev, "created tx queue %d\n", i);
 385         }
 386         for (i = 0; i < priv->rx_cfg.num_queues; i++) {
 387                 err = gve_adminq_create_rx_queue(priv, i);
 388                 if (err) {
 389                         netif_err(priv, drv, priv->dev, "failed to create rx queue %d\n",
 390                                   i);
 391                         /* This failure will trigger a reset - no need to clean
 392                          * up
 393                          */
 394                         return err;
 395                 }
 396                 /* Rx data ring has been prefilled with packet buffers at
 397                  * queue allocation time.
 398                  * Write the doorbell to provide descriptor slots and packet
 399                  * buffers to the NIC.
 400                  */
 401                 gve_rx_write_doorbell(priv, &priv->rx[i]);
 402                 netif_dbg(priv, drv, priv->dev, "created rx queue %d\n", i);
 403         }
 404
 405         return 0;
 406 }
 407
 408 static int gve_alloc_rings(struct gve_priv *priv)
 409 {
 410         int ntfy_idx;
 411         int err;
 412         int i;
 413
 414         /* Setup tx rings */
 415         priv->tx = kvzalloc(priv->tx_cfg.num_queues * sizeof(*priv->tx),
 416                             GFP_KERNEL);
 417         if (!priv->tx)
 418                 return -ENOMEM;
 419         err = gve_tx_alloc_rings(priv);
 420         if (err)
 421                 goto free_tx;
 422         /* Setup rx rings */
 423         priv->rx = kvzalloc(priv->rx_cfg.num_queues * sizeof(*priv->rx),
 424                             GFP_KERNEL);
 425         if (!priv->rx) {
 426                 err = -ENOMEM;
 427                 goto free_tx_queue;
 428         }
 429         err = gve_rx_alloc_rings(priv);
 430         if (err)
 431                 goto free_rx;
 432         /* Add tx napi & init sync stats*/
 433         for (i = 0; i < priv->tx_cfg.num_queues; i++) {
 434                 u64_stats_init(&priv->tx[i].statss);
 435                 ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
 436                 gve_add_napi(priv, ntfy_idx);
 437         }
 438         /* Add rx napi  & init sync stats*/
 439         for (i = 0; i < priv->rx_cfg.num_queues; i++) {
 440                 u64_stats_init(&priv->rx[i].statss);
 441                 ntfy_idx = gve_rx_idx_to_ntfy(priv, i);
 442                 gve_add_napi(priv, ntfy_idx);
 443         }
 444
 445         return 0;
 446
 447 free_rx:
 448         kvfree(priv->rx);
 449         priv->rx = NULL;
 450 free_tx_queue:
 451         gve_tx_free_rings(priv);
 452 free_tx:
 453         kvfree(priv->tx);
 454         priv->tx = NULL;
 455         return err;
 456 }
 457
 458 static int gve_destroy_rings(struct gve_priv *priv)
 459 {
 460         int err;
 461         int i;
 462
 463         for (i = 0; i < priv->tx_cfg.num_queues; i++) {
 464                 err = gve_adminq_destroy_tx_queue(priv, i);
 465                 if (err) {
 466                         netif_err(priv, drv, priv->dev,
 467                                   "failed to destroy tx queue %d\n",
 468                                   i);
 469                         /* This failure will trigger a reset - no need to clean
 470                          * up
 471                          */
 472                         return err;
 473                 }
 474                 netif_dbg(priv, drv, priv->dev, "destroyed tx queue %d\n", i);
 475         }
 476         for (i = 0; i < priv->rx_cfg.num_queues; i++) {
 477                 err = gve_adminq_destroy_rx_queue(priv, i);
 478                 if (err) {
 479                         netif_err(priv, drv, priv->dev,
 480                                   "failed to destroy rx queue %d\n",
 481                                   i);
 482                         /* This failure will trigger a reset - no need to clean
 483                          * up
 484                          */
 485                         return err;
 486                 }
 487                 netif_dbg(priv, drv, priv->dev, "destroyed rx queue %d\n", i);
 488         }
 489         return 0;
 490 }
 491
 492 static void gve_free_rings(struct gve_priv *priv)
 493 {
 494         int ntfy_idx;
 495         int i;
 496
 497         if (priv->tx) {
 498                 for (i = 0; i < priv->tx_cfg.num_queues; i++) {
 499                         ntfy_idx = gve_tx_idx_to_ntfy(priv, i);
 500                         gve_remove_napi(priv, ntfy_idx);
 501                 }
 502                 gve_tx_free_rings(priv);
 503                 kvfree(priv->tx);
 504                 priv->tx = NULL;
 505         }
 506         if (priv->rx) {
 507                 for (i = 0; i < priv->rx_cfg.num_queues; i++) {
 508                         ntfy_idx = gve_rx_idx_to_ntfy(priv, i);
 509                         gve_remove_napi(priv, ntfy_idx);
 510                 }
 511                 gve_rx_free_rings(priv);
 512                 kvfree(priv->rx);
 513                 priv->rx = NULL;
 514         }
 515 }
 516
 517 int gve_alloc_page(struct device *dev, struct page **page, dma_addr_t *dma,
 518                    enum dma_data_direction dir)
 519 {
 520         *page = alloc_page(GFP_KERNEL);
 521         if (!*page)
 522                 return -ENOMEM;
 523         *dma = dma_map_page(dev, *page, 0, PAGE_SIZE, dir);
 524         if (dma_mapping_error(dev, *dma)) {
 525                 put_page(*page);
 526                 return -ENOMEM;
 527         }
 528         return 0;
 529 }
 530
 531 static int gve_alloc_queue_page_list(struct gve_priv *priv, u32 id,
 532                                      int pages)
 533 {
 534         struct gve_queue_page_list *qpl = &priv->qpls[id];
 535         int err;
 536         int i;
 537
 538         if (pages + priv->num_registered_pages > priv->max_registered_pages) {
 539                 netif_err(priv, drv, priv->dev,
 540                           "Reached max number of registered pages %llu > %llu\n",
 541                           pages + priv->num_registered_pages,
 542                           priv->max_registered_pages);
 543                 return -EINVAL;
 544         }
 545
 546         qpl->id = id;
 547         qpl->num_entries = 0;
 548         qpl->pages = kvzalloc(pages * sizeof(*qpl->pages), GFP_KERNEL);
 549         /* caller handles clean up */
 550         if (!qpl->pages)
 551                 return -ENOMEM;
 552         qpl->page_buses = kvzalloc(pages * sizeof(*qpl->page_buses),
 553                                    GFP_KERNEL);
 554         /* caller handles clean up */
 555         if (!qpl->page_buses)
 556                 return -ENOMEM;
 557
 558         for (i = 0; i < pages; i++) {
 559                 err = gve_alloc_page(&priv->pdev->dev, &qpl->pages[i],
 560                                      &qpl->page_buses[i],
 561                                      gve_qpl_dma_dir(priv, id));
 562                 /* caller handles clean up */
 563                 if (err)
 564                         return -ENOMEM;
 565                 qpl->num_entries++;
 566         }
 567         priv->num_registered_pages += pages;
 568
 569         return 0;
 570 }
 571
 572 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma,
 573                    enum dma_data_direction dir)
 574 {
 575         if (!dma_mapping_error(dev, dma))
 576                 dma_unmap_page(dev, dma, PAGE_SIZE, dir);
 577         if (page)
 578                 put_page(page);
 579 }
 580
 581 static void gve_free_queue_page_list(struct gve_priv *priv,
 582                                      int id)
 583 {
 584         struct gve_queue_page_list *qpl = &priv->qpls[id];
 585         int i;
 586
 587         if (!qpl->pages)
 588                 return;
 589         if (!qpl->page_buses)
 590                 goto free_pages;
 591
 592         for (i = 0; i < qpl->num_entries; i++)
 593                 gve_free_page(&priv->pdev->dev, qpl->pages[i],
 594                               qpl->page_buses[i], gve_qpl_dma_dir(priv, id));
 595
 596         kvfree(qpl->page_buses);
 597 free_pages:
 598         kvfree(qpl->pages);
 599         priv->num_registered_pages -= qpl->num_entries;
 600 }
 601
 602 static int gve_alloc_qpls(struct gve_priv *priv)
 603 {
 604         int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
 605         int i, j;
 606         int err;
 607
 608         priv->qpls = kvzalloc(num_qpls * sizeof(*priv->qpls), GFP_KERNEL);
 609         if (!priv->qpls)
 610                 return -ENOMEM;
 611
 612         for (i = 0; i < gve_num_tx_qpls(priv); i++) {
 613                 err = gve_alloc_queue_page_list(priv, i,
 614                                                 priv->tx_pages_per_qpl);
 615                 if (err)
 616                         goto free_qpls;
 617         }
 618         for (; i < num_qpls; i++) {
 619                 err = gve_alloc_queue_page_list(priv, i,
 620                                                 priv->rx_pages_per_qpl);
 621                 if (err)
 622                         goto free_qpls;
 623         }
 624
 625         priv->qpl_cfg.qpl_map_size = BITS_TO_LONGS(num_qpls) *
 626                                      sizeof(unsigned long) * BITS_PER_BYTE;
 627         priv->qpl_cfg.qpl_id_map = kvzalloc(BITS_TO_LONGS(num_qpls) *
 628                                             sizeof(unsigned long), GFP_KERNEL);
 629         if (!priv->qpl_cfg.qpl_id_map) {
 630                 err = -ENOMEM;
 631                 goto free_qpls;
 632         }
 633
 634         return 0;
 635
 636 free_qpls:
 637         for (j = 0; j <= i; j++)
 638                 gve_free_queue_page_list(priv, j);
 639         kvfree(priv->qpls);
 640         return err;
 641 }
 642
 643 static void gve_free_qpls(struct gve_priv *priv)
 644 {
 645         int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv);
 646         int i;
 647
 648         kvfree(priv->qpl_cfg.qpl_id_map);
 649
 650         for (i = 0; i < num_qpls; i++)
 651                 gve_free_queue_page_list(priv, i);
 652
 653         kvfree(priv->qpls);
 654 }
 655
 656 /* Use this to schedule a reset when the device is capable of continuing
 657  * to handle other requests in its current state. If it is not, do a reset
 658  * in thread instead.
 659  */
 660 void gve_schedule_reset(struct gve_priv *priv)
 661 {
 662         gve_set_do_reset(priv);
 663         queue_work(priv->gve_wq, &priv->service_task);
 664 }
 665
 666 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up);
 667 static int gve_reset_recovery(struct gve_priv *priv, bool was_up);
 668 static void gve_turndown(struct gve_priv *priv);
 669 static void gve_turnup(struct gve_priv *priv);
 670
 671 static int gve_open(struct net_device *dev)
 672 {
 673         struct gve_priv *priv = netdev_priv(dev);
 674         int err;
 675
 676         err = gve_alloc_qpls(priv);
 677         if (err)
 678                 return err;
 679         err = gve_alloc_rings(priv);
 680         if (err)
 681                 goto free_qpls;
 682
 683         err = netif_set_real_num_tx_queues(dev, priv->tx_cfg.num_queues);
 684         if (err)
 685                 goto free_rings;
 686         err = netif_set_real_num_rx_queues(dev, priv->rx_cfg.num_queues);
 687         if (err)
 688                 goto free_rings;
 689
 690         err = gve_register_qpls(priv);
 691         if (err)
 692                 goto reset;
 693         err = gve_create_rings(priv);
 694         if (err)
 695                 goto reset;
 696         gve_set_device_rings_ok(priv);
 697
 698         gve_turnup(priv);
 699         netif_carrier_on(dev);
 700         return 0;
 701
 702 free_rings:
 703         gve_free_rings(priv);
 704 free_qpls:
 705         gve_free_qpls(priv);
 706         return err;
 707
 708 reset:
 709         /* This must have been called from a reset due to the rtnl lock
 710          * so just return at this point.
 711          */
 712         if (gve_get_reset_in_progress(priv))
 713                 return err;
 714         /* Otherwise reset before returning */
 715         gve_reset_and_teardown(priv, true);
 716         /* if this fails there is nothing we can do so just ignore the return */
 717         gve_reset_recovery(priv, false);
 718         /* return the original error */
 719         return err;
 720 }
 721
 722 static int gve_close(struct net_device *dev)
 723 {
 724         struct gve_priv *priv = netdev_priv(dev);
 725         int err;
 726
 727         netif_carrier_off(dev);
 728         if (gve_get_device_rings_ok(priv)) {
 729                 gve_turndown(priv);
 730                 err = gve_destroy_rings(priv);
 731                 if (err)
 732                         goto err;
 733                 err = gve_unregister_qpls(priv);
 734                 if (err)
 735                         goto err;
 736                 gve_clear_device_rings_ok(priv);
 737         }
 738
 739         gve_free_rings(priv);
 740         gve_free_qpls(priv);
 741         return 0;
 742
 743 err:
 744         /* This must have been called from a reset due to the rtnl lock
 745          * so just return at this point.
 746          */
 747         if (gve_get_reset_in_progress(priv))
 748                 return err;
 749         /* Otherwise reset before returning */
 750         gve_reset_and_teardown(priv, true);
 751         return gve_reset_recovery(priv, false);
 752 }
 753
 754 int gve_adjust_queues(struct gve_priv *priv,
 755                       struct gve_queue_config new_rx_config,
 756                       struct gve_queue_config new_tx_config)
 757 {
 758         int err;
 759
 760         if (netif_carrier_ok(priv->dev)) {
 761                 /* To make this process as simple as possible we teardown the
 762                  * device, set the new configuration, and then bring the device
 763                  * up again.
 764                  */
 765                 err = gve_close(priv->dev);
 766                 /* we have already tried to reset in close,
 767                  * just fail at this point
 768                  */
 769                 if (err)
 770                         return err;
 771                 priv->tx_cfg = new_tx_config;
 772                 priv->rx_cfg = new_rx_config;
 773
 774                 err = gve_open(priv->dev);
 775                 if (err)
 776                         goto err;
 777
 778                 return 0;
 779         }
 780         /* Set the config for the next up. */
 781         priv->tx_cfg = new_tx_config;
 782         priv->rx_cfg = new_rx_config;
 783
 784         return 0;
 785 err:
 786         netif_err(priv, drv, priv->dev,
 787                   "Adjust queues failed! !!! DISABLING ALL QUEUES !!!\n");
 788         gve_turndown(priv);
 789         return err;
 790 }
 791
 792 static void gve_turndown(struct gve_priv *priv)
 793 {
 794         int idx;
 795
 796         if (netif_carrier_ok(priv->dev))
 797                 netif_carrier_off(priv->dev);
 798
 799         if (!gve_get_napi_enabled(priv))
 800                 return;
 801
 802         /* Disable napi to prevent more work from coming in */
 803         for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
 804                 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
 805                 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 806
 807                 napi_disable(&block->napi);
 808         }
 809         for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
 810                 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
 811                 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 812
 813                 napi_disable(&block->napi);
 814         }
 815
 816         /* Stop tx queues */
 817         netif_tx_disable(priv->dev);
 818
 819         gve_clear_napi_enabled(priv);
 820 }
 821
 822 static void gve_turnup(struct gve_priv *priv)
 823 {
 824         int idx;
 825
 826         /* Start the tx queues */
 827         netif_tx_start_all_queues(priv->dev);
 828
 829         /* Enable napi and unmask interrupts for all queues */
 830         for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
 831                 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
 832                 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 833
 834                 napi_enable(&block->napi);
 835                 iowrite32be(0, gve_irq_doorbell(priv, block));
 836         }
 837         for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
 838                 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
 839                 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 840
 841                 napi_enable(&block->napi);
 842                 iowrite32be(0, gve_irq_doorbell(priv, block));
 843         }
 844
 845         gve_set_napi_enabled(priv);
 846 }
 847
 848 static void gve_tx_timeout(struct net_device *dev)
 849 {
 850         struct gve_priv *priv = netdev_priv(dev);
 851
 852         gve_schedule_reset(priv);
 853         priv->tx_timeo_cnt++;
 854 }
 855
 856 static const struct net_device_ops gve_netdev_ops = {
 857         .ndo_start_xmit         =       gve_tx,
 858         .ndo_open               =       gve_open,
 859         .ndo_stop               =       gve_close,
 860         .ndo_get_stats64        =       gve_get_stats,
 861         .ndo_tx_timeout         =       gve_tx_timeout,
 862 };
 863
 864 static void gve_handle_status(struct gve_priv *priv, u32 status)
 865 {
 866         if (GVE_DEVICE_STATUS_RESET_MASK & status) {
 867                 dev_info(&priv->pdev->dev, "Device requested reset.\n");
 868                 gve_set_do_reset(priv);
 869         }
 870 }
 871
 872 static void gve_handle_reset(struct gve_priv *priv)
 873 {
 874         /* A service task will be scheduled at the end of probe to catch any
 875          * resets that need to happen, and we don't want to reset until
 876          * probe is done.
 877          */
 878         if (gve_get_probe_in_progress(priv))
 879                 return;
 880
 881         if (gve_get_do_reset(priv)) {
 882                 rtnl_lock();
 883                 gve_reset(priv, false);
 884                 rtnl_unlock();
 885         }
 886 }
 887
 888 /* Handle NIC status register changes and reset requests */
 889 static void gve_service_task(struct work_struct *work)
 890 {
 891         struct gve_priv *priv = container_of(work, struct gve_priv,
 892                                              service_task);
 893
 894         gve_handle_status(priv,
 895                           ioread32be(&priv->reg_bar0->device_status));
 896
 897         gve_handle_reset(priv);
 898 }
 899
 900 static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device)
 901 {
 902         int num_ntfy;
 903         int err;
 904
 905         /* Set up the adminq */
 906         err = gve_adminq_alloc(&priv->pdev->dev, priv);
 907         if (err) {
 908                 dev_err(&priv->pdev->dev,
 909                         "Failed to alloc admin queue: err=%d\n", err);
 910                 return err;
 911         }
 912
 913         if (skip_describe_device)
 914                 goto setup_device;
 915
 916         /* Get the initial information we need from the device */
 917         err = gve_adminq_describe_device(priv);
 918         if (err) {
 919                 dev_err(&priv->pdev->dev,
 920                         "Could not get device information: err=%d\n", err);
 921                 goto err;
 922         }
 923         if (priv->dev->max_mtu > PAGE_SIZE) {
 924                 priv->dev->max_mtu = PAGE_SIZE;
 925                 err = gve_adminq_set_mtu(priv, priv->dev->mtu);
 926                 if (err) {
 927                         netif_err(priv, drv, priv->dev, "Could not set mtu");
 928                         goto err;
 929                 }
 930         }
 931         priv->dev->mtu = priv->dev->max_mtu;
 932         num_ntfy = pci_msix_vec_count(priv->pdev);
 933         if (num_ntfy <= 0) {
 934                 dev_err(&priv->pdev->dev,
 935                         "could not count MSI-x vectors: err=%d\n", num_ntfy);
 936                 err = num_ntfy;
 937                 goto err;
 938         } else if (num_ntfy < GVE_MIN_MSIX) {
 939                 dev_err(&priv->pdev->dev, "gve needs at least %d MSI-x vectors, but only has %d\n",
 940                         GVE_MIN_MSIX, num_ntfy);
 941                 err = -EINVAL;
 942                 goto err;
 943         }
 944
 945         priv->num_registered_pages = 0;
 946         priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
 947         /* gvnic has one Notification Block per MSI-x vector, except for the
 948          * management vector
 949          */
 950         priv->num_ntfy_blks = (num_ntfy - 1) & ~0x1;
 951         priv->mgmt_msix_idx = priv->num_ntfy_blks;
 952
 953         priv->tx_cfg.max_queues =
 954                 min_t(int, priv->tx_cfg.max_queues, priv->num_ntfy_blks / 2);
 955         priv->rx_cfg.max_queues =
 956                 min_t(int, priv->rx_cfg.max_queues, priv->num_ntfy_blks / 2);
 957
 958         priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
 959         priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
 960         if (priv->default_num_queues > 0) {
 961                 priv->tx_cfg.num_queues = min_t(int, priv->default_num_queues,
 962                                                 priv->tx_cfg.num_queues);
 963                 priv->rx_cfg.num_queues = min_t(int, priv->default_num_queues,
 964                                                 priv->rx_cfg.num_queues);
 965         }
 966
 967         netif_info(priv, drv, priv->dev, "TX queues %d, RX queues %d\n",
 968                    priv->tx_cfg.num_queues, priv->rx_cfg.num_queues);
 969         netif_info(priv, drv, priv->dev, "Max TX queues %d, Max RX queues %d\n",
 970                    priv->tx_cfg.max_queues, priv->rx_cfg.max_queues);
 971
 972 setup_device:
 973         err = gve_setup_device_resources(priv);
 974         if (!err)
 975                 return 0;
 976 err:
 977         gve_adminq_free(&priv->pdev->dev, priv);
 978         return err;
 979 }
 980
 981 static void gve_teardown_priv_resources(struct gve_priv *priv)
 982 {
 983         gve_teardown_device_resources(priv);
 984         gve_adminq_free(&priv->pdev->dev, priv);
 985 }
 986
 987 static void gve_trigger_reset(struct gve_priv *priv)
 988 {
 989         /* Reset the device by releasing the AQ */
 990         gve_adminq_release(priv);
 991 }
 992
 993 static void gve_reset_and_teardown(struct gve_priv *priv, bool was_up)
 994 {
 995         gve_trigger_reset(priv);
 996         /* With the reset having already happened, close cannot fail */
 997         if (was_up)
 998                 gve_close(priv->dev);
 999         gve_teardown_priv_resources(priv);
1000 }
1001
1002 static int gve_reset_recovery(struct gve_priv *priv, bool was_up)
1003 {
1004         int err;
1005
1006         err = gve_init_priv(priv, true);
1007         if (err)
1008                 goto err;
1009         if (was_up) {
1010                 err = gve_open(priv->dev);
1011                 if (err)
1012                         goto err;
1013         }
1014         return 0;
1015 err:
1016         dev_err(&priv->pdev->dev, "Reset failed! !!! DISABLING ALL QUEUES !!!\n");
1017         gve_turndown(priv);
1018         return err;
1019 }
1020
1021 int gve_reset(struct gve_priv *priv, bool attempt_teardown)
1022 {
1023         bool was_up = netif_carrier_ok(priv->dev);
1024         int err;
1025
1026         dev_info(&priv->pdev->dev, "Performing reset\n");
1027         gve_clear_do_reset(priv);
1028         gve_set_reset_in_progress(priv);
1029         /* If we aren't attempting to teardown normally, just go turndown and
1030          * reset right away.
1031          */
1032         if (!attempt_teardown) {
1033                 gve_turndown(priv);
1034                 gve_reset_and_teardown(priv, was_up);
1035         } else {
1036                 /* Otherwise attempt to close normally */
1037                 if (was_up) {
1038                         err = gve_close(priv->dev);
1039                         /* If that fails reset as we did above */
1040                         if (err)
1041                                 gve_reset_and_teardown(priv, was_up);
1042                 }
1043                 /* Clean up any remaining resources */
1044                 gve_teardown_priv_resources(priv);
1045         }
1046
1047         /* Set it all back up */
1048         err = gve_reset_recovery(priv, was_up);
1049         gve_clear_reset_in_progress(priv);
1050         return err;
1051 }
1052
1053 static void gve_write_version(u8 __iomem *driver_version_register)
1054 {
1055         const char *c = gve_version_prefix;
1056
1057         while (*c) {
1058                 writeb(*c, driver_version_register);
1059                 c++;
1060         }
1061
1062         c = gve_version_str;
1063         while (*c) {
1064                 writeb(*c, driver_version_register);
1065                 c++;
1066         }
1067         writeb('\n', driver_version_register);
1068 }
1069
1070 static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1071 {
1072         int max_tx_queues, max_rx_queues;
1073         struct net_device *dev;
1074         __be32 __iomem *db_bar;
1075         struct gve_registers __iomem *reg_bar;
1076         struct gve_priv *priv;
1077         int err;
1078
1079         err = pci_enable_device(pdev);
1080         if (err)
1081                 return -ENXIO;
1082
1083         err = pci_request_regions(pdev, "gvnic-cfg");
1084         if (err)
1085                 goto abort_with_enabled;
1086
1087         pci_set_master(pdev);
1088
1089         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1090         if (err) {
1091                 dev_err(&pdev->dev, "Failed to set dma mask: err=%d\n", err);
1092                 goto abort_with_pci_region;
1093         }
1094
1095         err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1096         if (err) {
1097                 dev_err(&pdev->dev,
1098                         "Failed to set consistent dma mask: err=%d\n", err);
1099                 goto abort_with_pci_region;
1100         }
1101
1102         reg_bar = pci_iomap(pdev, GVE_REGISTER_BAR, 0);
1103         if (!reg_bar) {
1104                 dev_err(&pdev->dev, "Failed to map pci bar!\n");
1105                 err = -ENOMEM;
1106                 goto abort_with_pci_region;
1107         }
1108
1109         db_bar = pci_iomap(pdev, GVE_DOORBELL_BAR, 0);
1110         if (!db_bar) {
1111                 dev_err(&pdev->dev, "Failed to map doorbell bar!\n");
1112                 err = -ENOMEM;
1113                 goto abort_with_reg_bar;
1114         }
1115
1116         gve_write_version(&reg_bar->driver_version);
1117         /* Get max queues to alloc etherdev */
1118         max_rx_queues = ioread32be(&reg_bar->max_tx_queues);
1119         max_tx_queues = ioread32be(&reg_bar->max_rx_queues);
1120         /* Alloc and setup the netdev and priv */
1121         dev = alloc_etherdev_mqs(sizeof(*priv), max_tx_queues, max_rx_queues);
1122         if (!dev) {
1123                 dev_err(&pdev->dev, "could not allocate netdev\n");
1124                 goto abort_with_db_bar;
1125         }
1126         SET_NETDEV_DEV(dev, &pdev->dev);
1127         pci_set_drvdata(pdev, dev);
1128         dev->ethtool_ops = &gve_ethtool_ops;
1129         dev->netdev_ops = &gve_netdev_ops;
1130         /* advertise features */
1131         dev->hw_features = NETIF_F_HIGHDMA;
1132         dev->hw_features |= NETIF_F_SG;
1133         dev->hw_features |= NETIF_F_HW_CSUM;
1134         dev->hw_features |= NETIF_F_TSO;
1135         dev->hw_features |= NETIF_F_TSO6;
1136         dev->hw_features |= NETIF_F_TSO_ECN;
1137         dev->hw_features |= NETIF_F_RXCSUM;
1138         dev->hw_features |= NETIF_F_RXHASH;
1139         dev->features = dev->hw_features;
1140         dev->watchdog_timeo = 5 * HZ;
1141         dev->min_mtu = ETH_MIN_MTU;
1142         netif_carrier_off(dev);
1143
1144         priv = netdev_priv(dev);
1145         priv->dev = dev;
1146         priv->pdev = pdev;
1147         priv->msg_enable = DEFAULT_MSG_LEVEL;
1148         priv->reg_bar0 = reg_bar;
1149         priv->db_bar2 = db_bar;
1150         priv->service_task_flags = 0x0;
1151         priv->state_flags = 0x0;
1152
1153         gve_set_probe_in_progress(priv);
1154         priv->gve_wq = alloc_ordered_workqueue("gve", 0);
1155         if (!priv->gve_wq) {
1156                 dev_err(&pdev->dev, "Could not allocate workqueue");
1157                 err = -ENOMEM;
1158                 goto abort_with_netdev;
1159         }
1160         INIT_WORK(&priv->service_task, gve_service_task);
1161         priv->tx_cfg.max_queues = max_tx_queues;
1162         priv->rx_cfg.max_queues = max_rx_queues;
1163
1164         err = gve_init_priv(priv, false);
1165         if (err)
1166                 goto abort_with_wq;
1167
1168         err = register_netdev(dev);
1169         if (err)
1170                 goto abort_with_wq;
1171
1172         dev_info(&pdev->dev, "GVE version %s\n", gve_version_str);
1173         gve_clear_probe_in_progress(priv);
1174         queue_work(priv->gve_wq, &priv->service_task);
1175         return 0;
1176
1177 abort_with_wq:
1178         destroy_workqueue(priv->gve_wq);
1179
1180 abort_with_netdev:
1181         free_netdev(dev);
1182
1183 abort_with_db_bar:
1184         pci_iounmap(pdev, db_bar);
1185
1186 abort_with_reg_bar:
1187         pci_iounmap(pdev, reg_bar);
1188
1189 abort_with_pci_region:
1190         pci_release_regions(pdev);
1191
1192 abort_with_enabled:
1193         pci_disable_device(pdev);
1194         return -ENXIO;
1195 }
1196
1197 static void gve_remove(struct pci_dev *pdev)
1198 {
1199         struct net_device *netdev = pci_get_drvdata(pdev);
1200         struct gve_priv *priv = netdev_priv(netdev);
1201         __be32 __iomem *db_bar = priv->db_bar2;
1202         void __iomem *reg_bar = priv->reg_bar0;
1203
1204         unregister_netdev(netdev);
1205         gve_teardown_priv_resources(priv);
1206         destroy_workqueue(priv->gve_wq);
1207         free_netdev(netdev);
1208         pci_iounmap(pdev, db_bar);
1209         pci_iounmap(pdev, reg_bar);
1210         pci_release_regions(pdev);
1211         pci_disable_device(pdev);
1212 }
1213
1214 static const struct pci_device_id gve_id_table[] = {
1215         { PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) },
1216         { }
1217 };
1218
1219 static struct pci_driver gvnic_driver = {
1220         .name           = "gvnic",
1221         .id_table       = gve_id_table,
1222         .probe          = gve_probe,
1223         .remove         = gve_remove,
1224 };
1225
1226 module_pci_driver(gvnic_driver);
1227
1228 MODULE_DEVICE_TABLE(pci, gve_id_table);
1229 MODULE_AUTHOR("Google, Inc.");
1230 MODULE_DESCRIPTION("gVNIC Driver");
1231 MODULE_LICENSE("Dual MIT/GPL");
1232 MODULE_VERSION(GVE_VERSION);