drivers/net/ethernet/amd/xgbe/xgbe-drv.c

   1 /*
   2  * AMD 10Gb Ethernet driver
   3  *
   4  * This file is available to you under your choice of the following two
   5  * licenses:
   6  *
   7  * License 1: GPLv2
   8  *
   9  * Copyright (c) 2014-2016 Advanced Micro Devices, Inc.
  10  *
  11  * This file is free software; you may copy, redistribute and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation, either version 2 of the License, or (at
  14  * your option) any later version.
  15  *
  16  * This file is distributed in the hope that it will be useful, but
  17  * WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  19  * General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License
  22  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  23  *
  24  * This file incorporates work covered by the following copyright and
  25  * permission notice:
  26  *     The Synopsys DWC ETHER XGMAC Software Driver and documentation
  27  *     (hereinafter "Software") is an unsupported proprietary work of Synopsys,
  28  *     Inc. unless otherwise expressly agreed to in writing between Synopsys
  29  *     and you.
  30  *
  31  *     The Software IS NOT an item of Licensed Software or Licensed Product
  32  *     under any End User Software License Agreement or Agreement for Licensed
  33  *     Product with Synopsys or any supplement thereto.  Permission is hereby
  34  *     granted, free of charge, to any person obtaining a copy of this software
  35  *     annotated with this license and the Software, to deal in the Software
  36  *     without restriction, including without limitation the rights to use,
  37  *     copy, modify, merge, publish, distribute, sublicense, and/or sell copies
  38  *     of the Software, and to permit persons to whom the Software is furnished
  39  *     to do so, subject to the following conditions:
  40  *
  41  *     The above copyright notice and this permission notice shall be included
  42  *     in all copies or substantial portions of the Software.
  43  *
  44  *     THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS"
  45  *     BASIS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  46  *     TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
  47  *     PARTICULAR PURPOSE ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS
  48  *     BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  49  *     CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  50  *     SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  51  *     INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  52  *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  53  *     ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  54  *     THE POSSIBILITY OF SUCH DAMAGE.
  55  *
  56  *
  57  * License 2: Modified BSD
  58  *
  59  * Copyright (c) 2014-2016 Advanced Micro Devices, Inc.
  60  * All rights reserved.
  61  *
  62  * Redistribution and use in source and binary forms, with or without
  63  * modification, are permitted provided that the following conditions are met:
  64  *     * Redistributions of source code must retain the above copyright
  65  *       notice, this list of conditions and the following disclaimer.
  66  *     * Redistributions in binary form must reproduce the above copyright
  67  *       notice, this list of conditions and the following disclaimer in the
  68  *       documentation and/or other materials provided with the distribution.
  69  *     * Neither the name of Advanced Micro Devices, Inc. nor the
  70  *       names of its contributors may be used to endorse or promote products
  71  *       derived from this software without specific prior written permission.
  72  *
  73  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  74  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  75  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  76  * ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
  77  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  78  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  79  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  80  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  81  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  82  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  83  *
  84  * This file incorporates work covered by the following copyright and
  85  * permission notice:
  86  *     The Synopsys DWC ETHER XGMAC Software Driver and documentation
  87  *     (hereinafter "Software") is an unsupported proprietary work of Synopsys,
  88  *     Inc. unless otherwise expressly agreed to in writing between Synopsys
  89  *     and you.
  90  *
  91  *     The Software IS NOT an item of Licensed Software or Licensed Product
  92  *     under any End User Software License Agreement or Agreement for Licensed
  93  *     Product with Synopsys or any supplement thereto.  Permission is hereby
  94  *     granted, free of charge, to any person obtaining a copy of this software
  95  *     annotated with this license and the Software, to deal in the Software
  96  *     without restriction, including without limitation the rights to use,
  97  *     copy, modify, merge, publish, distribute, sublicense, and/or sell copies
  98  *     of the Software, and to permit persons to whom the Software is furnished
  99  *     to do so, subject to the following conditions:
 100  *
 101  *     The above copyright notice and this permission notice shall be included
 102  *     in all copies or substantial portions of the Software.
 103  *
 104  *     THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS"
 105  *     BASIS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 106  *     TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
 107  *     PARTICULAR PURPOSE ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS
 108  *     BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 109  *     CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 110  *     SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 111  *     INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 112  *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 113  *     ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 114  *     THE POSSIBILITY OF SUCH DAMAGE.
 115  */
 116
 117 #include <linux/module.h>
 118 #include <linux/spinlock.h>
 119 #include <linux/tcp.h>
 120 #include <linux/if_vlan.h>
 121 #include <linux/interrupt.h>
 122 #include <net/busy_poll.h>
 123 #include <linux/clk.h>
 124 #include <linux/if_ether.h>
 125 #include <linux/net_tstamp.h>
 126 #include <linux/phy.h>
 127
 128 #include "xgbe.h"
 129 #include "xgbe-common.h"
 130
 131 static unsigned int ecc_sec_info_threshold = 10;
 132 static unsigned int ecc_sec_warn_threshold = 10000;
 133 static unsigned int ecc_sec_period = 600;
 134 static unsigned int ecc_ded_threshold = 2;
 135 static unsigned int ecc_ded_period = 600;
 136
 137 #ifdef CONFIG_AMD_XGBE_HAVE_ECC
 138 /* Only expose the ECC parameters if supported */
 139 module_param(ecc_sec_info_threshold, uint, S_IWUSR | S_IRUGO);
 140 MODULE_PARM_DESC(ecc_sec_info_threshold,
 141                  " ECC corrected error informational threshold setting");
 142
 143 module_param(ecc_sec_warn_threshold, uint, S_IWUSR | S_IRUGO);
 144 MODULE_PARM_DESC(ecc_sec_warn_threshold,
 145                  " ECC corrected error warning threshold setting");
 146
 147 module_param(ecc_sec_period, uint, S_IWUSR | S_IRUGO);
 148 MODULE_PARM_DESC(ecc_sec_period, " ECC corrected error period (in seconds)");
 149
 150 module_param(ecc_ded_threshold, uint, S_IWUSR | S_IRUGO);
 151 MODULE_PARM_DESC(ecc_ded_threshold, " ECC detected error threshold setting");
 152
 153 module_param(ecc_ded_period, uint, S_IWUSR | S_IRUGO);
 154 MODULE_PARM_DESC(ecc_ded_period, " ECC detected error period (in seconds)");
 155 #endif
 156
 157 static int xgbe_one_poll(struct napi_struct *, int);
 158 static int xgbe_all_poll(struct napi_struct *, int);
 159 static void xgbe_stop(struct xgbe_prv_data *);
 160
 161 static void *xgbe_alloc_node(size_t size, int node)
 162 {
 163         void *mem;
 164
 165         mem = kzalloc_node(size, GFP_KERNEL, node);
 166         if (!mem)
 167                 mem = kzalloc(size, GFP_KERNEL);
 168
 169         return mem;
 170 }
 171
 172 static void xgbe_free_channels(struct xgbe_prv_data *pdata)
 173 {
 174         unsigned int i;
 175
 176         for (i = 0; i < ARRAY_SIZE(pdata->channel); i++) {
 177                 if (!pdata->channel[i])
 178                         continue;
 179
 180                 kfree(pdata->channel[i]->rx_ring);
 181                 kfree(pdata->channel[i]->tx_ring);
 182                 kfree(pdata->channel[i]);
 183
 184                 pdata->channel[i] = NULL;
 185         }
 186
 187         pdata->channel_count = 0;
 188 }
 189
 190 static int xgbe_alloc_channels(struct xgbe_prv_data *pdata)
 191 {
 192         struct xgbe_channel *channel;
 193         struct xgbe_ring *ring;
 194         unsigned int count, i;
 195         unsigned int cpu;
 196         int node;
 197
 198         count = max_t(unsigned int, pdata->tx_ring_count, pdata->rx_ring_count);
 199         for (i = 0; i < count; i++) {
 200                 /* Attempt to use a CPU on the node the device is on */
 201                 cpu = cpumask_local_spread(i, dev_to_node(pdata->dev));
 202
 203                 /* Set the allocation node based on the returned CPU */
 204                 node = cpu_to_node(cpu);
 205
 206                 channel = xgbe_alloc_node(sizeof(*channel), node);
 207                 if (!channel)
 208                         goto err_mem;
 209                 pdata->channel[i] = channel;
 210
 211                 snprintf(channel->name, sizeof(channel->name), "channel-%u", i);
 212                 channel->pdata = pdata;
 213                 channel->queue_index = i;
 214                 channel->dma_regs = pdata->xgmac_regs + DMA_CH_BASE +
 215                                     (DMA_CH_INC * i);
 216                 channel->node = node;
 217                 cpumask_set_cpu(cpu, &channel->affinity_mask);
 218
 219                 if (pdata->per_channel_irq)
 220                         channel->dma_irq = pdata->channel_irq[i];
 221
 222                 if (i < pdata->tx_ring_count) {
 223                         ring = xgbe_alloc_node(sizeof(*ring), node);
 224                         if (!ring)
 225                                 goto err_mem;
 226
 227                         spin_lock_init(&ring->lock);
 228                         ring->node = node;
 229
 230                         channel->tx_ring = ring;
 231                 }
 232
 233                 if (i < pdata->rx_ring_count) {
 234                         ring = xgbe_alloc_node(sizeof(*ring), node);
 235                         if (!ring)
 236                                 goto err_mem;
 237
 238                         spin_lock_init(&ring->lock);
 239                         ring->node = node;
 240
 241                         channel->rx_ring = ring;
 242                 }
 243
 244                 netif_dbg(pdata, drv, pdata->netdev,
 245                           "%s: cpu=%u, node=%d\n", channel->name, cpu, node);
 246
 247                 netif_dbg(pdata, drv, pdata->netdev,
 248                           "%s: dma_regs=%p, dma_irq=%d, tx=%p, rx=%p\n",
 249                           channel->name, channel->dma_regs, channel->dma_irq,
 250                           channel->tx_ring, channel->rx_ring);
 251         }
 252
 253         pdata->channel_count = count;
 254
 255         return 0;
 256
 257 err_mem:
 258         xgbe_free_channels(pdata);
 259
 260         return -ENOMEM;
 261 }
 262
 263 static inline unsigned int xgbe_tx_avail_desc(struct xgbe_ring *ring)
 264 {
 265         return (ring->rdesc_count - (ring->cur - ring->dirty));
 266 }
 267
 268 static inline unsigned int xgbe_rx_dirty_desc(struct xgbe_ring *ring)
 269 {
 270         return (ring->cur - ring->dirty);
 271 }
 272
 273 static int xgbe_maybe_stop_tx_queue(struct xgbe_channel *channel,
 274                                     struct xgbe_ring *ring, unsigned int count)
 275 {
 276         struct xgbe_prv_data *pdata = channel->pdata;
 277
 278         if (count > xgbe_tx_avail_desc(ring)) {
 279                 netif_info(pdata, drv, pdata->netdev,
 280                            "Tx queue stopped, not enough descriptors available\n");
 281                 netif_stop_subqueue(pdata->netdev, channel->queue_index);
 282                 ring->tx.queue_stopped = 1;
 283
 284                 /* If we haven't notified the hardware because of xmit_more
 285                  * support, tell it now
 286                  */
 287                 if (ring->tx.xmit_more)
 288                         pdata->hw_if.tx_start_xmit(channel, ring);
 289
 290                 return NETDEV_TX_BUSY;
 291         }
 292
 293         return 0;
 294 }
 295
 296 static int xgbe_calc_rx_buf_size(struct net_device *netdev, unsigned int mtu)
 297 {
 298         unsigned int rx_buf_size;
 299
 300         rx_buf_size = mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
 301         rx_buf_size = clamp_val(rx_buf_size, XGBE_RX_MIN_BUF_SIZE, PAGE_SIZE);
 302
 303         rx_buf_size = (rx_buf_size + XGBE_RX_BUF_ALIGN - 1) &
 304                       ~(XGBE_RX_BUF_ALIGN - 1);
 305
 306         return rx_buf_size;
 307 }
 308
 309 static void xgbe_enable_rx_tx_int(struct xgbe_prv_data *pdata,
 310                                   struct xgbe_channel *channel)
 311 {
 312         struct xgbe_hw_if *hw_if = &pdata->hw_if;
 313         enum xgbe_int int_id;
 314
 315         if (channel->tx_ring && channel->rx_ring)
 316                 int_id = XGMAC_INT_DMA_CH_SR_TI_RI;
 317         else if (channel->tx_ring)
 318                 int_id = XGMAC_INT_DMA_CH_SR_TI;
 319         else if (channel->rx_ring)
 320                 int_id = XGMAC_INT_DMA_CH_SR_RI;
 321         else
 322                 return;
 323
 324         hw_if->enable_int(channel, int_id);
 325 }
 326
 327 static void xgbe_enable_rx_tx_ints(struct xgbe_prv_data *pdata)
 328 {
 329         unsigned int i;
 330
 331         for (i = 0; i < pdata->channel_count; i++)
 332                 xgbe_enable_rx_tx_int(pdata, pdata->channel[i]);
 333 }
 334
 335 static void xgbe_disable_rx_tx_int(struct xgbe_prv_data *pdata,
 336                                    struct xgbe_channel *channel)
 337 {
 338         struct xgbe_hw_if *hw_if = &pdata->hw_if;
 339         enum xgbe_int int_id;
 340
 341         if (channel->tx_ring && channel->rx_ring)
 342                 int_id = XGMAC_INT_DMA_CH_SR_TI_RI;
 343         else if (channel->tx_ring)
 344                 int_id = XGMAC_INT_DMA_CH_SR_TI;
 345         else if (channel->rx_ring)
 346                 int_id = XGMAC_INT_DMA_CH_SR_RI;
 347         else
 348                 return;
 349
 350         hw_if->disable_int(channel, int_id);
 351 }
 352
 353 static void xgbe_disable_rx_tx_ints(struct xgbe_prv_data *pdata)
 354 {
 355         unsigned int i;
 356
 357         for (i = 0; i < pdata->channel_count; i++)
 358                 xgbe_disable_rx_tx_int(pdata, pdata->channel[i]);
 359 }
 360
 361 static bool xgbe_ecc_sec(struct xgbe_prv_data *pdata, unsigned long *period,
 362                          unsigned int *count, const char *area)
 363 {
 364         if (time_before(jiffies, *period)) {
 365                 (*count)++;
 366         } else {
 367                 *period = jiffies + (ecc_sec_period * HZ);
 368                 *count = 1;
 369         }
 370
 371         if (*count > ecc_sec_info_threshold)
 372                 dev_warn_once(pdata->dev,
 373                               "%s ECC corrected errors exceed informational threshold\n",
 374                               area);
 375
 376         if (*count > ecc_sec_warn_threshold) {
 377                 dev_warn_once(pdata->dev,
 378                               "%s ECC corrected errors exceed warning threshold\n",
 379                               area);
 380                 return true;
 381         }
 382
 383         return false;
 384 }
 385
 386 static bool xgbe_ecc_ded(struct xgbe_prv_data *pdata, unsigned long *period,
 387                          unsigned int *count, const char *area)
 388 {
 389         if (time_before(jiffies, *period)) {
 390                 (*count)++;
 391         } else {
 392                 *period = jiffies + (ecc_ded_period * HZ);
 393                 *count = 1;
 394         }
 395
 396         if (*count > ecc_ded_threshold) {
 397                 netdev_alert(pdata->netdev,
 398                              "%s ECC detected errors exceed threshold\n",
 399                              area);
 400                 return true;
 401         }
 402
 403         return false;
 404 }
 405
 406 static void xgbe_ecc_isr_task(unsigned long data)
 407 {
 408         struct xgbe_prv_data *pdata = (struct xgbe_prv_data *)data;
 409         unsigned int ecc_isr;
 410         bool stop = false;
 411
 412         /* Mask status with only the interrupts we care about */
 413         ecc_isr = XP_IOREAD(pdata, XP_ECC_ISR);
 414         ecc_isr &= XP_IOREAD(pdata, XP_ECC_IER);
 415         netif_dbg(pdata, intr, pdata->netdev, "ECC_ISR=%#010x\n", ecc_isr);
 416
 417         if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, TX_DED)) {
 418                 stop |= xgbe_ecc_ded(pdata, &pdata->tx_ded_period,
 419                                      &pdata->tx_ded_count, "TX fifo");
 420         }
 421
 422         if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, RX_DED)) {
 423                 stop |= xgbe_ecc_ded(pdata, &pdata->rx_ded_period,
 424                                      &pdata->rx_ded_count, "RX fifo");
 425         }
 426
 427         if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, DESC_DED)) {
 428                 stop |= xgbe_ecc_ded(pdata, &pdata->desc_ded_period,
 429                                      &pdata->desc_ded_count,
 430                                      "descriptor cache");
 431         }
 432
 433         if (stop) {
 434                 pdata->hw_if.disable_ecc_ded(pdata);
 435                 schedule_work(&pdata->stopdev_work);
 436                 goto out;
 437         }
 438
 439         if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, TX_SEC)) {
 440                 if (xgbe_ecc_sec(pdata, &pdata->tx_sec_period,
 441                                  &pdata->tx_sec_count, "TX fifo"))
 442                         pdata->hw_if.disable_ecc_sec(pdata, XGBE_ECC_SEC_TX);
 443         }
 444
 445         if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, RX_SEC))
 446                 if (xgbe_ecc_sec(pdata, &pdata->rx_sec_period,
 447                                  &pdata->rx_sec_count, "RX fifo"))
 448                         pdata->hw_if.disable_ecc_sec(pdata, XGBE_ECC_SEC_RX);
 449
 450         if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, DESC_SEC))
 451                 if (xgbe_ecc_sec(pdata, &pdata->desc_sec_period,
 452                                  &pdata->desc_sec_count, "descriptor cache"))
 453                         pdata->hw_if.disable_ecc_sec(pdata, XGBE_ECC_SEC_DESC);
 454
 455 out:
 456         /* Clear all ECC interrupts */
 457         XP_IOWRITE(pdata, XP_ECC_ISR, ecc_isr);
 458
 459         /* Reissue interrupt if status is not clear */
 460         if (pdata->vdata->irq_reissue_support)
 461                 XP_IOWRITE(pdata, XP_INT_REISSUE_EN, 1 << 1);
 462 }
 463
 464 static irqreturn_t xgbe_ecc_isr(int irq, void *data)
 465 {
 466         struct xgbe_prv_data *pdata = data;
 467
 468         if (pdata->isr_as_tasklet)
 469                 tasklet_schedule(&pdata->tasklet_ecc);
 470         else
 471                 xgbe_ecc_isr_task((unsigned long)pdata);
 472
 473         return IRQ_HANDLED;
 474 }
 475
 476 static void xgbe_isr_task(unsigned long data)
 477 {
 478         struct xgbe_prv_data *pdata = (struct xgbe_prv_data *)data;
 479         struct xgbe_hw_if *hw_if = &pdata->hw_if;
 480         struct xgbe_channel *channel;
 481         unsigned int dma_isr, dma_ch_isr;
 482         unsigned int mac_isr, mac_tssr, mac_mdioisr;
 483         unsigned int i;
 484
 485         /* The DMA interrupt status register also reports MAC and MTL
 486          * interrupts. So for polling mode, we just need to check for
 487          * this register to be non-zero
 488          */
 489         dma_isr = XGMAC_IOREAD(pdata, DMA_ISR);
 490         if (!dma_isr)
 491                 goto isr_done;
 492
 493         netif_dbg(pdata, intr, pdata->netdev, "DMA_ISR=%#010x\n", dma_isr);
 494
 495         for (i = 0; i < pdata->channel_count; i++) {
 496                 if (!(dma_isr & (1 << i)))
 497                         continue;
 498
 499                 channel = pdata->channel[i];
 500
 501                 dma_ch_isr = XGMAC_DMA_IOREAD(channel, DMA_CH_SR);
 502                 netif_dbg(pdata, intr, pdata->netdev, "DMA_CH%u_ISR=%#010x\n",
 503                           i, dma_ch_isr);
 504
 505                 /* The TI or RI interrupt bits may still be set even if using
 506                  * per channel DMA interrupts. Check to be sure those are not
 507                  * enabled before using the private data napi structure.
 508                  */
 509                 if (!pdata->per_channel_irq &&
 510                     (XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, TI) ||
 511                      XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, RI))) {
 512                         if (napi_schedule_prep(&pdata->napi)) {
 513                                 /* Disable Tx and Rx interrupts */
 514                                 xgbe_disable_rx_tx_ints(pdata);
 515
 516                                 /* Turn on polling */
 517                                 __napi_schedule_irqoff(&pdata->napi);
 518                         }
 519                 } else {
 520                         /* Don't clear Rx/Tx status if doing per channel DMA
 521                          * interrupts, these will be cleared by the ISR for
 522                          * per channel DMA interrupts.
 523                          */
 524                         XGMAC_SET_BITS(dma_ch_isr, DMA_CH_SR, TI, 0);
 525                         XGMAC_SET_BITS(dma_ch_isr, DMA_CH_SR, RI, 0);
 526                 }
 527
 528                 if (XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, RBU))
 529                         pdata->ext_stats.rx_buffer_unavailable++;
 530
 531                 /* Restart the device on a Fatal Bus Error */
 532                 if (XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, FBE))
 533                         schedule_work(&pdata->restart_work);
 534
 535                 /* Clear interrupt signals */
 536                 XGMAC_DMA_IOWRITE(channel, DMA_CH_SR, dma_ch_isr);
 537         }
 538
 539         if (XGMAC_GET_BITS(dma_isr, DMA_ISR, MACIS)) {
 540                 mac_isr = XGMAC_IOREAD(pdata, MAC_ISR);
 541
 542                 netif_dbg(pdata, intr, pdata->netdev, "MAC_ISR=%#010x\n",
 543                           mac_isr);
 544
 545                 if (XGMAC_GET_BITS(mac_isr, MAC_ISR, MMCTXIS))
 546                         hw_if->tx_mmc_int(pdata);
 547
 548                 if (XGMAC_GET_BITS(mac_isr, MAC_ISR, MMCRXIS))
 549                         hw_if->rx_mmc_int(pdata);
 550
 551                 if (XGMAC_GET_BITS(mac_isr, MAC_ISR, TSIS)) {
 552                         mac_tssr = XGMAC_IOREAD(pdata, MAC_TSSR);
 553
 554                         netif_dbg(pdata, intr, pdata->netdev,
 555                                   "MAC_TSSR=%#010x\n", mac_tssr);
 556
 557                         if (XGMAC_GET_BITS(mac_tssr, MAC_TSSR, TXTSC)) {
 558                                 /* Read Tx Timestamp to clear interrupt */
 559                                 pdata->tx_tstamp =
 560                                         hw_if->get_tx_tstamp(pdata);
 561                                 queue_work(pdata->dev_workqueue,
 562                                            &pdata->tx_tstamp_work);
 563                         }
 564                 }
 565
 566                 if (XGMAC_GET_BITS(mac_isr, MAC_ISR, SMI)) {
 567                         mac_mdioisr = XGMAC_IOREAD(pdata, MAC_MDIOISR);
 568
 569                         netif_dbg(pdata, intr, pdata->netdev,
 570                                   "MAC_MDIOISR=%#010x\n", mac_mdioisr);
 571
 572                         if (XGMAC_GET_BITS(mac_mdioisr, MAC_MDIOISR,
 573                                            SNGLCOMPINT))
 574                                 complete(&pdata->mdio_complete);
 575                 }
 576         }
 577
 578 isr_done:
 579         /* If there is not a separate AN irq, handle it here */
 580         if (pdata->dev_irq == pdata->an_irq)
 581                 pdata->phy_if.an_isr(pdata);
 582
 583         /* If there is not a separate ECC irq, handle it here */
 584         if (pdata->vdata->ecc_support && (pdata->dev_irq == pdata->ecc_irq))
 585                 xgbe_ecc_isr_task((unsigned long)pdata);
 586
 587         /* If there is not a separate I2C irq, handle it here */
 588         if (pdata->vdata->i2c_support && (pdata->dev_irq == pdata->i2c_irq))
 589                 pdata->i2c_if.i2c_isr(pdata);
 590
 591         /* Reissue interrupt if status is not clear */
 592         if (pdata->vdata->irq_reissue_support) {
 593                 unsigned int reissue_mask;
 594
 595                 reissue_mask = 1 << 0;
 596                 if (!pdata->per_channel_irq)
 597                         reissue_mask |= 0xffff < 4;
 598
 599                 XP_IOWRITE(pdata, XP_INT_REISSUE_EN, reissue_mask);
 600         }
 601 }
 602
 603 static irqreturn_t xgbe_isr(int irq, void *data)
 604 {
 605         struct xgbe_prv_data *pdata = data;
 606
 607         if (pdata->isr_as_tasklet)
 608                 tasklet_schedule(&pdata->tasklet_dev);
 609         else
 610                 xgbe_isr_task((unsigned long)pdata);
 611
 612         return IRQ_HANDLED;
 613 }
 614
 615 static irqreturn_t xgbe_dma_isr(int irq, void *data)
 616 {
 617         struct xgbe_channel *channel = data;
 618         struct xgbe_prv_data *pdata = channel->pdata;
 619         unsigned int dma_status;
 620
 621         /* Per channel DMA interrupts are enabled, so we use the per
 622          * channel napi structure and not the private data napi structure
 623          */
 624         if (napi_schedule_prep(&channel->napi)) {
 625                 /* Disable Tx and Rx interrupts */
 626                 if (pdata->channel_irq_mode)
 627                         xgbe_disable_rx_tx_int(pdata, channel);
 628                 else
 629                         disable_irq_nosync(channel->dma_irq);
 630
 631                 /* Turn on polling */
 632                 __napi_schedule_irqoff(&channel->napi);
 633         }
 634
 635         /* Clear Tx/Rx signals */
 636         dma_status = 0;
 637         XGMAC_SET_BITS(dma_status, DMA_CH_SR, TI, 1);
 638         XGMAC_SET_BITS(dma_status, DMA_CH_SR, RI, 1);
 639         XGMAC_DMA_IOWRITE(channel, DMA_CH_SR, dma_status);
 640
 641         return IRQ_HANDLED;
 642 }
 643
 644 static void xgbe_tx_timer(unsigned long data)
 645 {
 646         struct xgbe_channel *channel = (struct xgbe_channel *)data;
 647         struct xgbe_prv_data *pdata = channel->pdata;
 648         struct napi_struct *napi;
 649
 650         DBGPR("-->xgbe_tx_timer\n");
 651
 652         napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
 653
 654         if (napi_schedule_prep(napi)) {
 655                 /* Disable Tx and Rx interrupts */
 656                 if (pdata->per_channel_irq)
 657                         if (pdata->channel_irq_mode)
 658                                 xgbe_disable_rx_tx_int(pdata, channel);
 659                         else
 660                                 disable_irq_nosync(channel->dma_irq);
 661                 else
 662                         xgbe_disable_rx_tx_ints(pdata);
 663
 664                 /* Turn on polling */
 665                 __napi_schedule(napi);
 666         }
 667
 668         channel->tx_timer_active = 0;
 669
 670         DBGPR("<--xgbe_tx_timer\n");
 671 }
 672
 673 static void xgbe_service(struct work_struct *work)
 674 {
 675         struct xgbe_prv_data *pdata = container_of(work,
 676                                                    struct xgbe_prv_data,
 677                                                    service_work);
 678
 679         pdata->phy_if.phy_status(pdata);
 680 }
 681
 682 static void xgbe_service_timer(unsigned long data)
 683 {
 684         struct xgbe_prv_data *pdata = (struct xgbe_prv_data *)data;
 685
 686         queue_work(pdata->dev_workqueue, &pdata->service_work);
 687
 688         mod_timer(&pdata->service_timer, jiffies + HZ);
 689 }
 690
 691 static void xgbe_init_timers(struct xgbe_prv_data *pdata)
 692 {
 693         struct xgbe_channel *channel;
 694         unsigned int i;
 695
 696         setup_timer(&pdata->service_timer, xgbe_service_timer,
 697                     (unsigned long)pdata);
 698
 699         for (i = 0; i < pdata->channel_count; i++) {
 700                 channel = pdata->channel[i];
 701                 if (!channel->tx_ring)
 702                         break;
 703
 704                 setup_timer(&channel->tx_timer, xgbe_tx_timer,
 705                             (unsigned long)channel);
 706         }
 707 }
 708
 709 static void xgbe_start_timers(struct xgbe_prv_data *pdata)
 710 {
 711         mod_timer(&pdata->service_timer, jiffies + HZ);
 712 }
 713
 714 static void xgbe_stop_timers(struct xgbe_prv_data *pdata)
 715 {
 716         struct xgbe_channel *channel;
 717         unsigned int i;
 718
 719         del_timer_sync(&pdata->service_timer);
 720
 721         for (i = 0; i < pdata->channel_count; i++) {
 722                 channel = pdata->channel[i];
 723                 if (!channel->tx_ring)
 724                         break;
 725
 726                 del_timer_sync(&channel->tx_timer);
 727         }
 728 }
 729
 730 void xgbe_get_all_hw_features(struct xgbe_prv_data *pdata)
 731 {
 732         unsigned int mac_hfr0, mac_hfr1, mac_hfr2;
 733         struct xgbe_hw_features *hw_feat = &pdata->hw_feat;
 734
 735         DBGPR("-->xgbe_get_all_hw_features\n");
 736
 737         mac_hfr0 = XGMAC_IOREAD(pdata, MAC_HWF0R);
 738         mac_hfr1 = XGMAC_IOREAD(pdata, MAC_HWF1R);
 739         mac_hfr2 = XGMAC_IOREAD(pdata, MAC_HWF2R);
 740
 741         memset(hw_feat, 0, sizeof(*hw_feat));
 742
 743         hw_feat->version = XGMAC_IOREAD(pdata, MAC_VR);
 744
 745         /* Hardware feature register 0 */
 746         hw_feat->gmii        = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, GMIISEL);
 747         hw_feat->vlhash      = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, VLHASH);
 748         hw_feat->sma         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, SMASEL);
 749         hw_feat->rwk         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, RWKSEL);
 750         hw_feat->mgk         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, MGKSEL);
 751         hw_feat->mmc         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, MMCSEL);
 752         hw_feat->aoe         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, ARPOFFSEL);
 753         hw_feat->ts          = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, TSSEL);
 754         hw_feat->eee         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, EEESEL);
 755         hw_feat->tx_coe      = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, TXCOESEL);
 756         hw_feat->rx_coe      = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, RXCOESEL);
 757         hw_feat->addn_mac    = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R,
 758                                               ADDMACADRSEL);
 759         hw_feat->ts_src      = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, TSSTSSEL);
 760         hw_feat->sa_vlan_ins = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, SAVLANINS);
 761
 762         /* Hardware feature register 1 */
 763         hw_feat->rx_fifo_size  = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
 764                                                 RXFIFOSIZE);
 765         hw_feat->tx_fifo_size  = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
 766                                                 TXFIFOSIZE);
 767         hw_feat->adv_ts_hi     = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, ADVTHWORD);
 768         hw_feat->dma_width     = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, ADDR64);
 769         hw_feat->dcb           = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, DCBEN);
 770         hw_feat->sph           = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, SPHEN);
 771         hw_feat->tso           = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, TSOEN);
 772         hw_feat->dma_debug     = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, DBGMEMA);
 773         hw_feat->rss           = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, RSSEN);
 774         hw_feat->tc_cnt        = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, NUMTC);
 775         hw_feat->hash_table_size = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
 776                                                   HASHTBLSZ);
 777         hw_feat->l3l4_filter_num = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
 778                                                   L3L4FNUM);
 779
 780         /* Hardware feature register 2 */
 781         hw_feat->rx_q_cnt     = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, RXQCNT);
 782         hw_feat->tx_q_cnt     = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, TXQCNT);
 783         hw_feat->rx_ch_cnt    = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, RXCHCNT);
 784         hw_feat->tx_ch_cnt    = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, TXCHCNT);
 785         hw_feat->pps_out_num  = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, PPSOUTNUM);
 786         hw_feat->aux_snap_num = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, AUXSNAPNUM);
 787
 788         /* Translate the Hash Table size into actual number */
 789         switch (hw_feat->hash_table_size) {
 790         case 0:
 791                 break;
 792         case 1:
 793                 hw_feat->hash_table_size = 64;
 794                 break;
 795         case 2:
 796                 hw_feat->hash_table_size = 128;
 797                 break;
 798         case 3:
 799                 hw_feat->hash_table_size = 256;
 800                 break;
 801         }
 802
 803         /* Translate the address width setting into actual number */
 804         switch (hw_feat->dma_width) {
 805         case 0:
 806                 hw_feat->dma_width = 32;
 807                 break;
 808         case 1:
 809                 hw_feat->dma_width = 40;
 810                 break;
 811         case 2:
 812                 hw_feat->dma_width = 48;
 813                 break;
 814         default:
 815                 hw_feat->dma_width = 32;
 816         }
 817
 818         /* The Queue, Channel and TC counts are zero based so increment them
 819          * to get the actual number
 820          */
 821         hw_feat->rx_q_cnt++;
 822         hw_feat->tx_q_cnt++;
 823         hw_feat->rx_ch_cnt++;
 824         hw_feat->tx_ch_cnt++;
 825         hw_feat->tc_cnt++;
 826
 827         /* Translate the fifo sizes into actual numbers */
 828         hw_feat->rx_fifo_size = 1 << (hw_feat->rx_fifo_size + 7);
 829         hw_feat->tx_fifo_size = 1 << (hw_feat->tx_fifo_size + 7);
 830
 831         DBGPR("<--xgbe_get_all_hw_features\n");
 832 }
 833
 834 static void xgbe_napi_enable(struct xgbe_prv_data *pdata, unsigned int add)
 835 {
 836         struct xgbe_channel *channel;
 837         unsigned int i;
 838
 839         if (pdata->per_channel_irq) {
 840                 for (i = 0; i < pdata->channel_count; i++) {
 841                         channel = pdata->channel[i];
 842                         if (add)
 843                                 netif_napi_add(pdata->netdev, &channel->napi,
 844                                                xgbe_one_poll, NAPI_POLL_WEIGHT);
 845
 846                         napi_enable(&channel->napi);
 847                 }
 848         } else {
 849                 if (add)
 850                         netif_napi_add(pdata->netdev, &pdata->napi,
 851                                        xgbe_all_poll, NAPI_POLL_WEIGHT);
 852
 853                 napi_enable(&pdata->napi);
 854         }
 855 }
 856
 857 static void xgbe_napi_disable(struct xgbe_prv_data *pdata, unsigned int del)
 858 {
 859         struct xgbe_channel *channel;
 860         unsigned int i;
 861
 862         if (pdata->per_channel_irq) {
 863                 for (i = 0; i < pdata->channel_count; i++) {
 864                         channel = pdata->channel[i];
 865                         napi_disable(&channel->napi);
 866
 867                         if (del)
 868                                 netif_napi_del(&channel->napi);
 869                 }
 870         } else {
 871                 napi_disable(&pdata->napi);
 872
 873                 if (del)
 874                         netif_napi_del(&pdata->napi);
 875         }
 876 }
 877
 878 static int xgbe_request_irqs(struct xgbe_prv_data *pdata)
 879 {
 880         struct xgbe_channel *channel;
 881         struct net_device *netdev = pdata->netdev;
 882         unsigned int i;
 883         int ret;
 884
 885         tasklet_init(&pdata->tasklet_dev, xgbe_isr_task, (unsigned long)pdata);
 886         tasklet_init(&pdata->tasklet_ecc, xgbe_ecc_isr_task,
 887                      (unsigned long)pdata);
 888
 889         ret = devm_request_irq(pdata->dev, pdata->dev_irq, xgbe_isr, 0,
 890                                netdev->name, pdata);
 891         if (ret) {
 892                 netdev_alert(netdev, "error requesting irq %d\n",
 893                              pdata->dev_irq);
 894                 return ret;
 895         }
 896
 897         if (pdata->vdata->ecc_support && (pdata->dev_irq != pdata->ecc_irq)) {
 898                 ret = devm_request_irq(pdata->dev, pdata->ecc_irq, xgbe_ecc_isr,
 899                                        0, pdata->ecc_name, pdata);
 900                 if (ret) {
 901                         netdev_alert(netdev, "error requesting ecc irq %d\n",
 902                                      pdata->ecc_irq);
 903                         goto err_dev_irq;
 904                 }
 905         }
 906
 907         if (!pdata->per_channel_irq)
 908                 return 0;
 909
 910         for (i = 0; i < pdata->channel_count; i++) {
 911                 channel = pdata->channel[i];
 912                 snprintf(channel->dma_irq_name,
 913                          sizeof(channel->dma_irq_name) - 1,
 914                          "%s-TxRx-%u", netdev_name(netdev),
 915                          channel->queue_index);
 916
 917                 ret = devm_request_irq(pdata->dev, channel->dma_irq,
 918                                        xgbe_dma_isr, 0,
 919                                        channel->dma_irq_name, channel);
 920                 if (ret) {
 921                         netdev_alert(netdev, "error requesting irq %d\n",
 922                                      channel->dma_irq);
 923                         goto err_dma_irq;
 924                 }
 925
 926                 irq_set_affinity_hint(channel->dma_irq,
 927                                       &channel->affinity_mask);
 928         }
 929
 930         return 0;
 931
 932 err_dma_irq:
 933         /* Using an unsigned int, 'i' will go to UINT_MAX and exit */
 934         for (i--; i < pdata->channel_count; i--) {
 935                 channel = pdata->channel[i];
 936
 937                 irq_set_affinity_hint(channel->dma_irq, NULL);
 938                 devm_free_irq(pdata->dev, channel->dma_irq, channel);
 939         }
 940
 941         if (pdata->vdata->ecc_support && (pdata->dev_irq != pdata->ecc_irq))
 942                 devm_free_irq(pdata->dev, pdata->ecc_irq, pdata);
 943
 944 err_dev_irq:
 945         devm_free_irq(pdata->dev, pdata->dev_irq, pdata);
 946
 947         return ret;
 948 }
 949
 950 static void xgbe_free_irqs(struct xgbe_prv_data *pdata)
 951 {
 952         struct xgbe_channel *channel;
 953         unsigned int i;
 954
 955         devm_free_irq(pdata->dev, pdata->dev_irq, pdata);
 956
 957         if (pdata->vdata->ecc_support && (pdata->dev_irq != pdata->ecc_irq))
 958                 devm_free_irq(pdata->dev, pdata->ecc_irq, pdata);
 959
 960         if (!pdata->per_channel_irq)
 961                 return;
 962
 963         for (i = 0; i < pdata->channel_count; i++) {
 964                 channel = pdata->channel[i];
 965
 966                 irq_set_affinity_hint(channel->dma_irq, NULL);
 967                 devm_free_irq(pdata->dev, channel->dma_irq, channel);
 968         }
 969 }
 970
 971 void xgbe_init_tx_coalesce(struct xgbe_prv_data *pdata)
 972 {
 973         struct xgbe_hw_if *hw_if = &pdata->hw_if;
 974
 975         DBGPR("-->xgbe_init_tx_coalesce\n");
 976
 977         pdata->tx_usecs = XGMAC_INIT_DMA_TX_USECS;
 978         pdata->tx_frames = XGMAC_INIT_DMA_TX_FRAMES;
 979
 980         hw_if->config_tx_coalesce(pdata);
 981
 982         DBGPR("<--xgbe_init_tx_coalesce\n");
 983 }
 984
 985 void xgbe_init_rx_coalesce(struct xgbe_prv_data *pdata)
 986 {
 987         struct xgbe_hw_if *hw_if = &pdata->hw_if;
 988
 989         DBGPR("-->xgbe_init_rx_coalesce\n");
 990
 991         pdata->rx_riwt = hw_if->usec_to_riwt(pdata, XGMAC_INIT_DMA_RX_USECS);
 992         pdata->rx_usecs = XGMAC_INIT_DMA_RX_USECS;
 993         pdata->rx_frames = XGMAC_INIT_DMA_RX_FRAMES;
 994
 995         hw_if->config_rx_coalesce(pdata);
 996
 997         DBGPR("<--xgbe_init_rx_coalesce\n");
 998 }
 999
1000 static void xgbe_free_tx_data(struct xgbe_prv_data *pdata)
1001 {
1002         struct xgbe_desc_if *desc_if = &pdata->desc_if;
1003         struct xgbe_ring *ring;
1004         struct xgbe_ring_data *rdata;
1005         unsigned int i, j;
1006
1007         DBGPR("-->xgbe_free_tx_data\n");
1008
1009         for (i = 0; i < pdata->channel_count; i++) {
1010                 ring = pdata->channel[i]->tx_ring;
1011                 if (!ring)
1012                         break;
1013
1014                 for (j = 0; j < ring->rdesc_count; j++) {
1015                         rdata = XGBE_GET_DESC_DATA(ring, j);
1016                         desc_if->unmap_rdata(pdata, rdata);
1017                 }
1018         }
1019
1020         DBGPR("<--xgbe_free_tx_data\n");
1021 }
1022
1023 static void xgbe_free_rx_data(struct xgbe_prv_data *pdata)
1024 {
1025         struct xgbe_desc_if *desc_if = &pdata->desc_if;
1026         struct xgbe_ring *ring;
1027         struct xgbe_ring_data *rdata;
1028         unsigned int i, j;
1029
1030         DBGPR("-->xgbe_free_rx_data\n");
1031
1032         for (i = 0; i < pdata->channel_count; i++) {
1033                 ring = pdata->channel[i]->rx_ring;
1034                 if (!ring)
1035                         break;
1036
1037                 for (j = 0; j < ring->rdesc_count; j++) {
1038                         rdata = XGBE_GET_DESC_DATA(ring, j);
1039                         desc_if->unmap_rdata(pdata, rdata);
1040                 }
1041         }
1042
1043         DBGPR("<--xgbe_free_rx_data\n");
1044 }
1045
1046 static int xgbe_phy_reset(struct xgbe_prv_data *pdata)
1047 {
1048         pdata->phy_link = -1;
1049         pdata->phy_speed = SPEED_UNKNOWN;
1050
1051         return pdata->phy_if.phy_reset(pdata);
1052 }
1053
1054 int xgbe_powerdown(struct net_device *netdev, unsigned int caller)
1055 {
1056         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1057         struct xgbe_hw_if *hw_if = &pdata->hw_if;
1058         unsigned long flags;
1059
1060         DBGPR("-->xgbe_powerdown\n");
1061
1062         if (!netif_running(netdev) ||
1063             (caller == XGMAC_IOCTL_CONTEXT && pdata->power_down)) {
1064                 netdev_alert(netdev, "Device is already powered down\n");
1065                 DBGPR("<--xgbe_powerdown\n");
1066                 return -EINVAL;
1067         }
1068
1069         spin_lock_irqsave(&pdata->lock, flags);
1070
1071         if (caller == XGMAC_DRIVER_CONTEXT)
1072                 netif_device_detach(netdev);
1073
1074         netif_tx_stop_all_queues(netdev);
1075
1076         xgbe_stop_timers(pdata);
1077         flush_workqueue(pdata->dev_workqueue);
1078
1079         hw_if->powerdown_tx(pdata);
1080         hw_if->powerdown_rx(pdata);
1081
1082         xgbe_napi_disable(pdata, 0);
1083
1084         pdata->power_down = 1;
1085
1086         spin_unlock_irqrestore(&pdata->lock, flags);
1087
1088         DBGPR("<--xgbe_powerdown\n");
1089
1090         return 0;
1091 }
1092
1093 int xgbe_powerup(struct net_device *netdev, unsigned int caller)
1094 {
1095         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1096         struct xgbe_hw_if *hw_if = &pdata->hw_if;
1097         unsigned long flags;
1098
1099         DBGPR("-->xgbe_powerup\n");
1100
1101         if (!netif_running(netdev) ||
1102             (caller == XGMAC_IOCTL_CONTEXT && !pdata->power_down)) {
1103                 netdev_alert(netdev, "Device is already powered up\n");
1104                 DBGPR("<--xgbe_powerup\n");
1105                 return -EINVAL;
1106         }
1107
1108         spin_lock_irqsave(&pdata->lock, flags);
1109
1110         pdata->power_down = 0;
1111
1112         xgbe_napi_enable(pdata, 0);
1113
1114         hw_if->powerup_tx(pdata);
1115         hw_if->powerup_rx(pdata);
1116
1117         if (caller == XGMAC_DRIVER_CONTEXT)
1118                 netif_device_attach(netdev);
1119
1120         netif_tx_start_all_queues(netdev);
1121
1122         xgbe_start_timers(pdata);
1123
1124         spin_unlock_irqrestore(&pdata->lock, flags);
1125
1126         DBGPR("<--xgbe_powerup\n");
1127
1128         return 0;
1129 }
1130
1131 static int xgbe_start(struct xgbe_prv_data *pdata)
1132 {
1133         struct xgbe_hw_if *hw_if = &pdata->hw_if;
1134         struct xgbe_phy_if *phy_if = &pdata->phy_if;
1135         struct net_device *netdev = pdata->netdev;
1136         int ret;
1137
1138         DBGPR("-->xgbe_start\n");
1139
1140         ret = hw_if->init(pdata);
1141         if (ret)
1142                 return ret;
1143
1144         xgbe_napi_enable(pdata, 1);
1145
1146         ret = xgbe_request_irqs(pdata);
1147         if (ret)
1148                 goto err_napi;
1149
1150         ret = phy_if->phy_start(pdata);
1151         if (ret)
1152                 goto err_irqs;
1153
1154         hw_if->enable_tx(pdata);
1155         hw_if->enable_rx(pdata);
1156
1157         netif_tx_start_all_queues(netdev);
1158
1159         xgbe_start_timers(pdata);
1160         queue_work(pdata->dev_workqueue, &pdata->service_work);
1161
1162         clear_bit(XGBE_STOPPED, &pdata->dev_state);
1163
1164         DBGPR("<--xgbe_start\n");
1165
1166         return 0;
1167
1168 err_irqs:
1169         xgbe_free_irqs(pdata);
1170
1171 err_napi:
1172         xgbe_napi_disable(pdata, 1);
1173
1174         hw_if->exit(pdata);
1175
1176         return ret;
1177 }
1178
1179 static void xgbe_stop(struct xgbe_prv_data *pdata)
1180 {
1181         struct xgbe_hw_if *hw_if = &pdata->hw_if;
1182         struct xgbe_phy_if *phy_if = &pdata->phy_if;
1183         struct xgbe_channel *channel;
1184         struct net_device *netdev = pdata->netdev;
1185         struct netdev_queue *txq;
1186         unsigned int i;
1187
1188         DBGPR("-->xgbe_stop\n");
1189
1190         if (test_bit(XGBE_STOPPED, &pdata->dev_state))
1191                 return;
1192
1193         netif_tx_stop_all_queues(netdev);
1194
1195         xgbe_stop_timers(pdata);
1196         flush_workqueue(pdata->dev_workqueue);
1197
1198         hw_if->disable_tx(pdata);
1199         hw_if->disable_rx(pdata);
1200
1201         phy_if->phy_stop(pdata);
1202
1203         xgbe_free_irqs(pdata);
1204
1205         xgbe_napi_disable(pdata, 1);
1206
1207         hw_if->exit(pdata);
1208
1209         for (i = 0; i < pdata->channel_count; i++) {
1210                 channel = pdata->channel[i];
1211                 if (!channel->tx_ring)
1212                         continue;
1213
1214                 txq = netdev_get_tx_queue(netdev, channel->queue_index);
1215                 netdev_tx_reset_queue(txq);
1216         }
1217
1218         set_bit(XGBE_STOPPED, &pdata->dev_state);
1219
1220         DBGPR("<--xgbe_stop\n");
1221 }
1222
1223 static void xgbe_stopdev(struct work_struct *work)
1224 {
1225         struct xgbe_prv_data *pdata = container_of(work,
1226                                                    struct xgbe_prv_data,
1227                                                    stopdev_work);
1228
1229         rtnl_lock();
1230
1231         xgbe_stop(pdata);
1232
1233         xgbe_free_tx_data(pdata);
1234         xgbe_free_rx_data(pdata);
1235
1236         rtnl_unlock();
1237
1238         netdev_alert(pdata->netdev, "device stopped\n");
1239 }
1240
1241 static void xgbe_restart_dev(struct xgbe_prv_data *pdata)
1242 {
1243         DBGPR("-->xgbe_restart_dev\n");
1244
1245         /* If not running, "restart" will happen on open */
1246         if (!netif_running(pdata->netdev))
1247                 return;
1248
1249         xgbe_stop(pdata);
1250
1251         xgbe_free_tx_data(pdata);
1252         xgbe_free_rx_data(pdata);
1253
1254         xgbe_start(pdata);
1255
1256         DBGPR("<--xgbe_restart_dev\n");
1257 }
1258
1259 static void xgbe_restart(struct work_struct *work)
1260 {
1261         struct xgbe_prv_data *pdata = container_of(work,
1262                                                    struct xgbe_prv_data,
1263                                                    restart_work);
1264
1265         rtnl_lock();
1266
1267         xgbe_restart_dev(pdata);
1268
1269         rtnl_unlock();
1270 }
1271
1272 static void xgbe_tx_tstamp(struct work_struct *work)
1273 {
1274         struct xgbe_prv_data *pdata = container_of(work,
1275                                                    struct xgbe_prv_data,
1276                                                    tx_tstamp_work);
1277         struct skb_shared_hwtstamps hwtstamps;
1278         u64 nsec;
1279         unsigned long flags;
1280
1281         spin_lock_irqsave(&pdata->tstamp_lock, flags);
1282         if (!pdata->tx_tstamp_skb)
1283                 goto unlock;
1284
1285         if (pdata->tx_tstamp) {
1286                 nsec = timecounter_cyc2time(&pdata->tstamp_tc,
1287                                             pdata->tx_tstamp);
1288
1289                 memset(&hwtstamps, 0, sizeof(hwtstamps));
1290                 hwtstamps.hwtstamp = ns_to_ktime(nsec);
1291                 skb_tstamp_tx(pdata->tx_tstamp_skb, &hwtstamps);
1292         }
1293
1294         dev_kfree_skb_any(pdata->tx_tstamp_skb);
1295
1296         pdata->tx_tstamp_skb = NULL;
1297
1298 unlock:
1299         spin_unlock_irqrestore(&pdata->tstamp_lock, flags);
1300 }
1301
1302 static int xgbe_get_hwtstamp_settings(struct xgbe_prv_data *pdata,
1303                                       struct ifreq *ifreq)
1304 {
1305         if (copy_to_user(ifreq->ifr_data, &pdata->tstamp_config,
1306                          sizeof(pdata->tstamp_config)))
1307                 return -EFAULT;
1308
1309         return 0;
1310 }
1311
1312 static int xgbe_set_hwtstamp_settings(struct xgbe_prv_data *pdata,
1313                                       struct ifreq *ifreq)
1314 {
1315         struct hwtstamp_config config;
1316         unsigned int mac_tscr;
1317
1318         if (copy_from_user(&config, ifreq->ifr_data, sizeof(config)))
1319                 return -EFAULT;
1320
1321         if (config.flags)
1322                 return -EINVAL;
1323
1324         mac_tscr = 0;
1325
1326         switch (config.tx_type) {
1327         case HWTSTAMP_TX_OFF:
1328                 break;
1329
1330         case HWTSTAMP_TX_ON:
1331                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1332                 break;
1333
1334         default:
1335                 return -ERANGE;
1336         }
1337
1338         switch (config.rx_filter) {
1339         case HWTSTAMP_FILTER_NONE:
1340                 break;
1341
1342         case HWTSTAMP_FILTER_NTP_ALL:
1343         case HWTSTAMP_FILTER_ALL:
1344                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENALL, 1);
1345                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1346                 break;
1347
1348         /* PTP v2, UDP, any kind of event packet */
1349         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1350                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSVER2ENA, 1);
1351         /* PTP v1, UDP, any kind of event packet */
1352         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1353                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV4ENA, 1);
1354                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV6ENA, 1);
1355                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, SNAPTYPSEL, 1);
1356                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1357                 break;
1358
1359         /* PTP v2, UDP, Sync packet */
1360         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1361                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSVER2ENA, 1);
1362         /* PTP v1, UDP, Sync packet */
1363         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1364                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV4ENA, 1);
1365                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV6ENA, 1);
1366                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSEVNTENA, 1);
1367                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1368                 break;
1369
1370         /* PTP v2, UDP, Delay_req packet */
1371         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1372                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSVER2ENA, 1);
1373         /* PTP v1, UDP, Delay_req packet */
1374         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1375                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV4ENA, 1);
1376                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV6ENA, 1);
1377                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSEVNTENA, 1);
1378                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSMSTRENA, 1);
1379                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1380                 break;
1381
1382         /* 802.AS1, Ethernet, any kind of event packet */
1383         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1384                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, AV8021ASMEN, 1);
1385                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, SNAPTYPSEL, 1);
1386                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1387                 break;
1388
1389         /* 802.AS1, Ethernet, Sync packet */
1390         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1391                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, AV8021ASMEN, 1);
1392                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSEVNTENA, 1);
1393                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1394                 break;
1395
1396         /* 802.AS1, Ethernet, Delay_req packet */
1397         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1398                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, AV8021ASMEN, 1);
1399                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSMSTRENA, 1);
1400                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSEVNTENA, 1);
1401                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1402                 break;
1403
1404         /* PTP v2/802.AS1, any layer, any kind of event packet */
1405         case HWTSTAMP_FILTER_PTP_V2_EVENT:
1406                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSVER2ENA, 1);
1407                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPENA, 1);
1408                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV4ENA, 1);
1409                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV6ENA, 1);
1410                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, SNAPTYPSEL, 1);
1411                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1412                 break;
1413
1414         /* PTP v2/802.AS1, any layer, Sync packet */
1415         case HWTSTAMP_FILTER_PTP_V2_SYNC:
1416                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSVER2ENA, 1);
1417                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPENA, 1);
1418                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV4ENA, 1);
1419                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV6ENA, 1);
1420                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSEVNTENA, 1);
1421                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1422                 break;
1423
1424         /* PTP v2/802.AS1, any layer, Delay_req packet */
1425         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1426                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSVER2ENA, 1);
1427                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPENA, 1);
1428                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV4ENA, 1);
1429                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSIPV6ENA, 1);
1430                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSMSTRENA, 1);
1431                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSEVNTENA, 1);
1432                 XGMAC_SET_BITS(mac_tscr, MAC_TSCR, TSENA, 1);
1433                 break;
1434
1435         default:
1436                 return -ERANGE;
1437         }
1438
1439         pdata->hw_if.config_tstamp(pdata, mac_tscr);
1440
1441         memcpy(&pdata->tstamp_config, &config, sizeof(config));
1442
1443         return 0;
1444 }
1445
1446 static void xgbe_prep_tx_tstamp(struct xgbe_prv_data *pdata,
1447                                 struct sk_buff *skb,
1448                                 struct xgbe_packet_data *packet)
1449 {
1450         unsigned long flags;
1451
1452         if (XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES, PTP)) {
1453                 spin_lock_irqsave(&pdata->tstamp_lock, flags);
1454                 if (pdata->tx_tstamp_skb) {
1455                         /* Another timestamp in progress, ignore this one */
1456                         XGMAC_SET_BITS(packet->attributes,
1457                                        TX_PACKET_ATTRIBUTES, PTP, 0);
1458                 } else {
1459                         pdata->tx_tstamp_skb = skb_get(skb);
1460                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1461                 }
1462                 spin_unlock_irqrestore(&pdata->tstamp_lock, flags);
1463         }
1464
1465         skb_tx_timestamp(skb);
1466 }
1467
1468 static void xgbe_prep_vlan(struct sk_buff *skb, struct xgbe_packet_data *packet)
1469 {
1470         if (skb_vlan_tag_present(skb))
1471                 packet->vlan_ctag = skb_vlan_tag_get(skb);
1472 }
1473
1474 static int xgbe_prep_tso(struct sk_buff *skb, struct xgbe_packet_data *packet)
1475 {
1476         int ret;
1477
1478         if (!XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1479                             TSO_ENABLE))
1480                 return 0;
1481
1482         ret = skb_cow_head(skb, 0);
1483         if (ret)
1484                 return ret;
1485
1486         packet->header_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1487         packet->tcp_header_len = tcp_hdrlen(skb);
1488         packet->tcp_payload_len = skb->len - packet->header_len;
1489         packet->mss = skb_shinfo(skb)->gso_size;
1490         DBGPR("  packet->header_len=%u\n", packet->header_len);
1491         DBGPR("  packet->tcp_header_len=%u, packet->tcp_payload_len=%u\n",
1492               packet->tcp_header_len, packet->tcp_payload_len);
1493         DBGPR("  packet->mss=%u\n", packet->mss);
1494
1495         /* Update the number of packets that will ultimately be transmitted
1496          * along with the extra bytes for each extra packet
1497          */
1498         packet->tx_packets = skb_shinfo(skb)->gso_segs;
1499         packet->tx_bytes += (packet->tx_packets - 1) * packet->header_len;
1500
1501         return 0;
1502 }
1503
1504 static int xgbe_is_tso(struct sk_buff *skb)
1505 {
1506         if (skb->ip_summed != CHECKSUM_PARTIAL)
1507                 return 0;
1508
1509         if (!skb_is_gso(skb))
1510                 return 0;
1511
1512         DBGPR("  TSO packet to be processed\n");
1513
1514         return 1;
1515 }
1516
1517 static void xgbe_packet_info(struct xgbe_prv_data *pdata,
1518                              struct xgbe_ring *ring, struct sk_buff *skb,
1519                              struct xgbe_packet_data *packet)
1520 {
1521         struct skb_frag_struct *frag;
1522         unsigned int context_desc;
1523         unsigned int len;
1524         unsigned int i;
1525
1526         packet->skb = skb;
1527
1528         context_desc = 0;
1529         packet->rdesc_count = 0;
1530
1531         packet->tx_packets = 1;
1532         packet->tx_bytes = skb->len;
1533
1534         if (xgbe_is_tso(skb)) {
1535                 /* TSO requires an extra descriptor if mss is different */
1536                 if (skb_shinfo(skb)->gso_size != ring->tx.cur_mss) {
1537                         context_desc = 1;
1538                         packet->rdesc_count++;
1539                 }
1540
1541                 /* TSO requires an extra descriptor for TSO header */
1542                 packet->rdesc_count++;
1543
1544                 XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1545                                TSO_ENABLE, 1);
1546                 XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1547                                CSUM_ENABLE, 1);
1548         } else if (skb->ip_summed == CHECKSUM_PARTIAL)
1549                 XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1550                                CSUM_ENABLE, 1);
1551
1552         if (skb_vlan_tag_present(skb)) {
1553                 /* VLAN requires an extra descriptor if tag is different */
1554                 if (skb_vlan_tag_get(skb) != ring->tx.cur_vlan_ctag)
1555                         /* We can share with the TSO context descriptor */
1556                         if (!context_desc) {
1557                                 context_desc = 1;
1558                                 packet->rdesc_count++;
1559                         }
1560
1561                 XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1562                                VLAN_CTAG, 1);
1563         }
1564
1565         if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1566             (pdata->tstamp_config.tx_type == HWTSTAMP_TX_ON))
1567                 XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1568                                PTP, 1);
1569
1570         for (len = skb_headlen(skb); len;) {
1571                 packet->rdesc_count++;
1572                 len -= min_t(unsigned int, len, XGBE_TX_MAX_BUF_SIZE);
1573         }
1574
1575         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1576                 frag = &skb_shinfo(skb)->frags[i];
1577                 for (len = skb_frag_size(frag); len; ) {
1578                         packet->rdesc_count++;
1579                         len -= min_t(unsigned int, len, XGBE_TX_MAX_BUF_SIZE);
1580                 }
1581         }
1582 }
1583
1584 static int xgbe_open(struct net_device *netdev)
1585 {
1586         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1587         struct xgbe_desc_if *desc_if = &pdata->desc_if;
1588         int ret;
1589
1590         DBGPR("-->xgbe_open\n");
1591
1592         /* Reset the phy settings */
1593         ret = xgbe_phy_reset(pdata);
1594         if (ret)
1595                 return ret;
1596
1597         /* Enable the clocks */
1598         ret = clk_prepare_enable(pdata->sysclk);
1599         if (ret) {
1600                 netdev_alert(netdev, "dma clk_prepare_enable failed\n");
1601                 return ret;
1602         }
1603
1604         ret = clk_prepare_enable(pdata->ptpclk);
1605         if (ret) {
1606                 netdev_alert(netdev, "ptp clk_prepare_enable failed\n");
1607                 goto err_sysclk;
1608         }
1609
1610         /* Calculate the Rx buffer size before allocating rings */
1611         ret = xgbe_calc_rx_buf_size(netdev, netdev->mtu);
1612         if (ret < 0)
1613                 goto err_ptpclk;
1614         pdata->rx_buf_size = ret;
1615
1616         /* Allocate the channel and ring structures */
1617         ret = xgbe_alloc_channels(pdata);
1618         if (ret)
1619                 goto err_ptpclk;
1620
1621         /* Allocate the ring descriptors and buffers */
1622         ret = desc_if->alloc_ring_resources(pdata);
1623         if (ret)
1624                 goto err_channels;
1625
1626         INIT_WORK(&pdata->service_work, xgbe_service);
1627         INIT_WORK(&pdata->restart_work, xgbe_restart);
1628         INIT_WORK(&pdata->stopdev_work, xgbe_stopdev);
1629         INIT_WORK(&pdata->tx_tstamp_work, xgbe_tx_tstamp);
1630         xgbe_init_timers(pdata);
1631
1632         ret = xgbe_start(pdata);
1633         if (ret)
1634                 goto err_rings;
1635
1636         clear_bit(XGBE_DOWN, &pdata->dev_state);
1637
1638         DBGPR("<--xgbe_open\n");
1639
1640         return 0;
1641
1642 err_rings:
1643         desc_if->free_ring_resources(pdata);
1644
1645 err_channels:
1646         xgbe_free_channels(pdata);
1647
1648 err_ptpclk:
1649         clk_disable_unprepare(pdata->ptpclk);
1650
1651 err_sysclk:
1652         clk_disable_unprepare(pdata->sysclk);
1653
1654         return ret;
1655 }
1656
1657 static int xgbe_close(struct net_device *netdev)
1658 {
1659         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1660         struct xgbe_desc_if *desc_if = &pdata->desc_if;
1661
1662         DBGPR("-->xgbe_close\n");
1663
1664         /* Stop the device */
1665         xgbe_stop(pdata);
1666
1667         /* Free the ring descriptors and buffers */
1668         desc_if->free_ring_resources(pdata);
1669
1670         /* Free the channel and ring structures */
1671         xgbe_free_channels(pdata);
1672
1673         /* Disable the clocks */
1674         clk_disable_unprepare(pdata->ptpclk);
1675         clk_disable_unprepare(pdata->sysclk);
1676
1677         set_bit(XGBE_DOWN, &pdata->dev_state);
1678
1679         DBGPR("<--xgbe_close\n");
1680
1681         return 0;
1682 }
1683
1684 static int xgbe_xmit(struct sk_buff *skb, struct net_device *netdev)
1685 {
1686         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1687         struct xgbe_hw_if *hw_if = &pdata->hw_if;
1688         struct xgbe_desc_if *desc_if = &pdata->desc_if;
1689         struct xgbe_channel *channel;
1690         struct xgbe_ring *ring;
1691         struct xgbe_packet_data *packet;
1692         struct netdev_queue *txq;
1693         int ret;
1694
1695         DBGPR("-->xgbe_xmit: skb->len = %d\n", skb->len);
1696
1697         channel = pdata->channel[skb->queue_mapping];
1698         txq = netdev_get_tx_queue(netdev, channel->queue_index);
1699         ring = channel->tx_ring;
1700         packet = &ring->packet_data;
1701
1702         ret = NETDEV_TX_OK;
1703
1704         if (skb->len == 0) {
1705                 netif_err(pdata, tx_err, netdev,
1706                           "empty skb received from stack\n");
1707                 dev_kfree_skb_any(skb);
1708                 goto tx_netdev_return;
1709         }
1710
1711         /* Calculate preliminary packet info */
1712         memset(packet, 0, sizeof(*packet));
1713         xgbe_packet_info(pdata, ring, skb, packet);
1714
1715         /* Check that there are enough descriptors available */
1716         ret = xgbe_maybe_stop_tx_queue(channel, ring, packet->rdesc_count);
1717         if (ret)
1718                 goto tx_netdev_return;
1719
1720         ret = xgbe_prep_tso(skb, packet);
1721         if (ret) {
1722                 netif_err(pdata, tx_err, netdev,
1723                           "error processing TSO packet\n");
1724                 dev_kfree_skb_any(skb);
1725                 goto tx_netdev_return;
1726         }
1727         xgbe_prep_vlan(skb, packet);
1728
1729         if (!desc_if->map_tx_skb(channel, skb)) {
1730                 dev_kfree_skb_any(skb);
1731                 goto tx_netdev_return;
1732         }
1733
1734         xgbe_prep_tx_tstamp(pdata, skb, packet);
1735
1736         /* Report on the actual number of bytes (to be) sent */
1737         netdev_tx_sent_queue(txq, packet->tx_bytes);
1738
1739         /* Configure required descriptor fields for transmission */
1740         hw_if->dev_xmit(channel);
1741
1742         if (netif_msg_pktdata(pdata))
1743                 xgbe_print_pkt(netdev, skb, true);
1744
1745         /* Stop the queue in advance if there may not be enough descriptors */
1746         xgbe_maybe_stop_tx_queue(channel, ring, XGBE_TX_MAX_DESCS);
1747
1748         ret = NETDEV_TX_OK;
1749
1750 tx_netdev_return:
1751         return ret;
1752 }
1753
1754 static void xgbe_set_rx_mode(struct net_device *netdev)
1755 {
1756         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1757         struct xgbe_hw_if *hw_if = &pdata->hw_if;
1758
1759         DBGPR("-->xgbe_set_rx_mode\n");
1760
1761         hw_if->config_rx_mode(pdata);
1762
1763         DBGPR("<--xgbe_set_rx_mode\n");
1764 }
1765
1766 static int xgbe_set_mac_address(struct net_device *netdev, void *addr)
1767 {
1768         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1769         struct xgbe_hw_if *hw_if = &pdata->hw_if;
1770         struct sockaddr *saddr = addr;
1771
1772         DBGPR("-->xgbe_set_mac_address\n");
1773
1774         if (!is_valid_ether_addr(saddr->sa_data))
1775                 return -EADDRNOTAVAIL;
1776
1777         memcpy(netdev->dev_addr, saddr->sa_data, netdev->addr_len);
1778
1779         hw_if->set_mac_address(pdata, netdev->dev_addr);
1780
1781         DBGPR("<--xgbe_set_mac_address\n");
1782
1783         return 0;
1784 }
1785
1786 static int xgbe_ioctl(struct net_device *netdev, struct ifreq *ifreq, int cmd)
1787 {
1788         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1789         int ret;
1790
1791         switch (cmd) {
1792         case SIOCGHWTSTAMP:
1793                 ret = xgbe_get_hwtstamp_settings(pdata, ifreq);
1794                 break;
1795
1796         case SIOCSHWTSTAMP:
1797                 ret = xgbe_set_hwtstamp_settings(pdata, ifreq);
1798                 break;
1799
1800         default:
1801                 ret = -EOPNOTSUPP;
1802         }
1803
1804         return ret;
1805 }
1806
1807 static int xgbe_change_mtu(struct net_device *netdev, int mtu)
1808 {
1809         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1810         int ret;
1811
1812         DBGPR("-->xgbe_change_mtu\n");
1813
1814         ret = xgbe_calc_rx_buf_size(netdev, mtu);
1815         if (ret < 0)
1816                 return ret;
1817
1818         pdata->rx_buf_size = ret;
1819         netdev->mtu = mtu;
1820
1821         xgbe_restart_dev(pdata);
1822
1823         DBGPR("<--xgbe_change_mtu\n");
1824
1825         return 0;
1826 }
1827
1828 static void xgbe_tx_timeout(struct net_device *netdev)
1829 {
1830         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1831
1832         netdev_warn(netdev, "tx timeout, device restarting\n");
1833         schedule_work(&pdata->restart_work);
1834 }
1835
1836 static void xgbe_get_stats64(struct net_device *netdev,
1837                              struct rtnl_link_stats64 *s)
1838 {
1839         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1840         struct xgbe_mmc_stats *pstats = &pdata->mmc_stats;
1841
1842         DBGPR("-->%s\n", __func__);
1843
1844         pdata->hw_if.read_mmc_stats(pdata);
1845
1846         s->rx_packets = pstats->rxframecount_gb;
1847         s->rx_bytes = pstats->rxoctetcount_gb;
1848         s->rx_errors = pstats->rxframecount_gb -
1849                        pstats->rxbroadcastframes_g -
1850                        pstats->rxmulticastframes_g -
1851                        pstats->rxunicastframes_g;
1852         s->multicast = pstats->rxmulticastframes_g;
1853         s->rx_length_errors = pstats->rxlengtherror;
1854         s->rx_crc_errors = pstats->rxcrcerror;
1855         s->rx_fifo_errors = pstats->rxfifooverflow;
1856
1857         s->tx_packets = pstats->txframecount_gb;
1858         s->tx_bytes = pstats->txoctetcount_gb;
1859         s->tx_errors = pstats->txframecount_gb - pstats->txframecount_g;
1860         s->tx_dropped = netdev->stats.tx_dropped;
1861
1862         DBGPR("<--%s\n", __func__);
1863 }
1864
1865 static int xgbe_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1866                                 u16 vid)
1867 {
1868         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1869         struct xgbe_hw_if *hw_if = &pdata->hw_if;
1870
1871         DBGPR("-->%s\n", __func__);
1872
1873         set_bit(vid, pdata->active_vlans);
1874         hw_if->update_vlan_hash_table(pdata);
1875
1876         DBGPR("<--%s\n", __func__);
1877
1878         return 0;
1879 }
1880
1881 static int xgbe_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1882                                  u16 vid)
1883 {
1884         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1885         struct xgbe_hw_if *hw_if = &pdata->hw_if;
1886
1887         DBGPR("-->%s\n", __func__);
1888
1889         clear_bit(vid, pdata->active_vlans);
1890         hw_if->update_vlan_hash_table(pdata);
1891
1892         DBGPR("<--%s\n", __func__);
1893
1894         return 0;
1895 }
1896
1897 #ifdef CONFIG_NET_POLL_CONTROLLER
1898 static void xgbe_poll_controller(struct net_device *netdev)
1899 {
1900         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1901         struct xgbe_channel *channel;
1902         unsigned int i;
1903
1904         DBGPR("-->xgbe_poll_controller\n");
1905
1906         if (pdata->per_channel_irq) {
1907                 for (i = 0; i < pdata->channel_count; i++) {
1908                         channel = pdata->channel[i];
1909                         xgbe_dma_isr(channel->dma_irq, channel);
1910                 }
1911         } else {
1912                 disable_irq(pdata->dev_irq);
1913                 xgbe_isr(pdata->dev_irq, pdata);
1914                 enable_irq(pdata->dev_irq);
1915         }
1916
1917         DBGPR("<--xgbe_poll_controller\n");
1918 }
1919 #endif /* End CONFIG_NET_POLL_CONTROLLER */
1920
1921 static int xgbe_setup_tc(struct net_device *netdev, u32 handle, u32 chain_index,
1922                          __be16 proto,
1923                          struct tc_to_netdev *tc_to_netdev)
1924 {
1925         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1926         u8 tc;
1927
1928         if (tc_to_netdev->type != TC_SETUP_MQPRIO)
1929                 return -EINVAL;
1930
1931         tc_to_netdev->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1932         tc = tc_to_netdev->mqprio->num_tc;
1933
1934         if (tc > pdata->hw_feat.tc_cnt)
1935                 return -EINVAL;
1936
1937         pdata->num_tcs = tc;
1938         pdata->hw_if.config_tc(pdata);
1939
1940         return 0;
1941 }
1942
1943 static int xgbe_set_features(struct net_device *netdev,
1944                              netdev_features_t features)
1945 {
1946         struct xgbe_prv_data *pdata = netdev_priv(netdev);
1947         struct xgbe_hw_if *hw_if = &pdata->hw_if;
1948         netdev_features_t rxhash, rxcsum, rxvlan, rxvlan_filter;
1949         int ret = 0;
1950
1951         rxhash = pdata->netdev_features & NETIF_F_RXHASH;
1952         rxcsum = pdata->netdev_features & NETIF_F_RXCSUM;
1953         rxvlan = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_RX;
1954         rxvlan_filter = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_FILTER;
1955
1956         if ((features & NETIF_F_RXHASH) && !rxhash)
1957                 ret = hw_if->enable_rss(pdata);
1958         else if (!(features & NETIF_F_RXHASH) && rxhash)
1959                 ret = hw_if->disable_rss(pdata);
1960         if (ret)
1961                 return ret;
1962
1963         if ((features & NETIF_F_RXCSUM) && !rxcsum)
1964                 hw_if->enable_rx_csum(pdata);
1965         else if (!(features & NETIF_F_RXCSUM) && rxcsum)
1966                 hw_if->disable_rx_csum(pdata);
1967
1968         if ((features & NETIF_F_HW_VLAN_CTAG_RX) && !rxvlan)
1969                 hw_if->enable_rx_vlan_stripping(pdata);
1970         else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) && rxvlan)
1971                 hw_if->disable_rx_vlan_stripping(pdata);
1972
1973         if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) && !rxvlan_filter)
1974                 hw_if->enable_rx_vlan_filtering(pdata);
1975         else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) && rxvlan_filter)
1976                 hw_if->disable_rx_vlan_filtering(pdata);
1977
1978         pdata->netdev_features = features;
1979
1980         DBGPR("<--xgbe_set_features\n");
1981
1982         return 0;
1983 }
1984
1985 static const struct net_device_ops xgbe_netdev_ops = {
1986         .ndo_open               = xgbe_open,
1987         .ndo_stop               = xgbe_close,
1988         .ndo_start_xmit         = xgbe_xmit,
1989         .ndo_set_rx_mode        = xgbe_set_rx_mode,
1990         .ndo_set_mac_address    = xgbe_set_mac_address,
1991         .ndo_validate_addr      = eth_validate_addr,
1992         .ndo_do_ioctl           = xgbe_ioctl,
1993         .ndo_change_mtu         = xgbe_change_mtu,
1994         .ndo_tx_timeout         = xgbe_tx_timeout,
1995         .ndo_get_stats64        = xgbe_get_stats64,
1996         .ndo_vlan_rx_add_vid    = xgbe_vlan_rx_add_vid,
1997         .ndo_vlan_rx_kill_vid   = xgbe_vlan_rx_kill_vid,
1998 #ifdef CONFIG_NET_POLL_CONTROLLER
1999         .ndo_poll_controller    = xgbe_poll_controller,
2000 #endif
2001         .ndo_setup_tc           = xgbe_setup_tc,
2002         .ndo_set_features       = xgbe_set_features,
2003 };
2004
2005 const struct net_device_ops *xgbe_get_netdev_ops(void)
2006 {
2007         return &xgbe_netdev_ops;
2008 }
2009
2010 static void xgbe_rx_refresh(struct xgbe_channel *channel)
2011 {
2012         struct xgbe_prv_data *pdata = channel->pdata;
2013         struct xgbe_hw_if *hw_if = &pdata->hw_if;
2014         struct xgbe_desc_if *desc_if = &pdata->desc_if;
2015         struct xgbe_ring *ring = channel->rx_ring;
2016         struct xgbe_ring_data *rdata;
2017
2018         while (ring->dirty != ring->cur) {
2019                 rdata = XGBE_GET_DESC_DATA(ring, ring->dirty);
2020
2021                 /* Reset rdata values */
2022                 desc_if->unmap_rdata(pdata, rdata);
2023
2024                 if (desc_if->map_rx_buffer(pdata, ring, rdata))
2025                         break;
2026
2027                 hw_if->rx_desc_reset(pdata, rdata, ring->dirty);
2028
2029                 ring->dirty++;
2030         }
2031
2032         /* Make sure everything is written before the register write */
2033         wmb();
2034
2035         /* Update the Rx Tail Pointer Register with address of
2036          * the last cleaned entry */
2037         rdata = XGBE_GET_DESC_DATA(ring, ring->dirty - 1);
2038         XGMAC_DMA_IOWRITE(channel, DMA_CH_RDTR_LO,
2039                           lower_32_bits(rdata->rdesc_dma));
2040 }
2041
2042 static struct sk_buff *xgbe_create_skb(struct xgbe_prv_data *pdata,
2043                                        struct napi_struct *napi,
2044                                        struct xgbe_ring_data *rdata,
2045                                        unsigned int len)
2046 {
2047         struct sk_buff *skb;
2048         u8 *packet;
2049
2050         skb = napi_alloc_skb(napi, rdata->rx.hdr.dma_len);
2051         if (!skb)
2052                 return NULL;
2053
2054         /* Pull in the header buffer which may contain just the header
2055          * or the header plus data
2056          */
2057         dma_sync_single_range_for_cpu(pdata->dev, rdata->rx.hdr.dma_base,
2058                                       rdata->rx.hdr.dma_off,
2059                                       rdata->rx.hdr.dma_len, DMA_FROM_DEVICE);
2060
2061         packet = page_address(rdata->rx.hdr.pa.pages) +
2062                  rdata->rx.hdr.pa.pages_offset;
2063         skb_copy_to_linear_data(skb, packet, len);
2064         skb_put(skb, len);
2065
2066         return skb;
2067 }
2068
2069 static unsigned int xgbe_rx_buf1_len(struct xgbe_ring_data *rdata,
2070                                      struct xgbe_packet_data *packet)
2071 {
2072         /* Always zero if not the first descriptor */
2073         if (!XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, FIRST))
2074                 return 0;
2075
2076         /* First descriptor with split header, return header length */
2077         if (rdata->rx.hdr_len)
2078                 return rdata->rx.hdr_len;
2079
2080         /* First descriptor but not the last descriptor and no split header,
2081          * so the full buffer was used
2082          */
2083         if (!XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, LAST))
2084                 return rdata->rx.hdr.dma_len;
2085
2086         /* First descriptor and last descriptor and no split header, so
2087          * calculate how much of the buffer was used
2088          */
2089         return min_t(unsigned int, rdata->rx.hdr.dma_len, rdata->rx.len);
2090 }
2091
2092 static unsigned int xgbe_rx_buf2_len(struct xgbe_ring_data *rdata,
2093                                      struct xgbe_packet_data *packet,
2094                                      unsigned int len)
2095 {
2096         /* Always the full buffer if not the last descriptor */
2097         if (!XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, LAST))
2098                 return rdata->rx.buf.dma_len;
2099
2100         /* Last descriptor so calculate how much of the buffer was used
2101          * for the last bit of data
2102          */
2103         return rdata->rx.len - len;
2104 }
2105
2106 static int xgbe_tx_poll(struct xgbe_channel *channel)
2107 {
2108         struct xgbe_prv_data *pdata = channel->pdata;
2109         struct xgbe_hw_if *hw_if = &pdata->hw_if;
2110         struct xgbe_desc_if *desc_if = &pdata->desc_if;
2111         struct xgbe_ring *ring = channel->tx_ring;
2112         struct xgbe_ring_data *rdata;
2113         struct xgbe_ring_desc *rdesc;
2114         struct net_device *netdev = pdata->netdev;
2115         struct netdev_queue *txq;
2116         int processed = 0;
2117         unsigned int tx_packets = 0, tx_bytes = 0;
2118         unsigned int cur;
2119
2120         DBGPR("-->xgbe_tx_poll\n");
2121
2122         /* Nothing to do if there isn't a Tx ring for this channel */
2123         if (!ring)
2124                 return 0;
2125
2126         cur = ring->cur;
2127
2128         /* Be sure we get ring->cur before accessing descriptor data */
2129         smp_rmb();
2130
2131         txq = netdev_get_tx_queue(netdev, channel->queue_index);
2132
2133         while ((processed < XGBE_TX_DESC_MAX_PROC) &&
2134                (ring->dirty != cur)) {
2135                 rdata = XGBE_GET_DESC_DATA(ring, ring->dirty);
2136                 rdesc = rdata->rdesc;
2137
2138                 if (!hw_if->tx_complete(rdesc))
2139                         break;
2140
2141                 /* Make sure descriptor fields are read after reading the OWN
2142                  * bit */
2143                 dma_rmb();
2144
2145                 if (netif_msg_tx_done(pdata))
2146                         xgbe_dump_tx_desc(pdata, ring, ring->dirty, 1, 0);
2147
2148                 if (hw_if->is_last_desc(rdesc)) {
2149                         tx_packets += rdata->tx.packets;
2150                         tx_bytes += rdata->tx.bytes;
2151                 }
2152
2153                 /* Free the SKB and reset the descriptor for re-use */
2154                 desc_if->unmap_rdata(pdata, rdata);
2155                 hw_if->tx_desc_reset(rdata);
2156
2157                 processed++;
2158                 ring->dirty++;
2159         }
2160
2161         if (!processed)
2162                 return 0;
2163
2164         netdev_tx_completed_queue(txq, tx_packets, tx_bytes);
2165
2166         if ((ring->tx.queue_stopped == 1) &&
2167             (xgbe_tx_avail_desc(ring) > XGBE_TX_DESC_MIN_FREE)) {
2168                 ring->tx.queue_stopped = 0;
2169                 netif_tx_wake_queue(txq);
2170         }
2171
2172         DBGPR("<--xgbe_tx_poll: processed=%d\n", processed);
2173
2174         return processed;
2175 }
2176
2177 static int xgbe_rx_poll(struct xgbe_channel *channel, int budget)
2178 {
2179         struct xgbe_prv_data *pdata = channel->pdata;
2180         struct xgbe_hw_if *hw_if = &pdata->hw_if;
2181         struct xgbe_ring *ring = channel->rx_ring;
2182         struct xgbe_ring_data *rdata;
2183         struct xgbe_packet_data *packet;
2184         struct net_device *netdev = pdata->netdev;
2185         struct napi_struct *napi;
2186         struct sk_buff *skb;
2187         struct skb_shared_hwtstamps *hwtstamps;
2188         unsigned int last, error, context_next, context;
2189         unsigned int len, buf1_len, buf2_len, max_len;
2190         unsigned int received = 0;
2191         int packet_count = 0;
2192
2193         DBGPR("-->xgbe_rx_poll: budget=%d\n", budget);
2194
2195         /* Nothing to do if there isn't a Rx ring for this channel */
2196         if (!ring)
2197                 return 0;
2198
2199         last = 0;
2200         context_next = 0;
2201
2202         napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
2203
2204         rdata = XGBE_GET_DESC_DATA(ring, ring->cur);
2205         packet = &ring->packet_data;
2206         while (packet_count < budget) {
2207                 DBGPR("  cur = %d\n", ring->cur);
2208
2209                 /* First time in loop see if we need to restore state */
2210                 if (!received && rdata->state_saved) {
2211                         skb = rdata->state.skb;
2212                         error = rdata->state.error;
2213                         len = rdata->state.len;
2214                 } else {
2215                         memset(packet, 0, sizeof(*packet));
2216                         skb = NULL;
2217                         error = 0;
2218                         len = 0;
2219                 }
2220
2221 read_again:
2222                 rdata = XGBE_GET_DESC_DATA(ring, ring->cur);
2223
2224                 if (xgbe_rx_dirty_desc(ring) > (XGBE_RX_DESC_CNT >> 3))
2225                         xgbe_rx_refresh(channel);
2226
2227                 if (hw_if->dev_read(channel))
2228                         break;
2229
2230                 received++;
2231                 ring->cur++;
2232
2233                 last = XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES,
2234                                       LAST);
2235                 context_next = XGMAC_GET_BITS(packet->attributes,
2236                                               RX_PACKET_ATTRIBUTES,
2237                                               CONTEXT_NEXT);
2238                 context = XGMAC_GET_BITS(packet->attributes,
2239                                          RX_PACKET_ATTRIBUTES,
2240                                          CONTEXT);
2241
2242                 /* Earlier error, just drain the remaining data */
2243                 if ((!last || context_next) && error)
2244                         goto read_again;
2245
2246                 if (error || packet->errors) {
2247                         if (packet->errors)
2248                                 netif_err(pdata, rx_err, netdev,
2249                                           "error in received packet\n");
2250                         dev_kfree_skb(skb);
2251                         goto next_packet;
2252                 }
2253
2254                 if (!context) {
2255                         /* Get the data length in the descriptor buffers */
2256                         buf1_len = xgbe_rx_buf1_len(rdata, packet);
2257                         len += buf1_len;
2258                         buf2_len = xgbe_rx_buf2_len(rdata, packet, len);
2259                         len += buf2_len;
2260
2261                         if (!skb) {
2262                                 skb = xgbe_create_skb(pdata, napi, rdata,
2263                                                       buf1_len);
2264                                 if (!skb) {
2265                                         error = 1;
2266                                         goto skip_data;
2267                                 }
2268                         }
2269
2270                         if (buf2_len) {
2271                                 dma_sync_single_range_for_cpu(pdata->dev,
2272                                                         rdata->rx.buf.dma_base,
2273                                                         rdata->rx.buf.dma_off,
2274                                                         rdata->rx.buf.dma_len,
2275                                                         DMA_FROM_DEVICE);
2276
2277                                 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
2278                                                 rdata->rx.buf.pa.pages,
2279                                                 rdata->rx.buf.pa.pages_offset,
2280                                                 buf2_len,
2281                                                 rdata->rx.buf.dma_len);
2282                                 rdata->rx.buf.pa.pages = NULL;
2283                         }
2284                 }
2285
2286 skip_data:
2287                 if (!last || context_next)
2288                         goto read_again;
2289
2290                 if (!skb)
2291                         goto next_packet;
2292
2293                 /* Be sure we don't exceed the configured MTU */
2294                 max_len = netdev->mtu + ETH_HLEN;
2295                 if (!(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
2296                     (skb->protocol == htons(ETH_P_8021Q)))
2297                         max_len += VLAN_HLEN;
2298
2299                 if (skb->len > max_len) {
2300                         netif_err(pdata, rx_err, netdev,
2301                                   "packet length exceeds configured MTU\n");
2302                         dev_kfree_skb(skb);
2303                         goto next_packet;
2304                 }
2305
2306                 if (netif_msg_pktdata(pdata))
2307                         xgbe_print_pkt(netdev, skb, false);
2308
2309                 skb_checksum_none_assert(skb);
2310                 if (XGMAC_GET_BITS(packet->attributes,
2311                                    RX_PACKET_ATTRIBUTES, CSUM_DONE))
2312                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2313
2314                 if (XGMAC_GET_BITS(packet->attributes,
2315                                    RX_PACKET_ATTRIBUTES, VLAN_CTAG))
2316                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
2317                                                packet->vlan_ctag);
2318
2319                 if (XGMAC_GET_BITS(packet->attributes,
2320                                    RX_PACKET_ATTRIBUTES, RX_TSTAMP)) {
2321                         u64 nsec;
2322
2323                         nsec = timecounter_cyc2time(&pdata->tstamp_tc,
2324                                                     packet->rx_tstamp);
2325                         hwtstamps = skb_hwtstamps(skb);
2326                         hwtstamps->hwtstamp = ns_to_ktime(nsec);
2327                 }
2328
2329                 if (XGMAC_GET_BITS(packet->attributes,
2330                                    RX_PACKET_ATTRIBUTES, RSS_HASH))
2331                         skb_set_hash(skb, packet->rss_hash,
2332                                      packet->rss_hash_type);
2333
2334                 skb->dev = netdev;
2335                 skb->protocol = eth_type_trans(skb, netdev);
2336                 skb_record_rx_queue(skb, channel->queue_index);
2337
2338                 napi_gro_receive(napi, skb);
2339
2340 next_packet:
2341                 packet_count++;
2342         }
2343
2344         /* Check if we need to save state before leaving */
2345         if (received && (!last || context_next)) {
2346                 rdata = XGBE_GET_DESC_DATA(ring, ring->cur);
2347                 rdata->state_saved = 1;
2348                 rdata->state.skb = skb;
2349                 rdata->state.len = len;
2350                 rdata->state.error = error;
2351         }
2352
2353         DBGPR("<--xgbe_rx_poll: packet_count = %d\n", packet_count);
2354
2355         return packet_count;
2356 }
2357
2358 static int xgbe_one_poll(struct napi_struct *napi, int budget)
2359 {
2360         struct xgbe_channel *channel = container_of(napi, struct xgbe_channel,
2361                                                     napi);
2362         struct xgbe_prv_data *pdata = channel->pdata;
2363         int processed = 0;
2364
2365         DBGPR("-->xgbe_one_poll: budget=%d\n", budget);
2366
2367         /* Cleanup Tx ring first */
2368         xgbe_tx_poll(channel);
2369
2370         /* Process Rx ring next */
2371         processed = xgbe_rx_poll(channel, budget);
2372
2373         /* If we processed everything, we are done */
2374         if ((processed < budget) && napi_complete_done(napi, processed)) {
2375                 /* Enable Tx and Rx interrupts */
2376                 if (pdata->channel_irq_mode)
2377                         xgbe_enable_rx_tx_int(pdata, channel);
2378                 else
2379                         enable_irq(channel->dma_irq);
2380         }
2381
2382         DBGPR("<--xgbe_one_poll: received = %d\n", processed);
2383
2384         return processed;
2385 }
2386
2387 static int xgbe_all_poll(struct napi_struct *napi, int budget)
2388 {
2389         struct xgbe_prv_data *pdata = container_of(napi, struct xgbe_prv_data,
2390                                                    napi);
2391         struct xgbe_channel *channel;
2392         int ring_budget;
2393         int processed, last_processed;
2394         unsigned int i;
2395
2396         DBGPR("-->xgbe_all_poll: budget=%d\n", budget);
2397
2398         processed = 0;
2399         ring_budget = budget / pdata->rx_ring_count;
2400         do {
2401                 last_processed = processed;
2402
2403                 for (i = 0; i < pdata->channel_count; i++) {
2404                         channel = pdata->channel[i];
2405
2406                         /* Cleanup Tx ring first */
2407                         xgbe_tx_poll(channel);
2408
2409                         /* Process Rx ring next */
2410                         if (ring_budget > (budget - processed))
2411                                 ring_budget = budget - processed;
2412                         processed += xgbe_rx_poll(channel, ring_budget);
2413                 }
2414         } while ((processed < budget) && (processed != last_processed));
2415
2416         /* If we processed everything, we are done */
2417         if ((processed < budget) && napi_complete_done(napi, processed)) {
2418                 /* Enable Tx and Rx interrupts */
2419                 xgbe_enable_rx_tx_ints(pdata);
2420         }
2421
2422         DBGPR("<--xgbe_all_poll: received = %d\n", processed);
2423
2424         return processed;
2425 }
2426
2427 void xgbe_dump_tx_desc(struct xgbe_prv_data *pdata, struct xgbe_ring *ring,
2428                        unsigned int idx, unsigned int count, unsigned int flag)
2429 {
2430         struct xgbe_ring_data *rdata;
2431         struct xgbe_ring_desc *rdesc;
2432
2433         while (count--) {
2434                 rdata = XGBE_GET_DESC_DATA(ring, idx);
2435                 rdesc = rdata->rdesc;
2436                 netdev_dbg(pdata->netdev,
2437                            "TX_NORMAL_DESC[%d %s] = %08x:%08x:%08x:%08x\n", idx,
2438                            (flag == 1) ? "QUEUED FOR TX" : "TX BY DEVICE",
2439                            le32_to_cpu(rdesc->desc0),
2440                            le32_to_cpu(rdesc->desc1),
2441                            le32_to_cpu(rdesc->desc2),
2442                            le32_to_cpu(rdesc->desc3));
2443                 idx++;
2444         }
2445 }
2446
2447 void xgbe_dump_rx_desc(struct xgbe_prv_data *pdata, struct xgbe_ring *ring,
2448                        unsigned int idx)
2449 {
2450         struct xgbe_ring_data *rdata;
2451         struct xgbe_ring_desc *rdesc;
2452
2453         rdata = XGBE_GET_DESC_DATA(ring, idx);
2454         rdesc = rdata->rdesc;
2455         netdev_dbg(pdata->netdev,
2456                    "RX_NORMAL_DESC[%d RX BY DEVICE] = %08x:%08x:%08x:%08x\n",
2457                    idx, le32_to_cpu(rdesc->desc0), le32_to_cpu(rdesc->desc1),
2458                    le32_to_cpu(rdesc->desc2), le32_to_cpu(rdesc->desc3));
2459 }
2460
2461 void xgbe_print_pkt(struct net_device *netdev, struct sk_buff *skb, bool tx_rx)
2462 {
2463         struct ethhdr *eth = (struct ethhdr *)skb->data;
2464         unsigned char *buf = skb->data;
2465         unsigned char buffer[128];
2466         unsigned int i, j;
2467
2468         netdev_dbg(netdev, "\n************** SKB dump ****************\n");
2469
2470         netdev_dbg(netdev, "%s packet of %d bytes\n",
2471                    (tx_rx ? "TX" : "RX"), skb->len);
2472
2473         netdev_dbg(netdev, "Dst MAC addr: %pM\n", eth->h_dest);
2474         netdev_dbg(netdev, "Src MAC addr: %pM\n", eth->h_source);
2475         netdev_dbg(netdev, "Protocol: %#06hx\n", ntohs(eth->h_proto));
2476
2477         for (i = 0, j = 0; i < skb->len;) {
2478                 j += snprintf(buffer + j, sizeof(buffer) - j, "%02hhx",
2479                               buf[i++]);
2480
2481                 if ((i % 32) == 0) {
2482                         netdev_dbg(netdev, "  %#06x: %s\n", i - 32, buffer);
2483                         j = 0;
2484                 } else if ((i % 16) == 0) {
2485                         buffer[j++] = ' ';
2486                         buffer[j++] = ' ';
2487                 } else if ((i % 4) == 0) {
2488                         buffer[j++] = ' ';
2489                 }
2490         }
2491         if (i % 32)
2492                 netdev_dbg(netdev, "  %#06x: %s\n", i - (i % 32), buffer);
2493
2494         netdev_dbg(netdev, "\n************** SKB dump ****************\n");
2495 }