drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2011 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/slab.h>
  40
  41 #include "nes.h"
  42
  43 static int wide_ppm_offset;
  44 module_param(wide_ppm_offset, int, 0644);
  45 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  46
  47 static u32 crit_err_count;
  48 u32 int_mod_timer_init;
  49 u32 int_mod_cq_depth_256;
  50 u32 int_mod_cq_depth_128;
  51 u32 int_mod_cq_depth_32;
  52 u32 int_mod_cq_depth_24;
  53 u32 int_mod_cq_depth_16;
  54 u32 int_mod_cq_depth_4;
  55 u32 int_mod_cq_depth_1;
  56 static const u8 nes_max_critical_error_count = 100;
  57 #include "nes_cm.h"
  58
  59 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  60 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  61 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  62                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  63 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  64 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  65 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  66 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  67                                    struct nes_hw_aeqe *aeqe);
  68 static void process_critical_error(struct nes_device *nesdev);
  69 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  70 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  71 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  72
  73 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  74 static unsigned char *nes_iwarp_state_str[] = {
  75         "Non-Existent",
  76         "Idle",
  77         "RTS",
  78         "Closing",
  79         "RSVD1",
  80         "Terminate",
  81         "Error",
  82         "RSVD2",
  83 };
  84
  85 static unsigned char *nes_tcp_state_str[] = {
  86         "Non-Existent",
  87         "Closed",
  88         "Listen",
  89         "SYN Sent",
  90         "SYN Rcvd",
  91         "Established",
  92         "Close Wait",
  93         "FIN Wait 1",
  94         "Closing",
  95         "Last Ack",
  96         "FIN Wait 2",
  97         "Time Wait",
  98         "RSVD1",
  99         "RSVD2",
 100         "RSVD3",
 101         "RSVD4",
 102 };
 103 #endif
 104
 105 static inline void print_ip(struct nes_cm_node *cm_node)
 106 {
 107         unsigned char *rem_addr;
 108         if (cm_node) {
 109                 rem_addr = (unsigned char *)&cm_node->rem_addr;
 110                 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
 111         }
 112 }
 113
 114 /**
 115  * nes_nic_init_timer_defaults
 116  */
 117 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 118 {
 119         unsigned long flags;
 120         struct nes_adapter *nesadapter = nesdev->nesadapter;
 121         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 122
 123         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 124
 125         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 126         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 127         if (jumbomode) {
 128                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 129                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 130                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 131         } else {
 132                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 133                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 134                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 135         }
 136
 137         /* todo use netdev->mtu to set thresholds */
 138         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 139 }
 140
 141
 142 /**
 143  * nes_nic_init_timer
 144  */
 145 static void  nes_nic_init_timer(struct nes_device *nesdev)
 146 {
 147         unsigned long flags;
 148         struct nes_adapter *nesadapter = nesdev->nesadapter;
 149         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 150
 151         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 152
 153         if (shared_timer->timer_in_use_old == 0) {
 154                 nesdev->deepcq_count = 0;
 155                 shared_timer->timer_direction_upward = 0;
 156                 shared_timer->timer_direction_downward = 0;
 157                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 158                 shared_timer->timer_in_use_old = 0;
 159
 160         }
 161         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 162                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 163                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 164                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 165         }
 166         /* todo use netdev->mtu to set thresholds */
 167         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 168 }
 169
 170
 171 /**
 172  * nes_nic_tune_timer
 173  */
 174 static void nes_nic_tune_timer(struct nes_device *nesdev)
 175 {
 176         unsigned long flags;
 177         struct nes_adapter *nesadapter = nesdev->nesadapter;
 178         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 179         u16 cq_count = nesdev->currcq_count;
 180
 181         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 182
 183         if (shared_timer->cq_count_old <= cq_count)
 184                 shared_timer->cq_direction_downward = 0;
 185         else
 186                 shared_timer->cq_direction_downward++;
 187         shared_timer->cq_count_old = cq_count;
 188         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 189                 if (cq_count <= shared_timer->threshold_low &&
 190                     shared_timer->threshold_low > 4) {
 191                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 192                         shared_timer->cq_direction_downward=0;
 193                         nesdev->currcq_count = 0;
 194                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 195                         return;
 196                 }
 197         }
 198
 199         if (cq_count > 1) {
 200                 nesdev->deepcq_count += cq_count;
 201                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 202                         shared_timer->timer_direction_upward++;
 203                         shared_timer->timer_direction_downward = 0;
 204                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 205                         shared_timer->timer_direction_upward = 0;
 206                         shared_timer->timer_direction_downward = 0;
 207                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 208                         shared_timer->timer_direction_downward++;
 209                         shared_timer->timer_direction_upward = 0;
 210                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 211                         shared_timer->timer_in_use -= 2;
 212                         shared_timer->timer_direction_upward = 0;
 213                         shared_timer->timer_direction_downward++;
 214                 } else {
 215                         shared_timer->timer_in_use -= 4;
 216                         shared_timer->timer_direction_upward = 0;
 217                         shared_timer->timer_direction_downward++;
 218                 }
 219
 220                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 221                         shared_timer->timer_in_use += 3;
 222                         shared_timer->timer_direction_upward = 0;
 223                         shared_timer->timer_direction_downward = 0;
 224                 }
 225                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 226                         shared_timer->timer_in_use -= 4 ;
 227                         shared_timer->timer_direction_downward = 0;
 228                         shared_timer->timer_direction_upward = 0;
 229                 }
 230         }
 231
 232         /* boundary checking */
 233         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 234                 shared_timer->timer_in_use = shared_timer->threshold_high;
 235         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 236                 shared_timer->timer_in_use = shared_timer->threshold_low;
 237
 238         nesdev->currcq_count = 0;
 239
 240         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 241 }
 242
 243
 244 /**
 245  * nes_init_adapter - initialize adapter
 246  */
 247 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 248         struct nes_adapter *nesadapter = NULL;
 249         unsigned long num_pds;
 250         u32 u32temp;
 251         u32 port_count;
 252         u16 max_rq_wrs;
 253         u16 max_sq_wrs;
 254         u32 max_mr;
 255         u32 max_256pbl;
 256         u32 max_4kpbl;
 257         u32 max_qp;
 258         u32 max_irrq;
 259         u32 max_cq;
 260         u32 hte_index_mask;
 261         u32 adapter_size;
 262         u32 arp_table_size;
 263         u16 vendor_id;
 264         u16 device_id;
 265         u8  OneG_Mode;
 266         u8  func_index;
 267
 268         /* search the list of existing adapters */
 269         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 270                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 271                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 272                                 nesdev->pcidev->devfn,
 273                                 PCI_SLOT(nesadapter->devfn),
 274                                 nesadapter->bus_number,
 275                                 PCI_SLOT(nesdev->pcidev->devfn),
 276                                 nesdev->pcidev->bus->number );
 277                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 278                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 279                         nesadapter->ref_count++;
 280                         return nesadapter;
 281                 }
 282         }
 283
 284         /* no adapter found */
 285         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 286         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 287                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 288                                 hw_rev);
 289                 return NULL;
 290         }
 291
 292         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 293                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 295                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 296                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 297
 298         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 299
 300
 301         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 302                 return NULL;
 303
 304         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 305         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 306
 307         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 308         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 309                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 310                                 max_qp, u32temp);
 311                 max_qp = (u32)1 << (u32temp & 0x001f);
 312         }
 313
 314         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 315         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 316                         max_qp, hte_index_mask);
 317
 318         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 319
 320         max_irrq = 1 << (u32temp & 0x001f);
 321
 322         if (max_qp > max_irrq) {
 323                 max_qp = max_irrq;
 324                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 325                                 max_qp);
 326         }
 327
 328         /* there should be no reason to allocate more pds than qps */
 329         if (num_pds > max_qp)
 330                 num_pds = max_qp;
 331
 332         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 333         max_mr = (u32)8192 << (u32temp & 0x7);
 334
 335         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 336         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 337         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 338         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 339
 340         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 341         arp_table_size = 1 << u32temp;
 342
 343         adapter_size = (sizeof(struct nes_adapter) +
 344                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 348         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 349         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 350         adapter_size += sizeof(struct nes_qp **) * max_qp;
 351
 352         /* allocate a new adapter struct */
 353         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 354         if (nesadapter == NULL) {
 355                 return NULL;
 356         }
 357
 358         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 359                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 360
 361         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 362                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 363                 kfree(nesadapter);
 364                 return NULL;
 365         }
 366
 367         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 368                                 (nesadapter->mac_addr_low >> 24);
 369
 370         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 371                                  PCI_DEVICE_ID, &device_id);
 372         nesadapter->vendor_part_id = device_id;
 373
 374         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 375                                                         OneG_Mode)) {
 376                 kfree(nesadapter);
 377                 return NULL;
 378         }
 379         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 380
 381         memset(nesadapter->pft_mcast_map, 255,
 382                sizeof nesadapter->pft_mcast_map);
 383
 384         /* populate the new nesadapter */
 385         nesadapter->devfn = nesdev->pcidev->devfn;
 386         nesadapter->bus_number = nesdev->pcidev->bus->number;
 387         nesadapter->ref_count = 1;
 388         nesadapter->timer_int_req = 0xffff0000;
 389         nesadapter->OneG_Mode = OneG_Mode;
 390         nesadapter->doorbell_start = nesdev->doorbell_region;
 391
 392         /* nesadapter->tick_delta = clk_divisor; */
 393         nesadapter->hw_rev = hw_rev;
 394         nesadapter->port_count = port_count;
 395
 396         nesadapter->max_qp = max_qp;
 397         nesadapter->hte_index_mask = hte_index_mask;
 398         nesadapter->max_irrq = max_irrq;
 399         nesadapter->max_mr = max_mr;
 400         nesadapter->max_256pbl = max_256pbl - 1;
 401         nesadapter->max_4kpbl = max_4kpbl - 1;
 402         nesadapter->max_cq = max_cq;
 403         nesadapter->free_256pbl = max_256pbl - 1;
 404         nesadapter->free_4kpbl = max_4kpbl - 1;
 405         nesadapter->max_pd = num_pds;
 406         nesadapter->arp_table_size = arp_table_size;
 407
 408         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 409         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 410                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 411                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 412                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 413         } else {
 414                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 415                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 416                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 417                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 418         }
 419         /* Setup and enable the periodic timer */
 420         if (nesadapter->et_rx_coalesce_usecs_irq)
 421                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 422                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 423         else
 424                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 425
 426         nesadapter->base_pd = 1;
 427
 428         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 429                                        IB_DEVICE_MEM_WINDOW |
 430                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 431
 432         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 433                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 434         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 435         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 436         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 437         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 438         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 439
 440
 441         /* mark the usual suspect QPs, MR and CQs as in use */
 442         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 443                 set_bit(u32temp, nesadapter->allocated_qps);
 444                 set_bit(u32temp, nesadapter->allocated_cqs);
 445         }
 446         set_bit(0, nesadapter->allocated_mrs);
 447
 448         for (u32temp = 0; u32temp < 20; u32temp++)
 449                 set_bit(u32temp, nesadapter->allocated_pds);
 450         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 451
 452         max_rq_wrs = ((u32temp >> 8) & 3);
 453         switch (max_rq_wrs) {
 454                 case 0:
 455                         max_rq_wrs = 4;
 456                         break;
 457                 case 1:
 458                         max_rq_wrs = 16;
 459                         break;
 460                 case 2:
 461                         max_rq_wrs = 32;
 462                         break;
 463                 case 3:
 464                         max_rq_wrs = 512;
 465                         break;
 466         }
 467
 468         max_sq_wrs = (u32temp & 3);
 469         switch (max_sq_wrs) {
 470                 case 0:
 471                         max_sq_wrs = 4;
 472                         break;
 473                 case 1:
 474                         max_sq_wrs = 16;
 475                         break;
 476                 case 2:
 477                         max_sq_wrs = 32;
 478                         break;
 479                 case 3:
 480                         max_sq_wrs = 512;
 481                         break;
 482         }
 483         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 484         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 485
 486         nesadapter->max_sge = 4;
 487         nesadapter->max_cqe = 32766;
 488
 489         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 490                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 491                 kfree(nesadapter);
 492                 return NULL;
 493         }
 494
 495         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 496         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 497                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 498
 499         /* setup port configuration */
 500         if (nesadapter->port_count == 1) {
 501                 nesadapter->log_port = 0x00000000;
 502                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 503                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 504                 else
 505                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 506         } else {
 507                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 508                         nesadapter->log_port = 0x000000D8;
 509                 } else {
 510                         if (nesadapter->port_count == 2)
 511                                 nesadapter->log_port = 0x00000044;
 512                         else
 513                                 nesadapter->log_port = 0x000000e4;
 514                 }
 515                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 516         }
 517
 518         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 519                                                 nesadapter->log_port);
 520         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 521                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 522
 523         spin_lock_init(&nesadapter->resource_lock);
 524         spin_lock_init(&nesadapter->phy_lock);
 525         spin_lock_init(&nesadapter->pbl_lock);
 526         spin_lock_init(&nesadapter->periodic_timer_lock);
 527
 528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 529         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 530         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 531         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 532
 533         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 534                 u32 pcs_control_status0, pcs_control_status1;
 535                 u32 reset_value;
 536                 u32 i = 0;
 537                 u32 int_cnt = 0;
 538                 u32 ext_cnt = 0;
 539                 unsigned long flags;
 540                 u32 j = 0;
 541
 542                 pcs_control_status0 = nes_read_indexed(nesdev,
 543                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 544                 pcs_control_status1 = nes_read_indexed(nesdev,
 545                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 546
 547                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 548                         pcs_control_status0 = nes_read_indexed(nesdev,
 549                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 550                         pcs_control_status1 = nes_read_indexed(nesdev,
 551                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 552                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 553                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 554                                 int_cnt++;
 555                         msleep(1);
 556                 }
 557                 if (int_cnt > 1) {
 558                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 559                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 560                         mh_detected++;
 561                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 562                         reset_value |= 0x0000003d;
 563                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 564
 565                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 566                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 567                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 568
 569                         pcs_control_status0 = nes_read_indexed(nesdev,
 570                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 571                         pcs_control_status1 = nes_read_indexed(nesdev,
 572                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 573
 574                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 575                                 pcs_control_status0 = nes_read_indexed(nesdev,
 576                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 577                                 pcs_control_status1 = nes_read_indexed(nesdev,
 578                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 579                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 580                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 581                                         if (++ext_cnt > int_cnt) {
 582                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 583                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 584                                                                 0x0000F088);
 585                                                 mh_detected++;
 586                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 587                                                 reset_value |= 0x0000003d;
 588                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 589
 590                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 591                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 592                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 593                                                 break;
 594                                         }
 595                                 }
 596                                 msleep(1);
 597                         }
 598                 }
 599         }
 600
 601         if (nesadapter->hw_rev == NE020_REV) {
 602                 init_timer(&nesadapter->mh_timer);
 603                 nesadapter->mh_timer.function = nes_mh_fix;
 604                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 605                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 606                 add_timer(&nesadapter->mh_timer);
 607         } else {
 608                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 609         }
 610
 611         init_timer(&nesadapter->lc_timer);
 612         nesadapter->lc_timer.function = nes_clc;
 613         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 614         nesadapter->lc_timer.data = (unsigned long)nesdev;
 615         add_timer(&nesadapter->lc_timer);
 616
 617         list_add_tail(&nesadapter->list, &nes_adapter_list);
 618
 619         for (func_index = 0; func_index < 8; func_index++) {
 620                 pci_bus_read_config_word(nesdev->pcidev->bus,
 621                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 622                                         func_index), 0, &vendor_id);
 623                 if (vendor_id == 0xffff)
 624                         break;
 625         }
 626         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 627                 func_index, pci_name(nesdev->pcidev));
 628         nesadapter->adapter_fcn_count = func_index;
 629
 630         return nesadapter;
 631 }
 632
 633
 634 /**
 635  * nes_reset_adapter_ne020
 636  */
 637 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 638 {
 639         u32 port_count;
 640         u32 u32temp;
 641         u32 i;
 642
 643         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 644         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 645         /* TODO: assuming that both SERDES are set the same for now */
 646         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 647         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 648                         u32temp, port_count);
 649         if (*OneG_Mode)
 650                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 651         u32temp &= 0xff00ffc0;
 652         switch (port_count) {
 653                 case 1:
 654                         u32temp |= 0x00ee0000;
 655                         break;
 656                 case 2:
 657                         u32temp |= 0x00cc0000;
 658                         break;
 659                 case 4:
 660                         u32temp |= 0x00000000;
 661                         break;
 662                 default:
 663                         return 0;
 664                         break;
 665         }
 666
 667         /* check and do full reset if needed */
 668         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 669                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 670                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 671
 672                 i = 0;
 673                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 674                         mdelay(1);
 675                 if (i > 10000) {
 676                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 677                         return 0;
 678                 }
 679
 680                 i = 0;
 681                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 682                         mdelay(1);
 683                 if (i > 10000) {
 684                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 685                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 686                         return 0;
 687                 }
 688         }
 689
 690         /* port reset */
 691         switch (port_count) {
 692                 case 1:
 693                         u32temp |= 0x00ee0010;
 694                         break;
 695                 case 2:
 696                         u32temp |= 0x00cc0030;
 697                         break;
 698                 case 4:
 699                         u32temp |= 0x00000030;
 700                         break;
 701         }
 702
 703         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 704         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 705
 706         i = 0;
 707         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 708                 mdelay(1);
 709         if (i > 10000) {
 710                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 711                 return 0;
 712         }
 713
 714         /* serdes 0 */
 715         i = 0;
 716         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 717                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 718                 mdelay(1);
 719         if (i > 5000) {
 720                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 721                 return 0;
 722         }
 723
 724         /* serdes 1 */
 725         if (port_count > 1) {
 726                 i = 0;
 727                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 728                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 729                         mdelay(1);
 730                 if (i > 5000) {
 731                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 732                         return 0;
 733                 }
 734         }
 735
 736         return port_count;
 737 }
 738
 739
 740 /**
 741  * nes_init_serdes
 742  */
 743 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 744                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 745 {
 746         int i;
 747         u32 u32temp;
 748         u32 sds;
 749
 750         if (hw_rev != NE020_REV) {
 751                 /* init serdes 0 */
 752                 switch (nesadapter->phy_type[0]) {
 753                 case NES_PHY_TYPE_CX4:
 754                         if (wide_ppm_offset)
 755                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 756                         else
 757                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 758                         break;
 759                 case NES_PHY_TYPE_KR:
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 762                         break;
 763                 case NES_PHY_TYPE_PUMA_1G:
 764                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 765                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 766                         sds |= 0x00000100;
 767                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 768                         break;
 769                 default:
 770                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 771                         break;
 772                 }
 773
 774                 if (!OneG_Mode)
 775                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 776
 777                 if (port_count < 2)
 778                         return 0;
 779
 780                 /* init serdes 1 */
 781                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 782                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 783
 784                 switch (nesadapter->phy_type[1]) {
 785                 case NES_PHY_TYPE_ARGUS:
 786                 case NES_PHY_TYPE_SFP_D:
 787                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 788                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 789                         break;
 790                 case NES_PHY_TYPE_CX4:
 791                         if (wide_ppm_offset)
 792                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 793                         break;
 794                 case NES_PHY_TYPE_KR:
 795                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 796                         break;
 797                 case NES_PHY_TYPE_PUMA_1G:
 798                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 799                         sds |= 0x000000100;
 800                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 801                 }
 802                 if (!OneG_Mode) {
 803                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 804                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 805                         sds &= 0xFFFFFFBF;
 806                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 807                 }
 808         } else {
 809                 /* init serdes 0 */
 810                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 811                 i = 0;
 812                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 813                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 814                         mdelay(1);
 815                 if (i > 5000) {
 816                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 817                         return 1;
 818                 }
 819                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 820                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 821                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 822                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 823                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 824                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 825                 if (OneG_Mode)
 826                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 827                 else
 828                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 829
 830                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 831                 if (port_count > 1) {
 832                         /* init serdes 1 */
 833                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 834                         i = 0;
 835                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 836                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 837                                 mdelay(1);
 838                         if (i > 5000) {
 839                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 840                                 /* return 1; */
 841                         }
 842                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 843                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 844                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 845                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 846                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 847                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 848                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 849                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 850                 }
 851         }
 852         return 0;
 853 }
 854
 855
 856 /**
 857  * nes_init_csr_ne020
 858  * Initialize registers for ne020 hardware
 859  */
 860 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 861 {
 862         u32 u32temp;
 863
 864         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 865
 866         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 867         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 868         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 869         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 870         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 871         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 872         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 873         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 874
 875         /* TODO: move these MAC register settings to NIC bringup */
 876         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 877         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 878         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 879         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 880         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 881         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 882         if (port_count > 1) {
 883                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 885                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 886                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 887                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 888                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 889                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 890         }
 891         if (port_count > 2) {
 892                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 893                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 894                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 895                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 896                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 897                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 898                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 899
 900                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 901                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 902                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 903                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 904                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 905                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 906                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 907         }
 908
 909         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 910         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 911         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 912                                                          0x00000001);
 913         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 914         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 915         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 916         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 917         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 918
 919         /* TODO: move this to code, get from EEPROM */
 920         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 921         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 922         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 923
 924         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 925         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 926
 927         if (hw_rev != NE020_REV) {
 928                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 929                 u32temp |= 0x80000000;
 930                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 931                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 932                 u32temp &= 0x7fffffff;
 933                 u32temp |= 0x7fff0010;
 934                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 935                 if (port_count > 1) {
 936                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 937                         u32temp &= 0x7fffffff;
 938                         u32temp |= 0x7fff0010;
 939                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 940                 }
 941         }
 942 }
 943
 944
 945 /**
 946  * nes_destroy_adapter - destroy the adapter structure
 947  */
 948 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 949 {
 950         struct nes_adapter *tmp_adapter;
 951
 952         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 953                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 954                                 tmp_adapter);
 955         }
 956
 957         nesadapter->ref_count--;
 958         if (!nesadapter->ref_count) {
 959                 if (nesadapter->hw_rev == NE020_REV) {
 960                         del_timer(&nesadapter->mh_timer);
 961                 }
 962                 del_timer(&nesadapter->lc_timer);
 963
 964                 list_del(&nesadapter->list);
 965                 kfree(nesadapter);
 966         }
 967 }
 968
 969
 970 /**
 971  * nes_init_cqp
 972  */
 973 int nes_init_cqp(struct nes_device *nesdev)
 974 {
 975         struct nes_adapter *nesadapter = nesdev->nesadapter;
 976         struct nes_hw_cqp_qp_context *cqp_qp_context;
 977         struct nes_hw_cqp_wqe *cqp_wqe;
 978         struct nes_hw_ceq *ceq;
 979         struct nes_hw_ceq *nic_ceq;
 980         struct nes_hw_aeq *aeq;
 981         void *vmem;
 982         dma_addr_t pmem;
 983         u32 count=0;
 984         u32 cqp_head;
 985         u64 u64temp;
 986         u32 u32temp;
 987
 988         /* allocate CQP memory */
 989         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 990         /* SQ is 512 byte aligned, others are 256 byte aligned */
 991         nesdev->cqp_mem_size = 512 +
 992                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 993                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 994                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 995                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 996                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 997                         sizeof(struct nes_hw_cqp_qp_context);
 998
 999         nesdev->cqp_vbase = pci_zalloc_consistent(nesdev->pcidev,
1000                                                   nesdev->cqp_mem_size,
1001                                                   &nesdev->cqp_pbase);
1002         if (!nesdev->cqp_vbase) {
1003                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1004                 return -ENOMEM;
1005         }
1006
1007         /* Allocate a twice the number of CQP requests as the SQ size */
1008         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1009                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1010         if (nesdev->nes_cqp_requests == NULL) {
1011                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1012                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1013                                 nesdev->cqp.sq_pbase);
1014                 return -ENOMEM;
1015         }
1016
1017         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1018                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1019
1020         spin_lock_init(&nesdev->cqp.lock);
1021         init_waitqueue_head(&nesdev->cqp.waitq);
1022
1023         /* Setup Various Structures */
1024         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1025                         ~(unsigned long)(512 - 1));
1026         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1027                         ~(unsigned long long)(512 - 1));
1028
1029         nesdev->cqp.sq_vbase = vmem;
1030         nesdev->cqp.sq_pbase = pmem;
1031         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1032         nesdev->cqp.sq_head = 0;
1033         nesdev->cqp.sq_tail = 0;
1034         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1035
1036         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1037         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1038
1039         nesdev->ccq.cq_vbase = vmem;
1040         nesdev->ccq.cq_pbase = pmem;
1041         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1042         nesdev->ccq.cq_head = 0;
1043         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1044         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1045
1046         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1047         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1048
1049         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1050         ceq = &nesadapter->ceq[nesdev->ceq_index];
1051         ceq->ceq_vbase = vmem;
1052         ceq->ceq_pbase = pmem;
1053         ceq->ceq_size = NES_CCEQ_SIZE;
1054         ceq->ceq_head = 0;
1055
1056         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1057         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1058
1059         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1060         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1061         nic_ceq->ceq_vbase = vmem;
1062         nic_ceq->ceq_pbase = pmem;
1063         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1064         nic_ceq->ceq_head = 0;
1065
1066         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1067         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1068
1069         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1070         aeq->aeq_vbase = vmem;
1071         aeq->aeq_pbase = pmem;
1072         aeq->aeq_size = nesadapter->max_qp;
1073         aeq->aeq_head = 0;
1074
1075         /* Setup QP Context */
1076         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1077         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1078
1079         cqp_qp_context = vmem;
1080         cqp_qp_context->context_words[0] =
1081                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1082         cqp_qp_context->context_words[1] = 0;
1083         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1084         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1085
1086
1087         /* Write the address to Create CQP */
1088         if ((sizeof(dma_addr_t) > 4)) {
1089                 nes_write_indexed(nesdev,
1090                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1091                                 ((u64)pmem) >> 32);
1092         } else {
1093                 nes_write_indexed(nesdev,
1094                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1095         }
1096         nes_write_indexed(nesdev,
1097                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1098                         (u32)pmem);
1099
1100         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1101         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1102
1103         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1104                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1105                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1106         }
1107
1108         /* Write Create CCQ WQE */
1109         cqp_head = nesdev->cqp.sq_head++;
1110         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1111         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1112         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1113                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1114                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1115         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1116                             (nesdev->ccq.cq_number |
1117                              ((u32)nesdev->ceq_index << 16)));
1118         u64temp = (u64)nesdev->ccq.cq_pbase;
1119         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1120         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1121         u64temp = (unsigned long)&nesdev->ccq;
1122         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1123                         cpu_to_le32((u32)(u64temp >> 1));
1124         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1125                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1126         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1127
1128         /* Write Create CEQ WQE */
1129         cqp_head = nesdev->cqp.sq_head++;
1130         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1131         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1132         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1133                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1134         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1135         u64temp = (u64)ceq->ceq_pbase;
1136         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1137
1138         /* Write Create AEQ WQE */
1139         cqp_head = nesdev->cqp.sq_head++;
1140         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1141         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1142         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1143                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1144         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1145         u64temp = (u64)aeq->aeq_pbase;
1146         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1147
1148         /* Write Create NIC CEQ WQE */
1149         cqp_head = nesdev->cqp.sq_head++;
1150         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1151         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1152         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1153                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1154         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1155         u64temp = (u64)nic_ceq->ceq_pbase;
1156         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1157
1158         /* Poll until CCQP done */
1159         count = 0;
1160         do {
1161                 if (count++ > 1000) {
1162                         printk(KERN_ERR PFX "Error creating CQP\n");
1163                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1164                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1165                         return -1;
1166                 }
1167                 udelay(10);
1168         } while (!(nes_read_indexed(nesdev,
1169                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1170
1171         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1172                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1173
1174         u32temp = 0x04800000;
1175         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1176
1177         /* wait for the CCQ, CEQ, and AEQ to get created */
1178         count = 0;
1179         do {
1180                 if (count++ > 1000) {
1181                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1182                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1183                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1184                         return -1;
1185                 }
1186                 udelay(10);
1187         } while (((nes_read_indexed(nesdev,
1188                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1189
1190         /* dump the QP status value */
1191         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1192                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1193
1194         nesdev->cqp.sq_tail++;
1195
1196         return 0;
1197 }
1198
1199
1200 /**
1201  * nes_destroy_cqp
1202  */
1203 int nes_destroy_cqp(struct nes_device *nesdev)
1204 {
1205         struct nes_hw_cqp_wqe *cqp_wqe;
1206         u32 count = 0;
1207         u32 cqp_head;
1208         unsigned long flags;
1209
1210         do {
1211                 if (count++ > 1000)
1212                         break;
1213                 udelay(10);
1214         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1215
1216         /* Reset CCQ */
1217         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1218                         nesdev->ccq.cq_number);
1219
1220         /* Disable device interrupts */
1221         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1222
1223         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1224
1225         /* Destroy the AEQ */
1226         cqp_head = nesdev->cqp.sq_head++;
1227         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1228         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1229         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1230                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1231         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1232
1233         /* Destroy the NIC CEQ */
1234         cqp_head = nesdev->cqp.sq_head++;
1235         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1236         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1237         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1238                         ((u32)nesdev->nic_ceq_index << 8));
1239
1240         /* Destroy the CEQ */
1241         cqp_head = nesdev->cqp.sq_head++;
1242         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1243         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1244         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1245                         (nesdev->ceq_index << 8));
1246
1247         /* Destroy the CCQ */
1248         cqp_head = nesdev->cqp.sq_head++;
1249         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1250         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1251         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1252         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1253                         ((u32)nesdev->ceq_index << 16));
1254
1255         /* Destroy CQP */
1256         cqp_head = nesdev->cqp.sq_head++;
1257         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1258         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1259         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1260                         NES_CQP_QP_TYPE_CQP);
1261         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1262
1263         barrier();
1264         /* Ring doorbell (5 WQEs) */
1265         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1266
1267         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1268
1269         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1270         count = 0;
1271         do {
1272                 if (count++ > 1000) {
1273                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1274                                         PCI_FUNC(nesdev->pcidev->devfn));
1275                         break;
1276                 }
1277                 udelay(10);
1278         } while (((nes_read_indexed(nesdev,
1279                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1280
1281         /* dump the QP status value */
1282         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1283                         PCI_FUNC(nesdev->pcidev->devfn),
1284                         nes_read_indexed(nesdev,
1285                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1286
1287         kfree(nesdev->nes_cqp_requests);
1288
1289         /* Free the control structures */
1290         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1291                         nesdev->cqp.sq_pbase);
1292
1293         return 0;
1294 }
1295
1296
1297 /**
1298  * nes_init_1g_phy
1299  */
1300 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1301 {
1302         u32 counter = 0;
1303         u16 phy_data;
1304         int ret = 0;
1305
1306         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1307         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1308
1309         /* Reset the PHY */
1310         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1311         udelay(100);
1312         counter = 0;
1313         do {
1314                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1315                 if (counter++ > 100) {
1316                         ret = -1;
1317                         break;
1318                 }
1319         } while (phy_data & 0x8000);
1320
1321         /* Setting no phy loopback */
1322         phy_data &= 0xbfff;
1323         phy_data |= 0x1140;
1324         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1325         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1326         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1327         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1328
1329         /* Setting the interrupt mask */
1330         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1331         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1332         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1333
1334         /* turning on flow control */
1335         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1336         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1337         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1338
1339         /* Clear Half duplex */
1340         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1341         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1342         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1343
1344         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1345         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1346
1347         return ret;
1348 }
1349
1350
1351 /**
1352  * nes_init_2025_phy
1353  */
1354 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1355 {
1356         u32 temp_phy_data = 0;
1357         u32 temp_phy_data2 = 0;
1358         u32 counter = 0;
1359         u32 sds;
1360         u32 mac_index = nesdev->mac_index;
1361         int ret = 0;
1362         unsigned int first_attempt = 1;
1363
1364         /* Check firmware heartbeat */
1365         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1366         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1367         udelay(1500);
1368         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1369         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1370
1371         if (temp_phy_data != temp_phy_data2) {
1372                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1373                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1374                 if ((temp_phy_data & 0xff) > 0x20)
1375                         return 0;
1376                 printk(PFX "Reinitialize external PHY\n");
1377         }
1378
1379         /* no heartbeat, configure the PHY */
1380         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1381         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1382         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1383         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1384
1385         switch (phy_type) {
1386         case NES_PHY_TYPE_ARGUS:
1387                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1388                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1389                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1390                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1391                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1392                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1393                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1394
1395                 /* setup LEDs */
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1397                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1398                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1399                 break;
1400
1401         case NES_PHY_TYPE_SFP_D:
1402                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1403                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1406                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1407                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1408                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1409
1410                 /* setup LEDs */
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1412                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1413                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1414                 break;
1415
1416         case NES_PHY_TYPE_KR:
1417                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1418                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1421                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1422                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1423                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1424
1425                 /* setup LEDs */
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1427                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1428                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1429
1430                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1431                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1432                 break;
1433         }
1434
1435         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1436
1437         /* Bring PHY out of reset */
1438         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1439
1440         /* Check for heartbeat */
1441         counter = 0;
1442         mdelay(690);
1443         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1444         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1445         do {
1446                 if (counter++ > 150) {
1447                         printk(PFX "No PHY heartbeat\n");
1448                         break;
1449                 }
1450                 mdelay(1);
1451                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1452                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1453         } while ((temp_phy_data2 == temp_phy_data));
1454
1455         /* wait for tracking */
1456         counter = 0;
1457         do {
1458                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1459                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1460                 if (counter++ > 300) {
1461                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1462                                 first_attempt = 0;
1463                                 counter = 0;
1464                                 /* reset AMCC PHY and try again */
1465                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1466                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1467                                 continue;
1468                         } else {
1469                                 ret = 1;
1470                                 break;
1471                         }
1472                 }
1473                 mdelay(10);
1474         } while ((temp_phy_data & 0xff) < 0x30);
1475
1476         /* setup signal integrity */
1477         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1478         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1479         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1480         if (phy_type == NES_PHY_TYPE_KR) {
1481                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1482         } else {
1483                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1484                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1485         }
1486
1487         /* reset serdes */
1488         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1489         sds |= 0x1;
1490         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1491         sds &= 0xfffffffe;
1492         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1493
1494         counter = 0;
1495         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1496                         && (counter++ < 5000))
1497                 ;
1498
1499         return ret;
1500 }
1501
1502
1503 /**
1504  * nes_init_phy
1505  */
1506 int nes_init_phy(struct nes_device *nesdev)
1507 {
1508         struct nes_adapter *nesadapter = nesdev->nesadapter;
1509         u32 mac_index = nesdev->mac_index;
1510         u32 tx_config = 0;
1511         unsigned long flags;
1512         u8  phy_type = nesadapter->phy_type[mac_index];
1513         u8  phy_index = nesadapter->phy_index[mac_index];
1514         int ret = 0;
1515
1516         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1517         if (phy_type == NES_PHY_TYPE_1G) {
1518                 /* setup 1G MDIO operation */
1519                 tx_config &= 0xFFFFFFE3;
1520                 tx_config |= 0x04;
1521         } else {
1522                 /* setup 10G MDIO operation */
1523                 tx_config &= 0xFFFFFFE3;
1524                 tx_config |= 0x1D;
1525         }
1526         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1527
1528         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1529
1530         switch (phy_type) {
1531         case NES_PHY_TYPE_1G:
1532                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1533                 break;
1534         case NES_PHY_TYPE_ARGUS:
1535         case NES_PHY_TYPE_SFP_D:
1536         case NES_PHY_TYPE_KR:
1537                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1538                 break;
1539         }
1540
1541         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1542
1543         return ret;
1544 }
1545
1546
1547 /**
1548  * nes_replenish_nic_rq
1549  */
1550 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1551 {
1552         unsigned long flags;
1553         dma_addr_t bus_address;
1554         struct sk_buff *skb;
1555         struct nes_hw_nic_rq_wqe *nic_rqe;
1556         struct nes_hw_nic *nesnic;
1557         struct nes_device *nesdev;
1558         struct nes_rskb_cb *cb;
1559         u32 rx_wqes_posted = 0;
1560
1561         nesnic = &nesvnic->nic;
1562         nesdev = nesvnic->nesdev;
1563         spin_lock_irqsave(&nesnic->rq_lock, flags);
1564         if (nesnic->replenishing_rq !=0) {
1565                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1566                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1567                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1568                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1569                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1570                         add_timer(&nesvnic->rq_wqes_timer);
1571                 } else
1572                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1573                 return;
1574         }
1575         nesnic->replenishing_rq = 1;
1576         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1577         do {
1578                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1579                 if (skb) {
1580                         skb->dev = nesvnic->netdev;
1581
1582                         bus_address = pci_map_single(nesdev->pcidev,
1583                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1584                         cb = (struct nes_rskb_cb *)&skb->cb[0];
1585                         cb->busaddr = bus_address;
1586                         cb->maplen = nesvnic->max_frame_size;
1587
1588                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1589                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1590                                         cpu_to_le32(nesvnic->max_frame_size);
1591                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1592                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1593                                         cpu_to_le32((u32)bus_address);
1594                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1595                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1596                         nesnic->rx_skb[nesnic->rq_head] = skb;
1597                         nesnic->rq_head++;
1598                         nesnic->rq_head &= nesnic->rq_size - 1;
1599                         atomic_dec(&nesvnic->rx_skbs_needed);
1600                         barrier();
1601                         if (++rx_wqes_posted == 255) {
1602                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1603                                 rx_wqes_posted = 0;
1604                         }
1605                 } else {
1606                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1607                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1608                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1609                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1610                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1611                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1612                                 add_timer(&nesvnic->rq_wqes_timer);
1613                         } else
1614                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1615                         break;
1616                 }
1617         } while (atomic_read(&nesvnic->rx_skbs_needed));
1618         barrier();
1619         if (rx_wqes_posted)
1620                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1621         nesnic->replenishing_rq = 0;
1622 }
1623
1624
1625 /**
1626  * nes_rq_wqes_timeout
1627  */
1628 static void nes_rq_wqes_timeout(unsigned long parm)
1629 {
1630         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1631         printk("%s: Timer fired.\n", __func__);
1632         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1633         if (atomic_read(&nesvnic->rx_skbs_needed))
1634                 nes_replenish_nic_rq(nesvnic);
1635 }
1636
1637
1638 /**
1639  * nes_init_nic_qp
1640  */
1641 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1642 {
1643         struct nes_hw_cqp_wqe *cqp_wqe;
1644         struct nes_hw_nic_sq_wqe *nic_sqe;
1645         struct nes_hw_nic_qp_context *nic_context;
1646         struct sk_buff *skb;
1647         struct nes_hw_nic_rq_wqe *nic_rqe;
1648         struct nes_vnic *nesvnic = netdev_priv(netdev);
1649         unsigned long flags;
1650         void *vmem;
1651         dma_addr_t pmem;
1652         u64 u64temp;
1653         int ret;
1654         u32 cqp_head;
1655         u32 counter;
1656         u32 wqe_count;
1657         struct nes_rskb_cb *cb;
1658         u8 jumbomode=0;
1659
1660         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1661         nesvnic->nic_mem_size = 256 +
1662                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1663                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1664                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1665                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1666                         sizeof(struct nes_hw_nic_qp_context);
1667
1668         nesvnic->nic_vbase = pci_zalloc_consistent(nesdev->pcidev,
1669                                                    nesvnic->nic_mem_size,
1670                                                    &nesvnic->nic_pbase);
1671         if (!nesvnic->nic_vbase) {
1672                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1673                 return -ENOMEM;
1674         }
1675         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1676                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1677
1678         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1679                         ~(unsigned long)(256 - 1));
1680         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1681                         ~(unsigned long long)(256 - 1));
1682
1683         /* Setup the first Fragment buffers */
1684         nesvnic->nic.first_frag_vbase = vmem;
1685
1686         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1687                 nesvnic->nic.frag_paddr[counter] = pmem;
1688                 pmem += sizeof(struct nes_first_frag);
1689         }
1690
1691         /* setup the SQ */
1692         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1693
1694         nesvnic->nic.sq_vbase = (void *)vmem;
1695         nesvnic->nic.sq_pbase = pmem;
1696         nesvnic->nic.sq_head = 0;
1697         nesvnic->nic.sq_tail = 0;
1698         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1699         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1700                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1701                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1702                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1703                                 NES_NIC_SQ_WQE_COMPLETION);
1704                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1705                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1706                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1707                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1708                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1709                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1710         }
1711
1712         nesvnic->get_cqp_request = nes_get_cqp_request;
1713         nesvnic->post_cqp_request = nes_post_cqp_request;
1714         nesvnic->mcrq_mcast_filter = NULL;
1715
1716         spin_lock_init(&nesvnic->nic.rq_lock);
1717
1718         /* setup the RQ */
1719         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1720         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1721
1722
1723         nesvnic->nic.rq_vbase = vmem;
1724         nesvnic->nic.rq_pbase = pmem;
1725         nesvnic->nic.rq_head = 0;
1726         nesvnic->nic.rq_tail = 0;
1727         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1728
1729         /* setup the CQ */
1730         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1731         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1732
1733         if (nesdev->nesadapter->netdev_count > 2)
1734                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1735         else
1736                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1737
1738         nesvnic->nic_cq.cq_vbase = vmem;
1739         nesvnic->nic_cq.cq_pbase = pmem;
1740         nesvnic->nic_cq.cq_head = 0;
1741         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1742
1743         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1744
1745         /* Send CreateCQ request to CQP */
1746         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1747         cqp_head = nesdev->cqp.sq_head;
1748
1749         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1750         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1751
1752         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1753                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1754                         ((u32)nesvnic->nic_cq.cq_size << 16));
1755         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1756                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1757         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1758         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1759         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1760         u64temp = (unsigned long)&nesvnic->nic_cq;
1761         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1762         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1763                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1764         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1765         if (++cqp_head >= nesdev->cqp.sq_size)
1766                 cqp_head = 0;
1767         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1768         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1769
1770         /* Send CreateQP request to CQP */
1771         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1772         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1773                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1774                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1775         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1776                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1777                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1778         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1779                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1780         }
1781
1782         u64temp = (u64)nesvnic->nic.sq_pbase;
1783         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1784         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1785         u64temp = (u64)nesvnic->nic.rq_pbase;
1786         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1787         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1788
1789         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1790                         NES_CQP_QP_TYPE_NIC);
1791         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1792         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1793                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1794         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1795
1796         if (++cqp_head >= nesdev->cqp.sq_size)
1797                 cqp_head = 0;
1798         nesdev->cqp.sq_head = cqp_head;
1799
1800         barrier();
1801
1802         /* Ring doorbell (2 WQEs) */
1803         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1804
1805         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1806         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1807                         nesvnic->nic.qp_id);
1808
1809         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1810                         NES_EVENT_TIMEOUT);
1811         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1812                         nesvnic->nic.qp_id, ret);
1813         if (!ret) {
1814                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1815                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1816                                 nesvnic->nic_pbase);
1817                 return -EIO;
1818         }
1819
1820         /* Populate the RQ */
1821         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1822                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1823                 if (!skb) {
1824                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1825
1826                         nes_destroy_nic_qp(nesvnic);
1827                         return -ENOMEM;
1828                 }
1829
1830                 skb->dev = netdev;
1831
1832                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1833                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1834                 cb = (struct nes_rskb_cb *)&skb->cb[0];
1835                 cb->busaddr = pmem;
1836                 cb->maplen = nesvnic->max_frame_size;
1837
1838                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1839                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1840                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1841                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1842                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1843                 nesvnic->nic.rx_skb[counter] = skb;
1844         }
1845
1846         wqe_count = NES_NIC_WQ_SIZE - 1;
1847         nesvnic->nic.rq_head = wqe_count;
1848         barrier();
1849         do {
1850                 counter = min(wqe_count, ((u32)255));
1851                 wqe_count -= counter;
1852                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1853         } while (wqe_count);
1854         init_timer(&nesvnic->rq_wqes_timer);
1855         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1856         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1857         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1858         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1859         {
1860                 nes_nic_init_timer(nesdev);
1861                 if (netdev->mtu > 1500)
1862                         jumbomode = 1;
1863                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1864         }
1865         if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1866                 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1867                         nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", netdev->name);
1868                         nes_destroy_nic_qp(nesvnic);
1869                 return -ENOMEM;
1870         }
1871
1872         return 0;
1873 }
1874
1875
1876 /**
1877  * nes_destroy_nic_qp
1878  */
1879 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1880 {
1881         u64 u64temp;
1882         dma_addr_t bus_address;
1883         struct nes_device *nesdev = nesvnic->nesdev;
1884         struct nes_hw_cqp_wqe *cqp_wqe;
1885         struct nes_hw_nic_sq_wqe *nic_sqe;
1886         __le16 *wqe_fragment_length;
1887         u16  wqe_fragment_index;
1888         u32 cqp_head;
1889         u32 wqm_cfg0;
1890         unsigned long flags;
1891         struct sk_buff *rx_skb;
1892         struct nes_rskb_cb *cb;
1893         int ret;
1894
1895         if (nesdev->nesadapter->allow_unaligned_fpdus)
1896                 nes_destroy_mgt(nesvnic);
1897
1898         /* clear wqe stall before destroying NIC QP */
1899         wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1900         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1901
1902         /* Free remaining NIC receive buffers */
1903         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1904                 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1905                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1906                 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1907                         PCI_DMA_FROMDEVICE);
1908
1909                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1910                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1911         }
1912
1913         /* Free remaining NIC transmit buffers */
1914         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1915                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1916                 wqe_fragment_index = 1;
1917                 wqe_fragment_length = (__le16 *)
1918                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1919                 /* bump past the vlan tag */
1920                 wqe_fragment_length++;
1921                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1922                         u64temp = (u64)le32_to_cpu(
1923                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1924                                 wqe_fragment_index*2]);
1925                         u64temp += ((u64)le32_to_cpu(
1926                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1927                                 + wqe_fragment_index*2]))<<32;
1928                         bus_address = (dma_addr_t)u64temp;
1929                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1930                                         nesvnic->nic.first_frag_overflow)) {
1931                                 pci_unmap_single(nesdev->pcidev,
1932                                                 bus_address,
1933                                                 le16_to_cpu(wqe_fragment_length[
1934                                                         wqe_fragment_index++]),
1935                                                 PCI_DMA_TODEVICE);
1936                         }
1937                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1938                                 if (wqe_fragment_length[wqe_fragment_index]) {
1939                                         u64temp = le32_to_cpu(
1940                                                 nic_sqe->wqe_words[
1941                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1942                                                 wqe_fragment_index*2]);
1943                                         u64temp += ((u64)le32_to_cpu(
1944                                                 nic_sqe->wqe_words[
1945                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1946                                                 wqe_fragment_index*2]))<<32;
1947                                         bus_address = (dma_addr_t)u64temp;
1948                                         pci_unmap_page(nesdev->pcidev,
1949                                                         bus_address,
1950                                                         le16_to_cpu(
1951                                                         wqe_fragment_length[
1952                                                         wqe_fragment_index]),
1953                                                         PCI_DMA_TODEVICE);
1954                                 } else
1955                                         break;
1956                         }
1957                 }
1958                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1959                         dev_kfree_skb(
1960                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1961
1962                 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1963                                         & (nesvnic->nic.sq_size - 1);
1964         }
1965
1966         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1967
1968         /* Destroy NIC QP */
1969         cqp_head = nesdev->cqp.sq_head;
1970         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1971         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1972
1973         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1974                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1975         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1976                 nesvnic->nic.qp_id);
1977
1978         if (++cqp_head >= nesdev->cqp.sq_size)
1979                 cqp_head = 0;
1980
1981         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1982
1983         /* Destroy NIC CQ */
1984         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1985         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1986                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1987         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1988                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1989
1990         if (++cqp_head >= nesdev->cqp.sq_size)
1991                 cqp_head = 0;
1992
1993         nesdev->cqp.sq_head = cqp_head;
1994         barrier();
1995
1996         /* Ring doorbell (2 WQEs) */
1997         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1998
1999         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2000         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2001                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2002                         cqp_head, nesdev->cqp.sq_head,
2003                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2004
2005         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2006                         NES_EVENT_TIMEOUT);
2007
2008         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2009                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2010                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2011         if (!ret) {
2012                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2013                                 nesvnic->nic.qp_id);
2014         }
2015
2016         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2017                         nesvnic->nic_pbase);
2018
2019         /* restore old wqm_cfg0 value */
2020         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2021 }
2022
2023 /**
2024  * nes_napi_isr
2025  */
2026 int nes_napi_isr(struct nes_device *nesdev)
2027 {
2028         struct nes_adapter *nesadapter = nesdev->nesadapter;
2029         u32 int_stat;
2030
2031         if (nesdev->napi_isr_ran) {
2032                 /* interrupt status has already been read in ISR */
2033                 int_stat = nesdev->int_stat;
2034         } else {
2035                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2036                 nesdev->int_stat = int_stat;
2037                 nesdev->napi_isr_ran = 1;
2038         }
2039
2040         int_stat &= nesdev->int_req;
2041         /* iff NIC, process here, else wait for DPC */
2042         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2043                 nesdev->napi_isr_ran = 0;
2044                 nes_write32(nesdev->regs + NES_INT_STAT,
2045                         (int_stat &
2046                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2047
2048                 /* Process the CEQs */
2049                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2050
2051                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2052                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2053                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2054                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2055                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2056                                 /* Enable Periodic timer interrupts */
2057                                 nesdev->int_req |= NES_INT_TIMER;
2058                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2059                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2060                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2061                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2062                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2063                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2064                         }
2065
2066                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2067                         {
2068                                 nes_nic_init_timer(nesdev);
2069                         }
2070                         /* Enable interrupts, except CEQs */
2071                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2072                 } else {
2073                         /* Enable interrupts, make sure timer is off */
2074                         nesdev->int_req &= ~NES_INT_TIMER;
2075                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2076                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2077                 }
2078                 nesdev->deepcq_count = 0;
2079                 return 1;
2080         } else {
2081                 return 0;
2082         }
2083 }
2084
2085 static void process_critical_error(struct nes_device *nesdev)
2086 {
2087         u32 debug_error;
2088         u32 nes_idx_debug_error_masks0 = 0;
2089         u16 error_module = 0;
2090
2091         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2092         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2093                         (u16)debug_error);
2094         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2095                         0x01010000 | (debug_error & 0x0000ffff));
2096         if (crit_err_count++ > 10)
2097                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2098         error_module = (u16) (debug_error & 0x1F00) >> 8;
2099         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2100                         nes_max_critical_error_count) {
2101                 printk(KERN_ERR PFX "Masking off critical error for module "
2102                         "0x%02X\n", (u16)error_module);
2103                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2104                         NES_IDX_DEBUG_ERROR_MASKS0);
2105                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2106                         nes_idx_debug_error_masks0 | (1 << error_module));
2107         }
2108 }
2109 /**
2110  * nes_dpc
2111  */
2112 void nes_dpc(unsigned long param)
2113 {
2114         struct nes_device *nesdev = (struct nes_device *)param;
2115         struct nes_adapter *nesadapter = nesdev->nesadapter;
2116         u32 counter;
2117         u32 loop_counter = 0;
2118         u32 int_status_bit;
2119         u32 int_stat;
2120         u32 timer_stat;
2121         u32 temp_int_stat;
2122         u32 intf_int_stat;
2123         u32 processed_intf_int = 0;
2124         u16 processed_timer_int = 0;
2125         u16 completion_ints = 0;
2126         u16 timer_ints = 0;
2127
2128         /* nes_debug(NES_DBG_ISR, "\n"); */
2129
2130         do {
2131                 timer_stat = 0;
2132                 if (nesdev->napi_isr_ran) {
2133                         nesdev->napi_isr_ran = 0;
2134                         int_stat = nesdev->int_stat;
2135                 } else
2136                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2137                 if (processed_intf_int != 0)
2138                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2139                 else
2140                         int_stat &= nesdev->int_req;
2141                 if (processed_timer_int == 0) {
2142                         processed_timer_int = 1;
2143                         if (int_stat & NES_INT_TIMER) {
2144                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2145                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2146                                         int_stat &= ~NES_INT_TIMER;
2147                                 }
2148                         }
2149                 } else {
2150                         int_stat &= ~NES_INT_TIMER;
2151                 }
2152
2153                 if (int_stat) {
2154                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2155                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2156                                 /* Ack the interrupts */
2157                                 nes_write32(nesdev->regs+NES_INT_STAT,
2158                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2159                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2160                         }
2161
2162                         temp_int_stat = int_stat;
2163                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2164                                 if (int_stat & int_status_bit) {
2165                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2166                                         temp_int_stat &= ~int_status_bit;
2167                                         completion_ints = 1;
2168                                 }
2169                                 if (!(temp_int_stat & 0x0000ffff))
2170                                         break;
2171                                 int_status_bit <<= 1;
2172                         }
2173
2174                         /* Process the AEQ for this pci function */
2175                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2176                         if (int_stat & int_status_bit) {
2177                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2178                         }
2179
2180                         /* Process the MAC interrupt for this pci function */
2181                         int_status_bit = 1 << (24 + nesdev->mac_index);
2182                         if (int_stat & int_status_bit) {
2183                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2184                         }
2185
2186                         if (int_stat & NES_INT_TIMER) {
2187                                 if (timer_stat & nesdev->timer_int_req) {
2188                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2189                                                         (timer_stat & nesdev->timer_int_req) |
2190                                                         ~(nesdev->nesadapter->timer_int_req));
2191                                         timer_ints = 1;
2192                                 }
2193                         }
2194
2195                         if (int_stat & NES_INT_INTF) {
2196                                 processed_intf_int = 1;
2197                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2198                                 intf_int_stat &= nesdev->intf_int_req;
2199                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2200                                         process_critical_error(nesdev);
2201                                 }
2202                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2203                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2204                                         BUG();
2205                                 }
2206                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2207                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2208                                         BUG();
2209                                 }
2210                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2211                         }
2212
2213                         if (int_stat & NES_INT_TSW) {
2214                         }
2215                 }
2216                 /* Don't use the interface interrupt bit stay in loop */
2217                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2218                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2219         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2220
2221         if (timer_ints == 1) {
2222                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2223                         if (completion_ints == 0) {
2224                                 nesdev->timer_only_int_count++;
2225                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2226                                         nesdev->timer_only_int_count = 0;
2227                                         nesdev->int_req &= ~NES_INT_TIMER;
2228                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2229                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2230                                 } else {
2231                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2232                                 }
2233                         } else {
2234                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2235                                 {
2236                                         nes_nic_init_timer(nesdev);
2237                                 }
2238                                 nesdev->timer_only_int_count = 0;
2239                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2240                         }
2241                 } else {
2242                         nesdev->timer_only_int_count = 0;
2243                         nesdev->int_req &= ~NES_INT_TIMER;
2244                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2245                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2246                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2247                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2248                 }
2249         } else {
2250                 if ( (completion_ints == 1) &&
2251                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2252                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2253                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2254                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2255                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2256                         nesdev->timer_only_int_count = 0;
2257                         nesdev->int_req |= NES_INT_TIMER;
2258                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2259                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2260                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2261                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2262                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2263                 } else {
2264                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2265                 }
2266         }
2267         nesdev->deepcq_count = 0;
2268 }
2269
2270
2271 /**
2272  * nes_process_ceq
2273  */
2274 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2275 {
2276         u64 u64temp;
2277         struct nes_hw_cq *cq;
2278         u32 head;
2279         u32 ceq_size;
2280
2281         /* nes_debug(NES_DBG_CQ, "\n"); */
2282         head = ceq->ceq_head;
2283         ceq_size = ceq->ceq_size;
2284
2285         do {
2286                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2287                                 NES_CEQE_VALID) {
2288                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2289                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2290                         u64temp <<= 1;
2291                         cq = *((struct nes_hw_cq **)&u64temp);
2292                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2293                         barrier();
2294                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2295
2296                         /* call the event handler */
2297                         cq->ce_handler(nesdev, cq);
2298
2299                         if (++head >= ceq_size)
2300                                 head = 0;
2301                 } else {
2302                         break;
2303                 }
2304
2305         } while (1);
2306
2307         ceq->ceq_head = head;
2308 }
2309
2310
2311 /**
2312  * nes_process_aeq
2313  */
2314 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2315 {
2316         /* u64 u64temp; */
2317         u32 head;
2318         u32 aeq_size;
2319         u32 aeqe_misc;
2320         u32 aeqe_cq_id;
2321         struct nes_hw_aeqe volatile *aeqe;
2322
2323         head = aeq->aeq_head;
2324         aeq_size = aeq->aeq_size;
2325
2326         do {
2327                 aeqe = &aeq->aeq_vbase[head];
2328                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2329                         break;
2330                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2331                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2332                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2333                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2334                                 /* dealing with an accelerated QP related AE */
2335                                 /*
2336                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2337                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2338                                  */
2339                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2340                         } else {
2341                                 /* TODO: dealing with a CQP related AE */
2342                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2343                                                 (u16)(aeqe_misc >> 16));
2344                         }
2345                 }
2346
2347                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2348
2349                 if (++head >= aeq_size)
2350                         head = 0;
2351
2352                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2353         }
2354         while (1);
2355         aeq->aeq_head = head;
2356 }
2357
2358 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2359 {
2360         struct nes_adapter *nesadapter = nesdev->nesadapter;
2361         u32 reset_value;
2362         u32 i=0;
2363         u32 u32temp;
2364
2365         if (nesadapter->hw_rev == NE020_REV) {
2366                 return;
2367         }
2368         mh_detected++;
2369
2370         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2371
2372         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2373                 reset_value |= 0x0000001d;
2374         else
2375                 reset_value |= 0x0000002d;
2376
2377         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2378                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2379                         nesadapter->link_interrupt_count[0] = 0;
2380                         nesadapter->link_interrupt_count[1] = 0;
2381                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2382                         if (0x00000040 & u32temp)
2383                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2384                         else
2385                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2386
2387                         reset_value |= 0x0000003d;
2388                 }
2389                 nesadapter->link_interrupt_count[mac_index] = 0;
2390         }
2391
2392         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2393
2394         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2395                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2396
2397         if (0x0000003d == (reset_value & 0x0000003d)) {
2398                 u32 pcs_control_status0, pcs_control_status1;
2399
2400                 for (i = 0; i < 10; i++) {
2401                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2402                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2403                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2404                              && (pcs_control_status0 & 0x00100000))
2405                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2406                                 && (pcs_control_status1 & 0x00100000)))
2407                                 continue;
2408                         else
2409                                 break;
2410                 }
2411                 if (10 == i) {
2412                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2413                         if (0x00000040 & u32temp)
2414                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2415                         else
2416                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2417
2418                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2419
2420                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2421                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2422                 }
2423         }
2424 }
2425
2426 /**
2427  * nes_process_mac_intr
2428  */
2429 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2430 {
2431         unsigned long flags;
2432         u32 pcs_control_status;
2433         struct nes_adapter *nesadapter = nesdev->nesadapter;
2434         struct nes_vnic *nesvnic;
2435         u32 mac_status;
2436         u32 mac_index = nesdev->mac_index;
2437         u32 u32temp;
2438         u16 phy_data;
2439         u16 temp_phy_data;
2440         u32 pcs_val  = 0x0f0f0000;
2441         u32 pcs_mask = 0x0f1f0000;
2442         u32 cdr_ctrl;
2443
2444         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2445         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2446                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2447                 return;
2448         }
2449         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2450
2451         /* ack the MAC interrupt */
2452         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2453         /* Clear the interrupt */
2454         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2455
2456         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2457
2458         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2459                 nesdev->link_status_interrupts++;
2460                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2461                         nes_reset_link(nesdev, mac_index);
2462
2463                 /* read the PHY interrupt status register */
2464                 if ((nesadapter->OneG_Mode) &&
2465                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2466                         do {
2467                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2468                                                 nesadapter->phy_index[mac_index], &phy_data);
2469                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2470                                                 nesadapter->phy_index[mac_index], phy_data);
2471                         } while (phy_data&0x8000);
2472
2473                         temp_phy_data = 0;
2474                         do {
2475                                 nes_read_1G_phy_reg(nesdev, 0x11,
2476                                                 nesadapter->phy_index[mac_index], &phy_data);
2477                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2478                                                 nesadapter->phy_index[mac_index], phy_data);
2479                                 if (temp_phy_data == phy_data)
2480                                         break;
2481                                 temp_phy_data = phy_data;
2482                         } while (1);
2483
2484                         nes_read_1G_phy_reg(nesdev, 0x1e,
2485                                         nesadapter->phy_index[mac_index], &phy_data);
2486                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2487                                         nesadapter->phy_index[mac_index], phy_data);
2488
2489                         nes_read_1G_phy_reg(nesdev, 1,
2490                                         nesadapter->phy_index[mac_index], &phy_data);
2491                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2492                                         nesadapter->phy_index[mac_index], phy_data);
2493
2494                         if (temp_phy_data & 0x1000) {
2495                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2496                                 phy_data = 4;
2497                         } else {
2498                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2499                         }
2500                 }
2501                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2502                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2503                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2504
2505                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2506                         switch (mac_index) {
2507                         case 1:
2508                         case 3:
2509                                 pcs_control_status = nes_read_indexed(nesdev,
2510                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2511                                 break;
2512                         default:
2513                                 pcs_control_status = nes_read_indexed(nesdev,
2514                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2515                                 break;
2516                         }
2517                 } else {
2518                         pcs_control_status = nes_read_indexed(nesdev,
2519                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2520                         pcs_control_status = nes_read_indexed(nesdev,
2521                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2522                 }
2523
2524                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2525                                 mac_index, pcs_control_status);
2526                 if ((nesadapter->OneG_Mode) &&
2527                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2528                         u32temp = 0x01010000;
2529                         if (nesadapter->port_count > 2) {
2530                                 u32temp |= 0x02020000;
2531                         }
2532                         if ((pcs_control_status & u32temp)!= u32temp) {
2533                                 phy_data = 0;
2534                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2535                         }
2536                 } else {
2537                         switch (nesadapter->phy_type[mac_index]) {
2538                         case NES_PHY_TYPE_ARGUS:
2539                         case NES_PHY_TYPE_SFP_D:
2540                         case NES_PHY_TYPE_KR:
2541                                 /* clear the alarms */
2542                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2543                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2544                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2545                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2546                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2547                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2548                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2549                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2550                                 /* check link status */
2551                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2552                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2553
2554                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2555                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2556                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2557                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2558
2559                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2560
2561                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2562                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2563                                 break;
2564
2565                         case NES_PHY_TYPE_PUMA_1G:
2566                                 if (mac_index < 2)
2567                                         pcs_val = pcs_mask = 0x01010000;
2568                                 else
2569                                         pcs_val = pcs_mask = 0x02020000;
2570                                 /* fall through */
2571                         default:
2572                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2573                                 break;
2574                         }
2575                 }
2576
2577                 if (phy_data & 0x0004) {
2578                         if (wide_ppm_offset &&
2579                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2580                             (nesadapter->hw_rev != NE020_REV)) {
2581                                 cdr_ctrl = nes_read_indexed(nesdev,
2582                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2583                                                             mac_index * 0x200);
2584                                 nes_write_indexed(nesdev,
2585                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2586                                                   mac_index * 0x200,
2587                                                   cdr_ctrl | 0x000F0000);
2588                         }
2589                         nesadapter->mac_link_down[mac_index] = 0;
2590                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2591                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2592                                                 nesvnic->linkup);
2593                                 if (nesvnic->linkup == 0) {
2594                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2595                                                         nesvnic->netdev->name, nesvnic->netdev);
2596                                         if (netif_queue_stopped(nesvnic->netdev))
2597                                                 netif_start_queue(nesvnic->netdev);
2598                                         nesvnic->linkup = 1;
2599                                         netif_carrier_on(nesvnic->netdev);
2600
2601                                         spin_lock(&nesvnic->port_ibevent_lock);
2602                                         if (nesvnic->of_device_registered) {
2603                                                 if (nesdev->iw_status == 0) {
2604                                                         nesdev->iw_status = 1;
2605                                                         nes_port_ibevent(nesvnic);
2606                                                 }
2607                                         }
2608                                         spin_unlock(&nesvnic->port_ibevent_lock);
2609                                 }
2610                         }
2611                 } else {
2612                         if (wide_ppm_offset &&
2613                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2614                             (nesadapter->hw_rev != NE020_REV)) {
2615                                 cdr_ctrl = nes_read_indexed(nesdev,
2616                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2617                                                             mac_index * 0x200);
2618                                 nes_write_indexed(nesdev,
2619                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2620                                                   mac_index * 0x200,
2621                                                   cdr_ctrl & 0xFFF0FFFF);
2622                         }
2623                         nesadapter->mac_link_down[mac_index] = 1;
2624                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2625                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2626                                                 nesvnic->linkup);
2627                                 if (nesvnic->linkup == 1) {
2628                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2629                                                         nesvnic->netdev->name, nesvnic->netdev);
2630                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2631                                                 netif_stop_queue(nesvnic->netdev);
2632                                         nesvnic->linkup = 0;
2633                                         netif_carrier_off(nesvnic->netdev);
2634
2635                                         spin_lock(&nesvnic->port_ibevent_lock);
2636                                         if (nesvnic->of_device_registered) {
2637                                                 if (nesdev->iw_status == 1) {
2638                                                         nesdev->iw_status = 0;
2639                                                         nes_port_ibevent(nesvnic);
2640                                                 }
2641                                         }
2642                                         spin_unlock(&nesvnic->port_ibevent_lock);
2643                                 }
2644                         }
2645                 }
2646                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2647                         nesdev->link_recheck = 1;
2648                         mod_delayed_work(system_wq, &nesdev->work,
2649                                          NES_LINK_RECHECK_DELAY);
2650                 }
2651         }
2652
2653         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2654
2655         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2656 }
2657
2658 void nes_recheck_link_status(struct work_struct *work)
2659 {
2660         unsigned long flags;
2661         struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2662         struct nes_adapter *nesadapter = nesdev->nesadapter;
2663         struct nes_vnic *nesvnic;
2664         u32 mac_index = nesdev->mac_index;
2665         u16 phy_data;
2666         u16 temp_phy_data;
2667
2668         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2669
2670         /* check link status */
2671         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2672         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2673
2674         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2675         nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2676         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2677         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2678
2679         phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2680
2681         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2682                 __func__, phy_data,
2683                 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2684
2685         if (phy_data & 0x0004) {
2686                 nesadapter->mac_link_down[mac_index] = 0;
2687                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2688                         if (nesvnic->linkup == 0) {
2689                                 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2690                                                 nesvnic->netdev->name, nesvnic->netdev);
2691                                 if (netif_queue_stopped(nesvnic->netdev))
2692                                         netif_start_queue(nesvnic->netdev);
2693                                 nesvnic->linkup = 1;
2694                                 netif_carrier_on(nesvnic->netdev);
2695
2696                                 spin_lock(&nesvnic->port_ibevent_lock);
2697                                 if (nesvnic->of_device_registered) {
2698                                         if (nesdev->iw_status == 0) {
2699                                                 nesdev->iw_status = 1;
2700                                                 nes_port_ibevent(nesvnic);
2701                                         }
2702                                 }
2703                                 spin_unlock(&nesvnic->port_ibevent_lock);
2704                         }
2705                 }
2706
2707         } else {
2708                 nesadapter->mac_link_down[mac_index] = 1;
2709                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2710                         if (nesvnic->linkup == 1) {
2711                                 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2712                                                 nesvnic->netdev->name, nesvnic->netdev);
2713                                 if (!(netif_queue_stopped(nesvnic->netdev)))
2714                                         netif_stop_queue(nesvnic->netdev);
2715                                 nesvnic->linkup = 0;
2716                                 netif_carrier_off(nesvnic->netdev);
2717
2718                                 spin_lock(&nesvnic->port_ibevent_lock);
2719                                 if (nesvnic->of_device_registered) {
2720                                         if (nesdev->iw_status == 1) {
2721                                                 nesdev->iw_status = 0;
2722                                                 nes_port_ibevent(nesvnic);
2723                                         }
2724                                 }
2725                                 spin_unlock(&nesvnic->port_ibevent_lock);
2726                         }
2727                 }
2728         }
2729         if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2730                 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2731         else
2732                 nesdev->link_recheck = 0;
2733
2734         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2735 }
2736
2737
2738 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2739 {
2740         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2741
2742         napi_schedule(&nesvnic->napi);
2743 }
2744
2745
2746 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2747 * getting out of nic_ce_handler
2748 */
2749 #define MAX_RQES_TO_PROCESS     384
2750
2751 /**
2752  * nes_nic_ce_handler
2753  */
2754 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2755 {
2756         u64 u64temp;
2757         dma_addr_t bus_address;
2758         struct nes_hw_nic *nesnic;
2759         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2760         struct nes_adapter *nesadapter = nesdev->nesadapter;
2761         struct nes_hw_nic_rq_wqe *nic_rqe;
2762         struct nes_hw_nic_sq_wqe *nic_sqe;
2763         struct sk_buff *skb;
2764         struct sk_buff *rx_skb;
2765         struct nes_rskb_cb *cb;
2766         __le16 *wqe_fragment_length;
2767         u32 head;
2768         u32 cq_size;
2769         u32 rx_pkt_size;
2770         u32 cqe_count=0;
2771         u32 cqe_errv;
2772         u32 cqe_misc;
2773         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2774         u16 vlan_tag;
2775         u16 pkt_type;
2776         u16 rqes_processed = 0;
2777         u8 sq_cqes = 0;
2778
2779         head = cq->cq_head;
2780         cq_size = cq->cq_size;
2781         cq->cqes_pending = 1;
2782         do {
2783                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2784                                 NES_NIC_CQE_VALID) {
2785                         nesnic = &nesvnic->nic;
2786                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2787                         if (cqe_misc & NES_NIC_CQE_SQ) {
2788                                 sq_cqes++;
2789                                 wqe_fragment_index = 1;
2790                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2791                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2792                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2793                                 /* bump past the vlan tag */
2794                                 wqe_fragment_length++;
2795                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2796                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2797                                                         wqe_fragment_index * 2]);
2798                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2799                                                         wqe_fragment_index * 2])) << 32;
2800                                         bus_address = (dma_addr_t)u64temp;
2801                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2802                                                 pci_unmap_single(nesdev->pcidev,
2803                                                                 bus_address,
2804                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2805                                                                 PCI_DMA_TODEVICE);
2806                                         }
2807                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2808                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2809                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2810                                                                                 wqe_fragment_index * 2]);
2811                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2812                                                                                 + wqe_fragment_index * 2])) <<32;
2813                                                         bus_address = (dma_addr_t)u64temp;
2814                                                         pci_unmap_page(nesdev->pcidev,
2815                                                                         bus_address,
2816                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2817                                                                         PCI_DMA_TODEVICE);
2818                                                 } else
2819                                                         break;
2820                                         }
2821                                 }
2822                                 if (skb)
2823                                         dev_kfree_skb_any(skb);
2824                                 nesnic->sq_tail++;
2825                                 nesnic->sq_tail &= nesnic->sq_size-1;
2826                                 if (sq_cqes > 128) {
2827                                         barrier();
2828                                         /* restart the queue if it had been stopped */
2829                                         if (netif_queue_stopped(nesvnic->netdev))
2830                                                 netif_wake_queue(nesvnic->netdev);
2831                                         sq_cqes = 0;
2832                                 }
2833                         } else {
2834                                 rqes_processed ++;
2835
2836                                 cq->rx_cqes_completed++;
2837                                 cq->rx_pkts_indicated++;
2838                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2839                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2840                                 /* Get the skb */
2841                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2842                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2843                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2844                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2845                                 pci_unmap_single(nesdev->pcidev, bus_address,
2846                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2847                                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2848                                 cb->busaddr = 0;
2849                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2850                                 /* rx_skb->len = rx_pkt_size; */
2851                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2852                                 skb_put(rx_skb, rx_pkt_size);
2853                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2854                                 nesnic->rq_tail++;
2855                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2856
2857                                 atomic_inc(&nesvnic->rx_skbs_needed);
2858                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2859                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2860                                                         cq->cq_number | (cqe_count << 16));
2861                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2862                                         nesdev->currcq_count += cqe_count;
2863                                         cqe_count = 0;
2864                                         nes_replenish_nic_rq(nesvnic);
2865                                 }
2866                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2867                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2868                                 rx_skb->ip_summed = CHECKSUM_NONE;
2869
2870                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2871                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2872                                         if ((cqe_errv &
2873                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2874                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2875                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2876                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2877                                         } else
2878                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2879                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2880                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2881
2882                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2883                                         if ((cqe_errv &
2884                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2885                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2886                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2887                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2888                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2889                                                                   nesvnic->netdev->name); */
2890                                                 }
2891                                         } else
2892                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2893                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2894                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2895                                         }
2896                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2897                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2898
2899                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2900                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2901                                                 rx_skb = NULL;
2902                                 }
2903                                 if (rx_skb == NULL)
2904                                         goto skip_rx_indicate0;
2905
2906
2907                                 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2908                                         vlan_tag = (u16)(le32_to_cpu(
2909                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2910                                                         >> 16);
2911                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2912                                                         nesvnic->netdev->name, vlan_tag);
2913
2914                                         __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag);
2915                                 }
2916                                 napi_gro_receive(&nesvnic->napi, rx_skb);
2917
2918 skip_rx_indicate0:
2919                                 ;
2920                                 /* nesvnic->netstats.rx_packets++; */
2921                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2922                         }
2923
2924                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2925                         /* Accounting... */
2926                         cqe_count++;
2927                         if (++head >= cq_size)
2928                                 head = 0;
2929                         if (cqe_count == 255) {
2930                                 /* Replenish Nic CQ */
2931                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2932                                                 cq->cq_number | (cqe_count << 16));
2933                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2934                                 nesdev->currcq_count += cqe_count;
2935                                 cqe_count = 0;
2936                         }
2937
2938                         if (cq->rx_cqes_completed >= nesvnic->budget)
2939                                 break;
2940                 } else {
2941                         cq->cqes_pending = 0;
2942                         break;
2943                 }
2944
2945         } while (1);
2946
2947         if (sq_cqes) {
2948                 barrier();
2949                 /* restart the queue if it had been stopped */
2950                 if (netif_queue_stopped(nesvnic->netdev))
2951                         netif_wake_queue(nesvnic->netdev);
2952         }
2953         cq->cq_head = head;
2954         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2955                         cq->cq_number, cqe_count, cq->cq_head); */
2956         cq->cqe_allocs_pending = cqe_count;
2957         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2958         {
2959                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2960                 nesdev->currcq_count += cqe_count;
2961                 nes_nic_tune_timer(nesdev);
2962         }
2963         if (atomic_read(&nesvnic->rx_skbs_needed))
2964                 nes_replenish_nic_rq(nesvnic);
2965 }
2966
2967
2968
2969 /**
2970  * nes_cqp_ce_handler
2971  */
2972 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2973 {
2974         u64 u64temp;
2975         unsigned long flags;
2976         struct nes_hw_cqp *cqp = NULL;
2977         struct nes_cqp_request *cqp_request;
2978         struct nes_hw_cqp_wqe *cqp_wqe;
2979         u32 head;
2980         u32 cq_size;
2981         u32 cqe_count=0;
2982         u32 error_code;
2983         u32 opcode;
2984         u32 ctx_index;
2985         /* u32 counter; */
2986
2987         head = cq->cq_head;
2988         cq_size = cq->cq_size;
2989
2990         do {
2991                 /* process the CQE */
2992                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2993                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2994
2995                 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
2996                 if (opcode & NES_CQE_VALID) {
2997                         cqp = &nesdev->cqp;
2998
2999                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
3000                         if (error_code) {
3001                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
3002                                                 " Major/Minor codes = 0x%04X:%04X.\n",
3003                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3004                                                 (u16)(error_code >> 16),
3005                                                 (u16)error_code);
3006                         }
3007
3008                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3009                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3010                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
3011                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3012
3013                         cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3014                         if (cqp_request) {
3015                                 if (cqp_request->waiting) {
3016                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3017                                         cqp_request->major_code = (u16)(error_code >> 16);
3018                                         cqp_request->minor_code = (u16)error_code;
3019                                         barrier();
3020                                         cqp_request->request_done = 1;
3021                                         wake_up(&cqp_request->waitq);
3022                                         nes_put_cqp_request(nesdev, cqp_request);
3023                                 } else {
3024                                         if (cqp_request->callback)
3025                                                 cqp_request->cqp_callback(nesdev, cqp_request);
3026                                         nes_free_cqp_request(nesdev, cqp_request);
3027                                 }
3028                         } else {
3029                                 wake_up(&nesdev->cqp.waitq);
3030                         }
3031
3032                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3033                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3034                         if (++cqp->sq_tail >= cqp->sq_size)
3035                                 cqp->sq_tail = 0;
3036
3037                         /* Accounting... */
3038                         cqe_count++;
3039                         if (++head >= cq_size)
3040                                 head = 0;
3041                 } else {
3042                         break;
3043                 }
3044         } while (1);
3045         cq->cq_head = head;
3046
3047         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3048         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3049                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3050                         (nesdev->cqp.sq_size - 1)) != 1)) {
3051                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3052                                 struct nes_cqp_request, list);
3053                 list_del_init(&cqp_request->list);
3054                 head = nesdev->cqp.sq_head++;
3055                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3056                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3057                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3058                 barrier();
3059
3060                 opcode = cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX];
3061                 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3062                         ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3063                 else
3064                         ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3065                 cqp_wqe->wqe_words[ctx_index] =
3066                         cpu_to_le32((u32)((unsigned long)cqp_request));
3067                 cqp_wqe->wqe_words[ctx_index + 1] =
3068                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3069                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3070                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3071                 /* Ring doorbell (1 WQEs) */
3072                 barrier();
3073                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3074         }
3075         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3076
3077         /* Arm the CCQ */
3078         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3079                         cq->cq_number);
3080         nes_read32(nesdev->regs+NES_CQE_ALLOC);
3081 }
3082
3083 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3084 {
3085         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3086                 /* skip over ethernet header */
3087                 pkt += ETH_HLEN;
3088
3089                 /* Skip over IP and TCP headers */
3090                 pkt += 4 * (pkt[0] & 0x0f);
3091                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3092         }
3093         return pkt;
3094 }
3095
3096 /* Determine if incoming error pkt is rdma layer */
3097 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3098 {
3099         u8 *pkt;
3100         u16 *mpa;
3101         u32 opcode = 0xffffffff;
3102
3103         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3104                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3105                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3106                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3107         }
3108
3109         return opcode;
3110 }
3111
3112 /* Build iWARP terminate header */
3113 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3114 {
3115         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3116         u16 ddp_seg_len;
3117         int copy_len = 0;
3118         u8 is_tagged = 0;
3119         u8 flush_code = 0;
3120         struct nes_terminate_hdr *termhdr;
3121
3122         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3123         memset(termhdr, 0, 64);
3124
3125         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3126
3127                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3128                 pkt = locate_mpa(pkt, aeq_info);
3129                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3130                 if (ddp_seg_len) {
3131                         copy_len = 2;
3132                         termhdr->hdrct = DDP_LEN_FLAG;
3133                         if (pkt[2] & 0x80) {
3134                                 is_tagged = 1;
3135                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3136                                         copy_len += TERM_DDP_LEN_TAGGED;
3137                                         termhdr->hdrct |= DDP_HDR_FLAG;
3138                                 }
3139                         } else {
3140                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3141                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3142                                         termhdr->hdrct |= DDP_HDR_FLAG;
3143                                 }
3144
3145                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3146                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3147                                                 copy_len += TERM_RDMA_LEN;
3148                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3149                                         }
3150                                 }
3151                         }
3152                 }
3153         }
3154
3155         switch (async_event_id) {
3156         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3157                 switch (iwarp_opcode(nesqp, aeq_info)) {
3158                 case IWARP_OPCODE_WRITE:
3159                         flush_code = IB_WC_LOC_PROT_ERR;
3160                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3161                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3162                         break;
3163                 default:
3164                         flush_code = IB_WC_REM_ACCESS_ERR;
3165                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3166                         termhdr->error_code = RDMAP_INV_STAG;
3167                 }
3168                 break;
3169         case NES_AEQE_AEID_AMP_INVALID_STAG:
3170                 flush_code = IB_WC_REM_ACCESS_ERR;
3171                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3172                 termhdr->error_code = RDMAP_INV_STAG;
3173                 break;
3174         case NES_AEQE_AEID_AMP_BAD_QP:
3175                 flush_code = IB_WC_LOC_QP_OP_ERR;
3176                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3177                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3178                 break;
3179         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3180         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3181                 switch (iwarp_opcode(nesqp, aeq_info)) {
3182                 case IWARP_OPCODE_SEND_INV:
3183                 case IWARP_OPCODE_SEND_SE_INV:
3184                         flush_code = IB_WC_REM_OP_ERR;
3185                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3186                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3187                         break;
3188                 default:
3189                         flush_code = IB_WC_REM_ACCESS_ERR;
3190                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3191                         termhdr->error_code = RDMAP_INV_STAG;
3192                 }
3193                 break;
3194         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3195                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3196                         flush_code = IB_WC_LOC_PROT_ERR;
3197                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3198                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3199                 } else {
3200                         flush_code = IB_WC_REM_ACCESS_ERR;
3201                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3202                         termhdr->error_code = RDMAP_INV_BOUNDS;
3203                 }
3204                 break;
3205         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3206         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3207         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3208                 flush_code = IB_WC_REM_ACCESS_ERR;
3209                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3210                 termhdr->error_code = RDMAP_ACCESS;
3211                 break;
3212         case NES_AEQE_AEID_AMP_TO_WRAP:
3213                 flush_code = IB_WC_REM_ACCESS_ERR;
3214                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3215                 termhdr->error_code = RDMAP_TO_WRAP;
3216                 break;
3217         case NES_AEQE_AEID_AMP_BAD_PD:
3218                 switch (iwarp_opcode(nesqp, aeq_info)) {
3219                 case IWARP_OPCODE_WRITE:
3220                         flush_code = IB_WC_LOC_PROT_ERR;
3221                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3222                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3223                         break;
3224                 case IWARP_OPCODE_SEND_INV:
3225                 case IWARP_OPCODE_SEND_SE_INV:
3226                         flush_code = IB_WC_REM_ACCESS_ERR;
3227                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3228                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3229                         break;
3230                 default:
3231                         flush_code = IB_WC_REM_ACCESS_ERR;
3232                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3233                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3234                 }
3235                 break;
3236         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3237                 flush_code = IB_WC_LOC_LEN_ERR;
3238                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3239                 termhdr->error_code = MPA_MARKER;
3240                 break;
3241         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3242                 flush_code = IB_WC_GENERAL_ERR;
3243                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3244                 termhdr->error_code = MPA_CRC;
3245                 break;
3246         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3247         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3248                 flush_code = IB_WC_LOC_LEN_ERR;
3249                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3250                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3251                 break;
3252         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3253         case NES_AEQE_AEID_DDP_NO_L_BIT:
3254                 flush_code = IB_WC_FATAL_ERR;
3255                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3256                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3257                 break;
3258         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3259         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3260                 flush_code = IB_WC_GENERAL_ERR;
3261                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3262                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3263                 break;
3264         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3265                 flush_code = IB_WC_LOC_LEN_ERR;
3266                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3267                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3268                 break;
3269         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3270                 flush_code = IB_WC_GENERAL_ERR;
3271                 if (is_tagged) {
3272                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3273                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3274                 } else {
3275                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3276                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3277                 }
3278                 break;
3279         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3280                 flush_code = IB_WC_GENERAL_ERR;
3281                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3282                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3283                 break;
3284         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3285                 flush_code = IB_WC_REM_OP_ERR;
3286                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3287                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3288                 break;
3289         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3290                 flush_code = IB_WC_GENERAL_ERR;
3291                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3292                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3293                 break;
3294         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3295                 flush_code = IB_WC_GENERAL_ERR;
3296                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3297                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3298                 break;
3299         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3300                 flush_code = IB_WC_LOC_QP_OP_ERR;
3301                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3302                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3303                 break;
3304         default:
3305                 flush_code = IB_WC_FATAL_ERR;
3306                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3307                 termhdr->error_code = RDMAP_UNSPECIFIED;
3308                 break;
3309         }
3310
3311         if (copy_len)
3312                 memcpy(termhdr + 1, pkt, copy_len);
3313
3314         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3315                 if (aeq_info & NES_AEQE_SQ)
3316                         nesqp->term_sq_flush_code = flush_code;
3317                 else
3318                         nesqp->term_rq_flush_code = flush_code;
3319         }
3320
3321         return sizeof(struct nes_terminate_hdr) + copy_len;
3322 }
3323
3324 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3325                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3326 {
3327         u64 context;
3328         unsigned long flags;
3329         u32 aeq_info;
3330         u16 async_event_id;
3331         u8 tcp_state;
3332         u8 iwarp_state;
3333         u32 termlen = 0;
3334         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3335                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3336         struct nes_adapter *nesadapter = nesdev->nesadapter;
3337
3338         if (nesqp->term_flags & NES_TERM_SENT)
3339                 return; /* Sanity check */
3340
3341         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3342         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3343         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3344         async_event_id = (u16)aeq_info;
3345
3346         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3347                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3348         if (!context) {
3349                 WARN_ON(!context);
3350                 return;
3351         }
3352
3353         nesqp = (struct nes_qp *)(unsigned long)context;
3354         spin_lock_irqsave(&nesqp->lock, flags);
3355         nesqp->hw_iwarp_state = iwarp_state;
3356         nesqp->hw_tcp_state = tcp_state;
3357         nesqp->last_aeq = async_event_id;
3358         nesqp->terminate_eventtype = eventtype;
3359         spin_unlock_irqrestore(&nesqp->lock, flags);
3360
3361         if (nesadapter->send_term_ok)
3362                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3363         else
3364                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3365
3366         if (!nesdev->iw_status)  {
3367                 nesqp->term_flags = NES_TERM_DONE;
3368                 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3369                 nes_cm_disconn(nesqp);
3370         } else {
3371                 nes_terminate_start_timer(nesqp);
3372                 nesqp->term_flags |= NES_TERM_SENT;
3373                 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3374         }
3375 }
3376
3377 static void nes_terminate_send_fin(struct nes_device *nesdev,
3378                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3379 {
3380         u32 aeq_info;
3381         u16 async_event_id;
3382         u8 tcp_state;
3383         u8 iwarp_state;
3384         unsigned long flags;
3385
3386         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3387         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3388         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3389         async_event_id = (u16)aeq_info;
3390
3391         spin_lock_irqsave(&nesqp->lock, flags);
3392         nesqp->hw_iwarp_state = iwarp_state;
3393         nesqp->hw_tcp_state = tcp_state;
3394         nesqp->last_aeq = async_event_id;
3395         spin_unlock_irqrestore(&nesqp->lock, flags);
3396
3397         /* Send the fin only */
3398         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3399                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3400 }
3401
3402 /* Cleanup after a terminate sent or received */
3403 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3404 {
3405         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3406         unsigned long flags;
3407         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3408         struct nes_device *nesdev = nesvnic->nesdev;
3409         u8 first_time = 0;
3410
3411         spin_lock_irqsave(&nesqp->lock, flags);
3412         if (nesqp->hte_added) {
3413                 nesqp->hte_added = 0;
3414                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3415         }
3416
3417         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3418         nesqp->term_flags |= NES_TERM_DONE;
3419         spin_unlock_irqrestore(&nesqp->lock, flags);
3420
3421         /* Make sure we go through this only once */
3422         if (first_time) {
3423                 if (timeout_occurred == 0)
3424                         del_timer(&nesqp->terminate_timer);
3425                 else
3426                         next_iwarp_state |= NES_CQP_QP_RESET;
3427
3428                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3429                 nes_cm_disconn(nesqp);
3430         }
3431 }
3432
3433 static void nes_terminate_received(struct nes_device *nesdev,
3434                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3435 {
3436         u32 aeq_info;
3437         u8 *pkt;
3438         u32 *mpa;
3439         u8 ddp_ctl;
3440         u8 rdma_ctl;
3441         u16 aeq_id = 0;
3442
3443         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3444         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3445                 /* Terminate is not a performance path so the silicon */
3446                 /* did not validate the frame - do it now */
3447                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3448                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3449                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3450                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3451                 if ((ddp_ctl & 0xc0) != 0x40)
3452                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3453                 else if ((ddp_ctl & 0x03) != 1)
3454                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3455                 else if (be32_to_cpu(mpa[2]) != 2)
3456                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3457                 else if (be32_to_cpu(mpa[3]) != 1)
3458                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3459                 else if (be32_to_cpu(mpa[4]) != 0)
3460                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3461                 else if ((rdma_ctl & 0xc0) != 0x40)
3462                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3463
3464                 if (aeq_id) {
3465                         /* Bad terminate recvd - send back a terminate */
3466                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3467                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3468                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3469                         return;
3470                 }
3471         }
3472
3473         nesqp->term_flags |= NES_TERM_RCVD;
3474         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3475         nes_terminate_start_timer(nesqp);
3476         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3477 }
3478
3479 /* Timeout routine in case terminate fails to complete */
3480 void nes_terminate_timeout(unsigned long context)
3481 {
3482         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3483
3484         nes_terminate_done(nesqp, 1);
3485 }
3486
3487 /* Set a timer in case hw cannot complete the terminate sequence */
3488 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3489 {
3490         mod_timer(&nesqp->terminate_timer, (jiffies + HZ));
3491 }
3492
3493 /**
3494  * nes_process_iwarp_aeqe
3495  */
3496 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3497                                    struct nes_hw_aeqe *aeqe)
3498 {
3499         u64 context;
3500         unsigned long flags;
3501         struct nes_qp *nesqp;
3502         struct nes_hw_cq *hw_cq;
3503         struct nes_cq *nescq;
3504         int resource_allocated;
3505         struct nes_adapter *nesadapter = nesdev->nesadapter;
3506         u32 aeq_info;
3507         u32 next_iwarp_state = 0;
3508         u32 aeqe_cq_id;
3509         u16 async_event_id;
3510         u8 tcp_state;
3511         u8 iwarp_state;
3512         struct ib_event ibevent;
3513
3514         nes_debug(NES_DBG_AEQ, "\n");
3515         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3516         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3517                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3518                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3519         } else {
3520                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3521                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3522                 BUG_ON(!context);
3523         }
3524
3525         /* context is nesqp unless async_event_id == CQ ERROR */
3526         nesqp = (struct nes_qp *)(unsigned long)context;
3527         async_event_id = (u16)aeq_info;
3528         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3529         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3530         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3531                         " Tcp state = %s, iWARP state = %s\n",
3532                         async_event_id,
3533                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3534                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3535
3536         aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3537         if (aeq_info & NES_AEQE_QP) {
3538                 if (!nes_is_resource_allocated(nesadapter,
3539                                 nesadapter->allocated_qps,
3540                                 aeqe_cq_id))
3541                         return;
3542         }
3543
3544         switch (async_event_id) {
3545                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3546                         if (nesqp->term_flags)
3547                                 return; /* Ignore it, wait for close complete */
3548
3549                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3550                                 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3551                                         (nesqp->ibqp_state == IB_QPS_RTS)) {
3552                                         spin_lock_irqsave(&nesqp->lock, flags);
3553                                         nesqp->hw_iwarp_state = iwarp_state;
3554                                         nesqp->hw_tcp_state = tcp_state;
3555                                         nesqp->last_aeq = async_event_id;
3556                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3557                                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3558                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3559                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3560                                         nes_cm_disconn(nesqp);
3561                                 }
3562                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3563                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3564                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3565                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3566                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3567                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3568                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3569                                                 async_event_id, nesqp->last_aeq, tcp_state);
3570                         }
3571                         break;
3572                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3573                         spin_lock_irqsave(&nesqp->lock, flags);
3574                         nesqp->hw_iwarp_state = iwarp_state;
3575                         nesqp->hw_tcp_state = tcp_state;
3576                         nesqp->last_aeq = async_event_id;
3577                         spin_unlock_irqrestore(&nesqp->lock, flags);
3578                         nes_cm_disconn(nesqp);
3579                         break;
3580
3581                 case NES_AEQE_AEID_RESET_SENT:
3582                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3583                         spin_lock_irqsave(&nesqp->lock, flags);
3584                         nesqp->hw_iwarp_state = iwarp_state;
3585                         nesqp->hw_tcp_state = tcp_state;
3586                         nesqp->last_aeq = async_event_id;
3587                         nesqp->hte_added = 0;
3588                         spin_unlock_irqrestore(&nesqp->lock, flags);
3589                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3590                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3591                         nes_cm_disconn(nesqp);
3592                         break;
3593
3594                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3595                         if (atomic_read(&nesqp->close_timer_started))
3596                                 return;
3597                         spin_lock_irqsave(&nesqp->lock, flags);
3598                         nesqp->hw_iwarp_state = iwarp_state;
3599                         nesqp->hw_tcp_state = tcp_state;
3600                         nesqp->last_aeq = async_event_id;
3601                         spin_unlock_irqrestore(&nesqp->lock, flags);
3602                         nes_cm_disconn(nesqp);
3603                         break;
3604
3605                 case NES_AEQE_AEID_TERMINATE_SENT:
3606                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3607                         break;
3608
3609                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3610                         nes_terminate_received(nesdev, nesqp, aeqe);
3611                         break;
3612
3613                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3614                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3615                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3616                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3617                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3618                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3619                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3620                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3621                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3622                 case NES_AEQE_AEID_AMP_TO_WRAP:
3623                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3624                                         nesqp->hwqp.qp_id, async_event_id);
3625                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3626                         break;
3627
3628                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3629                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3630                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3631                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3632                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3633                                 aeq_info &= 0xffff0000;
3634                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3635                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3636                         }
3637
3638                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3639                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3640                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3641                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3642                 case NES_AEQE_AEID_AMP_BAD_QP:
3643                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3644                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3645                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3646                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3647                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3648                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3649                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3650                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3651                 case NES_AEQE_AEID_AMP_BAD_PD:
3652                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3653                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3654                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3655                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3656                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3657                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3658                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3659                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3660                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3661                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3662                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3663                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3664                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3665                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3666                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3667                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3668                 case NES_AEQE_AEID_BAD_CLOSE:
3669                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3670                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3671                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3672                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3673                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3674                                         nesqp->hwqp.qp_id, async_event_id);
3675                         print_ip(nesqp->cm_node);
3676                         if (!atomic_read(&nesqp->close_timer_started))
3677                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3678                         break;
3679
3680                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3681                         context <<= 1;
3682                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3683                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3684                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3685                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3686                         if (resource_allocated) {
3687                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3688                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3689                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3690                                 if (hw_cq) {
3691                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3692                                         if (nescq->ibcq.event_handler) {
3693                                                 ibevent.device = nescq->ibcq.device;
3694                                                 ibevent.event = IB_EVENT_CQ_ERR;
3695                                                 ibevent.element.cq = &nescq->ibcq;
3696                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3697                                         }
3698                                 }
3699                         }
3700                         break;
3701
3702                 default:
3703                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3704                                         async_event_id);
3705                         break;
3706         }
3707
3708 }
3709
3710 /**
3711  * nes_iwarp_ce_handler
3712  */
3713 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3714 {
3715         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3716
3717         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3718                         nescq->hw_cq.cq_number); */
3719         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3720
3721         if (nescq->ibcq.comp_handler)
3722                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3723
3724         return;
3725 }
3726
3727
3728 /**
3729  * nes_manage_apbvt()
3730  */
3731 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3732                 u32 nic_index, u32 add_port)
3733 {
3734         struct nes_device *nesdev = nesvnic->nesdev;
3735         struct nes_hw_cqp_wqe *cqp_wqe;
3736         struct nes_cqp_request *cqp_request;
3737         int ret = 0;
3738         u16 major_code;
3739
3740         /* Send manage APBVT request to CQP */
3741         cqp_request = nes_get_cqp_request(nesdev);
3742         if (cqp_request == NULL) {
3743                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3744                 return -ENOMEM;
3745         }
3746         cqp_request->waiting = 1;
3747         cqp_wqe = &cqp_request->cqp_wqe;
3748
3749         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3750                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3751                         accel_local_port, accel_local_port, nic_index);
3752
3753         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3754         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3755                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3756         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3757                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3758
3759         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3760
3761         atomic_set(&cqp_request->refcount, 2);
3762         nes_post_cqp_request(nesdev, cqp_request);
3763
3764         if (add_port == NES_MANAGE_APBVT_ADD)
3765                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3766                                 NES_EVENT_TIMEOUT);
3767         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3768                         ret, cqp_request->major_code, cqp_request->minor_code);
3769         major_code = cqp_request->major_code;
3770
3771         nes_put_cqp_request(nesdev, cqp_request);
3772
3773         if (!ret)
3774                 return -ETIME;
3775         else if (major_code)
3776                 return -EIO;
3777         else
3778                 return 0;
3779 }
3780
3781
3782 /**
3783  * nes_manage_arp_cache
3784  */
3785 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3786                 u32 ip_addr, u32 action)
3787 {
3788         struct nes_hw_cqp_wqe *cqp_wqe;
3789         struct nes_vnic *nesvnic = netdev_priv(netdev);
3790         struct nes_device *nesdev;
3791         struct nes_cqp_request *cqp_request;
3792         int arp_index;
3793
3794         nesdev = nesvnic->nesdev;
3795         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3796         if (arp_index == -1) {
3797                 return;
3798         }
3799
3800         /* update the ARP entry */
3801         cqp_request = nes_get_cqp_request(nesdev);
3802         if (cqp_request == NULL) {
3803                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3804                 return;
3805         }
3806         cqp_request->waiting = 0;
3807         cqp_wqe = &cqp_request->cqp_wqe;
3808         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3809
3810         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3811                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3812         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3813                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3814         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3815
3816         if (action == NES_ARP_ADD) {
3817                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3818                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3819                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3820                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3821                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3822                                 (((u32)mac_addr[0]) << 8) | (u32)mac_addr[1]);
3823         } else {
3824                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3825                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3826         }
3827
3828         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3829                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3830
3831         atomic_set(&cqp_request->refcount, 1);
3832         nes_post_cqp_request(nesdev, cqp_request);
3833 }
3834
3835
3836 /**
3837  * flush_wqes
3838  */
3839 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3840                 u32 which_wq, u32 wait_completion)
3841 {
3842         struct nes_cqp_request *cqp_request;
3843         struct nes_hw_cqp_wqe *cqp_wqe;
3844         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3845         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3846         int ret;
3847
3848         cqp_request = nes_get_cqp_request(nesdev);
3849         if (cqp_request == NULL) {
3850                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3851                 return;
3852         }
3853         if (wait_completion) {
3854                 cqp_request->waiting = 1;
3855                 atomic_set(&cqp_request->refcount, 2);
3856         } else {
3857                 cqp_request->waiting = 0;
3858         }
3859         cqp_wqe = &cqp_request->cqp_wqe;
3860         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3861
3862         /* If wqe in error was identified, set code to be put into cqe */
3863         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3864                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3865                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3866                 nesqp->term_sq_flush_code = 0;
3867         }
3868
3869         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3870                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3871                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3872                 nesqp->term_rq_flush_code = 0;
3873         }
3874
3875         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3876                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3877                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3878         }
3879
3880         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3881                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3882         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3883
3884         nes_post_cqp_request(nesdev, cqp_request);
3885
3886         if (wait_completion) {
3887                 /* Wait for CQP */
3888                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3889                                 NES_EVENT_TIMEOUT);
3890                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3891                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3892                                 ret, cqp_request->major_code, cqp_request->minor_code);
3893                 nes_put_cqp_request(nesdev, cqp_request);
3894         }
3895 }