drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2011 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/slab.h>
  40
  41 #include "nes.h"
  42
  43 static int wide_ppm_offset;
  44 module_param(wide_ppm_offset, int, 0644);
  45 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  46
  47 static u32 crit_err_count;
  48 u32 int_mod_timer_init;
  49 u32 int_mod_cq_depth_256;
  50 u32 int_mod_cq_depth_128;
  51 u32 int_mod_cq_depth_32;
  52 u32 int_mod_cq_depth_24;
  53 u32 int_mod_cq_depth_16;
  54 u32 int_mod_cq_depth_4;
  55 u32 int_mod_cq_depth_1;
  56 static const u8 nes_max_critical_error_count = 100;
  57 #include "nes_cm.h"
  58
  59 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  60 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  61 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  62                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  63 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  64 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  65 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  66 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  67                                    struct nes_hw_aeqe *aeqe);
  68 static void process_critical_error(struct nes_device *nesdev);
  69 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  70 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  71 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  72
  73 static const char *const nes_iwarp_state_str[] = {
  74         "Non-Existent",
  75         "Idle",
  76         "RTS",
  77         "Closing",
  78         "RSVD1",
  79         "Terminate",
  80         "Error",
  81         "RSVD2",
  82 };
  83
  84 static const char *const nes_tcp_state_str[] = {
  85         "Non-Existent",
  86         "Closed",
  87         "Listen",
  88         "SYN Sent",
  89         "SYN Rcvd",
  90         "Established",
  91         "Close Wait",
  92         "FIN Wait 1",
  93         "Closing",
  94         "Last Ack",
  95         "FIN Wait 2",
  96         "Time Wait",
  97         "RSVD1",
  98         "RSVD2",
  99         "RSVD3",
 100         "RSVD4",
 101 };
 102
 103 static inline void print_ip(struct nes_cm_node *cm_node)
 104 {
 105         unsigned char *rem_addr;
 106         if (cm_node) {
 107                 rem_addr = (unsigned char *)&cm_node->rem_addr;
 108                 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
 109         }
 110 }
 111
 112 /**
 113  * nes_nic_init_timer_defaults
 114  */
 115 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 116 {
 117         unsigned long flags;
 118         struct nes_adapter *nesadapter = nesdev->nesadapter;
 119         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 120
 121         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 122
 123         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 124         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 125         if (jumbomode) {
 126                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 127                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 128                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 129         } else {
 130                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 131                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 132                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 133         }
 134
 135         /* todo use netdev->mtu to set thresholds */
 136         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 137 }
 138
 139
 140 /**
 141  * nes_nic_init_timer
 142  */
 143 static void  nes_nic_init_timer(struct nes_device *nesdev)
 144 {
 145         unsigned long flags;
 146         struct nes_adapter *nesadapter = nesdev->nesadapter;
 147         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 148
 149         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 150
 151         if (shared_timer->timer_in_use_old == 0) {
 152                 nesdev->deepcq_count = 0;
 153                 shared_timer->timer_direction_upward = 0;
 154                 shared_timer->timer_direction_downward = 0;
 155                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 156                 shared_timer->timer_in_use_old = 0;
 157
 158         }
 159         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 160                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 161                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 162                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 163         }
 164         /* todo use netdev->mtu to set thresholds */
 165         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 166 }
 167
 168
 169 /**
 170  * nes_nic_tune_timer
 171  */
 172 static void nes_nic_tune_timer(struct nes_device *nesdev)
 173 {
 174         unsigned long flags;
 175         struct nes_adapter *nesadapter = nesdev->nesadapter;
 176         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 177         u16 cq_count = nesdev->currcq_count;
 178
 179         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 180
 181         if (shared_timer->cq_count_old <= cq_count)
 182                 shared_timer->cq_direction_downward = 0;
 183         else
 184                 shared_timer->cq_direction_downward++;
 185         shared_timer->cq_count_old = cq_count;
 186         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 187                 if (cq_count <= shared_timer->threshold_low &&
 188                     shared_timer->threshold_low > 4) {
 189                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 190                         shared_timer->cq_direction_downward=0;
 191                         nesdev->currcq_count = 0;
 192                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 193                         return;
 194                 }
 195         }
 196
 197         if (cq_count > 1) {
 198                 nesdev->deepcq_count += cq_count;
 199                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 200                         shared_timer->timer_direction_upward++;
 201                         shared_timer->timer_direction_downward = 0;
 202                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 203                         shared_timer->timer_direction_upward = 0;
 204                         shared_timer->timer_direction_downward = 0;
 205                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 206                         shared_timer->timer_direction_downward++;
 207                         shared_timer->timer_direction_upward = 0;
 208                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 209                         shared_timer->timer_in_use -= 2;
 210                         shared_timer->timer_direction_upward = 0;
 211                         shared_timer->timer_direction_downward++;
 212                 } else {
 213                         shared_timer->timer_in_use -= 4;
 214                         shared_timer->timer_direction_upward = 0;
 215                         shared_timer->timer_direction_downward++;
 216                 }
 217
 218                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 219                         shared_timer->timer_in_use += 3;
 220                         shared_timer->timer_direction_upward = 0;
 221                         shared_timer->timer_direction_downward = 0;
 222                 }
 223                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 224                         shared_timer->timer_in_use -= 4 ;
 225                         shared_timer->timer_direction_downward = 0;
 226                         shared_timer->timer_direction_upward = 0;
 227                 }
 228         }
 229
 230         /* boundary checking */
 231         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 232                 shared_timer->timer_in_use = shared_timer->threshold_high;
 233         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 234                 shared_timer->timer_in_use = shared_timer->threshold_low;
 235
 236         nesdev->currcq_count = 0;
 237
 238         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 239 }
 240
 241
 242 /**
 243  * nes_init_adapter - initialize adapter
 244  */
 245 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 246         struct nes_adapter *nesadapter = NULL;
 247         unsigned long num_pds;
 248         u32 u32temp;
 249         u32 port_count;
 250         u16 max_rq_wrs;
 251         u16 max_sq_wrs;
 252         u32 max_mr;
 253         u32 max_256pbl;
 254         u32 max_4kpbl;
 255         u32 max_qp;
 256         u32 max_irrq;
 257         u32 max_cq;
 258         u32 hte_index_mask;
 259         u32 adapter_size;
 260         u32 arp_table_size;
 261         u16 vendor_id;
 262         u16 device_id;
 263         u8  OneG_Mode;
 264         u8  func_index;
 265
 266         /* search the list of existing adapters */
 267         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 268                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 269                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 270                                 nesdev->pcidev->devfn,
 271                                 PCI_SLOT(nesadapter->devfn),
 272                                 nesadapter->bus_number,
 273                                 PCI_SLOT(nesdev->pcidev->devfn),
 274                                 nesdev->pcidev->bus->number );
 275                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 276                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 277                         nesadapter->ref_count++;
 278                         return nesadapter;
 279                 }
 280         }
 281
 282         /* no adapter found */
 283         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 284         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 285                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 286                                 hw_rev);
 287                 return NULL;
 288         }
 289
 290         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 291                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 292                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 295
 296         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 297
 298
 299         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 300                 return NULL;
 301
 302         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 303         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 304
 305         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 306         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 307                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 308                                 max_qp, u32temp);
 309                 max_qp = (u32)1 << (u32temp & 0x001f);
 310         }
 311
 312         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 313         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 314                         max_qp, hte_index_mask);
 315
 316         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 317
 318         max_irrq = 1 << (u32temp & 0x001f);
 319
 320         if (max_qp > max_irrq) {
 321                 max_qp = max_irrq;
 322                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 323                                 max_qp);
 324         }
 325
 326         /* there should be no reason to allocate more pds than qps */
 327         if (num_pds > max_qp)
 328                 num_pds = max_qp;
 329
 330         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 331         max_mr = (u32)8192 << (u32temp & 0x7);
 332
 333         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 334         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 335         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 336         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 337
 338         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 339         arp_table_size = 1 << u32temp;
 340
 341         adapter_size = (sizeof(struct nes_adapter) +
 342                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 343         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 348         adapter_size += sizeof(struct nes_qp **) * max_qp;
 349
 350         /* allocate a new adapter struct */
 351         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 352         if (!nesadapter)
 353                 return NULL;
 354
 355         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 356                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 357
 358         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 359                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 360                 kfree(nesadapter);
 361                 return NULL;
 362         }
 363
 364         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 365                                 (nesadapter->mac_addr_low >> 24);
 366
 367         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 368                                  PCI_DEVICE_ID, &device_id);
 369         nesadapter->vendor_part_id = device_id;
 370
 371         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 372                                                         OneG_Mode)) {
 373                 kfree(nesadapter);
 374                 return NULL;
 375         }
 376         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 377
 378         memset(nesadapter->pft_mcast_map, 255,
 379                sizeof nesadapter->pft_mcast_map);
 380
 381         /* populate the new nesadapter */
 382         nesadapter->nesdev = nesdev;
 383         nesadapter->devfn = nesdev->pcidev->devfn;
 384         nesadapter->bus_number = nesdev->pcidev->bus->number;
 385         nesadapter->ref_count = 1;
 386         nesadapter->timer_int_req = 0xffff0000;
 387         nesadapter->OneG_Mode = OneG_Mode;
 388         nesadapter->doorbell_start = nesdev->doorbell_region;
 389
 390         /* nesadapter->tick_delta = clk_divisor; */
 391         nesadapter->hw_rev = hw_rev;
 392         nesadapter->port_count = port_count;
 393
 394         nesadapter->max_qp = max_qp;
 395         nesadapter->hte_index_mask = hte_index_mask;
 396         nesadapter->max_irrq = max_irrq;
 397         nesadapter->max_mr = max_mr;
 398         nesadapter->max_256pbl = max_256pbl - 1;
 399         nesadapter->max_4kpbl = max_4kpbl - 1;
 400         nesadapter->max_cq = max_cq;
 401         nesadapter->free_256pbl = max_256pbl - 1;
 402         nesadapter->free_4kpbl = max_4kpbl - 1;
 403         nesadapter->max_pd = num_pds;
 404         nesadapter->arp_table_size = arp_table_size;
 405
 406         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 407         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 408                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 409                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 410                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 411         } else {
 412                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 413                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 414                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 415                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 416         }
 417         /* Setup and enable the periodic timer */
 418         if (nesadapter->et_rx_coalesce_usecs_irq)
 419                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 420                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 421         else
 422                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 423
 424         nesadapter->base_pd = 1;
 425
 426         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 427                                        IB_DEVICE_MEM_WINDOW |
 428                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 429
 430         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 431                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 432         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 433         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 434         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 435         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 436         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 437
 438
 439         /* mark the usual suspect QPs, MR and CQs as in use */
 440         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 441                 set_bit(u32temp, nesadapter->allocated_qps);
 442                 set_bit(u32temp, nesadapter->allocated_cqs);
 443         }
 444         set_bit(0, nesadapter->allocated_mrs);
 445
 446         for (u32temp = 0; u32temp < 20; u32temp++)
 447                 set_bit(u32temp, nesadapter->allocated_pds);
 448         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 449
 450         max_rq_wrs = ((u32temp >> 8) & 3);
 451         switch (max_rq_wrs) {
 452                 case 0:
 453                         max_rq_wrs = 4;
 454                         break;
 455                 case 1:
 456                         max_rq_wrs = 16;
 457                         break;
 458                 case 2:
 459                         max_rq_wrs = 32;
 460                         break;
 461                 case 3:
 462                         max_rq_wrs = 512;
 463                         break;
 464         }
 465
 466         max_sq_wrs = (u32temp & 3);
 467         switch (max_sq_wrs) {
 468                 case 0:
 469                         max_sq_wrs = 4;
 470                         break;
 471                 case 1:
 472                         max_sq_wrs = 16;
 473                         break;
 474                 case 2:
 475                         max_sq_wrs = 32;
 476                         break;
 477                 case 3:
 478                         max_sq_wrs = 512;
 479                         break;
 480         }
 481         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 482         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 483
 484         nesadapter->max_sge = 4;
 485         nesadapter->max_cqe = 32766;
 486
 487         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 488                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 489                 kfree(nesadapter);
 490                 return NULL;
 491         }
 492
 493         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 494         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 495                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 496
 497         /* setup port configuration */
 498         if (nesadapter->port_count == 1) {
 499                 nesadapter->log_port = 0x00000000;
 500                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 501                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 502                 else
 503                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 504         } else {
 505                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 506                         nesadapter->log_port = 0x000000D8;
 507                 } else {
 508                         if (nesadapter->port_count == 2)
 509                                 nesadapter->log_port = 0x00000044;
 510                         else
 511                                 nesadapter->log_port = 0x000000e4;
 512                 }
 513                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 514         }
 515
 516         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 517                                                 nesadapter->log_port);
 518         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 519                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 520
 521         spin_lock_init(&nesadapter->resource_lock);
 522         spin_lock_init(&nesadapter->phy_lock);
 523         spin_lock_init(&nesadapter->pbl_lock);
 524         spin_lock_init(&nesadapter->periodic_timer_lock);
 525
 526         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 527         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 529         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 530
 531         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 532                 u32 pcs_control_status0, pcs_control_status1;
 533                 u32 reset_value;
 534                 u32 i = 0;
 535                 u32 int_cnt = 0;
 536                 u32 ext_cnt = 0;
 537                 unsigned long flags;
 538                 u32 j = 0;
 539
 540                 pcs_control_status0 = nes_read_indexed(nesdev,
 541                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 542                 pcs_control_status1 = nes_read_indexed(nesdev,
 543                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 544
 545                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 546                         pcs_control_status0 = nes_read_indexed(nesdev,
 547                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 548                         pcs_control_status1 = nes_read_indexed(nesdev,
 549                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 550                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 551                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 552                                 int_cnt++;
 553                         usleep_range(1000, 2000);
 554                 }
 555                 if (int_cnt > 1) {
 556                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 557                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 558                         mh_detected++;
 559                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 560                         reset_value |= 0x0000003d;
 561                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 562
 563                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 564                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 565                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 566
 567                         pcs_control_status0 = nes_read_indexed(nesdev,
 568                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 569                         pcs_control_status1 = nes_read_indexed(nesdev,
 570                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 571
 572                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 573                                 pcs_control_status0 = nes_read_indexed(nesdev,
 574                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 575                                 pcs_control_status1 = nes_read_indexed(nesdev,
 576                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 577                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 578                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 579                                         if (++ext_cnt > int_cnt) {
 580                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 581                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 582                                                                 0x0000F088);
 583                                                 mh_detected++;
 584                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 585                                                 reset_value |= 0x0000003d;
 586                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 587
 588                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 589                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 590                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 591                                                 break;
 592                                         }
 593                                 }
 594                                 usleep_range(1000, 2000);
 595                         }
 596                 }
 597         }
 598
 599         if (nesadapter->hw_rev == NE020_REV) {
 600                 timer_setup(&nesadapter->mh_timer, nes_mh_fix, 0);
 601                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 602                 add_timer(&nesadapter->mh_timer);
 603         } else {
 604                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 605         }
 606
 607         timer_setup(&nesadapter->lc_timer, nes_clc, 0);
 608         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 609         add_timer(&nesadapter->lc_timer);
 610
 611         list_add_tail(&nesadapter->list, &nes_adapter_list);
 612
 613         for (func_index = 0; func_index < 8; func_index++) {
 614                 pci_bus_read_config_word(nesdev->pcidev->bus,
 615                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 616                                         func_index), 0, &vendor_id);
 617                 if (vendor_id == 0xffff)
 618                         break;
 619         }
 620         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 621                 func_index, pci_name(nesdev->pcidev));
 622         nesadapter->adapter_fcn_count = func_index;
 623
 624         return nesadapter;
 625 }
 626
 627
 628 /**
 629  * nes_reset_adapter_ne020
 630  */
 631 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 632 {
 633         u32 port_count;
 634         u32 u32temp;
 635         u32 i;
 636
 637         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 638         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 639         /* TODO: assuming that both SERDES are set the same for now */
 640         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 641         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 642                         u32temp, port_count);
 643         if (*OneG_Mode)
 644                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 645         u32temp &= 0xff00ffc0;
 646         switch (port_count) {
 647                 case 1:
 648                         u32temp |= 0x00ee0000;
 649                         break;
 650                 case 2:
 651                         u32temp |= 0x00cc0000;
 652                         break;
 653                 case 4:
 654                         u32temp |= 0x00000000;
 655                         break;
 656                 default:
 657                         return 0;
 658                         break;
 659         }
 660
 661         /* check and do full reset if needed */
 662         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 663                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 664                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 665
 666                 i = 0;
 667                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 668                         mdelay(1);
 669                 if (i > 10000) {
 670                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 671                         return 0;
 672                 }
 673
 674                 i = 0;
 675                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 676                         mdelay(1);
 677                 if (i > 10000) {
 678                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 679                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 680                         return 0;
 681                 }
 682         }
 683
 684         /* port reset */
 685         switch (port_count) {
 686                 case 1:
 687                         u32temp |= 0x00ee0010;
 688                         break;
 689                 case 2:
 690                         u32temp |= 0x00cc0030;
 691                         break;
 692                 case 4:
 693                         u32temp |= 0x00000030;
 694                         break;
 695         }
 696
 697         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 698         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 699
 700         i = 0;
 701         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 702                 mdelay(1);
 703         if (i > 10000) {
 704                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 705                 return 0;
 706         }
 707
 708         /* serdes 0 */
 709         i = 0;
 710         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 711                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 712                 mdelay(1);
 713         if (i > 5000) {
 714                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 715                 return 0;
 716         }
 717
 718         /* serdes 1 */
 719         if (port_count > 1) {
 720                 i = 0;
 721                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 722                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 723                         mdelay(1);
 724                 if (i > 5000) {
 725                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 726                         return 0;
 727                 }
 728         }
 729
 730         return port_count;
 731 }
 732
 733
 734 /**
 735  * nes_init_serdes
 736  */
 737 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 738                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 739 {
 740         int i;
 741         u32 u32temp;
 742         u32 sds;
 743
 744         if (hw_rev != NE020_REV) {
 745                 /* init serdes 0 */
 746                 switch (nesadapter->phy_type[0]) {
 747                 case NES_PHY_TYPE_CX4:
 748                         if (wide_ppm_offset)
 749                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 750                         else
 751                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 752                         break;
 753                 case NES_PHY_TYPE_KR:
 754                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 755                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 756                         break;
 757                 case NES_PHY_TYPE_PUMA_1G:
 758                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 759                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 760                         sds |= 0x00000100;
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 762                         break;
 763                 default:
 764                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 765                         break;
 766                 }
 767
 768                 if (!OneG_Mode)
 769                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 770
 771                 if (port_count < 2)
 772                         return 0;
 773
 774                 /* init serdes 1 */
 775                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 776                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 777
 778                 switch (nesadapter->phy_type[1]) {
 779                 case NES_PHY_TYPE_ARGUS:
 780                 case NES_PHY_TYPE_SFP_D:
 781                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 782                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 783                         break;
 784                 case NES_PHY_TYPE_CX4:
 785                         if (wide_ppm_offset)
 786                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 787                         break;
 788                 case NES_PHY_TYPE_KR:
 789                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 790                         break;
 791                 case NES_PHY_TYPE_PUMA_1G:
 792                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 793                         sds |= 0x000000100;
 794                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 795                 }
 796                 if (!OneG_Mode) {
 797                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 798                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 799                         sds &= 0xFFFFFFBF;
 800                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 801                 }
 802         } else {
 803                 /* init serdes 0 */
 804                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 805                 i = 0;
 806                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 807                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 808                         mdelay(1);
 809                 if (i > 5000) {
 810                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 811                         return 1;
 812                 }
 813                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 814                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 815                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 816                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 817                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 818                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 819                 if (OneG_Mode)
 820                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 821                 else
 822                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 823
 824                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 825                 if (port_count > 1) {
 826                         /* init serdes 1 */
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 828                         i = 0;
 829                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 830                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 831                                 mdelay(1);
 832                         if (i > 5000) {
 833                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 834                                 /* return 1; */
 835                         }
 836                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 837                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 838                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 839                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 840                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 841                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 842                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 843                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 844                 }
 845         }
 846         return 0;
 847 }
 848
 849
 850 /**
 851  * nes_init_csr_ne020
 852  * Initialize registers for ne020 hardware
 853  */
 854 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 855 {
 856         u32 u32temp;
 857
 858         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 859
 860         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 861         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 862         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 863         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 864         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 865         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 866         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 867         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 868
 869         /* TODO: move these MAC register settings to NIC bringup */
 870         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 871         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 872         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 873         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 874         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 875         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 876         if (port_count > 1) {
 877                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 878                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 879                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 880                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 881                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 882                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 883                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 884         }
 885         if (port_count > 2) {
 886                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 887                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 888                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 889                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 890                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 891                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 892                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 893
 894                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 895                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 896                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 897                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 898                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 899                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 900                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 901         }
 902
 903         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 904         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 905         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 906                                                          0x00000001);
 907         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 908         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 909         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 910         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 911         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 912
 913         /* TODO: move this to code, get from EEPROM */
 914         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 915         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 916         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 917
 918         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 919         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 920
 921         if (hw_rev != NE020_REV) {
 922                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 923                 u32temp |= 0x80000000;
 924                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 925                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 926                 u32temp &= 0x7fffffff;
 927                 u32temp |= 0x7fff0010;
 928                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 929                 if (port_count > 1) {
 930                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 931                         u32temp &= 0x7fffffff;
 932                         u32temp |= 0x7fff0010;
 933                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 934                 }
 935         }
 936 }
 937
 938
 939 /**
 940  * nes_destroy_adapter - destroy the adapter structure
 941  */
 942 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 943 {
 944         struct nes_adapter *tmp_adapter;
 945
 946         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 947                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 948                                 tmp_adapter);
 949         }
 950
 951         nesadapter->ref_count--;
 952         if (!nesadapter->ref_count) {
 953                 if (nesadapter->hw_rev == NE020_REV) {
 954                         del_timer(&nesadapter->mh_timer);
 955                 }
 956                 del_timer(&nesadapter->lc_timer);
 957
 958                 list_del(&nesadapter->list);
 959                 kfree(nesadapter);
 960         }
 961 }
 962
 963
 964 /**
 965  * nes_init_cqp
 966  */
 967 int nes_init_cqp(struct nes_device *nesdev)
 968 {
 969         struct nes_adapter *nesadapter = nesdev->nesadapter;
 970         struct nes_hw_cqp_qp_context *cqp_qp_context;
 971         struct nes_hw_cqp_wqe *cqp_wqe;
 972         struct nes_hw_ceq *ceq;
 973         struct nes_hw_ceq *nic_ceq;
 974         struct nes_hw_aeq *aeq;
 975         void *vmem;
 976         dma_addr_t pmem;
 977         u32 count=0;
 978         u32 cqp_head;
 979         u64 u64temp;
 980         u32 u32temp;
 981
 982         /* allocate CQP memory */
 983         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 984         /* SQ is 512 byte aligned, others are 256 byte aligned */
 985         nesdev->cqp_mem_size = 512 +
 986                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 987                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 988                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 989                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 990                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 991                         sizeof(struct nes_hw_cqp_qp_context);
 992
 993         nesdev->cqp_vbase = pci_zalloc_consistent(nesdev->pcidev,
 994                                                   nesdev->cqp_mem_size,
 995                                                   &nesdev->cqp_pbase);
 996         if (!nesdev->cqp_vbase) {
 997                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 998                 return -ENOMEM;
 999         }
1000
1001         /* Allocate a twice the number of CQP requests as the SQ size */
1002         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1003                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1004         if (!nesdev->nes_cqp_requests) {
1005                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1006                                 nesdev->cqp.sq_pbase);
1007                 return -ENOMEM;
1008         }
1009
1010         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1011                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1012
1013         spin_lock_init(&nesdev->cqp.lock);
1014         init_waitqueue_head(&nesdev->cqp.waitq);
1015
1016         /* Setup Various Structures */
1017         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1018                         ~(unsigned long)(512 - 1));
1019         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1020                         ~(unsigned long long)(512 - 1));
1021
1022         nesdev->cqp.sq_vbase = vmem;
1023         nesdev->cqp.sq_pbase = pmem;
1024         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1025         nesdev->cqp.sq_head = 0;
1026         nesdev->cqp.sq_tail = 0;
1027         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1028
1029         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1030         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1031
1032         nesdev->ccq.cq_vbase = vmem;
1033         nesdev->ccq.cq_pbase = pmem;
1034         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1035         nesdev->ccq.cq_head = 0;
1036         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1037         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1038
1039         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1040         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1041
1042         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1043         ceq = &nesadapter->ceq[nesdev->ceq_index];
1044         ceq->ceq_vbase = vmem;
1045         ceq->ceq_pbase = pmem;
1046         ceq->ceq_size = NES_CCEQ_SIZE;
1047         ceq->ceq_head = 0;
1048
1049         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1050         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1051
1052         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1053         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1054         nic_ceq->ceq_vbase = vmem;
1055         nic_ceq->ceq_pbase = pmem;
1056         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1057         nic_ceq->ceq_head = 0;
1058
1059         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1060         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1061
1062         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1063         aeq->aeq_vbase = vmem;
1064         aeq->aeq_pbase = pmem;
1065         aeq->aeq_size = nesadapter->max_qp;
1066         aeq->aeq_head = 0;
1067
1068         /* Setup QP Context */
1069         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1070         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1071
1072         cqp_qp_context = vmem;
1073         cqp_qp_context->context_words[0] =
1074                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1075         cqp_qp_context->context_words[1] = 0;
1076         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1077         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1078
1079
1080         /* Write the address to Create CQP */
1081         if ((sizeof(dma_addr_t) > 4)) {
1082                 nes_write_indexed(nesdev,
1083                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1084                                 ((u64)pmem) >> 32);
1085         } else {
1086                 nes_write_indexed(nesdev,
1087                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1088         }
1089         nes_write_indexed(nesdev,
1090                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1091                         (u32)pmem);
1092
1093         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1094         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1095
1096         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1097                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1098                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1099         }
1100
1101         /* Write Create CCQ WQE */
1102         cqp_head = nesdev->cqp.sq_head++;
1103         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1104         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1105         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1106                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1107                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1108         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1109                             (nesdev->ccq.cq_number |
1110                              ((u32)nesdev->ceq_index << 16)));
1111         u64temp = (u64)nesdev->ccq.cq_pbase;
1112         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1113         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1114         u64temp = (unsigned long)&nesdev->ccq;
1115         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1116                         cpu_to_le32((u32)(u64temp >> 1));
1117         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1118                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1119         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1120
1121         /* Write Create CEQ WQE */
1122         cqp_head = nesdev->cqp.sq_head++;
1123         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1124         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1125         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1126                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1127         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1128         u64temp = (u64)ceq->ceq_pbase;
1129         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1130
1131         /* Write Create AEQ WQE */
1132         cqp_head = nesdev->cqp.sq_head++;
1133         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1134         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1135         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1136                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1137         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1138         u64temp = (u64)aeq->aeq_pbase;
1139         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1140
1141         /* Write Create NIC CEQ WQE */
1142         cqp_head = nesdev->cqp.sq_head++;
1143         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1144         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1145         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1146                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1147         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1148         u64temp = (u64)nic_ceq->ceq_pbase;
1149         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1150
1151         /* Poll until CCQP done */
1152         count = 0;
1153         do {
1154                 if (count++ > 1000) {
1155                         printk(KERN_ERR PFX "Error creating CQP\n");
1156                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1157                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1158                         return -1;
1159                 }
1160                 udelay(10);
1161         } while (!(nes_read_indexed(nesdev,
1162                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1163
1164         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1165                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1166
1167         u32temp = 0x04800000;
1168         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1169
1170         /* wait for the CCQ, CEQ, and AEQ to get created */
1171         count = 0;
1172         do {
1173                 if (count++ > 1000) {
1174                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1175                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1176                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1177                         return -1;
1178                 }
1179                 udelay(10);
1180         } while (((nes_read_indexed(nesdev,
1181                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1182
1183         /* dump the QP status value */
1184         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1185                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1186
1187         nesdev->cqp.sq_tail++;
1188
1189         return 0;
1190 }
1191
1192
1193 /**
1194  * nes_destroy_cqp
1195  */
1196 int nes_destroy_cqp(struct nes_device *nesdev)
1197 {
1198         struct nes_hw_cqp_wqe *cqp_wqe;
1199         u32 count = 0;
1200         u32 cqp_head;
1201         unsigned long flags;
1202
1203         do {
1204                 if (count++ > 1000)
1205                         break;
1206                 udelay(10);
1207         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1208
1209         /* Reset CCQ */
1210         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1211                         nesdev->ccq.cq_number);
1212
1213         /* Disable device interrupts */
1214         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1215
1216         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1217
1218         /* Destroy the AEQ */
1219         cqp_head = nesdev->cqp.sq_head++;
1220         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1221         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1222         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1223                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1224         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1225
1226         /* Destroy the NIC CEQ */
1227         cqp_head = nesdev->cqp.sq_head++;
1228         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1229         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1230         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1231                         ((u32)nesdev->nic_ceq_index << 8));
1232
1233         /* Destroy the CEQ */
1234         cqp_head = nesdev->cqp.sq_head++;
1235         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1236         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1237         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1238                         (nesdev->ceq_index << 8));
1239
1240         /* Destroy the CCQ */
1241         cqp_head = nesdev->cqp.sq_head++;
1242         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1243         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1244         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1245         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1246                         ((u32)nesdev->ceq_index << 16));
1247
1248         /* Destroy CQP */
1249         cqp_head = nesdev->cqp.sq_head++;
1250         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1251         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1252         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1253                         NES_CQP_QP_TYPE_CQP);
1254         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1255
1256         barrier();
1257         /* Ring doorbell (5 WQEs) */
1258         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1259
1260         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1261
1262         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1263         count = 0;
1264         do {
1265                 if (count++ > 1000) {
1266                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1267                                         PCI_FUNC(nesdev->pcidev->devfn));
1268                         break;
1269                 }
1270                 udelay(10);
1271         } while (((nes_read_indexed(nesdev,
1272                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1273
1274         /* dump the QP status value */
1275         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1276                         PCI_FUNC(nesdev->pcidev->devfn),
1277                         nes_read_indexed(nesdev,
1278                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1279
1280         kfree(nesdev->nes_cqp_requests);
1281
1282         /* Free the control structures */
1283         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1284                         nesdev->cqp.sq_pbase);
1285
1286         return 0;
1287 }
1288
1289
1290 /**
1291  * nes_init_1g_phy
1292  */
1293 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1294 {
1295         u32 counter = 0;
1296         u16 phy_data;
1297         int ret = 0;
1298
1299         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1300         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1301
1302         /* Reset the PHY */
1303         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1304         udelay(100);
1305         counter = 0;
1306         do {
1307                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1308                 if (counter++ > 100) {
1309                         ret = -1;
1310                         break;
1311                 }
1312         } while (phy_data & 0x8000);
1313
1314         /* Setting no phy loopback */
1315         phy_data &= 0xbfff;
1316         phy_data |= 0x1140;
1317         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1318         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1319         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1320         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1321
1322         /* Setting the interrupt mask */
1323         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1324         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1325         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1326
1327         /* turning on flow control */
1328         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1329         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1330         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1331
1332         /* Clear Half duplex */
1333         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1334         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1335         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1336
1337         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1338         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1339
1340         return ret;
1341 }
1342
1343
1344 /**
1345  * nes_init_2025_phy
1346  */
1347 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1348 {
1349         u32 temp_phy_data = 0;
1350         u32 temp_phy_data2 = 0;
1351         u32 counter = 0;
1352         u32 sds;
1353         u32 mac_index = nesdev->mac_index;
1354         int ret = 0;
1355         unsigned int first_attempt = 1;
1356
1357         /* Check firmware heartbeat */
1358         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1359         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1360         udelay(1500);
1361         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1362         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1363
1364         if (temp_phy_data != temp_phy_data2) {
1365                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1366                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1367                 if ((temp_phy_data & 0xff) > 0x20)
1368                         return 0;
1369                 printk(PFX "Reinitialize external PHY\n");
1370         }
1371
1372         /* no heartbeat, configure the PHY */
1373         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1374         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1375         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1376         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1377
1378         switch (phy_type) {
1379         case NES_PHY_TYPE_ARGUS:
1380                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1381                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1382                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1383                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1384                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1385                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1386                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1387
1388                 /* setup LEDs */
1389                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1390                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1391                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1392                 break;
1393
1394         case NES_PHY_TYPE_SFP_D:
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1397                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1398                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1399                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1400                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1401                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1402
1403                 /* setup LEDs */
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1406                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1407                 break;
1408
1409         case NES_PHY_TYPE_KR:
1410                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1412                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1413                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1414                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1415                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1416                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1417
1418                 /* setup LEDs */
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1421                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1422
1423                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1424                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1425                 break;
1426         }
1427
1428         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1429
1430         /* Bring PHY out of reset */
1431         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1432
1433         /* Check for heartbeat */
1434         counter = 0;
1435         mdelay(690);
1436         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1437         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1438         do {
1439                 if (counter++ > 150) {
1440                         printk(PFX "No PHY heartbeat\n");
1441                         break;
1442                 }
1443                 mdelay(1);
1444                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1445                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1446         } while ((temp_phy_data2 == temp_phy_data));
1447
1448         /* wait for tracking */
1449         counter = 0;
1450         do {
1451                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1452                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1453                 if (counter++ > 300) {
1454                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1455                                 first_attempt = 0;
1456                                 counter = 0;
1457                                 /* reset AMCC PHY and try again */
1458                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1459                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1460                                 continue;
1461                         } else {
1462                                 ret = 1;
1463                                 break;
1464                         }
1465                 }
1466                 mdelay(10);
1467         } while ((temp_phy_data & 0xff) < 0x30);
1468
1469         /* setup signal integrity */
1470         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1471         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1472         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1473         if (phy_type == NES_PHY_TYPE_KR) {
1474                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1475         } else {
1476                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1477                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1478         }
1479
1480         /* reset serdes */
1481         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1482         sds |= 0x1;
1483         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1484         sds &= 0xfffffffe;
1485         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1486
1487         counter = 0;
1488         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1489                         && (counter++ < 5000))
1490                 ;
1491
1492         return ret;
1493 }
1494
1495
1496 /**
1497  * nes_init_phy
1498  */
1499 int nes_init_phy(struct nes_device *nesdev)
1500 {
1501         struct nes_adapter *nesadapter = nesdev->nesadapter;
1502         u32 mac_index = nesdev->mac_index;
1503         u32 tx_config = 0;
1504         unsigned long flags;
1505         u8  phy_type = nesadapter->phy_type[mac_index];
1506         u8  phy_index = nesadapter->phy_index[mac_index];
1507         int ret = 0;
1508
1509         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1510         if (phy_type == NES_PHY_TYPE_1G) {
1511                 /* setup 1G MDIO operation */
1512                 tx_config &= 0xFFFFFFE3;
1513                 tx_config |= 0x04;
1514         } else {
1515                 /* setup 10G MDIO operation */
1516                 tx_config &= 0xFFFFFFE3;
1517                 tx_config |= 0x1D;
1518         }
1519         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1520
1521         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1522
1523         switch (phy_type) {
1524         case NES_PHY_TYPE_1G:
1525                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1526                 break;
1527         case NES_PHY_TYPE_ARGUS:
1528         case NES_PHY_TYPE_SFP_D:
1529         case NES_PHY_TYPE_KR:
1530                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1531                 break;
1532         }
1533
1534         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1535
1536         return ret;
1537 }
1538
1539
1540 /**
1541  * nes_replenish_nic_rq
1542  */
1543 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1544 {
1545         unsigned long flags;
1546         dma_addr_t bus_address;
1547         struct sk_buff *skb;
1548         struct nes_hw_nic_rq_wqe *nic_rqe;
1549         struct nes_hw_nic *nesnic;
1550         struct nes_device *nesdev;
1551         struct nes_rskb_cb *cb;
1552         u32 rx_wqes_posted = 0;
1553
1554         nesnic = &nesvnic->nic;
1555         nesdev = nesvnic->nesdev;
1556         spin_lock_irqsave(&nesnic->rq_lock, flags);
1557         if (nesnic->replenishing_rq !=0) {
1558                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1559                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1560                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1561                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1562                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1563                         add_timer(&nesvnic->rq_wqes_timer);
1564                 } else
1565                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1566                 return;
1567         }
1568         nesnic->replenishing_rq = 1;
1569         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1570         do {
1571                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1572                 if (skb) {
1573                         skb->dev = nesvnic->netdev;
1574
1575                         bus_address = pci_map_single(nesdev->pcidev,
1576                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1577                         cb = (struct nes_rskb_cb *)&skb->cb[0];
1578                         cb->busaddr = bus_address;
1579                         cb->maplen = nesvnic->max_frame_size;
1580
1581                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1582                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1583                                         cpu_to_le32(nesvnic->max_frame_size);
1584                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1585                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1586                                         cpu_to_le32((u32)bus_address);
1587                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1588                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1589                         nesnic->rx_skb[nesnic->rq_head] = skb;
1590                         nesnic->rq_head++;
1591                         nesnic->rq_head &= nesnic->rq_size - 1;
1592                         atomic_dec(&nesvnic->rx_skbs_needed);
1593                         barrier();
1594                         if (++rx_wqes_posted == 255) {
1595                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1596                                 rx_wqes_posted = 0;
1597                         }
1598                 } else {
1599                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1600                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1601                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1602                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1603                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1604                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1605                                 add_timer(&nesvnic->rq_wqes_timer);
1606                         } else
1607                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1608                         break;
1609                 }
1610         } while (atomic_read(&nesvnic->rx_skbs_needed));
1611         barrier();
1612         if (rx_wqes_posted)
1613                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1614         nesnic->replenishing_rq = 0;
1615 }
1616
1617
1618 /**
1619  * nes_rq_wqes_timeout
1620  */
1621 static void nes_rq_wqes_timeout(struct timer_list *t)
1622 {
1623         struct nes_vnic *nesvnic = from_timer(nesvnic, t, rq_wqes_timer);
1624         printk("%s: Timer fired.\n", __func__);
1625         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1626         if (atomic_read(&nesvnic->rx_skbs_needed))
1627                 nes_replenish_nic_rq(nesvnic);
1628 }
1629
1630
1631 /**
1632  * nes_init_nic_qp
1633  */
1634 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1635 {
1636         struct nes_hw_cqp_wqe *cqp_wqe;
1637         struct nes_hw_nic_sq_wqe *nic_sqe;
1638         struct nes_hw_nic_qp_context *nic_context;
1639         struct sk_buff *skb;
1640         struct nes_hw_nic_rq_wqe *nic_rqe;
1641         struct nes_vnic *nesvnic = netdev_priv(netdev);
1642         unsigned long flags;
1643         void *vmem;
1644         dma_addr_t pmem;
1645         u64 u64temp;
1646         int ret;
1647         u32 cqp_head;
1648         u32 counter;
1649         u32 wqe_count;
1650         struct nes_rskb_cb *cb;
1651         u8 jumbomode=0;
1652
1653         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1654         nesvnic->nic_mem_size = 256 +
1655                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1656                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1657                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1658                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1659                         sizeof(struct nes_hw_nic_qp_context);
1660
1661         nesvnic->nic_vbase = pci_zalloc_consistent(nesdev->pcidev,
1662                                                    nesvnic->nic_mem_size,
1663                                                    &nesvnic->nic_pbase);
1664         if (!nesvnic->nic_vbase) {
1665                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1666                 return -ENOMEM;
1667         }
1668         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1669                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1670
1671         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1672                         ~(unsigned long)(256 - 1));
1673         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1674                         ~(unsigned long long)(256 - 1));
1675
1676         /* Setup the first Fragment buffers */
1677         nesvnic->nic.first_frag_vbase = vmem;
1678
1679         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1680                 nesvnic->nic.frag_paddr[counter] = pmem;
1681                 pmem += sizeof(struct nes_first_frag);
1682         }
1683
1684         /* setup the SQ */
1685         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1686
1687         nesvnic->nic.sq_vbase = (void *)vmem;
1688         nesvnic->nic.sq_pbase = pmem;
1689         nesvnic->nic.sq_head = 0;
1690         nesvnic->nic.sq_tail = 0;
1691         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1692         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1693                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1694                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1695                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1696                                 NES_NIC_SQ_WQE_COMPLETION);
1697                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1698                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1699                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1700                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1701                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1702                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1703         }
1704
1705         nesvnic->get_cqp_request = nes_get_cqp_request;
1706         nesvnic->post_cqp_request = nes_post_cqp_request;
1707         nesvnic->mcrq_mcast_filter = NULL;
1708
1709         spin_lock_init(&nesvnic->nic.rq_lock);
1710
1711         /* setup the RQ */
1712         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1713         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1714
1715
1716         nesvnic->nic.rq_vbase = vmem;
1717         nesvnic->nic.rq_pbase = pmem;
1718         nesvnic->nic.rq_head = 0;
1719         nesvnic->nic.rq_tail = 0;
1720         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1721
1722         /* setup the CQ */
1723         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1724         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1725
1726         if (nesdev->nesadapter->netdev_count > 2)
1727                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1728         else
1729                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1730
1731         nesvnic->nic_cq.cq_vbase = vmem;
1732         nesvnic->nic_cq.cq_pbase = pmem;
1733         nesvnic->nic_cq.cq_head = 0;
1734         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1735
1736         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1737
1738         /* Send CreateCQ request to CQP */
1739         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1740         cqp_head = nesdev->cqp.sq_head;
1741
1742         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1743         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1744
1745         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1746                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1747                         ((u32)nesvnic->nic_cq.cq_size << 16));
1748         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1749                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1750         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1751         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1752         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1753         u64temp = (unsigned long)&nesvnic->nic_cq;
1754         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1755         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1756                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1757         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1758         if (++cqp_head >= nesdev->cqp.sq_size)
1759                 cqp_head = 0;
1760         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1761         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1762
1763         /* Send CreateQP request to CQP */
1764         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1765         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1766                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1767                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1768         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1769                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1770                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1771         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1772                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1773         }
1774
1775         u64temp = (u64)nesvnic->nic.sq_pbase;
1776         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1777         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1778         u64temp = (u64)nesvnic->nic.rq_pbase;
1779         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1780         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1781
1782         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1783                         NES_CQP_QP_TYPE_NIC);
1784         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1785         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1786                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1787         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1788
1789         if (++cqp_head >= nesdev->cqp.sq_size)
1790                 cqp_head = 0;
1791         nesdev->cqp.sq_head = cqp_head;
1792
1793         barrier();
1794
1795         /* Ring doorbell (2 WQEs) */
1796         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1797
1798         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1799         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1800                         nesvnic->nic.qp_id);
1801
1802         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1803                         NES_EVENT_TIMEOUT);
1804         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1805                         nesvnic->nic.qp_id, ret);
1806         if (!ret) {
1807                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1808                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1809                                 nesvnic->nic_pbase);
1810                 return -EIO;
1811         }
1812
1813         /* Populate the RQ */
1814         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1815                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1816                 if (!skb) {
1817                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1818
1819                         nes_destroy_nic_qp(nesvnic);
1820                         return -ENOMEM;
1821                 }
1822
1823                 skb->dev = netdev;
1824
1825                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1826                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1827                 cb = (struct nes_rskb_cb *)&skb->cb[0];
1828                 cb->busaddr = pmem;
1829                 cb->maplen = nesvnic->max_frame_size;
1830
1831                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1832                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1833                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1834                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1835                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1836                 nesvnic->nic.rx_skb[counter] = skb;
1837         }
1838
1839         wqe_count = NES_NIC_WQ_SIZE - 1;
1840         nesvnic->nic.rq_head = wqe_count;
1841         barrier();
1842         do {
1843                 counter = min(wqe_count, ((u32)255));
1844                 wqe_count -= counter;
1845                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1846         } while (wqe_count);
1847         timer_setup(&nesvnic->rq_wqes_timer, nes_rq_wqes_timeout, 0);
1848         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1849         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1850         {
1851                 nes_nic_init_timer(nesdev);
1852                 if (netdev->mtu > 1500)
1853                         jumbomode = 1;
1854                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1855         }
1856         if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1857                 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1858                 nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n",
1859                           netdev->name);
1860                 nes_destroy_nic_qp(nesvnic);
1861                 return -ENOMEM;
1862         }
1863
1864         return 0;
1865 }
1866
1867
1868 /**
1869  * nes_destroy_nic_qp
1870  */
1871 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1872 {
1873         u64 u64temp;
1874         dma_addr_t bus_address;
1875         struct nes_device *nesdev = nesvnic->nesdev;
1876         struct nes_hw_cqp_wqe *cqp_wqe;
1877         struct nes_hw_nic_sq_wqe *nic_sqe;
1878         __le16 *wqe_fragment_length;
1879         u16  wqe_fragment_index;
1880         u32 cqp_head;
1881         u32 wqm_cfg0;
1882         unsigned long flags;
1883         struct sk_buff *rx_skb;
1884         struct nes_rskb_cb *cb;
1885         int ret;
1886
1887         if (nesdev->nesadapter->allow_unaligned_fpdus)
1888                 nes_destroy_mgt(nesvnic);
1889
1890         /* clear wqe stall before destroying NIC QP */
1891         wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1892         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1893
1894         /* Free remaining NIC receive buffers */
1895         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1896                 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1897                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1898                 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1899                         PCI_DMA_FROMDEVICE);
1900
1901                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1902                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1903         }
1904
1905         /* Free remaining NIC transmit buffers */
1906         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1907                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1908                 wqe_fragment_index = 1;
1909                 wqe_fragment_length = (__le16 *)
1910                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1911                 /* bump past the vlan tag */
1912                 wqe_fragment_length++;
1913                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1914                         u64temp = (u64)le32_to_cpu(
1915                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1916                                 wqe_fragment_index*2]);
1917                         u64temp += ((u64)le32_to_cpu(
1918                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1919                                 + wqe_fragment_index*2]))<<32;
1920                         bus_address = (dma_addr_t)u64temp;
1921                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1922                                         nesvnic->nic.first_frag_overflow)) {
1923                                 pci_unmap_single(nesdev->pcidev,
1924                                                 bus_address,
1925                                                 le16_to_cpu(wqe_fragment_length[
1926                                                         wqe_fragment_index++]),
1927                                                 PCI_DMA_TODEVICE);
1928                         }
1929                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1930                                 if (wqe_fragment_length[wqe_fragment_index]) {
1931                                         u64temp = le32_to_cpu(
1932                                                 nic_sqe->wqe_words[
1933                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1934                                                 wqe_fragment_index*2]);
1935                                         u64temp += ((u64)le32_to_cpu(
1936                                                 nic_sqe->wqe_words[
1937                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1938                                                 wqe_fragment_index*2]))<<32;
1939                                         bus_address = (dma_addr_t)u64temp;
1940                                         pci_unmap_page(nesdev->pcidev,
1941                                                         bus_address,
1942                                                         le16_to_cpu(
1943                                                         wqe_fragment_length[
1944                                                         wqe_fragment_index]),
1945                                                         PCI_DMA_TODEVICE);
1946                                 } else
1947                                         break;
1948                         }
1949                 }
1950                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1951                         dev_kfree_skb(
1952                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1953
1954                 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1955                                         & (nesvnic->nic.sq_size - 1);
1956         }
1957
1958         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1959
1960         /* Destroy NIC QP */
1961         cqp_head = nesdev->cqp.sq_head;
1962         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1963         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1964
1965         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1966                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1967         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1968                 nesvnic->nic.qp_id);
1969
1970         if (++cqp_head >= nesdev->cqp.sq_size)
1971                 cqp_head = 0;
1972
1973         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1974
1975         /* Destroy NIC CQ */
1976         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1977         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1978                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1979         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1980                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1981
1982         if (++cqp_head >= nesdev->cqp.sq_size)
1983                 cqp_head = 0;
1984
1985         nesdev->cqp.sq_head = cqp_head;
1986         barrier();
1987
1988         /* Ring doorbell (2 WQEs) */
1989         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1990
1991         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1992         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1993                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1994                         cqp_head, nesdev->cqp.sq_head,
1995                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1996
1997         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1998                         NES_EVENT_TIMEOUT);
1999
2000         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2001                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2002                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2003         if (!ret) {
2004                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2005                                 nesvnic->nic.qp_id);
2006         }
2007
2008         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2009                         nesvnic->nic_pbase);
2010
2011         /* restore old wqm_cfg0 value */
2012         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2013 }
2014
2015 /**
2016  * nes_napi_isr
2017  */
2018 int nes_napi_isr(struct nes_device *nesdev)
2019 {
2020         struct nes_adapter *nesadapter = nesdev->nesadapter;
2021         u32 int_stat;
2022
2023         if (nesdev->napi_isr_ran) {
2024                 /* interrupt status has already been read in ISR */
2025                 int_stat = nesdev->int_stat;
2026         } else {
2027                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2028                 nesdev->int_stat = int_stat;
2029                 nesdev->napi_isr_ran = 1;
2030         }
2031
2032         int_stat &= nesdev->int_req;
2033         /* iff NIC, process here, else wait for DPC */
2034         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2035                 nesdev->napi_isr_ran = 0;
2036                 nes_write32(nesdev->regs + NES_INT_STAT,
2037                         (int_stat &
2038                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2039
2040                 /* Process the CEQs */
2041                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2042
2043                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2044                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2045                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2046                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2047                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2048                                 /* Enable Periodic timer interrupts */
2049                                 nesdev->int_req |= NES_INT_TIMER;
2050                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2051                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2052                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2053                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2054                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2055                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2056                         }
2057
2058                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2059                         {
2060                                 nes_nic_init_timer(nesdev);
2061                         }
2062                         /* Enable interrupts, except CEQs */
2063                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2064                 } else {
2065                         /* Enable interrupts, make sure timer is off */
2066                         nesdev->int_req &= ~NES_INT_TIMER;
2067                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2068                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2069                 }
2070                 nesdev->deepcq_count = 0;
2071                 return 1;
2072         } else {
2073                 return 0;
2074         }
2075 }
2076
2077 static void process_critical_error(struct nes_device *nesdev)
2078 {
2079         u32 debug_error;
2080         u32 nes_idx_debug_error_masks0 = 0;
2081         u16 error_module = 0;
2082
2083         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2084         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2085                         (u16)debug_error);
2086         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2087                         0x01010000 | (debug_error & 0x0000ffff));
2088         if (crit_err_count++ > 10)
2089                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2090         error_module = (u16) (debug_error & 0x1F00) >> 8;
2091         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2092                         nes_max_critical_error_count) {
2093                 printk(KERN_ERR PFX "Masking off critical error for module "
2094                         "0x%02X\n", (u16)error_module);
2095                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2096                         NES_IDX_DEBUG_ERROR_MASKS0);
2097                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2098                         nes_idx_debug_error_masks0 | (1 << error_module));
2099         }
2100 }
2101 /**
2102  * nes_dpc
2103  */
2104 void nes_dpc(unsigned long param)
2105 {
2106         struct nes_device *nesdev = (struct nes_device *)param;
2107         struct nes_adapter *nesadapter = nesdev->nesadapter;
2108         u32 counter;
2109         u32 loop_counter = 0;
2110         u32 int_status_bit;
2111         u32 int_stat;
2112         u32 timer_stat;
2113         u32 temp_int_stat;
2114         u32 intf_int_stat;
2115         u32 processed_intf_int = 0;
2116         u16 processed_timer_int = 0;
2117         u16 completion_ints = 0;
2118         u16 timer_ints = 0;
2119
2120         /* nes_debug(NES_DBG_ISR, "\n"); */
2121
2122         do {
2123                 timer_stat = 0;
2124                 if (nesdev->napi_isr_ran) {
2125                         nesdev->napi_isr_ran = 0;
2126                         int_stat = nesdev->int_stat;
2127                 } else
2128                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2129                 if (processed_intf_int != 0)
2130                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2131                 else
2132                         int_stat &= nesdev->int_req;
2133                 if (processed_timer_int == 0) {
2134                         processed_timer_int = 1;
2135                         if (int_stat & NES_INT_TIMER) {
2136                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2137                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2138                                         int_stat &= ~NES_INT_TIMER;
2139                                 }
2140                         }
2141                 } else {
2142                         int_stat &= ~NES_INT_TIMER;
2143                 }
2144
2145                 if (int_stat) {
2146                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2147                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2148                                 /* Ack the interrupts */
2149                                 nes_write32(nesdev->regs+NES_INT_STAT,
2150                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2151                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2152                         }
2153
2154                         temp_int_stat = int_stat;
2155                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2156                                 if (int_stat & int_status_bit) {
2157                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2158                                         temp_int_stat &= ~int_status_bit;
2159                                         completion_ints = 1;
2160                                 }
2161                                 if (!(temp_int_stat & 0x0000ffff))
2162                                         break;
2163                                 int_status_bit <<= 1;
2164                         }
2165
2166                         /* Process the AEQ for this pci function */
2167                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2168                         if (int_stat & int_status_bit) {
2169                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2170                         }
2171
2172                         /* Process the MAC interrupt for this pci function */
2173                         int_status_bit = 1 << (24 + nesdev->mac_index);
2174                         if (int_stat & int_status_bit) {
2175                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2176                         }
2177
2178                         if (int_stat & NES_INT_TIMER) {
2179                                 if (timer_stat & nesdev->timer_int_req) {
2180                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2181                                                         (timer_stat & nesdev->timer_int_req) |
2182                                                         ~(nesdev->nesadapter->timer_int_req));
2183                                         timer_ints = 1;
2184                                 }
2185                         }
2186
2187                         if (int_stat & NES_INT_INTF) {
2188                                 processed_intf_int = 1;
2189                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2190                                 intf_int_stat &= nesdev->intf_int_req;
2191                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2192                                         process_critical_error(nesdev);
2193                                 }
2194                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2195                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2196                                         BUG();
2197                                 }
2198                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2199                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2200                                         BUG();
2201                                 }
2202                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2203                         }
2204
2205                         if (int_stat & NES_INT_TSW) {
2206                         }
2207                 }
2208                 /* Don't use the interface interrupt bit stay in loop */
2209                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2210                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2211         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2212
2213         if (timer_ints == 1) {
2214                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2215                         if (completion_ints == 0) {
2216                                 nesdev->timer_only_int_count++;
2217                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2218                                         nesdev->timer_only_int_count = 0;
2219                                         nesdev->int_req &= ~NES_INT_TIMER;
2220                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2221                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2222                                 } else {
2223                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2224                                 }
2225                         } else {
2226                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2227                                 {
2228                                         nes_nic_init_timer(nesdev);
2229                                 }
2230                                 nesdev->timer_only_int_count = 0;
2231                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2232                         }
2233                 } else {
2234                         nesdev->timer_only_int_count = 0;
2235                         nesdev->int_req &= ~NES_INT_TIMER;
2236                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2237                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2238                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2239                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2240                 }
2241         } else {
2242                 if ( (completion_ints == 1) &&
2243                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2244                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2245                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2246                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2247                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2248                         nesdev->timer_only_int_count = 0;
2249                         nesdev->int_req |= NES_INT_TIMER;
2250                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2251                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2252                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2253                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2254                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2255                 } else {
2256                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2257                 }
2258         }
2259         nesdev->deepcq_count = 0;
2260 }
2261
2262
2263 /**
2264  * nes_process_ceq
2265  */
2266 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2267 {
2268         u64 u64temp;
2269         struct nes_hw_cq *cq;
2270         u32 head;
2271         u32 ceq_size;
2272
2273         /* nes_debug(NES_DBG_CQ, "\n"); */
2274         head = ceq->ceq_head;
2275         ceq_size = ceq->ceq_size;
2276
2277         do {
2278                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2279                                 NES_CEQE_VALID) {
2280                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2281                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2282                         u64temp <<= 1;
2283                         cq = *((struct nes_hw_cq **)&u64temp);
2284                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2285                         barrier();
2286                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2287
2288                         /* call the event handler */
2289                         cq->ce_handler(nesdev, cq);
2290
2291                         if (++head >= ceq_size)
2292                                 head = 0;
2293                 } else {
2294                         break;
2295                 }
2296
2297         } while (1);
2298
2299         ceq->ceq_head = head;
2300 }
2301
2302
2303 /**
2304  * nes_process_aeq
2305  */
2306 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2307 {
2308         /* u64 u64temp; */
2309         u32 head;
2310         u32 aeq_size;
2311         u32 aeqe_misc;
2312         u32 aeqe_cq_id;
2313         struct nes_hw_aeqe volatile *aeqe;
2314
2315         head = aeq->aeq_head;
2316         aeq_size = aeq->aeq_size;
2317
2318         do {
2319                 aeqe = &aeq->aeq_vbase[head];
2320                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2321                         break;
2322                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2323                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2324                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2325                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2326                                 /* dealing with an accelerated QP related AE */
2327                                 /*
2328                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2329                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2330                                  */
2331                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2332                         } else {
2333                                 /* TODO: dealing with a CQP related AE */
2334                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2335                                                 (u16)(aeqe_misc >> 16));
2336                         }
2337                 }
2338
2339                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2340
2341                 if (++head >= aeq_size)
2342                         head = 0;
2343
2344                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2345         }
2346         while (1);
2347         aeq->aeq_head = head;
2348 }
2349
2350 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2351 {
2352         struct nes_adapter *nesadapter = nesdev->nesadapter;
2353         u32 reset_value;
2354         u32 i=0;
2355         u32 u32temp;
2356
2357         if (nesadapter->hw_rev == NE020_REV) {
2358                 return;
2359         }
2360         mh_detected++;
2361
2362         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2363
2364         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2365                 reset_value |= 0x0000001d;
2366         else
2367                 reset_value |= 0x0000002d;
2368
2369         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2370                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2371                         nesadapter->link_interrupt_count[0] = 0;
2372                         nesadapter->link_interrupt_count[1] = 0;
2373                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2374                         if (0x00000040 & u32temp)
2375                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2376                         else
2377                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2378
2379                         reset_value |= 0x0000003d;
2380                 }
2381                 nesadapter->link_interrupt_count[mac_index] = 0;
2382         }
2383
2384         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2385
2386         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2387                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2388
2389         if (0x0000003d == (reset_value & 0x0000003d)) {
2390                 u32 pcs_control_status0, pcs_control_status1;
2391
2392                 for (i = 0; i < 10; i++) {
2393                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2394                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2395                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2396                              && (pcs_control_status0 & 0x00100000))
2397                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2398                                 && (pcs_control_status1 & 0x00100000)))
2399                                 continue;
2400                         else
2401                                 break;
2402                 }
2403                 if (10 == i) {
2404                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2405                         if (0x00000040 & u32temp)
2406                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2407                         else
2408                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2409
2410                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2411
2412                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2413                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2414                 }
2415         }
2416 }
2417
2418 /**
2419  * nes_process_mac_intr
2420  */
2421 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2422 {
2423         unsigned long flags;
2424         u32 pcs_control_status;
2425         struct nes_adapter *nesadapter = nesdev->nesadapter;
2426         struct nes_vnic *nesvnic;
2427         u32 mac_status;
2428         u32 mac_index = nesdev->mac_index;
2429         u32 u32temp;
2430         u16 phy_data;
2431         u16 temp_phy_data;
2432         u32 pcs_val  = 0x0f0f0000;
2433         u32 pcs_mask = 0x0f1f0000;
2434         u32 cdr_ctrl;
2435
2436         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2437         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2438                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2439                 return;
2440         }
2441         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2442
2443         /* ack the MAC interrupt */
2444         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2445         /* Clear the interrupt */
2446         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2447
2448         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2449
2450         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2451                 nesdev->link_status_interrupts++;
2452                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2453                         nes_reset_link(nesdev, mac_index);
2454
2455                 /* read the PHY interrupt status register */
2456                 if ((nesadapter->OneG_Mode) &&
2457                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2458                         do {
2459                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2460                                                 nesadapter->phy_index[mac_index], &phy_data);
2461                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2462                                                 nesadapter->phy_index[mac_index], phy_data);
2463                         } while (phy_data&0x8000);
2464
2465                         temp_phy_data = 0;
2466                         do {
2467                                 nes_read_1G_phy_reg(nesdev, 0x11,
2468                                                 nesadapter->phy_index[mac_index], &phy_data);
2469                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2470                                                 nesadapter->phy_index[mac_index], phy_data);
2471                                 if (temp_phy_data == phy_data)
2472                                         break;
2473                                 temp_phy_data = phy_data;
2474                         } while (1);
2475
2476                         nes_read_1G_phy_reg(nesdev, 0x1e,
2477                                         nesadapter->phy_index[mac_index], &phy_data);
2478                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2479                                         nesadapter->phy_index[mac_index], phy_data);
2480
2481                         nes_read_1G_phy_reg(nesdev, 1,
2482                                         nesadapter->phy_index[mac_index], &phy_data);
2483                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2484                                         nesadapter->phy_index[mac_index], phy_data);
2485
2486                         if (temp_phy_data & 0x1000) {
2487                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2488                                 phy_data = 4;
2489                         } else {
2490                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2491                         }
2492                 }
2493                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2494                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2495                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2496
2497                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2498                         switch (mac_index) {
2499                         case 1:
2500                         case 3:
2501                                 pcs_control_status = nes_read_indexed(nesdev,
2502                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2503                                 break;
2504                         default:
2505                                 pcs_control_status = nes_read_indexed(nesdev,
2506                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2507                                 break;
2508                         }
2509                 } else {
2510                         pcs_control_status = nes_read_indexed(nesdev,
2511                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2512                         pcs_control_status = nes_read_indexed(nesdev,
2513                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2514                 }
2515
2516                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2517                                 mac_index, pcs_control_status);
2518                 if ((nesadapter->OneG_Mode) &&
2519                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2520                         u32temp = 0x01010000;
2521                         if (nesadapter->port_count > 2) {
2522                                 u32temp |= 0x02020000;
2523                         }
2524                         if ((pcs_control_status & u32temp)!= u32temp) {
2525                                 phy_data = 0;
2526                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2527                         }
2528                 } else {
2529                         switch (nesadapter->phy_type[mac_index]) {
2530                         case NES_PHY_TYPE_ARGUS:
2531                         case NES_PHY_TYPE_SFP_D:
2532                         case NES_PHY_TYPE_KR:
2533                                 /* clear the alarms */
2534                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2535                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2536                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2537                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2538                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2539                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2540                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2541                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2542                                 /* check link status */
2543                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2544                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2545
2546                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2547                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2548                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2549                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2550
2551                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2552
2553                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2554                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2555                                 break;
2556
2557                         case NES_PHY_TYPE_PUMA_1G:
2558                                 if (mac_index < 2)
2559                                         pcs_val = pcs_mask = 0x01010000;
2560                                 else
2561                                         pcs_val = pcs_mask = 0x02020000;
2562                                 /* fall through */
2563                         default:
2564                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2565                                 break;
2566                         }
2567                 }
2568
2569                 if (phy_data & 0x0004) {
2570                         if (wide_ppm_offset &&
2571                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2572                             (nesadapter->hw_rev != NE020_REV)) {
2573                                 cdr_ctrl = nes_read_indexed(nesdev,
2574                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2575                                                             mac_index * 0x200);
2576                                 nes_write_indexed(nesdev,
2577                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2578                                                   mac_index * 0x200,
2579                                                   cdr_ctrl | 0x000F0000);
2580                         }
2581                         nesadapter->mac_link_down[mac_index] = 0;
2582                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2583                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2584                                                 nesvnic->linkup);
2585                                 if (nesvnic->linkup == 0) {
2586                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2587                                                         nesvnic->netdev->name, nesvnic->netdev);
2588                                         if (netif_queue_stopped(nesvnic->netdev))
2589                                                 netif_start_queue(nesvnic->netdev);
2590                                         nesvnic->linkup = 1;
2591                                         netif_carrier_on(nesvnic->netdev);
2592
2593                                         spin_lock(&nesvnic->port_ibevent_lock);
2594                                         if (nesvnic->of_device_registered) {
2595                                                 if (nesdev->iw_status == 0) {
2596                                                         nesdev->iw_status = 1;
2597                                                         nes_port_ibevent(nesvnic);
2598                                                 }
2599                                         }
2600                                         spin_unlock(&nesvnic->port_ibevent_lock);
2601                                 }
2602                         }
2603                 } else {
2604                         if (wide_ppm_offset &&
2605                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2606                             (nesadapter->hw_rev != NE020_REV)) {
2607                                 cdr_ctrl = nes_read_indexed(nesdev,
2608                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2609                                                             mac_index * 0x200);
2610                                 nes_write_indexed(nesdev,
2611                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2612                                                   mac_index * 0x200,
2613                                                   cdr_ctrl & 0xFFF0FFFF);
2614                         }
2615                         nesadapter->mac_link_down[mac_index] = 1;
2616                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2617                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2618                                                 nesvnic->linkup);
2619                                 if (nesvnic->linkup == 1) {
2620                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2621                                                         nesvnic->netdev->name, nesvnic->netdev);
2622                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2623                                                 netif_stop_queue(nesvnic->netdev);
2624                                         nesvnic->linkup = 0;
2625                                         netif_carrier_off(nesvnic->netdev);
2626
2627                                         spin_lock(&nesvnic->port_ibevent_lock);
2628                                         if (nesvnic->of_device_registered) {
2629                                                 if (nesdev->iw_status == 1) {
2630                                                         nesdev->iw_status = 0;
2631                                                         nes_port_ibevent(nesvnic);
2632                                                 }
2633                                         }
2634                                         spin_unlock(&nesvnic->port_ibevent_lock);
2635                                 }
2636                         }
2637                 }
2638                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2639                         nesdev->link_recheck = 1;
2640                         mod_delayed_work(system_wq, &nesdev->work,
2641                                          NES_LINK_RECHECK_DELAY);
2642                 }
2643         }
2644
2645         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2646
2647         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2648 }
2649
2650 void nes_recheck_link_status(struct work_struct *work)
2651 {
2652         unsigned long flags;
2653         struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2654         struct nes_adapter *nesadapter = nesdev->nesadapter;
2655         struct nes_vnic *nesvnic;
2656         u32 mac_index = nesdev->mac_index;
2657         u16 phy_data;
2658         u16 temp_phy_data;
2659
2660         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2661
2662         /* check link status */
2663         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2664         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2665
2666         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2667         nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2668         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2669         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2670
2671         phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2672
2673         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2674                 __func__, phy_data,
2675                 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2676
2677         if (phy_data & 0x0004) {
2678                 nesadapter->mac_link_down[mac_index] = 0;
2679                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2680                         if (nesvnic->linkup == 0) {
2681                                 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2682                                                 nesvnic->netdev->name, nesvnic->netdev);
2683                                 if (netif_queue_stopped(nesvnic->netdev))
2684                                         netif_start_queue(nesvnic->netdev);
2685                                 nesvnic->linkup = 1;
2686                                 netif_carrier_on(nesvnic->netdev);
2687
2688                                 spin_lock(&nesvnic->port_ibevent_lock);
2689                                 if (nesvnic->of_device_registered) {
2690                                         if (nesdev->iw_status == 0) {
2691                                                 nesdev->iw_status = 1;
2692                                                 nes_port_ibevent(nesvnic);
2693                                         }
2694                                 }
2695                                 spin_unlock(&nesvnic->port_ibevent_lock);
2696                         }
2697                 }
2698
2699         } else {
2700                 nesadapter->mac_link_down[mac_index] = 1;
2701                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2702                         if (nesvnic->linkup == 1) {
2703                                 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2704                                                 nesvnic->netdev->name, nesvnic->netdev);
2705                                 if (!(netif_queue_stopped(nesvnic->netdev)))
2706                                         netif_stop_queue(nesvnic->netdev);
2707                                 nesvnic->linkup = 0;
2708                                 netif_carrier_off(nesvnic->netdev);
2709
2710                                 spin_lock(&nesvnic->port_ibevent_lock);
2711                                 if (nesvnic->of_device_registered) {
2712                                         if (nesdev->iw_status == 1) {
2713                                                 nesdev->iw_status = 0;
2714                                                 nes_port_ibevent(nesvnic);
2715                                         }
2716                                 }
2717                                 spin_unlock(&nesvnic->port_ibevent_lock);
2718                         }
2719                 }
2720         }
2721         if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2722                 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2723         else
2724                 nesdev->link_recheck = 0;
2725
2726         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2727 }
2728
2729
2730 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2731 {
2732         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2733
2734         napi_schedule(&nesvnic->napi);
2735 }
2736
2737
2738 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2739 * getting out of nic_ce_handler
2740 */
2741 #define MAX_RQES_TO_PROCESS     384
2742
2743 /**
2744  * nes_nic_ce_handler
2745  */
2746 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2747 {
2748         u64 u64temp;
2749         dma_addr_t bus_address;
2750         struct nes_hw_nic *nesnic;
2751         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2752         struct nes_adapter *nesadapter = nesdev->nesadapter;
2753         struct nes_hw_nic_rq_wqe *nic_rqe;
2754         struct nes_hw_nic_sq_wqe *nic_sqe;
2755         struct sk_buff *skb;
2756         struct sk_buff *rx_skb;
2757         struct nes_rskb_cb *cb;
2758         __le16 *wqe_fragment_length;
2759         u32 head;
2760         u32 cq_size;
2761         u32 rx_pkt_size;
2762         u32 cqe_count=0;
2763         u32 cqe_errv;
2764         u32 cqe_misc;
2765         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2766         u16 vlan_tag;
2767         u16 pkt_type;
2768         u16 rqes_processed = 0;
2769         u8 sq_cqes = 0;
2770
2771         head = cq->cq_head;
2772         cq_size = cq->cq_size;
2773         cq->cqes_pending = 1;
2774         do {
2775                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2776                                 NES_NIC_CQE_VALID) {
2777                         nesnic = &nesvnic->nic;
2778                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2779                         if (cqe_misc & NES_NIC_CQE_SQ) {
2780                                 sq_cqes++;
2781                                 wqe_fragment_index = 1;
2782                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2783                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2784                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2785                                 /* bump past the vlan tag */
2786                                 wqe_fragment_length++;
2787                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2788                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2789                                                         wqe_fragment_index * 2]);
2790                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2791                                                         wqe_fragment_index * 2])) << 32;
2792                                         bus_address = (dma_addr_t)u64temp;
2793                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2794                                                 pci_unmap_single(nesdev->pcidev,
2795                                                                 bus_address,
2796                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2797                                                                 PCI_DMA_TODEVICE);
2798                                         }
2799                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2800                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2801                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2802                                                                                 wqe_fragment_index * 2]);
2803                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2804                                                                                 + wqe_fragment_index * 2])) <<32;
2805                                                         bus_address = (dma_addr_t)u64temp;
2806                                                         pci_unmap_page(nesdev->pcidev,
2807                                                                         bus_address,
2808                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2809                                                                         PCI_DMA_TODEVICE);
2810                                                 } else
2811                                                         break;
2812                                         }
2813                                 }
2814                                 if (skb)
2815                                         dev_kfree_skb_any(skb);
2816                                 nesnic->sq_tail++;
2817                                 nesnic->sq_tail &= nesnic->sq_size-1;
2818                                 if (sq_cqes > 128) {
2819                                         barrier();
2820                                         /* restart the queue if it had been stopped */
2821                                         if (netif_queue_stopped(nesvnic->netdev))
2822                                                 netif_wake_queue(nesvnic->netdev);
2823                                         sq_cqes = 0;
2824                                 }
2825                         } else {
2826                                 rqes_processed ++;
2827
2828                                 cq->rx_cqes_completed++;
2829                                 cq->rx_pkts_indicated++;
2830                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2831                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2832                                 /* Get the skb */
2833                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2834                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2835                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2836                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2837                                 pci_unmap_single(nesdev->pcidev, bus_address,
2838                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2839                                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2840                                 cb->busaddr = 0;
2841                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2842                                 /* rx_skb->len = rx_pkt_size; */
2843                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2844                                 skb_put(rx_skb, rx_pkt_size);
2845                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2846                                 nesnic->rq_tail++;
2847                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2848
2849                                 atomic_inc(&nesvnic->rx_skbs_needed);
2850                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2851                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2852                                                         cq->cq_number | (cqe_count << 16));
2853                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2854                                         nesdev->currcq_count += cqe_count;
2855                                         cqe_count = 0;
2856                                         nes_replenish_nic_rq(nesvnic);
2857                                 }
2858                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2859                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2860                                 rx_skb->ip_summed = CHECKSUM_NONE;
2861
2862                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2863                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2864                                         if ((cqe_errv &
2865                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2866                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2867                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2868                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2869                                         } else
2870                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2871                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2872                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2873
2874                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2875                                         if ((cqe_errv &
2876                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2877                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2878                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2879                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2880                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2881                                                                   nesvnic->netdev->name); */
2882                                                 }
2883                                         } else
2884                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2885                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2886                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2887                                         }
2888                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2889                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2890
2891                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2892                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2893                                                 rx_skb = NULL;
2894                                 }
2895                                 if (rx_skb == NULL)
2896                                         goto skip_rx_indicate0;
2897
2898
2899                                 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2900                                         vlan_tag = (u16)(le32_to_cpu(
2901                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2902                                                         >> 16);
2903                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2904                                                         nesvnic->netdev->name, vlan_tag);
2905
2906                                         __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag);
2907                                 }
2908                                 napi_gro_receive(&nesvnic->napi, rx_skb);
2909
2910 skip_rx_indicate0:
2911                                 ;
2912                                 /* nesvnic->netstats.rx_packets++; */
2913                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2914                         }
2915
2916                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2917                         /* Accounting... */
2918                         cqe_count++;
2919                         if (++head >= cq_size)
2920                                 head = 0;
2921                         if (cqe_count == 255) {
2922                                 /* Replenish Nic CQ */
2923                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2924                                                 cq->cq_number | (cqe_count << 16));
2925                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2926                                 nesdev->currcq_count += cqe_count;
2927                                 cqe_count = 0;
2928                         }
2929
2930                         if (cq->rx_cqes_completed >= nesvnic->budget)
2931                                 break;
2932                 } else {
2933                         cq->cqes_pending = 0;
2934                         break;
2935                 }
2936
2937         } while (1);
2938
2939         if (sq_cqes) {
2940                 barrier();
2941                 /* restart the queue if it had been stopped */
2942                 if (netif_queue_stopped(nesvnic->netdev))
2943                         netif_wake_queue(nesvnic->netdev);
2944         }
2945         cq->cq_head = head;
2946         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2947                         cq->cq_number, cqe_count, cq->cq_head); */
2948         cq->cqe_allocs_pending = cqe_count;
2949         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2950         {
2951                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2952                 nesdev->currcq_count += cqe_count;
2953                 nes_nic_tune_timer(nesdev);
2954         }
2955         if (atomic_read(&nesvnic->rx_skbs_needed))
2956                 nes_replenish_nic_rq(nesvnic);
2957 }
2958
2959
2960
2961 /**
2962  * nes_cqp_ce_handler
2963  */
2964 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2965 {
2966         u64 u64temp;
2967         unsigned long flags;
2968         struct nes_hw_cqp *cqp = NULL;
2969         struct nes_cqp_request *cqp_request;
2970         struct nes_hw_cqp_wqe *cqp_wqe;
2971         u32 head;
2972         u32 cq_size;
2973         u32 cqe_count=0;
2974         u32 error_code;
2975         u32 opcode;
2976         u32 ctx_index;
2977         /* u32 counter; */
2978
2979         head = cq->cq_head;
2980         cq_size = cq->cq_size;
2981
2982         do {
2983                 /* process the CQE */
2984                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2985                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2986
2987                 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
2988                 if (opcode & NES_CQE_VALID) {
2989                         cqp = &nesdev->cqp;
2990
2991                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2992                         if (error_code) {
2993                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2994                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2995                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2996                                                 (u16)(error_code >> 16),
2997                                                 (u16)error_code);
2998                         }
2999
3000                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3001                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3002                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
3003                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3004
3005                         cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3006                         if (cqp_request) {
3007                                 if (cqp_request->waiting) {
3008                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3009                                         cqp_request->major_code = (u16)(error_code >> 16);
3010                                         cqp_request->minor_code = (u16)error_code;
3011                                         barrier();
3012                                         cqp_request->request_done = 1;
3013                                         wake_up(&cqp_request->waitq);
3014                                         nes_put_cqp_request(nesdev, cqp_request);
3015                                 } else {
3016                                         if (cqp_request->callback)
3017                                                 cqp_request->cqp_callback(nesdev, cqp_request);
3018                                         nes_free_cqp_request(nesdev, cqp_request);
3019                                 }
3020                         } else {
3021                                 wake_up(&nesdev->cqp.waitq);
3022                         }
3023
3024                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3025                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3026                         if (++cqp->sq_tail >= cqp->sq_size)
3027                                 cqp->sq_tail = 0;
3028
3029                         /* Accounting... */
3030                         cqe_count++;
3031                         if (++head >= cq_size)
3032                                 head = 0;
3033                 } else {
3034                         break;
3035                 }
3036         } while (1);
3037         cq->cq_head = head;
3038
3039         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3040         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3041                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3042                         (nesdev->cqp.sq_size - 1)) != 1)) {
3043                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3044                                 struct nes_cqp_request, list);
3045                 list_del_init(&cqp_request->list);
3046                 head = nesdev->cqp.sq_head++;
3047                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3048                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3049                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3050                 barrier();
3051
3052                 opcode = le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX]);
3053                 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3054                         ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3055                 else
3056                         ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3057                 cqp_wqe->wqe_words[ctx_index] =
3058                         cpu_to_le32((u32)((unsigned long)cqp_request));
3059                 cqp_wqe->wqe_words[ctx_index + 1] =
3060                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3061                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3062                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3063                 /* Ring doorbell (1 WQEs) */
3064                 barrier();
3065                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3066         }
3067         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3068
3069         /* Arm the CCQ */
3070         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3071                         cq->cq_number);
3072         nes_read32(nesdev->regs+NES_CQE_ALLOC);
3073 }
3074
3075 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3076 {
3077         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3078                 /* skip over ethernet header */
3079                 pkt += ETH_HLEN;
3080
3081                 /* Skip over IP and TCP headers */
3082                 pkt += 4 * (pkt[0] & 0x0f);
3083                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3084         }
3085         return pkt;
3086 }
3087
3088 /* Determine if incoming error pkt is rdma layer */
3089 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3090 {
3091         u8 *pkt;
3092         u16 *mpa;
3093         u32 opcode = 0xffffffff;
3094
3095         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3096                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3097                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3098                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3099         }
3100
3101         return opcode;
3102 }
3103
3104 /* Build iWARP terminate header */
3105 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3106 {
3107         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3108         u16 ddp_seg_len;
3109         int copy_len = 0;
3110         u8 is_tagged = 0;
3111         u8 flush_code = 0;
3112         struct nes_terminate_hdr *termhdr;
3113
3114         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3115         memset(termhdr, 0, 64);
3116
3117         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3118
3119                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3120                 pkt = locate_mpa(pkt, aeq_info);
3121                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3122                 if (ddp_seg_len) {
3123                         copy_len = 2;
3124                         termhdr->hdrct = DDP_LEN_FLAG;
3125                         if (pkt[2] & 0x80) {
3126                                 is_tagged = 1;
3127                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3128                                         copy_len += TERM_DDP_LEN_TAGGED;
3129                                         termhdr->hdrct |= DDP_HDR_FLAG;
3130                                 }
3131                         } else {
3132                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3133                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3134                                         termhdr->hdrct |= DDP_HDR_FLAG;
3135                                 }
3136
3137                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3138                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3139                                                 copy_len += TERM_RDMA_LEN;
3140                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3141                                         }
3142                                 }
3143                         }
3144                 }
3145         }
3146
3147         switch (async_event_id) {
3148         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3149                 switch (iwarp_opcode(nesqp, aeq_info)) {
3150                 case IWARP_OPCODE_WRITE:
3151                         flush_code = IB_WC_LOC_PROT_ERR;
3152                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3153                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3154                         break;
3155                 default:
3156                         flush_code = IB_WC_REM_ACCESS_ERR;
3157                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3158                         termhdr->error_code = RDMAP_INV_STAG;
3159                 }
3160                 break;
3161         case NES_AEQE_AEID_AMP_INVALID_STAG:
3162                 flush_code = IB_WC_REM_ACCESS_ERR;
3163                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3164                 termhdr->error_code = RDMAP_INV_STAG;
3165                 break;
3166         case NES_AEQE_AEID_AMP_BAD_QP:
3167                 flush_code = IB_WC_LOC_QP_OP_ERR;
3168                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3169                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3170                 break;
3171         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3172         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3173                 switch (iwarp_opcode(nesqp, aeq_info)) {
3174                 case IWARP_OPCODE_SEND_INV:
3175                 case IWARP_OPCODE_SEND_SE_INV:
3176                         flush_code = IB_WC_REM_OP_ERR;
3177                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3178                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3179                         break;
3180                 default:
3181                         flush_code = IB_WC_REM_ACCESS_ERR;
3182                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3183                         termhdr->error_code = RDMAP_INV_STAG;
3184                 }
3185                 break;
3186         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3187                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3188                         flush_code = IB_WC_LOC_PROT_ERR;
3189                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3190                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3191                 } else {
3192                         flush_code = IB_WC_REM_ACCESS_ERR;
3193                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3194                         termhdr->error_code = RDMAP_INV_BOUNDS;
3195                 }
3196                 break;
3197         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3198         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3199         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3200                 flush_code = IB_WC_REM_ACCESS_ERR;
3201                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3202                 termhdr->error_code = RDMAP_ACCESS;
3203                 break;
3204         case NES_AEQE_AEID_AMP_TO_WRAP:
3205                 flush_code = IB_WC_REM_ACCESS_ERR;
3206                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3207                 termhdr->error_code = RDMAP_TO_WRAP;
3208                 break;
3209         case NES_AEQE_AEID_AMP_BAD_PD:
3210                 switch (iwarp_opcode(nesqp, aeq_info)) {
3211                 case IWARP_OPCODE_WRITE:
3212                         flush_code = IB_WC_LOC_PROT_ERR;
3213                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3214                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3215                         break;
3216                 case IWARP_OPCODE_SEND_INV:
3217                 case IWARP_OPCODE_SEND_SE_INV:
3218                         flush_code = IB_WC_REM_ACCESS_ERR;
3219                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3220                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3221                         break;
3222                 default:
3223                         flush_code = IB_WC_REM_ACCESS_ERR;
3224                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3225                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3226                 }
3227                 break;
3228         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3229                 flush_code = IB_WC_LOC_LEN_ERR;
3230                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3231                 termhdr->error_code = MPA_MARKER;
3232                 break;
3233         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3234                 flush_code = IB_WC_GENERAL_ERR;
3235                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3236                 termhdr->error_code = MPA_CRC;
3237                 break;
3238         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3239         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3240                 flush_code = IB_WC_LOC_LEN_ERR;
3241                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3242                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3243                 break;
3244         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3245         case NES_AEQE_AEID_DDP_NO_L_BIT:
3246                 flush_code = IB_WC_FATAL_ERR;
3247                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3248                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3249                 break;
3250         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3251         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3252                 flush_code = IB_WC_GENERAL_ERR;
3253                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3254                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3255                 break;
3256         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3257                 flush_code = IB_WC_LOC_LEN_ERR;
3258                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3259                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3260                 break;
3261         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3262                 flush_code = IB_WC_GENERAL_ERR;
3263                 if (is_tagged) {
3264                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3265                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3266                 } else {
3267                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3268                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3269                 }
3270                 break;
3271         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3272                 flush_code = IB_WC_GENERAL_ERR;
3273                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3274                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3275                 break;
3276         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3277                 flush_code = IB_WC_REM_OP_ERR;
3278                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3279                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3280                 break;
3281         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3282                 flush_code = IB_WC_GENERAL_ERR;
3283                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3284                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3285                 break;
3286         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3287                 flush_code = IB_WC_GENERAL_ERR;
3288                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3289                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3290                 break;
3291         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3292                 flush_code = IB_WC_LOC_QP_OP_ERR;
3293                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3294                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3295                 break;
3296         default:
3297                 flush_code = IB_WC_FATAL_ERR;
3298                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3299                 termhdr->error_code = RDMAP_UNSPECIFIED;
3300                 break;
3301         }
3302
3303         if (copy_len)
3304                 memcpy(termhdr + 1, pkt, copy_len);
3305
3306         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3307                 if (aeq_info & NES_AEQE_SQ)
3308                         nesqp->term_sq_flush_code = flush_code;
3309                 else
3310                         nesqp->term_rq_flush_code = flush_code;
3311         }
3312
3313         return sizeof(struct nes_terminate_hdr) + copy_len;
3314 }
3315
3316 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3317                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3318 {
3319         u64 context;
3320         unsigned long flags;
3321         u32 aeq_info;
3322         u16 async_event_id;
3323         u8 tcp_state;
3324         u8 iwarp_state;
3325         u32 termlen = 0;
3326         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3327                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3328         struct nes_adapter *nesadapter = nesdev->nesadapter;
3329
3330         if (nesqp->term_flags & NES_TERM_SENT)
3331                 return; /* Sanity check */
3332
3333         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3334         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3335         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3336         async_event_id = (u16)aeq_info;
3337
3338         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3339                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3340         if (!context) {
3341                 WARN_ON(!context);
3342                 return;
3343         }
3344
3345         nesqp = (struct nes_qp *)(unsigned long)context;
3346         spin_lock_irqsave(&nesqp->lock, flags);
3347         nesqp->hw_iwarp_state = iwarp_state;
3348         nesqp->hw_tcp_state = tcp_state;
3349         nesqp->last_aeq = async_event_id;
3350         nesqp->terminate_eventtype = eventtype;
3351         spin_unlock_irqrestore(&nesqp->lock, flags);
3352
3353         if (nesadapter->send_term_ok)
3354                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3355         else
3356                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3357
3358         if (!nesdev->iw_status)  {
3359                 nesqp->term_flags = NES_TERM_DONE;
3360                 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3361                 nes_cm_disconn(nesqp);
3362         } else {
3363                 nes_terminate_start_timer(nesqp);
3364                 nesqp->term_flags |= NES_TERM_SENT;
3365                 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3366         }
3367 }
3368
3369 static void nes_terminate_send_fin(struct nes_device *nesdev,
3370                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3371 {
3372         u32 aeq_info;
3373         u16 async_event_id;
3374         u8 tcp_state;
3375         u8 iwarp_state;
3376         unsigned long flags;
3377
3378         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3379         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3380         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3381         async_event_id = (u16)aeq_info;
3382
3383         spin_lock_irqsave(&nesqp->lock, flags);
3384         nesqp->hw_iwarp_state = iwarp_state;
3385         nesqp->hw_tcp_state = tcp_state;
3386         nesqp->last_aeq = async_event_id;
3387         spin_unlock_irqrestore(&nesqp->lock, flags);
3388
3389         /* Send the fin only */
3390         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3391                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3392 }
3393
3394 /* Cleanup after a terminate sent or received */
3395 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3396 {
3397         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3398         unsigned long flags;
3399         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3400         struct nes_device *nesdev = nesvnic->nesdev;
3401         u8 first_time = 0;
3402
3403         spin_lock_irqsave(&nesqp->lock, flags);
3404         if (nesqp->hte_added) {
3405                 nesqp->hte_added = 0;
3406                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3407         }
3408
3409         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3410         nesqp->term_flags |= NES_TERM_DONE;
3411         spin_unlock_irqrestore(&nesqp->lock, flags);
3412
3413         /* Make sure we go through this only once */
3414         if (first_time) {
3415                 if (timeout_occurred == 0)
3416                         del_timer(&nesqp->terminate_timer);
3417                 else
3418                         next_iwarp_state |= NES_CQP_QP_RESET;
3419
3420                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3421                 nes_cm_disconn(nesqp);
3422         }
3423 }
3424
3425 static void nes_terminate_received(struct nes_device *nesdev,
3426                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3427 {
3428         u32 aeq_info;
3429         u8 *pkt;
3430         u32 *mpa;
3431         u8 ddp_ctl;
3432         u8 rdma_ctl;
3433         u16 aeq_id = 0;
3434
3435         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3436         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3437                 /* Terminate is not a performance path so the silicon */
3438                 /* did not validate the frame - do it now */
3439                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3440                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3441                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3442                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3443                 if ((ddp_ctl & 0xc0) != 0x40)
3444                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3445                 else if ((ddp_ctl & 0x03) != 1)
3446                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3447                 else if (be32_to_cpu(mpa[2]) != 2)
3448                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3449                 else if (be32_to_cpu(mpa[3]) != 1)
3450                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3451                 else if (be32_to_cpu(mpa[4]) != 0)
3452                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3453                 else if ((rdma_ctl & 0xc0) != 0x40)
3454                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3455
3456                 if (aeq_id) {
3457                         /* Bad terminate recvd - send back a terminate */
3458                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3459                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3460                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3461                         return;
3462                 }
3463         }
3464
3465         nesqp->term_flags |= NES_TERM_RCVD;
3466         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3467         nes_terminate_start_timer(nesqp);
3468         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3469 }
3470
3471 /* Timeout routine in case terminate fails to complete */
3472 void nes_terminate_timeout(struct timer_list *t)
3473 {
3474         struct nes_qp *nesqp = from_timer(nesqp, t, terminate_timer);
3475
3476         nes_terminate_done(nesqp, 1);
3477 }
3478
3479 /* Set a timer in case hw cannot complete the terminate sequence */
3480 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3481 {
3482         mod_timer(&nesqp->terminate_timer, (jiffies + HZ));
3483 }
3484
3485 /**
3486  * nes_process_iwarp_aeqe
3487  */
3488 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3489                                    struct nes_hw_aeqe *aeqe)
3490 {
3491         u64 context;
3492         unsigned long flags;
3493         struct nes_qp *nesqp;
3494         struct nes_hw_cq *hw_cq;
3495         struct nes_cq *nescq;
3496         int resource_allocated;
3497         struct nes_adapter *nesadapter = nesdev->nesadapter;
3498         u32 aeq_info;
3499         u32 next_iwarp_state = 0;
3500         u32 aeqe_cq_id;
3501         u16 async_event_id;
3502         u8 tcp_state;
3503         u8 iwarp_state;
3504         struct ib_event ibevent;
3505
3506         nes_debug(NES_DBG_AEQ, "\n");
3507         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3508         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3509                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3510                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3511         } else {
3512                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3513                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3514                 BUG_ON(!context);
3515         }
3516
3517         /* context is nesqp unless async_event_id == CQ ERROR */
3518         nesqp = (struct nes_qp *)(unsigned long)context;
3519         async_event_id = (u16)aeq_info;
3520         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3521         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3522         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3523                         " Tcp state = %s, iWARP state = %s\n",
3524                         async_event_id,
3525                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3526                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3527
3528         aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3529         if (aeq_info & NES_AEQE_QP) {
3530                 if (!nes_is_resource_allocated(nesadapter,
3531                                 nesadapter->allocated_qps,
3532                                 aeqe_cq_id))
3533                         return;
3534         }
3535
3536         switch (async_event_id) {
3537                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3538                         if (nesqp->term_flags)
3539                                 return; /* Ignore it, wait for close complete */
3540
3541                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3542                                 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3543                                         (nesqp->ibqp_state == IB_QPS_RTS)) {
3544                                         spin_lock_irqsave(&nesqp->lock, flags);
3545                                         nesqp->hw_iwarp_state = iwarp_state;
3546                                         nesqp->hw_tcp_state = tcp_state;
3547                                         nesqp->last_aeq = async_event_id;
3548                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3549                                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3550                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3551                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3552                                         nes_cm_disconn(nesqp);
3553                                 }
3554                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3555                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3556                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3557                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3558                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3559                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3560                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3561                                                 async_event_id, nesqp->last_aeq, tcp_state);
3562                         }
3563                         break;
3564                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3565                         spin_lock_irqsave(&nesqp->lock, flags);
3566                         nesqp->hw_iwarp_state = iwarp_state;
3567                         nesqp->hw_tcp_state = tcp_state;
3568                         nesqp->last_aeq = async_event_id;
3569                         spin_unlock_irqrestore(&nesqp->lock, flags);
3570                         nes_cm_disconn(nesqp);
3571                         break;
3572
3573                 case NES_AEQE_AEID_RESET_SENT:
3574                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3575                         spin_lock_irqsave(&nesqp->lock, flags);
3576                         nesqp->hw_iwarp_state = iwarp_state;
3577                         nesqp->hw_tcp_state = tcp_state;
3578                         nesqp->last_aeq = async_event_id;
3579                         nesqp->hte_added = 0;
3580                         spin_unlock_irqrestore(&nesqp->lock, flags);
3581                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3582                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3583                         nes_cm_disconn(nesqp);
3584                         break;
3585
3586                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3587                         if (atomic_read(&nesqp->close_timer_started))
3588                                 return;
3589                         spin_lock_irqsave(&nesqp->lock, flags);
3590                         nesqp->hw_iwarp_state = iwarp_state;
3591                         nesqp->hw_tcp_state = tcp_state;
3592                         nesqp->last_aeq = async_event_id;
3593                         spin_unlock_irqrestore(&nesqp->lock, flags);
3594                         nes_cm_disconn(nesqp);
3595                         break;
3596
3597                 case NES_AEQE_AEID_TERMINATE_SENT:
3598                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3599                         break;
3600
3601                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3602                         nes_terminate_received(nesdev, nesqp, aeqe);
3603                         break;
3604
3605                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3606                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3607                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3608                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3609                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3610                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3611                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3612                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3613                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3614                 case NES_AEQE_AEID_AMP_TO_WRAP:
3615                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3616                                         nesqp->hwqp.qp_id, async_event_id);
3617                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3618                         break;
3619
3620                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3621                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3622                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3623                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3624                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3625                                 aeq_info &= 0xffff0000;
3626                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3627                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3628                         }
3629                         /* fall through */
3630                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3631                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3632                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3633                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3634                 case NES_AEQE_AEID_AMP_BAD_QP:
3635                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3636                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3637                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3638                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3639                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3640                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3641                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3642                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3643                 case NES_AEQE_AEID_AMP_BAD_PD:
3644                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3645                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3646                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3647                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3648                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3649                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3650                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3651                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3652                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3653                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3654                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3655                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3656                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3657                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3658                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3659                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3660                 case NES_AEQE_AEID_BAD_CLOSE:
3661                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3662                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3663                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3664                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3665                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3666                                         nesqp->hwqp.qp_id, async_event_id);
3667                         print_ip(nesqp->cm_node);
3668                         if (!atomic_read(&nesqp->close_timer_started))
3669                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3670                         break;
3671
3672                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3673                         context <<= 1;
3674                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3675                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3676                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3677                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3678                         if (resource_allocated) {
3679                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3680                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3681                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3682                                 if (hw_cq) {
3683                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3684                                         if (nescq->ibcq.event_handler) {
3685                                                 ibevent.device = nescq->ibcq.device;
3686                                                 ibevent.event = IB_EVENT_CQ_ERR;
3687                                                 ibevent.element.cq = &nescq->ibcq;
3688                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3689                                         }
3690                                 }
3691                         }
3692                         break;
3693
3694                 default:
3695                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3696                                         async_event_id);
3697                         break;
3698         }
3699
3700 }
3701
3702 /**
3703  * nes_iwarp_ce_handler
3704  */
3705 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3706 {
3707         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3708
3709         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3710                         nescq->hw_cq.cq_number); */
3711         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3712
3713         if (nescq->ibcq.comp_handler)
3714                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3715
3716         return;
3717 }
3718
3719
3720 /**
3721  * nes_manage_apbvt()
3722  */
3723 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3724                 u32 nic_index, u32 add_port)
3725 {
3726         struct nes_device *nesdev = nesvnic->nesdev;
3727         struct nes_hw_cqp_wqe *cqp_wqe;
3728         struct nes_cqp_request *cqp_request;
3729         int ret = 0;
3730         u16 major_code;
3731
3732         /* Send manage APBVT request to CQP */
3733         cqp_request = nes_get_cqp_request(nesdev);
3734         if (cqp_request == NULL) {
3735                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3736                 return -ENOMEM;
3737         }
3738         cqp_request->waiting = 1;
3739         cqp_wqe = &cqp_request->cqp_wqe;
3740
3741         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3742                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3743                         accel_local_port, accel_local_port, nic_index);
3744
3745         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3746         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3747                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3748         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3749                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3750
3751         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3752
3753         atomic_set(&cqp_request->refcount, 2);
3754         nes_post_cqp_request(nesdev, cqp_request);
3755
3756         if (add_port == NES_MANAGE_APBVT_ADD)
3757                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3758                                 NES_EVENT_TIMEOUT);
3759         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3760                         ret, cqp_request->major_code, cqp_request->minor_code);
3761         major_code = cqp_request->major_code;
3762
3763         nes_put_cqp_request(nesdev, cqp_request);
3764
3765         if (!ret)
3766                 return -ETIME;
3767         else if (major_code)
3768                 return -EIO;
3769         else
3770                 return 0;
3771 }
3772
3773
3774 /**
3775  * nes_manage_arp_cache
3776  */
3777 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3778                 u32 ip_addr, u32 action)
3779 {
3780         struct nes_hw_cqp_wqe *cqp_wqe;
3781         struct nes_vnic *nesvnic = netdev_priv(netdev);
3782         struct nes_device *nesdev;
3783         struct nes_cqp_request *cqp_request;
3784         int arp_index;
3785
3786         nesdev = nesvnic->nesdev;
3787         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3788         if (arp_index == -1) {
3789                 return;
3790         }
3791
3792         /* update the ARP entry */
3793         cqp_request = nes_get_cqp_request(nesdev);
3794         if (cqp_request == NULL) {
3795                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3796                 return;
3797         }
3798         cqp_request->waiting = 0;
3799         cqp_wqe = &cqp_request->cqp_wqe;
3800         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3801
3802         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3803                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3804         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3805                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3806         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3807
3808         if (action == NES_ARP_ADD) {
3809                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3810                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3811                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3812                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3813                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3814                                 (((u32)mac_addr[0]) << 8) | (u32)mac_addr[1]);
3815         } else {
3816                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3817                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3818         }
3819
3820         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3821                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3822
3823         atomic_set(&cqp_request->refcount, 1);
3824         nes_post_cqp_request(nesdev, cqp_request);
3825 }
3826
3827
3828 /**
3829  * flush_wqes
3830  */
3831 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3832                 u32 which_wq, u32 wait_completion)
3833 {
3834         struct nes_cqp_request *cqp_request;
3835         struct nes_hw_cqp_wqe *cqp_wqe;
3836         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3837         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3838         int ret;
3839
3840         cqp_request = nes_get_cqp_request(nesdev);
3841         if (cqp_request == NULL) {
3842                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3843                 return;
3844         }
3845         if (wait_completion) {
3846                 cqp_request->waiting = 1;
3847                 atomic_set(&cqp_request->refcount, 2);
3848         } else {
3849                 cqp_request->waiting = 0;
3850         }
3851         cqp_wqe = &cqp_request->cqp_wqe;
3852         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3853
3854         /* If wqe in error was identified, set code to be put into cqe */
3855         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3856                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3857                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3858                 nesqp->term_sq_flush_code = 0;
3859         }
3860
3861         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3862                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3863                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3864                 nesqp->term_rq_flush_code = 0;
3865         }
3866
3867         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3868                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3869                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3870         }
3871
3872         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3873                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3874         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3875
3876         nes_post_cqp_request(nesdev, cqp_request);
3877
3878         if (wait_completion) {
3879                 /* Wait for CQP */
3880                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3881                                 NES_EVENT_TIMEOUT);
3882                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3883                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3884                                 ret, cqp_request->major_code, cqp_request->minor_code);
3885                 nes_put_cqp_request(nesdev, cqp_request);
3886         }
3887 }