drivers/infiniband/core/verbs.c

   1 /*
   2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
   3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
   4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
   5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
   7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   8  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
   9  *
  10  * This software is available to you under a choice of one of two
  11  * licenses.  You may choose to be licensed under the terms of the GNU
  12  * General Public License (GPL) Version 2, available from the file
  13  * COPYING in the main directory of this source tree, or the
  14  * OpenIB.org BSD license below:
  15  *
  16  *     Redistribution and use in source and binary forms, with or
  17  *     without modification, are permitted provided that the following
  18  *     conditions are met:
  19  *
  20  *      - Redistributions of source code must retain the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer.
  23  *
  24  *      - Redistributions in binary form must reproduce the above
  25  *        copyright notice, this list of conditions and the following
  26  *        disclaimer in the documentation and/or other materials
  27  *        provided with the distribution.
  28  *
  29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  36  * SOFTWARE.
  37  */
  38
  39 #include <linux/errno.h>
  40 #include <linux/err.h>
  41 #include <linux/export.h>
  42 #include <linux/string.h>
  43 #include <linux/slab.h>
  44
  45 #include <rdma/ib_verbs.h>
  46 #include <rdma/ib_cache.h>
  47 #include <rdma/ib_addr.h>
  48
  49 #include "core_priv.h"
  50
  51 __attribute_const__ int ib_rate_to_mult(enum ib_rate rate)
  52 {
  53         switch (rate) {
  54         case IB_RATE_2_5_GBPS: return  1;
  55         case IB_RATE_5_GBPS:   return  2;
  56         case IB_RATE_10_GBPS:  return  4;
  57         case IB_RATE_20_GBPS:  return  8;
  58         case IB_RATE_30_GBPS:  return 12;
  59         case IB_RATE_40_GBPS:  return 16;
  60         case IB_RATE_60_GBPS:  return 24;
  61         case IB_RATE_80_GBPS:  return 32;
  62         case IB_RATE_120_GBPS: return 48;
  63         default:               return -1;
  64         }
  65 }
  66 EXPORT_SYMBOL(ib_rate_to_mult);
  67
  68 __attribute_const__ enum ib_rate mult_to_ib_rate(int mult)
  69 {
  70         switch (mult) {
  71         case 1:  return IB_RATE_2_5_GBPS;
  72         case 2:  return IB_RATE_5_GBPS;
  73         case 4:  return IB_RATE_10_GBPS;
  74         case 8:  return IB_RATE_20_GBPS;
  75         case 12: return IB_RATE_30_GBPS;
  76         case 16: return IB_RATE_40_GBPS;
  77         case 24: return IB_RATE_60_GBPS;
  78         case 32: return IB_RATE_80_GBPS;
  79         case 48: return IB_RATE_120_GBPS;
  80         default: return IB_RATE_PORT_CURRENT;
  81         }
  82 }
  83 EXPORT_SYMBOL(mult_to_ib_rate);
  84
  85 __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)
  86 {
  87         switch (rate) {
  88         case IB_RATE_2_5_GBPS: return 2500;
  89         case IB_RATE_5_GBPS:   return 5000;
  90         case IB_RATE_10_GBPS:  return 10000;
  91         case IB_RATE_20_GBPS:  return 20000;
  92         case IB_RATE_30_GBPS:  return 30000;
  93         case IB_RATE_40_GBPS:  return 40000;
  94         case IB_RATE_60_GBPS:  return 60000;
  95         case IB_RATE_80_GBPS:  return 80000;
  96         case IB_RATE_120_GBPS: return 120000;
  97         case IB_RATE_14_GBPS:  return 14062;
  98         case IB_RATE_56_GBPS:  return 56250;
  99         case IB_RATE_112_GBPS: return 112500;
 100         case IB_RATE_168_GBPS: return 168750;
 101         case IB_RATE_25_GBPS:  return 25781;
 102         case IB_RATE_100_GBPS: return 103125;
 103         case IB_RATE_200_GBPS: return 206250;
 104         case IB_RATE_300_GBPS: return 309375;
 105         default:               return -1;
 106         }
 107 }
 108 EXPORT_SYMBOL(ib_rate_to_mbps);
 109
 110 __attribute_const__ enum rdma_transport_type
 111 rdma_node_get_transport(enum rdma_node_type node_type)
 112 {
 113         switch (node_type) {
 114         case RDMA_NODE_IB_CA:
 115         case RDMA_NODE_IB_SWITCH:
 116         case RDMA_NODE_IB_ROUTER:
 117                 return RDMA_TRANSPORT_IB;
 118         case RDMA_NODE_RNIC:
 119                 return RDMA_TRANSPORT_IWARP;
 120         case RDMA_NODE_USNIC:
 121                 return RDMA_TRANSPORT_USNIC;
 122         case RDMA_NODE_USNIC_UDP:
 123                 return RDMA_TRANSPORT_USNIC_UDP;
 124         default:
 125                 BUG();
 126                 return 0;
 127         }
 128 }
 129 EXPORT_SYMBOL(rdma_node_get_transport);
 130
 131 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
 132 {
 133         if (device->get_link_layer)
 134                 return device->get_link_layer(device, port_num);
 135
 136         switch (rdma_node_get_transport(device->node_type)) {
 137         case RDMA_TRANSPORT_IB:
 138                 return IB_LINK_LAYER_INFINIBAND;
 139         case RDMA_TRANSPORT_IWARP:
 140         case RDMA_TRANSPORT_USNIC:
 141         case RDMA_TRANSPORT_USNIC_UDP:
 142                 return IB_LINK_LAYER_ETHERNET;
 143         default:
 144                 return IB_LINK_LAYER_UNSPECIFIED;
 145         }
 146 }
 147 EXPORT_SYMBOL(rdma_port_get_link_layer);
 148
 149 /* Protection domains */
 150
 151 struct ib_pd *ib_alloc_pd(struct ib_device *device)
 152 {
 153         struct ib_pd *pd;
 154
 155         pd = device->alloc_pd(device, NULL, NULL);
 156
 157         if (!IS_ERR(pd)) {
 158                 pd->device  = device;
 159                 pd->uobject = NULL;
 160                 atomic_set(&pd->usecnt, 0);
 161         }
 162
 163         return pd;
 164 }
 165 EXPORT_SYMBOL(ib_alloc_pd);
 166
 167 int ib_dealloc_pd(struct ib_pd *pd)
 168 {
 169         if (atomic_read(&pd->usecnt))
 170                 return -EBUSY;
 171
 172         return pd->device->dealloc_pd(pd);
 173 }
 174 EXPORT_SYMBOL(ib_dealloc_pd);
 175
 176 /* Address handles */
 177
 178 struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
 179 {
 180         struct ib_ah *ah;
 181
 182         ah = pd->device->create_ah(pd, ah_attr);
 183
 184         if (!IS_ERR(ah)) {
 185                 ah->device  = pd->device;
 186                 ah->pd      = pd;
 187                 ah->uobject = NULL;
 188                 atomic_inc(&pd->usecnt);
 189         }
 190
 191         return ah;
 192 }
 193 EXPORT_SYMBOL(ib_create_ah);
 194
 195 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc,
 196                        struct ib_grh *grh, struct ib_ah_attr *ah_attr)
 197 {
 198         u32 flow_class;
 199         u16 gid_index;
 200         int ret;
 201         int is_eth = (rdma_port_get_link_layer(device, port_num) ==
 202                         IB_LINK_LAYER_ETHERNET);
 203
 204         memset(ah_attr, 0, sizeof *ah_attr);
 205         if (is_eth) {
 206                 if (!(wc->wc_flags & IB_WC_GRH))
 207                         return -EPROTOTYPE;
 208
 209                 if (wc->wc_flags & IB_WC_WITH_SMAC &&
 210                     wc->wc_flags & IB_WC_WITH_VLAN) {
 211                         memcpy(ah_attr->dmac, wc->smac, ETH_ALEN);
 212                         ah_attr->vlan_id = wc->vlan_id;
 213                 } else {
 214                         ret = rdma_addr_find_dmac_by_grh(&grh->dgid, &grh->sgid,
 215                                         ah_attr->dmac, &ah_attr->vlan_id);
 216                         if (ret)
 217                                 return ret;
 218                 }
 219         } else {
 220                 ah_attr->vlan_id = 0xffff;
 221         }
 222
 223         ah_attr->dlid = wc->slid;
 224         ah_attr->sl = wc->sl;
 225         ah_attr->src_path_bits = wc->dlid_path_bits;
 226         ah_attr->port_num = port_num;
 227
 228         if (wc->wc_flags & IB_WC_GRH) {
 229                 ah_attr->ah_flags = IB_AH_GRH;
 230                 ah_attr->grh.dgid = grh->sgid;
 231
 232                 ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
 233                                          &gid_index);
 234                 if (ret)
 235                         return ret;
 236
 237                 ah_attr->grh.sgid_index = (u8) gid_index;
 238                 flow_class = be32_to_cpu(grh->version_tclass_flow);
 239                 ah_attr->grh.flow_label = flow_class & 0xFFFFF;
 240                 ah_attr->grh.hop_limit = 0xFF;
 241                 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
 242         }
 243         return 0;
 244 }
 245 EXPORT_SYMBOL(ib_init_ah_from_wc);
 246
 247 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc,
 248                                    struct ib_grh *grh, u8 port_num)
 249 {
 250         struct ib_ah_attr ah_attr;
 251         int ret;
 252
 253         ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
 254         if (ret)
 255                 return ERR_PTR(ret);
 256
 257         return ib_create_ah(pd, &ah_attr);
 258 }
 259 EXPORT_SYMBOL(ib_create_ah_from_wc);
 260
 261 int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
 262 {
 263         return ah->device->modify_ah ?
 264                 ah->device->modify_ah(ah, ah_attr) :
 265                 -ENOSYS;
 266 }
 267 EXPORT_SYMBOL(ib_modify_ah);
 268
 269 int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
 270 {
 271         return ah->device->query_ah ?
 272                 ah->device->query_ah(ah, ah_attr) :
 273                 -ENOSYS;
 274 }
 275 EXPORT_SYMBOL(ib_query_ah);
 276
 277 int ib_destroy_ah(struct ib_ah *ah)
 278 {
 279         struct ib_pd *pd;
 280         int ret;
 281
 282         pd = ah->pd;
 283         ret = ah->device->destroy_ah(ah);
 284         if (!ret)
 285                 atomic_dec(&pd->usecnt);
 286
 287         return ret;
 288 }
 289 EXPORT_SYMBOL(ib_destroy_ah);
 290
 291 /* Shared receive queues */
 292
 293 struct ib_srq *ib_create_srq(struct ib_pd *pd,
 294                              struct ib_srq_init_attr *srq_init_attr)
 295 {
 296         struct ib_srq *srq;
 297
 298         if (!pd->device->create_srq)
 299                 return ERR_PTR(-ENOSYS);
 300
 301         srq = pd->device->create_srq(pd, srq_init_attr, NULL);
 302
 303         if (!IS_ERR(srq)) {
 304                 srq->device        = pd->device;
 305                 srq->pd            = pd;
 306                 srq->uobject       = NULL;
 307                 srq->event_handler = srq_init_attr->event_handler;
 308                 srq->srq_context   = srq_init_attr->srq_context;
 309                 srq->srq_type      = srq_init_attr->srq_type;
 310                 if (srq->srq_type == IB_SRQT_XRC) {
 311                         srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
 312                         srq->ext.xrc.cq   = srq_init_attr->ext.xrc.cq;
 313                         atomic_inc(&srq->ext.xrc.xrcd->usecnt);
 314                         atomic_inc(&srq->ext.xrc.cq->usecnt);
 315                 }
 316                 atomic_inc(&pd->usecnt);
 317                 atomic_set(&srq->usecnt, 0);
 318         }
 319
 320         return srq;
 321 }
 322 EXPORT_SYMBOL(ib_create_srq);
 323
 324 int ib_modify_srq(struct ib_srq *srq,
 325                   struct ib_srq_attr *srq_attr,
 326                   enum ib_srq_attr_mask srq_attr_mask)
 327 {
 328         return srq->device->modify_srq ?
 329                 srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) :
 330                 -ENOSYS;
 331 }
 332 EXPORT_SYMBOL(ib_modify_srq);
 333
 334 int ib_query_srq(struct ib_srq *srq,
 335                  struct ib_srq_attr *srq_attr)
 336 {
 337         return srq->device->query_srq ?
 338                 srq->device->query_srq(srq, srq_attr) : -ENOSYS;
 339 }
 340 EXPORT_SYMBOL(ib_query_srq);
 341
 342 int ib_destroy_srq(struct ib_srq *srq)
 343 {
 344         struct ib_pd *pd;
 345         enum ib_srq_type srq_type;
 346         struct ib_xrcd *uninitialized_var(xrcd);
 347         struct ib_cq *uninitialized_var(cq);
 348         int ret;
 349
 350         if (atomic_read(&srq->usecnt))
 351                 return -EBUSY;
 352
 353         pd = srq->pd;
 354         srq_type = srq->srq_type;
 355         if (srq_type == IB_SRQT_XRC) {
 356                 xrcd = srq->ext.xrc.xrcd;
 357                 cq = srq->ext.xrc.cq;
 358         }
 359
 360         ret = srq->device->destroy_srq(srq);
 361         if (!ret) {
 362                 atomic_dec(&pd->usecnt);
 363                 if (srq_type == IB_SRQT_XRC) {
 364                         atomic_dec(&xrcd->usecnt);
 365                         atomic_dec(&cq->usecnt);
 366                 }
 367         }
 368
 369         return ret;
 370 }
 371 EXPORT_SYMBOL(ib_destroy_srq);
 372
 373 /* Queue pairs */
 374
 375 static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
 376 {
 377         struct ib_qp *qp = context;
 378         unsigned long flags;
 379
 380         spin_lock_irqsave(&qp->device->event_handler_lock, flags);
 381         list_for_each_entry(event->element.qp, &qp->open_list, open_list)
 382                 if (event->element.qp->event_handler)
 383                         event->element.qp->event_handler(event, event->element.qp->qp_context);
 384         spin_unlock_irqrestore(&qp->device->event_handler_lock, flags);
 385 }
 386
 387 static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
 388 {
 389         mutex_lock(&xrcd->tgt_qp_mutex);
 390         list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
 391         mutex_unlock(&xrcd->tgt_qp_mutex);
 392 }
 393
 394 static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
 395                                   void (*event_handler)(struct ib_event *, void *),
 396                                   void *qp_context)
 397 {
 398         struct ib_qp *qp;
 399         unsigned long flags;
 400
 401         qp = kzalloc(sizeof *qp, GFP_KERNEL);
 402         if (!qp)
 403                 return ERR_PTR(-ENOMEM);
 404
 405         qp->real_qp = real_qp;
 406         atomic_inc(&real_qp->usecnt);
 407         qp->device = real_qp->device;
 408         qp->event_handler = event_handler;
 409         qp->qp_context = qp_context;
 410         qp->qp_num = real_qp->qp_num;
 411         qp->qp_type = real_qp->qp_type;
 412
 413         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
 414         list_add(&qp->open_list, &real_qp->open_list);
 415         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
 416
 417         return qp;
 418 }
 419
 420 struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
 421                          struct ib_qp_open_attr *qp_open_attr)
 422 {
 423         struct ib_qp *qp, *real_qp;
 424
 425         if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
 426                 return ERR_PTR(-EINVAL);
 427
 428         qp = ERR_PTR(-EINVAL);
 429         mutex_lock(&xrcd->tgt_qp_mutex);
 430         list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
 431                 if (real_qp->qp_num == qp_open_attr->qp_num) {
 432                         qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
 433                                           qp_open_attr->qp_context);
 434                         break;
 435                 }
 436         }
 437         mutex_unlock(&xrcd->tgt_qp_mutex);
 438         return qp;
 439 }
 440 EXPORT_SYMBOL(ib_open_qp);
 441
 442 struct ib_qp *ib_create_qp(struct ib_pd *pd,
 443                            struct ib_qp_init_attr *qp_init_attr)
 444 {
 445         struct ib_qp *qp, *real_qp;
 446         struct ib_device *device;
 447
 448         device = pd ? pd->device : qp_init_attr->xrcd->device;
 449         qp = device->create_qp(pd, qp_init_attr, NULL);
 450
 451         if (!IS_ERR(qp)) {
 452                 qp->device     = device;
 453                 qp->real_qp    = qp;
 454                 qp->uobject    = NULL;
 455                 qp->qp_type    = qp_init_attr->qp_type;
 456
 457                 atomic_set(&qp->usecnt, 0);
 458                 if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
 459                         qp->event_handler = __ib_shared_qp_event_handler;
 460                         qp->qp_context = qp;
 461                         qp->pd = NULL;
 462                         qp->send_cq = qp->recv_cq = NULL;
 463                         qp->srq = NULL;
 464                         qp->xrcd = qp_init_attr->xrcd;
 465                         atomic_inc(&qp_init_attr->xrcd->usecnt);
 466                         INIT_LIST_HEAD(&qp->open_list);
 467
 468                         real_qp = qp;
 469                         qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
 470                                           qp_init_attr->qp_context);
 471                         if (!IS_ERR(qp))
 472                                 __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
 473                         else
 474                                 real_qp->device->destroy_qp(real_qp);
 475                 } else {
 476                         qp->event_handler = qp_init_attr->event_handler;
 477                         qp->qp_context = qp_init_attr->qp_context;
 478                         if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
 479                                 qp->recv_cq = NULL;
 480                                 qp->srq = NULL;
 481                         } else {
 482                                 qp->recv_cq = qp_init_attr->recv_cq;
 483                                 atomic_inc(&qp_init_attr->recv_cq->usecnt);
 484                                 qp->srq = qp_init_attr->srq;
 485                                 if (qp->srq)
 486                                         atomic_inc(&qp_init_attr->srq->usecnt);
 487                         }
 488
 489                         qp->pd      = pd;
 490                         qp->send_cq = qp_init_attr->send_cq;
 491                         qp->xrcd    = NULL;
 492
 493                         atomic_inc(&pd->usecnt);
 494                         atomic_inc(&qp_init_attr->send_cq->usecnt);
 495                 }
 496         }
 497
 498         return qp;
 499 }
 500 EXPORT_SYMBOL(ib_create_qp);
 501
 502 static const struct {
 503         int                     valid;
 504         enum ib_qp_attr_mask    req_param[IB_QPT_MAX];
 505         enum ib_qp_attr_mask    req_param_add_eth[IB_QPT_MAX];
 506         enum ib_qp_attr_mask    opt_param[IB_QPT_MAX];
 507         enum ib_qp_attr_mask    opt_param_add_eth[IB_QPT_MAX];
 508 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
 509         [IB_QPS_RESET] = {
 510                 [IB_QPS_RESET] = { .valid = 1 },
 511                 [IB_QPS_INIT]  = {
 512                         .valid = 1,
 513                         .req_param = {
 514                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 515                                                 IB_QP_PORT                      |
 516                                                 IB_QP_QKEY),
 517                                 [IB_QPT_RAW_PACKET] = IB_QP_PORT,
 518                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
 519                                                 IB_QP_PORT                      |
 520                                                 IB_QP_ACCESS_FLAGS),
 521                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
 522                                                 IB_QP_PORT                      |
 523                                                 IB_QP_ACCESS_FLAGS),
 524                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
 525                                                 IB_QP_PORT                      |
 526                                                 IB_QP_ACCESS_FLAGS),
 527                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
 528                                                 IB_QP_PORT                      |
 529                                                 IB_QP_ACCESS_FLAGS),
 530                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 531                                                 IB_QP_QKEY),
 532                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 533                                                 IB_QP_QKEY),
 534                         }
 535                 },
 536         },
 537         [IB_QPS_INIT]  = {
 538                 [IB_QPS_RESET] = { .valid = 1 },
 539                 [IB_QPS_ERR] =   { .valid = 1 },
 540                 [IB_QPS_INIT]  = {
 541                         .valid = 1,
 542                         .opt_param = {
 543                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 544                                                 IB_QP_PORT                      |
 545                                                 IB_QP_QKEY),
 546                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
 547                                                 IB_QP_PORT                      |
 548                                                 IB_QP_ACCESS_FLAGS),
 549                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
 550                                                 IB_QP_PORT                      |
 551                                                 IB_QP_ACCESS_FLAGS),
 552                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
 553                                                 IB_QP_PORT                      |
 554                                                 IB_QP_ACCESS_FLAGS),
 555                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
 556                                                 IB_QP_PORT                      |
 557                                                 IB_QP_ACCESS_FLAGS),
 558                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 559                                                 IB_QP_QKEY),
 560                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 561                                                 IB_QP_QKEY),
 562                         }
 563                 },
 564                 [IB_QPS_RTR]   = {
 565                         .valid = 1,
 566                         .req_param = {
 567                                 [IB_QPT_UC]  = (IB_QP_AV                        |
 568                                                 IB_QP_PATH_MTU                  |
 569                                                 IB_QP_DEST_QPN                  |
 570                                                 IB_QP_RQ_PSN),
 571                                 [IB_QPT_RC]  = (IB_QP_AV                        |
 572                                                 IB_QP_PATH_MTU                  |
 573                                                 IB_QP_DEST_QPN                  |
 574                                                 IB_QP_RQ_PSN                    |
 575                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 576                                                 IB_QP_MIN_RNR_TIMER),
 577                                 [IB_QPT_XRC_INI] = (IB_QP_AV                    |
 578                                                 IB_QP_PATH_MTU                  |
 579                                                 IB_QP_DEST_QPN                  |
 580                                                 IB_QP_RQ_PSN),
 581                                 [IB_QPT_XRC_TGT] = (IB_QP_AV                    |
 582                                                 IB_QP_PATH_MTU                  |
 583                                                 IB_QP_DEST_QPN                  |
 584                                                 IB_QP_RQ_PSN                    |
 585                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 586                                                 IB_QP_MIN_RNR_TIMER),
 587                         },
 588                         .req_param_add_eth = {
 589                                 [IB_QPT_RC]  = (IB_QP_SMAC),
 590                                 [IB_QPT_UC]  = (IB_QP_SMAC),
 591                                 [IB_QPT_XRC_INI]  = (IB_QP_SMAC),
 592                                 [IB_QPT_XRC_TGT]  = (IB_QP_SMAC)
 593                         },
 594                         .opt_param = {
 595                                  [IB_QPT_UD]  = (IB_QP_PKEY_INDEX               |
 596                                                  IB_QP_QKEY),
 597                                  [IB_QPT_UC]  = (IB_QP_ALT_PATH                 |
 598                                                  IB_QP_ACCESS_FLAGS             |
 599                                                  IB_QP_PKEY_INDEX),
 600                                  [IB_QPT_RC]  = (IB_QP_ALT_PATH                 |
 601                                                  IB_QP_ACCESS_FLAGS             |
 602                                                  IB_QP_PKEY_INDEX),
 603                                  [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH             |
 604                                                  IB_QP_ACCESS_FLAGS             |
 605                                                  IB_QP_PKEY_INDEX),
 606                                  [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH             |
 607                                                  IB_QP_ACCESS_FLAGS             |
 608                                                  IB_QP_PKEY_INDEX),
 609                                  [IB_QPT_SMI] = (IB_QP_PKEY_INDEX               |
 610                                                  IB_QP_QKEY),
 611                                  [IB_QPT_GSI] = (IB_QP_PKEY_INDEX               |
 612                                                  IB_QP_QKEY),
 613                          },
 614                         .opt_param_add_eth = {
 615                                 [IB_QPT_RC]  = (IB_QP_ALT_SMAC                  |
 616                                                 IB_QP_VID                       |
 617                                                 IB_QP_ALT_VID),
 618                                 [IB_QPT_UC]  = (IB_QP_ALT_SMAC                  |
 619                                                 IB_QP_VID                       |
 620                                                 IB_QP_ALT_VID),
 621                                 [IB_QPT_XRC_INI]  = (IB_QP_ALT_SMAC                     |
 622                                                 IB_QP_VID                       |
 623                                                 IB_QP_ALT_VID),
 624                                 [IB_QPT_XRC_TGT]  = (IB_QP_ALT_SMAC                     |
 625                                                 IB_QP_VID                       |
 626                                                 IB_QP_ALT_VID)
 627                         }
 628                 }
 629         },
 630         [IB_QPS_RTR]   = {
 631                 [IB_QPS_RESET] = { .valid = 1 },
 632                 [IB_QPS_ERR] =   { .valid = 1 },
 633                 [IB_QPS_RTS]   = {
 634                         .valid = 1,
 635                         .req_param = {
 636                                 [IB_QPT_UD]  = IB_QP_SQ_PSN,
 637                                 [IB_QPT_UC]  = IB_QP_SQ_PSN,
 638                                 [IB_QPT_RC]  = (IB_QP_TIMEOUT                   |
 639                                                 IB_QP_RETRY_CNT                 |
 640                                                 IB_QP_RNR_RETRY                 |
 641                                                 IB_QP_SQ_PSN                    |
 642                                                 IB_QP_MAX_QP_RD_ATOMIC),
 643                                 [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT               |
 644                                                 IB_QP_RETRY_CNT                 |
 645                                                 IB_QP_RNR_RETRY                 |
 646                                                 IB_QP_SQ_PSN                    |
 647                                                 IB_QP_MAX_QP_RD_ATOMIC),
 648                                 [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT               |
 649                                                 IB_QP_SQ_PSN),
 650                                 [IB_QPT_SMI] = IB_QP_SQ_PSN,
 651                                 [IB_QPT_GSI] = IB_QP_SQ_PSN,
 652                         },
 653                         .opt_param = {
 654                                  [IB_QPT_UD]  = (IB_QP_CUR_STATE                |
 655                                                  IB_QP_QKEY),
 656                                  [IB_QPT_UC]  = (IB_QP_CUR_STATE                |
 657                                                  IB_QP_ALT_PATH                 |
 658                                                  IB_QP_ACCESS_FLAGS             |
 659                                                  IB_QP_PATH_MIG_STATE),
 660                                  [IB_QPT_RC]  = (IB_QP_CUR_STATE                |
 661                                                  IB_QP_ALT_PATH                 |
 662                                                  IB_QP_ACCESS_FLAGS             |
 663                                                  IB_QP_MIN_RNR_TIMER            |
 664                                                  IB_QP_PATH_MIG_STATE),
 665                                  [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE            |
 666                                                  IB_QP_ALT_PATH                 |
 667                                                  IB_QP_ACCESS_FLAGS             |
 668                                                  IB_QP_PATH_MIG_STATE),
 669                                  [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE            |
 670                                                  IB_QP_ALT_PATH                 |
 671                                                  IB_QP_ACCESS_FLAGS             |
 672                                                  IB_QP_MIN_RNR_TIMER            |
 673                                                  IB_QP_PATH_MIG_STATE),
 674                                  [IB_QPT_SMI] = (IB_QP_CUR_STATE                |
 675                                                  IB_QP_QKEY),
 676                                  [IB_QPT_GSI] = (IB_QP_CUR_STATE                |
 677                                                  IB_QP_QKEY),
 678                          }
 679                 }
 680         },
 681         [IB_QPS_RTS]   = {
 682                 [IB_QPS_RESET] = { .valid = 1 },
 683                 [IB_QPS_ERR] =   { .valid = 1 },
 684                 [IB_QPS_RTS]   = {
 685                         .valid = 1,
 686                         .opt_param = {
 687                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 688                                                 IB_QP_QKEY),
 689                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 690                                                 IB_QP_ACCESS_FLAGS              |
 691                                                 IB_QP_ALT_PATH                  |
 692                                                 IB_QP_PATH_MIG_STATE),
 693                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
 694                                                 IB_QP_ACCESS_FLAGS              |
 695                                                 IB_QP_ALT_PATH                  |
 696                                                 IB_QP_PATH_MIG_STATE            |
 697                                                 IB_QP_MIN_RNR_TIMER),
 698                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
 699                                                 IB_QP_ACCESS_FLAGS              |
 700                                                 IB_QP_ALT_PATH                  |
 701                                                 IB_QP_PATH_MIG_STATE),
 702                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
 703                                                 IB_QP_ACCESS_FLAGS              |
 704                                                 IB_QP_ALT_PATH                  |
 705                                                 IB_QP_PATH_MIG_STATE            |
 706                                                 IB_QP_MIN_RNR_TIMER),
 707                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 708                                                 IB_QP_QKEY),
 709                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 710                                                 IB_QP_QKEY),
 711                         }
 712                 },
 713                 [IB_QPS_SQD]   = {
 714                         .valid = 1,
 715                         .opt_param = {
 716                                 [IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 717                                 [IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 718                                 [IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 719                                 [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
 720                                 [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
 721                                 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
 722                                 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
 723                         }
 724                 },
 725         },
 726         [IB_QPS_SQD]   = {
 727                 [IB_QPS_RESET] = { .valid = 1 },
 728                 [IB_QPS_ERR] =   { .valid = 1 },
 729                 [IB_QPS_RTS]   = {
 730                         .valid = 1,
 731                         .opt_param = {
 732                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 733                                                 IB_QP_QKEY),
 734                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 735                                                 IB_QP_ALT_PATH                  |
 736                                                 IB_QP_ACCESS_FLAGS              |
 737                                                 IB_QP_PATH_MIG_STATE),
 738                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
 739                                                 IB_QP_ALT_PATH                  |
 740                                                 IB_QP_ACCESS_FLAGS              |
 741                                                 IB_QP_MIN_RNR_TIMER             |
 742                                                 IB_QP_PATH_MIG_STATE),
 743                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
 744                                                 IB_QP_ALT_PATH                  |
 745                                                 IB_QP_ACCESS_FLAGS              |
 746                                                 IB_QP_PATH_MIG_STATE),
 747                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
 748                                                 IB_QP_ALT_PATH                  |
 749                                                 IB_QP_ACCESS_FLAGS              |
 750                                                 IB_QP_MIN_RNR_TIMER             |
 751                                                 IB_QP_PATH_MIG_STATE),
 752                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 753                                                 IB_QP_QKEY),
 754                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 755                                                 IB_QP_QKEY),
 756                         }
 757                 },
 758                 [IB_QPS_SQD]   = {
 759                         .valid = 1,
 760                         .opt_param = {
 761                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 762                                                 IB_QP_QKEY),
 763                                 [IB_QPT_UC]  = (IB_QP_AV                        |
 764                                                 IB_QP_ALT_PATH                  |
 765                                                 IB_QP_ACCESS_FLAGS              |
 766                                                 IB_QP_PKEY_INDEX                |
 767                                                 IB_QP_PATH_MIG_STATE),
 768                                 [IB_QPT_RC]  = (IB_QP_PORT                      |
 769                                                 IB_QP_AV                        |
 770                                                 IB_QP_TIMEOUT                   |
 771                                                 IB_QP_RETRY_CNT                 |
 772                                                 IB_QP_RNR_RETRY                 |
 773                                                 IB_QP_MAX_QP_RD_ATOMIC          |
 774                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 775                                                 IB_QP_ALT_PATH                  |
 776                                                 IB_QP_ACCESS_FLAGS              |
 777                                                 IB_QP_PKEY_INDEX                |
 778                                                 IB_QP_MIN_RNR_TIMER             |
 779                                                 IB_QP_PATH_MIG_STATE),
 780                                 [IB_QPT_XRC_INI] = (IB_QP_PORT                  |
 781                                                 IB_QP_AV                        |
 782                                                 IB_QP_TIMEOUT                   |
 783                                                 IB_QP_RETRY_CNT                 |
 784                                                 IB_QP_RNR_RETRY                 |
 785                                                 IB_QP_MAX_QP_RD_ATOMIC          |
 786                                                 IB_QP_ALT_PATH                  |
 787                                                 IB_QP_ACCESS_FLAGS              |
 788                                                 IB_QP_PKEY_INDEX                |
 789                                                 IB_QP_PATH_MIG_STATE),
 790                                 [IB_QPT_XRC_TGT] = (IB_QP_PORT                  |
 791                                                 IB_QP_AV                        |
 792                                                 IB_QP_TIMEOUT                   |
 793                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 794                                                 IB_QP_ALT_PATH                  |
 795                                                 IB_QP_ACCESS_FLAGS              |
 796                                                 IB_QP_PKEY_INDEX                |
 797                                                 IB_QP_MIN_RNR_TIMER             |
 798                                                 IB_QP_PATH_MIG_STATE),
 799                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 800                                                 IB_QP_QKEY),
 801                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 802                                                 IB_QP_QKEY),
 803                         }
 804                 }
 805         },
 806         [IB_QPS_SQE]   = {
 807                 [IB_QPS_RESET] = { .valid = 1 },
 808                 [IB_QPS_ERR] =   { .valid = 1 },
 809                 [IB_QPS_RTS]   = {
 810                         .valid = 1,
 811                         .opt_param = {
 812                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 813                                                 IB_QP_QKEY),
 814                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 815                                                 IB_QP_ACCESS_FLAGS),
 816                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 817                                                 IB_QP_QKEY),
 818                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 819                                                 IB_QP_QKEY),
 820                         }
 821                 }
 822         },
 823         [IB_QPS_ERR] = {
 824                 [IB_QPS_RESET] = { .valid = 1 },
 825                 [IB_QPS_ERR] =   { .valid = 1 }
 826         }
 827 };
 828
 829 int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
 830                        enum ib_qp_type type, enum ib_qp_attr_mask mask,
 831                        enum rdma_link_layer ll)
 832 {
 833         enum ib_qp_attr_mask req_param, opt_param;
 834
 835         if (cur_state  < 0 || cur_state  > IB_QPS_ERR ||
 836             next_state < 0 || next_state > IB_QPS_ERR)
 837                 return 0;
 838
 839         if (mask & IB_QP_CUR_STATE  &&
 840             cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
 841             cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
 842                 return 0;
 843
 844         if (!qp_state_table[cur_state][next_state].valid)
 845                 return 0;
 846
 847         req_param = qp_state_table[cur_state][next_state].req_param[type];
 848         opt_param = qp_state_table[cur_state][next_state].opt_param[type];
 849
 850         if (ll == IB_LINK_LAYER_ETHERNET) {
 851                 req_param |= qp_state_table[cur_state][next_state].
 852                         req_param_add_eth[type];
 853                 opt_param |= qp_state_table[cur_state][next_state].
 854                         opt_param_add_eth[type];
 855         }
 856
 857         if ((mask & req_param) != req_param)
 858                 return 0;
 859
 860         if (mask & ~(req_param | opt_param | IB_QP_STATE))
 861                 return 0;
 862
 863         return 1;
 864 }
 865 EXPORT_SYMBOL(ib_modify_qp_is_ok);
 866
 867 int ib_resolve_eth_l2_attrs(struct ib_qp *qp,
 868                             struct ib_qp_attr *qp_attr, int *qp_attr_mask)
 869 {
 870         int           ret = 0;
 871         union ib_gid  sgid;
 872
 873         if ((*qp_attr_mask & IB_QP_AV)  &&
 874             (rdma_port_get_link_layer(qp->device, qp_attr->ah_attr.port_num) == IB_LINK_LAYER_ETHERNET)) {
 875                 ret = ib_query_gid(qp->device, qp_attr->ah_attr.port_num,
 876                                    qp_attr->ah_attr.grh.sgid_index, &sgid);
 877                 if (ret)
 878                         goto out;
 879                 if (rdma_link_local_addr((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw)) {
 880                         rdma_get_ll_mac((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw, qp_attr->ah_attr.dmac);
 881                         rdma_get_ll_mac((struct in6_addr *)sgid.raw, qp_attr->smac);
 882                         qp_attr->vlan_id = rdma_get_vlan_id(&sgid);
 883                 } else {
 884                         ret = rdma_addr_find_dmac_by_grh(&sgid, &qp_attr->ah_attr.grh.dgid,
 885                                         qp_attr->ah_attr.dmac, &qp_attr->vlan_id);
 886                         if (ret)
 887                                 goto out;
 888                         ret = rdma_addr_find_smac_by_sgid(&sgid, qp_attr->smac, NULL);
 889                         if (ret)
 890                                 goto out;
 891                 }
 892                 *qp_attr_mask |= IB_QP_SMAC;
 893                 if (qp_attr->vlan_id < 0xFFFF)
 894                         *qp_attr_mask |= IB_QP_VID;
 895         }
 896 out:
 897         return ret;
 898 }
 899 EXPORT_SYMBOL(ib_resolve_eth_l2_attrs);
 900
 901
 902 int ib_modify_qp(struct ib_qp *qp,
 903                  struct ib_qp_attr *qp_attr,
 904                  int qp_attr_mask)
 905 {
 906         int ret;
 907
 908         ret = ib_resolve_eth_l2_attrs(qp, qp_attr, &qp_attr_mask);
 909         if (ret)
 910                 return ret;
 911
 912         return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
 913 }
 914 EXPORT_SYMBOL(ib_modify_qp);
 915
 916 int ib_query_qp(struct ib_qp *qp,
 917                 struct ib_qp_attr *qp_attr,
 918                 int qp_attr_mask,
 919                 struct ib_qp_init_attr *qp_init_attr)
 920 {
 921         return qp->device->query_qp ?
 922                 qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) :
 923                 -ENOSYS;
 924 }
 925 EXPORT_SYMBOL(ib_query_qp);
 926
 927 int ib_close_qp(struct ib_qp *qp)
 928 {
 929         struct ib_qp *real_qp;
 930         unsigned long flags;
 931
 932         real_qp = qp->real_qp;
 933         if (real_qp == qp)
 934                 return -EINVAL;
 935
 936         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
 937         list_del(&qp->open_list);
 938         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
 939
 940         atomic_dec(&real_qp->usecnt);
 941         kfree(qp);
 942
 943         return 0;
 944 }
 945 EXPORT_SYMBOL(ib_close_qp);
 946
 947 static int __ib_destroy_shared_qp(struct ib_qp *qp)
 948 {
 949         struct ib_xrcd *xrcd;
 950         struct ib_qp *real_qp;
 951         int ret;
 952
 953         real_qp = qp->real_qp;
 954         xrcd = real_qp->xrcd;
 955
 956         mutex_lock(&xrcd->tgt_qp_mutex);
 957         ib_close_qp(qp);
 958         if (atomic_read(&real_qp->usecnt) == 0)
 959                 list_del(&real_qp->xrcd_list);
 960         else
 961                 real_qp = NULL;
 962         mutex_unlock(&xrcd->tgt_qp_mutex);
 963
 964         if (real_qp) {
 965                 ret = ib_destroy_qp(real_qp);
 966                 if (!ret)
 967                         atomic_dec(&xrcd->usecnt);
 968                 else
 969                         __ib_insert_xrcd_qp(xrcd, real_qp);
 970         }
 971
 972         return 0;
 973 }
 974
 975 int ib_destroy_qp(struct ib_qp *qp)
 976 {
 977         struct ib_pd *pd;
 978         struct ib_cq *scq, *rcq;
 979         struct ib_srq *srq;
 980         int ret;
 981
 982         if (atomic_read(&qp->usecnt))
 983                 return -EBUSY;
 984
 985         if (qp->real_qp != qp)
 986                 return __ib_destroy_shared_qp(qp);
 987
 988         pd   = qp->pd;
 989         scq  = qp->send_cq;
 990         rcq  = qp->recv_cq;
 991         srq  = qp->srq;
 992
 993         ret = qp->device->destroy_qp(qp);
 994         if (!ret) {
 995                 if (pd)
 996                         atomic_dec(&pd->usecnt);
 997                 if (scq)
 998                         atomic_dec(&scq->usecnt);
 999                 if (rcq)
1000                         atomic_dec(&rcq->usecnt);
1001                 if (srq)
1002                         atomic_dec(&srq->usecnt);
1003         }
1004
1005         return ret;
1006 }
1007 EXPORT_SYMBOL(ib_destroy_qp);
1008
1009 /* Completion queues */
1010
1011 struct ib_cq *ib_create_cq(struct ib_device *device,
1012                            ib_comp_handler comp_handler,
1013                            void (*event_handler)(struct ib_event *, void *),
1014                            void *cq_context, int cqe, int comp_vector)
1015 {
1016         struct ib_cq *cq;
1017
1018         cq = device->create_cq(device, cqe, comp_vector, NULL, NULL);
1019
1020         if (!IS_ERR(cq)) {
1021                 cq->device        = device;
1022                 cq->uobject       = NULL;
1023                 cq->comp_handler  = comp_handler;
1024                 cq->event_handler = event_handler;
1025                 cq->cq_context    = cq_context;
1026                 atomic_set(&cq->usecnt, 0);
1027         }
1028
1029         return cq;
1030 }
1031 EXPORT_SYMBOL(ib_create_cq);
1032
1033 int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
1034 {
1035         return cq->device->modify_cq ?
1036                 cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS;
1037 }
1038 EXPORT_SYMBOL(ib_modify_cq);
1039
1040 int ib_destroy_cq(struct ib_cq *cq)
1041 {
1042         if (atomic_read(&cq->usecnt))
1043                 return -EBUSY;
1044
1045         return cq->device->destroy_cq(cq);
1046 }
1047 EXPORT_SYMBOL(ib_destroy_cq);
1048
1049 int ib_resize_cq(struct ib_cq *cq, int cqe)
1050 {
1051         return cq->device->resize_cq ?
1052                 cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS;
1053 }
1054 EXPORT_SYMBOL(ib_resize_cq);
1055
1056 /* Memory regions */
1057
1058 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
1059 {
1060         struct ib_mr *mr;
1061         int err;
1062
1063         err = ib_check_mr_access(mr_access_flags);
1064         if (err)
1065                 return ERR_PTR(err);
1066
1067         mr = pd->device->get_dma_mr(pd, mr_access_flags);
1068
1069         if (!IS_ERR(mr)) {
1070                 mr->device  = pd->device;
1071                 mr->pd      = pd;
1072                 mr->uobject = NULL;
1073                 atomic_inc(&pd->usecnt);
1074                 atomic_set(&mr->usecnt, 0);
1075         }
1076
1077         return mr;
1078 }
1079 EXPORT_SYMBOL(ib_get_dma_mr);
1080
1081 struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
1082                              struct ib_phys_buf *phys_buf_array,
1083                              int num_phys_buf,
1084                              int mr_access_flags,
1085                              u64 *iova_start)
1086 {
1087         struct ib_mr *mr;
1088         int err;
1089
1090         err = ib_check_mr_access(mr_access_flags);
1091         if (err)
1092                 return ERR_PTR(err);
1093
1094         if (!pd->device->reg_phys_mr)
1095                 return ERR_PTR(-ENOSYS);
1096
1097         mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
1098                                      mr_access_flags, iova_start);
1099
1100         if (!IS_ERR(mr)) {
1101                 mr->device  = pd->device;
1102                 mr->pd      = pd;
1103                 mr->uobject = NULL;
1104                 atomic_inc(&pd->usecnt);
1105                 atomic_set(&mr->usecnt, 0);
1106         }
1107
1108         return mr;
1109 }
1110 EXPORT_SYMBOL(ib_reg_phys_mr);
1111
1112 int ib_rereg_phys_mr(struct ib_mr *mr,
1113                      int mr_rereg_mask,
1114                      struct ib_pd *pd,
1115                      struct ib_phys_buf *phys_buf_array,
1116                      int num_phys_buf,
1117                      int mr_access_flags,
1118                      u64 *iova_start)
1119 {
1120         struct ib_pd *old_pd;
1121         int ret;
1122
1123         ret = ib_check_mr_access(mr_access_flags);
1124         if (ret)
1125                 return ret;
1126
1127         if (!mr->device->rereg_phys_mr)
1128                 return -ENOSYS;
1129
1130         if (atomic_read(&mr->usecnt))
1131                 return -EBUSY;
1132
1133         old_pd = mr->pd;
1134
1135         ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
1136                                         phys_buf_array, num_phys_buf,
1137                                         mr_access_flags, iova_start);
1138
1139         if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
1140                 atomic_dec(&old_pd->usecnt);
1141                 atomic_inc(&pd->usecnt);
1142         }
1143
1144         return ret;
1145 }
1146 EXPORT_SYMBOL(ib_rereg_phys_mr);
1147
1148 int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
1149 {
1150         return mr->device->query_mr ?
1151                 mr->device->query_mr(mr, mr_attr) : -ENOSYS;
1152 }
1153 EXPORT_SYMBOL(ib_query_mr);
1154
1155 int ib_dereg_mr(struct ib_mr *mr)
1156 {
1157         struct ib_pd *pd;
1158         int ret;
1159
1160         if (atomic_read(&mr->usecnt))
1161                 return -EBUSY;
1162
1163         pd = mr->pd;
1164         ret = mr->device->dereg_mr(mr);
1165         if (!ret)
1166                 atomic_dec(&pd->usecnt);
1167
1168         return ret;
1169 }
1170 EXPORT_SYMBOL(ib_dereg_mr);
1171
1172 struct ib_mr *ib_create_mr(struct ib_pd *pd,
1173                            struct ib_mr_init_attr *mr_init_attr)
1174 {
1175         struct ib_mr *mr;
1176
1177         if (!pd->device->create_mr)
1178                 return ERR_PTR(-ENOSYS);
1179
1180         mr = pd->device->create_mr(pd, mr_init_attr);
1181
1182         if (!IS_ERR(mr)) {
1183                 mr->device  = pd->device;
1184                 mr->pd      = pd;
1185                 mr->uobject = NULL;
1186                 atomic_inc(&pd->usecnt);
1187                 atomic_set(&mr->usecnt, 0);
1188         }
1189
1190         return mr;
1191 }
1192 EXPORT_SYMBOL(ib_create_mr);
1193
1194 int ib_destroy_mr(struct ib_mr *mr)
1195 {
1196         struct ib_pd *pd;
1197         int ret;
1198
1199         if (atomic_read(&mr->usecnt))
1200                 return -EBUSY;
1201
1202         pd = mr->pd;
1203         ret = mr->device->destroy_mr(mr);
1204         if (!ret)
1205                 atomic_dec(&pd->usecnt);
1206
1207         return ret;
1208 }
1209 EXPORT_SYMBOL(ib_destroy_mr);
1210
1211 struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
1212 {
1213         struct ib_mr *mr;
1214
1215         if (!pd->device->alloc_fast_reg_mr)
1216                 return ERR_PTR(-ENOSYS);
1217
1218         mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len);
1219
1220         if (!IS_ERR(mr)) {
1221                 mr->device  = pd->device;
1222                 mr->pd      = pd;
1223                 mr->uobject = NULL;
1224                 atomic_inc(&pd->usecnt);
1225                 atomic_set(&mr->usecnt, 0);
1226         }
1227
1228         return mr;
1229 }
1230 EXPORT_SYMBOL(ib_alloc_fast_reg_mr);
1231
1232 struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device,
1233                                                           int max_page_list_len)
1234 {
1235         struct ib_fast_reg_page_list *page_list;
1236
1237         if (!device->alloc_fast_reg_page_list)
1238                 return ERR_PTR(-ENOSYS);
1239
1240         page_list = device->alloc_fast_reg_page_list(device, max_page_list_len);
1241
1242         if (!IS_ERR(page_list)) {
1243                 page_list->device = device;
1244                 page_list->max_page_list_len = max_page_list_len;
1245         }
1246
1247         return page_list;
1248 }
1249 EXPORT_SYMBOL(ib_alloc_fast_reg_page_list);
1250
1251 void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
1252 {
1253         page_list->device->free_fast_reg_page_list(page_list);
1254 }
1255 EXPORT_SYMBOL(ib_free_fast_reg_page_list);
1256
1257 /* Memory windows */
1258
1259 struct ib_mw *ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
1260 {
1261         struct ib_mw *mw;
1262
1263         if (!pd->device->alloc_mw)
1264                 return ERR_PTR(-ENOSYS);
1265
1266         mw = pd->device->alloc_mw(pd, type);
1267         if (!IS_ERR(mw)) {
1268                 mw->device  = pd->device;
1269                 mw->pd      = pd;
1270                 mw->uobject = NULL;
1271                 mw->type    = type;
1272                 atomic_inc(&pd->usecnt);
1273         }
1274
1275         return mw;
1276 }
1277 EXPORT_SYMBOL(ib_alloc_mw);
1278
1279 int ib_dealloc_mw(struct ib_mw *mw)
1280 {
1281         struct ib_pd *pd;
1282         int ret;
1283
1284         pd = mw->pd;
1285         ret = mw->device->dealloc_mw(mw);
1286         if (!ret)
1287                 atomic_dec(&pd->usecnt);
1288
1289         return ret;
1290 }
1291 EXPORT_SYMBOL(ib_dealloc_mw);
1292
1293 /* "Fast" memory regions */
1294
1295 struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
1296                             int mr_access_flags,
1297                             struct ib_fmr_attr *fmr_attr)
1298 {
1299         struct ib_fmr *fmr;
1300
1301         if (!pd->device->alloc_fmr)
1302                 return ERR_PTR(-ENOSYS);
1303
1304         fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
1305         if (!IS_ERR(fmr)) {
1306                 fmr->device = pd->device;
1307                 fmr->pd     = pd;
1308                 atomic_inc(&pd->usecnt);
1309         }
1310
1311         return fmr;
1312 }
1313 EXPORT_SYMBOL(ib_alloc_fmr);
1314
1315 int ib_unmap_fmr(struct list_head *fmr_list)
1316 {
1317         struct ib_fmr *fmr;
1318
1319         if (list_empty(fmr_list))
1320                 return 0;
1321
1322         fmr = list_entry(fmr_list->next, struct ib_fmr, list);
1323         return fmr->device->unmap_fmr(fmr_list);
1324 }
1325 EXPORT_SYMBOL(ib_unmap_fmr);
1326
1327 int ib_dealloc_fmr(struct ib_fmr *fmr)
1328 {
1329         struct ib_pd *pd;
1330         int ret;
1331
1332         pd = fmr->pd;
1333         ret = fmr->device->dealloc_fmr(fmr);
1334         if (!ret)
1335                 atomic_dec(&pd->usecnt);
1336
1337         return ret;
1338 }
1339 EXPORT_SYMBOL(ib_dealloc_fmr);
1340
1341 /* Multicast groups */
1342
1343 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1344 {
1345         int ret;
1346
1347         if (!qp->device->attach_mcast)
1348                 return -ENOSYS;
1349         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1350                 return -EINVAL;
1351
1352         ret = qp->device->attach_mcast(qp, gid, lid);
1353         if (!ret)
1354                 atomic_inc(&qp->usecnt);
1355         return ret;
1356 }
1357 EXPORT_SYMBOL(ib_attach_mcast);
1358
1359 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1360 {
1361         int ret;
1362
1363         if (!qp->device->detach_mcast)
1364                 return -ENOSYS;
1365         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1366                 return -EINVAL;
1367
1368         ret = qp->device->detach_mcast(qp, gid, lid);
1369         if (!ret)
1370                 atomic_dec(&qp->usecnt);
1371         return ret;
1372 }
1373 EXPORT_SYMBOL(ib_detach_mcast);
1374
1375 struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
1376 {
1377         struct ib_xrcd *xrcd;
1378
1379         if (!device->alloc_xrcd)
1380                 return ERR_PTR(-ENOSYS);
1381
1382         xrcd = device->alloc_xrcd(device, NULL, NULL);
1383         if (!IS_ERR(xrcd)) {
1384                 xrcd->device = device;
1385                 xrcd->inode = NULL;
1386                 atomic_set(&xrcd->usecnt, 0);
1387                 mutex_init(&xrcd->tgt_qp_mutex);
1388                 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
1389         }
1390
1391         return xrcd;
1392 }
1393 EXPORT_SYMBOL(ib_alloc_xrcd);
1394
1395 int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
1396 {
1397         struct ib_qp *qp;
1398         int ret;
1399
1400         if (atomic_read(&xrcd->usecnt))
1401                 return -EBUSY;
1402
1403         while (!list_empty(&xrcd->tgt_qp_list)) {
1404                 qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
1405                 ret = ib_destroy_qp(qp);
1406                 if (ret)
1407                         return ret;
1408         }
1409
1410         return xrcd->device->dealloc_xrcd(xrcd);
1411 }
1412 EXPORT_SYMBOL(ib_dealloc_xrcd);
1413
1414 struct ib_flow *ib_create_flow(struct ib_qp *qp,
1415                                struct ib_flow_attr *flow_attr,
1416                                int domain)
1417 {
1418         struct ib_flow *flow_id;
1419         if (!qp->device->create_flow)
1420                 return ERR_PTR(-ENOSYS);
1421
1422         flow_id = qp->device->create_flow(qp, flow_attr, domain);
1423         if (!IS_ERR(flow_id))
1424                 atomic_inc(&qp->usecnt);
1425         return flow_id;
1426 }
1427 EXPORT_SYMBOL(ib_create_flow);
1428
1429 int ib_destroy_flow(struct ib_flow *flow_id)
1430 {
1431         int err;
1432         struct ib_qp *qp = flow_id->qp;
1433
1434         err = qp->device->destroy_flow(flow_id);
1435         if (!err)
1436                 atomic_dec(&qp->usecnt);
1437         return err;
1438 }
1439 EXPORT_SYMBOL(ib_destroy_flow);
1440
1441 int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
1442                        struct ib_mr_status *mr_status)
1443 {
1444         return mr->device->check_mr_status ?
1445                 mr->device->check_mr_status(mr, check_mask, mr_status) : -ENOSYS;
1446 }
1447 EXPORT_SYMBOL(ib_check_mr_status);