drivers/infiniband/core/uverbs_cmd.c

   1 /*
   2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.
   4  * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
   5  * Copyright (c) 2006 Mellanox Technologies.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/file.h>
  37 #include <linux/fs.h>
  38 #include <linux/slab.h>
  39 #include <linux/sched.h>
  40
  41 #include <linux/uaccess.h>
  42
  43 #include <rdma/uverbs_types.h>
  44 #include <rdma/uverbs_std_types.h>
  45 #include "rdma_core.h"
  46
  47 #include "uverbs.h"
  48 #include "core_priv.h"
  49
  50 static struct ib_uverbs_completion_event_file *
  51 ib_uverbs_lookup_comp_file(int fd, struct ib_ucontext *context)
  52 {
  53         struct ib_uobject *uobj = uobj_get_read(uobj_get_type(comp_channel),
  54                                                 fd, context);
  55         struct ib_uobject_file *uobj_file;
  56
  57         if (IS_ERR(uobj))
  58                 return (void *)uobj;
  59
  60         uverbs_uobject_get(uobj);
  61         uobj_put_read(uobj);
  62
  63         uobj_file = container_of(uobj, struct ib_uobject_file, uobj);
  64         return container_of(uobj_file, struct ib_uverbs_completion_event_file,
  65                             uobj_file);
  66 }
  67
  68 ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
  69                               struct ib_device *ib_dev,
  70                               const char __user *buf,
  71                               int in_len, int out_len)
  72 {
  73         struct ib_uverbs_get_context      cmd;
  74         struct ib_uverbs_get_context_resp resp;
  75         struct ib_udata                   udata;
  76         struct ib_ucontext               *ucontext;
  77         struct file                      *filp;
  78         struct ib_rdmacg_object          cg_obj;
  79         int ret;
  80
  81         if (out_len < sizeof resp)
  82                 return -ENOSPC;
  83
  84         if (copy_from_user(&cmd, buf, sizeof cmd))
  85                 return -EFAULT;
  86
  87         mutex_lock(&file->mutex);
  88
  89         if (file->ucontext) {
  90                 ret = -EINVAL;
  91                 goto err;
  92         }
  93
  94         INIT_UDATA(&udata, buf + sizeof(cmd),
  95                    (unsigned long) cmd.response + sizeof(resp),
  96                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
  97                    out_len - sizeof(resp));
  98
  99         ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
 100         if (ret)
 101                 goto err;
 102
 103         ucontext = ib_dev->alloc_ucontext(ib_dev, &udata);
 104         if (IS_ERR(ucontext)) {
 105                 ret = PTR_ERR(ucontext);
 106                 goto err_alloc;
 107         }
 108
 109         ucontext->device = ib_dev;
 110         ucontext->cg_obj = cg_obj;
 111         /* ufile is required when some objects are released */
 112         ucontext->ufile = file;
 113         uverbs_initialize_ucontext(ucontext);
 114
 115         rcu_read_lock();
 116         ucontext->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
 117         rcu_read_unlock();
 118         ucontext->closing = 0;
 119
 120 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
 121         ucontext->umem_tree = RB_ROOT_CACHED;
 122         init_rwsem(&ucontext->umem_rwsem);
 123         ucontext->odp_mrs_count = 0;
 124         INIT_LIST_HEAD(&ucontext->no_private_counters);
 125
 126         if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
 127                 ucontext->invalidate_range = NULL;
 128
 129 #endif
 130
 131         resp.num_comp_vectors = file->device->num_comp_vectors;
 132
 133         ret = get_unused_fd_flags(O_CLOEXEC);
 134         if (ret < 0)
 135                 goto err_free;
 136         resp.async_fd = ret;
 137
 138         filp = ib_uverbs_alloc_async_event_file(file, ib_dev);
 139         if (IS_ERR(filp)) {
 140                 ret = PTR_ERR(filp);
 141                 goto err_fd;
 142         }
 143
 144         if (copy_to_user((void __user *) (unsigned long) cmd.response,
 145                          &resp, sizeof resp)) {
 146                 ret = -EFAULT;
 147                 goto err_file;
 148         }
 149
 150         file->ucontext = ucontext;
 151
 152         fd_install(resp.async_fd, filp);
 153
 154         mutex_unlock(&file->mutex);
 155
 156         return in_len;
 157
 158 err_file:
 159         ib_uverbs_free_async_event_file(file);
 160         fput(filp);
 161
 162 err_fd:
 163         put_unused_fd(resp.async_fd);
 164
 165 err_free:
 166         put_pid(ucontext->tgid);
 167         ib_dev->dealloc_ucontext(ucontext);
 168
 169 err_alloc:
 170         ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE);
 171
 172 err:
 173         mutex_unlock(&file->mutex);
 174         return ret;
 175 }
 176
 177 static void copy_query_dev_fields(struct ib_uverbs_file *file,
 178                                   struct ib_device *ib_dev,
 179                                   struct ib_uverbs_query_device_resp *resp,
 180                                   struct ib_device_attr *attr)
 181 {
 182         resp->fw_ver            = attr->fw_ver;
 183         resp->node_guid         = ib_dev->node_guid;
 184         resp->sys_image_guid    = attr->sys_image_guid;
 185         resp->max_mr_size       = attr->max_mr_size;
 186         resp->page_size_cap     = attr->page_size_cap;
 187         resp->vendor_id         = attr->vendor_id;
 188         resp->vendor_part_id    = attr->vendor_part_id;
 189         resp->hw_ver            = attr->hw_ver;
 190         resp->max_qp            = attr->max_qp;
 191         resp->max_qp_wr         = attr->max_qp_wr;
 192         resp->device_cap_flags  = lower_32_bits(attr->device_cap_flags);
 193         resp->max_sge           = attr->max_sge;
 194         resp->max_sge_rd        = attr->max_sge_rd;
 195         resp->max_cq            = attr->max_cq;
 196         resp->max_cqe           = attr->max_cqe;
 197         resp->max_mr            = attr->max_mr;
 198         resp->max_pd            = attr->max_pd;
 199         resp->max_qp_rd_atom    = attr->max_qp_rd_atom;
 200         resp->max_ee_rd_atom    = attr->max_ee_rd_atom;
 201         resp->max_res_rd_atom   = attr->max_res_rd_atom;
 202         resp->max_qp_init_rd_atom       = attr->max_qp_init_rd_atom;
 203         resp->max_ee_init_rd_atom       = attr->max_ee_init_rd_atom;
 204         resp->atomic_cap                = attr->atomic_cap;
 205         resp->max_ee                    = attr->max_ee;
 206         resp->max_rdd                   = attr->max_rdd;
 207         resp->max_mw                    = attr->max_mw;
 208         resp->max_raw_ipv6_qp           = attr->max_raw_ipv6_qp;
 209         resp->max_raw_ethy_qp           = attr->max_raw_ethy_qp;
 210         resp->max_mcast_grp             = attr->max_mcast_grp;
 211         resp->max_mcast_qp_attach       = attr->max_mcast_qp_attach;
 212         resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach;
 213         resp->max_ah                    = attr->max_ah;
 214         resp->max_fmr                   = attr->max_fmr;
 215         resp->max_map_per_fmr           = attr->max_map_per_fmr;
 216         resp->max_srq                   = attr->max_srq;
 217         resp->max_srq_wr                = attr->max_srq_wr;
 218         resp->max_srq_sge               = attr->max_srq_sge;
 219         resp->max_pkeys                 = attr->max_pkeys;
 220         resp->local_ca_ack_delay        = attr->local_ca_ack_delay;
 221         resp->phys_port_cnt             = ib_dev->phys_port_cnt;
 222 }
 223
 224 ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file,
 225                                struct ib_device *ib_dev,
 226                                const char __user *buf,
 227                                int in_len, int out_len)
 228 {
 229         struct ib_uverbs_query_device      cmd;
 230         struct ib_uverbs_query_device_resp resp;
 231
 232         if (out_len < sizeof resp)
 233                 return -ENOSPC;
 234
 235         if (copy_from_user(&cmd, buf, sizeof cmd))
 236                 return -EFAULT;
 237
 238         memset(&resp, 0, sizeof resp);
 239         copy_query_dev_fields(file, ib_dev, &resp, &ib_dev->attrs);
 240
 241         if (copy_to_user((void __user *) (unsigned long) cmd.response,
 242                          &resp, sizeof resp))
 243                 return -EFAULT;
 244
 245         return in_len;
 246 }
 247
 248 ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file,
 249                              struct ib_device *ib_dev,
 250                              const char __user *buf,
 251                              int in_len, int out_len)
 252 {
 253         struct ib_uverbs_query_port      cmd;
 254         struct ib_uverbs_query_port_resp resp;
 255         struct ib_port_attr              attr;
 256         int                              ret;
 257
 258         if (out_len < sizeof resp)
 259                 return -ENOSPC;
 260
 261         if (copy_from_user(&cmd, buf, sizeof cmd))
 262                 return -EFAULT;
 263
 264         ret = ib_query_port(ib_dev, cmd.port_num, &attr);
 265         if (ret)
 266                 return ret;
 267
 268         memset(&resp, 0, sizeof resp);
 269
 270         resp.state           = attr.state;
 271         resp.max_mtu         = attr.max_mtu;
 272         resp.active_mtu      = attr.active_mtu;
 273         resp.gid_tbl_len     = attr.gid_tbl_len;
 274         resp.port_cap_flags  = attr.port_cap_flags;
 275         resp.max_msg_sz      = attr.max_msg_sz;
 276         resp.bad_pkey_cntr   = attr.bad_pkey_cntr;
 277         resp.qkey_viol_cntr  = attr.qkey_viol_cntr;
 278         resp.pkey_tbl_len    = attr.pkey_tbl_len;
 279
 280         if (rdma_cap_opa_ah(ib_dev, cmd.port_num)) {
 281                 resp.lid     = OPA_TO_IB_UCAST_LID(attr.lid);
 282                 resp.sm_lid  = OPA_TO_IB_UCAST_LID(attr.sm_lid);
 283         } else {
 284                 resp.lid     = ib_lid_cpu16(attr.lid);
 285                 resp.sm_lid  = ib_lid_cpu16(attr.sm_lid);
 286         }
 287         resp.lmc             = attr.lmc;
 288         resp.max_vl_num      = attr.max_vl_num;
 289         resp.sm_sl           = attr.sm_sl;
 290         resp.subnet_timeout  = attr.subnet_timeout;
 291         resp.init_type_reply = attr.init_type_reply;
 292         resp.active_width    = attr.active_width;
 293         resp.active_speed    = attr.active_speed;
 294         resp.phys_state      = attr.phys_state;
 295         resp.link_layer      = rdma_port_get_link_layer(ib_dev,
 296                                                         cmd.port_num);
 297
 298         if (copy_to_user((void __user *) (unsigned long) cmd.response,
 299                          &resp, sizeof resp))
 300                 return -EFAULT;
 301
 302         return in_len;
 303 }
 304
 305 ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file,
 306                            struct ib_device *ib_dev,
 307                            const char __user *buf,
 308                            int in_len, int out_len)
 309 {
 310         struct ib_uverbs_alloc_pd      cmd;
 311         struct ib_uverbs_alloc_pd_resp resp;
 312         struct ib_udata                udata;
 313         struct ib_uobject             *uobj;
 314         struct ib_pd                  *pd;
 315         int                            ret;
 316
 317         if (out_len < sizeof resp)
 318                 return -ENOSPC;
 319
 320         if (copy_from_user(&cmd, buf, sizeof cmd))
 321                 return -EFAULT;
 322
 323         INIT_UDATA(&udata, buf + sizeof(cmd),
 324                    (unsigned long) cmd.response + sizeof(resp),
 325                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
 326                    out_len - sizeof(resp));
 327
 328         uobj  = uobj_alloc(uobj_get_type(pd), file->ucontext);
 329         if (IS_ERR(uobj))
 330                 return PTR_ERR(uobj);
 331
 332         pd = ib_dev->alloc_pd(ib_dev, file->ucontext, &udata);
 333         if (IS_ERR(pd)) {
 334                 ret = PTR_ERR(pd);
 335                 goto err;
 336         }
 337
 338         pd->device  = ib_dev;
 339         pd->uobject = uobj;
 340         pd->__internal_mr = NULL;
 341         atomic_set(&pd->usecnt, 0);
 342
 343         uobj->object = pd;
 344         memset(&resp, 0, sizeof resp);
 345         resp.pd_handle = uobj->id;
 346
 347         if (copy_to_user((void __user *) (unsigned long) cmd.response,
 348                          &resp, sizeof resp)) {
 349                 ret = -EFAULT;
 350                 goto err_copy;
 351         }
 352
 353         uobj_alloc_commit(uobj);
 354
 355         return in_len;
 356
 357 err_copy:
 358         ib_dealloc_pd(pd);
 359
 360 err:
 361         uobj_alloc_abort(uobj);
 362         return ret;
 363 }
 364
 365 ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file,
 366                              struct ib_device *ib_dev,
 367                              const char __user *buf,
 368                              int in_len, int out_len)
 369 {
 370         struct ib_uverbs_dealloc_pd cmd;
 371         struct ib_uobject          *uobj;
 372         int                         ret;
 373
 374         if (copy_from_user(&cmd, buf, sizeof cmd))
 375                 return -EFAULT;
 376
 377         uobj  = uobj_get_write(uobj_get_type(pd), cmd.pd_handle,
 378                                file->ucontext);
 379         if (IS_ERR(uobj))
 380                 return PTR_ERR(uobj);
 381
 382         ret = uobj_remove_commit(uobj);
 383
 384         return ret ?: in_len;
 385 }
 386
 387 struct xrcd_table_entry {
 388         struct rb_node  node;
 389         struct ib_xrcd *xrcd;
 390         struct inode   *inode;
 391 };
 392
 393 static int xrcd_table_insert(struct ib_uverbs_device *dev,
 394                             struct inode *inode,
 395                             struct ib_xrcd *xrcd)
 396 {
 397         struct xrcd_table_entry *entry, *scan;
 398         struct rb_node **p = &dev->xrcd_tree.rb_node;
 399         struct rb_node *parent = NULL;
 400
 401         entry = kmalloc(sizeof *entry, GFP_KERNEL);
 402         if (!entry)
 403                 return -ENOMEM;
 404
 405         entry->xrcd  = xrcd;
 406         entry->inode = inode;
 407
 408         while (*p) {
 409                 parent = *p;
 410                 scan = rb_entry(parent, struct xrcd_table_entry, node);
 411
 412                 if (inode < scan->inode) {
 413                         p = &(*p)->rb_left;
 414                 } else if (inode > scan->inode) {
 415                         p = &(*p)->rb_right;
 416                 } else {
 417                         kfree(entry);
 418                         return -EEXIST;
 419                 }
 420         }
 421
 422         rb_link_node(&entry->node, parent, p);
 423         rb_insert_color(&entry->node, &dev->xrcd_tree);
 424         igrab(inode);
 425         return 0;
 426 }
 427
 428 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev,
 429                                                   struct inode *inode)
 430 {
 431         struct xrcd_table_entry *entry;
 432         struct rb_node *p = dev->xrcd_tree.rb_node;
 433
 434         while (p) {
 435                 entry = rb_entry(p, struct xrcd_table_entry, node);
 436
 437                 if (inode < entry->inode)
 438                         p = p->rb_left;
 439                 else if (inode > entry->inode)
 440                         p = p->rb_right;
 441                 else
 442                         return entry;
 443         }
 444
 445         return NULL;
 446 }
 447
 448 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode)
 449 {
 450         struct xrcd_table_entry *entry;
 451
 452         entry = xrcd_table_search(dev, inode);
 453         if (!entry)
 454                 return NULL;
 455
 456         return entry->xrcd;
 457 }
 458
 459 static void xrcd_table_delete(struct ib_uverbs_device *dev,
 460                               struct inode *inode)
 461 {
 462         struct xrcd_table_entry *entry;
 463
 464         entry = xrcd_table_search(dev, inode);
 465         if (entry) {
 466                 iput(inode);
 467                 rb_erase(&entry->node, &dev->xrcd_tree);
 468                 kfree(entry);
 469         }
 470 }
 471
 472 ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
 473                             struct ib_device *ib_dev,
 474                             const char __user *buf, int in_len,
 475                             int out_len)
 476 {
 477         struct ib_uverbs_open_xrcd      cmd;
 478         struct ib_uverbs_open_xrcd_resp resp;
 479         struct ib_udata                 udata;
 480         struct ib_uxrcd_object         *obj;
 481         struct ib_xrcd                 *xrcd = NULL;
 482         struct fd                       f = {NULL, 0};
 483         struct inode                   *inode = NULL;
 484         int                             ret = 0;
 485         int                             new_xrcd = 0;
 486
 487         if (out_len < sizeof resp)
 488                 return -ENOSPC;
 489
 490         if (copy_from_user(&cmd, buf, sizeof cmd))
 491                 return -EFAULT;
 492
 493         INIT_UDATA(&udata, buf + sizeof(cmd),
 494                    (unsigned long) cmd.response + sizeof(resp),
 495                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
 496                    out_len - sizeof(resp));
 497
 498         mutex_lock(&file->device->xrcd_tree_mutex);
 499
 500         if (cmd.fd != -1) {
 501                 /* search for file descriptor */
 502                 f = fdget(cmd.fd);
 503                 if (!f.file) {
 504                         ret = -EBADF;
 505                         goto err_tree_mutex_unlock;
 506                 }
 507
 508                 inode = file_inode(f.file);
 509                 xrcd = find_xrcd(file->device, inode);
 510                 if (!xrcd && !(cmd.oflags & O_CREAT)) {
 511                         /* no file descriptor. Need CREATE flag */
 512                         ret = -EAGAIN;
 513                         goto err_tree_mutex_unlock;
 514                 }
 515
 516                 if (xrcd && cmd.oflags & O_EXCL) {
 517                         ret = -EINVAL;
 518                         goto err_tree_mutex_unlock;
 519                 }
 520         }
 521
 522         obj  = (struct ib_uxrcd_object *)uobj_alloc(uobj_get_type(xrcd),
 523                                                     file->ucontext);
 524         if (IS_ERR(obj)) {
 525                 ret = PTR_ERR(obj);
 526                 goto err_tree_mutex_unlock;
 527         }
 528
 529         if (!xrcd) {
 530                 xrcd = ib_dev->alloc_xrcd(ib_dev, file->ucontext, &udata);
 531                 if (IS_ERR(xrcd)) {
 532                         ret = PTR_ERR(xrcd);
 533                         goto err;
 534                 }
 535
 536                 xrcd->inode   = inode;
 537                 xrcd->device  = ib_dev;
 538                 atomic_set(&xrcd->usecnt, 0);
 539                 mutex_init(&xrcd->tgt_qp_mutex);
 540                 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
 541                 new_xrcd = 1;
 542         }
 543
 544         atomic_set(&obj->refcnt, 0);
 545         obj->uobject.object = xrcd;
 546         memset(&resp, 0, sizeof resp);
 547         resp.xrcd_handle = obj->uobject.id;
 548
 549         if (inode) {
 550                 if (new_xrcd) {
 551                         /* create new inode/xrcd table entry */
 552                         ret = xrcd_table_insert(file->device, inode, xrcd);
 553                         if (ret)
 554                                 goto err_dealloc_xrcd;
 555                 }
 556                 atomic_inc(&xrcd->usecnt);
 557         }
 558
 559         if (copy_to_user((void __user *) (unsigned long) cmd.response,
 560                          &resp, sizeof resp)) {
 561                 ret = -EFAULT;
 562                 goto err_copy;
 563         }
 564
 565         if (f.file)
 566                 fdput(f);
 567
 568         mutex_unlock(&file->device->xrcd_tree_mutex);
 569
 570         uobj_alloc_commit(&obj->uobject);
 571
 572         return in_len;
 573
 574 err_copy:
 575         if (inode) {
 576                 if (new_xrcd)
 577                         xrcd_table_delete(file->device, inode);
 578                 atomic_dec(&xrcd->usecnt);
 579         }
 580
 581 err_dealloc_xrcd:
 582         ib_dealloc_xrcd(xrcd);
 583
 584 err:
 585         uobj_alloc_abort(&obj->uobject);
 586
 587 err_tree_mutex_unlock:
 588         if (f.file)
 589                 fdput(f);
 590
 591         mutex_unlock(&file->device->xrcd_tree_mutex);
 592
 593         return ret;
 594 }
 595
 596 ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file,
 597                              struct ib_device *ib_dev,
 598                              const char __user *buf, int in_len,
 599                              int out_len)
 600 {
 601         struct ib_uverbs_close_xrcd cmd;
 602         struct ib_uobject           *uobj;
 603         int                         ret = 0;
 604
 605         if (copy_from_user(&cmd, buf, sizeof cmd))
 606                 return -EFAULT;
 607
 608         uobj  = uobj_get_write(uobj_get_type(xrcd), cmd.xrcd_handle,
 609                                file->ucontext);
 610         if (IS_ERR(uobj))
 611                 return PTR_ERR(uobj);
 612
 613         ret = uobj_remove_commit(uobj);
 614         return ret ?: in_len;
 615 }
 616
 617 int ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev,
 618                            struct ib_xrcd *xrcd,
 619                            enum rdma_remove_reason why)
 620 {
 621         struct inode *inode;
 622         int ret;
 623
 624         inode = xrcd->inode;
 625         if (inode && !atomic_dec_and_test(&xrcd->usecnt))
 626                 return 0;
 627
 628         ret = ib_dealloc_xrcd(xrcd);
 629
 630         if (why == RDMA_REMOVE_DESTROY && ret)
 631                 atomic_inc(&xrcd->usecnt);
 632         else if (inode)
 633                 xrcd_table_delete(dev, inode);
 634
 635         return ret;
 636 }
 637
 638 ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file,
 639                          struct ib_device *ib_dev,
 640                          const char __user *buf, int in_len,
 641                          int out_len)
 642 {
 643         struct ib_uverbs_reg_mr      cmd;
 644         struct ib_uverbs_reg_mr_resp resp;
 645         struct ib_udata              udata;
 646         struct ib_uobject           *uobj;
 647         struct ib_pd                *pd;
 648         struct ib_mr                *mr;
 649         int                          ret;
 650
 651         if (out_len < sizeof resp)
 652                 return -ENOSPC;
 653
 654         if (copy_from_user(&cmd, buf, sizeof cmd))
 655                 return -EFAULT;
 656
 657         INIT_UDATA(&udata, buf + sizeof(cmd),
 658                    (unsigned long) cmd.response + sizeof(resp),
 659                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
 660                    out_len - sizeof(resp));
 661
 662         if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))
 663                 return -EINVAL;
 664
 665         ret = ib_check_mr_access(cmd.access_flags);
 666         if (ret)
 667                 return ret;
 668
 669         uobj  = uobj_alloc(uobj_get_type(mr), file->ucontext);
 670         if (IS_ERR(uobj))
 671                 return PTR_ERR(uobj);
 672
 673         pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext);
 674         if (!pd) {
 675                 ret = -EINVAL;
 676                 goto err_free;
 677         }
 678
 679         if (cmd.access_flags & IB_ACCESS_ON_DEMAND) {
 680                 if (!(pd->device->attrs.device_cap_flags &
 681                       IB_DEVICE_ON_DEMAND_PAGING)) {
 682                         pr_debug("ODP support not available\n");
 683                         ret = -EINVAL;
 684                         goto err_put;
 685                 }
 686         }
 687
 688         mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
 689                                      cmd.access_flags, &udata);
 690         if (IS_ERR(mr)) {
 691                 ret = PTR_ERR(mr);
 692                 goto err_put;
 693         }
 694
 695         mr->device  = pd->device;
 696         mr->pd      = pd;
 697         mr->uobject = uobj;
 698         atomic_inc(&pd->usecnt);
 699
 700         uobj->object = mr;
 701
 702         memset(&resp, 0, sizeof resp);
 703         resp.lkey      = mr->lkey;
 704         resp.rkey      = mr->rkey;
 705         resp.mr_handle = uobj->id;
 706
 707         if (copy_to_user((void __user *) (unsigned long) cmd.response,
 708                          &resp, sizeof resp)) {
 709                 ret = -EFAULT;
 710                 goto err_copy;
 711         }
 712
 713         uobj_put_obj_read(pd);
 714
 715         uobj_alloc_commit(uobj);
 716
 717         return in_len;
 718
 719 err_copy:
 720         ib_dereg_mr(mr);
 721
 722 err_put:
 723         uobj_put_obj_read(pd);
 724
 725 err_free:
 726         uobj_alloc_abort(uobj);
 727         return ret;
 728 }
 729
 730 ssize_t ib_uverbs_rereg_mr(struct ib_uverbs_file *file,
 731                            struct ib_device *ib_dev,
 732                            const char __user *buf, int in_len,
 733                            int out_len)
 734 {
 735         struct ib_uverbs_rereg_mr      cmd;
 736         struct ib_uverbs_rereg_mr_resp resp;
 737         struct ib_udata              udata;
 738         struct ib_pd                *pd = NULL;
 739         struct ib_mr                *mr;
 740         struct ib_pd                *old_pd;
 741         int                          ret;
 742         struct ib_uobject           *uobj;
 743
 744         if (out_len < sizeof(resp))
 745                 return -ENOSPC;
 746
 747         if (copy_from_user(&cmd, buf, sizeof(cmd)))
 748                 return -EFAULT;
 749
 750         INIT_UDATA(&udata, buf + sizeof(cmd),
 751                    (unsigned long) cmd.response + sizeof(resp),
 752                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
 753                    out_len - sizeof(resp));
 754
 755         if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags)
 756                 return -EINVAL;
 757
 758         if ((cmd.flags & IB_MR_REREG_TRANS) &&
 759             (!cmd.start || !cmd.hca_va || 0 >= cmd.length ||
 760              (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)))
 761                         return -EINVAL;
 762
 763         uobj  = uobj_get_write(uobj_get_type(mr), cmd.mr_handle,
 764                                file->ucontext);
 765         if (IS_ERR(uobj))
 766                 return PTR_ERR(uobj);
 767
 768         mr = uobj->object;
 769
 770         if (cmd.flags & IB_MR_REREG_ACCESS) {
 771                 ret = ib_check_mr_access(cmd.access_flags);
 772                 if (ret)
 773                         goto put_uobjs;
 774         }
 775
 776         if (cmd.flags & IB_MR_REREG_PD) {
 777                 pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext);
 778                 if (!pd) {
 779                         ret = -EINVAL;
 780                         goto put_uobjs;
 781                 }
 782         }
 783
 784         old_pd = mr->pd;
 785         ret = mr->device->rereg_user_mr(mr, cmd.flags, cmd.start,
 786                                         cmd.length, cmd.hca_va,
 787                                         cmd.access_flags, pd, &udata);
 788         if (!ret) {
 789                 if (cmd.flags & IB_MR_REREG_PD) {
 790                         atomic_inc(&pd->usecnt);
 791                         mr->pd = pd;
 792                         atomic_dec(&old_pd->usecnt);
 793                 }
 794         } else {
 795                 goto put_uobj_pd;
 796         }
 797
 798         memset(&resp, 0, sizeof(resp));
 799         resp.lkey      = mr->lkey;
 800         resp.rkey      = mr->rkey;
 801
 802         if (copy_to_user((void __user *)(unsigned long)cmd.response,
 803                          &resp, sizeof(resp)))
 804                 ret = -EFAULT;
 805         else
 806                 ret = in_len;
 807
 808 put_uobj_pd:
 809         if (cmd.flags & IB_MR_REREG_PD)
 810                 uobj_put_obj_read(pd);
 811
 812 put_uobjs:
 813         uobj_put_write(uobj);
 814
 815         return ret;
 816 }
 817
 818 ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file,
 819                            struct ib_device *ib_dev,
 820                            const char __user *buf, int in_len,
 821                            int out_len)
 822 {
 823         struct ib_uverbs_dereg_mr cmd;
 824         struct ib_uobject        *uobj;
 825         int                       ret = -EINVAL;
 826
 827         if (copy_from_user(&cmd, buf, sizeof cmd))
 828                 return -EFAULT;
 829
 830         uobj  = uobj_get_write(uobj_get_type(mr), cmd.mr_handle,
 831                                file->ucontext);
 832         if (IS_ERR(uobj))
 833                 return PTR_ERR(uobj);
 834
 835         ret = uobj_remove_commit(uobj);
 836
 837         return ret ?: in_len;
 838 }
 839
 840 ssize_t ib_uverbs_alloc_mw(struct ib_uverbs_file *file,
 841                            struct ib_device *ib_dev,
 842                            const char __user *buf, int in_len,
 843                            int out_len)
 844 {
 845         struct ib_uverbs_alloc_mw      cmd;
 846         struct ib_uverbs_alloc_mw_resp resp;
 847         struct ib_uobject             *uobj;
 848         struct ib_pd                  *pd;
 849         struct ib_mw                  *mw;
 850         struct ib_udata                udata;
 851         int                            ret;
 852
 853         if (out_len < sizeof(resp))
 854                 return -ENOSPC;
 855
 856         if (copy_from_user(&cmd, buf, sizeof(cmd)))
 857                 return -EFAULT;
 858
 859         uobj  = uobj_alloc(uobj_get_type(mw), file->ucontext);
 860         if (IS_ERR(uobj))
 861                 return PTR_ERR(uobj);
 862
 863         pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext);
 864         if (!pd) {
 865                 ret = -EINVAL;
 866                 goto err_free;
 867         }
 868
 869         INIT_UDATA(&udata, buf + sizeof(cmd),
 870                    (unsigned long)cmd.response + sizeof(resp),
 871                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
 872                    out_len - sizeof(resp));
 873
 874         mw = pd->device->alloc_mw(pd, cmd.mw_type, &udata);
 875         if (IS_ERR(mw)) {
 876                 ret = PTR_ERR(mw);
 877                 goto err_put;
 878         }
 879
 880         mw->device  = pd->device;
 881         mw->pd      = pd;
 882         mw->uobject = uobj;
 883         atomic_inc(&pd->usecnt);
 884
 885         uobj->object = mw;
 886
 887         memset(&resp, 0, sizeof(resp));
 888         resp.rkey      = mw->rkey;
 889         resp.mw_handle = uobj->id;
 890
 891         if (copy_to_user((void __user *)(unsigned long)cmd.response,
 892                          &resp, sizeof(resp))) {
 893                 ret = -EFAULT;
 894                 goto err_copy;
 895         }
 896
 897         uobj_put_obj_read(pd);
 898         uobj_alloc_commit(uobj);
 899
 900         return in_len;
 901
 902 err_copy:
 903         uverbs_dealloc_mw(mw);
 904 err_put:
 905         uobj_put_obj_read(pd);
 906 err_free:
 907         uobj_alloc_abort(uobj);
 908         return ret;
 909 }
 910
 911 ssize_t ib_uverbs_dealloc_mw(struct ib_uverbs_file *file,
 912                              struct ib_device *ib_dev,
 913                              const char __user *buf, int in_len,
 914                              int out_len)
 915 {
 916         struct ib_uverbs_dealloc_mw cmd;
 917         struct ib_uobject          *uobj;
 918         int                         ret = -EINVAL;
 919
 920         if (copy_from_user(&cmd, buf, sizeof(cmd)))
 921                 return -EFAULT;
 922
 923         uobj  = uobj_get_write(uobj_get_type(mw), cmd.mw_handle,
 924                                file->ucontext);
 925         if (IS_ERR(uobj))
 926                 return PTR_ERR(uobj);
 927
 928         ret = uobj_remove_commit(uobj);
 929         return ret ?: in_len;
 930 }
 931
 932 ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file,
 933                                       struct ib_device *ib_dev,
 934                                       const char __user *buf, int in_len,
 935                                       int out_len)
 936 {
 937         struct ib_uverbs_create_comp_channel       cmd;
 938         struct ib_uverbs_create_comp_channel_resp  resp;
 939         struct ib_uobject                         *uobj;
 940         struct ib_uverbs_completion_event_file    *ev_file;
 941
 942         if (out_len < sizeof resp)
 943                 return -ENOSPC;
 944
 945         if (copy_from_user(&cmd, buf, sizeof cmd))
 946                 return -EFAULT;
 947
 948         uobj = uobj_alloc(uobj_get_type(comp_channel), file->ucontext);
 949         if (IS_ERR(uobj))
 950                 return PTR_ERR(uobj);
 951
 952         resp.fd = uobj->id;
 953
 954         ev_file = container_of(uobj, struct ib_uverbs_completion_event_file,
 955                                uobj_file.uobj);
 956         ib_uverbs_init_event_queue(&ev_file->ev_queue);
 957
 958         if (copy_to_user((void __user *) (unsigned long) cmd.response,
 959                          &resp, sizeof resp)) {
 960                 uobj_alloc_abort(uobj);
 961                 return -EFAULT;
 962         }
 963
 964         uobj_alloc_commit(uobj);
 965         return in_len;
 966 }
 967
 968 static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file,
 969                                         struct ib_device *ib_dev,
 970                                        struct ib_udata *ucore,
 971                                        struct ib_udata *uhw,
 972                                        struct ib_uverbs_ex_create_cq *cmd,
 973                                        size_t cmd_sz,
 974                                        int (*cb)(struct ib_uverbs_file *file,
 975                                                  struct ib_ucq_object *obj,
 976                                                  struct ib_uverbs_ex_create_cq_resp *resp,
 977                                                  struct ib_udata *udata,
 978                                                  void *context),
 979                                        void *context)
 980 {
 981         struct ib_ucq_object           *obj;
 982         struct ib_uverbs_completion_event_file    *ev_file = NULL;
 983         struct ib_cq                   *cq;
 984         int                             ret;
 985         struct ib_uverbs_ex_create_cq_resp resp;
 986         struct ib_cq_init_attr attr = {};
 987
 988         if (cmd->comp_vector >= file->device->num_comp_vectors)
 989                 return ERR_PTR(-EINVAL);
 990
 991         obj  = (struct ib_ucq_object *)uobj_alloc(uobj_get_type(cq),
 992                                                   file->ucontext);
 993         if (IS_ERR(obj))
 994                 return obj;
 995
 996         if (cmd->comp_channel >= 0) {
 997                 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel,
 998                                                      file->ucontext);
 999                 if (IS_ERR(ev_file)) {
1000                         ret = PTR_ERR(ev_file);
1001                         goto err;
1002                 }
1003         }
1004
1005         obj->uobject.user_handle = cmd->user_handle;
1006         obj->uverbs_file           = file;
1007         obj->comp_events_reported  = 0;
1008         obj->async_events_reported = 0;
1009         INIT_LIST_HEAD(&obj->comp_list);
1010         INIT_LIST_HEAD(&obj->async_list);
1011
1012         attr.cqe = cmd->cqe;
1013         attr.comp_vector = cmd->comp_vector;
1014
1015         if (cmd_sz > offsetof(typeof(*cmd), flags) + sizeof(cmd->flags))
1016                 attr.flags = cmd->flags;
1017
1018         cq = ib_dev->create_cq(ib_dev, &attr, file->ucontext, uhw);
1019         if (IS_ERR(cq)) {
1020                 ret = PTR_ERR(cq);
1021                 goto err_file;
1022         }
1023
1024         cq->device        = ib_dev;
1025         cq->uobject       = &obj->uobject;
1026         cq->comp_handler  = ib_uverbs_comp_handler;
1027         cq->event_handler = ib_uverbs_cq_event_handler;
1028         cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;
1029         atomic_set(&cq->usecnt, 0);
1030
1031         obj->uobject.object = cq;
1032         memset(&resp, 0, sizeof resp);
1033         resp.base.cq_handle = obj->uobject.id;
1034         resp.base.cqe       = cq->cqe;
1035
1036         resp.response_length = offsetof(typeof(resp), response_length) +
1037                 sizeof(resp.response_length);
1038
1039         ret = cb(file, obj, &resp, ucore, context);
1040         if (ret)
1041                 goto err_cb;
1042
1043         uobj_alloc_commit(&obj->uobject);
1044
1045         return obj;
1046
1047 err_cb:
1048         ib_destroy_cq(cq);
1049
1050 err_file:
1051         if (ev_file)
1052                 ib_uverbs_release_ucq(file, ev_file, obj);
1053
1054 err:
1055         uobj_alloc_abort(&obj->uobject);
1056
1057         return ERR_PTR(ret);
1058 }
1059
1060 static int ib_uverbs_create_cq_cb(struct ib_uverbs_file *file,
1061                                   struct ib_ucq_object *obj,
1062                                   struct ib_uverbs_ex_create_cq_resp *resp,
1063                                   struct ib_udata *ucore, void *context)
1064 {
1065         if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base)))
1066                 return -EFAULT;
1067
1068         return 0;
1069 }
1070
1071 ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file,
1072                             struct ib_device *ib_dev,
1073                             const char __user *buf, int in_len,
1074                             int out_len)
1075 {
1076         struct ib_uverbs_create_cq      cmd;
1077         struct ib_uverbs_ex_create_cq   cmd_ex;
1078         struct ib_uverbs_create_cq_resp resp;
1079         struct ib_udata                 ucore;
1080         struct ib_udata                 uhw;
1081         struct ib_ucq_object           *obj;
1082
1083         if (out_len < sizeof(resp))
1084                 return -ENOSPC;
1085
1086         if (copy_from_user(&cmd, buf, sizeof(cmd)))
1087                 return -EFAULT;
1088
1089         INIT_UDATA(&ucore, buf, (unsigned long)cmd.response, sizeof(cmd), sizeof(resp));
1090
1091         INIT_UDATA(&uhw, buf + sizeof(cmd),
1092                    (unsigned long)cmd.response + sizeof(resp),
1093                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1094                    out_len - sizeof(resp));
1095
1096         memset(&cmd_ex, 0, sizeof(cmd_ex));
1097         cmd_ex.user_handle = cmd.user_handle;
1098         cmd_ex.cqe = cmd.cqe;
1099         cmd_ex.comp_vector = cmd.comp_vector;
1100         cmd_ex.comp_channel = cmd.comp_channel;
1101
1102         obj = create_cq(file, ib_dev, &ucore, &uhw, &cmd_ex,
1103                         offsetof(typeof(cmd_ex), comp_channel) +
1104                         sizeof(cmd.comp_channel), ib_uverbs_create_cq_cb,
1105                         NULL);
1106
1107         if (IS_ERR(obj))
1108                 return PTR_ERR(obj);
1109
1110         return in_len;
1111 }
1112
1113 static int ib_uverbs_ex_create_cq_cb(struct ib_uverbs_file *file,
1114                                      struct ib_ucq_object *obj,
1115                                      struct ib_uverbs_ex_create_cq_resp *resp,
1116                                      struct ib_udata *ucore, void *context)
1117 {
1118         if (ib_copy_to_udata(ucore, resp, resp->response_length))
1119                 return -EFAULT;
1120
1121         return 0;
1122 }
1123
1124 int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file,
1125                          struct ib_device *ib_dev,
1126                            struct ib_udata *ucore,
1127                            struct ib_udata *uhw)
1128 {
1129         struct ib_uverbs_ex_create_cq_resp resp;
1130         struct ib_uverbs_ex_create_cq  cmd;
1131         struct ib_ucq_object           *obj;
1132         int err;
1133
1134         if (ucore->inlen < sizeof(cmd))
1135                 return -EINVAL;
1136
1137         err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
1138         if (err)
1139                 return err;
1140
1141         if (cmd.comp_mask)
1142                 return -EINVAL;
1143
1144         if (cmd.reserved)
1145                 return -EINVAL;
1146
1147         if (ucore->outlen < (offsetof(typeof(resp), response_length) +
1148                              sizeof(resp.response_length)))
1149                 return -ENOSPC;
1150
1151         obj = create_cq(file, ib_dev, ucore, uhw, &cmd,
1152                         min(ucore->inlen, sizeof(cmd)),
1153                         ib_uverbs_ex_create_cq_cb, NULL);
1154
1155         if (IS_ERR(obj))
1156                 return PTR_ERR(obj);
1157
1158         return 0;
1159 }
1160
1161 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file,
1162                             struct ib_device *ib_dev,
1163                             const char __user *buf, int in_len,
1164                             int out_len)
1165 {
1166         struct ib_uverbs_resize_cq      cmd;
1167         struct ib_uverbs_resize_cq_resp resp = {};
1168         struct ib_udata                 udata;
1169         struct ib_cq                    *cq;
1170         int                             ret = -EINVAL;
1171
1172         if (copy_from_user(&cmd, buf, sizeof cmd))
1173                 return -EFAULT;
1174
1175         INIT_UDATA(&udata, buf + sizeof(cmd),
1176                    (unsigned long) cmd.response + sizeof(resp),
1177                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1178                    out_len - sizeof(resp));
1179
1180         cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext);
1181         if (!cq)
1182                 return -EINVAL;
1183
1184         ret = cq->device->resize_cq(cq, cmd.cqe, &udata);
1185         if (ret)
1186                 goto out;
1187
1188         resp.cqe = cq->cqe;
1189
1190         if (copy_to_user((void __user *) (unsigned long) cmd.response,
1191                          &resp, sizeof resp.cqe))
1192                 ret = -EFAULT;
1193
1194 out:
1195         uobj_put_obj_read(cq);
1196
1197         return ret ? ret : in_len;
1198 }
1199
1200 static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,
1201                            struct ib_wc *wc)
1202 {
1203         struct ib_uverbs_wc tmp;
1204
1205         tmp.wr_id               = wc->wr_id;
1206         tmp.status              = wc->status;
1207         tmp.opcode              = wc->opcode;
1208         tmp.vendor_err          = wc->vendor_err;
1209         tmp.byte_len            = wc->byte_len;
1210         tmp.ex.imm_data         = (__u32 __force) wc->ex.imm_data;
1211         tmp.qp_num              = wc->qp->qp_num;
1212         tmp.src_qp              = wc->src_qp;
1213         tmp.wc_flags            = wc->wc_flags;
1214         tmp.pkey_index          = wc->pkey_index;
1215         if (rdma_cap_opa_ah(ib_dev, wc->port_num))
1216                 tmp.slid        = OPA_TO_IB_UCAST_LID(wc->slid);
1217         else
1218                 tmp.slid        = ib_lid_cpu16(wc->slid);
1219         tmp.sl                  = wc->sl;
1220         tmp.dlid_path_bits      = wc->dlid_path_bits;
1221         tmp.port_num            = wc->port_num;
1222         tmp.reserved            = 0;
1223
1224         if (copy_to_user(dest, &tmp, sizeof tmp))
1225                 return -EFAULT;
1226
1227         return 0;
1228 }
1229
1230 ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file,
1231                           struct ib_device *ib_dev,
1232                           const char __user *buf, int in_len,
1233                           int out_len)
1234 {
1235         struct ib_uverbs_poll_cq       cmd;
1236         struct ib_uverbs_poll_cq_resp  resp;
1237         u8 __user                     *header_ptr;
1238         u8 __user                     *data_ptr;
1239         struct ib_cq                  *cq;
1240         struct ib_wc                   wc;
1241         int                            ret;
1242
1243         if (copy_from_user(&cmd, buf, sizeof cmd))
1244                 return -EFAULT;
1245
1246         cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext);
1247         if (!cq)
1248                 return -EINVAL;
1249
1250         /* we copy a struct ib_uverbs_poll_cq_resp to user space */
1251         header_ptr = (void __user *)(unsigned long) cmd.response;
1252         data_ptr = header_ptr + sizeof resp;
1253
1254         memset(&resp, 0, sizeof resp);
1255         while (resp.count < cmd.ne) {
1256                 ret = ib_poll_cq(cq, 1, &wc);
1257                 if (ret < 0)
1258                         goto out_put;
1259                 if (!ret)
1260                         break;
1261
1262                 ret = copy_wc_to_user(ib_dev, data_ptr, &wc);
1263                 if (ret)
1264                         goto out_put;
1265
1266                 data_ptr += sizeof(struct ib_uverbs_wc);
1267                 ++resp.count;
1268         }
1269
1270         if (copy_to_user(header_ptr, &resp, sizeof resp)) {
1271                 ret = -EFAULT;
1272                 goto out_put;
1273         }
1274
1275         ret = in_len;
1276
1277 out_put:
1278         uobj_put_obj_read(cq);
1279         return ret;
1280 }
1281
1282 ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file,
1283                                 struct ib_device *ib_dev,
1284                                 const char __user *buf, int in_len,
1285                                 int out_len)
1286 {
1287         struct ib_uverbs_req_notify_cq cmd;
1288         struct ib_cq                  *cq;
1289
1290         if (copy_from_user(&cmd, buf, sizeof cmd))
1291                 return -EFAULT;
1292
1293         cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext);
1294         if (!cq)
1295                 return -EINVAL;
1296
1297         ib_req_notify_cq(cq, cmd.solicited_only ?
1298                          IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
1299
1300         uobj_put_obj_read(cq);
1301
1302         return in_len;
1303 }
1304
1305 ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file,
1306                              struct ib_device *ib_dev,
1307                              const char __user *buf, int in_len,
1308                              int out_len)
1309 {
1310         struct ib_uverbs_destroy_cq      cmd;
1311         struct ib_uverbs_destroy_cq_resp resp;
1312         struct ib_uobject               *uobj;
1313         struct ib_cq                    *cq;
1314         struct ib_ucq_object            *obj;
1315         int                              ret = -EINVAL;
1316
1317         if (copy_from_user(&cmd, buf, sizeof cmd))
1318                 return -EFAULT;
1319
1320         uobj  = uobj_get_write(uobj_get_type(cq), cmd.cq_handle,
1321                                file->ucontext);
1322         if (IS_ERR(uobj))
1323                 return PTR_ERR(uobj);
1324
1325         /*
1326          * Make sure we don't free the memory in remove_commit as we still
1327          * needs the uobject memory to create the response.
1328          */
1329         uverbs_uobject_get(uobj);
1330         cq      = uobj->object;
1331         obj     = container_of(cq->uobject, struct ib_ucq_object, uobject);
1332
1333         memset(&resp, 0, sizeof(resp));
1334
1335         ret = uobj_remove_commit(uobj);
1336         if (ret) {
1337                 uverbs_uobject_put(uobj);
1338                 return ret;
1339         }
1340
1341         resp.comp_events_reported  = obj->comp_events_reported;
1342         resp.async_events_reported = obj->async_events_reported;
1343
1344         uverbs_uobject_put(uobj);
1345         if (copy_to_user((void __user *) (unsigned long) cmd.response,
1346                          &resp, sizeof resp))
1347                 return -EFAULT;
1348
1349         return in_len;
1350 }
1351
1352 static int create_qp(struct ib_uverbs_file *file,
1353                      struct ib_udata *ucore,
1354                      struct ib_udata *uhw,
1355                      struct ib_uverbs_ex_create_qp *cmd,
1356                      size_t cmd_sz,
1357                      int (*cb)(struct ib_uverbs_file *file,
1358                                struct ib_uverbs_ex_create_qp_resp *resp,
1359                                struct ib_udata *udata),
1360                      void *context)
1361 {
1362         struct ib_uqp_object            *obj;
1363         struct ib_device                *device;
1364         struct ib_pd                    *pd = NULL;
1365         struct ib_xrcd                  *xrcd = NULL;
1366         struct ib_uobject               *xrcd_uobj = ERR_PTR(-ENOENT);
1367         struct ib_cq                    *scq = NULL, *rcq = NULL;
1368         struct ib_srq                   *srq = NULL;
1369         struct ib_qp                    *qp;
1370         char                            *buf;
1371         struct ib_qp_init_attr          attr = {};
1372         struct ib_uverbs_ex_create_qp_resp resp;
1373         int                             ret;
1374         struct ib_rwq_ind_table *ind_tbl = NULL;
1375         bool has_sq = true;
1376
1377         if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
1378                 return -EPERM;
1379
1380         obj  = (struct ib_uqp_object *)uobj_alloc(uobj_get_type(qp),
1381                                                   file->ucontext);
1382         if (IS_ERR(obj))
1383                 return PTR_ERR(obj);
1384         obj->uxrcd = NULL;
1385         obj->uevent.uobject.user_handle = cmd->user_handle;
1386         mutex_init(&obj->mcast_lock);
1387
1388         if (cmd_sz >= offsetof(typeof(*cmd), rwq_ind_tbl_handle) +
1389                       sizeof(cmd->rwq_ind_tbl_handle) &&
1390                       (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE)) {
1391                 ind_tbl = uobj_get_obj_read(rwq_ind_table,
1392                                             cmd->rwq_ind_tbl_handle,
1393                                             file->ucontext);
1394                 if (!ind_tbl) {
1395                         ret = -EINVAL;
1396                         goto err_put;
1397                 }
1398
1399                 attr.rwq_ind_tbl = ind_tbl;
1400         }
1401
1402         if (cmd_sz > sizeof(*cmd) &&
1403             !ib_is_udata_cleared(ucore, sizeof(*cmd),
1404                                  cmd_sz - sizeof(*cmd))) {
1405                 ret = -EOPNOTSUPP;
1406                 goto err_put;
1407         }
1408
1409         if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) {
1410                 ret = -EINVAL;
1411                 goto err_put;
1412         }
1413
1414         if (ind_tbl && !cmd->max_send_wr)
1415                 has_sq = false;
1416
1417         if (cmd->qp_type == IB_QPT_XRC_TGT) {
1418                 xrcd_uobj = uobj_get_read(uobj_get_type(xrcd), cmd->pd_handle,
1419                                           file->ucontext);
1420
1421                 if (IS_ERR(xrcd_uobj)) {
1422                         ret = -EINVAL;
1423                         goto err_put;
1424                 }
1425
1426                 xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1427                 if (!xrcd) {
1428                         ret = -EINVAL;
1429                         goto err_put;
1430                 }
1431                 device = xrcd->device;
1432         } else {
1433                 if (cmd->qp_type == IB_QPT_XRC_INI) {
1434                         cmd->max_recv_wr = 0;
1435                         cmd->max_recv_sge = 0;
1436                 } else {
1437                         if (cmd->is_srq) {
1438                                 srq = uobj_get_obj_read(srq, cmd->srq_handle,
1439                                                         file->ucontext);
1440                                 if (!srq || srq->srq_type == IB_SRQT_XRC) {
1441                                         ret = -EINVAL;
1442                                         goto err_put;
1443                                 }
1444                         }
1445
1446                         if (!ind_tbl) {
1447                                 if (cmd->recv_cq_handle != cmd->send_cq_handle) {
1448                                         rcq = uobj_get_obj_read(cq, cmd->recv_cq_handle,
1449                                                                 file->ucontext);
1450                                         if (!rcq) {
1451                                                 ret = -EINVAL;
1452                                                 goto err_put;
1453                                         }
1454                                 }
1455                         }
1456                 }
1457
1458                 if (has_sq)
1459                         scq = uobj_get_obj_read(cq, cmd->send_cq_handle,
1460                                                 file->ucontext);
1461                 if (!ind_tbl)
1462                         rcq = rcq ?: scq;
1463                 pd  = uobj_get_obj_read(pd, cmd->pd_handle, file->ucontext);
1464                 if (!pd || (!scq && has_sq)) {
1465                         ret = -EINVAL;
1466                         goto err_put;
1467                 }
1468
1469                 device = pd->device;
1470         }
1471
1472         attr.event_handler = ib_uverbs_qp_event_handler;
1473         attr.qp_context    = file;
1474         attr.send_cq       = scq;
1475         attr.recv_cq       = rcq;
1476         attr.srq           = srq;
1477         attr.xrcd          = xrcd;
1478         attr.sq_sig_type   = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR :
1479                                               IB_SIGNAL_REQ_WR;
1480         attr.qp_type       = cmd->qp_type;
1481         attr.create_flags  = 0;
1482
1483         attr.cap.max_send_wr     = cmd->max_send_wr;
1484         attr.cap.max_recv_wr     = cmd->max_recv_wr;
1485         attr.cap.max_send_sge    = cmd->max_send_sge;
1486         attr.cap.max_recv_sge    = cmd->max_recv_sge;
1487         attr.cap.max_inline_data = cmd->max_inline_data;
1488
1489         obj->uevent.events_reported     = 0;
1490         INIT_LIST_HEAD(&obj->uevent.event_list);
1491         INIT_LIST_HEAD(&obj->mcast_list);
1492
1493         if (cmd_sz >= offsetof(typeof(*cmd), create_flags) +
1494                       sizeof(cmd->create_flags))
1495                 attr.create_flags = cmd->create_flags;
1496
1497         if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
1498                                 IB_QP_CREATE_CROSS_CHANNEL |
1499                                 IB_QP_CREATE_MANAGED_SEND |
1500                                 IB_QP_CREATE_MANAGED_RECV |
1501                                 IB_QP_CREATE_SCATTER_FCS |
1502                                 IB_QP_CREATE_CVLAN_STRIPPING |
1503                                 IB_QP_CREATE_SOURCE_QPN)) {
1504                 ret = -EINVAL;
1505                 goto err_put;
1506         }
1507
1508         if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {
1509                 if (!capable(CAP_NET_RAW)) {
1510                         ret = -EPERM;
1511                         goto err_put;
1512                 }
1513
1514                 attr.source_qpn = cmd->source_qpn;
1515         }
1516
1517         buf = (void *)cmd + sizeof(*cmd);
1518         if (cmd_sz > sizeof(*cmd))
1519                 if (!(buf[0] == 0 && !memcmp(buf, buf + 1,
1520                                              cmd_sz - sizeof(*cmd) - 1))) {
1521                         ret = -EINVAL;
1522                         goto err_put;
1523                 }
1524
1525         if (cmd->qp_type == IB_QPT_XRC_TGT)
1526                 qp = ib_create_qp(pd, &attr);
1527         else
1528                 qp = device->create_qp(pd, &attr, uhw);
1529
1530         if (IS_ERR(qp)) {
1531                 ret = PTR_ERR(qp);
1532                 goto err_put;
1533         }
1534
1535         if (cmd->qp_type != IB_QPT_XRC_TGT) {
1536                 ret = ib_create_qp_security(qp, device);
1537                 if (ret)
1538                         goto err_cb;
1539
1540                 qp->real_qp       = qp;
1541                 qp->device        = device;
1542                 qp->pd            = pd;
1543                 qp->send_cq       = attr.send_cq;
1544                 qp->recv_cq       = attr.recv_cq;
1545                 qp->srq           = attr.srq;
1546                 qp->rwq_ind_tbl   = ind_tbl;
1547                 qp->event_handler = attr.event_handler;
1548                 qp->qp_context    = attr.qp_context;
1549                 qp->qp_type       = attr.qp_type;
1550                 atomic_set(&qp->usecnt, 0);
1551                 atomic_inc(&pd->usecnt);
1552                 qp->port = 0;
1553                 if (attr.send_cq)
1554                         atomic_inc(&attr.send_cq->usecnt);
1555                 if (attr.recv_cq)
1556                         atomic_inc(&attr.recv_cq->usecnt);
1557                 if (attr.srq)
1558                         atomic_inc(&attr.srq->usecnt);
1559                 if (ind_tbl)
1560                         atomic_inc(&ind_tbl->usecnt);
1561         }
1562         qp->uobject = &obj->uevent.uobject;
1563
1564         obj->uevent.uobject.object = qp;
1565
1566         memset(&resp, 0, sizeof resp);
1567         resp.base.qpn             = qp->qp_num;
1568         resp.base.qp_handle       = obj->uevent.uobject.id;
1569         resp.base.max_recv_sge    = attr.cap.max_recv_sge;
1570         resp.base.max_send_sge    = attr.cap.max_send_sge;
1571         resp.base.max_recv_wr     = attr.cap.max_recv_wr;
1572         resp.base.max_send_wr     = attr.cap.max_send_wr;
1573         resp.base.max_inline_data = attr.cap.max_inline_data;
1574
1575         resp.response_length = offsetof(typeof(resp), response_length) +
1576                                sizeof(resp.response_length);
1577
1578         ret = cb(file, &resp, ucore);
1579         if (ret)
1580                 goto err_cb;
1581
1582         if (xrcd) {
1583                 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1584                                           uobject);
1585                 atomic_inc(&obj->uxrcd->refcnt);
1586                 uobj_put_read(xrcd_uobj);
1587         }
1588
1589         if (pd)
1590                 uobj_put_obj_read(pd);
1591         if (scq)
1592                 uobj_put_obj_read(scq);
1593         if (rcq && rcq != scq)
1594                 uobj_put_obj_read(rcq);
1595         if (srq)
1596                 uobj_put_obj_read(srq);
1597         if (ind_tbl)
1598                 uobj_put_obj_read(ind_tbl);
1599
1600         uobj_alloc_commit(&obj->uevent.uobject);
1601
1602         return 0;
1603 err_cb:
1604         ib_destroy_qp(qp);
1605
1606 err_put:
1607         if (!IS_ERR(xrcd_uobj))
1608                 uobj_put_read(xrcd_uobj);
1609         if (pd)
1610                 uobj_put_obj_read(pd);
1611         if (scq)
1612                 uobj_put_obj_read(scq);
1613         if (rcq && rcq != scq)
1614                 uobj_put_obj_read(rcq);
1615         if (srq)
1616                 uobj_put_obj_read(srq);
1617         if (ind_tbl)
1618                 uobj_put_obj_read(ind_tbl);
1619
1620         uobj_alloc_abort(&obj->uevent.uobject);
1621         return ret;
1622 }
1623
1624 static int ib_uverbs_create_qp_cb(struct ib_uverbs_file *file,
1625                                   struct ib_uverbs_ex_create_qp_resp *resp,
1626                                   struct ib_udata *ucore)
1627 {
1628         if (ib_copy_to_udata(ucore, &resp->base, sizeof(resp->base)))
1629                 return -EFAULT;
1630
1631         return 0;
1632 }
1633
1634 ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
1635                             struct ib_device *ib_dev,
1636                             const char __user *buf, int in_len,
1637                             int out_len)
1638 {
1639         struct ib_uverbs_create_qp      cmd;
1640         struct ib_uverbs_ex_create_qp   cmd_ex;
1641         struct ib_udata                 ucore;
1642         struct ib_udata                 uhw;
1643         ssize_t resp_size = sizeof(struct ib_uverbs_create_qp_resp);
1644         int                             err;
1645
1646         if (out_len < resp_size)
1647                 return -ENOSPC;
1648
1649         if (copy_from_user(&cmd, buf, sizeof(cmd)))
1650                 return -EFAULT;
1651
1652         INIT_UDATA(&ucore, buf, (unsigned long)cmd.response, sizeof(cmd),
1653                    resp_size);
1654         INIT_UDATA(&uhw, buf + sizeof(cmd),
1655                    (unsigned long)cmd.response + resp_size,
1656                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1657                    out_len - resp_size);
1658
1659         memset(&cmd_ex, 0, sizeof(cmd_ex));
1660         cmd_ex.user_handle = cmd.user_handle;
1661         cmd_ex.pd_handle = cmd.pd_handle;
1662         cmd_ex.send_cq_handle = cmd.send_cq_handle;
1663         cmd_ex.recv_cq_handle = cmd.recv_cq_handle;
1664         cmd_ex.srq_handle = cmd.srq_handle;
1665         cmd_ex.max_send_wr = cmd.max_send_wr;
1666         cmd_ex.max_recv_wr = cmd.max_recv_wr;
1667         cmd_ex.max_send_sge = cmd.max_send_sge;
1668         cmd_ex.max_recv_sge = cmd.max_recv_sge;
1669         cmd_ex.max_inline_data = cmd.max_inline_data;
1670         cmd_ex.sq_sig_all = cmd.sq_sig_all;
1671         cmd_ex.qp_type = cmd.qp_type;
1672         cmd_ex.is_srq = cmd.is_srq;
1673
1674         err = create_qp(file, &ucore, &uhw, &cmd_ex,
1675                         offsetof(typeof(cmd_ex), is_srq) +
1676                         sizeof(cmd.is_srq), ib_uverbs_create_qp_cb,
1677                         NULL);
1678
1679         if (err)
1680                 return err;
1681
1682         return in_len;
1683 }
1684
1685 static int ib_uverbs_ex_create_qp_cb(struct ib_uverbs_file *file,
1686                                      struct ib_uverbs_ex_create_qp_resp *resp,
1687                                      struct ib_udata *ucore)
1688 {
1689         if (ib_copy_to_udata(ucore, resp, resp->response_length))
1690                 return -EFAULT;
1691
1692         return 0;
1693 }
1694
1695 int ib_uverbs_ex_create_qp(struct ib_uverbs_file *file,
1696                            struct ib_device *ib_dev,
1697                            struct ib_udata *ucore,
1698                            struct ib_udata *uhw)
1699 {
1700         struct ib_uverbs_ex_create_qp_resp resp;
1701         struct ib_uverbs_ex_create_qp cmd = {0};
1702         int err;
1703
1704         if (ucore->inlen < (offsetof(typeof(cmd), comp_mask) +
1705                             sizeof(cmd.comp_mask)))
1706                 return -EINVAL;
1707
1708         err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
1709         if (err)
1710                 return err;
1711
1712         if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK)
1713                 return -EINVAL;
1714
1715         if (cmd.reserved)
1716                 return -EINVAL;
1717
1718         if (ucore->outlen < (offsetof(typeof(resp), response_length) +
1719                              sizeof(resp.response_length)))
1720                 return -ENOSPC;
1721
1722         err = create_qp(file, ucore, uhw, &cmd,
1723                         min(ucore->inlen, sizeof(cmd)),
1724                         ib_uverbs_ex_create_qp_cb, NULL);
1725
1726         if (err)
1727                 return err;
1728
1729         return 0;
1730 }
1731
1732 ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file,
1733                           struct ib_device *ib_dev,
1734                           const char __user *buf, int in_len, int out_len)
1735 {
1736         struct ib_uverbs_open_qp        cmd;
1737         struct ib_uverbs_create_qp_resp resp;
1738         struct ib_udata                 udata;
1739         struct ib_uqp_object           *obj;
1740         struct ib_xrcd                 *xrcd;
1741         struct ib_uobject              *uninitialized_var(xrcd_uobj);
1742         struct ib_qp                   *qp;
1743         struct ib_qp_open_attr          attr;
1744         int ret;
1745
1746         if (out_len < sizeof resp)
1747                 return -ENOSPC;
1748
1749         if (copy_from_user(&cmd, buf, sizeof cmd))
1750                 return -EFAULT;
1751
1752         INIT_UDATA(&udata, buf + sizeof(cmd),
1753                    (unsigned long) cmd.response + sizeof(resp),
1754                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
1755                    out_len - sizeof(resp));
1756
1757         obj  = (struct ib_uqp_object *)uobj_alloc(uobj_get_type(qp),
1758                                                   file->ucontext);
1759         if (IS_ERR(obj))
1760                 return PTR_ERR(obj);
1761
1762         xrcd_uobj = uobj_get_read(uobj_get_type(xrcd), cmd.pd_handle,
1763                                   file->ucontext);
1764         if (IS_ERR(xrcd_uobj)) {
1765                 ret = -EINVAL;
1766                 goto err_put;
1767         }
1768
1769         xrcd = (struct ib_xrcd *)xrcd_uobj->object;
1770         if (!xrcd) {
1771                 ret = -EINVAL;
1772                 goto err_xrcd;
1773         }
1774
1775         attr.event_handler = ib_uverbs_qp_event_handler;
1776         attr.qp_context    = file;
1777         attr.qp_num        = cmd.qpn;
1778         attr.qp_type       = cmd.qp_type;
1779
1780         obj->uevent.events_reported = 0;
1781         INIT_LIST_HEAD(&obj->uevent.event_list);
1782         INIT_LIST_HEAD(&obj->mcast_list);
1783
1784         qp = ib_open_qp(xrcd, &attr);
1785         if (IS_ERR(qp)) {
1786                 ret = PTR_ERR(qp);
1787                 goto err_xrcd;
1788         }
1789
1790         obj->uevent.uobject.object = qp;
1791         obj->uevent.uobject.user_handle = cmd.user_handle;
1792
1793         memset(&resp, 0, sizeof resp);
1794         resp.qpn       = qp->qp_num;
1795         resp.qp_handle = obj->uevent.uobject.id;
1796
1797         if (copy_to_user((void __user *) (unsigned long) cmd.response,
1798                          &resp, sizeof resp)) {
1799                 ret = -EFAULT;
1800                 goto err_destroy;
1801         }
1802
1803         obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1804         atomic_inc(&obj->uxrcd->refcnt);
1805         qp->uobject = &obj->uevent.uobject;
1806         uobj_put_read(xrcd_uobj);
1807
1808
1809         uobj_alloc_commit(&obj->uevent.uobject);
1810
1811         return in_len;
1812
1813 err_destroy:
1814         ib_destroy_qp(qp);
1815 err_xrcd:
1816         uobj_put_read(xrcd_uobj);
1817 err_put:
1818         uobj_alloc_abort(&obj->uevent.uobject);
1819         return ret;
1820 }
1821
1822 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr,
1823                                    struct rdma_ah_attr *rdma_attr)
1824 {
1825         const struct ib_global_route   *grh;
1826
1827         uverb_attr->dlid              = rdma_ah_get_dlid(rdma_attr);
1828         uverb_attr->sl                = rdma_ah_get_sl(rdma_attr);
1829         uverb_attr->src_path_bits     = rdma_ah_get_path_bits(rdma_attr);
1830         uverb_attr->static_rate       = rdma_ah_get_static_rate(rdma_attr);
1831         uverb_attr->is_global         = !!(rdma_ah_get_ah_flags(rdma_attr) &
1832                                          IB_AH_GRH);
1833         if (uverb_attr->is_global) {
1834                 grh = rdma_ah_read_grh(rdma_attr);
1835                 memcpy(uverb_attr->dgid, grh->dgid.raw, 16);
1836                 uverb_attr->flow_label        = grh->flow_label;
1837                 uverb_attr->sgid_index        = grh->sgid_index;
1838                 uverb_attr->hop_limit         = grh->hop_limit;
1839                 uverb_attr->traffic_class     = grh->traffic_class;
1840         }
1841         uverb_attr->port_num          = rdma_ah_get_port_num(rdma_attr);
1842 }
1843
1844 ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file,
1845                            struct ib_device *ib_dev,
1846                            const char __user *buf, int in_len,
1847                            int out_len)
1848 {
1849         struct ib_uverbs_query_qp      cmd;
1850         struct ib_uverbs_query_qp_resp resp;
1851         struct ib_qp                   *qp;
1852         struct ib_qp_attr              *attr;
1853         struct ib_qp_init_attr         *init_attr;
1854         int                            ret;
1855
1856         if (copy_from_user(&cmd, buf, sizeof cmd))
1857                 return -EFAULT;
1858
1859         attr      = kmalloc(sizeof *attr, GFP_KERNEL);
1860         init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL);
1861         if (!attr || !init_attr) {
1862                 ret = -ENOMEM;
1863                 goto out;
1864         }
1865
1866         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
1867         if (!qp) {
1868                 ret = -EINVAL;
1869                 goto out;
1870         }
1871
1872         ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr);
1873
1874         uobj_put_obj_read(qp);
1875
1876         if (ret)
1877                 goto out;
1878
1879         memset(&resp, 0, sizeof resp);
1880
1881         resp.qp_state               = attr->qp_state;
1882         resp.cur_qp_state           = attr->cur_qp_state;
1883         resp.path_mtu               = attr->path_mtu;
1884         resp.path_mig_state         = attr->path_mig_state;
1885         resp.qkey                   = attr->qkey;
1886         resp.rq_psn                 = attr->rq_psn;
1887         resp.sq_psn                 = attr->sq_psn;
1888         resp.dest_qp_num            = attr->dest_qp_num;
1889         resp.qp_access_flags        = attr->qp_access_flags;
1890         resp.pkey_index             = attr->pkey_index;
1891         resp.alt_pkey_index         = attr->alt_pkey_index;
1892         resp.sq_draining            = attr->sq_draining;
1893         resp.max_rd_atomic          = attr->max_rd_atomic;
1894         resp.max_dest_rd_atomic     = attr->max_dest_rd_atomic;
1895         resp.min_rnr_timer          = attr->min_rnr_timer;
1896         resp.port_num               = attr->port_num;
1897         resp.timeout                = attr->timeout;
1898         resp.retry_cnt              = attr->retry_cnt;
1899         resp.rnr_retry              = attr->rnr_retry;
1900         resp.alt_port_num           = attr->alt_port_num;
1901         resp.alt_timeout            = attr->alt_timeout;
1902
1903         copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr);
1904         copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr);
1905
1906         resp.max_send_wr            = init_attr->cap.max_send_wr;
1907         resp.max_recv_wr            = init_attr->cap.max_recv_wr;
1908         resp.max_send_sge           = init_attr->cap.max_send_sge;
1909         resp.max_recv_sge           = init_attr->cap.max_recv_sge;
1910         resp.max_inline_data        = init_attr->cap.max_inline_data;
1911         resp.sq_sig_all             = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1912
1913         if (copy_to_user((void __user *) (unsigned long) cmd.response,
1914                          &resp, sizeof resp))
1915                 ret = -EFAULT;
1916
1917 out:
1918         kfree(attr);
1919         kfree(init_attr);
1920
1921         return ret ? ret : in_len;
1922 }
1923
1924 /* Remove ignored fields set in the attribute mask */
1925 static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
1926 {
1927         switch (qp_type) {
1928         case IB_QPT_XRC_INI:
1929                 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
1930         case IB_QPT_XRC_TGT:
1931                 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
1932                                 IB_QP_RNR_RETRY);
1933         default:
1934                 return mask;
1935         }
1936 }
1937
1938 static void copy_ah_attr_from_uverbs(struct ib_device *dev,
1939                                      struct rdma_ah_attr *rdma_attr,
1940                                      struct ib_uverbs_qp_dest *uverb_attr)
1941 {
1942         rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num);
1943         if (uverb_attr->is_global) {
1944                 rdma_ah_set_grh(rdma_attr, NULL,
1945                                 uverb_attr->flow_label,
1946                                 uverb_attr->sgid_index,
1947                                 uverb_attr->hop_limit,
1948                                 uverb_attr->traffic_class);
1949                 rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid);
1950         } else {
1951                 rdma_ah_set_ah_flags(rdma_attr, 0);
1952         }
1953         rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid);
1954         rdma_ah_set_sl(rdma_attr, uverb_attr->sl);
1955         rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits);
1956         rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate);
1957         rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num);
1958         rdma_ah_set_make_grd(rdma_attr, false);
1959 }
1960
1961 static int modify_qp(struct ib_uverbs_file *file,
1962                      struct ib_uverbs_ex_modify_qp *cmd, struct ib_udata *udata)
1963 {
1964         struct ib_qp_attr *attr;
1965         struct ib_qp *qp;
1966         int ret;
1967
1968         attr = kmalloc(sizeof *attr, GFP_KERNEL);
1969         if (!attr)
1970                 return -ENOMEM;
1971
1972         qp = uobj_get_obj_read(qp, cmd->base.qp_handle, file->ucontext);
1973         if (!qp) {
1974                 ret = -EINVAL;
1975                 goto out;
1976         }
1977
1978         if ((cmd->base.attr_mask & IB_QP_PORT) &&
1979             !rdma_is_port_valid(qp->device, cmd->base.port_num)) {
1980                 ret = -EINVAL;
1981                 goto release_qp;
1982         }
1983
1984         if ((cmd->base.attr_mask & IB_QP_AV)) {
1985                 if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) {
1986                         ret = -EINVAL;
1987                         goto release_qp;
1988                 }
1989
1990                 if (cmd->base.attr_mask & IB_QP_STATE &&
1991                     cmd->base.qp_state == IB_QPS_RTR) {
1992                 /* We are in INIT->RTR TRANSITION (if we are not,
1993                  * this transition will be rejected in subsequent checks).
1994                  * In the INIT->RTR transition, we cannot have IB_QP_PORT set,
1995                  * but the IB_QP_STATE flag is required.
1996                  *
1997                  * Since kernel 3.14 (commit dbf727de7440), the uverbs driver,
1998                  * when IB_QP_AV is set, has required inclusion of a valid
1999                  * port number in the primary AV. (AVs are created and handled
2000                  * differently for infiniband and ethernet (RoCE) ports).
2001                  *
2002                  * Check the port number included in the primary AV against
2003                  * the port number in the qp struct, which was set (and saved)
2004                  * in the RST->INIT transition.
2005                  */
2006                         if (cmd->base.dest.port_num != qp->real_qp->port) {
2007                                 ret = -EINVAL;
2008                                 goto release_qp;
2009                         }
2010                 } else {
2011                 /* We are in SQD->SQD. (If we are not, this transition will
2012                  * be rejected later in the verbs layer checks).
2013                  * Check for both IB_QP_PORT and IB_QP_AV, these can be set
2014                  * together in the SQD->SQD transition.
2015                  *
2016                  * If only IP_QP_AV was set, add in IB_QP_PORT as well (the
2017                  * verbs layer driver does not track primary port changes
2018                  * resulting from path migration. Thus, in SQD, if the primary
2019                  * AV is modified, the primary port should also be modified).
2020                  *
2021                  * Note that in this transition, the IB_QP_STATE flag
2022                  * is not allowed.
2023                  */
2024                         if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
2025                              == (IB_QP_AV | IB_QP_PORT)) &&
2026                             cmd->base.port_num != cmd->base.dest.port_num) {
2027                                 ret = -EINVAL;
2028                                 goto release_qp;
2029                         }
2030                         if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT))
2031                             == IB_QP_AV) {
2032                                 cmd->base.attr_mask |= IB_QP_PORT;
2033                                 cmd->base.port_num = cmd->base.dest.port_num;
2034                         }
2035                 }
2036         }
2037
2038         if ((cmd->base.attr_mask & IB_QP_ALT_PATH) &&
2039             (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) ||
2040             !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) ||
2041             cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) {
2042                 ret = -EINVAL;
2043                 goto release_qp;
2044         }
2045
2046         attr->qp_state            = cmd->base.qp_state;
2047         attr->cur_qp_state        = cmd->base.cur_qp_state;
2048         attr->path_mtu            = cmd->base.path_mtu;
2049         attr->path_mig_state      = cmd->base.path_mig_state;
2050         attr->qkey                = cmd->base.qkey;
2051         attr->rq_psn              = cmd->base.rq_psn;
2052         attr->sq_psn              = cmd->base.sq_psn;
2053         attr->dest_qp_num         = cmd->base.dest_qp_num;
2054         attr->qp_access_flags     = cmd->base.qp_access_flags;
2055         attr->pkey_index          = cmd->base.pkey_index;
2056         attr->alt_pkey_index      = cmd->base.alt_pkey_index;
2057         attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify;
2058         attr->max_rd_atomic       = cmd->base.max_rd_atomic;
2059         attr->max_dest_rd_atomic  = cmd->base.max_dest_rd_atomic;
2060         attr->min_rnr_timer       = cmd->base.min_rnr_timer;
2061         attr->port_num            = cmd->base.port_num;
2062         attr->timeout             = cmd->base.timeout;
2063         attr->retry_cnt           = cmd->base.retry_cnt;
2064         attr->rnr_retry           = cmd->base.rnr_retry;
2065         attr->alt_port_num        = cmd->base.alt_port_num;
2066         attr->alt_timeout         = cmd->base.alt_timeout;
2067         attr->rate_limit          = cmd->rate_limit;
2068
2069         if (cmd->base.attr_mask & IB_QP_AV)
2070                 copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr,
2071                                          &cmd->base.dest);
2072
2073         if (cmd->base.attr_mask & IB_QP_ALT_PATH)
2074                 copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr,
2075                                          &cmd->base.alt_dest);
2076
2077         ret = ib_modify_qp_with_udata(qp, attr,
2078                                       modify_qp_mask(qp->qp_type,
2079                                                      cmd->base.attr_mask),
2080                                       udata);
2081
2082 release_qp:
2083         uobj_put_obj_read(qp);
2084 out:
2085         kfree(attr);
2086
2087         return ret;
2088 }
2089
2090 ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
2091                             struct ib_device *ib_dev,
2092                             const char __user *buf, int in_len,
2093                             int out_len)
2094 {
2095         struct ib_uverbs_ex_modify_qp cmd = {};
2096         struct ib_udata udata;
2097         int ret;
2098
2099         if (copy_from_user(&cmd.base, buf, sizeof(cmd.base)))
2100                 return -EFAULT;
2101
2102         if (cmd.base.attr_mask &
2103             ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1))
2104                 return -EOPNOTSUPP;
2105
2106         INIT_UDATA(&udata, buf + sizeof(cmd.base), NULL,
2107                    in_len - sizeof(cmd.base) - sizeof(struct ib_uverbs_cmd_hdr),
2108                    out_len);
2109
2110         ret = modify_qp(file, &cmd, &udata);
2111         if (ret)
2112                 return ret;
2113
2114         return in_len;
2115 }
2116
2117 int ib_uverbs_ex_modify_qp(struct ib_uverbs_file *file,
2118                            struct ib_device *ib_dev,
2119                            struct ib_udata *ucore,
2120                            struct ib_udata *uhw)
2121 {
2122         struct ib_uverbs_ex_modify_qp cmd = {};
2123         int ret;
2124
2125         /*
2126          * Last bit is reserved for extending the attr_mask by
2127          * using another field.
2128          */
2129         BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31));
2130
2131         if (ucore->inlen < sizeof(cmd.base))
2132                 return -EINVAL;
2133
2134         ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
2135         if (ret)
2136                 return ret;
2137
2138         if (cmd.base.attr_mask &
2139             ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1))
2140                 return -EOPNOTSUPP;
2141
2142         if (ucore->inlen > sizeof(cmd)) {
2143                 if (!ib_is_udata_cleared(ucore, sizeof(cmd),
2144                                          ucore->inlen - sizeof(cmd)))
2145                         return -EOPNOTSUPP;
2146         }
2147
2148         ret = modify_qp(file, &cmd, uhw);
2149
2150         return ret;
2151 }
2152
2153 ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
2154                              struct ib_device *ib_dev,
2155                              const char __user *buf, int in_len,
2156                              int out_len)
2157 {
2158         struct ib_uverbs_destroy_qp      cmd;
2159         struct ib_uverbs_destroy_qp_resp resp;
2160         struct ib_uobject               *uobj;
2161         struct ib_uqp_object            *obj;
2162         int                              ret = -EINVAL;
2163
2164         if (copy_from_user(&cmd, buf, sizeof cmd))
2165                 return -EFAULT;
2166
2167         memset(&resp, 0, sizeof resp);
2168
2169         uobj  = uobj_get_write(uobj_get_type(qp), cmd.qp_handle,
2170                                file->ucontext);
2171         if (IS_ERR(uobj))
2172                 return PTR_ERR(uobj);
2173
2174         obj = container_of(uobj, struct ib_uqp_object, uevent.uobject);
2175         /*
2176          * Make sure we don't free the memory in remove_commit as we still
2177          * needs the uobject memory to create the response.
2178          */
2179         uverbs_uobject_get(uobj);
2180
2181         ret = uobj_remove_commit(uobj);
2182         if (ret) {
2183                 uverbs_uobject_put(uobj);
2184                 return ret;
2185         }
2186
2187         resp.events_reported = obj->uevent.events_reported;
2188         uverbs_uobject_put(uobj);
2189
2190         if (copy_to_user((void __user *) (unsigned long) cmd.response,
2191                          &resp, sizeof resp))
2192                 return -EFAULT;
2193
2194         return in_len;
2195 }
2196
2197 static void *alloc_wr(size_t wr_size, __u32 num_sge)
2198 {
2199         if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) /
2200                        sizeof (struct ib_sge))
2201                 return NULL;
2202
2203         return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) +
2204                          num_sge * sizeof (struct ib_sge), GFP_KERNEL);
2205 }
2206
2207 ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file,
2208                             struct ib_device *ib_dev,
2209                             const char __user *buf, int in_len,
2210                             int out_len)
2211 {
2212         struct ib_uverbs_post_send      cmd;
2213         struct ib_uverbs_post_send_resp resp;
2214         struct ib_uverbs_send_wr       *user_wr;
2215         struct ib_send_wr              *wr = NULL, *last, *next, *bad_wr;
2216         struct ib_qp                   *qp;
2217         int                             i, sg_ind;
2218         int                             is_ud;
2219         ssize_t                         ret = -EINVAL;
2220         size_t                          next_size;
2221
2222         if (copy_from_user(&cmd, buf, sizeof cmd))
2223                 return -EFAULT;
2224
2225         if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count +
2226             cmd.sge_count * sizeof (struct ib_uverbs_sge))
2227                 return -EINVAL;
2228
2229         if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr))
2230                 return -EINVAL;
2231
2232         user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL);
2233         if (!user_wr)
2234                 return -ENOMEM;
2235
2236         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
2237         if (!qp)
2238                 goto out;
2239
2240         is_ud = qp->qp_type == IB_QPT_UD;
2241         sg_ind = 0;
2242         last = NULL;
2243         for (i = 0; i < cmd.wr_count; ++i) {
2244                 if (copy_from_user(user_wr,
2245                                    buf + sizeof cmd + i * cmd.wqe_size,
2246                                    cmd.wqe_size)) {
2247                         ret = -EFAULT;
2248                         goto out_put;
2249                 }
2250
2251                 if (user_wr->num_sge + sg_ind > cmd.sge_count) {
2252                         ret = -EINVAL;
2253                         goto out_put;
2254                 }
2255
2256                 if (is_ud) {
2257                         struct ib_ud_wr *ud;
2258
2259                         if (user_wr->opcode != IB_WR_SEND &&
2260                             user_wr->opcode != IB_WR_SEND_WITH_IMM) {
2261                                 ret = -EINVAL;
2262                                 goto out_put;
2263                         }
2264
2265                         next_size = sizeof(*ud);
2266                         ud = alloc_wr(next_size, user_wr->num_sge);
2267                         if (!ud) {
2268                                 ret = -ENOMEM;
2269                                 goto out_put;
2270                         }
2271
2272                         ud->ah = uobj_get_obj_read(ah, user_wr->wr.ud.ah,
2273                                                    file->ucontext);
2274                         if (!ud->ah) {
2275                                 kfree(ud);
2276                                 ret = -EINVAL;
2277                                 goto out_put;
2278                         }
2279                         ud->remote_qpn = user_wr->wr.ud.remote_qpn;
2280                         ud->remote_qkey = user_wr->wr.ud.remote_qkey;
2281
2282                         next = &ud->wr;
2283                 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2284                            user_wr->opcode == IB_WR_RDMA_WRITE ||
2285                            user_wr->opcode == IB_WR_RDMA_READ) {
2286                         struct ib_rdma_wr *rdma;
2287
2288                         next_size = sizeof(*rdma);
2289                         rdma = alloc_wr(next_size, user_wr->num_sge);
2290                         if (!rdma) {
2291                                 ret = -ENOMEM;
2292                                 goto out_put;
2293                         }
2294
2295                         rdma->remote_addr = user_wr->wr.rdma.remote_addr;
2296                         rdma->rkey = user_wr->wr.rdma.rkey;
2297
2298                         next = &rdma->wr;
2299                 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2300                            user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2301                         struct ib_atomic_wr *atomic;
2302
2303                         next_size = sizeof(*atomic);
2304                         atomic = alloc_wr(next_size, user_wr->num_sge);
2305                         if (!atomic) {
2306                                 ret = -ENOMEM;
2307                                 goto out_put;
2308                         }
2309
2310                         atomic->remote_addr = user_wr->wr.atomic.remote_addr;
2311                         atomic->compare_add = user_wr->wr.atomic.compare_add;
2312                         atomic->swap = user_wr->wr.atomic.swap;
2313                         atomic->rkey = user_wr->wr.atomic.rkey;
2314
2315                         next = &atomic->wr;
2316                 } else if (user_wr->opcode == IB_WR_SEND ||
2317                            user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2318                            user_wr->opcode == IB_WR_SEND_WITH_INV) {
2319                         next_size = sizeof(*next);
2320                         next = alloc_wr(next_size, user_wr->num_sge);
2321                         if (!next) {
2322                                 ret = -ENOMEM;
2323                                 goto out_put;
2324                         }
2325                 } else {
2326                         ret = -EINVAL;
2327                         goto out_put;
2328                 }
2329
2330                 if (user_wr->opcode == IB_WR_SEND_WITH_IMM ||
2331                     user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
2332                         next->ex.imm_data =
2333                                         (__be32 __force) user_wr->ex.imm_data;
2334                 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) {
2335                         next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey;
2336                 }
2337
2338                 if (!last)
2339                         wr = next;
2340                 else
2341                         last->next = next;
2342                 last = next;
2343
2344                 next->next       = NULL;
2345                 next->wr_id      = user_wr->wr_id;
2346                 next->num_sge    = user_wr->num_sge;
2347                 next->opcode     = user_wr->opcode;
2348                 next->send_flags = user_wr->send_flags;
2349
2350                 if (next->num_sge) {
2351                         next->sg_list = (void *) next +
2352                                 ALIGN(next_size, sizeof(struct ib_sge));
2353                         if (copy_from_user(next->sg_list,
2354                                            buf + sizeof cmd +
2355                                            cmd.wr_count * cmd.wqe_size +
2356                                            sg_ind * sizeof (struct ib_sge),
2357                                            next->num_sge * sizeof (struct ib_sge))) {
2358                                 ret = -EFAULT;
2359                                 goto out_put;
2360                         }
2361                         sg_ind += next->num_sge;
2362                 } else
2363                         next->sg_list = NULL;
2364         }
2365
2366         resp.bad_wr = 0;
2367         ret = qp->device->post_send(qp->real_qp, wr, &bad_wr);
2368         if (ret)
2369                 for (next = wr; next; next = next->next) {
2370                         ++resp.bad_wr;
2371                         if (next == bad_wr)
2372                                 break;
2373                 }
2374
2375         if (copy_to_user((void __user *) (unsigned long) cmd.response,
2376                          &resp, sizeof resp))
2377                 ret = -EFAULT;
2378
2379 out_put:
2380         uobj_put_obj_read(qp);
2381
2382         while (wr) {
2383                 if (is_ud && ud_wr(wr)->ah)
2384                         uobj_put_obj_read(ud_wr(wr)->ah);
2385                 next = wr->next;
2386                 kfree(wr);
2387                 wr = next;
2388         }
2389
2390 out:
2391         kfree(user_wr);
2392
2393         return ret ? ret : in_len;
2394 }
2395
2396 static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf,
2397                                                     int in_len,
2398                                                     u32 wr_count,
2399                                                     u32 sge_count,
2400                                                     u32 wqe_size)
2401 {
2402         struct ib_uverbs_recv_wr *user_wr;
2403         struct ib_recv_wr        *wr = NULL, *last, *next;
2404         int                       sg_ind;
2405         int                       i;
2406         int                       ret;
2407
2408         if (in_len < wqe_size * wr_count +
2409             sge_count * sizeof (struct ib_uverbs_sge))
2410                 return ERR_PTR(-EINVAL);
2411
2412         if (wqe_size < sizeof (struct ib_uverbs_recv_wr))
2413                 return ERR_PTR(-EINVAL);
2414
2415         user_wr = kmalloc(wqe_size, GFP_KERNEL);
2416         if (!user_wr)
2417                 return ERR_PTR(-ENOMEM);
2418
2419         sg_ind = 0;
2420         last = NULL;
2421         for (i = 0; i < wr_count; ++i) {
2422                 if (copy_from_user(user_wr, buf + i * wqe_size,
2423                                    wqe_size)) {
2424                         ret = -EFAULT;
2425                         goto err;
2426                 }
2427
2428                 if (user_wr->num_sge + sg_ind > sge_count) {
2429                         ret = -EINVAL;
2430                         goto err;
2431                 }
2432
2433                 if (user_wr->num_sge >=
2434                     (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) /
2435                     sizeof (struct ib_sge)) {
2436                         ret = -EINVAL;
2437                         goto err;
2438                 }
2439
2440                 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) +
2441                                user_wr->num_sge * sizeof (struct ib_sge),
2442                                GFP_KERNEL);
2443                 if (!next) {
2444                         ret = -ENOMEM;
2445                         goto err;
2446                 }
2447
2448                 if (!last)
2449                         wr = next;
2450                 else
2451                         last->next = next;
2452                 last = next;
2453
2454                 next->next       = NULL;
2455                 next->wr_id      = user_wr->wr_id;
2456                 next->num_sge    = user_wr->num_sge;
2457
2458                 if (next->num_sge) {
2459                         next->sg_list = (void *) next +
2460                                 ALIGN(sizeof *next, sizeof (struct ib_sge));
2461                         if (copy_from_user(next->sg_list,
2462                                            buf + wr_count * wqe_size +
2463                                            sg_ind * sizeof (struct ib_sge),
2464                                            next->num_sge * sizeof (struct ib_sge))) {
2465                                 ret = -EFAULT;
2466                                 goto err;
2467                         }
2468                         sg_ind += next->num_sge;
2469                 } else
2470                         next->sg_list = NULL;
2471         }
2472
2473         kfree(user_wr);
2474         return wr;
2475
2476 err:
2477         kfree(user_wr);
2478
2479         while (wr) {
2480                 next = wr->next;
2481                 kfree(wr);
2482                 wr = next;
2483         }
2484
2485         return ERR_PTR(ret);
2486 }
2487
2488 ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file,
2489                             struct ib_device *ib_dev,
2490                             const char __user *buf, int in_len,
2491                             int out_len)
2492 {
2493         struct ib_uverbs_post_recv      cmd;
2494         struct ib_uverbs_post_recv_resp resp;
2495         struct ib_recv_wr              *wr, *next, *bad_wr;
2496         struct ib_qp                   *qp;
2497         ssize_t                         ret = -EINVAL;
2498
2499         if (copy_from_user(&cmd, buf, sizeof cmd))
2500                 return -EFAULT;
2501
2502         wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2503                                        in_len - sizeof cmd, cmd.wr_count,
2504                                        cmd.sge_count, cmd.wqe_size);
2505         if (IS_ERR(wr))
2506                 return PTR_ERR(wr);
2507
2508         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
2509         if (!qp)
2510                 goto out;
2511
2512         resp.bad_wr = 0;
2513         ret = qp->device->post_recv(qp->real_qp, wr, &bad_wr);
2514
2515         uobj_put_obj_read(qp);
2516         if (ret) {
2517                 for (next = wr; next; next = next->next) {
2518                         ++resp.bad_wr;
2519                         if (next == bad_wr)
2520                                 break;
2521                 }
2522         }
2523
2524         if (copy_to_user((void __user *) (unsigned long) cmd.response,
2525                          &resp, sizeof resp))
2526                 ret = -EFAULT;
2527
2528 out:
2529         while (wr) {
2530                 next = wr->next;
2531                 kfree(wr);
2532                 wr = next;
2533         }
2534
2535         return ret ? ret : in_len;
2536 }
2537
2538 ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file,
2539                                 struct ib_device *ib_dev,
2540                                 const char __user *buf, int in_len,
2541                                 int out_len)
2542 {
2543         struct ib_uverbs_post_srq_recv      cmd;
2544         struct ib_uverbs_post_srq_recv_resp resp;
2545         struct ib_recv_wr                  *wr, *next, *bad_wr;
2546         struct ib_srq                      *srq;
2547         ssize_t                             ret = -EINVAL;
2548
2549         if (copy_from_user(&cmd, buf, sizeof cmd))
2550                 return -EFAULT;
2551
2552         wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd,
2553                                        in_len - sizeof cmd, cmd.wr_count,
2554                                        cmd.sge_count, cmd.wqe_size);
2555         if (IS_ERR(wr))
2556                 return PTR_ERR(wr);
2557
2558         srq = uobj_get_obj_read(srq, cmd.srq_handle, file->ucontext);
2559         if (!srq)
2560                 goto out;
2561
2562         resp.bad_wr = 0;
2563         ret = srq->device->post_srq_recv(srq, wr, &bad_wr);
2564
2565         uobj_put_obj_read(srq);
2566
2567         if (ret)
2568                 for (next = wr; next; next = next->next) {
2569                         ++resp.bad_wr;
2570                         if (next == bad_wr)
2571                                 break;
2572                 }
2573
2574         if (copy_to_user((void __user *) (unsigned long) cmd.response,
2575                          &resp, sizeof resp))
2576                 ret = -EFAULT;
2577
2578 out:
2579         while (wr) {
2580                 next = wr->next;
2581                 kfree(wr);
2582                 wr = next;
2583         }
2584
2585         return ret ? ret : in_len;
2586 }
2587
2588 ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
2589                             struct ib_device *ib_dev,
2590                             const char __user *buf, int in_len,
2591                             int out_len)
2592 {
2593         struct ib_uverbs_create_ah       cmd;
2594         struct ib_uverbs_create_ah_resp  resp;
2595         struct ib_uobject               *uobj;
2596         struct ib_pd                    *pd;
2597         struct ib_ah                    *ah;
2598         struct rdma_ah_attr             attr;
2599         int ret;
2600         struct ib_udata                   udata;
2601         u8                              *dmac;
2602
2603         if (out_len < sizeof resp)
2604                 return -ENOSPC;
2605
2606         if (copy_from_user(&cmd, buf, sizeof cmd))
2607                 return -EFAULT;
2608
2609         if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num))
2610                 return -EINVAL;
2611
2612         INIT_UDATA(&udata, buf + sizeof(cmd),
2613                    (unsigned long)cmd.response + sizeof(resp),
2614                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
2615                    out_len - sizeof(resp));
2616
2617         uobj  = uobj_alloc(uobj_get_type(ah), file->ucontext);
2618         if (IS_ERR(uobj))
2619                 return PTR_ERR(uobj);
2620
2621         pd = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext);
2622         if (!pd) {
2623                 ret = -EINVAL;
2624                 goto err;
2625         }
2626
2627         attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num);
2628         rdma_ah_set_make_grd(&attr, false);
2629         rdma_ah_set_dlid(&attr, cmd.attr.dlid);
2630         rdma_ah_set_sl(&attr, cmd.attr.sl);
2631         rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits);
2632         rdma_ah_set_static_rate(&attr, cmd.attr.static_rate);
2633         rdma_ah_set_port_num(&attr, cmd.attr.port_num);
2634
2635         if (cmd.attr.is_global) {
2636                 rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label,
2637                                 cmd.attr.grh.sgid_index,
2638                                 cmd.attr.grh.hop_limit,
2639                                 cmd.attr.grh.traffic_class);
2640                 rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid);
2641         } else {
2642                 rdma_ah_set_ah_flags(&attr, 0);
2643         }
2644         dmac = rdma_ah_retrieve_dmac(&attr);
2645         if (dmac)
2646                 memset(dmac, 0, ETH_ALEN);
2647
2648         ah = pd->device->create_ah(pd, &attr, &udata);
2649
2650         if (IS_ERR(ah)) {
2651                 ret = PTR_ERR(ah);
2652                 goto err_put;
2653         }
2654
2655         ah->device  = pd->device;
2656         ah->pd      = pd;
2657         atomic_inc(&pd->usecnt);
2658         ah->uobject  = uobj;
2659         uobj->user_handle = cmd.user_handle;
2660         uobj->object = ah;
2661
2662         resp.ah_handle = uobj->id;
2663
2664         if (copy_to_user((void __user *) (unsigned long) cmd.response,
2665                          &resp, sizeof resp)) {
2666                 ret = -EFAULT;
2667                 goto err_copy;
2668         }
2669
2670         uobj_put_obj_read(pd);
2671         uobj_alloc_commit(uobj);
2672
2673         return in_len;
2674
2675 err_copy:
2676         rdma_destroy_ah(ah);
2677
2678 err_put:
2679         uobj_put_obj_read(pd);
2680
2681 err:
2682         uobj_alloc_abort(uobj);
2683         return ret;
2684 }
2685
2686 ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file,
2687                              struct ib_device *ib_dev,
2688                              const char __user *buf, int in_len, int out_len)
2689 {
2690         struct ib_uverbs_destroy_ah cmd;
2691         struct ib_uobject          *uobj;
2692         int                         ret;
2693
2694         if (copy_from_user(&cmd, buf, sizeof cmd))
2695                 return -EFAULT;
2696
2697         uobj  = uobj_get_write(uobj_get_type(ah), cmd.ah_handle,
2698                                file->ucontext);
2699         if (IS_ERR(uobj))
2700                 return PTR_ERR(uobj);
2701
2702         ret = uobj_remove_commit(uobj);
2703         return ret ?: in_len;
2704 }
2705
2706 ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file,
2707                                struct ib_device *ib_dev,
2708                                const char __user *buf, int in_len,
2709                                int out_len)
2710 {
2711         struct ib_uverbs_attach_mcast cmd;
2712         struct ib_qp                 *qp;
2713         struct ib_uqp_object         *obj;
2714         struct ib_uverbs_mcast_entry *mcast;
2715         int                           ret;
2716
2717         if (copy_from_user(&cmd, buf, sizeof cmd))
2718                 return -EFAULT;
2719
2720         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
2721         if (!qp)
2722                 return -EINVAL;
2723
2724         obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2725
2726         mutex_lock(&obj->mcast_lock);
2727         list_for_each_entry(mcast, &obj->mcast_list, list)
2728                 if (cmd.mlid == mcast->lid &&
2729                     !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2730                         ret = 0;
2731                         goto out_put;
2732                 }
2733
2734         mcast = kmalloc(sizeof *mcast, GFP_KERNEL);
2735         if (!mcast) {
2736                 ret = -ENOMEM;
2737                 goto out_put;
2738         }
2739
2740         mcast->lid = cmd.mlid;
2741         memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw);
2742
2743         ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid);
2744         if (!ret)
2745                 list_add_tail(&mcast->list, &obj->mcast_list);
2746         else
2747                 kfree(mcast);
2748
2749 out_put:
2750         mutex_unlock(&obj->mcast_lock);
2751         uobj_put_obj_read(qp);
2752
2753         return ret ? ret : in_len;
2754 }
2755
2756 ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file,
2757                                struct ib_device *ib_dev,
2758                                const char __user *buf, int in_len,
2759                                int out_len)
2760 {
2761         struct ib_uverbs_detach_mcast cmd;
2762         struct ib_uqp_object         *obj;
2763         struct ib_qp                 *qp;
2764         struct ib_uverbs_mcast_entry *mcast;
2765         int                           ret = -EINVAL;
2766         bool                          found = false;
2767
2768         if (copy_from_user(&cmd, buf, sizeof cmd))
2769                 return -EFAULT;
2770
2771         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
2772         if (!qp)
2773                 return -EINVAL;
2774
2775         obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject);
2776         mutex_lock(&obj->mcast_lock);
2777
2778         list_for_each_entry(mcast, &obj->mcast_list, list)
2779                 if (cmd.mlid == mcast->lid &&
2780                     !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) {
2781                         list_del(&mcast->list);
2782                         kfree(mcast);
2783                         found = true;
2784                         break;
2785                 }
2786
2787         if (!found) {
2788                 ret = -EINVAL;
2789                 goto out_put;
2790         }
2791
2792         ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid);
2793
2794 out_put:
2795         mutex_unlock(&obj->mcast_lock);
2796         uobj_put_obj_read(qp);
2797         return ret ? ret : in_len;
2798 }
2799
2800 static int kern_spec_to_ib_spec_action(struct ib_uverbs_flow_spec *kern_spec,
2801                                        union ib_flow_spec *ib_spec)
2802 {
2803         ib_spec->type = kern_spec->type;
2804         switch (ib_spec->type) {
2805         case IB_FLOW_SPEC_ACTION_TAG:
2806                 if (kern_spec->flow_tag.size !=
2807                     sizeof(struct ib_uverbs_flow_spec_action_tag))
2808                         return -EINVAL;
2809
2810                 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag);
2811                 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id;
2812                 break;
2813         case IB_FLOW_SPEC_ACTION_DROP:
2814                 if (kern_spec->drop.size !=
2815                     sizeof(struct ib_uverbs_flow_spec_action_drop))
2816                         return -EINVAL;
2817
2818                 ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop);
2819                 break;
2820         default:
2821                 return -EINVAL;
2822         }
2823         return 0;
2824 }
2825
2826 static size_t kern_spec_filter_sz(struct ib_uverbs_flow_spec_hdr *spec)
2827 {
2828         /* Returns user space filter size, includes padding */
2829         return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2;
2830 }
2831
2832 static ssize_t spec_filter_size(void *kern_spec_filter, u16 kern_filter_size,
2833                                 u16 ib_real_filter_sz)
2834 {
2835         /*
2836          * User space filter structures must be 64 bit aligned, otherwise this
2837          * may pass, but we won't handle additional new attributes.
2838          */
2839
2840         if (kern_filter_size > ib_real_filter_sz) {
2841                 if (memchr_inv(kern_spec_filter +
2842                                ib_real_filter_sz, 0,
2843                                kern_filter_size - ib_real_filter_sz))
2844                         return -EINVAL;
2845                 return ib_real_filter_sz;
2846         }
2847         return kern_filter_size;
2848 }
2849
2850 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec,
2851                                        union ib_flow_spec *ib_spec)
2852 {
2853         ssize_t actual_filter_sz;
2854         ssize_t kern_filter_sz;
2855         ssize_t ib_filter_sz;
2856         void *kern_spec_mask;
2857         void *kern_spec_val;
2858
2859         if (kern_spec->reserved)
2860                 return -EINVAL;
2861
2862         ib_spec->type = kern_spec->type;
2863
2864         kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr);
2865         /* User flow spec size must be aligned to 4 bytes */
2866         if (kern_filter_sz != ALIGN(kern_filter_sz, 4))
2867                 return -EINVAL;
2868
2869         kern_spec_val = (void *)kern_spec +
2870                 sizeof(struct ib_uverbs_flow_spec_hdr);
2871         kern_spec_mask = kern_spec_val + kern_filter_sz;
2872         if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL))
2873                 return -EINVAL;
2874
2875         switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
2876         case IB_FLOW_SPEC_ETH:
2877                 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz);
2878                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2879                                                     kern_filter_sz,
2880                                                     ib_filter_sz);
2881                 if (actual_filter_sz <= 0)
2882                         return -EINVAL;
2883                 ib_spec->size = sizeof(struct ib_flow_spec_eth);
2884                 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz);
2885                 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz);
2886                 break;
2887         case IB_FLOW_SPEC_IPV4:
2888                 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz);
2889                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2890                                                     kern_filter_sz,
2891                                                     ib_filter_sz);
2892                 if (actual_filter_sz <= 0)
2893                         return -EINVAL;
2894                 ib_spec->size = sizeof(struct ib_flow_spec_ipv4);
2895                 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz);
2896                 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz);
2897                 break;
2898         case IB_FLOW_SPEC_IPV6:
2899                 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz);
2900                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2901                                                     kern_filter_sz,
2902                                                     ib_filter_sz);
2903                 if (actual_filter_sz <= 0)
2904                         return -EINVAL;
2905                 ib_spec->size = sizeof(struct ib_flow_spec_ipv6);
2906                 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz);
2907                 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz);
2908
2909                 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) ||
2910                     (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20))
2911                         return -EINVAL;
2912                 break;
2913         case IB_FLOW_SPEC_TCP:
2914         case IB_FLOW_SPEC_UDP:
2915                 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz);
2916                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2917                                                     kern_filter_sz,
2918                                                     ib_filter_sz);
2919                 if (actual_filter_sz <= 0)
2920                         return -EINVAL;
2921                 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp);
2922                 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz);
2923                 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz);
2924                 break;
2925         case IB_FLOW_SPEC_VXLAN_TUNNEL:
2926                 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz);
2927                 actual_filter_sz = spec_filter_size(kern_spec_mask,
2928                                                     kern_filter_sz,
2929                                                     ib_filter_sz);
2930                 if (actual_filter_sz <= 0)
2931                         return -EINVAL;
2932                 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel);
2933                 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz);
2934                 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz);
2935
2936                 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) ||
2937                     (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24))
2938                         return -EINVAL;
2939                 break;
2940         default:
2941                 return -EINVAL;
2942         }
2943         return 0;
2944 }
2945
2946 static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec,
2947                                 union ib_flow_spec *ib_spec)
2948 {
2949         if (kern_spec->reserved)
2950                 return -EINVAL;
2951
2952         if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG)
2953                 return kern_spec_to_ib_spec_action(kern_spec, ib_spec);
2954         else
2955                 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec);
2956 }
2957
2958 int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file,
2959                            struct ib_device *ib_dev,
2960                            struct ib_udata *ucore,
2961                            struct ib_udata *uhw)
2962 {
2963         struct ib_uverbs_ex_create_wq     cmd = {};
2964         struct ib_uverbs_ex_create_wq_resp resp = {};
2965         struct ib_uwq_object           *obj;
2966         int err = 0;
2967         struct ib_cq *cq;
2968         struct ib_pd *pd;
2969         struct ib_wq *wq;
2970         struct ib_wq_init_attr wq_init_attr = {};
2971         size_t required_cmd_sz;
2972         size_t required_resp_len;
2973
2974         required_cmd_sz = offsetof(typeof(cmd), max_sge) + sizeof(cmd.max_sge);
2975         required_resp_len = offsetof(typeof(resp), wqn) + sizeof(resp.wqn);
2976
2977         if (ucore->inlen < required_cmd_sz)
2978                 return -EINVAL;
2979
2980         if (ucore->outlen < required_resp_len)
2981                 return -ENOSPC;
2982
2983         if (ucore->inlen > sizeof(cmd) &&
2984             !ib_is_udata_cleared(ucore, sizeof(cmd),
2985                                  ucore->inlen - sizeof(cmd)))
2986                 return -EOPNOTSUPP;
2987
2988         err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
2989         if (err)
2990                 return err;
2991
2992         if (cmd.comp_mask)
2993                 return -EOPNOTSUPP;
2994
2995         obj  = (struct ib_uwq_object *)uobj_alloc(uobj_get_type(wq),
2996                                                   file->ucontext);
2997         if (IS_ERR(obj))
2998                 return PTR_ERR(obj);
2999
3000         pd  = uobj_get_obj_read(pd, cmd.pd_handle, file->ucontext);
3001         if (!pd) {
3002                 err = -EINVAL;
3003                 goto err_uobj;
3004         }
3005
3006         cq = uobj_get_obj_read(cq, cmd.cq_handle, file->ucontext);
3007         if (!cq) {
3008                 err = -EINVAL;
3009                 goto err_put_pd;
3010         }
3011
3012         wq_init_attr.cq = cq;
3013         wq_init_attr.max_sge = cmd.max_sge;
3014         wq_init_attr.max_wr = cmd.max_wr;
3015         wq_init_attr.wq_context = file;
3016         wq_init_attr.wq_type = cmd.wq_type;
3017         wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
3018         if (ucore->inlen >= (offsetof(typeof(cmd), create_flags) +
3019                              sizeof(cmd.create_flags)))
3020                 wq_init_attr.create_flags = cmd.create_flags;
3021         obj->uevent.events_reported = 0;
3022         INIT_LIST_HEAD(&obj->uevent.event_list);
3023         wq = pd->device->create_wq(pd, &wq_init_attr, uhw);
3024         if (IS_ERR(wq)) {
3025                 err = PTR_ERR(wq);
3026                 goto err_put_cq;
3027         }
3028
3029         wq->uobject = &obj->uevent.uobject;
3030         obj->uevent.uobject.object = wq;
3031         wq->wq_type = wq_init_attr.wq_type;
3032         wq->cq = cq;
3033         wq->pd = pd;
3034         wq->device = pd->device;
3035         wq->wq_context = wq_init_attr.wq_context;
3036         atomic_set(&wq->usecnt, 0);
3037         atomic_inc(&pd->usecnt);
3038         atomic_inc(&cq->usecnt);
3039         wq->uobject = &obj->uevent.uobject;
3040         obj->uevent.uobject.object = wq;
3041
3042         memset(&resp, 0, sizeof(resp));
3043         resp.wq_handle = obj->uevent.uobject.id;
3044         resp.max_sge = wq_init_attr.max_sge;
3045         resp.max_wr = wq_init_attr.max_wr;
3046         resp.wqn = wq->wq_num;
3047         resp.response_length = required_resp_len;
3048         err = ib_copy_to_udata(ucore,
3049                                &resp, resp.response_length);
3050         if (err)
3051                 goto err_copy;
3052
3053         uobj_put_obj_read(pd);
3054         uobj_put_obj_read(cq);
3055         uobj_alloc_commit(&obj->uevent.uobject);
3056         return 0;
3057
3058 err_copy:
3059         ib_destroy_wq(wq);
3060 err_put_cq:
3061         uobj_put_obj_read(cq);
3062 err_put_pd:
3063         uobj_put_obj_read(pd);
3064 err_uobj:
3065         uobj_alloc_abort(&obj->uevent.uobject);
3066
3067         return err;
3068 }
3069
3070 int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file,
3071                             struct ib_device *ib_dev,
3072                             struct ib_udata *ucore,
3073                             struct ib_udata *uhw)
3074 {
3075         struct ib_uverbs_ex_destroy_wq  cmd = {};
3076         struct ib_uverbs_ex_destroy_wq_resp     resp = {};
3077         struct ib_uobject               *uobj;
3078         struct ib_uwq_object            *obj;
3079         size_t required_cmd_sz;
3080         size_t required_resp_len;
3081         int                             ret;
3082
3083         required_cmd_sz = offsetof(typeof(cmd), wq_handle) + sizeof(cmd.wq_handle);
3084         required_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
3085
3086         if (ucore->inlen < required_cmd_sz)
3087                 return -EINVAL;
3088
3089         if (ucore->outlen < required_resp_len)
3090                 return -ENOSPC;
3091
3092         if (ucore->inlen > sizeof(cmd) &&
3093             !ib_is_udata_cleared(ucore, sizeof(cmd),
3094                                  ucore->inlen - sizeof(cmd)))
3095                 return -EOPNOTSUPP;
3096
3097         ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
3098         if (ret)
3099                 return ret;
3100
3101         if (cmd.comp_mask)
3102                 return -EOPNOTSUPP;
3103
3104         resp.response_length = required_resp_len;
3105         uobj  = uobj_get_write(uobj_get_type(wq), cmd.wq_handle,
3106                                file->ucontext);
3107         if (IS_ERR(uobj))
3108                 return PTR_ERR(uobj);
3109
3110         obj = container_of(uobj, struct ib_uwq_object, uevent.uobject);
3111         /*
3112          * Make sure we don't free the memory in remove_commit as we still
3113          * needs the uobject memory to create the response.
3114          */
3115         uverbs_uobject_get(uobj);
3116
3117         ret = uobj_remove_commit(uobj);
3118         resp.events_reported = obj->uevent.events_reported;
3119         uverbs_uobject_put(uobj);
3120         if (ret)
3121                 return ret;
3122
3123         return ib_copy_to_udata(ucore, &resp, resp.response_length);
3124 }
3125
3126 int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file,
3127                            struct ib_device *ib_dev,
3128                            struct ib_udata *ucore,
3129                            struct ib_udata *uhw)
3130 {
3131         struct ib_uverbs_ex_modify_wq cmd = {};
3132         struct ib_wq *wq;
3133         struct ib_wq_attr wq_attr = {};
3134         size_t required_cmd_sz;
3135         int ret;
3136
3137         required_cmd_sz = offsetof(typeof(cmd), curr_wq_state) + sizeof(cmd.curr_wq_state);
3138         if (ucore->inlen < required_cmd_sz)
3139                 return -EINVAL;
3140
3141         if (ucore->inlen > sizeof(cmd) &&
3142             !ib_is_udata_cleared(ucore, sizeof(cmd),
3143                                  ucore->inlen - sizeof(cmd)))
3144                 return -EOPNOTSUPP;
3145
3146         ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
3147         if (ret)
3148                 return ret;
3149
3150         if (!cmd.attr_mask)
3151                 return -EINVAL;
3152
3153         if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS))
3154                 return -EINVAL;
3155
3156         wq = uobj_get_obj_read(wq, cmd.wq_handle, file->ucontext);
3157         if (!wq)
3158                 return -EINVAL;
3159
3160         wq_attr.curr_wq_state = cmd.curr_wq_state;
3161         wq_attr.wq_state = cmd.wq_state;
3162         if (cmd.attr_mask & IB_WQ_FLAGS) {
3163                 wq_attr.flags = cmd.flags;
3164                 wq_attr.flags_mask = cmd.flags_mask;
3165         }
3166         ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw);
3167         uobj_put_obj_read(wq);
3168         return ret;
3169 }
3170
3171 int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file,
3172                                       struct ib_device *ib_dev,
3173                                       struct ib_udata *ucore,
3174                                       struct ib_udata *uhw)
3175 {
3176         struct ib_uverbs_ex_create_rwq_ind_table          cmd = {};
3177         struct ib_uverbs_ex_create_rwq_ind_table_resp  resp = {};
3178         struct ib_uobject                 *uobj;
3179         int err = 0;
3180         struct ib_rwq_ind_table_init_attr init_attr = {};
3181         struct ib_rwq_ind_table *rwq_ind_tbl;
3182         struct ib_wq    **wqs = NULL;
3183         u32 *wqs_handles = NULL;
3184         struct ib_wq    *wq = NULL;
3185         int i, j, num_read_wqs;
3186         u32 num_wq_handles;
3187         u32 expected_in_size;
3188         size_t required_cmd_sz_header;
3189         size_t required_resp_len;
3190
3191         required_cmd_sz_header = offsetof(typeof(cmd), log_ind_tbl_size) + sizeof(cmd.log_ind_tbl_size);
3192         required_resp_len = offsetof(typeof(resp), ind_tbl_num) + sizeof(resp.ind_tbl_num);
3193
3194         if (ucore->inlen < required_cmd_sz_header)
3195                 return -EINVAL;
3196
3197         if (ucore->outlen < required_resp_len)
3198                 return -ENOSPC;
3199
3200         err = ib_copy_from_udata(&cmd, ucore, required_cmd_sz_header);
3201         if (err)
3202                 return err;
3203
3204         ucore->inbuf += required_cmd_sz_header;
3205         ucore->inlen -= required_cmd_sz_header;
3206
3207         if (cmd.comp_mask)
3208                 return -EOPNOTSUPP;
3209
3210         if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE)
3211                 return -EINVAL;
3212
3213         num_wq_handles = 1 << cmd.log_ind_tbl_size;
3214         expected_in_size = num_wq_handles * sizeof(__u32);
3215         if (num_wq_handles == 1)
3216                 /* input size for wq handles is u64 aligned */
3217                 expected_in_size += sizeof(__u32);
3218
3219         if (ucore->inlen < expected_in_size)
3220                 return -EINVAL;
3221
3222         if (ucore->inlen > expected_in_size &&
3223             !ib_is_udata_cleared(ucore, expected_in_size,
3224                                  ucore->inlen - expected_in_size))
3225                 return -EOPNOTSUPP;
3226
3227         wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles),
3228                               GFP_KERNEL);
3229         if (!wqs_handles)
3230                 return -ENOMEM;
3231
3232         err = ib_copy_from_udata(wqs_handles, ucore,
3233                                  num_wq_handles * sizeof(__u32));
3234         if (err)
3235                 goto err_free;
3236
3237         wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL);
3238         if (!wqs) {
3239                 err = -ENOMEM;
3240                 goto  err_free;
3241         }
3242
3243         for (num_read_wqs = 0; num_read_wqs < num_wq_handles;
3244                         num_read_wqs++) {
3245                 wq = uobj_get_obj_read(wq, wqs_handles[num_read_wqs],
3246                                        file->ucontext);
3247                 if (!wq) {
3248                         err = -EINVAL;
3249                         goto put_wqs;
3250                 }
3251
3252                 wqs[num_read_wqs] = wq;
3253         }
3254
3255         uobj  = uobj_alloc(uobj_get_type(rwq_ind_table), file->ucontext);
3256         if (IS_ERR(uobj)) {
3257                 err = PTR_ERR(uobj);
3258                 goto put_wqs;
3259         }
3260
3261         init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
3262         init_attr.ind_tbl = wqs;
3263         rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw);
3264
3265         if (IS_ERR(rwq_ind_tbl)) {
3266                 err = PTR_ERR(rwq_ind_tbl);
3267                 goto err_uobj;
3268         }
3269
3270         rwq_ind_tbl->ind_tbl = wqs;
3271         rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size;
3272         rwq_ind_tbl->uobject = uobj;
3273         uobj->object = rwq_ind_tbl;
3274         rwq_ind_tbl->device = ib_dev;
3275         atomic_set(&rwq_ind_tbl->usecnt, 0);
3276
3277         for (i = 0; i < num_wq_handles; i++)
3278                 atomic_inc(&wqs[i]->usecnt);
3279
3280         resp.ind_tbl_handle = uobj->id;
3281         resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num;
3282         resp.response_length = required_resp_len;
3283
3284         err = ib_copy_to_udata(ucore,
3285                                &resp, resp.response_length);
3286         if (err)
3287                 goto err_copy;
3288
3289         kfree(wqs_handles);
3290
3291         for (j = 0; j < num_read_wqs; j++)
3292                 uobj_put_obj_read(wqs[j]);
3293
3294         uobj_alloc_commit(uobj);
3295         return 0;
3296
3297 err_copy:
3298         ib_destroy_rwq_ind_table(rwq_ind_tbl);
3299 err_uobj:
3300         uobj_alloc_abort(uobj);
3301 put_wqs:
3302         for (j = 0; j < num_read_wqs; j++)
3303                 uobj_put_obj_read(wqs[j]);
3304 err_free:
3305         kfree(wqs_handles);
3306         kfree(wqs);
3307         return err;
3308 }
3309
3310 int ib_uverbs_ex_destroy_rwq_ind_table(struct ib_uverbs_file *file,
3311                                        struct ib_device *ib_dev,
3312                                        struct ib_udata *ucore,
3313                                        struct ib_udata *uhw)
3314 {
3315         struct ib_uverbs_ex_destroy_rwq_ind_table       cmd = {};
3316         struct ib_uobject               *uobj;
3317         int                     ret;
3318         size_t required_cmd_sz;
3319
3320         required_cmd_sz = offsetof(typeof(cmd), ind_tbl_handle) + sizeof(cmd.ind_tbl_handle);
3321
3322         if (ucore->inlen < required_cmd_sz)
3323                 return -EINVAL;
3324
3325         if (ucore->inlen > sizeof(cmd) &&
3326             !ib_is_udata_cleared(ucore, sizeof(cmd),
3327                                  ucore->inlen - sizeof(cmd)))
3328                 return -EOPNOTSUPP;
3329
3330         ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
3331         if (ret)
3332                 return ret;
3333
3334         if (cmd.comp_mask)
3335                 return -EOPNOTSUPP;
3336
3337         uobj  = uobj_get_write(uobj_get_type(rwq_ind_table), cmd.ind_tbl_handle,
3338                                file->ucontext);
3339         if (IS_ERR(uobj))
3340                 return PTR_ERR(uobj);
3341
3342         return uobj_remove_commit(uobj);
3343 }
3344
3345 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
3346                              struct ib_device *ib_dev,
3347                              struct ib_udata *ucore,
3348                              struct ib_udata *uhw)
3349 {
3350         struct ib_uverbs_create_flow      cmd;
3351         struct ib_uverbs_create_flow_resp resp;
3352         struct ib_uobject                 *uobj;
3353         struct ib_flow                    *flow_id;
3354         struct ib_uverbs_flow_attr        *kern_flow_attr;
3355         struct ib_flow_attr               *flow_attr;
3356         struct ib_qp                      *qp;
3357         int err = 0;
3358         void *kern_spec;
3359         void *ib_spec;
3360         int i;
3361
3362         if (ucore->inlen < sizeof(cmd))
3363                 return -EINVAL;
3364
3365         if (ucore->outlen < sizeof(resp))
3366                 return -ENOSPC;
3367
3368         err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3369         if (err)
3370                 return err;
3371
3372         ucore->inbuf += sizeof(cmd);
3373         ucore->inlen -= sizeof(cmd);
3374
3375         if (cmd.comp_mask)
3376                 return -EINVAL;
3377
3378         if (!capable(CAP_NET_RAW))
3379                 return -EPERM;
3380
3381         if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)
3382                 return -EINVAL;
3383
3384         if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) &&
3385             ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) ||
3386              (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT)))
3387                 return -EINVAL;
3388
3389         if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS)
3390                 return -EINVAL;
3391
3392         if (cmd.flow_attr.size > ucore->inlen ||
3393             cmd.flow_attr.size >
3394             (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec)))
3395                 return -EINVAL;
3396
3397         if (cmd.flow_attr.reserved[0] ||
3398             cmd.flow_attr.reserved[1])
3399                 return -EINVAL;
3400
3401         if (cmd.flow_attr.num_of_specs) {
3402                 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size,
3403                                          GFP_KERNEL);
3404                 if (!kern_flow_attr)
3405                         return -ENOMEM;
3406
3407                 memcpy(kern_flow_attr, &cmd.flow_attr, sizeof(*kern_flow_attr));
3408                 err = ib_copy_from_udata(kern_flow_attr + 1, ucore,
3409                                          cmd.flow_attr.size);
3410                 if (err)
3411                         goto err_free_attr;
3412         } else {
3413                 kern_flow_attr = &cmd.flow_attr;
3414         }
3415
3416         uobj  = uobj_alloc(uobj_get_type(flow), file->ucontext);
3417         if (IS_ERR(uobj)) {
3418                 err = PTR_ERR(uobj);
3419                 goto err_free_attr;
3420         }
3421
3422         qp = uobj_get_obj_read(qp, cmd.qp_handle, file->ucontext);
3423         if (!qp) {
3424                 err = -EINVAL;
3425                 goto err_uobj;
3426         }
3427
3428         if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) {
3429                 err = -EINVAL;
3430                 goto err_put;
3431         }
3432
3433         flow_attr = kzalloc(sizeof(*flow_attr) + cmd.flow_attr.num_of_specs *
3434                             sizeof(union ib_flow_spec), GFP_KERNEL);
3435         if (!flow_attr) {
3436                 err = -ENOMEM;
3437                 goto err_put;
3438         }
3439
3440         flow_attr->type = kern_flow_attr->type;
3441         flow_attr->priority = kern_flow_attr->priority;
3442         flow_attr->num_of_specs = kern_flow_attr->num_of_specs;
3443         flow_attr->port = kern_flow_attr->port;
3444         flow_attr->flags = kern_flow_attr->flags;
3445         flow_attr->size = sizeof(*flow_attr);
3446
3447         kern_spec = kern_flow_attr + 1;
3448         ib_spec = flow_attr + 1;
3449         for (i = 0; i < flow_attr->num_of_specs &&
3450              cmd.flow_attr.size > offsetof(struct ib_uverbs_flow_spec, reserved) &&
3451              cmd.flow_attr.size >=
3452              ((struct ib_uverbs_flow_spec *)kern_spec)->size; i++) {
3453                 err = kern_spec_to_ib_spec(kern_spec, ib_spec);
3454                 if (err)
3455                         goto err_free;
3456                 flow_attr->size +=
3457                         ((union ib_flow_spec *) ib_spec)->size;
3458                 cmd.flow_attr.size -= ((struct ib_uverbs_flow_spec *)kern_spec)->size;
3459                 kern_spec += ((struct ib_uverbs_flow_spec *) kern_spec)->size;
3460                 ib_spec += ((union ib_flow_spec *) ib_spec)->size;
3461         }
3462         if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) {
3463                 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n",
3464                         i, cmd.flow_attr.size);
3465                 err = -EINVAL;
3466                 goto err_free;
3467         }
3468         flow_id = ib_create_flow(qp, flow_attr, IB_FLOW_DOMAIN_USER);
3469         if (IS_ERR(flow_id)) {
3470                 err = PTR_ERR(flow_id);
3471                 goto err_free;
3472         }
3473         flow_id->uobject = uobj;
3474         uobj->object = flow_id;
3475
3476         memset(&resp, 0, sizeof(resp));
3477         resp.flow_handle = uobj->id;
3478
3479         err = ib_copy_to_udata(ucore,
3480                                &resp, sizeof(resp));
3481         if (err)
3482                 goto err_copy;
3483
3484         uobj_put_obj_read(qp);
3485         uobj_alloc_commit(uobj);
3486         kfree(flow_attr);
3487         if (cmd.flow_attr.num_of_specs)
3488                 kfree(kern_flow_attr);
3489         return 0;
3490 err_copy:
3491         ib_destroy_flow(flow_id);
3492 err_free:
3493         kfree(flow_attr);
3494 err_put:
3495         uobj_put_obj_read(qp);
3496 err_uobj:
3497         uobj_alloc_abort(uobj);
3498 err_free_attr:
3499         if (cmd.flow_attr.num_of_specs)
3500                 kfree(kern_flow_attr);
3501         return err;
3502 }
3503
3504 int ib_uverbs_ex_destroy_flow(struct ib_uverbs_file *file,
3505                               struct ib_device *ib_dev,
3506                               struct ib_udata *ucore,
3507                               struct ib_udata *uhw)
3508 {
3509         struct ib_uverbs_destroy_flow   cmd;
3510         struct ib_uobject               *uobj;
3511         int                             ret;
3512
3513         if (ucore->inlen < sizeof(cmd))
3514                 return -EINVAL;
3515
3516         ret = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3517         if (ret)
3518                 return ret;
3519
3520         if (cmd.comp_mask)
3521                 return -EINVAL;
3522
3523         uobj  = uobj_get_write(uobj_get_type(flow), cmd.flow_handle,
3524                                file->ucontext);
3525         if (IS_ERR(uobj))
3526                 return PTR_ERR(uobj);
3527
3528         ret = uobj_remove_commit(uobj);
3529         return ret;
3530 }
3531
3532 static int __uverbs_create_xsrq(struct ib_uverbs_file *file,
3533                                 struct ib_device *ib_dev,
3534                                 struct ib_uverbs_create_xsrq *cmd,
3535                                 struct ib_udata *udata)
3536 {
3537         struct ib_uverbs_create_srq_resp resp;
3538         struct ib_usrq_object           *obj;
3539         struct ib_pd                    *pd;
3540         struct ib_srq                   *srq;
3541         struct ib_uobject               *uninitialized_var(xrcd_uobj);
3542         struct ib_srq_init_attr          attr;
3543         int ret;
3544
3545         obj  = (struct ib_usrq_object *)uobj_alloc(uobj_get_type(srq),
3546                                                    file->ucontext);
3547         if (IS_ERR(obj))
3548                 return PTR_ERR(obj);
3549
3550         if (cmd->srq_type == IB_SRQT_TM)
3551                 attr.ext.tag_matching.max_num_tags = cmd->max_num_tags;
3552
3553         if (cmd->srq_type == IB_SRQT_XRC) {
3554                 xrcd_uobj = uobj_get_read(uobj_get_type(xrcd), cmd->xrcd_handle,
3555                                           file->ucontext);
3556                 if (IS_ERR(xrcd_uobj)) {
3557                         ret = -EINVAL;
3558                         goto err;
3559                 }
3560
3561                 attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object;
3562                 if (!attr.ext.xrc.xrcd) {
3563                         ret = -EINVAL;
3564                         goto err_put_xrcd;
3565                 }
3566
3567                 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
3568                 atomic_inc(&obj->uxrcd->refcnt);
3569         }
3570
3571         if (ib_srq_has_cq(cmd->srq_type)) {
3572                 attr.ext.cq  = uobj_get_obj_read(cq, cmd->cq_handle,
3573                                                  file->ucontext);
3574                 if (!attr.ext.cq) {
3575                         ret = -EINVAL;
3576                         goto err_put_xrcd;
3577                 }
3578         }
3579
3580         pd  = uobj_get_obj_read(pd, cmd->pd_handle, file->ucontext);
3581         if (!pd) {
3582                 ret = -EINVAL;
3583                 goto err_put_cq;
3584         }
3585
3586         attr.event_handler  = ib_uverbs_srq_event_handler;
3587         attr.srq_context    = file;
3588         attr.srq_type       = cmd->srq_type;
3589         attr.attr.max_wr    = cmd->max_wr;
3590         attr.attr.max_sge   = cmd->max_sge;
3591         attr.attr.srq_limit = cmd->srq_limit;
3592
3593         obj->uevent.events_reported = 0;
3594         INIT_LIST_HEAD(&obj->uevent.event_list);
3595
3596         srq = pd->device->create_srq(pd, &attr, udata);
3597         if (IS_ERR(srq)) {
3598                 ret = PTR_ERR(srq);
3599                 goto err_put;
3600         }
3601
3602         srq->device        = pd->device;
3603         srq->pd            = pd;
3604         srq->srq_type      = cmd->srq_type;
3605         srq->uobject       = &obj->uevent.uobject;
3606         srq->event_handler = attr.event_handler;
3607         srq->srq_context   = attr.srq_context;
3608
3609         if (ib_srq_has_cq(cmd->srq_type)) {
3610                 srq->ext.cq       = attr.ext.cq;
3611                 atomic_inc(&attr.ext.cq->usecnt);
3612         }
3613
3614         if (cmd->srq_type == IB_SRQT_XRC) {
3615                 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd;
3616                 atomic_inc(&attr.ext.xrc.xrcd->usecnt);
3617         }
3618
3619         atomic_inc(&pd->usecnt);
3620         atomic_set(&srq->usecnt, 0);
3621
3622         obj->uevent.uobject.object = srq;
3623         obj->uevent.uobject.user_handle = cmd->user_handle;
3624
3625         memset(&resp, 0, sizeof resp);
3626         resp.srq_handle = obj->uevent.uobject.id;
3627         resp.max_wr     = attr.attr.max_wr;
3628         resp.max_sge    = attr.attr.max_sge;
3629         if (cmd->srq_type == IB_SRQT_XRC)
3630                 resp.srqn = srq->ext.xrc.srq_num;
3631
3632         if (copy_to_user((void __user *) (unsigned long) cmd->response,
3633                          &resp, sizeof resp)) {
3634                 ret = -EFAULT;
3635                 goto err_copy;
3636         }
3637
3638         if (cmd->srq_type == IB_SRQT_XRC)
3639                 uobj_put_read(xrcd_uobj);
3640
3641         if (ib_srq_has_cq(cmd->srq_type))
3642                 uobj_put_obj_read(attr.ext.cq);
3643
3644         uobj_put_obj_read(pd);
3645         uobj_alloc_commit(&obj->uevent.uobject);
3646
3647         return 0;
3648
3649 err_copy:
3650         ib_destroy_srq(srq);
3651
3652 err_put:
3653         uobj_put_obj_read(pd);
3654
3655 err_put_cq:
3656         if (ib_srq_has_cq(cmd->srq_type))
3657                 uobj_put_obj_read(attr.ext.cq);
3658
3659 err_put_xrcd:
3660         if (cmd->srq_type == IB_SRQT_XRC) {
3661                 atomic_dec(&obj->uxrcd->refcnt);
3662                 uobj_put_read(xrcd_uobj);
3663         }
3664
3665 err:
3666         uobj_alloc_abort(&obj->uevent.uobject);
3667         return ret;
3668 }
3669
3670 ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file,
3671                              struct ib_device *ib_dev,
3672                              const char __user *buf, int in_len,
3673                              int out_len)
3674 {
3675         struct ib_uverbs_create_srq      cmd;
3676         struct ib_uverbs_create_xsrq     xcmd;
3677         struct ib_uverbs_create_srq_resp resp;
3678         struct ib_udata                  udata;
3679         int ret;
3680
3681         if (out_len < sizeof resp)
3682                 return -ENOSPC;
3683
3684         if (copy_from_user(&cmd, buf, sizeof cmd))
3685                 return -EFAULT;
3686
3687         memset(&xcmd, 0, sizeof(xcmd));
3688         xcmd.response    = cmd.response;
3689         xcmd.user_handle = cmd.user_handle;
3690         xcmd.srq_type    = IB_SRQT_BASIC;
3691         xcmd.pd_handle   = cmd.pd_handle;
3692         xcmd.max_wr      = cmd.max_wr;
3693         xcmd.max_sge     = cmd.max_sge;
3694         xcmd.srq_limit   = cmd.srq_limit;
3695
3696         INIT_UDATA(&udata, buf + sizeof(cmd),
3697                    (unsigned long) cmd.response + sizeof(resp),
3698                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
3699                    out_len - sizeof(resp));
3700
3701         ret = __uverbs_create_xsrq(file, ib_dev, &xcmd, &udata);
3702         if (ret)
3703                 return ret;
3704
3705         return in_len;
3706 }
3707
3708 ssize_t ib_uverbs_create_xsrq(struct ib_uverbs_file *file,
3709                               struct ib_device *ib_dev,
3710                               const char __user *buf, int in_len, int out_len)
3711 {
3712         struct ib_uverbs_create_xsrq     cmd;
3713         struct ib_uverbs_create_srq_resp resp;
3714         struct ib_udata                  udata;
3715         int ret;
3716
3717         if (out_len < sizeof resp)
3718                 return -ENOSPC;
3719
3720         if (copy_from_user(&cmd, buf, sizeof cmd))
3721                 return -EFAULT;
3722
3723         INIT_UDATA(&udata, buf + sizeof(cmd),
3724                    (unsigned long) cmd.response + sizeof(resp),
3725                    in_len - sizeof(cmd) - sizeof(struct ib_uverbs_cmd_hdr),
3726                    out_len - sizeof(resp));
3727
3728         ret = __uverbs_create_xsrq(file, ib_dev, &cmd, &udata);
3729         if (ret)
3730                 return ret;
3731
3732         return in_len;
3733 }
3734
3735 ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file,
3736                              struct ib_device *ib_dev,
3737                              const char __user *buf, int in_len,
3738                              int out_len)
3739 {
3740         struct ib_uverbs_modify_srq cmd;
3741         struct ib_udata             udata;
3742         struct ib_srq              *srq;
3743         struct ib_srq_attr          attr;
3744         int                         ret;
3745
3746         if (copy_from_user(&cmd, buf, sizeof cmd))
3747                 return -EFAULT;
3748
3749         INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
3750                    out_len);
3751
3752         srq = uobj_get_obj_read(srq, cmd.srq_handle, file->ucontext);
3753         if (!srq)
3754                 return -EINVAL;
3755
3756         attr.max_wr    = cmd.max_wr;
3757         attr.srq_limit = cmd.srq_limit;
3758
3759         ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata);
3760
3761         uobj_put_obj_read(srq);
3762
3763         return ret ? ret : in_len;
3764 }
3765
3766 ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file,
3767                             struct ib_device *ib_dev,
3768                             const char __user *buf,
3769                             int in_len, int out_len)
3770 {
3771         struct ib_uverbs_query_srq      cmd;
3772         struct ib_uverbs_query_srq_resp resp;
3773         struct ib_srq_attr              attr;
3774         struct ib_srq                   *srq;
3775         int                             ret;
3776
3777         if (out_len < sizeof resp)
3778                 return -ENOSPC;
3779
3780         if (copy_from_user(&cmd, buf, sizeof cmd))
3781                 return -EFAULT;
3782
3783         srq = uobj_get_obj_read(srq, cmd.srq_handle, file->ucontext);
3784         if (!srq)
3785                 return -EINVAL;
3786
3787         ret = ib_query_srq(srq, &attr);
3788
3789         uobj_put_obj_read(srq);
3790
3791         if (ret)
3792                 return ret;
3793
3794         memset(&resp, 0, sizeof resp);
3795
3796         resp.max_wr    = attr.max_wr;
3797         resp.max_sge   = attr.max_sge;
3798         resp.srq_limit = attr.srq_limit;
3799
3800         if (copy_to_user((void __user *) (unsigned long) cmd.response,
3801                          &resp, sizeof resp))
3802                 return -EFAULT;
3803
3804         return in_len;
3805 }
3806
3807 ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
3808                               struct ib_device *ib_dev,
3809                               const char __user *buf, int in_len,
3810                               int out_len)
3811 {
3812         struct ib_uverbs_destroy_srq      cmd;
3813         struct ib_uverbs_destroy_srq_resp resp;
3814         struct ib_uobject                *uobj;
3815         struct ib_uevent_object          *obj;
3816         int                               ret = -EINVAL;
3817
3818         if (copy_from_user(&cmd, buf, sizeof cmd))
3819                 return -EFAULT;
3820
3821         uobj  = uobj_get_write(uobj_get_type(srq), cmd.srq_handle,
3822                                file->ucontext);
3823         if (IS_ERR(uobj))
3824                 return PTR_ERR(uobj);
3825
3826         obj = container_of(uobj, struct ib_uevent_object, uobject);
3827         /*
3828          * Make sure we don't free the memory in remove_commit as we still
3829          * needs the uobject memory to create the response.
3830          */
3831         uverbs_uobject_get(uobj);
3832
3833         memset(&resp, 0, sizeof(resp));
3834
3835         ret = uobj_remove_commit(uobj);
3836         if (ret) {
3837                 uverbs_uobject_put(uobj);
3838                 return ret;
3839         }
3840         resp.events_reported = obj->events_reported;
3841         uverbs_uobject_put(uobj);
3842         if (copy_to_user((void __user *)(unsigned long)cmd.response,
3843                          &resp, sizeof(resp)))
3844                 return -EFAULT;
3845
3846         return in_len;
3847 }
3848
3849 int ib_uverbs_ex_query_device(struct ib_uverbs_file *file,
3850                               struct ib_device *ib_dev,
3851                               struct ib_udata *ucore,
3852                               struct ib_udata *uhw)
3853 {
3854         struct ib_uverbs_ex_query_device_resp resp = { {0} };
3855         struct ib_uverbs_ex_query_device  cmd;
3856         struct ib_device_attr attr = {0};
3857         int err;
3858
3859         if (ucore->inlen < sizeof(cmd))
3860                 return -EINVAL;
3861
3862         err = ib_copy_from_udata(&cmd, ucore, sizeof(cmd));
3863         if (err)
3864                 return err;
3865
3866         if (cmd.comp_mask)
3867                 return -EINVAL;
3868
3869         if (cmd.reserved)
3870                 return -EINVAL;
3871
3872         resp.response_length = offsetof(typeof(resp), odp_caps);
3873
3874         if (ucore->outlen < resp.response_length)
3875                 return -ENOSPC;
3876
3877         err = ib_dev->query_device(ib_dev, &attr, uhw);
3878         if (err)
3879                 return err;
3880
3881         copy_query_dev_fields(file, ib_dev, &resp.base, &attr);
3882
3883         if (ucore->outlen < resp.response_length + sizeof(resp.odp_caps))
3884                 goto end;
3885
3886 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
3887         resp.odp_caps.general_caps = attr.odp_caps.general_caps;
3888         resp.odp_caps.per_transport_caps.rc_odp_caps =
3889                 attr.odp_caps.per_transport_caps.rc_odp_caps;
3890         resp.odp_caps.per_transport_caps.uc_odp_caps =
3891                 attr.odp_caps.per_transport_caps.uc_odp_caps;
3892         resp.odp_caps.per_transport_caps.ud_odp_caps =
3893                 attr.odp_caps.per_transport_caps.ud_odp_caps;
3894 #endif
3895         resp.response_length += sizeof(resp.odp_caps);
3896
3897         if (ucore->outlen < resp.response_length + sizeof(resp.timestamp_mask))
3898                 goto end;
3899
3900         resp.timestamp_mask = attr.timestamp_mask;
3901         resp.response_length += sizeof(resp.timestamp_mask);
3902
3903         if (ucore->outlen < resp.response_length + sizeof(resp.hca_core_clock))
3904                 goto end;
3905
3906         resp.hca_core_clock = attr.hca_core_clock;
3907         resp.response_length += sizeof(resp.hca_core_clock);
3908
3909         if (ucore->outlen < resp.response_length + sizeof(resp.device_cap_flags_ex))
3910                 goto end;
3911
3912         resp.device_cap_flags_ex = attr.device_cap_flags;
3913         resp.response_length += sizeof(resp.device_cap_flags_ex);
3914
3915         if (ucore->outlen < resp.response_length + sizeof(resp.rss_caps))
3916                 goto end;
3917
3918         resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts;
3919         resp.rss_caps.max_rwq_indirection_tables =
3920                 attr.rss_caps.max_rwq_indirection_tables;
3921         resp.rss_caps.max_rwq_indirection_table_size =
3922                 attr.rss_caps.max_rwq_indirection_table_size;
3923
3924         resp.response_length += sizeof(resp.rss_caps);
3925
3926         if (ucore->outlen < resp.response_length + sizeof(resp.max_wq_type_rq))
3927                 goto end;
3928
3929         resp.max_wq_type_rq = attr.max_wq_type_rq;
3930         resp.response_length += sizeof(resp.max_wq_type_rq);
3931
3932         if (ucore->outlen < resp.response_length + sizeof(resp.raw_packet_caps))
3933                 goto end;
3934
3935         resp.raw_packet_caps = attr.raw_packet_caps;
3936         resp.response_length += sizeof(resp.raw_packet_caps);
3937
3938         if (ucore->outlen < resp.response_length + sizeof(resp.tm_caps))
3939                 goto end;
3940
3941         resp.tm_caps.max_rndv_hdr_size  = attr.tm_caps.max_rndv_hdr_size;
3942         resp.tm_caps.max_num_tags       = attr.tm_caps.max_num_tags;
3943         resp.tm_caps.max_ops            = attr.tm_caps.max_ops;
3944         resp.tm_caps.max_sge            = attr.tm_caps.max_sge;
3945         resp.tm_caps.flags              = attr.tm_caps.flags;
3946         resp.response_length += sizeof(resp.tm_caps);
3947 end:
3948         err = ib_copy_to_udata(ucore, &resp, resp.response_length);
3949         return err;
3950 }